diff --git a/NX2_GPU_THREAD_SPLIT.md b/NX2_GPU_THREAD_SPLIT.md new file mode 100644 index 0000000000..75fd0e4105 --- /dev/null +++ b/NX2_GPU_THREAD_SPLIT.md @@ -0,0 +1,63 @@ +# Splitting the GPU thread + +Title 1 (Mario Kart World) is bound by one host thread, the GPU thread: ~100% busy while the +host GPU is 20-40% busy and the game's own CPU thread mostly polls the counter. Per second of +3D scenes (profiled, 2026-10-29): command processing and register writes ~335 ms, macros +~200 ms (1.3 million calls), draw setup ~345 ms (55,000 draws), compute ~70 ms. Speeding up the +pieces (NX2_NOTES rows 123-139) took frames from ~22-38 ms to ~18-26 ms; a locked 60 fps on the +heavy courses needs work on more than one core. + +## Split point: method calls + +- **Front thread** (the current GPU thread): takes the CPU's command queue, parses GPFIFO + entries and command lists (DmaPusher), runs macros (compiled, macro_turing_jit.h), and keeps + what macros read: a mirror of each engine's registers, its macro code and data RAM. +- **Back thread**: executes the resulting stream of method writes in order: Maxwell3D / + KeplerCompute / Fermi2D / DMA engines' CallMethod and everything behind them (register + state, dirty flags, constant buffer and inline uploads, draws, dispatches, queries, fences, + the rasterizer and its caches). Only the back thread touches the rasterizer. +- The stream: an SPSC ring of {engine, method, argument} plus copied data for multi-word + methods (command lists are often guest memory read in place, which the guest reuses). + CPU-side commands (flush/invalidate requests, RunInOrder actions, presentation) are + forwarded into the same ring so they keep their order. + +## What the front needs without waiting + +- Register values macros read (MME_READ): the front mirrors every register write it sends. + Back-side changes to registers it must reproduce: constant buffer offset (+4 per + const_buffer.buffer word), shadow_scratch[0] = 1 (falcon[4]), shadow RAM control (Track / + Replay change the effective value), macro upload pointers (the front handles load_mme + itself). Compute: native_registers (0xad is changed temporarily by the inline QMD path). +- Macro code uploads and binds (load_mme.*, compute 0x45-0x48): handled by the front. + +## Where the front waits for the back (sync points) + +- Macro DMA reads (MME_DMA_READ / _FIFOED, ~450 a second): read guest memory as of their + position, so the back must have executed everything before them. +- Semaphore acquires that are not satisfied yet (~10,500 a second, cross-channel waits on QMD + releases): the release is executed by the back, so the front waits until the semaphore + holds (or parks the channel and runs another one, as the pending-channel code does now). +- Syncpoint waits, WaitForIdle-style methods, and anything else that reads guest memory or + host state on the front. +- Expected cost: each wait drains the back only up to the dependency, then the back idles + until the front sends the next method (a few microseconds), so even ~10,000 waits a second + leave most of the overlap. Ceiling: front ~35-40% of the work, back ~60-65% -> ~1.4-1.6x. + +## Stages + +1. Instrument: count sync points per second and the front/back share precisely (zones). +2. Ring + back thread executing method writes; the front still does everything else. + Correctness first: an EDEN_NX2_SPLIT=0 switch runs both on one thread. +3. Move macros to the front with the register mirrors; verify mirrors against the back's + registers in a debug mode (compare on every MME_READ). +4. Sync points; crash and screenshot tests across courses; then title 4. + +## Status (2026-10-29) + +Stages 1-3 are in (babd395, `gpu_split.h`), off by default: `EDEN_NX2_SPLIT=1` runs engine methods +on the back, `EDEN_NX2_SPLIT=2` also runs the 3D and compute macros on the front against register +copies. Measured in title 1 (NX2_NOTES rows 142-144): ~44,000 draws a second on one thread, ~47,400 +with stage 2, 50,000-66,500 with stage 3. The front waits ~4,000-5,000 times a second (semaphore +acquires, MME DMA reads); the back still spends ~200 ms a second outside its zones (ring dispatch, +method sink). Clean A/B (NX2_NOTES row 145): 1.46x throughput, frames 17% shorter; on by +default since. The back is ~92% busy, ~53% of it draw setup: the next step is there, not the ring. diff --git a/NX2_NOTES.md b/NX2_NOTES.md new file mode 100644 index 0000000000..a50b037b85 --- /dev/null +++ b/NX2_NOTES.md @@ -0,0 +1,264 @@ +# Eden on Switch 2 titles (branch `nx2`) + +Local experiment: get Eden (Switch 1 emulator) to load a Switch 2 title's loose ExeFS and push +boot as far as it goes. Nothing here is upstreamed or pushed. Graphics (Ampere / NVN2) is out of +scope for this pass. + +## Setup + +- Source + build live in `C:\dev\eden` (a worktree of `eden-nx2/`, branch `nx2`). The original + working folder path is ~180 chars, and `cl.exe` can't open its PDB from a build folder there + (C1041), so the build moved to a short path. +- Toolchain: VS 2026 (MSVC 14.51) via `vcvars64.bat`, Ninja from VS, CMake 4.4. +- Configure: + `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INTERMEDIATE_DIR_STRATEGY=SHORT -DENABLE_QT=OFF -DYUZU_CMD=ON -DENABLE_WEB_SERVICE=OFF -DYUZU_ROOM=OFF -DYUZU_TESTS=OFF -DENABLE_WERROR=OFF -DENABLE_UPDATE_CHECKER=OFF` +- Build: `cmake --build build --target yuzu-cmd` -> `build/bin/eden-cli.exe`. +- Run (portable mode, `build/bin/user/` holds config/logs; no keys or firmware installed): + `eden-cli.exe -g "\exefs\main"` +- Debug helpers: + - `EDEN_NX2_SVC_TRACE=1` logs every SVC with arguments and results. + - `EDEN_NX2_IPC_TRACE=1` logs every dispatched CMIF command and non-zero result words. + - `EDEN_NX2_AUTO_STUB=1` answers every unknown command with an empty success (the + `use_auto_stub` setting gets reset by eden-cli's config rewrite). Used for all runs below. + - `eden-cli --filter "<log filter>"` sets the log filter for one run (e.g. `*:Info Loader:Debug` + to get module load addresses). + - `tools/nx2/nso_syms.py <nso> addr|got|find ...` maps module offsets to exported symbols, GOT + slots to imports, or searches symbol names. Eden's backtrace shows "unknown" modules for some + titles; with the `loaded module X @ addr` lines this turns frames into SDK names. + - `tools/nx2/a64_coverage.py <a64.inc> <nso>...` lists encodings in an NSO's .text that + dynarmic's decoder table doesn't match. + - `tools/nx2/npdm_info.py <main.npdm>` dumps the META/ACI0 header, services and kernel caps. + - `tools/nx2/nso_dis.py <llvm bin> <nso> <text_off> <count> <workdir>` disassembles part of an + NSO's .text (clang + llvm-objdump from VS 2022's LLVM). + - Under cdb, stop only on second-chance AVs (`sxd -c2 "..." av`): dynarmic fastmem uses + first-chance access violations. + +## Titles + +All titles share the same `main.npdm` layout: 64-bit, 39-bit address space, ThreadInfo cores +0..5 (core mask 0x3f), priorities 28..59, KernelVersion cap 9.1, SVCs 0x01-0x36 plus 0x80, +Switch 2 ACID layout. + +All four are **Switch 2 builds**: none is a plain Switch 1 sanity check. Instruction counts that +stock dynarmic did not decode (`a64_coverage.py` against the unpatched `a64.inc`): + +| Title (ExeFS) | Program id | RETAA/B | LDAPR | LSE atomics | CAS | SWP | other | +|---|---|---|---|---|---|---|---| +| Mario Kart World `exefs` | 0400C3F00006E000 | 114,767 | 37,560 | 24,718 | 6,640 | 2,470 | CASP 6, XPAC 7, PAC 2, STG 1 | +| Pokopia `exefs` and `updates/exefs` | 0400A3A01EE4C000 | 12 (rtld only) | - | - | - | - | main/sdk/subsdk0 don't decompress (broken dump) | +| Splatoon Raiders `exefs` (update `updates/65536` ~same) | 0400380020056000 | 134,911 | 100,520 | 32,110 | 8,667 | 3,733 | PACGA 58, PAC 7, CASP 6, XPAC 3, STG 1 | + +With the patched decoder, games 1/3/4 only leave ~60-110 rare SIMD encodings in `sdk` undecoded +(e.g. `5f119f9c`, `4ea05c42`, `0f413044`, FP16/v8.2 forms). + +## Blockers hit, in order + +| # | Where it stopped | Cause | Fix (commit) | +|---|---|---|---| +| 1 | CMake try-compile | C1041, PDB path ~290 chars | build in `C:\dev\eden` | +| 2 | Compile | MSVC 14.51: `reinterpret_cast<u64>(ULONG_PTR)`, C4505 on amalgamated zstd.c, C4800 int->bool | Fix build with MSVC 14.51 | +| 3 | Decoder | dynarmic had no LSE atomics, CAS/CASP, LDAPR, RETAA/RETAB, BRA*/BLRA*, LDRA*, PAC*/AUT*/XPAC*, PACGA, MTE ops. Title 1 has ~115k RETAB, ~37k LDAPR, ~25k LSE, ~6.6k CAS, ~2.5k SWP | dynarmic: decode ARMv8.1-8.5 instructions (atomics are exclusive load/op/store with retry; PAC is identity; MTE tags are 0) | +| 4 | Loader | `main` identified as a bare NSO, so main.npdm/rtld/sdk were skipped, then a crash writing into an uninitialised process | loader: load a loose ExeFS when given its main | +| 5 | NPDM parse (ErrorBadFileAccessControl) | Switch 2 ACID has a 0x300-byte signature/key area (Switch 1: 0x200); FAC/FAH offsets were also read without the block base | npdm: parse the Switch 2 ACID layout | +| 6 | eden-cli crash on start/exit | `*appstate` never set, SDL3 callbacks returned SUCCESS (quit), InputSubsystem was a local of SDL_AppInit, mouse timer before mouse driver | eden-cli: fix SDL3 main-callback lifetime bugs | +| 7 | `nnosInitialize` abort | `svcGetInfo` 29/30 unknown (region between alias and ASLR) | kernel: stub ids 29/30 (empty region after alias) | +| 8 | `VammManager::InitializeIfEnabled` abort | `svcGetInfo` 33 unknown; it's a page granule (module ranges are checked against `value-1`; VAMM maps one granule first) | return 0x1000 | +| 9 | `nn::am::InitializeApplicationInterfaces` | `appletOE` cmd 10 (Switch 2 OpenApplicationProxy) | alias to cmd 0 | +| 10 | same | `IApplicationProxy` cmds 60 and 40: new getters | return a generic `nx2:` stub interface (`nx2_stub_interface.h`); calls on it are auto-stubbed and logged | +| 11 | `nn::oe::SetupGpuErrorHandler` -> CreateThread | thread on virtual core 5; Eden emulates 4 cores | fold virtual cores 4/5 onto 0/1 in CreateThread/SetThreadCoreMask | +| 12 | (title 3) `nn::oe` init | `IApplicationFunctions` 320 has no handler | empty-success stub | +| 13 | (titles 3/4) `nn::fs::QueryMountRomCacheSize` -> `GetFileSystemProxyServiceObject` abort 2002-1002 | fsp-srv `SetCurrentProcess` returned ResultTargetNotFound: the loose-ExeFS loader never registered the process with the fs controller, and never set its RomFS | loader: give loose ExeFS boots a RomFS (sibling `romfs/`) and an fs registration | +| 14 | (title 4) `IApplicationFunctions` 321 | unknown | empty-success stub | +| 15 | (title 4) `nn::oe::GetCurrentGraphicsScale` UnexpectedDefault | `IApplicationProxy` 60 is a graphics-scale interface; its cmd 10 must return 10/20/30 | `IGraphicsScaleController` returning 10 | +| 16 | (title 4) `nn::gs::sdk::BootGraphicsSystem` -> `nn::os::GetRemoteMemoryUsage` abort | `svcGetInfo` 31/32 (remote memory for gsv/nvsrv) | return 0 for both (logged) | +| 17 | (title 4) `nn::gsv::BootWithRemoteMemory` abort | it sends TIPC on a session handle from `ISelfController` 2000, which had no handler | `ISelfController` 2000 returns a raw session to `INx2GraphicsSystemSession` (`nx2_gsv.h`), which accepts every command | +| 18 | (title 3) `nn::oe::GetPseudoDeviceId` abort | needs control.nacp, which loose ExeFS lacks | seed from program id | +| 19 | (title 4) `nn::syncpt::sdk::InitializeEx` | gsv 202 must return the syncpt session; gsv 204 takes two transfer memories and must leave a header (version 1, count, offsets) in the read-only one and return two events; syncpt 1 returns a u32 < 0x400 | `INx2SyncptSession`, gsv 202/204 | +| 20 | (title 4) `nn::gpu::sdk::BootWithRemoteMemory` | gsv 497 returns the GPU session; gpu 11 (id range), 12(id) event, 21(id) start + wait on that event, 22 another session | `INx2GpuSession` | +| 21 | (title 4) `nn::vi2::sdk::Initialize` / `nn::vi::CreateLayer` | gsv 305 returns the vi2 session; vi2 1/3/5 return small u32s (5 = timestamp recorder count, 7 needed by LayerHistory), 11/31/32 return handles | `INx2Vi2Session` | +| 22 | all TIPC replies with data | Eden bug: `ResponseBuilder::Push(Result)` writes the 2-word CMIF result, so TIPC data lands one word late and past the copied size | `TipcReplyWords` | +| 23 | (title 4) NVN -> NvRm device open | nvsrv is the nvdrv INvDrvServices protocol; serve it with Eden's nvdrv, initialised without `Initialize` | nvdrv-backed `nx2:nvsrv` | +| 24 | NvRm chip check | Switch 2 NvRm asks for a 0xD0-byte characteristics block and accepts chips 0x17E/0x12B/0x12E, but NVN needs the Ampere one; s16 interface versions at 0x70..0x80 must be >= 6 etc. | report chip 0x17E, Ampere classes, versions 0x40; unknown ctrl-gpu ioctls zero-stubbed for such callers | +| 25 | NvRm address space | AllocAsEx(flags 5, no big pages, [0x8000000, 0xC000000000), split 0); NVN then reserves 4 KiB VA up to 512 GiB; Eden's split/limit handling made the reported regions unusable | small pages up to the end, absolute big-page limit | +| 26 | (title 4) NvRm channel setup | nvhost-gpu ioctls 0x1F/0x20, `/dev/nvhost-tsg-gpu`, GetEngineInfo, object class 0 | logged echo for unknown ioctls once an nx2 driver is seen; GR + async-copy engines; Ampere/class-0 objects on Maxwell slots (70f1651, 82dbd88) | +| 27 | GPU memory | svc 0x80 (make memory GPU-accessible), gpu sub-session 101 (register range) and 211 (job) | stubs (608f0d6) | +| 28 | `syncpt::sdk::InitializeEx` -> `InterProcessEvent::Signal` abort E401 | gsv 204's first handle is attached *writable* (client -> server request), the second readable; the shared page has u64 values (RO+0xc), armed bits (RW+0x18, RO+0x1c) and summary words | gsv 204 returns a writable event, lays out the page, a 500 us poller mirrors host1x values and acknowledges armed waits (88367ea) | +| 29 | `NvOsEventPoolWait` E401 | syncpt 22(slot) must return an event; 201(slot, ~syncpoint, value, mode) / 202(slot) request/cancel a signal | slot events via host1x guest actions (88367ea) | +| 30 | GPU never executed submissions | Switch 2 never maps through nvhost-as-gpu: memory registered with gpu sub 101 is used at GPU VA == CPU VA | `Nx2GpuMemoryRegistry` identity-maps every range into each GMMU; SMMU physical range 33 -> 34 bits for 10/12 GB layouts (88367ea) | +| 31 | Puller | binds Ampere classes C797/C7C0/C7B5 and class 0; NVN2 uploads Turing+ (MME64) macros, which Eden's Fermi macro JIT runs as garbage until it crashes | Ampere classes run on the Maxwell engines; macros skipped while `Tegra::g_nx2_ampere` (88367ea) | +| 32 | NVN device init waits on syncpoint 1 >= 1 | submits use `increment_value` (bit 8) and signal completion only with semaphore releases | Eden appends the syncpoint increments for such submits (88367ea) | +| 33 | `wsi::ounce::NativeWindow::Acquire` on a null window | gsv 1510 is `gsv::shim::CalculateRequirements`; size 0 made window init fail | block-linear size/alignment, 4 bytes/pixel (5fb37b5) | +| 34 | `NvOsEventPoolWait` E401 (second channel) | nvhost-gpu QueryEvent 7 | per-channel event (5fb37b5) | +| 35 | CPU spins in NVN for `[0x2106028010] >= 3` | queues release progress to page P+0x1000 and wait (GPU acquire + CPU poll) on the same offset in page P | puller mirrors releases into the first two-page gpu region (5fb37b5; guess, works) | +| 36 | font header check on a wild pointer | `pl:u` 2010 `GetBuiltInFontData` (bool at offset 0, then {flags, u32 offset, u32 size}) and 2000 (font shared memory + size) | plain TTF copies of Eden's shared fonts after the BFTTF data (5fb37b5) | +| 37 | null "SoundResource" heap (300 MB) | `nninitStartup` sizes the heap as total - used - 0x195000000; Eden capped total at the 8 GiB heap region, leaving 1.55 GiB | Switch 2 processes count the alias region in max memory (5fb37b5) | +| 38 | game waits for service `aud:u` | Switch 2 audio entry (`nn::audio::CreateUserClientSessionByHipc`); cmd 2 returns an IAudioRendererManager | `INx2AudioUserClient` backed by Eden's audren (2e6c39c) | +| 39 | every audio `RequestUpdate` fails | updates tagged `REv?` (lowercase v), renderer opened with `REV?` | compare revision numbers (2e6c39c) | +| 40 | main thread parked in `ExecuteVirtualMachineMapBufferEx` | gpu sub 211's job slot is request word 17, not word 2 | signal word 17's event (2e6c39c) | +| 41 | "too many files open" reading RomFS | eden-cli never raised the CRT stdio limit | `_setmaxstdio(8192)` (2e6c39c) | +| 42 | GPU memory model | gpu sub 101 = `CreateGpuAccessibleMemory` (reply word 3 = GpuMemoryHandle); 211 = `NormalMapGpuMemory4K` (words 3-4 first VA, 5-6 last, 7 handle, 8-9 offset, 17 slot), also into sparse regions at 0x81_xxxx_xxxx; 102/213 (`NormalUnmapGpuMemory4K`)/214 (`CreateSparseRegion`) complete through slot events (words 3/8/11) | real handles, `Nx2GpuMemoryRegistry::MapAt` with SMMU range reuse, slot signals | +| 43 | (title 1) `nn::syncpt::sdk::Initialize` abort | newer SDK opens syncpt with TIPC 0(u64 token) and aborts unless it is echoed | handshake echo | +| 44 | (title 1) NVN closes the GPU device after GetVARegions | newer SDK asks gpu 201/202(id) for the user/gsv partitions (two u64s each, stored as min/max) instead of using 203 | 201 = [0x8000000, 0xC000000000), 202 empty | +| 45 | NVN command buffer with no command memory (~12 s, titles 1 and 4) | `NvRmGpuTSGSubcontextCreate` failed (0x11): characteristics reported one VEID | flags bit 8/22 (TSG, TSG_SUBCONTEXTS) and 64 subcontexts at characteristics +0xb4 (12125bc) | +| 46 | NVN asserts in `GetSyncpointShimRoAddress` | Switch 2 reads syncpoints through a read-only GPU page (flag bit 26, nvhost-as ioctl 0xB) | shim at the top of the VA range; GPU reads of it return the syncpoint value (12125bc, from changes.patch) | +| 47 | window/image creation | gsv 111/112/114/423/429/430/435/1510 native memory and images; svc 0x80 is SetGpuMemoryAttribute | native object registry + layout calc (from changes.patch), real svc 0x80 (12125bc) | +| 48 | presents go nowhere | vi2 229 (word 5 image id, ~word 22/word 24 fence) | composite the block-linear image with native GOB tiling (f63ff90, from changes.patch) | +| 49 | crash in `DmaPusher::CallMethod` (null engine) | NVN uses subchannel 2 (inline-to-memory) without binding a class | bind KeplerMemory there by default; drop methods on unbound subchannels (9f30683) | +| 50 | GPU parses garbage, then stalls (~14-19 s) | NVN2 gates each command buffer with a SEMAPHORED acquire the CPU releases after writing it; Eden read past unmet acquires and ran stale commands (`EDEN_NX2_PB_CHECK` confirmed entries changed after submit) | suspend the channel on an unmet acquire and resume it from the same word (9f30683, approach from changes.patch) | +| 51 | desync after a called command buffer | NVN2 ends a GPFIFO entry inside a method's data, calls another buffer, and continues at the next address | set the unfinished method aside and resume it when the stream continues (9f30683) | +| 52 | semaphore words next to releases get clobbered | 4-byte releases (SEMAPHORED bit 24) were written as 16 bytes | 4-byte releases write 4 bytes (9f30683) | +| 53 | no draws: 3D macros skipped | NVN2 drives 3D/compute state and draws through Turing MME64 macros; compute dispatches through QMD 3.0 | changes.patch's MME interpreter (one macro context shared by the process's channels, also on the compute class), native compute and QMD scheduler (d7826ce) | +| 54 | draws hit Eden's Maxwell decoder ("Invalid insn") | shaders are SM86 SASS | changes.patch's Ampere-to-Maxwell lowering, widened until all 32 of title 4's programs lower; `EDEN_NX2_SHADER_DUMP` + `C:\dev\nx2work\lowertest` test it offline (d7826ce) | +| 55 | GPU desyncs again after ~15-30 s | the game ran a full ring of command memory ahead of Eden's GPU thread: the vi2 229 reply reports all buffers free, so nothing paced it | vi2 229 waits (bounded) for the frame's GPU fence (325cac7) | +| 56 | macro arguments lost | NVN interleaves several command buffers, so more than one split method can be pending | several parked methods keyed by resume address, dropped at the end of the submission (325cac7) | +| 57 | frames all black although draws run | gpu 214 creates GPU-only memory (six 300 MB regions at 0x81_0000_0000) that the graphics server backs itself; the render targets live there and were unmapped | back each region with application-pool pages mapped into the alias code region (325cac7). Title 4 now shows its loading screen and a menu screen | +| 58 | (title 4) hang after confirming difficulty: macro 0x142 "parameter FIFO underflow" | NVN2's conditional-render macro reads two query counters with MME_DMA_READ_FIFOED (3D methods 0x154/0x155 address, 0x158 count) and loads them after a load barrier | MME DMA reads into the load FIFO / data RAM (8cbe854) | +| 59 | equal-value gate semaphore missed (~24 s) | the CPU side ran ~96 command buffers ahead when the 250 ms present wait timed out during shader compiles | present waits up to 4 s for the frame fence (8cbe854) | +| 60 | 3D queue waits forever on a word the compute queue should release | NVN2 patches a third QMD release slot at word 52 for queue-to-queue syncs | fire release slots 24, 28 and 52 (8cbe854) | +| 61 | 3D queue waits for N compute completions, 4 missing | compute jobs whose shader Eden rejected never completed, so their +1 releases never fired | rejected QMDs still complete and release (a84ec80); IMAD -UR addend, LDSM lowered | +| 62 | queue waits for ≥ a value that keeps moving backwards | QMD release slot 0 (flag bit 19) is a max release into a word a 3D macro also advances | bit-19 QMD releases write max(old, payload) (a84ec80) | +| 63 | in-game draws refused (~20k per run), black screen behind the dialogue | strong LDG (bit 77), 2D TLD4, RED MAX, AST of RZ unsupported | lowered (a84ec80); 210/214 post-menu shaders lower | +| 64 | intro scene black (DLSS-style pass) | title 4 renders at 2560x1440 and upscales to 3840x2160 with a tensor-core (HMMA/MOVM/LDSM) compute network; emulating it takes the host driver minutes to compile and would cost seconds per frame | bilinear blit from its input (cbuf0+0x90) to its output (cbuf0+0xa0) when the program is rejected (22333bd); full emulation opt-in (EDEN_NX2_TENSOR) | +| 65 | fullscreen blit passes read constant bank 24 | console-provided blit parameter block, never bound by the game | bank 24 -> 17, synthesized from the render area by BlitConstantsPass in vertex and fragment shaders (22333bd, d75e57d) | +| 66 | synthesized blit sizes were 0 | the game never programs surface_clip, which feeds Eden's render area | fall back to colour target 0's size (d75e57d) | +| 67 | tonemap output black | the 3D colour-grading LUT is drawn once per scene by a pixel shader using CALL.REL.NOINC; capture treated it as a real call, never found the end and rejected the shader | NOINC calls count as forward branches in capture (4d58a74). **Intro cutscene renders correctly** | +| 68 | crash ~5 s after the save that follows character customization (garbage pointers in two threads) | nx2:gpu 213 (unmap) and 102 (destroy GPU memory) were no-ops; the game reuses the freed memory as heap and stale GPU write-backs overwrote it | unmap from every address space, drop freed semaphore regions (bed817c); not yet confirmed past customization: frame-locked replays desync across loading screens | +| 69 | hatched edges on UI portraits, buttons, squid cursor | UI textures are ASTC; the GPU ASTC decoder ignored the Switch 2 GOB swizzle | apply it in astc_decoder.comp (7fa2b46) | +| 70 | freeze ~15 s after the post-customization save; channels 2/4 wait on 0x2115c98000 (equal 0x5293, now 0x5310) | a 3D queue ticks a counter two queues acquire value by value; Eden wrote several ticks between their re-checks | releases latch the waiters they meet (8e37f58) | +| 71 | audio broken, game at 90-160 fps | vi2 presents were never paced | 60 Hz vblank pacing | +| 72 | stuck loading the next scene ("too many files open" for route_wallClimb.pack.zs, save.dat) | VFS open-file cache cap equal to the CRT limit (8192) | cap 512 below _getmaxstdio() | +| 73 | loading screen hangs, host GPU TDR | macro parameters split across GP entries were parked; method data continues across entries | continuous split methods (e8d060b) | +| 74 | rejected ATOMG/STL/LDL with small immediate offsets | lowering only took a zero offset | s20 ATOM offset, 24-bit STL/LDL offset (6302be8) | +| 75 | same crash 3 runs (415 s, 429 s, 9960 s): read 0x29 at pc 0x85096d10 (recursive mutex lock), lr 0x82ada310 | worker job runs main+0x2aba2b0 on object C = [A+0x18] whose manager [C+0x720] is null; C's destructor (main+0x2ab8e20, vtable main+0x4b660f8) nulls it before A's teardown (main+0x2b37b50) clears A+0x18; teardown and per-frame update are both jobs (via 0x8365c634); NFC Finalize comes 14-22 s before each crash; LSE atomics (exclusive pairs) and DMB (mfence) checked fine | open: catch all threads at the abort with the gdbstub (`nx2work/dbg_crash_threads.rsp`) | +| 76 | debugger run, crash at 406 s in the same scene load: writes through a null result at pc 0x82b471fc (main+0x2b271fc) | the scene's frame heap "PaintSceneProcHeap.26" (fixed 90 MiB + 0xfc8 parent, head 0x12c9525d18, tail 0x12c9800208) had 2.98 MB left for a 0x302088 allocation (A init main+0x2b37de0 -> main+0x2b27020); about 160 KB short, so some allocation of that scene is bigger than on the console (only 11.7 MB of the heap is nx2:gpu memory); the main thread was waiting in svc 0x34 for job workers | open: log that heap's allocations to find the oversized one | +| 77 | why the scene heap overflowed (run 6) | the heap size is a constant (mov w1, #0x5a00000 at main+0x2b32070); its fixed contents are 71.3 MiB (main+0x2ab9218) + 1.05 MiB + 1.25 MiB + 3 MiB; in runs 3, 4, 5 and the save test the scene's two 0x514000 GPU pools come from the root heap (next to the 0x2ee000 pool), in run 6 from the scene heap (current heap of the loading thread), 10 MiB it has no room for; which heap a job allocates from depends on the thread running it. The game's npdm asks for cores 0..5 (mask 0x3f, reported as is by GetInfo CoreMask) and Eden folds 4, 5 onto 0, 1, so two cores run two workers each | candidate: report or emulate the real core layout; ruled out: memory layout (12 GB), controllers (one), GPU pool alignment | +| 78 | six emulated cores (4fbfd5b), run 7 with the gdbstub: same scene-heap overflow at 292 s, the 0x514000 pools again in the scene heap | both overflow runs (6, 7) had the gdbstub on (memory accesses checked, slower JIT); the three runs without it (3, 4, 5) put those pools in the root heap and crashed later on the torn-down object instead, so the debugger itself shifts this timing | open: a six-core run without the gdbstub | +| 79 | run 8 (six cores, no gdbstub): original crash at 338.6 s, 11 s after the paint scene set up (327 s, pools in the root heap) | C's init (main+0x2ab9120) creates "ColPaintBuild.%u" (main+0x147d3c0 -> main+0x34f3850) sized by [C+0x738], 100 MiB by default (ctor main+0x2ab874c); when that returns null, init returns before setting [C+0x720], and the next state request (main+0x2aba230) crashes the update. So every crash at this load is a memory shortage: 100 MiB (no gdbstub) or 3 MiB in the scene heap (gdbstub). Root heap is 0x19506f000 at 0x1145e10020 | open: why memory is short (reported total memory for Switch 2, frees delayed behind the GPU) | +| 80 | the memory shortage behind rows 75-79 | title 4 is told 9.55 GiB (GetInfo 21) and plans all of it: SetHeapSize 3.125 GiB plus a 6.33 GiB VAMM-backed sead root heap in the alias region; Eden backed the six 300 MB gpu 214 regions from the application pool, 1.8 GB the kernel still reported as free | 214 backing from the non-secure system pool, +8 GiB DRAM on the 12 GB layout, device memory tracker 35 physical bits (5cff620); run 9 passed the crash spot (NFC Finalize 272.8 s, ran to 388 s), no allocation failures | +| 81 | title 1 hangs before its first frame (~17 s): main logic stops, two cores spin, only audio and hid keep running, almost no GPU submits | it asks gpu 214 for ~20 GiB (two 0x16ad80000 regions, these two still fail with the new pool); gets further than before (save data, friend list) | open: thread snapshot at the hang; 214 regions this large must be sparse on the console | +| 82 | title 1 hang (row 81) | `monitor handles` (a4051e4) showed the main thread waiting forever on the nx2:vi2 32 event (loop at main+0x25ec5e4 until [obj+0x88] >= 2); the stub never signalled it | 31 and 32 events signalled at 60 Hz (6196a11); title 1 now presents (241 frames by 95 s) | +| 83 | title 1 after the hang: ~3 fps with "fence not reached in 4 s" stalls | unimplemented vi2 104/105 (every frame) and gpu 215, gpu 214 still short for the two 5.7 GiB regions, one Nx2Map "out of SMMU space", rejected shaders (formatted surface store, IPA mode, LEA carry) | open | +| 84 | title 1 GPU stalls (row 83): channels wait on semaphores 0x21f6093140/3040 until the 4 s timeout | NVN2 queue syncs run through a ring of QMDs (dependency 2 each: the previous entry's launch and macro 0xda, which patches release slot 52 and schedules/decrements via PCAS 3/7); macro 0xda reads the ring base from MME data RAM word 0x27, which each channel sets (0x21098d39, 0x213e8ff8, 0x213ea168) but Eden shared one data RAM for all channels, so every channel drove the same ring | per-channel MME data RAM, one QMD table for all channels (6c20e37); no stalls, title 1 reaches its title screen (world map with logo) | +| 85 | title 1 address space | 214 = CreateSparseRegion, 215 = DestroySparseRegion (SDK symbols); SMMU space (16 GiB) ran out | 64 GiB device space (71b38ab), 215 handled (cc0c400) | +| 86 | title 1 after its title screen: black frames | tens of thousands of rejected shaders: IPA interpolation modes, integer carry/third source, uniform comparison, formatted surface load/store, uniform constant load, unsupported opcodes | open | +| 87 | title 1 loading crash: Vulkan out of memory | an 8192x8192x68-layer virtual texture (~23 GB) | images over 2^28 texels are created sparse with no memory (`Unbacked`: no uploads/downloads) (96a9b81); the game reaches its real title screen ("Press L + R"), ground black (that texture has no data) | +| 88 | title 1 shader coverage | checked against nvdisasm (`nx2work/nvd.py` disassembles one instruction at a time, since a single illegal word stops nvdisasm); calls: RET may restore the link through IMAD.MOV with code between, library functions may EXIT, `@Pg EXIT Pc`; PLOP3/UPLOP3 any table, UP2UR, ULOP3 predicate output; UIMAD.WIDE, uniform USHF/FMNMX/I2F/F2I, LDS `[Ra.Xn+URb]`, ATOMG/RED/STG forms, PRMT, FSWZADD, REDUX.MAX | 1351c73, 52cda01, 49396b7: 1699 of 1712 dumped programs lower; left: 12 task/mesh programs (ISBERD/ISBEWR, row 89) and one SURED | +| 89 | task/mesh shaders | title 1 draws some geometry with NV-style task + mesh programs: the task program culls a batch and writes its payload with `ISBEWR.O.ATTR [Rn+0x10]` and the mesh count with `ISBEWR [0x4]`; mesh programs read it back with `ISBERD.ATTR.SKEW`. The most-drawn rejected program (0x114c459500) is a task program. Eden has no mesh pipeline | open: needs a task/mesh pipeline (VK_EXT_mesh_shader) or a compute + indirect-draw emulation | +| 90 | each rejected program was lowered and logged again on every draw (1.7M log lines in 13 min) | no negative cache | the shader cache keeps the first exception per address until that memory is written; logs deduplicated (bcb4ff0) | +| 91 | a compute program with a jump table: "Failed to track indirect branch" | Eden's tracker wants IMNMX imm -> SHL imm -> LDC -> BRX; IMAD.SHL was lowered to an XMAD multiply | IMAD by a power of two lowers to SHL (49396b7+) | +| 92 | with rows 88's compute forms, title 1 stays on its loading map (file reads stop ~46 s in) | two compute programs failing in Eden's compiler: a jump table whose targets lie past the first EXIT, and a 128-bit store of RZ from unaligned scratch registers | programs with BRX end at the compiler's closing branch-to-self; R248-R251 for 128-bit zero stores (3df6a81) | +| 93 | title 1 colour cast (magenta/cyan, red/green, changing per course) | the sky/atmosphere 32^3 volume shader starts with an unguarded R2UR; R2UR/MATCH/HFMA2.RELU saved P6 with the uniform-guard mask `1 << (6 - 7)` (undefined), which shifted into the opcode bits | always save/restore P6 (2ed0c48). Found with `EDEN_NX2_DUMP` (flush GPU ranges at a present to files) and `nx2work/vol3d.py` / `volslices.py` (untile Switch 2 3D textures) | +| 94 | near terrain black | 8192x8192x68 ASTC 8x8 virtual textures: 1.5 GB in guest memory, 23 GB each as Eden's RGBA8 decode | huge GPU-decoded ASTC arrays keep levels from the first that fits 512 MiB (here 3) on the host (60c4791) | +| 95 | wind/foliage shards | SM70+ MUFU.SIN/COS take turns (sin(2pi x)); Eden's Maxwell MUFU takes radians | scale by 2pi (245e6ad); jungle canopy not re-checked yet | +| 96 | title 1 aborts after ~1.5-4 min at the title or when a race starts (nn::diag::Abort from main+0x19e5df0) | the movie player (main+0x19e5860) lays out its decoder frames with nn::codec::CalculateDecoderImagePoolBuffer, which sends nx2:gsv 991; auto-stubbed, the reply kept the request words, so the frame size read as 0x167027f'00000001 and the budget check at main+0x19e5b58 failed. (Not the "video-frame-count" < 9 check: the game sets that from a two-slot registry, at most 2.) | gsv 991 answers a block-linear NV12 size (640x360 -> 0x64000) | +| 97 | pop-in, objects flicker in and out, low-quality meshes | high-detail LODs drawn through task/mesh programs (bound in stage slot 1, header type VS, ISBERD/ISBEWR), which are rejected and skipped | open: task/mesh support | +| 98 | ~12 fps at the title | host GPU ~20% busy, emulated cores ~0.9 host cores, GPU thread 45-70% busy, much of it waiting on the host GPU: semaphore acquires wait for host fences under GPU accuracy High; ASTC uploads end in Finish() (a Flush() there crashed at boot); a 1 ms poll sleep while channels are suspended | gpu_accuracy=Low: ~16 fps; SDL main loop capped at 240 Hz (88f04ad). Open: ASTC upload without a full stall (and per-layer re-upload of the trimmed virtual textures), native MME interpreter cost, draw path | +| 99 | VRAM over the card (8.4-9.8 GB used on an 8 GB RTX 4070 Laptop), texture cache always collecting | ~700 ASTC 8x8 textures decoded to RGBA8 (16x their size, ~4.5 GB) | astc_recompression=Bc3 + accelerate_astc=CpuAsynchronous in the config; huge ASTC arrays stay on the GPU decoder as trimmed RGBA8 (ed0a3c9): 4.6-5.6 GB, ~10 -> 16 fps | +| 100 | 8 ImageTranscode threads at ~44% each (3.5 cores) | Eden's default pool is half the hardware threads | an eighth, at least 2 (`EDEN_NX2_TRANSCODE_THREADS` overrides): ~19 fps on the attract loop | +| 101 | Medal's capture hook inside Eden (`medal_hook64!OBS_Negotiate` in GPU-thread stacks) | Medal registers a per-user Vulkan implicit layer that loads into every Vulkan app | launches set `DISABLE_VULKAN_MEDAL_OBS_CAPTURE=1` (the layer's own off switch) | +| 102 | host crash (log stops, segfault) at 113 s in one run; under cdb "Free Heap block modified after it was freed" at 243 s, twice | Eden's TickAsyncDecode erased a finished decode while still holding its mutex (unlocked freed memory); only reached with accelerate_astc=CpuAsynchronous (the BC3 setup) | release the lock first, cancel decodes of deleted images (8766a72); clean past 330 s. Also made nx2 service callbacks (vsync timer, syncpt actions, capture/dump) independent of their sessions' lifetime (190747a) | +| 103 | (title 1) texture churn: the collector evicted ~1,250 decoded ASTC textures per 600 frames and they were decoded again (lag and pop-in turning the camera) | device memory sat above the collector's critical line (5.4 GB of a 7.0 GB budget): ASTC 8x8 is 4x larger as BC3, and a bloom chain at unmapped GPU 0xf0000-0x15b600 was re-created on every use (each lookup got a fresh stand-in CPU address), ~5,400 images per 600 frames on the map | unmapped GPU addresses get a stand-in address fixed by the GPU address, so images are found again; ASTC whose blocks carry no alpha (exact endpoint-mode scan, ~70% of them) goes to BC1 (`EDEN_NX2_ASTC_BC1=0` turns it off). Attract loop: 4.8-5.1 GB, no costly evictions, ~17 -> 22-23 fps | +| 104 | garbage textures (title logo, road patches) and a host crash 90-120 s in that skipped the crash handler (fail-fast) | an image re-decoded while a decode was in flight has two; the first to finish cleared IsDecoding, so deleting the image did not cancel the second, which was then uploaded into the next image to take the slot (another texture's data; a staging overrun when that image was smaller) | every pending decode of a deleted image is cancelled; a decode larger than its image is dropped and logged. Frames clean; crash: verifying with long runs | +| 105 | (title 1) movie abort moved on: `nn::codec::InitializeVp9Decoder` failed with 2001-0114 (InvalidHandle) | gsv 901/902/1001/1002 (`nn::codec::video::detail::ipc::Initialize`) were auto-stubbed; 902/1002 must return sessions (handle word 3, result word 4) whose TIPC 1 (CreateVideoDecoderService) returns another | sessions returned (2970100); the movie plays with no picture. Open: decode VP9 (vdecsvc 200/101/3/311 are auto-stubbed) | +| 106 | GPU thread ~50% in the Turing macro interpreter | every instruction re-decoded on every run, register file copied per instruction | decode once per PC, operands read up front (a97d26d): GPU thread ~68% -> ~45-50% busy | +| 107 | streamed textures shown as garbage/black for seconds; 2 transcode threads pegged | ASTC was decoded and re-compressed to BC on the CPU, ~40 textures/s on the attract loop | accelerate_astc=Gpu (config value 1) + BC recompression: GPU ASTC decode into a temporary RGBA8 image and `bc_encoder.comp` (69cc66b); textures ready the frame they are used | +| 108 | frame time is CPU work + GPU-thread work in series | nx2:vi2 229 blocks the game until Eden's GPU thread has processed the frame (WaitForFrame), and RequestComposite waits for the GPU thread to drain its queue too; the present timeline log shows 8-56 ms of each 34-105 ms frame blocked | `EDEN_NX2_FRAMES_AHEAD=1` (wait for the previous frame) deadlocks at the first scene change: the main thread waits on syncpoint 1 (present fences) for a value no queued work reaches. Open | +| 109 | ~1,400 SVCs per frame | the job system's WaitForAddress (0x34) / SignalToAddress (0x35) pairs and yields, all through Eden's global scheduler lock (3 of 6 cores seen blocked on it) | idle-core spinning (`EDEN_NX2_IDLE_SPIN_US`) gave no gain. Open | +| 110 | (title 1) GPU thread saturated, 3D attract scenes ~22 fps | profiled with the new GPU zone timers (`EDEN_NX2_GPU_PROFILE`) and the in-process sampler (`EDEN_NX2_SAMPLE`): per second, ~355 ms command parsing outside macros, ~210 ms in 745k macro calls, ~325 ms draw setup (33k draws), ~89 ms compute (4.4k dispatches) | see 111-115; now ~27-33 ms per frame in the same scenes | +| 111 | the guest clock (CNTPCT) cost ~10% of the main core | AMD CPUs have no CPUID leaf 0x15, so the TSC frequency stayed 0 and every read went through QueryPerformanceCounter | measure the TSC when the leaf is missing (948a391) | +| 112 | C++ exceptions on the GPU thread (~1-2%) | a rejected native shader (mesh/task programs) was rethrown twice every time it was bound | return false for remembered rejections (948a391) | +| 113 | debug aids on the hot path (~4% of the GPU thread) | semaphore watch per command word, FNV hash of every pushbuffer entry, first-write trace, two debug logs per macro call | flag checks unless their variable is set (948a391) | +| 114 | texture churn: 1,400-2,600 textures deleted and recreated per 600 frames, one purge of 4,300 | ~38% of device memory was empty space in 256 MiB VMA blocks; usage came from the driver, so evictions did not lower it | 64 MiB blocks and usage minus VMA block slack (7b0ff0b): 0-224 per 600 frames, 6.24 of 6.97 GB used | +| 115 | main core: ~12% slow-path reads of GPU-tracked pages, GPU thread and CPU fighting over the caches' mutex | `use_reactive_flushing` (on by default on desktop) protects tracked pages against reads too | off in the test config: the slow reads are gone, no visual differences seen in the attract loop. Kernel lock spins before sleeping (5c99031): ~20-25% less time blocked | +| 116 | macro interpreter is ~11-13% of the GPU thread | operand fetch is not the cost: a slot-based rewrite matched the old interpreter on 200,000 recorded calls (`EDEN_NX2_MME_RECORD`, `C:\dev\nx2work\mme_test`) but was no faster | open: needs a JIT or pre-built handlers | +| 117 | macro calls: ~1,000,000 a second, 7 instructions each, 96% exact repeats | the per-call interpreter cost was the time, not the instructions | memo of calls that depend only on their parameters and the data RAM words they read (1cd4a3c, b75fdf8, c28ea9a for compute macros): 97% replayed, verified on 200,000 recorded calls | +| 118 | crashes ~26 s in (fencing thread walking a corrupt operation list; query cache Barriers; Vulkan worker SubmitExecution) | `async_presentation=true` (config): 4 crashes in 8 runs with it, 0 in 6 without; a race between the present thread and the Vulkan scheduler | keep it off (test build 3b). The fence manager's SyncOperation also appended to a list SignalFence moved from another thread: own lock now | +| 119 | (tester, RTX 4070 SUPER) 50 MB log; textures evicted with 3 GB VRAM unused | nx2:vi2 104/105 called ~75 times a second through the logging auto stub; collector thresholds from Eden's ~7 GB device memory cap | quiet stubs (649fb56); thresholds from the driver budget (49225dd) | +| 120 | GPU thread asleep ~8% on the texture/buffer cache mutexes | holders: the fencing thread (it took both caches for every fence, one per compute dispatch, and holds the buffer cache while it copies GPU-written buffers back) and CPU write traps; std::recursive_mutex slept at once | spinning recursive mutex, per-cache fence flush flags, separate locks (f10767f): asleep 3.7%. Buffer downloads at every fence stay: the native MME DMA reads of indirect arguments depend on them (no synchronous flush logged with them on) | +| 121 | first-run freezes from shader translation (one 28,102-instruction compute shader: 3.4 s on the GPU thread) | BreadthFirstSearch's visited list was a linear scan (quadratic), and GlobalMemoryToStorageBufferPass searched again for every access | hash set past 64, per-start memo (ba173b6): 0.85 s; runtime builds now ~2-4 ms each. Translating on workers was not done: environments read live GPU state (texture types, constant buffer values) while translating | +| 122 | final clean run (attract loop): 21-25 ms per frame in most 3D stretches, 26-38 ms in compute-heavy courses (industrial, highway), host GPU ~38% busy | the GPU thread is still the limit: draw setup ~25%, command parsing and register writes ~15%, compute dispatches/inline uploads (heavy courses) | open: per-draw binding cache, uniform buffer copies (staging slices are recycled by tick, so reuse needs care), buffer download interval sets | +| 123 | (title 1) 5,500-7,000 fences a second, one per compute release, each made in full (both cache locks twice, a fence object, a fencing-thread wake-up) even with nothing to wait for: SignalFence 7.6% of the GPU thread | with fast GPU fences the release callback runs at once; ~20% of the fences had no downloads, queries or operations. OverlapRangeSet::Subtract restarted its search after every erase | empty fences skipped (one in 64 still made), one lock per cache per fence, single-pass Subtract (9905916): SignalFence 3.9% | +| 124 | GPU thread ~9% spinning on, sleeping on and waking for the cache locks | the fencing thread took the buffer cache once per fence to write 250-330 MB/s of downloaded buffer data back (13,000-20,000 ranges a second) | signaled fences released together (31cd7b2, `EDEN_NX2_FENCE_BATCH=0` off): spinning 7.0% -> 5.5%; fence cost per call unchanged within noise (longer holds). Open: the downloads themselves | +| 125 | crash every few minutes: BindHostGraphicsUniformBuffer -> ReadBlockUnsafe, access violation at 0x600 (2 of 4 runs) | DeviceMemoryManager's translation cache was shared by all threads; Map on a service thread (nx2:gpu 211 job mappings, logged at both crashes) cleared it between the GPU thread's null check and its copy | per-thread cache invalidated by a generation counter (1a497fa): 0 crashes in 6 runs | +| 126 | macro parameters appended to a std::vector per call (~2% of the GPU thread) | ~1,000,000 calls a second, nearly all whole in one method run | stack copy (3caf8ea, `EDEN_NX2_MACRO_COPY=1` off). Passing the command list words in place is unsafe: they are often guest memory read in place, and the memo stores the parameters after the call | +| 127 | main core: ~23% of its samples read CNTPCT (~17 million reads a second) | the game polls the counter while it waits for the GPU; a spinning thread slows its SMT sibling (8-core/16-thread Ryzen) | PAUSE between counter reads less than ~1.7 us apart (8abeb0b, `EDEN_NX2_CNTPCT_PAUSE=0` off); effect not separable from run-to-run noise | +| 128 | Vulkan worker woken with a system call every 8 draws (~2% of the GPU thread) | DispatchWork notified even when the worker was busy | every 32 draws, notify only a waiting worker (d9d7e94) | +| 129 | macro memo arena (16 MiB) cleared every ~2 s | records stored again with another shape left their old copies | compaction (9a37142); live records are ~12 MiB, so most compactions still end in a clear; hits stay ~97% (46% on the last call's slot) | +| 130 | measurements | a stuck `nvidia-smi -l` loop had spun one core for 12 h (every run since the evening of 10-28 had one core less); the attract loop shows different courses each run (frame means vary +-15%); stopped test loops kept running and overlapped | compare per-call costs (`C:\dev\nx2work\zones.py` on `EDEN_NX2_GPU_PROFILE` logs); `ab_env.sh` and `hangcheck.sh` refuse to start while eden-cli runs; `stopall.ps1` stops every loop | +| 131 | (title 1) 250-330 MB/s of buffer data copied back at fences (13,000-20,000 ranges a second) | Eden downloads every GPU-written range at the next fence. Measured: the native MME's DMA reads never touched those pages (`EDEN_NX2_GPU_PROFILE` counter), and with reactive flushing the guest CPU read GPU-written memory at most 64 times in 200 s | off for Switch 2 titles (d530bc4, `EDEN_NX2_BUFFER_DOWNLOADS=0/1` forces): fences ~1.1 us (were ~5), draw setup -7%, cache-lock spinning 1.8% | +| 132 | inline uploads 10.5% of the GPU thread | IsRegionGpuModified visited every GPU-written range in the query (they stay fragmented without downloads); std::hash of integers and pointers is FNV-1a on MSVC in every Common::unordered_map | RangeSet::Intersects (one lookup) and a pass-through hash for integer/pointer keys (0fe581f): 7.5% | +| 133 | some courses (desert/canyon, dense city, highway) run 30-45 ms with presents blocked 9-15 ms on the previous frame's fence (also before today: t14) | not the host GPU (nvidia-smi: 20-40% busy, ~22 W) but the GPU thread: ~100% busy there, its command processing 370-420 ms/s instead of 300-350 (more commands per frame). The 8 GB laptop GPU is also near its budget (6.6-6.9 of 7.1 GB): the texture collector evicts hundreds of decoded ASTC images per 600 frames, the buffer collector GPU-written buffers every frame | `EDEN_NX2_TC_MIN_AGE=120` (0f0fbb9) made no difference over three runs; buffer evictions batched into one GPU wait (db4c539), still ~2.7% on those courses (open: asynchronous eviction) | +| 134 | each inline write invalidated the buffer, texture, shader and query caches on its own | thousands of small writes between draws | guest memory written at once, pending query results dropped at once, GPU-written buffer data uploaded in place; the other caches get merged ranges at the next FlushCaching (c06ac80, `EDEN_NX2_INLINE_DEFER=0` off): inline writes 4.4% | +| 135 | ~6 million 3D register writes a second, ~60% writing the value the register already holds (mostly macro memo replays) | each went through shadow RAM, dirty flags (re-checked at the next draw) and the method switch | state registers (not in execution_mask) written with their value are dropped after shadow RAM processing (f265911, `EDEN_NX2_SKIP_SAME=0` off): macro calls 169 ns instead of 185, draw setup 5.80 us instead of 5.95. For non-executable registers ProcessMethodCall is skipped as well (it acts only on executable ones) | +| 136 | native compute: ~20% of the GPU thread under KeplerCompute::ProcessNativeMethod (dispatches 8%, inline QMD/uploads 4%) | its compute macros replay ~64 QMD register writes per dispatch, each through the macro/QMD/upload/semaphore checks | plain native registers stored directly from CallMethod (f7a82b2): its own time 2.2% -> 1.6% | +| 137 | 2D blits (Fermi2D) 3.6% on heavy courses | mip chains 64x64..4x1 copied near the GPU-only regions; neither image is GPU-written, so the software blitter (un)swizzles them on the CPU | open (expected for small copies; many of them) | +| 138 | `async_presentation=true` crashed 4 of 8 runs ~26 s in (row 118) | retested after the fence-operation race (row 118) and translation cache race (row 125) were fixed: 0 crashes in 4 runs, screenshots correct | on in the dev config (`async_presentation\default=false`, `async_presentation=true`); every later run is a crash test for it | +| 139 | buffer collector: up to 64 buffers evicted every frame, GPU-written ones downloaded with a GPU wait (~5% of the GPU thread on 8 GB) | it compared the whole device's use (textures, ~6.8 GB) with thresholds from the device size (6.14 GiB reported): aggressive every frame, while the buffers were ~200 MiB | Switch 2 titles collect on the buffers' own size (1 GiB, aggressive 2 GiB) and keep GPU-written buffers outside aggressive mode (c4eaa4b): 0%, presents no longer blocked on those courses (22-26 ms in t23) | +| 140 | macros ~20% of the GPU thread; the memo's lookups (~76 ns a call in the game, 48 ns in a warm harness) missed the CPU caches | memo records and its map fall out of cache between calls; 4-way last-call slots raised slot hits 48% -> 63% but saved nothing | x86-64 compiler for the Turing MME (a95cedd, `macro_turing_jit.h`, `EDEN_NX2_MME_JIT=0` off): identical to the interpreter on 200,000 recorded calls (`C:\dev\nx2work\mme_test\jit_test.cpp`); in the game 148 ns a call instead of 174, 0 failures, 0 crashes in 3 runs | +| 141 | draw setup ~6.4 us a draw (55,000 a second) | spread over ~12 steps: uniform buffer copies (2.4%, the fast path streams them every draw and bypasses write tracking), texture views (8.5% incl. ~3.6% streaming new course textures), handle reads, vertex/storage buffer lookups | no single cache-able step; open: small per-step trims. The way past it is a second thread (NX2_GPU_THREAD_SPLIT.md) | +| 142 | GPU thread ~100% busy, host GPU 20-40%: one thread parses, runs macros, sets state and records draws | the work is sequential but only a few points need a result from what came before (semaphore acquires, MME DMA reads) | GPU thread split (babd395, `gpu_split.h`, `EDEN_NX2_SPLIT=1`): engine methods run in order on a second thread fed by a lock-free ring; releases, syncpoints, wait-for-idle, flushes and channel binds are deferred into the ring, only acquires wait (~2,400 drains a second). ~47,400 draws a second against ~44,000 (3 A/B rounds), same per-call costs | +| 143 | crashes with the split: CommandChunk::Record (null chunk), vkCmdEndRenderPass on the Vulkan worker | two threads recording into the scheduler: (a) the DMA pusher's split-method debug log read guest memory with the cache-flushing ReadBlock on the front; (b) with async presentation, Swapchain::AcquireNextImage -> Scheduler::Wait flushed the main scheduler from the present thread whenever its tick was not submitted yet (pre-existing race, the split made it frequent) | (a) ReadBlockUnsafe; (b) e4523ab: the present thread waits up to 20 ms for the GPU thread to submit, never flushes. Found with `EDEN_NX2_SPLIT_CHECK=1` (stack of any other thread using the scheduler while the back runs) | +| 144 | macros on the split's front (`EDEN_NX2_SPLIT=2`) need the registers they read before the back has them | the front keeps copies: every write it sends plus what the back changes on its own (constant buffer offset, shadow_scratch[0], macro bind pointer, shadow RAM). `EDEN_NX2_SPLIT_VERIFY=1` found three differences: writes still in the back's method sink at the handover; a multi-word run that changes the shadow RAM mode mid-run; and shadow_state.shadow_ram_control being the shadow copy of register 0x49 itself (a Replay write reads the mode back) | all three modelled: 0 differences in 40,193 checks (~165 million macro calls). Front: macros 128 ns a call (180 on one thread, cache-hot), drops ~55% of 3D state writes as same-value. Back: 50,000-66,500 draws a second against 41,500-47,000 without the split; frames 20.5 / 25.5 ms on two courses where the unsplit runs measured 25.6-40.7 ms. Default stays off until a clean A/B | +| 145 | is the split worth having on by default | clean A/B, nothing else running, 3 rounds each (profiler on in both): one thread 41,100 / 48,600 / 37,000 draws a second, frames 30.5 / 35.0 / 27.1 ms; `EDEN_NX2_SPLIT=2` 59,200 / 62,400 / 62,700 draws a second, frames 28.6 / 23.3 / 25.1 ms. Throughput 1.46x, frames 17% shorter (the back is now ~92% busy, ~53% of it draw setup); 0 crashes, 0 register differences in 57,089 checks after the front's fast paths (no guarded statics, inline ring writes) | on by default for Switch 2 titles (`EDEN_NX2_SPLIT=0` one thread, `=1` macros on the back). Default run without the profiler: 16.9 ms frames (~59 fps) on a light course | +| 146 | title 4 aborts with InvalidHandle (0xE401) at its first movie, ~30 min into a new game | newer SDKs (`nn::codec::video::detail::InitializeServerContext`) open the decoder with gsv 906(u8, u64 library version 0x3'00030000) instead of 902; auto-stubbed, no handle came back | 906 hands out the 902 session (d8d11ec) | +| 147 | title 1: Grand Prix cup-opening screen stays black, decoder registered but never started; then an abort with 2111-3604 (0x1c286f) when it did start | `nn::syncpt::sdk::CancelSyncPointEvent` sends 202 and always waits on the slot's event; we signalled it only for pending requests. Then the movie player reused slot 0, which NVN's fence waits had left signalled, read it as a decoded frame and found none | 202 always signals, 201 clears the slot's event first (b09629f) | +| 148 | title 1/4 movies have no picture | the video decoder service (nn::codec::video, gsv 902/906/1002 -> vdec -> vdecsvc) completed frames without decoding | FFmpeg decodes on a thread per decoder service (`video_core/host1x/nx2_video_decoder`): 101 names the codec ("video/vp9") in an A buffer, 109 submits frames (A buffer, pts us). 311/314 reply NativeSyncHandles {~syncpoint, 0, value} on a syncpoint the service allocates, incremented per decoded frame, frames filling images in register/release order. gsv 911 creates the pool's images, 961 returns the CpuUserImageDescriptor that `ReadDecoderOutputImage*` copy out with (pitch-linear NV12/RGBA/BGRA, kinds 10/8/9). Title 1's Mushroom Cup opening (2560x1440 VP9, 1423 frames) decodes in real time and the game goes on to the race. The picture still stayed on the first frame: see 149. `EDEN_NX2_VDEC_DUMP=<dir>` saves decoded frames | +| 149 | title 1: the cup-opening movie's picture stays on its first frame although every frame decodes and reaches GPU memory | not the texture cache: the movie is drawn into a 2560x1440 render target whose NVN2 texture has GPU address 0 (also the 640x360 and 512x512 passes), so the draws go nowhere. NVN2's texture init (sdk+0x5ca240) places a texture in a compressible pool (flags 0x080000a1) only after `NvRmGpuMappingModify` switched the pool's mapping (0x815f0d0000, a gpu 214 region) to the texture's kind; NvRm fails that call with 2 unless the characteristics flags have NVGPU_GPU_FLAGS_SUPPORT_MAPPING_MODIFY (bit 34; ours were 0x4400155), and the texture keeps address 0 | report bit 34; the modify then runs through nn::gpu::sdk's VirtualMachine as gpu sub 212 (NormalModifyMemoryKind4K: words 3-6 range, 7 wanted kind, 8 fallback, 10 job slot; reply word 3 the kind applied), answered with the wanted kind and the slot signalled. The movie plays on screen | + +Run settings used (in `build/bin/user/config/sdl2-config.ini`): `memory_layout_mode=4` +(12 GB). Debug knobs: `EDEN_NX2_TC_EXPECTED_MIB` / `EDEN_NX2_TC_CRITICAL_MIB` (collector thresholds), +`EDEN_NX2_ASTC_FLUSH=1` (GPU ASTC decode without the full wait), `EDEN_NX2_CRASH_DUMP=<dir>` (prints the +faulting stack as module+offset, then a minidump), `EDEN_NX2_AUTO_STUB`, `EDEN_NX2_IPC_TRACE`, `EDEN_NX2_SVC_TRACE`, +`EDEN_NX2_PULLER_TRACE`, `EDEN_NX2_METHOD_TRACE`, `EDEN_NX2_HIDE_WINDOW`, +`EDEN_NX2_GPU_PROFILE` (GPU-thread zone timers every 10 s), `EDEN_NX2_SAMPLE=<threads>` with +`_START`/`_SECONDS`/`_OUT` (sampling profiler; `C:\dev\nx2work\folded.py` reads its .folded stacks), +`EDEN_NX2_MME_RECORD=<file>` (macro calls for `C:\dev\nx2work\mme_test`), +`EDEN_NX2_FENCE_BATCH=0`, `EDEN_NX2_MACRO_COPY=1`, `EDEN_NX2_CNTPCT_PAUSE=0` (turn rows 124, 126, 127 off), +`EDEN_NX2_MME_MEMO_STATS` (memo counters), +`EDEN_NX2_BUFFER_DOWNLOADS=0/1`, `EDEN_NX2_INLINE_DEFER=0`, `EDEN_NX2_TC_MIN_AGE=<frames>` (rows 131, 134, 133), +`EDEN_NX2_SKIP_SAME=0` (row 135), +`EDEN_NX2_MME_JIT=0` (row 140), +`EDEN_NX2_SPLIT=0/1` (default 2), `EDEN_NX2_SPLIT_VERIFY=1`, `EDEN_NX2_SPLIT_CHECK=1`, `EDEN_NX2_SPLIT_SKIP=0` (rows 142-145), +`EDEN_NX2_FRAME_CAPTURE=<dir>` (every 120th presented frame as TGA), `EDEN_NX2_PB_CHECK` +(pushbuffer entries changed between submit and execution), `EDEN_NX2_QUERY_TRACE` (every GPU +report/semaphore write). `C:\dev\crashrun.sh` runs eden-cli under cdb and prints the crash stack; +`C:\dev\stackdump.sh <main> <seconds>` dumps host thread stacks of a running test. + +## Where it stops now + +All runs: `EDEN_NX2_AUTO_STUB=1`, 12 GB memory layout, loose ExeFS with the sibling `romfs/`. + +**Title 4** (`fourth switch game\exefs`, 0400380020056000) **plays its intro cutscene correctly +rendered**: boot, loading screen, menu (difficulty select, keyboard input), confirm, then the intro +(plane cockpit, newspaper/map) lit, colour-graded and subtitled, matching console footage. The +DLSS-style upscale is a bilinear stand-in. In a played session the game went on to a later loading +screen that did not finish; open there: a compute shader with a 64-bit add (UIADD3 carry-out to +UP0), TMML (0xb69) in two graphics shaders, "reserved attribute 23" (a[0x5c]) reads. Scripted runs +(`C:\dev\run.sh`) move any title-4 save to `C:\dev\save_backups` first so they start a new game. +Scripted input: `EDEN_NX2_KEY_SCRIPT="50:4,50.5:-4,55:79,55.4:-79,57:4,57.5:-4"`. + +**Title 3**: last checked before the graphics work; should now track title 4. + +**Title 1** (`first switch game\exefs`, 0400C3F00006E000, `Drive.nss`): boots, shows the world-map +loading screen and reaches its title screen ("Press L + R to start") with sky, buildings, props, +kart and logo drawn. Wrong: magenta/cyan colour cast, black ground (unbacked virtual texture, row +87), task/mesh geometry missing (row 89), ~10-17 fps. Launch with frame capture: +`EDEN_NX2_FRAME_CAPTURE=C:/dev/frames_t1x EDEN_NX2_AUTO_STUB=1 EDEN_NX2_SHADER_DUMP=C:/dev/shaders_title1 ./eden-cli.exe -g "C:/switch/first switch game/exefs/main"`. +Offline shader check: `nx2work/lowertest/lowertest.exe C:/dev/shaders_title1 [-all [op]]`. + +**Title 2**: main/sdk/subsdk0 fail LZ4 and SHA-256 checks; can't be run from these files. + +## Prioritized next steps + +1. **Title 4 past the intro**: lower the 64-bit add with carry (UIADD3/IADD3 carry-out and .X + carry-in), TMML, and attribute 0x5c; then whatever the next area needs (dump + lowertest). + Squid/paint-splat hatch on the menu. A faster tensor-core emulation (shared-memory HMMA) would + replace the bilinear upscale stand-in. +2. **Title 1 / title 3** retests with the current tree; dump their shaders and widen the lowering. +3. Replace guesses with real protocol knowledge: syncpt page layout, the semaphore mirror page, + gsv 214 (GPU-only memory) and vi2 present buffer availability (real buffer release instead of + the fence wait), gsv 1510 formats, svcGetInfo 29-33. +4. **Six cores** instead of folding 4/5 onto 0/1; rare SIMD encodings still undecoded. +5. Upstream-worthy pieces independent of Switch 2: MSVC 14.51 build fixes, eden-cli SDL3 + lifetime fixes and `_setmaxstdio`, ExeFS loader fixes, FAC/FAH offset fix, GetPseudoDeviceId + without NACP. diff --git a/README.md b/README.md index 7d3fe5572e..c0d8b65d8a 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ For information on provided development tooling, see the [Tools directory](./too ## Download -You can download the latest releases from [our release page](https://git.eden-emu.dev/eden-emu/eden/releases). +You can download the latest releases from [our release page](https://git.eden-emu.dev/Takoroka/eden-nx2/releases). Save us some bandwidth! We have [mirrors available](./docs/user/ThirdParty.md#mirrors) as well. diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp index a88eae91ad..ca3eaece94 100644 --- a/src/audio_core/renderer/behavior/info_updater.cpp +++ b/src/audio_core/renderer/behavior/info_updater.cpp @@ -588,7 +588,11 @@ Result InfoUpdater::UpdateBehaviorInfo(BehaviorInfo& behaviour_) { return Service::Audio::ResultInvalidUpdateInfo; } - if (in_params->revision != behaviour_.GetUserRevision()) { + // Compare revision numbers: Switch 2 opens the renderer with 'REV?' but tags its updates + // 'REv?' (lowercase v) for the same revision. + if (GetRevisionNum(in_params->revision) != GetRevisionNum(behaviour_.GetUserRevision())) { + LOG_ERROR(Service_Audio, "BehaviorInfo revision {:08X} does not match the renderer's {:08X}", + in_params->revision, behaviour_.GetUserRevision()); return Service::Audio::ResultInvalidUpdateInfo; } diff --git a/src/audio_core/sink/sink_stream.cpp b/src/audio_core/sink/sink_stream.cpp index c70020e9da..9dfd3bf747 100644 --- a/src/audio_core/sink/sink_stream.cpp +++ b/src/audio_core/sink/sink_stream.cpp @@ -4,8 +4,10 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> #include <array> #include <atomic> +#include <cstdlib> #include <memory> #include <span> #include <vector> @@ -213,6 +215,19 @@ void SinkStream::ProcessAudioOutAndRender(std::span<s16> output_buffer, std::siz std::memcpy(last_frame.data(), &output_buffer[(frames_written - 1) * frame_size], frame_size_bytes); + // EDEN_NX2_AUDIO_LEVEL: log the output peak about every two seconds (is a game audible?). + if (static const bool meter = std::getenv("EDEN_NX2_AUDIO_LEVEL") != nullptr; meter) { + for (size_t i = 0; i < num_frames * frame_size; ++i) { + level_peak = std::max<s32>(level_peak, std::abs(static_cast<s32>(output_buffer[i]))); + } + level_frames += num_frames; + if (level_frames >= TargetSampleRate * 2) { + LOG_INFO(Audio, "Audio output peak over {} frames: {} of 32767", level_frames, level_peak); + level_frames = 0; + level_peak = 0; + } + } + { std::scoped_lock lk{sample_count_lock}; last_sample_count_update_time = system.CoreTiming().GetGlobalTimeNs(); diff --git a/src/audio_core/sink/sink_stream.h b/src/audio_core/sink/sink_stream.h index acb7537dda..280b7747ec 100644 --- a/src/audio_core/sink/sink_stream.h +++ b/src/audio_core/sink/sink_stream.h @@ -245,6 +245,9 @@ private: SinkBuffer playing_buffer{}; /// The last played (or received) frame of audio, used when the callback underruns std::array<s16, MaxChannels> last_frame{}; + /// EDEN_NX2_AUDIO_LEVEL meter state. + s32 level_peak{}; + u64 level_frames{}; /// Number of buffers waiting to be played std::atomic<u32> queued_buffers{}; /// The ring size for audio out buffers (usually 4, rarely 2 or 8) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 5d1b30a549..d3e7b03cd2 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -96,6 +96,7 @@ add_library( ${CMAKE_CURRENT_BINARY_DIR}/scm_rev.cpp scm_rev.h scope_exit.h + spin_recursive_mutex.h scratch_buffer.h settings.cpp settings.h @@ -151,7 +152,7 @@ add_library( set_source_files_properties(zbic_compression.cpp PROPERTIES INCLUDE_DIRECTORIES "${ZBIC_INCLUDE_DIR}" - COMPILE_OPTIONS "$<$<CXX_COMPILER_ID:Clang,GNU>:-Wno-unused-function;-Wno-missing-declarations;-Wno-shadow>") + COMPILE_OPTIONS "$<$<CXX_COMPILER_ID:Clang,GNU>:-Wno-unused-function;-Wno-missing-declarations;-Wno-shadow>;$<$<CXX_COMPILER_ID:MSVC>:/wd4505;/wd5245>") if(WIN32) target_sources(common PRIVATE windows/timer_resolution.cpp diff --git a/src/common/container/unordered_map.h b/src/common/container/unordered_map.h index 7d9e49fd80..c4d4a72663 100644 --- a/src/common/container/unordered_map.h +++ b/src/common/container/unordered_map.h @@ -9,7 +9,7 @@ namespace Common { -template <class Key, class T, class Hash = std::hash<Key>, class Pred = std::equal_to<Key>, +template <class Key, class T, class Hash = FastHash<Key>, class Pred = std::equal_to<Key>, class Allocator = std::allocator<std::pair<const Key, T>>> using unordered_map = boost::unordered::unordered_flat_map<Key, T, Hash, Pred, Allocator>; diff --git a/src/common/container/unordered_set.h b/src/common/container/unordered_set.h index 54d71406a1..690c33286f 100644 --- a/src/common/container/unordered_set.h +++ b/src/common/container/unordered_set.h @@ -9,7 +9,7 @@ namespace Common { -template <class Key, class Hash = std::hash<Key>, class Pred = std::equal_to<Key>, +template <class Key, class Hash = FastHash<Key>, class Pred = std::equal_to<Key>, class Allocator = std::allocator<Key>> using unordered_set = boost::unordered::unordered_flat_set<Key, Hash, Pred, Allocator>; diff --git a/src/common/container_hash.h b/src/common/container_hash.h index d7477f7e4f..f06f69ad8b 100644 --- a/src/common/container_hash.h +++ b/src/common/container_hash.h @@ -19,6 +19,28 @@ namespace Common { +/// The default hash of Common::unordered_map/set. std::hash of an integer or a pointer is a +/// byte-wise FNV-1a on MSVC, and boost's flat containers mix a hash that does not say it +/// avalanches anyway: pass those keys through (the GPU thread's page maps looked keys up +/// tens of thousands of times a second with it). Other keys keep std::hash. +template <class Key> +struct FastHash : std::hash<Key> {}; + +template <class Key> + requires std::is_integral_v<Key> +struct FastHash<Key> { + std::size_t operator()(Key key) const noexcept { + return static_cast<std::size_t>(key); + } +}; + +template <class T> +struct FastHash<T*> { + std::size_t operator()(T* pointer) const noexcept { + return reinterpret_cast<std::size_t>(pointer); + } +}; + namespace detail { template <typename T> diff --git a/src/common/cpu_features.cpp b/src/common/cpu_features.cpp index 8ccb6847f7..52fd003c44 100644 --- a/src/common/cpu_features.cpp +++ b/src/common/cpu_features.cpp @@ -225,6 +225,13 @@ const CPUCaps g_cpu_caps = [] { } } + // AMD CPUs have no leaf 0x15 (Zen 4 stops at 0x10): measure their invariant TSC too, or every + // guest clock read (CNTPCT, which games read thousands of times a frame) goes through the + // host's much slower QueryPerformanceCounter. + if (caps.invariant_tsc && caps.tsc_frequency == 0) { + caps.tsc_frequency = X64::EstimateRDTSCFrequency(); + } + if (max_std_fn >= 0x16) { __cpuid(cpu_id, 0x16); caps.base_frequency = cpu_id[0]; diff --git a/src/common/range_sets.h b/src/common/range_sets.h index f8fcee4837..e5cb53b058 100644 --- a/src/common/range_sets.h +++ b/src/common/range_sets.h @@ -26,6 +26,9 @@ public: void Clear(); bool Empty() const; + /// Whether any range overlaps [base_address, base_address + size): one lookup. + bool Intersects(AddressType base_address, size_t size) const; + template <typename Func> void ForEach(Func&& func) const; diff --git a/src/common/range_sets.inc b/src/common/range_sets.inc index 3edd8c8a43..aeda7bc351 100644 --- a/src/common/range_sets.inc +++ b/src/common/range_sets.inc @@ -116,28 +116,25 @@ struct OverlapRangeSet<AddressType>::OverlapRangeSetImpl { } AddressType end_address = base_address + static_cast<AddressType>(size); IntervalType interval{base_address, end_address}; - bool any_removals = false; m_split_ranges_set += std::make_pair(interval, -amount); - do { - any_removals = false; - auto it = m_split_ranges_set.lower_bound(interval); - if (it == m_split_ranges_set.end()) { - return; + // One pass, erasing the segments that dropped to zero or below in address order (the + // search restarted after every erase: quadratic in the segments of the range, and the + // GPU fencing thread does this for each downloaded buffer range while holding the buffer + // cache's lock). Erasing a node leaves the other iterators valid. + auto it = m_split_ranges_set.lower_bound(interval); + const auto end_it = m_split_ranges_set.upper_bound(interval); + while (it != end_it) { + if (it->second > 0) { + ++it; + continue; } - auto end_it = m_split_ranges_set.upper_bound(interval); - for (; it != end_it; it++) { - if (it->second <= 0) { - if constexpr (has_on_delete) { - if (it->second == 0) { - on_delete(it->first.lower(), it->first.upper()); - } - } - any_removals = true; - m_split_ranges_set.erase(it); - break; + if constexpr (has_on_delete) { + if (it->second == 0) { + on_delete(it->first.lower(), it->first.upper()); } } - } while (any_removals); + m_split_ranges_set.erase(it++); + } } template <typename Func> @@ -223,6 +220,18 @@ bool RangeSet<AddressType>::Empty() const { return m_impl->m_ranges_set.empty(); } +template <typename AddressType> +bool RangeSet<AddressType>::Intersects(AddressType base_address, size_t size) const { + const auto& set = m_impl->m_ranges_set; + if (set.empty() || size == 0) { + return false; + } + const AddressType end_address = base_address + static_cast<AddressType>(size); + // The first range not entirely below the query; it overlaps when it starts before its end. + const auto it = set.lower_bound(typename RangeSetImpl::IntervalType{base_address, end_address}); + return it != set.end() && it->lower() < end_address; +} + template <typename AddressType> template <typename Func> void RangeSet<AddressType>::ForEach(Func&& func) const { diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp index eee8ff6d18..b96340999d 100644 --- a/src/common/sparse_large_vector.cpp +++ b/src/common/sparse_large_vector.cpp @@ -26,7 +26,7 @@ static std::mutex vector_regions_mutex {}; // Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { DWORD code = info->ExceptionRecord->ExceptionCode; - u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]); + u64 exception_addr = static_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]); if (code != EXCEPTION_ACCESS_VIOLATION || info->ExceptionRecord->ExceptionInformation[0] == 1) { // Not our problem diff --git a/src/common/spin_recursive_mutex.h b/src/common/spin_recursive_mutex.h new file mode 100644 index 0000000000..291dd4e41e --- /dev/null +++ b/src/common/spin_recursive_mutex.h @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <atomic> +#include <cstdint> +#include <thread> + +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) +#include <immintrin.h> +#endif + +namespace Common { + +/// A recursive mutex that spins briefly before sleeping. The GPU caches' locks are taken tens of +/// thousands of times a second by the GPU thread, the fencing thread and CPU write traps, and +/// held for microseconds: std::recursive_mutex put a waiting thread to sleep at once, and its +/// wake-up took longer than the critical section (title 1's GPU thread spent ~8% asleep there). +class SpinRecursiveMutex { +public: + void lock() { + const std::thread::id self = std::this_thread::get_id(); + if (owner.load(std::memory_order_relaxed) == self) { + ++depth; + return; + } + std::uint32_t expected = 0; + if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + LockContended(); + } + owner.store(self, std::memory_order_relaxed); + depth = 1; + } + + bool try_lock() { + const std::thread::id self = std::this_thread::get_id(); + if (owner.load(std::memory_order_relaxed) == self) { + ++depth; + return true; + } + std::uint32_t expected = 0; + if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return false; + } + owner.store(self, std::memory_order_relaxed); + depth = 1; + return true; + } + + void unlock() { + if (--depth != 0) { + return; + } + owner.store(std::thread::id{}, std::memory_order_relaxed); + if (state.exchange(0, std::memory_order_release) == 2) { + state.notify_one(); + } + } + +private: + // About 10 microseconds of spinning before sleeping. + static constexpr int SpinCount = 1024; + + static void Pause() { +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) + _mm_pause(); +#elif defined(__aarch64__) + asm volatile("yield"); +#endif + } + + void LockContended() { + for (int spin = 0; spin < SpinCount; ++spin) { + Pause(); + if (state.load(std::memory_order_relaxed) == 0) { + std::uint32_t expected = 0; + if (state.compare_exchange_weak(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return; + } + } + } + // 2: held with (possible) sleepers; taken this way it stays marked, so its unlock wakes + // the next sleeper. + while (state.exchange(2, std::memory_order_acquire) != 0) { + state.wait(2, std::memory_order_relaxed); + } + } + + std::atomic<std::uint32_t> state{0}; ///< 0 free, 1 held, 2 held with sleepers + std::atomic<std::thread::id> owner{}; + std::uint32_t depth{}; ///< Recursion depth, touched only by the owner +}; + +} // namespace Common diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1b578f67ba..56fa653738 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -457,6 +457,7 @@ add_library(core STATIC hle/service/am/service/application_functions.cpp hle/service/am/service/application_functions.h hle/service/am/service/application_proxy.cpp + hle/service/am/service/graphics_scale_controller.h hle/service/am/service/application_proxy.h hle/service/am/service/application_proxy_service.cpp hle/service/am/service/application_proxy_service.h @@ -526,6 +527,7 @@ add_library(core STATIC hle/service/audio/audio_out.h hle/service/audio/audio_renderer_manager.cpp hle/service/audio/audio_renderer_manager.h + hle/service/audio/nx2_audio_user_client.h hle/service/audio/audio_renderer.cpp hle/service/audio/audio_renderer.h hle/service/audio/audio.cpp @@ -865,6 +867,7 @@ add_library(core STATIC hle/service/nvdrv/devices/nvhost_vic.h hle/service/nvdrv/devices/nvmap.cpp hle/service/nvdrv/devices/nvmap.h + hle/service/nvdrv/devices/nx2_stub_device.h hle/service/nvdrv/nvdata.h hle/service/nvdrv/nvdrv.cpp hle/service/nvdrv/nvdrv.h @@ -1033,6 +1036,8 @@ add_library(core STATIC hle/service/server_manager.h hle/service/service.cpp hle/service/service.h + hle/service/nx2_stub_interface.h + hle/service/nx2_gsv.h hle/service/services.cpp hle/service/services.h hle/service/set/factory_settings_server.cpp diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 894857a86d..a2c0124f33 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -4,6 +4,11 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <atomic> +#include <cstdlib> +#if defined(_M_X64) || defined(__x86_64__) +#include <immintrin.h> +#endif #include "common/settings.h" #include "core/arm/dynarmic/arm_dynarmic.h" #include "core/arm/dynarmic/arm_dynarmic_64.h" @@ -24,13 +29,24 @@ DynarmicCallbacks64::DynarmicCallbacks64(ArmDynarmic64& parent, Kernel::KProcess u64 DynarmicCallbacks64::MemoryRead(u64 vaddr, size_t size) { CheckMemoryAccess(vaddr, size, Kernel::DebugWatchpointType::Read); + u64 value{}; switch (size) { - case sizeof(u64): return m_memory.Read64(vaddr); - case sizeof(u32): return m_memory.Read32(vaddr); - case sizeof(u16): return m_memory.Read16(vaddr); - case sizeof(u8): return m_memory.Read8(vaddr); + case sizeof(u64): value = m_memory.Read64(vaddr); break; + case sizeof(u32): value = m_memory.Read32(vaddr); break; + case sizeof(u16): value = m_memory.Read16(vaddr); break; + case sizeof(u8): value = m_memory.Read8(vaddr); break; default: UNREACHABLE(); } + // With memory aborts ignored the guest keeps running on zeros: say where it read from. + if (auto& unmapped = Core::Memory::UnmappedReadFlag(); unmapped != 0) { + unmapped = 0; + static std::atomic<u32> logged{}; + if (logged.fetch_add(1) < 32) { + LOG_ERROR(Core_ARM, "Unmapped read {:#x} ({} bytes) at pc {:#x} lr {:#x} sp {:#x}", vaddr, size, + m_parent.m_jit->GetPC(), m_parent.m_jit->GetRegister(30), m_parent.m_jit->GetSP()); + } + } + return value; } Dynarmic::A64::Vector DynarmicCallbacks64::MemoryRead128(u64 vaddr) { CheckMemoryAccess(vaddr, 16, Kernel::DebugWatchpointType::Read); @@ -154,7 +170,26 @@ u64 DynarmicCallbacks64::GetTicksRemaining() { } u64 DynarmicCallbacks64::GetCNTPCT() { - return m_parent.m_system.CoreTiming().GetClockTicks(); + const u64 ticks = m_parent.m_system.CoreTiming().GetClockTicks(); + // Guest code that waits by polling the counter (title 1's main thread reads it ~17 million + // times a second while waiting for the GPU) keeps a host core busy at full speed, and the + // GPU thread is often on the other half of that SMT core. Two reads within ~1.7 us of each + // other are such a loop: pause, which hands the core's resources to the sibling thread for a + // moment. The guest sees the same counter. EDEN_NX2_CNTPCT_PAUSE=0 turns it off. + static const bool pause_spins = [] { + const char* value = std::getenv("EDEN_NX2_CNTPCT_PAUSE"); + return value == nullptr || value[0] != '0'; + }(); + if (pause_spins && ticks - m_last_cntpct < 32) { +#if defined(_M_X64) || defined(__x86_64__) + _mm_pause(); + _mm_pause(); +#elif defined(__aarch64__) + asm volatile("yield"); +#endif + } + m_last_cntpct = ticks; + return ticks; } bool DynarmicCallbacks64::CheckMemoryAccess(u64 addr, u64 size, Kernel::DebugWatchpointType type) { diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.h b/src/core/arm/dynarmic/arm_dynarmic_64.h index 5ea5dde8a6..acd1fe773f 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.h +++ b/src/core/arm/dynarmic/arm_dynarmic_64.h @@ -65,6 +65,7 @@ public: Core::Memory::Memory& m_memory; u64 m_tpidrro_el0{}; u64 m_tpidr_el0{}; + u64 m_last_cntpct{}; ///< The previous counter read (GetCNTPCT spin detection) Kernel::KProcess* m_process{}; const bool m_debugger_enabled{}; const bool m_check_memory_access{}; diff --git a/src/core/core.cpp b/src/core/core.cpp index 06543b0833..f3d224b1ce 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -24,6 +24,7 @@ #include "core/device_memory.h" #include "core/file_sys/fs_filesystem.h" #include "core/file_sys/patch_manager.h" +#include "core/file_sys/program_metadata.h" #include "core/file_sys/registered_cache.h" #include "core/file_sys/romfs_factory.h" #include "core/file_sys/savedata_factory.h" @@ -68,6 +69,8 @@ namespace Core { +std::atomic<u64> g_nx2_present_count{}; + FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs, const std::string& path) { // To account for split 00+01+etc files. @@ -107,6 +110,19 @@ FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs, return vfs->OpenFile(path, FileSys::OpenMode::Read); } +namespace { + +// Switch 2 titles boot as a loose ExeFS, with main.npdm next to main. +bool IsSwitch2Program(const FileSys::VirtualFile& file) { + const auto dir = file != nullptr ? file->GetContainingDirectory() : nullptr; + const auto npdm = dir != nullptr ? dir->GetFile("main.npdm") : nullptr; + FileSys::ProgramMetadata metadata; + return npdm != nullptr && metadata.Load(npdm) == Loader::ResultStatus::Success && + metadata.IsSwitch2(); +} + +} // namespace + struct System::Impl { explicit Impl(System& system) : kernel{system}, fs_controller{system}, hid_core{kernel}, cpu_manager{system}, @@ -297,6 +313,16 @@ struct System::Impl { fs_controller.InitTempStorage(); } + // Switch 2 titles size their heap from the total memory and need the 12 GiB layout. Set it + // as a per-game value, so RestoreGlobalState brings back the configured layout afterwards. + if (virtual_filesystem != nullptr && + Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_12Gb && + IsSwitch2Program(GetGameFileFromPath(virtual_filesystem, filepath))) { + LOG_INFO(Core, "Switch 2 program: using the 12 GiB memory layout"); + Settings::values.memory_layout_mode.SetGlobal(false); + Settings::values.memory_layout_mode.SetValue(Settings::MemoryLayout::Memory_12Gb); + } + InitializeKernel(system); if (params.applet_type == Service::AM::AppletType::Application) { diff --git a/src/core/core.h b/src/core/core.h index d88ed64151..0276679d76 100644 --- a/src/core/core.h +++ b/src/core/core.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <cstddef> #include <deque> #include <functional> @@ -115,6 +116,9 @@ class RenderdocAPI; namespace Core { +/// Frames the guest has presented (Switch 2 vi2 present calls); input replay is locked to it. +extern std::atomic<u64> g_nx2_present_count; + class CpuManager; class Debugger; class DeviceMemory; diff --git a/src/core/debugger/gdbstub.cpp b/src/core/debugger/gdbstub.cpp index 44ddf11c39..71af2acee3 100644 --- a/src/core/debugger/gdbstub.cpp +++ b/src/core/debugger/gdbstub.cpp @@ -25,7 +25,10 @@ #include "core/debugger/gdbstub_arch.h" #include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_readable_event.h" #include "core/hle/kernel/k_thread.h" +#include "core/hle/service/kernel_helpers.h" #include "core/loader/loader.h" #include "core/memory.h" @@ -731,8 +734,26 @@ void GDBStub::HandleRcmd(const std::vector<u8>& command) { break; cur_addr = next_address; } + } else if (command_str == "handles" || command_str == "get handles") { + // Every handle of the process: its object type and, for service events, their name. + reply = "Handles:\n"; + auto& handle_table = debug_process->GetHandleTable(); + for (size_t i = 0; i < handle_table.GetTableSize(); ++i) { + Kernel::Handle handle{}; + auto object = handle_table.GetObjectByIndex(system.Kernel(), std::addressof(handle), i); + if (object.IsNull()) { + continue; + } + std::string name; + if (auto* readable = object->DynamicCast<Kernel::KReadableEvent*>(); readable != nullptr) { + name = Service::KernelHelpers::ServiceContext::GetEventName(readable->GetParent()); + } else if (auto* event = object->DynamicCast<Kernel::KEvent*>(); event != nullptr) { + name = Service::KernelHelpers::ServiceContext::GetEventName(event); + } + reply += fmt::format(" {:#010x} {} {}\n", handle, object->GetTypeObj().GetName(), name); + } } else { - reply += "Commands: fastmem, info, mappings\n"; + reply += "Commands: fastmem, info, mappings, handles\n"; } std::span<const u8> reply_span{reinterpret_cast<u8*>(&reply.front()), reply.size()}; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index b4b3b46088..46cda14e78 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -132,6 +132,27 @@ private: u8* host_ptr{}; }; + /// A thread's last four page translations of one manager. Several threads read through + /// them (the GPU thread, the fencing thread, service threads): one cache shared by all was + /// cleared by Map on a service thread between the GPU thread's null check of an entry and + /// its use, and the GPU thread copied from address 0 (title 1 crashed every few minutes). + struct TranslationCache { + const void* owner{}; + u64 generation{}; + std::array<TranslationEntry, 4> slots{}; + u32 cursor{}; + }; + static inline thread_local TranslationCache t_cache{}; + + /// This thread's translations, emptied when Map or Unmap ran since they were made. + TranslationCache& Translations() { + const u64 current = translation_generation.load(std::memory_order_acquire); + if (t_cache.owner != this || t_cache.generation != current) { + t_cache = TranslationCache{.owner = this, .generation = current}; + } + return t_cache; + } + // Internal helper that performs the update assuming the caller already holds the necessary lock. void UpdatePagesCachedCountNoLock(DAddr addr, size_t size, s32 delta); @@ -139,7 +160,7 @@ private: static constexpr size_t device_virtual_bits = Traits::device_virtual_bits; static constexpr size_t device_as_size = 1ULL << device_virtual_bits; static constexpr size_t physical_min_bits = 32; - static constexpr size_t physical_max_bits = 33; + static constexpr size_t physical_max_bits = 35; // covers the 12 GiB layout plus nx2 graphics server memory static constexpr size_t page_bits = 12; static constexpr size_t page_size = 1ULL << page_bits; static constexpr size_t page_mask = page_size - 1ULL; @@ -204,8 +225,7 @@ private: tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); } - std::array<TranslationEntry, 4> t_slot{}; - u32 cache_cursor = 0; + std::atomic<u64> translation_generation{1}; ///< Bumped by Map and Unmap using CounterType = u8; using CounterAtomicType = std::atomic_uint8_t; static constexpr size_t subentries = 8 / sizeof(CounterType); diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 97c2706abb..d70444ef8c 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -234,7 +234,7 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size } impl->multi_dev_address.Register(new_dev, start_id); } - t_slot = {}; + translation_generation.fetch_add(1, std::memory_order_release); if (track) { TrackContinuityImpl(address, virtual_address, size, asid); } @@ -268,7 +268,7 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) { base_dev = new_start | MULTI_FLAG; } } - t_slot = {}; + translation_generation.fetch_add(1, std::memory_order_release); } template <typename Traits> void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtual_address, @@ -404,9 +404,10 @@ void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, s const std::size_t page_offset = address & Memory::YUZU_PAGEMASK; if (size <= Memory::YUZU_PAGESIZE - page_offset) { const DAddr guest_page = address & ~static_cast<DAddr>(Memory::YUZU_PAGEMASK); - for (size_t i = 0; i < 4; ++i) { - if (t_slot[i].guest_page == guest_page && t_slot[i].host_ptr != nullptr) { - std::memcpy(dest_pointer, t_slot[i].host_ptr + page_offset, size); + TranslationCache& cache = Translations(); + for (const TranslationEntry& entry : cache.slots) { + if (entry.guest_page == guest_page && entry.host_ptr != nullptr) { + std::memcpy(dest_pointer, entry.host_ptr + page_offset, size); return; } } @@ -415,8 +416,8 @@ void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, s const auto phys_addr = tracked_entries[page_index].compressed_physical_ptr; if (phys_addr != 0) { auto* const mem_ptr = GetPointerFromRaw<u8>((PAddr(phys_addr - 1) << Memory::YUZU_PAGEBITS)); - t_slot[cache_cursor % t_slot.size()] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr}; - cache_cursor = (cache_cursor + 1) & 3U; + cache.slots[cache.cursor] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr}; + cache.cursor = (cache.cursor + 1) & 3U; std::memcpy(dest_pointer, mem_ptr + page_offset, size); return; } @@ -462,9 +463,10 @@ void DeviceMemoryManager<Traits>::ReadBlockUnsafe(DAddr address, void* dest_poin const std::size_t page_offset = address & Memory::YUZU_PAGEMASK; if (size <= Memory::YUZU_PAGESIZE - page_offset) { const DAddr guest_page = address & ~static_cast<DAddr>(Memory::YUZU_PAGEMASK); - for (size_t i = 0; i < 4; ++i) { - if (t_slot[i].guest_page == guest_page && t_slot[i].host_ptr != nullptr) { - std::memcpy(dest_pointer, t_slot[i].host_ptr + page_offset, size); + TranslationCache& cache = Translations(); + for (const TranslationEntry& entry : cache.slots) { + if (entry.guest_page == guest_page && entry.host_ptr != nullptr) { + std::memcpy(dest_pointer, entry.host_ptr + page_offset, size); return; } } @@ -473,8 +475,8 @@ void DeviceMemoryManager<Traits>::ReadBlockUnsafe(DAddr address, void* dest_poin const auto phys_addr = tracked_entries[page_index].compressed_physical_ptr; if (phys_addr != 0) { auto* const mem_ptr = GetPointerFromRaw<u8>((PAddr(phys_addr - 1) << Memory::YUZU_PAGEBITS)); - t_slot[cache_cursor % t_slot.size()] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr}; - cache_cursor = (cache_cursor + 1) & 3U; + cache.slots[cache.cursor] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr}; + cache.cursor = (cache.cursor + 1) & 3U; std::memcpy(dest_pointer, mem_ptr + page_offset, size); return; } diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp index 63ca9d3fe3..464e496ea3 100644 --- a/src/core/file_sys/program_metadata.cpp +++ b/src/core/file_sys/program_metadata.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <cstddef> +#include <cstring> #include <vector> #include "common/logging.h" @@ -33,13 +34,30 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { return Loader::ResultStatus::ErrorBadACIDHeader; } + // Switch 2 ACID blocks have a 0x300-byte signature/key area instead of 0x200, so the + // header starts 0x100 bytes later. Offsets inside the header are unchanged. + if (std::memcmp(acid_header.magic.data(), "ACID", 4) != 0) { + constexpr std::size_t acid_sig_size_v1 = 0x200; + constexpr std::size_t acid_sig_size_v2 = 0x300; + constexpr std::size_t acid_body_size = sizeof(AcidHeader) - acid_sig_size_v1; + auto* const body = reinterpret_cast<u8*>(&acid_header) + acid_sig_size_v1; + if (file->ReadBytes(body, acid_body_size, npdm_header.acid_offset + acid_sig_size_v2) != + acid_body_size || + std::memcmp(acid_header.magic.data(), "ACID", 4) != 0) { + return Loader::ResultStatus::ErrorBadACIDHeader; + } + LOG_INFO(Loader, "NPDM uses the Switch 2 ACID layout"); + is_switch2 = true; + } + if (sizeof(AciHeader) != file->ReadObject(&aci_header, npdm_header.aci_offset)) { return Loader::ResultStatus::ErrorBadACIHeader; } // Load acid_file_access per-component instead of the entire struct, since this struct does not // reflect the layout of the real data. - std::size_t current_offset = acid_header.fac_offset; + // FAC/FAH offsets are relative to the start of the ACID/ACI block, like the KAC offsets. + std::size_t current_offset = npdm_header.acid_offset + acid_header.fac_offset; if (sizeof(FileAccessControl::version) != file->ReadBytes(&acid_file_access.version, sizeof(FileAccessControl::version), current_offset)) { @@ -57,7 +75,7 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { } // Load aci_file_access per-component instead of the entire struct, same as acid_file_access - current_offset = aci_header.fah_offset; + current_offset = npdm_header.aci_offset + aci_header.fah_offset; if (sizeof(FileAccessHeader::version) != file->ReadBytes(&aci_file_access.version, sizeof(FileAccessHeader::version), current_offset)) { diff --git a/src/core/file_sys/program_metadata.h b/src/core/file_sys/program_metadata.h index 115e6d6cd1..5ce32af1a4 100644 --- a/src/core/file_sys/program_metadata.h +++ b/src/core/file_sys/program_metadata.h @@ -72,6 +72,10 @@ public: KernelCapabilityDescriptors capabilities); bool Is64BitProgram() const; + /// True when the NPDM uses the Switch 2 ACID layout. + bool IsSwitch2() const { + return is_switch2; + } ProgramAddressSpaceType GetAddressSpaceType() const; u8 GetMainThreadPriority() const; u8 GetMainThreadCore() const; @@ -179,6 +183,7 @@ private: Header npdm_header{}; AciHeader aci_header{}; AcidHeader acid_header{}; + bool is_switch2{}; FileAccessControl acid_file_access{}; FileAccessHeader aci_file_access{}; diff --git a/src/core/file_sys/vfs/vfs_real.cpp b/src/core/file_sys/vfs/vfs_real.cpp index 14e0caad3c..fd36752fee 100644 --- a/src/core/file_sys/vfs/vfs_real.cpp +++ b/src/core/file_sys/vfs/vfs_real.cpp @@ -19,6 +19,10 @@ // For FileTimeStampRaw #include <sys/stat.h> +#ifdef _WIN32 +#include <cstdio> // _getmaxstdio +#endif + #ifdef _MSC_VER #define stat _stat64 #endif @@ -33,7 +37,17 @@ namespace FS = Common::FS; namespace { -constexpr size_t MaxOpenFiles = 8192; +// Files the cache keeps open. On Windows it has to stay below the C runtime's open-file limit with +// room to spare: at the limit every other open fails (saves, shader caches, and the cache's own +// next file, which it only evicts for once it reaches its cap). +size_t MaxOpenFiles() { +#ifdef _WIN32 + static const size_t limit = static_cast<size_t>((std::max)(_getmaxstdio() - 512, 64)); + return (std::min)(size_t{8192}, limit); +#else + return 8192; +#endif +} bool IsWithinRoot(std::string_view root, std::string_view full_path) { if (root.empty()) @@ -251,7 +265,7 @@ void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference } void RealVfsFilesystem::EvictSingleReferenceLocked() { - if (num_open_files < MaxOpenFiles || open_references.empty()) { + if (num_open_files < MaxOpenFiles() || open_references.empty()) { return; } diff --git a/src/core/hardware_properties.h b/src/core/hardware_properties.h index 3f870becbf..aeea5f858f 100644 --- a/src/core/hardware_properties.h +++ b/src/core/hardware_properties.h @@ -15,11 +15,14 @@ namespace Hardware { constexpr u64 BASE_CLOCK_RATE = 1'020'000'000; // Default CPU Frequency = 1020 MHz constexpr u64 CNTFREQ = 19'200'000; // CNTPCT_EL0 Frequency = 19.2 MHz -constexpr u32 NUM_CPU_CORES = 4; // Number of CPU Cores - sync with dynarmic exclusive_monitor.h +// Switch 1 applications get cores 0..3. Switch 2 applications ask for 0..5 (title 4's npdm), and +// its job system starts a worker per core, so emulate six: with four, cores 0 and 1 each ran two +// of its workers. Switch 1 titles leave cores 4 and 5 idle. +constexpr u32 NUM_CPU_CORES = 6; // Number of CPU Cores - sync with dynarmic exclusive_monitor.h // Virtual to Physical core map. constexpr std::array<s32, Common::BitSize<u64>()> VirtualToPhysicalCoreMap{ - 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, }; diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp index 3343d1d282..ba4737e25f 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp @@ -94,7 +94,7 @@ size_t KSystemControl::Init::GetIntendedMemorySize() { case Smc::MemorySize_10GB: return 10_GiB; case Smc::MemorySize_12GB: - return 12_GiB; + return 12_GiB + KSystemControl::Nx2GraphicsServerMemorySize; } } diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h index 41a25ba1c8..cc5572bde8 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h @@ -21,6 +21,10 @@ public: // This can be overridden as needed. static constexpr size_t SecureAppletMemorySize = 4 * 1024 * 1024; // 4_MB + // Extra DRAM on the 12 GB layout for memory the Switch 2 graphics server keeps itself (nx2:gpu + // 214 GPU-only regions). It sits in the non-secure system pool, outside the application pool. + static constexpr size_t Nx2GraphicsServerMemorySize = 8ULL * 1024 * 1024 * 1024; // 8_GiB + public: class Init { public: diff --git a/src/core/hle/kernel/global_scheduler_context.cpp b/src/core/hle/kernel/global_scheduler_context.cpp index 6d1cd030e0..98d14c8549 100644 --- a/src/core/hle/kernel/global_scheduler_context.cpp +++ b/src/core/hle/kernel/global_scheduler_context.cpp @@ -40,12 +40,15 @@ void GlobalSchedulerContext::RemoveThread(KThread* thread) noexcept { /// @note This operation happens every 10ms. void GlobalSchedulerContext::PreemptThreads(KernelCore& kernel) noexcept { // The priority levels at which the global scheduler preempts threads every 10 ms. They are - // ordered from Core 0 to Core 3. + // ordered from Core 0 to Core 5; core 3 is shared with the system, 4 and 5 are application + // cores like 0..2. static constexpr std::array<u32, Core::Hardware::NUM_CPU_CORES> per_core{ 59, 59, 59, 63, + 59, + 59, }; ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); for (u32 core_id = 0; core_id < per_core.size(); core_id++) diff --git a/src/core/hle/kernel/k_memory_block.h b/src/core/hle/kernel/k_memory_block.h index f6543bdb3a..2abca31d87 100644 --- a/src/core/hle/kernel/k_memory_block.h +++ b/src/core/hle/kernel/k_memory_block.h @@ -195,7 +195,9 @@ enum class KMemoryAttribute : u8 { Uncached = static_cast<u8>(Svc::MemoryAttribute::Uncached), PermissionLocked = static_cast<u8>(Svc::MemoryAttribute::PermissionLocked), - SetMask = Uncached | PermissionLocked, + GpuAccessible = static_cast<u8>(Svc::MemoryAttribute::GpuAccessible), + + SetMask = Uncached | PermissionLocked | GpuAccessible, }; DECLARE_ENUM_FLAG_OPERATORS(KMemoryAttribute); diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp index 2a0a7d49d5..fd5d2c2dce 100644 --- a/src/core/hle/kernel/k_page_table_base.cpp +++ b/src/core/hle/kernel/k_page_table_base.cpp @@ -2102,7 +2102,8 @@ Result KPageTableBase::SetMemoryAttribute(KProcessAddress addr, size_t size, KMe constexpr KMemoryAttribute AttributeTestMask = ~(KMemoryAttribute::SetMask | KMemoryAttribute::DeviceShared); const KMemoryState state_test_mask = - (True(mask & KMemoryAttribute::Uncached) ? KMemoryState::FlagCanChangeAttribute + (True(mask & (KMemoryAttribute::Uncached | KMemoryAttribute::GpuAccessible)) + ? KMemoryState::FlagCanChangeAttribute : KMemoryState::None) | (True(mask & KMemoryAttribute::PermissionLocked) ? KMemoryState::FlagCanPermissionLock : KMemoryState::None); diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index 27d1b1bfc4..c3a3b73559 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -1261,6 +1261,14 @@ Result KProcess::LoadFromMetadata(KernelCore& kernel, const FileSys::ProgramMeta // Assign remaining properties. m_ideal_core_id = metadata.GetMainThreadCore(); + // Switch 2 applications get more memory than a 64-bit heap region holds (nninitStartup sizes + // its heap as total memory minus ~6.3 GiB), so count the alias region as well, bounded by the + // pool through the resource limit. + if (metadata.IsSwitch2()) { + m_is_switch2 = true; + m_max_process_memory = m_page_table.GetHeapRegionSize() + m_page_table.GetAliasRegionSize(); + } + // Set up emulation context. this->InitializeInterfaces(kernel); diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h index b5693751c3..d20711c222 100644 --- a/src/core/hle/kernel/k_process.h +++ b/src/core/hle/kernel/k_process.h @@ -135,6 +135,7 @@ private: bool m_is_signaled : 1 = false; bool m_is_initialized : 1 = false; bool m_is_application : 1 = false; + bool m_is_switch2 : 1 = false; bool m_is_default_application_system_resource : 1 = false; bool m_is_suspended : 1 = false; bool m_is_immortal : 1 = false; @@ -248,6 +249,10 @@ public: return m_is_application; } + bool IsSwitch2() const { + return m_is_switch2; + } + bool IsDefaultApplicationSystemResource() const { return m_is_default_application_system_resource; } diff --git a/src/core/hle/kernel/k_spin_lock.cpp b/src/core/hle/kernel/k_spin_lock.cpp index 8525320374..107d6f1aa3 100644 --- a/src/core/hle/kernel/k_spin_lock.cpp +++ b/src/core/hle/kernel/k_spin_lock.cpp @@ -3,18 +3,60 @@ #include "core/hle/kernel/k_spin_lock.h" +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) +#include <immintrin.h> +#endif + namespace Kernel { +namespace { + +// Spins of about 30-140 cycles each before a contended Lock sleeps (a few microseconds). +constexpr int SpinCount = 128; + +void Pause() { +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) + _mm_pause(); +#elif defined(__aarch64__) + asm volatile("yield"); +#endif +} + +} // Anonymous namespace + void KSpinLock::Lock() { - m_lock.lock(); + u32 expected = 0; + if (m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return; + } + for (int spin = 0; spin < SpinCount; ++spin) { + Pause(); + if (m_state.load(std::memory_order_relaxed) == 0) { + expected = 0; + if (m_state.compare_exchange_weak(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return; + } + } + } + // Mark the lock contended and sleep until it is released; taken this way it stays marked, + // so its Unlock wakes the next sleeper. + while (m_state.exchange(2, std::memory_order_acquire) != 0) { + m_state.wait(2, std::memory_order_relaxed); + } } void KSpinLock::Unlock() { - m_lock.unlock(); + if (m_state.exchange(0, std::memory_order_release) == 2) { + m_state.notify_one(); + } } bool KSpinLock::TryLock() { - return m_lock.try_lock(); + u32 expected = 0; + return m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed); } } // namespace Kernel diff --git a/src/core/hle/kernel/k_spin_lock.h b/src/core/hle/kernel/k_spin_lock.h index 094a1e6be0..f7bc8b4d05 100644 --- a/src/core/hle/kernel/k_spin_lock.h +++ b/src/core/hle/kernel/k_spin_lock.h @@ -3,9 +3,11 @@ #pragma once -#include <mutex> +#include <atomic> +#include <mutex> // Kernel headers that use std::mutex get it through this one #include "common/common_funcs.h" +#include "common/common_types.h" #include "core/hle/kernel/k_scoped_lock.h" namespace Kernel { @@ -22,7 +24,11 @@ public: bool TryLock(); private: - std::mutex m_lock; + // 0 free, 1 held, 2 held with (possible) sleepers. The scheduler lock is taken by every + // core thousands of times a frame and held for microseconds: a contended Lock spins briefly + // before sleeping, instead of putting the host thread to sleep at once and paying its + // wake-up latency (std::mutex did). + std::atomic<u32> m_state{0}; }; // TODO(bunnei): Alias for now, in case we want to implement these accurately in the future. diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 4f1d849209..b437dd4b4e 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -171,7 +171,7 @@ struct KernelCore::Impl { schedulers[core_id].reset(); } - // Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others + // Next host thead ID to use, IDs below NUM_CPU_CORES represent core threads, the rest others next_host_thread_id = Core::Hardware::NUM_CPU_CORES; // Close kernel objects that were not freed on shutdown diff --git a/src/core/hle/kernel/physical_core.cpp b/src/core/hle/kernel/physical_core.cpp index c5236c64e3..c3ae7629aa 100644 --- a/src/core/hle/kernel/physical_core.cpp +++ b/src/core/hle/kernel/physical_core.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <chrono> +#include <cstdlib> +#include <thread> #include "common/scope_exit.h" #include "common/settings.h" #include "core/core.h" @@ -222,8 +225,22 @@ void PhysicalCore::LogBacktrace(KernelCore& kernel) { } void PhysicalCore::Idle() { + // Guest job systems hand work between cores many times per frame; blocking at once adds the + // host's thread wake-up latency to every handoff. EDEN_NX2_IDLE_SPIN_US=<n>: poll for that + // many microseconds before blocking. + static const auto spin = std::chrono::microseconds{[] { + const char* text = std::getenv("EDEN_NX2_IDLE_SPIN_US"); + return text ? std::strtoll(text, nullptr, 10) : 0LL; + }()}; + if (spin.count() > 0) { + const auto deadline = std::chrono::steady_clock::now() + spin; + while (!m_is_interrupted.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + } std::unique_lock lk{m_guard}; - m_on_interrupt.wait(lk, [this] { return m_is_interrupted; }); + m_on_interrupt.wait(lk, [this] { return m_is_interrupted.load(); }); } bool PhysicalCore::IsInterrupted() const { diff --git a/src/core/hle/kernel/physical_core.h b/src/core/hle/kernel/physical_core.h index b92201204a..20371d6717 100644 --- a/src/core/hle/kernel/physical_core.h +++ b/src/core/hle/kernel/physical_core.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <condition_variable> #include <cstddef> #include <memory> @@ -71,7 +72,7 @@ private: std::condition_variable m_on_interrupt; Core::ArmInterface* m_arm_interface{}; KThread* m_current_thread{}; - bool m_is_interrupted{}; + std::atomic<bool> m_is_interrupted{}; bool m_is_single_core{}; }; diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 3542d5a3b6..53d964ecbc 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -6,6 +6,8 @@ // This file is automatically generated using svc_generator.py. // DO NOT MODIFY IT MANUALLY +#include <cstdlib> +#include <cstring> #include <type_traits> #include "core/arm/arm_interface.h" @@ -2505,11 +2507,60 @@ void Call(Core::System& system, u32 imm) { LOG_TRACE(Kernel_SVC, "{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x} [6]={:#x}", imm, GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2), GetArg64(args, 3), GetArg64(args, 4), GetArg64(args, 5), GetArg64(args, 6)); - if (process.Is64Bit()) { + // Set EDEN_NX2_SVC_TRACE=1 to log every SVC with its arguments and results, or + // EDEN_NX2_SVC_TRACE=<first>:<last> for presents first..last only (a frame-time profile). + static const std::pair<u64, u64> trace_range = [] { + const char* text = std::getenv("EDEN_NX2_SVC_TRACE"); + if (text == nullptr) return std::pair<u64, u64>{1, 0}; + const char* colon = std::strchr(text, ':'); + if (colon == nullptr) return std::pair<u64, u64>{0, ~u64{}}; + return std::pair<u64, u64>{std::strtoull(text, nullptr, 10), std::strtoull(colon + 1, nullptr, 10)}; + }(); + const u64 presents = Core::g_nx2_present_count.load(std::memory_order_relaxed); + const bool svc_trace = presents >= trace_range.first && presents <= trace_range.second; + if (svc_trace) { + LOG_INFO(Kernel_SVC, "svc {:#04x} in t{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x}", imm, + GetCurrentThread(kernel).GetThreadId(), + GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2), GetArg64(args, 3)); + } + if (imm == 0x80) { + // Switch 2 only: svcSetGpuMemoryAttribute(address, size, mask, attr). nn::gpu::sdk and + // nn::gs native memory set the new "GPU accessible" attribute (0x20) before registering + // memory with the GPU and clear it on destruction; NVN can see it through QueryMemory. + // Same checks as svcSetMemoryAttribute, restricted to that bit (as in changes.patch). + const u64 address = GetArg64(args, 0); + const u64 size = GetArg64(args, 1); + const u32 mask = static_cast<u32>(GetArg64(args, 2)); + const u32 attr = static_cast<u32>(GetArg64(args, 3)); + constexpr u32 Supported = static_cast<u32>(Svc::MemoryAttribute::GpuAccessible); + auto& page_table = process.GetPageTable(); + Result result = ResultSuccess; + if (!Common::IsAligned(address, PageSize)) { + result = ResultInvalidAddress; + } else if (size == 0 || !Common::IsAligned(size, PageSize)) { + result = ResultInvalidSize; + } else if ((attr & ~mask) != 0 || ((mask | attr) & ~Supported) != 0) { + result = ResultInvalidCombination; + } else if (address + size <= address || !page_table.Contains(address, size)) { + result = ResultInvalidCurrentMemory; + } else { + result = page_table.SetMemoryAttribute(address, size, + static_cast<KMemoryAttribute>(mask), + static_cast<KMemoryAttribute>(attr)); + } + LOG_DEBUG(Kernel_SVC, "Switch 2 svc 0x80 addr={:#x} size={:#x} mask={:#x} attr={:#x} -> {:#x}", + address, size, mask, attr, result.raw); + args[0] = result.raw; + } else if (process.Is64Bit()) { Call64(system, imm, args); } else { Call32(system, imm, args); } + if (svc_trace) { + LOG_INFO(Kernel_SVC, "svc {:#04x} out t{} [0]={:#x} [1]={:#x} [2]={:#x}", imm, + GetCurrentThread(kernel).GetThreadId(), GetArg64(args, 0), GetArg64(args, 1), + GetArg64(args, 2)); + } //kernel.ExitSVCProfile(); kernel.CurrentPhysicalCore().LoadSvcArguments(process, args); } diff --git a/src/core/hle/kernel/svc/svc_info.cpp b/src/core/hle/kernel/svc/svc_info.cpp index 863bb58fa8..bbd7d32445 100644 --- a/src/core/hle/kernel/svc/svc_info.cpp +++ b/src/core/hle/kernel/svc/svc_info.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "core/core.h" #include "core/core_timing.h" #include "core/hle/kernel/k_memory_manager.h" @@ -40,6 +42,11 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle case InfoType::UsedNonSystemMemorySize: case InfoType::IsApplication: case InfoType::FreeThreadCount: + case InfoType::Nx2RegionAddress: + case InfoType::Nx2RegionSize: + case InfoType::Nx2VammSize: + case InfoType::Nx2RemoteMemorySize: + case InfoType::Nx2RemoteMemoryUsed: case InfoType::AliasRegionExtraSize: { R_UNLESS(info_sub_id == 0, ResultInvalidEnumValue); @@ -66,6 +73,35 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle case InfoType::HeapRegionSize: *result = process->GetPageTable().GetHeapRegionSize(); R_SUCCEED(); + case InfoType::Nx2RegionAddress: + case InfoType::Nx2RegionSize: { + // Looks like the region Switch 2 VAMM reserves address space in (a title asks for a + // ~7.5 GiB reservation). VAMM backs it with MapPhysicalMemory, which only works in + // the alias region here, so hand out the upper half of the alias region. + auto& page_table = process->GetPageTable(); + const u64 half = page_table.GetAliasRegionSize() / 2; + *result = info_id_type == InfoType::Nx2RegionAddress + ? GetInteger(page_table.GetAliasRegionStart()) + half + : half; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType {}, returning {:#x}", + static_cast<u32>(info_id_type), *result); + R_SUCCEED(); + } + case InfoType::Nx2RemoteMemorySize: + // Unknown real value; nothing backs it yet. + *result = 0; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 31 (remote memory size), returning 0"); + R_SUCCEED(); + case InfoType::Nx2RemoteMemoryUsed: + *result = 0; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 32 (remote memory used), returning 0"); + R_SUCCEED(); + case InfoType::Nx2VammSize: + // A page granule: the SDK checks module addresses/sizes against (value - 1) and + // VAMM maps one granule up front. 4 KiB matches what Switch 1 kernels use. + *result = Core::Memory::YUZU_PAGESIZE; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 33, returning {:#x}", *result); + R_SUCCEED(); case InfoType::AslrRegionAddress: *result = GetInteger(process->GetPageTable().GetAliasCodeRegionStart()); R_SUCCEED(); @@ -310,7 +346,16 @@ Result GetSystemInfo(Core::System& system, uint64_t* out, SystemInfoType info_ty Result GetInfo64(Core::System& system, uint64_t* out, InfoType info_type, Handle handle, uint64_t info_subtype) { - R_RETURN(GetInfo(system, out, info_type, handle, info_subtype)); + const Result result = GetInfo(system, out, info_type, handle, info_subtype); + // EDEN_NX2_MEM_TRACE: log the memory layout and size queries of a title (diagnostics). + static const bool trace = std::getenv("EDEN_NX2_MEM_TRACE") != nullptr; + const u32 type = static_cast<u32>(info_type); + if (static u32 logs{}; trace && logs < 200 && (type <= 7 || (type >= 12 && type <= 33))) { + ++logs; + LOG_INFO(Kernel_SVC, "mem trace: GetInfo type {} sub {:#x} -> {:#x} = {:#x}", type, + info_subtype, result.raw, *out); + } + R_RETURN(result); } Result GetSystemInfo64(Core::System& system, uint64_t* out, SystemInfoType info_type, Handle handle, diff --git a/src/core/hle/kernel/svc/svc_physical_memory.cpp b/src/core/hle/kernel/svc/svc_physical_memory.cpp index 48915b2b1e..4fde85fdaa 100644 --- a/src/core/hle/kernel/svc/svc_physical_memory.cpp +++ b/src/core/hle/kernel/svc/svc_physical_memory.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "core/core.h" #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/svc.h" @@ -142,16 +144,47 @@ Result SetUnsafeLimit(Core::System& system, uint64_t limit) { R_THROW(ResultNotImplemented); } +// EDEN_NX2_MEM_TRACE: log the heap and physical memory requests of a title (diagnostics). +static bool MemTrace() { + static const bool trace = std::getenv("EDEN_NX2_MEM_TRACE") != nullptr; + return trace; +} + Result SetHeapSize64(Core::System& system, uint64_t* out_address, uint64_t size) { - R_RETURN(SetHeapSize(system, out_address, size)); + const Result result = SetHeapSize(system, out_address, size); + if (MemTrace()) { + LOG_INFO(Kernel_SVC, "mem trace: SetHeapSize {:#x} -> {:#x} at {:#x}", size, result.raw, + *out_address); + } + R_RETURN(result); } Result MapPhysicalMemory64(Core::System& system, uint64_t address, uint64_t size) { - R_RETURN(MapPhysicalMemory(system, address, size)); + const Result result = MapPhysicalMemory(system, address, size); + static u64 total{}; + if (MemTrace()) { + if (result.IsSuccess()) { + total += size; + } + static u32 logs{}; + if (result.IsError() || logs++ < 400) { + LOG_INFO(Kernel_SVC, "mem trace: MapPhysicalMemory {:#x}+{:#x} -> {:#x} (mapped total {:#x})", + address, size, result.raw, total); + } + } + R_RETURN(result); } Result UnmapPhysicalMemory64(Core::System& system, uint64_t address, uint64_t size) { - R_RETURN(UnmapPhysicalMemory(system, address, size)); + const Result result = UnmapPhysicalMemory(system, address, size); + if (MemTrace()) { + static u32 logs{}; + if (result.IsError() || logs++ < 400) { + LOG_INFO(Kernel_SVC, "mem trace: UnmapPhysicalMemory {:#x}+{:#x} -> {:#x}", address, + size, result.raw); + } + } + R_RETURN(result); } Result MapPhysicalMemoryUnsafe64(Core::System& system, uint64_t address, uint64_t size) { diff --git a/src/core/hle/kernel/svc/svc_thread.cpp b/src/core/hle/kernel/svc/svc_thread.cpp index dbca13dd26..7f6b959256 100644 --- a/src/core/hle/kernel/svc/svc_thread.cpp +++ b/src/core/hle/kernel/svc/svc_thread.cpp @@ -20,6 +20,18 @@ constexpr bool IsValidVirtualCoreId(int32_t core_id) { return (0 <= core_id && core_id < static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES)); } +// Switch 2 hardware has eight cores. Six are emulated (the most an application is given), so +// fold any id past them back onto the low cores, both in core ids and in affinity masks. +constexpr int32_t FoldExtraCoreId(int32_t core_id) { + constexpr auto num_cores = static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES); + return core_id >= num_cores && core_id < 2 * num_cores ? core_id - num_cores : core_id; +} + +constexpr u64 FoldExtraCoreMask(u64 mask) { + constexpr u64 low = (1ULL << Core::Hardware::NUM_CPU_CORES) - 1; + return (mask & low) | ((mask >> Core::Hardware::NUM_CPU_CORES) & low); +} + } // Anonymous namespace /// Creates a new thread @@ -36,6 +48,7 @@ Result CreateThread(Core::System& system, Handle* out_handle, u64 entry_point, u if (core_id == IdealCoreUseProcessValue) { core_id = process.GetIdealCoreId(); } + core_id = FoldExtraCoreId(core_id); // Validate arguments. R_UNLESS(IsValidVirtualCoreId(core_id), ResultInvalidCoreId); @@ -261,6 +274,10 @@ Result SetThreadCoreMask(Core::System& system, Handle thread_handle, s32 core_id R_UNLESS((affinity_mask | process_core_mask) == process_core_mask, ResultInvalidCoreId); R_UNLESS(affinity_mask != 0, ResultInvalidCombination); + // Validated against the real mask above; fold to the emulated cores from here on. + affinity_mask = FoldExtraCoreMask(affinity_mask); + core_id = FoldExtraCoreId(core_id); + // Validate the core id. if (IsValidVirtualCoreId(core_id)) { R_UNLESS(((1ULL << core_id) & affinity_mask) != 0, ResultInvalidCombination); diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h index b0f4796da4..0bf2aaaa2d 100644 --- a/src/core/hle/kernel/svc_types.h +++ b/src/core/hle/kernel/svc_types.h @@ -50,6 +50,7 @@ enum class MemoryAttribute : u32 { DeviceShared = (1 << 2), Uncached = (1 << 3), PermissionLocked = (1 << 4), + GpuAccessible = (1 << 5), // Switch 2 (svc 0x80) }; DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute); @@ -157,6 +158,17 @@ enum class InfoType : u32 { IsSvcPermitted = 26, IoRegionHint = 27, AliasRegionExtraSize = 28, + // Switch 2 kernels: address/size of a region the SDK queries between the alias and + // ASLR regions during nnosInitialize; likely the VAMM address-space region. + Nx2RegionAddress = 29, + Nx2RegionSize = 30, + // Switch 2 kernels: nn::os::GetRemoteMemoryUsage returns {id 31, id 32}; used by + // nn::gsv/nn::nvsrv BootWithRemoteMemory (memory set aside for the graphics stack). + Nx2RemoteMemorySize = 31, + Nx2RemoteMemoryUsed = 32, + // Switch 2 kernels: a page granule. The SDK aligns module ranges and VAMM's first + // MapPhysicalMemory to it. + Nx2VammSize = 33, TransferMemoryHint = 34, diff --git a/src/core/hle/service/am/service/application_functions.cpp b/src/core/hle/service/am/service/application_functions.cpp index 90d292c4a2..d4f7cd6041 100644 --- a/src/core/hle/service/am/service/application_functions.cpp +++ b/src/core/hle/service/am/service/application_functions.cpp @@ -118,7 +118,8 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_ {220, nullptr, "Unknown220"}, // [20.0.0+] {300, nullptr, "CreateMovieWriter"}, // [19.0.0+] {310, nullptr, "Unknown310"}, // [20.0.0+] - {320, nullptr, "Unknown320"}, // [20.0.0+] + {320, D<&IApplicationFunctions::Unknown320>, "Unknown320"}, // [20.0.0+] + {321, D<&IApplicationFunctions::Unknown321>, "Unknown321"}, // Switch 2 {330, D<&IApplicationFunctions::Unknown330>, "Unknown330"}, // [20.0.0+] {500, nullptr, "StartContinuousRecordingFlushForDebug"}, {1000, nullptr, "CreateMovieMaker"}, @@ -352,10 +353,11 @@ Result IApplicationFunctions::GetPseudoDeviceId(Out<Common::UUID> out_pseudo_dev // This should be hashed with the device specific hash // for now this will do const auto res = FileSys::PatchManager::GetMetadataFromBaseOrUpdate(system, m_applet->program_id); - R_UNLESS(res.first != nullptr, ResultUnknown); std::array<u8, EVP_MAX_MD_SIZE> hash; unsigned int hash_len = 0; - auto const seed = res.first->raw.seed_for_pseudo_device_id; + // Loose ExeFS boots have no control.nacp; seed from the program id instead of failing. + auto const seed = res.first != nullptr ? res.first->raw.seed_for_pseudo_device_id + : m_applet->program_id; EVP_MD_CTX *ctx = EVP_MD_CTX_new(); auto const algorithm = EVP_sha1(); EVP_DigestInit_ex(ctx, algorithm, nullptr); @@ -544,6 +546,18 @@ Result IApplicationFunctions::GetUnknownEvent210( R_SUCCEED(); } +Result IApplicationFunctions::Unknown320() { + // Called by Switch 2 titles during nn::oe::Initialize; arguments unknown. + LOG_WARNING(Service_AM, "(STUBBED) called"); + R_SUCCEED(); +} + +Result IApplicationFunctions::Unknown321() { + // Called by Switch 2 titles right after 320; arguments unknown. + LOG_WARNING(Service_AM, "(STUBBED) called"); + R_SUCCEED(); +} + Result IApplicationFunctions::Unknown330(Out<u8> out) { LOG_DEBUG(Service_AM, "called"); *out = 0; diff --git a/src/core/hle/service/am/service/application_functions.h b/src/core/hle/service/am/service/application_functions.h index b97ed7e25e..72bf8998b1 100644 --- a/src/core/hle/service/am/service/application_functions.h +++ b/src/core/hle/service/am/service/application_functions.h @@ -82,6 +82,8 @@ private: Result GetHealthWarningDisappearedSystemEvent(OutCopyHandle<Kernel::KReadableEvent> out_event); Result CreateApplicationAndRequestToStart(u64 application_id); Result GetUnknownEvent210(OutCopyHandle<Kernel::KReadableEvent> out_event); + Result Unknown320(); + Result Unknown321(); Result Unknown330(Out<u8> out); Result PrepareForJit(); diff --git a/src/core/hle/service/am/service/application_proxy.cpp b/src/core/hle/service/am/service/application_proxy.cpp index df438e1e17..430f3c6888 100644 --- a/src/core/hle/service/am/service/application_proxy.cpp +++ b/src/core/hle/service/am/service/application_proxy.cpp @@ -11,11 +11,13 @@ #include "core/hle/service/am/service/common_state_getter.h" #include "core/hle/service/am/service/debug_functions.h" #include "core/hle/service/am/service/display_controller.h" +#include "core/hle/service/am/service/graphics_scale_controller.h" #include "core/hle/service/am/service/library_applet_creator.h" #include "core/hle/service/am/service/process_winding_controller.h" #include "core/hle/service/am/service/self_controller.h" #include "core/hle/service/am/service/window_controller.h" #include "core/hle/service/cmif_serialization.h" +#include "core/hle/service/nx2_stub_interface.h" namespace Service::AM { @@ -33,6 +35,13 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl {10, D<&IApplicationProxy::GetProcessWindingController>, "GetProcessWindingController"}, {11, D<&IApplicationProxy::GetLibraryAppletCreator>, "GetLibraryAppletCreator"}, {20, D<&IApplicationProxy::GetApplicationFunctions>, "GetApplicationFunctions"}, + // Switch 2-only getters; see nx2_stub_interface.h. + {40, &IApplicationProxy::GetNx2Interface, "Nx2Getter40"}, + {50, &IApplicationProxy::GetNx2Interface, "Nx2Getter50"}, + {60, D<&IApplicationProxy::GetGraphicsScaleController>, "GetGraphicsScaleController (Switch 2)"}, + {70, &IApplicationProxy::GetNx2Interface, "Nx2Getter70"}, + {80, &IApplicationProxy::GetNx2Interface, "Nx2Getter80"}, + {90, &IApplicationProxy::GetNx2Interface, "Nx2Getter90"}, {1000, D<&IApplicationProxy::GetDebugFunctions>, "GetDebugFunctions"}, }; // clang-format on @@ -42,6 +51,17 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl IApplicationProxy::~IApplicationProxy() = default; +Result IApplicationProxy::GetGraphicsScaleController( + Out<SharedPointer<IGraphicsScaleController>> out_controller) { + LOG_DEBUG(Service_AM, "called"); + *out_controller = std::make_shared<IGraphicsScaleController>(system); + R_SUCCEED(); +} + +void IApplicationProxy::GetNx2Interface(HLERequestContext& ctx) { + ReturnNx2StubInterface(system, ctx, "IApplicationProxy"); +} + Result IApplicationProxy::GetAudioController( Out<SharedPointer<IAudioController>> out_audio_controller) { LOG_DEBUG(Service_AM, "called"); diff --git a/src/core/hle/service/am/service/application_proxy.h b/src/core/hle/service/am/service/application_proxy.h index b56f597605..0a706be4fd 100644 --- a/src/core/hle/service/am/service/application_proxy.h +++ b/src/core/hle/service/am/service/application_proxy.h @@ -17,6 +17,7 @@ class IApplicationFunctions; class ICommonStateGetter; class IDebugFunctions; class IDisplayController; +class IGraphicsScaleController; class ILibraryAppletCreator; class IProcessWindingController; class ISelfController; @@ -30,6 +31,8 @@ public: ~IApplicationProxy(); private: + void GetNx2Interface(HLERequestContext& ctx); + Result GetGraphicsScaleController(Out<SharedPointer<IGraphicsScaleController>> out_controller); Result GetAudioController(Out<SharedPointer<IAudioController>> out_audio_controller); Result GetDisplayController(Out<SharedPointer<IDisplayController>> out_display_controller); Result GetProcessWindingController( diff --git a/src/core/hle/service/am/service/application_proxy_service.cpp b/src/core/hle/service/am/service/application_proxy_service.cpp index 7727951d8c..67306aadc0 100644 --- a/src/core/hle/service/am/service/application_proxy_service.cpp +++ b/src/core/hle/service/am/service/application_proxy_service.cpp @@ -19,6 +19,8 @@ IApplicationProxyService::IApplicationProxyService(Core::System& system_, : ServiceFramework{system_, "appletOE"}, m_window_system{window_system} { static const FunctionInfo functions[] = { {0, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy"}, + // Switch 2 SDKs open the proxy with command 10, same arguments. + {10, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy (Switch 2)"}, }; RegisterHandlers(functions); } diff --git a/src/core/hle/service/am/service/graphics_scale_controller.h b/src/core/hle/service/am/service/graphics_scale_controller.h new file mode 100644 index 0000000000..07e38c6ac8 --- /dev/null +++ b/src/core/hle/service/am/service/graphics_scale_controller.h @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "core/hle/service/cmif_types.h" +#include "core/hle/service/nx2_stub_interface.h" +#include "core/hle/service/service.h" + +namespace Service::AM { + +// Switch 2 interface returned by IApplicationProxy command 60 (name is a guess). nn::oe's +// GetCurrentGraphicsScale reads command 10 and accepts 10, 20 or 30 (scale enum 0, 1, 2). +// Commands without a handler are auto-stubbed because the name carries the nx2: prefix. +class IGraphicsScaleController final : public ServiceFramework<IGraphicsScaleController> { +public: + explicit IGraphicsScaleController(Core::System& system_) + : ServiceFramework{system_, "nx2:IGraphicsScaleController"} { + // clang-format off + static const FunctionInfo functions[] = { + {10, D<&IGraphicsScaleController::GetCurrentGraphicsScale>, "GetCurrentGraphicsScale"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + Result GetCurrentGraphicsScale(Out<u32> out_scale) { + LOG_WARNING(Service_AM, "(STUBBED) called, returning 10"); + *out_scale = 10; + R_SUCCEED(); + } +}; + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/self_controller.cpp b/src/core/hle/service/am/service/self_controller.cpp index 3a654b9daa..a8078e6048 100644 --- a/src/core/hle/service/am/service/self_controller.cpp +++ b/src/core/hle/service/am/service/self_controller.cpp @@ -12,6 +12,7 @@ #include "core/hle/service/caps/caps_su.h" #include "core/hle/service/cmif_serialization.h" #include "core/hle/service/nvnflinger/nvnflinger.h" +#include "core/hle/service/nx2_gsv.h" #include "core/hle/service/sm/sm.h" #include "core/hle/service/vi/vi_results.h" @@ -73,6 +74,7 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> {230, D<&ISelfController::Unknown230>, "Unknown230"}, {240, D<&ISelfController::Unknown240>, "Unknown240"}, {1000, nullptr, "GetDebugStorageChannel"}, + {2000, &ISelfController::GetNx2GraphicsSystemSession, "GetGraphicsSystemSession (Switch 2, guessed)"}, }; // clang-format on @@ -83,6 +85,12 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> m_applet->library_applet_mode); } +void ISelfController::GetNx2GraphicsSystemSession(HLERequestContext& ctx) { + // nn::gsv::BootWithRemoteMemory sends TIPC on the handle this returns. + LOG_WARNING(Service_AM, "(STUBBED) Switch 2 ISelfController::2000, returning a nx2:gsv session"); + ReturnRawSession(system, ctx, std::make_shared<INx2GraphicsSystemSession>(system)); +} + ISelfController::~ISelfController() { std::scoped_lock lk{m_applet->lock}; m_applet->display_layer_manager.Finalize(); diff --git a/src/core/hle/service/am/service/self_controller.h b/src/core/hle/service/am/service/self_controller.h index c9f480f575..221fbb0981 100644 --- a/src/core/hle/service/am/service/self_controller.h +++ b/src/core/hle/service/am/service/self_controller.h @@ -30,6 +30,7 @@ public: ~ISelfController() override; private: + void GetNx2GraphicsSystemSession(HLERequestContext& ctx); Result Exit(); Result LockExit(); Result UnlockExit(); diff --git a/src/core/hle/service/audio/audio.cpp b/src/core/hle/service/audio/audio.cpp index fde766a275..211e1cb6bc 100644 --- a/src/core/hle/service/audio/audio.cpp +++ b/src/core/hle/service/audio/audio.cpp @@ -6,6 +6,7 @@ #include "core/core.h" #include "core/hle/service/audio/audio.h" +#include "core/hle/service/audio/nx2_audio_user_client.h" #include "core/hle/service/audio/audio_controller.h" #include "core/hle/service/audio/audio_in_manager.h" #include "core/hle/service/audio/audio_out_manager.h" @@ -203,6 +204,8 @@ void LoopProcess(Core::System& system) { server_manager->RegisterNamedService("audrec:a", std::make_shared<IFinalOutputRecorderManagerForApplet>(system), 30); server_manager->RegisterNamedService("audrec:u", std::make_shared<IFinalOutputRecorderManager>(system), 30); server_manager->RegisterNamedService("audren:u", std::make_shared<IAudioRendererManager>(system), 30); + // Switch 2 audio entry point (nn::audio::CreateUserClientSessionByHipc). + server_manager->RegisterNamedService("aud:u", std::make_shared<INx2AudioUserClient>(system), 30); server_manager->RegisterNamedService("audren:a", std::make_shared<IAudioRendererManagerForApplet>(system), 30); server_manager->RegisterNamedService("hwopus", std::make_shared<IHardwareOpusDecoderManager>(system), 25); ServerManager::RunServer(std::move(server_manager)); diff --git a/src/core/hle/service/audio/nx2_audio_user_client.h b/src/core/hle/service/audio/nx2_audio_user_client.h new file mode 100644 index 0000000000..2d1a158c84 --- /dev/null +++ b/src/core/hle/service/audio/nx2_audio_user_client.h @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "core/hle/service/audio/audio_in_manager.h" +#include "core/hle/service/audio/audio_out_manager.h" +#include "core/hle/service/audio/audio_renderer_manager.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/service.h" + +namespace Service::Audio { + +// Switch 2 "aud:u" (nn::audio::CreateUserClientSessionByHipc). The SDK reaches the Switch 1 +// audio interfaces through it: command 2 returns an object used exactly like +// IAudioRendererManager (vtable slot 1 = GetWorkBufferSize with a Switch 1 parameter block), so +// it is backed by Eden's audren:u implementation. The nx2: name keeps unknown commands +// auto-stubbed and logged. +class INx2AudioUserClient final : public ServiceFramework<INx2AudioUserClient> { +public: + explicit INx2AudioUserClient(Core::System& system_) + : ServiceFramework{system_, "nx2:aud:u"} { + // clang-format off + static const FunctionInfo functions[] = { + {2, &INx2AudioUserClient::GetAudioRendererManager, "GetAudioRendererManager (guessed)"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + void GetAudioRendererManager(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const u32 arg = rp.Pop<u32>(); + LOG_WARNING(Service_Audio, "(STUBBED) Switch 2 aud:u 2 arg={}, returning IAudioRendererManager", + arg); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<IAudioRendererManager>(ctx, system); + } +}; + +} // namespace Service::Audio diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp index 9689bce8dd..04b4663bc2 100644 --- a/src/core/hle/service/hid/hid_server.cpp +++ b/src/core/hle/service/hid/hid_server.cpp @@ -132,6 +132,7 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {210, C<&IHidServer::EndPermitVibrationSession>, "EndPermitVibrationSession"}, {211, C<&IHidServer::IsVibrationDeviceMounted>, "IsVibrationDeviceMounted"}, {212, C<&IHidServer::SendVibrationValueInBool>, "SendVibrationValueInBool"}, + {214, C<&IHidServer::SendNx2VibrationValues>, "SendNx2VibrationValues (guessed)"}, {300, C<&IHidServer::ActivateConsoleSixAxisSensor>, "ActivateConsoleSixAxisSensor"}, {301, C<&IHidServer::StartConsoleSixAxisSensor>, "StartConsoleSixAxisSensor"}, {302, C<&IHidServer::StopConsoleSixAxisSensor>, "StopConsoleSixAxisSensor"}, @@ -1069,6 +1070,11 @@ Result IHidServer::IsVibrationDeviceMounted( R_SUCCEED(); } +Result IHidServer::SendNx2VibrationValues() { + LOG_DEBUG(Service_HID, "(STUBBED) called"); + R_SUCCEED(); +} + Result IHidServer::SendVibrationValueInBool( bool is_vibrating, Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid) { diff --git a/src/core/hle/service/hid/hid_server.h b/src/core/hle/service/hid/hid_server.h index 57a1a04b9c..574d54c415 100644 --- a/src/core/hle/service/hid/hid_server.h +++ b/src/core/hle/service/hid/hid_server.h @@ -189,6 +189,9 @@ private: Result SendVibrationValueInBool(bool is_vibrating, Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid); + // Switch 2 sends vibration every frame with 214 (pointer buffers of device handles and + // values, like 206); Eden doesn't emulate vibration, so accept and drop it. + Result SendNx2VibrationValues(); Result ActivateConsoleSixAxisSensor(ClientAppletResourceUserId aruid); Result StartConsoleSixAxisSensor(Core::HID::ConsoleSixAxisSensorHandle console_sixaxis_handle, ClientAppletResourceUserId aruid); diff --git a/src/core/hle/service/kernel_helpers.cpp b/src/core/hle/service/kernel_helpers.cpp index f7a5cb7049..e4d6925f19 100644 --- a/src/core/hle/service/kernel_helpers.cpp +++ b/src/core/hle/service/kernel_helpers.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <mutex> +#include <unordered_map> + #include "core/core.h" #include "core/core_timing.h" #include "core/hle/kernel/k_event.h" @@ -16,6 +19,12 @@ namespace Service::KernelHelpers { +namespace { +// Event names for the debugger's handle listing. +std::mutex event_names_mutex; +std::unordered_map<const Kernel::KEvent*, std::string> event_names; +} // namespace + ServiceContext::ServiceContext(Core::System& system_, std::string name_) : kernel(system_.Kernel()) { @@ -64,14 +73,28 @@ Kernel::KEvent* ServiceContext::CreateEvent(std::string&& name) { // Register the event. Kernel::KEvent::Register(kernel, event); + { + std::scoped_lock lk{event_names_mutex}; + event_names[event] = std::move(name); + } return event; } void ServiceContext::CloseEvent(Kernel::KEvent* event) { if (event) { + { + std::scoped_lock lk{event_names_mutex}; + event_names.erase(event); + } event->GetReadableEvent().Close(kernel); event->Close(kernel); } } +std::string ServiceContext::GetEventName(const Kernel::KEvent* event) { + std::scoped_lock lk{event_names_mutex}; + const auto it = event_names.find(event); + return it == event_names.end() ? std::string{} : it->second; +} + } // namespace Service::KernelHelpers diff --git a/src/core/hle/service/kernel_helpers.h b/src/core/hle/service/kernel_helpers.h index 4e21c8857f..a62dd838aa 100644 --- a/src/core/hle/service/kernel_helpers.h +++ b/src/core/hle/service/kernel_helpers.h @@ -29,6 +29,9 @@ public: void CloseEvent(Kernel::KEvent* event); + /// Name an event was created with (debugger handle listing); empty when unknown. + static std::string GetEventName(const Kernel::KEvent* event); + Kernel::KernelCore& kernel; Kernel::KProcess* process{}; bool process_created{false}; diff --git a/src/core/hle/service/ns/platform_service_manager.cpp b/src/core/hle/service/ns/platform_service_manager.cpp index bf93a2edfa..88aac56b1c 100644 --- a/src/core/hle/service/ns/platform_service_manager.cpp +++ b/src/core/hle/service/ns/platform_service_manager.cpp @@ -22,7 +22,9 @@ #include "core/hle/kernel/k_shared_memory.h" #include "core/hle/kernel/kernel.h" #include "core/hle/service/cmif_serialization.h" +#include "common/alignment.h" #include "core/hle/service/filesystem/filesystem.h" +#include "core/hle/service/ipc_helpers.h" #include "core/hle/service/ns/platform_service_manager.h" namespace Service::NS { @@ -83,6 +85,8 @@ struct IPlatformServiceManager::Impl { // Automatically populated based on shared_fonts dump or system archives. // 6 builtin fonts + extra 2 for whatever may come after boost::container::static_vector<FontRegion, 8> shared_font_regions; + /// Plain TTF copies of the same fonts (Switch 2 GetBuiltInFontData), after the BFTTF data. + boost::container::static_vector<FontRegion, 8> builtin_font_regions; /// Backing memory for the shared font data std::array<u8, SHARED_FONT_MEM_SIZE> shared_font; }; @@ -107,6 +111,8 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch {106, nullptr, "GetFunctionBlackListVersion"}, {1000, nullptr, "LoadNgWordDataForPlatformRegionChina"}, {1001, nullptr, "GetNgWordDataSizeForPlatformRegionChina"}, + {2000, &IPlatformServiceManager::GetBuiltInFontSharedMemory, "GetBuiltInFontSharedMemory (Switch 2)"}, + {2010, &IPlatformServiceManager::GetBuiltInFontData, "GetBuiltInFontData (Switch 2)"}, }; // clang-format on RegisterHandlers(functions); @@ -116,6 +122,7 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch // Attempt to load shared font data from disk const auto* nand = fsc.GetSystemNANDContents(); std::size_t offset = 0; + std::vector<std::vector<u8>> plain_fonts; // Rebuild shared fonts from data ncas or synthesize for (auto& font : SHARED_FONTS) { FileSys::VirtualFile romfs; @@ -151,9 +158,58 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch Common::swap32); // Font offset and size do not account for the header const FontRegion region{u32(offset + 8), u32((font_data_u32.size() * sizeof(u32)) - 8)}; + std::vector<u8> plain(region.size); + DecryptSharedFontToTTF(font_data_u32, plain); + plain_fonts.push_back(std::move(plain)); DecryptSharedFont(font_data_u32, impl->shared_font, offset); impl->shared_font_regions.push_back(region); } + + for (const auto& plain : plain_fonts) { + offset = Common::AlignUp(offset, 0x1000); + if (offset + plain.size() > SHARED_FONT_MEM_SIZE) { + LOG_WARNING(Service_NS, "No shared memory left for plain built-in font copies"); + break; + } + std::memcpy(impl->shared_font.data() + offset, plain.data(), plain.size()); + impl->builtin_font_regions.push_back({u32(offset), u32(plain.size())}); + offset += plain.size(); + } +} + +// Switch 2: the shared memory GetBuiltInFontData offsets point into (copy handle, u32 size); the +// SDK maps it read-only. Eden serves its font shared memory, which holds plain copies too. +void IPlatformServiceManager::GetBuiltInFontSharedMemory(HLERequestContext& ctx) { + LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontSharedMemory"); + std::memcpy(kernel.GetFontSharedMem().GetPointer(), impl->shared_font.data(), impl->shared_font.size()); + IPC::ResponseBuilder rb{ctx, 3, 1}; + rb.Push(ResultSuccess); + rb.Push<u32>(static_cast<u32>(SHARED_FONT_MEM_SIZE)); + rb.PushCopyObjects(ctx, kernel.GetFontSharedMem()); +} + +// nn::pl::GetBuiltInFontData(BuiltInFontDataOunce*, BuiltInFontTypeOunce, bool): returns a 16-byte +// bool that says whether the font exists and a 16-byte descriptor (u8 flags, u8 flag bit, u16, +// u32 offset into the font shared memory, u32 size, u32). The client expects plain TrueType/OpenType data +// there. The type numbering is unknown, so types map onto Eden's shared fonts in order. +void IPlatformServiceManager::GetBuiltInFontData(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const u32 type = rp.Pop<u32>(); + const bool flag = rp.Pop<u8>() != 0; + const auto& regions = impl->builtin_font_regions; + const bool found = !regions.empty(); + const FontRegion region = found ? regions[std::min<size_t>(type, regions.size() - 1)] : EMPTY_REGION; + LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontData type={} flag={} -> offset={:#x} size={:#x}", + type, flag, region.offset, region.size); + // Raw reply: the bool at offset 0, the descriptor at offset 8. + IPC::ResponseBuilder rb{ctx, 2 + 6}; + rb.Push(ResultSuccess); + rb.Push<u32>(found ? 1 : 0); + rb.Push<u32>(0); + rb.Push<u32>(0); + rb.Push<u32>(region.offset); + rb.Push<u32>(region.size); + rb.Push<u32>(0); } IPlatformServiceManager::~IPlatformServiceManager() = default; diff --git a/src/core/hle/service/ns/platform_service_manager.h b/src/core/hle/service/ns/platform_service_manager.h index 73521abbc2..ed5b1c1e39 100644 --- a/src/core/hle/service/ns/platform_service_manager.h +++ b/src/core/hle/service/ns/platform_service_manager.h @@ -73,6 +73,9 @@ private: Out<bool> out_fonts_are_loaded, Out<u32> out_font_count, Set::LanguageCode language_code); + void GetBuiltInFontSharedMemory(HLERequestContext& ctx); + void GetBuiltInFontData(HLERequestContext& ctx); + struct Impl; std::unique_ptr<Impl> impl; }; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 3c9e42f602..2634902641 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -14,13 +14,16 @@ #include "common/logging.h" #include "core/core.h" #include "core/hle/service/nvdrv/core/container.h" +#include "core/hle/service/nvdrv/core/heap_mapper.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_as_gpu.h" #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "video_core/control/channel_state.h" #include "video_core/gpu.h" +#include "video_core/host1x/host1x.h" #include "video_core/memory_manager.h" #include "video_core/rasterizer_interface.h" @@ -30,7 +33,9 @@ nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Con : nvdevice{system_}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{}, gmmu{} {} -nvhost_as_gpu::~nvhost_as_gpu() = default; +nvhost_as_gpu::~nvhost_as_gpu() { + Nx2GpuMemoryRegistry::Get().RemoveListener(this); +} NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output) { @@ -49,8 +54,24 @@ NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> i return WrapFixed(this, &nvhost_as_gpu::MapBufferEx, input, output); case 0x8: return WrapFixed(this, &nvhost_as_gpu::GetVARegions1, input, output); - case 0x9: - return WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output); + case 0x9: { + const auto result = WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output); + if (result == NvResult::Success && g_nx2_driver) { + Nx2GpuMemoryRegistry::Get().AddListener( + this, + [this](u64 gpu_address, u64 cpu_address, u64 size) { + Nx2Map(gpu_address, cpu_address, size); + }, + [this](u64 gpu_address, u64 size) { Nx2Unmap(gpu_address, size); }, + [this](u64 cpu_address, u64 size) { Nx2Release(cpu_address, size); }); + } + return result; + } + case 0xb: + if (g_nx2_driver) { + return WrapFixed(this, &nvhost_as_gpu::Nx2GetSyncpointShimInfo, input, output); + } + break; case 0x14: return WrapVariable(this, &nvhost_as_gpu::Remap, input, output); default: @@ -90,11 +111,105 @@ NvResult nvhost_as_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> i return NvResult::NotImplemented; } -void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {} +void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) { + nx2_session = session_id; +} + +void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) { + std::scoped_lock lock(mutex); + auto* session = container.GetSession(nx2_session); + if (!gmmu || session == nullptr) { + LOG_ERROR(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x}: no address space or session", + gpu_address, size); + return; + } + // The same CPU range is often mapped twice (at its own address and in a sparse region), so + // reuse its SMMU mapping. + DAddr device_address{}; + bool heap = false; + if (const auto it = nx2_device_ranges.find({cpu_address, size}); it != nx2_device_ranges.end()) { + device_address = it->second.address; + heap = it->second.heap; + } else if (session->has_preallocated_area && session->mapper->IsInBounds(cpu_address, size)) { + device_address = session->mapper->Map(cpu_address, size); + heap = true; + } else { + auto& smmu = system.Host1x().MemoryManager(); + device_address = smmu.Allocate(Common::AlignUp(size, 0x20000)); + if (device_address == 0) { + LOG_CRITICAL(Service_NVDRV, "Switch 2 GPU memory {:#x}+{:#x}: out of SMMU space", + cpu_address, size); + return; + } + smmu.Map(device_address, cpu_address, size, session->asid, true); + } + nx2_device_ranges[{cpu_address, size}] = {device_address, heap}; + Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address); + gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + LOG_DEBUG(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x} -> CPU {:#x}", gpu_address, size, + cpu_address); +} + +void nvhost_as_gpu::Nx2Unmap(u64 gpu_address, u64 size) { + std::scoped_lock lock(mutex); + if (!gmmu) { + return; + } + // The GPU thread may still be running command lists that use the range. + system.GPU().RunInOrder([gmmu = gmmu.get(), gpu_address, size] { gmmu->Unmap(gpu_address, size); }); + LOG_DEBUG(Service_NVDRV, "Switch 2 GPU unmapping {:#x}+{:#x}", gpu_address, size); +} + +void nvhost_as_gpu::Nx2Release(u64 cpu_address, u64 size) { + std::scoped_lock lock(mutex); + const auto it = nx2_device_ranges.find({cpu_address, size}); + if (it == nx2_device_ranges.end()) { + return; + } + const Nx2DeviceRange range = it->second; + nx2_device_ranges.erase(it); + auto* session = container.GetSession(nx2_session); + auto* const mapper = range.heap && session != nullptr ? session->mapper.get() : nullptr; + auto& smmu = system.Host1x().MemoryManager(); + // Queued after the GPU unmaps of the range, so the GPU thread is done with it. + system.GPU().RunInOrder([&smmu, mapper, range, cpu_address, size] { + if (range.heap) { + if (mapper != nullptr) { + mapper->Unmap(cpu_address, size); + } + return; + } + smmu.Unmap(range.address, size); + smmu.Free(range.address, Common::AlignUp(size, 0x20000)); + }); + LOG_DEBUG(Service_NVDRV, "Switch 2 released backing {:#x}+{:#x}", cpu_address, size); +} + void nvhost_as_gpu::OnClose(DeviceFD fd) {} +// Switch 2 NvRm (with characteristics flag 26) asks for the read-only syncpoint shim: a GPU page +// whose u32 at id * stride reads as that syncpoint's value; NVN waits on syncpoints through it and +// asserts without one. Same scheme as changes.patch, placed on the last page of the VA range. +NvResult nvhost_as_gpu::Nx2GetSyncpointShimInfo(IoctlNx2SyncpointShimInfo& params) { + std::scoped_lock lock(mutex); + if (!vm.initialised || !gmmu) { + return NvResult::NotSupported; + } + const u64 address = vm.va_range_end - VM::YUZU_PAGESIZE; + if (!nx2_syncpoint_shim) { + nx2_syncpoint_shim = true; + gmmu->SetNativeSyncpointShim(address); + } + params.address = address; + params.stride = sizeof(u32); + params.count = Tegra::MemoryManager::NumShimSyncpoints; + LOG_INFO(Service_NVDRV, "Switch 2 syncpoint shim at {:#x} ({} syncpoints)", address, + params.count); + return NvResult::Success; +} + NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { - LOG_DEBUG(Service_NVDRV, "called, big_page_size={:#x}", params.big_page_size); + LOG_DEBUG(Service_NVDRV, "called, flags={:#x} big_page_size={:#x} va_range_start={:#x} va_range_end={:#x} va_range_split={:#x}", params.flags, params.big_page_size, params.va_range_start, params.va_range_end, params.va_range_split); std::scoped_lock lock(mutex); @@ -125,6 +240,15 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { vm.va_range_start = params.va_range_start; vm.va_range_split = params.va_range_split; vm.va_range_end = params.va_range_end; + if (vm.va_range_split == 0) { + // Switch 2 NvRm (flags 5, no big page size) asks for small pages over the whole range + // and then reserves 4 KiB VA up to 512 GiB. Keep a small big-page region past the end + // so both allocators stay valid. + vm.va_range_split = vm.va_range_end; + vm.va_range_end += 4ULL << 30; + vm.absolute_big_page_limit = true; + LOG_WARNING(Service_NVDRV, "No VA split given; small pages up to {:#x}", vm.va_range_split); + } } const u64 max_big_page_bits = Common::Log2Ceil(vm.va_range_end); @@ -134,8 +258,11 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { vm.small_page_allocator.emplace(start_pages, end_pages); const auto start_big_pages{static_cast<u32>(vm.va_range_split >> vm.big_page_size_bits)}; - const auto end_big_pages{ - static_cast<u32>((vm.va_range_end - vm.va_range_split) >> vm.big_page_size_bits)}; + // The Switch 1 layout passes the big-page region length as the limit (so the limit stays above + // the start there); a split past the region length needs the absolute end instead. + const auto end_big_pages{static_cast<u32>( + (vm.absolute_big_page_limit ? vm.va_range_end : vm.va_range_end - vm.va_range_split) >> + vm.big_page_size_bits)}; vm.big_page_allocator.emplace(start_big_pages, end_big_pages); gmmu = std::make_unique<Tegra::MemoryManager>(system, max_big_page_bits, vm.va_range_split, vm.big_page_size_bits, VM::PAGE_SIZE_BITS); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index b13b1c9efa..d64ec85dec 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -218,8 +218,29 @@ private: std::optional<Allocator> small_page_allocator; //! Shared as this is also used by nvhost::GpuChannel bool initialised{}; + bool absolute_big_page_limit{}; //!< Switch 2 layout: big-page limit is va_range_end } vm; std::unique_ptr<Tegra::MemoryManager> gmmu; + + struct IoctlNx2SyncpointShimInfo { + u64_le address; + u32_le stride; + u32_le count; + }; + static_assert(sizeof(IoctlNx2SyncpointShimInfo) == 16); + NvResult Nx2GetSyncpointShimInfo(IoctlNx2SyncpointShimInfo& params); + bool nx2_syncpoint_shim{}; + + // Switch 2: maps GPU-accessible process memory (CPU range) at a GPU address. + void Nx2Map(u64 gpu_address, u64 cpu_address, u64 size); + void Nx2Unmap(u64 gpu_address, u64 size); + NvCore::SessionId nx2_session{}; + struct Nx2DeviceRange { + DAddr address; + bool heap; ///< Mapped through the session's heap mapper (else an SMMU allocation) + }; + std::map<std::pair<u64, u64>, Nx2DeviceRange> nx2_device_ranges; + void Nx2Release(u64 cpu_address, u64 size); }; } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index 14cdd8fda4..5db984b53d 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <array> +#include <algorithm> #include <cstring> #include "common/assert.h" #include "common/logging.h" @@ -11,10 +13,16 @@ #include "core/core_timing.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" +#include "video_core/nx2.h" #include "core/hle/service/nvdrv/nvdrv.h" namespace Service::Nvidia::Devices { +namespace { +void ApplyNx2CharacteristicsTail(std::span<u8> block); +} // namespace + nvhost_ctrl_gpu::nvhost_ctrl_gpu(Core::System& system_, EventInterface& events_interface_) : nvdevice{system_}, events_interface{events_interface_} { error_notifier_event = events_interface.CreateEvent("CtrlGpuErrorNotifier"); @@ -93,9 +101,14 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> switch (command.group) { case 'G': switch (command.cmd) { - case 0x5: - return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3, input, output, - inline_output); + case 0x5: { + const auto result = WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3, + input, output, inline_output); + if (nx2_caller) { + ApplyNx2CharacteristicsTail(inline_output); + } + return result; + } case 0x6: return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetTPCMasks3, input, output, inline_output); @@ -112,14 +125,113 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> default: break; } + if (nx2_caller && command.group == 'G' && command.cmd == 0x1F) { + return Nx2GetEngineInfo(input, output, inline_output); + } + if (nx2_caller) { + // Switch 2 NvRm uses ioctls Switch 1 never had; answer them with a zeroed success so + // device setup can continue, and log each one. + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 ioctl={:08X} group={:01X} command={:01X}", + command.raw, command.group, command.cmd); + std::fill(output.begin(), output.end(), u8{0}); + std::fill(inline_output.begin(), inline_output.end(), u8{0}); + return NvResult::Success; + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}, group={:01X}, command={:01X}", command.raw, command.group, command.cmd); return NvResult::NotImplemented; } +// nvgpu GET_ENGINE_INFO as the Switch 2 NvRm uses it: args {u32 buf_size, u32 reserved, +// u64 buf_addr}, queried once for the size and once with an inline output buffer of 16-byte items +// {u32 engine_id, u32 engine_instance, s32 runlist_id, u32 reserved}. NvRm picks the channel +// runlist from this list. Report the graphics engine (runlist 0) and an async copy engine (1). +NvResult nvhost_ctrl_gpu::Nx2GetEngineInfo(std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output) { + struct Item { + u32 engine_id; + u32 engine_instance; + s32 runlist_id; + u32 reserved; + }; + static constexpr std::array<Item, 2> items{{ + {0, 0, 0, 0}, // NVGPU_ENGINE_GR + {2, 0, 1, 0}, // NVGPU_ENGINE_ASYNC_COPY + }}; + constexpr u32 needed = static_cast<u32>(items.size() * sizeof(Item)); + + u32 buf_size{}; + if (input.size() >= sizeof(buf_size)) { + std::memcpy(&buf_size, input.data(), sizeof(buf_size)); + } + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 GetEngineInfo buf_size={:#x} inline_out={:#x}", + buf_size, inline_output.size()); + if (buf_size >= needed && inline_output.size() >= needed) { + std::memcpy(inline_output.data(), items.data(), needed); + } + std::fill(output.begin(), output.end(), u8{0}); + std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin()); + if (output.size() >= sizeof(needed)) { + std::memcpy(output.data(), &needed, sizeof(needed)); + } + return NvResult::Success; +} + void nvhost_ctrl_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {} void nvhost_ctrl_gpu::OnClose(DeviceFD fd) {} +namespace { + +// Switch 2 SDKs ask for a 0xD0-byte characteristics block (Switch 1: 0xA0). Their NvRm only +// accepts chip ids 0x17E, 0x12B and 0x12E and its NVN path rejects the Maxwell ones, so report the +// Switch 2 Ampere chip. Fields past the Switch 1 layout are left zero. Values are best guesses. +constexpr u64 Nx2CharacteristicsSize = 0xD0; + +template <typename GC> +void ApplyNx2Characteristics(GC& gc) { + gc.arch = 0x170; // Ampere + gc.impl = 0xE; // chip id 0x17E as listed in the Switch 2 NvRm chip table + gc.num_tpc_per_gpc = 0x6; + gc.sm_arch_sm_version = 0x808; // NVN2 only accepts SM 0x808 (its class table lookup) + gc.sm_arch_spa_version = 0x808; + gc.threed_class = 0xC797; // AMPERE_B + gc.compute_class = 0xC7C0; // AMPERE_COMPUTE_B + gc.gpfifo_class = 0xC56F; // AMPERE_CHANNEL_GPFIFO_A + gc.dma_copy_class = 0xC7B5; // AMPERE_DMA_COPY_B + gc.chipname = 0x6631306167; // "ga10f" + gc.flags |= 0x100; // NVGPU_GPU_FLAGS_SUPPORT_TSG: NvRm binds channels to a TSG + gc.flags |= 1ULL << 22; // NVGPU_GPU_FLAGS_SUPPORT_TSG_SUBCONTEXTS (see below) + gc.flags |= 1ULL << 26; // NVGPU_GPU_FLAGS_SUPPORT_SYNCPOINT_ADDRESS: syncpoint shim + // NVGPU_GPU_FLAGS_SUPPORT_MAPPING_MODIFY: NVN2 gives a texture in a compressible pool its + // address only after NvRmGpuMappingModify switched the pool's mapping to the texture's kind. + // Without the flag NvRm fails that call and the texture stays at GPU address 0 (title 1's + // movie and the passes rendered into compressible pools never showed). + gc.flags |= 1ULL << 34; + + // The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's + // *_ioctl_nr_last); its NvRm requires e.g. [0x78] >= 6 and enables features above 0xa, 0x1f + // and 0x26. In the Switch 1 layout these bytes are max_fbps_count..max_tex_per_tpc. + auto* const raw = reinterpret_cast<u8*>(&gc); + for (size_t off = 0x70; off <= 0x80; off += 2) { + const s16 version = 0x40; + std::memcpy(raw + off, &version, sizeof(version)); + } +} + +// Fields of the Switch 2 block past the Switch 1 layout. NvRm sizes each TSG's subcontext (VEID) +// allocator from the u32 at 0xb4, but only when the TSG interface version (0x72) is above 0xa and +// flags bits 8 and 22 are set; otherwise every TSG gets one subcontext and a second queue's +// NvRmGpuTSGSubcontextCreate fails (0x11), leaving that NVN queue without command memory. +void ApplyNx2CharacteristicsTail(std::span<u8> block) { + constexpr size_t MaxSubcontextsOffset = 0xb4; + constexpr u32 MaxSubcontexts = 64; // Ampere VEIDs per TSG + if (block.size() >= MaxSubcontextsOffset + sizeof(MaxSubcontexts)) { + std::memcpy(block.data() + MaxSubcontextsOffset, &MaxSubcontexts, sizeof(MaxSubcontexts)); + } +} + +} // namespace + NvResult nvhost_ctrl_gpu::GetCharacteristics1(IoctlCharacteristics& params) { LOG_DEBUG(Service_NVDRV, "called"); params.gc.arch = 0x120; @@ -164,7 +276,8 @@ NvResult nvhost_ctrl_gpu::GetCharacteristics1(IoctlCharacteristics& params) { NvResult nvhost_ctrl_gpu::GetCharacteristics3( IoctlCharacteristics& params, std::span<IoctlGpuCharacteristics> gpu_characteristics) { - LOG_DEBUG(Service_NVDRV, "called"); + LOG_DEBUG(Service_NVDRV, "called, buf_size={:#x}", params.gpu_characteristics_buf_size); + const bool nx2 = params.gpu_characteristics_buf_size >= Nx2CharacteristicsSize; params.gc.arch = 0x120; params.gc.impl = 0xb; @@ -203,6 +316,14 @@ NvResult nvhost_ctrl_gpu::GetCharacteristics3( params.gc.gr_compbit_store_base_hw = 0x0; params.gpu_characteristics_buf_size = 0xA0; params.gpu_characteristics_buf_addr = 0xdeadbeef; // Cannot be 0 (UNUSED) + if (nx2) { + nx2_caller = true; + g_nx2_driver = true; + Tegra::g_nx2_ampere = true; + ApplyNx2Characteristics(params.gc); + params.gpu_characteristics_buf_size = Nx2CharacteristicsSize; + LOG_WARNING(Service_NVDRV, "Switch 2 caller, reporting Ampere chip 0x17E"); + } if (!gpu_characteristics.empty()) { gpu_characteristics.front() = params.gc; } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h index 80b7abd6a1..507cdac71d 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h @@ -208,6 +208,11 @@ private: EventInterface& events_interface; // Events + NvResult Nx2GetEngineInfo(std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output); + + // Set once a caller asked for the Switch 2 characteristics layout. + bool nx2_caller{}; Kernel::KEvent* error_notifier_event; Kernel::KEvent* unknown_event; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 1e571be4c9..2a144cc7ab 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -4,7 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> #include <cstring> +#include <vector> #include "common/assert.h" #include "common/logging.h" #include "core/core.h" @@ -14,6 +16,7 @@ #include "core/hle/service/nvdrv/core/syncpoint_manager.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/memory.h" #include "video_core/control/channel_state.h" @@ -48,6 +51,9 @@ nvhost_gpu::~nvhost_gpu() { events_interface.FreeEvent(sm_exception_breakpoint_int_report_event); events_interface.FreeEvent(sm_exception_breakpoint_pause_report_event); events_interface.FreeEvent(error_notifier_event); + for (auto& [id, event] : nx2_events) { + events_interface.FreeEvent(event); + } syncpoint_manager.FreeSyncpoint(channel_syncpoint); } @@ -101,6 +107,9 @@ NvResult nvhost_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> inpu } break; } + if (g_nx2_driver) { + return Nx2UnknownIoctl("nvhost-gpu", command, input, output); + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw); return NvResult::NotImplemented; }; @@ -116,12 +125,19 @@ NvResult nvhost_gpu::Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> inpu } break; } + if (g_nx2_driver) { + return Nx2UnknownIoctl("nvhost-gpu", command, input, output); + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw); return NvResult::NotImplemented; } NvResult nvhost_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output, std::span<u8> inline_output) { + if (g_nx2_driver) { + std::fill(inline_output.begin(), inline_output.end(), u8{0}); + return Nx2UnknownIoctl("nvhost-gpu", command, input, output); + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw); return NvResult::NotImplemented; } @@ -278,6 +294,35 @@ NvResult nvhost_gpu::AllocateObjectContext(IoctlAllocObjCtx& params) { params.flags = allowed_mask; } + if (g_nx2_driver) { + // Switch 2 NvRm asks for Ampere classes, or class 0 (it reads the class numbers from a + // characteristics layout we don't know yet). Map them onto the Maxwell engine slots. + u32 mapped = params.class_num; + switch (params.class_num) { + case 0xC797: case 0xC697: case 0xC597: mapped = static_cast<u32>(CtxClasses::Ctx3D); break; + case 0xC7C0: case 0xC6C0: case 0xC5C0: mapped = static_cast<u32>(CtxClasses::CtxCompute); break; + case 0xC7B5: case 0xC6B5: case 0xC5B5: mapped = static_cast<u32>(CtxClasses::CtxDMA); break; + case 0xC56F: case 0xC76F: mapped = static_cast<u32>(CtxClasses::CtxChannelGPFIFO); break; + case 0: { + for (const auto candidate : {CtxClasses::Ctx3D, CtxClasses::CtxCompute, CtxClasses::CtxDMA, + CtxClasses::Ctx2D, CtxClasses::CtxKepler}) { + const auto index = GetObjectContextClassNumberIndex(candidate); + if (index >= 0 && !ctxObjs[index].has_value()) { + mapped = static_cast<u32>(candidate); + break; + } + } + break; + } + default: + break; + } + if (mapped != params.class_num) { + LOG_WARNING(Service_NVDRV, "Switch 2 object class {:#x} -> {:#x}", params.class_num, mapped); + params.class_num = mapped; + } + } + s32_le ctx_class_number_index = GetObjectContextClassNumberIndex(static_cast<CtxClasses>(params.class_num)); if (ctx_class_number_index < 0) { @@ -327,6 +372,27 @@ static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementC return result; } +// Switch 2 pushbuffers signal completion with semaphore releases, never host1x syncpoint methods, +// but the driver still has the kernel raise the syncpoint max by the submission's increment value +// and then waits for that value. Append the increments after its commands. +static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementCountCommandList( + NvFence fence, u32 count) { + boost::container::small_vector<Tegra::CommandHeader, 512> result{ + Tegra::BuildCommandHeader(Tegra::BufferMethods::WaitForIdle, 1, + Tegra::SubmissionMode::Increasing), + {}, + Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointPayload, 1, + Tegra::SubmissionMode::Increasing), + {}}; + for (u32 i = 0; i < count; ++i) { + result.push_back(Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointOperation, 1, + Tegra::SubmissionMode::Increasing)); + result.push_back( + BuildFenceAction(Tegra::Engines::Puller::FenceOperation::Increment, fence.id)); + } + return result; +} + static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementWithWfiCommandList( NvFence fence) { boost::container::small_vector<Tegra::CommandHeader, 512> result{ @@ -353,6 +419,23 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL const auto bind_id = channel_state->bind_id; + if (g_nx2_driver) { + for (const auto& entry : entries.command_lists) { + LOG_TRACE(Service_NVDRV, "nx2 GPFIFO entry raw={:#018x} addr={:#x} size={:#x}", entry.raw, + entry.addr.Value(), entry.size.Value()); + } + static const bool check = std::getenv("EDEN_NX2_PB_CHECK") != nullptr; + if (check) { + std::vector<u32> words; + for (const auto& entry : entries.command_lists) { + words.resize(entry.size); + system.ApplicationMemory().ReadBlock(entry.addr.Value(), words.data(), + words.size() * sizeof(u32)); + entries.submit_hashes.push_back(Tegra::HashPushbufferWords(words)); + } + } + } + auto& flags = params.flags; if (flags.fence_wait.Value()) { @@ -372,6 +455,11 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL params.fence.value = syncpoint_manager.IncrementSyncpointMaxExt(channel_syncpoint, increment); gpu.PushGPUEntries(bind_id, std::move(entries)); + if (g_nx2_driver && !flags.fence_increment.Value() && increment > 0) { + gpu.PushGPUEntries(bind_id, + Tegra::CommandList{BuildIncrementCountCommandList(params.fence, increment)}); + } + if (flags.fence_increment.Value()) { if (flags.suppress_wfi.Value()) { gpu.PushGPUEntries(bind_id, @@ -382,6 +470,12 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL } } + if (g_nx2_driver) { + LOG_TRACE(Service_NVDRV, + "nx2 SubmitGPFIFO entries={} flags={:#x} fence_out=({}, {}) guest_value={}", + params.num_entries, flags.raw, params.fence.id, params.fence.value, + system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(params.fence.id)); + } flags.raw = 0; return NvResult::Success; @@ -464,6 +558,16 @@ Kernel::KEvent* nvhost_gpu::QueryEvent(u32 event_id) { case 3: return error_notifier_event; default: + if (g_nx2_driver) { + // Switch 2 NVN also asks for event 7 and multi-waits on it with its syncpoint events. + // Nothing signals it yet. + auto*& event = nx2_events[event_id]; + if (event == nullptr) { + event = events_interface.CreateEvent(fmt::format("GpuChannelNx2Event{}", event_id)); + } + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 channel event {}", event_id); + return event; + } LOG_CRITICAL(Service_NVDRV, "Unknown Ctrl GPU Event {}", event_id); return nullptr; } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h index 70a6b1bd68..ab968d6c7e 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h @@ -6,6 +6,7 @@ #pragma once +#include <map> #include <memory> #include <vector> #include "common/bit_field.h" @@ -225,6 +226,7 @@ private: Kernel::KEvent* sm_exception_breakpoint_int_report_event; Kernel::KEvent* sm_exception_breakpoint_pause_report_event; Kernel::KEvent* error_notifier_event; + std::map<u32, Kernel::KEvent*> nx2_events; }; } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h new file mode 100644 index 0000000000..b91a929f68 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <atomic> +#include <functional> +#include <map> +#include <mutex> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include "common/logging.h" +#include "core/hle/service/nvdrv/devices/nvdevice.h" + +namespace Service::Nvidia::Devices { + +// Set once a Switch 2 NvRm has identified itself (0xD0-byte GetCharacteristics). Devices then +// answer ioctls they don't know with a logged, input-echoing success instead of NotImplemented. +inline std::atomic<bool> g_nx2_driver{false}; + +inline NvResult Nx2UnknownIoctl(const char* device, Ioctl command, std::span<const u8> input, + std::span<u8> output) { + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 {} ioctl={:08X} group={:01X} command={:02X} in={:#x}", + device, command.raw, command.group, command.cmd, input.size()); + std::fill(output.begin(), output.end(), u8{0}); + std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin()); + return NvResult::Success; +} + +// GPU mappings a Switch 2 process made through nx2:gpu: CreateGpuAccessibleMemory (sub 101) maps +// the memory at GPU address == CPU address, and NormalMapGpuMemory4K (211) maps a memory handle at +// a chosen GPU address (also in sparse regions). Every nvhost-as-gpu address space applies each +// mapping, including ones made before the address space existed. +class Nx2GpuMemoryRegistry { +public: + using Listener = std::function<void(u64 gpu_address, u64 cpu_address, u64 size)>; + using UnmapListener = std::function<void(u64 gpu_address, u64 size)>; + using ReleaseListener = std::function<void(u64 cpu_address, u64 size)>; + + static Nx2GpuMemoryRegistry& Get() { + static Nx2GpuMemoryRegistry registry; + return registry; + } + + void Register(u64 address, u64 size) { + MapAt(address, address, size); + } + + void MapAt(u64 gpu_address, u64 cpu_address, u64 size) { + std::scoped_lock lk{mutex}; + TrimMappings(gpu_address, size); + mappings.push_back({gpu_address, cpu_address, size}); + for (auto& [owner, listener] : listeners) { + listener.map(gpu_address, cpu_address, size); + } + } + + // NormalUnmapGpuMemory4K, or destroying the memory: the GPU range stops reaching it. + void Unmap(u64 gpu_address, u64 size) { + std::scoped_lock lk{mutex}; + TrimMappings(gpu_address, size); + for (auto& [owner, listener] : listeners) { + listener.unmap(gpu_address, size); + } + } + + // Backing memory [cpu_address, +size) is about to be freed (its GPU mappings are gone): each + // address space drops its SMMU mapping of it, in GPU-thread order. + void ReleaseBacking(u64 cpu_address, u64 size) { + { + std::scoped_lock lk{device_mutex}; + device_ranges.erase(cpu_address); + } + std::scoped_lock lk{mutex}; + for (auto& [owner, listener] : listeners) { + if (listener.release) { + listener.release(cpu_address, size); + } + } + } + + void AddListener(const void* owner, Listener listener, UnmapListener unmap_listener, + ReleaseListener release_listener = {}) { + std::scoped_lock lk{mutex}; + for (const auto& m : mappings) { + listener(m.gpu_address, m.cpu_address, m.size); + } + listeners.insert_or_assign(owner, Listeners{std::move(listener), std::move(unmap_listener), + std::move(release_listener)}); + } + + void RemoveListener(const void* owner) { + std::scoped_lock lk{mutex}; + listeners.erase(owner); + } + + // Device (SMMU) address of a CPU range an address space mapped; used to present images. + void RecordDeviceRange(u64 cpu_address, u64 size, u64 device_address) { + std::scoped_lock lk{device_mutex}; + device_ranges[cpu_address] = {size, device_address}; + } + + std::optional<u64> DeviceAddressOf(u64 cpu_address) { + std::scoped_lock lk{device_mutex}; + auto it = device_ranges.upper_bound(cpu_address); + if (it == device_ranges.begin()) { + return std::nullopt; + } + --it; + const auto& [start, range] = *it; + if (cpu_address - start >= range.first) { + return std::nullopt; + } + return range.second + (cpu_address - start); + } + +private: + // Separate lock: ranges are recorded from listeners that run under `mutex`. + std::mutex device_mutex; + std::map<u64, std::pair<u64, u64>> device_ranges; // cpu start -> (size, device address) + + struct Mapping { + u64 gpu_address; + u64 cpu_address; + u64 size; + }; + + struct Listeners { + Listener map; + UnmapListener unmap; + ReleaseListener release; + }; + + // Drops [gpu_address, +size) from the recorded mappings, keeping the parts outside it. + void TrimMappings(u64 gpu_address, u64 size) { + const u64 end = gpu_address + size; + std::vector<Mapping> kept; + kept.reserve(mappings.size() + 1); + for (const auto& m : mappings) { + const u64 m_end = m.gpu_address + m.size; + if (m_end <= gpu_address || m.gpu_address >= end) { + kept.push_back(m); + continue; + } + if (m.gpu_address < gpu_address) { + kept.push_back({m.gpu_address, m.cpu_address, gpu_address - m.gpu_address}); + } + if (m_end > end) { + kept.push_back({end, m.cpu_address + (end - m.gpu_address), m_end - end}); + } + } + mappings = std::move(kept); + } + + std::mutex mutex; + std::vector<Mapping> mappings; + std::map<const void*, Listeners> listeners; +}; + +// Device node that Switch 2 nvgpu drivers open but Switch 1 never had (e.g. /dev/nvhost-tsg-gpu). +// Every ioctl succeeds with a zeroed reply and is logged, so the driver's use of it shows up. +class nx2_stub_device final : public nvdevice { +public: + explicit nx2_stub_device(Core::System& system_, std::string name_) + : nvdevice{system_}, name{std::move(name_)} {} + + NvResult Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<u8> output) override { + return Stub(fd, command, input, output, {}); + } + NvResult Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<const u8> inline_input, std::span<u8> output) override { + return Stub(fd, command, input, output, {}); + } + NvResult Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output) override { + return Stub(fd, command, input, output, inline_output); + } + + void OnOpen(NvCore::SessionId session_id, DeviceFD fd) override { + LOG_WARNING(Service_NVDRV, "(STUBBED) {} opened as fd {}", name, fd); + } + void OnClose(DeviceFD fd) override {} + +private: + NvResult Stub(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output) { + LOG_WARNING(Service_NVDRV, "(STUBBED) {} fd={} ioctl={:08X} group={:01X} command={:02X} in={:#x}", + name, fd, command.raw, command.group, command.cmd, input.size()); + // Echo the input (in/out ioctls keep their fields), zero the rest. + std::fill(output.begin(), output.end(), u8{0}); + std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin()); + std::fill(inline_output.begin(), inline_output.end(), u8{0}); + return NvResult::Success; + } + + std::string name; +}; + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index b252a2efa5..5cac2693c5 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -23,6 +23,7 @@ #include "core/hle/service/nvdrv/devices/nvhost_nvjpg.h" #include "core/hle/service/nvdrv/devices/nvhost_vic.h" #include "core/hle/service/nvdrv/devices/nvmap.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/hle/service/nvdrv/nvdrv_interface.h" #include "core/hle/service/nvdrv/nvmemp.h" @@ -125,6 +126,11 @@ void LoopProcess(Core::System& system) { Module::Module(Core::System& system) : container{system.Host1x()}, service_context{system, "nvdrv"}, events_interface{*this} { + nx2_stub_builder = [this, &system](DeviceFD fd, const std::string& name) { + auto device = std::make_shared<Devices::nx2_stub_device>(system, name); + open_files.emplace(fd, device); + return std::static_pointer_cast<Devices::nvdevice>(device); + }; builders["/dev/nvhost-as-gpu"] = [this, &system](DeviceFD fd) { auto device = std::make_shared<Devices::nvhost_as_gpu>(system, *this, container); return open_files.emplace(fd, std::move(device)).first; @@ -181,6 +187,15 @@ NvResult Module::VerifyFD(DeviceFD fd) const { DeviceFD Module::Open(const std::string& device_name, NvCore::SessionId session_id) { auto it = builders.find(device_name); + if (it == builders.end() && device_name.starts_with("/dev/nvhost-")) { + // Newer nvgpu nodes used by Switch 2 drivers (e.g. /dev/nvhost-tsg-gpu): serve them with + // a logged stub instead of failing the open. + LOG_WARNING(Service_NVDRV, "Opening unknown device {} as a Switch 2 stub", device_name); + const DeviceFD fd = next_fd++; + auto device = nx2_stub_builder(fd, device_name); + device->OnOpen(session_id, fd); + return fd; + } if (it == builders.end()) { LOG_ERROR(Service_NVDRV, "Trying to open unknown device {}", device_name); return INVALID_NVDRV_FD; diff --git a/src/core/hle/service/nvdrv/nvdrv.h b/src/core/hle/service/nvdrv/nvdrv.h index f503f108c3..ba8ab6a40e 100644 --- a/src/core/hle/service/nvdrv/nvdrv.h +++ b/src/core/hle/service/nvdrv/nvdrv.h @@ -98,6 +98,8 @@ private: friend class EventInterface; /// Manages syncpoints on the host + /// Builds a logged stub for nvgpu device nodes only Switch 2 drivers use. + std::function<std::shared_ptr<Devices::nvdevice>(DeviceFD, const std::string&)> nx2_stub_builder; NvCore::Container container; /// Id to use for the next open file descriptor. diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.cpp b/src/core/hle/service/nvdrv/nvdrv_interface.cpp index 5fae68adbb..020036d74b 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.cpp +++ b/src/core/hle/service/nvdrv/nvdrv_interface.cpp @@ -34,6 +34,7 @@ void NVDRV::Open(HLERequestContext& ctx) { const auto& buffer = ctx.ReadBuffer(); const std::string device_name(Common::StringFromBuffer(buffer)); + LOG_DEBUG(Service_NVDRV, "device_name={}", device_name); if (device_name == "/dev/nvhost-prof-gpu") { rb.Push<DeviceFD>(0); @@ -181,6 +182,14 @@ void NVDRV::Initialize(HLERequestContext& ctx) { is_initialized = true; } +void NVDRV::InitializeForProcess(Kernel::KProcess* process) { + if (is_initialized) { + return; + } + session_id = nvdrv->GetContainer().OpenSession(process); + is_initialized = true; +} + void NVDRV::QueryEvent(HLERequestContext& ctx) { IPC::RequestParser rp{ctx}; const auto fd = rp.Pop<DeviceFD>(); diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.h b/src/core/hle/service/nvdrv/nvdrv_interface.h index f628fca7df..8fe3196176 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.h +++ b/src/core/hle/service/nvdrv/nvdrv_interface.h @@ -23,6 +23,9 @@ public: return nvdrv; } + // For Switch 2 nvsrv sessions, which are set up through gsv instead of Initialize. + void InitializeForProcess(Kernel::KProcess* process); + private: void Open(HLERequestContext& ctx); void Ioctl1(HLERequestContext& ctx); diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h new file mode 100644 index 0000000000..e249b4911e --- /dev/null +++ b/src/core/hle/service/nx2_gsv.h @@ -0,0 +1,2068 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <thread> +#include <chrono> +#include <condition_variable> +#include <cstring> +#include <set> +#include <span> +#include <string> +#include <cstdlib> +#include <cstdio> +#include <initializer_list> +#include <atomic> +#include <deque> +#include <map> +#include <optional> +#include <memory> +#include <mutex> +#include <vector> + +#include "common/alignment.h" +#include "common/settings.h" +#include "common/thread.h" +#include "core/core.h" +#include "video_core/rasterizer_interface.h" +#include "video_core/renderer_base.h" +#include "core/frontend/framebuffer_layout.h" +#include "core/perf_stats.h" +#include "video_core/gpu.h" +#include "video_core/framebuffer_config.h" +#include "core/core_timing.h" +#include "core/hle/kernel/k_client_session.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/k_system_resource.h" +#include "core/hle/kernel/k_memory_manager.h" +#include "core/hle/kernel/k_page_group.h" +#include "core/hle/kernel/k_resource_limit.h" +#include "core/hle/kernel/k_server_session.h" +#include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_readable_event.h" +#include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_transfer_memory.h" +#include "core/memory.h" +#include "video_core/host1x/host1x.h" +#include "video_core/host1x/nx2_video_decoder.h" +#include "video_core/nx2.h" +#include "core/hle/service/hle_ipc.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/kernel_helpers.h" +#include "core/hle/service/nvdrv/core/container.h" +#include "core/hle/service/nvdrv/core/syncpoint_manager.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" +#include "core/hle/service/nvdrv/nvdrv.h" +#include "core/hle/service/nvdrv/nvdrv_interface.h" +#include "core/hle/service/nx2_stub_interface.h" +#include "core/hle/service/sm/sm.h" +#include "core/hle/service/server_manager.h" +#include "core/hle/service/service.h" + +namespace Service { + +// Creates a session served by `iface` and returns its client end. Unlike PushIpcInterface this +// works from a domain too, for callers that want a raw session handle. +template <class T> +inline Kernel::KClientSession* CreateRawSession(Core::System& system, HLERequestContext& ctx, + std::shared_ptr<T> iface) { + auto& kernel = system.Kernel(); + ASSERT(Kernel::GetCurrentProcess(kernel).GetResourceLimit()->Reserve( + kernel, Kernel::LimitableResource::SessionCountMax, 1)); + + auto* session = Kernel::KSession::Create(kernel); + session->Initialize(kernel, nullptr, 0); + Kernel::KSession::Register(kernel, session); + + auto manager = + std::make_shared<SessionRequestManager>(kernel, ctx.GetManager()->GetServerManager()); + manager->SetSessionHandler(std::move(iface)); + ctx.GetManager()->GetServerManager().RegisterSession(&session->GetServerSession(), manager); + return &session->GetClientSession(); +} + +// Replies to a TIPC request with success and raw u32 data words. ResponseBuilder::Push(Result) +// writes the CMIF two-word result, which shifts TIPC data one word past where the client reads +// it (and past the copied size); TIPC has a one-word result followed directly by the data. +inline void TipcReplyWords(HLERequestContext& ctx, std::span<const u32> words) { + IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())}; + rb.PushRaw<u32>(ResultSuccess.raw); + for (const u32 w : words) { + rb.Push<u32>(w); + } +} + +inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> words) { + TipcReplyWords(ctx, std::span<const u32>{words.begin(), words.size()}); +} + +// Replies with success and a copied handle to a new session served by `iface`. +template <class T> +inline void ReturnRawSession(Core::System& system, HLERequestContext& ctx, std::shared_ptr<T> iface) { + auto* client = CreateRawSession(system, ctx, std::move(iface)); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, *client); +} + +// Placeholder for the Switch 2 NVIDIA driver server (nn::nvsrv). It has no handlers, so every +// command (CMIF or TIPC) gets an empty success and is logged through the nx2: prefix. +class INx2NvsrvSession final : public ServiceFramework<INx2NvsrvSession> { +public: + explicit INx2NvsrvSession(Core::System& system_) : ServiceFramework{system_, "nx2:nvsrv"} {} +}; + +// Placeholder for the Switch 2 syncpoint server (nn::syncpt), handed out by gsv 202. TIPC 1 +// returns a u32 the SDK requires to be < 0x400 and then passes min(value, 63) to TIPC 21. +// Everything else is auto-stubbed through the nx2: prefix. +// - TIPC 0(u64): handshake, echo the token (newer SDKs). +// - TIPC 22(u32 slot): an event handle for that slot (copy handle at word 3, result at word 4). +// NVN's NvOsEventPool asks for slots and multi-waits on them. +// - TIPC 201(slot, counter, u64 value, mode): signal the slot's event once the counter reaches +// value. The counter is a SyncCounterId, the bitwise NOT of the host1x syncpoint id; mode 0 +// compares 32-bit wrapping, mode 1 64-bit. Eden's guest syncpoints back it. +// - TIPC 202(slot): cancel the slot's request. +class INx2SyncptSession final : public ServiceFramework<INx2SyncptSession> { +public: + explicit INx2SyncptSession(Core::System& system_) + : ServiceFramework{system_, "nx2:syncpt"}, m_context{system_, "nx2:syncpt"} { + RegisterHandlersTipc({ + {0, &INx2SyncptSession::Handshake, "Handshake"}, + {1, &INx2SyncptSession::GetCount, "GetCount (guessed)"}, + {22, &INx2SyncptSession::GetSlotEvent, "GetSlotEvent (guessed)"}, + {103, &INx2SyncptSession::QueryCounter, "QueryCounter"}, + {201, &INx2SyncptSession::RequestSlotEvent, "RequestSlotEvent (guessed)"}, + {202, &INx2SyncptSession::CancelSlotEvent, "CancelSlotEvent (guessed)"}, + }); + } + + ~INx2SyncptSession() override { + // A registered action may be running on the GPU thread right now: mark the session dead + // under the lock it signals under, so it can't touch a request or event closed below. + { + std::scoped_lock lk{m_shared->mutex}; + m_shared->alive = false; + } + for (auto& [slot, request] : m_requests) { + Cancel(request); + } + for (auto& [slot, event] : m_events) { + m_context.CloseEvent(event); + } + } + +private: + // Newer SDKs (title 1) open with TIPC 0(u64 token) and abort unless the reply echoes it. + void Handshake(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "nx2:syncpt TIPC 0 handshake {:#x}", u64{w[2]} | (u64{w[3]} << 32)); + TipcReplyWords(ctx, {w[2], w[3]}); + } + + void GetCount(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:syncpt TIPC 1, returning 32"); + TipcReplyWords(ctx, {32}); + } + + // From changes.patch: the counter is the bitwise NOT of the host1x syncpoint id. + void QueryCounter(HLERequestContext& ctx) { + const u32 syncpoint = ~ctx.CommandBuffer()[2]; + const u64 value = syncpoint < HostSyncpoints + ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(syncpoint) + : 0; + LOG_DEBUG(Service, "nx2:syncpt 103 syncpoint={} value={:#x}", syncpoint, value); + TipcReplyWords(ctx, {static_cast<u32>(value), static_cast<u32>(value >> 32), 0}); + } + + void GetSlotEvent(HLERequestContext& ctx) { + const u32 slot = ctx.CommandBuffer()[2]; + LOG_WARNING(Service, "(STUBBED) nx2:syncpt TIPC 22 slot={:#x}, returning an event", slot); + auto*& event = m_events[slot]; + if (event == nullptr) { + event = m_context.CreateEvent(fmt::format("nx2:syncpt:Slot{:x}", slot)); + } + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + // Map nodes are stable, so a registered action can hold a reference to its request. + struct SlotRequest { + u32 syncpoint{}; + std::atomic<bool> pending{}; + Tegra::Host1x::SyncpointManager::ActionHandle handle{}; + }; + + // Outlives the session (actions hold a reference): signalling happens under `mutex` and only + // while `alive`. + struct SharedState { + std::mutex mutex; + bool alive{true}; + }; + + void RequestSlotEvent(HLERequestContext& ctx) { + const auto* cmd_buf = ctx.CommandBuffer(); + const u32 slot = cmd_buf[2]; + const u32 syncpoint = ~cmd_buf[3]; + const u64 value = u64{cmd_buf[4]} | (u64{cmd_buf[5]} << 32); + const u32 mode = cmd_buf[6]; + LOG_INFO(Service, "nx2:syncpt TIPC 201 slot={:#x} syncpoint={} value={:#x} mode={}", slot, + syncpoint, value, mode); + auto*& event = m_events[slot]; + if (event == nullptr) { + event = m_context.CreateEvent(fmt::format("nx2:syncpt:Slot{:x}", slot)); + } + if (syncpoint >= HostSyncpoints) { + LOG_WARNING(Service, "nx2:syncpt TIPC 201 syncpoint {} out of range, signalling", + syncpoint); + event->Signal(system.Kernel()); + TipcReplyWords(ctx, {}); + return; + } + auto& request = m_requests[slot]; + Cancel(request); + // A new request starts unsignalled. Slots are reused by different client objects: title 1's + // movie player got slot 0 after NVN's fence waits had left it signalled, took that as a + // decoded frame and aborted when the decoder had none. + event->Clear(system.Kernel()); + request.syncpoint = syncpoint; + request.pending = true; + auto& manager = system.Host1x().GetSyncpointManager(); + // The action runs on the GPU thread under the manager's lock, or inline if the value is + // already reached. + request.handle = manager.RegisterGuestAction( + syncpoint, static_cast<u32>(value), + [shared = m_shared, &kernel = system.Kernel(), &request, event] { + std::scoped_lock lk{shared->mutex}; + if (!shared->alive) { + return; + } + request.pending = false; + event->Signal(kernel); + }); + TipcReplyWords(ctx, {}); + } + + void CancelSlotEvent(HLERequestContext& ctx) { + const u32 slot = ctx.CommandBuffer()[2]; + LOG_INFO(Service, "nx2:syncpt TIPC 202 slot={:#x}", slot); + if (auto it = m_requests.find(slot); it != m_requests.end()) { + Cancel(it->second); + } + // nn::syncpt::sdk's cancel always waits on the slot's event after this reply, so a cancel + // signals it like a completion, also when nothing was pending; the client rechecks the + // counter (TIPC 103) itself. Title 1 hung forever there when the GPU thread ran a frame + // behind (EDEN_NX2_FRAMES_AHEAD) and a mid-frame wait outlasted NvOsEventPool's 100 ms, + // and its movie player cancels each slot before its first request: signalled only for + // pending requests, the player never got to start a movie. + auto*& event = m_events[slot]; + if (event == nullptr) { + event = m_context.CreateEvent(fmt::format("nx2:syncpt:Slot{:x}", slot)); + } + event->Signal(system.Kernel()); + TipcReplyWords(ctx, {}); + } + + /// Returns true if the request was still pending. + bool Cancel(SlotRequest& request) { + if (request.pending.exchange(false)) { + system.Host1x().GetSyncpointManager().DeregisterGuestAction(request.syncpoint, + request.handle); + return true; + } + return false; + } + + static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager + + KernelHelpers::ServiceContext m_context; + std::map<u32, Kernel::KEvent*> m_events; + std::map<u32, SlotRequest> m_requests; + std::shared_ptr<SharedState> m_shared = std::make_shared<SharedState>(); +}; + +// Placeholder for the Switch 2 GPU session (nn::gpu::sdk), handed out by gsv 497. +// - TIPC 11: [first, last] id range (first <= min(last, 64)); ids outside it are masked out. +// - TIPC 12(id): event handle for that id (handle at word 3, result at word 4); the SDK attaches +// it as a SystemEvent. +// - TIPC 21(id): starts an operation; the SDK then waits on the id's event. Nothing runs here, so +// the event is signalled at once. +// - TIPC 22: another handle the SDK keeps like a session; a placeholder session is returned. +class INx2GpuSession final : public ServiceFramework<INx2GpuSession> { +public: + explicit INx2GpuSession(Core::System& system_) + : ServiceFramework{system_, "nx2:gpu"}, m_context{system_, "nx2:gpu"} { + RegisterHandlersTipc({ + {11, &INx2GpuSession::GetIdRange, "GetIdRange (guessed)"}, + {12, &INx2GpuSession::GetIdEvent, "GetIdEvent (guessed)"}, + {21, &INx2GpuSession::Start, "Start (guessed)"}, + {22, &INx2GpuSession::OpenSubSession, "OpenSubSession (guessed)"}, + {33, &INx2GpuSession::GetCount33, "Unknown33 (u32 count)"}, + {201, &INx2GpuSession::GetUserPartition, "GetUserPartition (newer SDKs)"}, + {202, &INx2GpuSession::GetGsvPartition, "GetGsvPartition (newer SDKs)"}, + {203, &INx2GpuSession::GetVaRange, "GetVaRange (guessed)"}, + // Sub-session commands (gpu 22 hands out a second session to this same object). + {101, &INx2GpuSession::SubRegisterMemory, "CreateGpuAccessibleMemory"}, + {102, &INx2GpuSession::SubDestroyMemory, "DestroyGpuAccessibleMemory (guessed)"}, + {211, &INx2GpuSession::SubSubmitJob, "NormalMapGpuMemory4K"}, + {212, &INx2GpuSession::SubModifyKind, "NormalModifyMemoryKind4K"}, + {213, &INx2GpuSession::SubUnmapJob, "NormalUnmapGpuMemory4K"}, + {214, &INx2GpuSession::SubCreateGpuOnlyRegion, "CreateSparseRegion"}, + {215, &INx2GpuSession::SubDestroySparseRegion, "DestroySparseRegion"}, + }); + } + + ~INx2GpuSession() override { + for (auto* event : m_events) { + if (event != nullptr) { + m_context.CloseEvent(event); + } + } + } + +private: + static constexpr u32 NumIds = 32; + + Kernel::KEvent* EventFor(u32 id) { + id %= NumIds; + if (m_events[id] == nullptr) { + m_events[id] = m_context.CreateEvent(fmt::format("nx2:gpu:Event{}", id)); + } + return m_events[id]; + } + + void GetIdRange(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 11, returning [0, {}]", NumIds - 1); + TipcReplyWords(ctx, {0, NumIds - 1}); + } + + void GetIdEvent(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 12 id={}", id); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, EventFor(id)->GetReadableEvent()); + } + + void Start(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 21 id={}, signalling completion", id); + EventFor(id)->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void GetCount33(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 33 [2]={:#x}, returning 1", ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {1}); + } + + // Used by nn::gpu::sdk::CreateGpuAddressSpace: three u64s after the result, a kind (< 5, 0 = + // none) and the GPU VA range of the in-process VirtualMachine that serves AllocSpace + // (nn::gpu::sdk::util::ExecuteVirtualMachineAllocSpace). NVN reserves fixed VA from 0x8000000 + // up to 512 GiB, so report the same range NvRm gives AllocAsEx: [0x8000000, 0xC000000000). + void GetVaRange(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 203 [2]={:#x}", ctx.CommandBuffer()[2]); + constexpr u64 kind = 1; + constexpr u64 start = 0x8000000ULL; + constexpr u64 size = 0xC000000000ULL - start; + TipcReplyWords(ctx, {static_cast<u32>(kind), static_cast<u32>(kind >> 32), + static_cast<u32>(start), static_cast<u32>(start >> 32), + static_cast<u32>(size), static_cast<u32>(size >> 32)}); + } + + // Newer SDKs (title 1) query the address space's user and gsv partitions with 201/202(id): + // two u64s each, stored as the partition's min/max. The user value works whether it is read + // as (start, size) or (min, max). Title 4 never needed the gsv partition, so it stays empty. + void GetUserPartition(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 201 [2]={:#x}", ctx.CommandBuffer()[2]); + constexpr u64 first = 0x8000000ULL; + constexpr u64 second = 0xC000000000ULL - first; + TipcReplyWords(ctx, {static_cast<u32>(first), static_cast<u32>(first >> 32), + static_cast<u32>(second), static_cast<u32>(second >> 32)}); + } + + void GetGsvPartition(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 202 [2]={:#x}, empty", ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {0, 0, 0, 0}); + } + + void OpenSubSession(HLERequestContext& ctx) { + // Served by this object so sub-session jobs can signal the per-id events. + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 22, returning a sub-session on nx2:gpu"); + ReturnRawSession(system, ctx, std::static_pointer_cast<INx2GpuSession>(shared_from_this())); + } + + // Sub 101: (start u64, last u64, flags...) after CreateGpuAccessibleMemory's svc 0x80. + void SubRegisterMemory(HLERequestContext& ctx) { + const auto* b = ctx.CommandBuffer(); + const u64 start = u64{b[2]} | (u64{b[3]} << 32); + const u64 last = u64{b[4]} | (u64{b[5]} << 32); + LOG_INFO(Service, "nx2:gpu sub 101 start={:#x} last={:#x} [6]={:#x} [7]={:#x}", start, + last, b[6], b[7]); + if (last > start) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(start, last - start + 1); + // Two-page regions hold queue semaphores: GPU releases go to the second page, waits + // read the first (see video_core/nx2.h). + if (last - start + 1 == 0x2000) { + Tegra::Nx2AddSemaphoreRegion(start); + LOG_INFO(Service, "nx2:gpu semaphore page {:#x} mirrored to {:#x}", start + 0x1000, + start); + } + } + // nn::gpu::sdk::CreateGpuAccessibleMemory stores word 3 of the reply as the + // GpuMemoryHandle that later map jobs (211) name. + const u32 handle = m_next_memory_handle++; + m_memory[handle] = {start, last - start + 1}; + TipcReplyWords(ctx, {handle}); + } + + // Jobs that complete through a job-slot event (word Slot of the request), like 211. + template <size_t Slot> + void SubSlotJob(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "(STUBBED) nx2:gpu {} words {:08x} slot={}", ctx.GetCommand(), + fmt::join(std::span<const u32>{w + 2, 18}, " "), w[Slot]); + EventFor(w[Slot])->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 211: job submission (nn::gpu::sdk job tracking). Word 2 is a constant id; word 17 is + // the job slot whose event (gpu TIPC 12) the SDK then waits on. Nothing runs here, so + // complete it at once. + void SubSubmitJob(HLERequestContext& ctx) { + const auto* cmd_buf = ctx.CommandBuffer(); + const u32 slot = cmd_buf[17]; + const u64 a = u64{cmd_buf[3]} | (u64{cmd_buf[4]} << 32); + const u64 b = u64{cmd_buf[5]} | (u64{cmd_buf[6]} << 32); + const u64 c = u64{cmd_buf[8]} | (u64{cmd_buf[9]} << 32); + // Words 3-4: first GPU address, 5-6: last, 7: memory handle, 8-9: offset into it. + const u32 handle = cmd_buf[7]; + LOG_INFO(Service, "nx2:gpu 211 map [{:#x}, {:#x}] handle={} offset={:#x} slot={}", a, b, + handle, c, slot); + if (const auto it = m_memory.find(handle); it != m_memory.end() && b > a) { + // Also at the memory's own address: an earlier 213 may have unmapped the mapping 101 + // made there. + const u64 cpu = it->second.address + c; + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(a, cpu, b - a + 1); + } else { + LOG_WARNING(Service, "nx2:gpu 211 map of unknown handle {}", handle); + } + EventFor(slot)->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 212 (nn::gpu::sdk::NormalModifyMemoryKind4K, behind NvRmGpuMappingModify): word 2 the + // address space, 3-4 first GPU address, 5-6 last, 7 the wanted (compressible) kind, 8 the + // fallback kind, 9 a sync flag, 10 the job slot. The reply's word 3 is the kind applied. Kinds + // do not matter to the emulated GPU, so keep the mapping and report the wanted kind. + void SubModifyKind(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 first = u64{w[3]} | (u64{w[4]} << 32); + const u64 last = u64{w[5]} | (u64{w[6]} << 32); + const u32 kind = w[7] & 0xff; + LOG_DEBUG(Service, "nx2:gpu 212 modify [{:#x}, {:#x}] kind={:#x} fallback={:#x} slot={}", + first, last, kind, w[8] & 0xff, w[10]); + EventFor(w[10])->Signal(system.Kernel()); + TipcReplyWords(ctx, {kind}); + } + + // Sub 213: words 3-4 first GPU address, 5-6 last, job slot in word 8. + void SubUnmapJob(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 first = u64{w[3]} | (u64{w[4]} << 32); + const u64 last = u64{w[5]} | (u64{w[6]} << 32); + LOG_DEBUG(Service, "nx2:gpu 213 unmap [{:#x}, {:#x}] slot={}", first, last, w[8]); + if (last > first) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1); + } + SignalAfterGpu(w[8]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Signals a job slot's event once the GPU thread has run everything queued so far, as the + // console's graphics server completes unmap jobs behind the submitted work. + void SignalAfterGpu(u32 slot) { + auto* const event = EventFor(slot); + system.GPU().RunInOrder([this, event] { event->Signal(system.Kernel()); }); + } + + // Sub 102: word 2 is the memory handle 101 returned, word 3 the job slot. The game frees the + // memory (svc 0x80 clears GPU access) and reuses it as ordinary heap right after, so nothing + // on the GPU side may keep it: drop the mapping 101 made at its own address. + void SubDestroyMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 handle = w[2]; + if (const auto it = m_memory.find(handle); it != m_memory.end()) { + const auto [address, size] = it->second; + LOG_DEBUG(Service, "nx2:gpu 102 destroy handle={} [{:#x}, +{:#x}] slot={}", handle, + address, size, w[3]); + if (size != 0) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(address, size); + } + if (size == 0x2000) { + Tegra::Nx2RemoveSemaphoreRegion(address); + } + m_memory.erase(it); + } else { + LOG_WARNING(Service, "nx2:gpu 102 destroy of unknown handle {}", handle); + } + SignalAfterGpu(w[3]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 214: GPU-only memory [start, last] (words 3-6, job slot in word 11). The graphics + // server backs it itself: the client never maps anything there but renders into it (title 4 + // keeps its render targets in six 300 MB regions at 0x81_0000_0000). Back each region with + // kernel memory placed in the process's alias code region and map that for the GPU. + void SubCreateGpuOnlyRegion(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 start = u64{w[3]} | (u64{w[4]} << 32); + const u64 last = u64{w[5]} | (u64{w[6]} << 32); + const u32 slot = w[11]; + LOG_INFO(Service, "nx2:gpu 214 words {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}", + w[1], w[2], w[3], w[4], w[5], w[6], w[7], w[8], w[9], w[10], w[11], w[12]); + if (last > start && ((last + 1 - start) % Core::Memory::YUZU_PAGESIZE) == 0) { + const u64 size = last + 1 - start; + FreeReleasedRegions(); + if (const auto address = BackGpuOnlyRegion(start, size)) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(start, *address, size); + LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed at {:#x}", + start, last, *address); + } + } else { + LOG_WARNING(Service, "nx2:gpu 214 unexpected region [{:#x}, {:#x}]", start, last); + } + EventFor(slot)->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 215 (nn::gpu::sdk::DestroySparseRegion): words 3-6 the range, word 8 the job slot, like + // 213. Drop the GPU mapping, then the backing 214 added: its SMMU mapping (in GPU-thread + // order) and, once the GPU thread is past that, its pages. Kept allocated, title 1 ran the + // pool dry within minutes and later regions had no memory at all (GPU writes to them lost). + void SubDestroySparseRegion(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 first = u64{w[3]} | (u64{w[4]} << 32); + const u64 last = u64{w[5]} | (u64{w[6]} << 32); + LOG_INFO(Service, "nx2:gpu 215 destroy sparse region [{:#x}, {:#x}] words {:#x} {:#x} {:#x} {:#x}", + first, last, w[1], w[2], w[7], w[8]); + FreeReleasedRegions(); + if (last > first) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1); + const auto region = m_gpu_only_regions.find(first); + if (region != m_gpu_only_regions.end() && region->second.size == last - first + 1) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().ReleaseBacking( + region->second.cpu_address, region->second.size); + auto done = std::make_shared<std::atomic<bool>>(false); + system.GPU().RunInOrder([done] { done->store(true, std::memory_order_release); }); + m_released_regions.push_back({std::move(region->second), std::move(done)}); + m_gpu_only_regions.erase(region); + } + } + SignalAfterGpu(w[8]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Returns the pages of destroyed regions the GPU thread is done with to the kernel. + void FreeReleasedRegions() { + auto* const process = system.ApplicationProcess(); + std::erase_if(m_released_regions, [&](ReleasedRegion& released) { + if (!released.done->load(std::memory_order_acquire)) { + return false; + } + auto& region = released.region; + if (process != nullptr && + R_FAILED(process->GetPageTable().UnmapPageGroup( + region.cpu_address, *region.pages, Kernel::KMemoryState::IoMemory))) { + LOG_ERROR(Service, "nx2:gpu could not unmap GPU-only backing {:#x}+{:#x}", + region.cpu_address, region.size); + return true; // still mapped: keep its pages + } + region.pages->Close(system.Kernel()); + return true; + }); + } + + std::optional<u64> BackGpuOnlyRegion(u64 gpu_address, u64 size) { + using Core::Memory::YUZU_PAGESIZE; + auto& kernel_core = system.Kernel(); + auto* const process = system.ApplicationProcess(); + if (process == nullptr) { + return std::nullopt; + } + auto pg = std::make_unique<Kernel::KPageGroup>( + kernel_core, std::addressof(kernel_core.GetSystemSystemResource().GetBlockInfoManager())); + // The graphics server's own memory, not the application's: taken from the application + // pool, title 4's six regions left it 1.8 GB less than the kernel reports to it. Not + // cleared: the GPU writes these before reading them, and touching 300 MB would commit it + // on the host. + if (R_FAILED(kernel_core.MemoryManager().AllocateAndOpen( + pg.get(), size / YUZU_PAGESIZE, + Kernel::KMemoryManager::EncodeOption(Kernel::KMemoryManager::Pool::SystemNonSecure, + Kernel::KMemoryManager::Direction::FromBack)))) { + LOG_ERROR(Service, "nx2:gpu 214 could not allocate {:#x} bytes", size); + return std::nullopt; + } + auto& page_table = process->GetPageTable(); + const u64 region_start = GetInteger(page_table.GetAliasCodeRegionStart()); + const u64 region_size = page_table.GetAliasCodeRegionSize(); + constexpr u64 Alignment = 0x200000; // 2 MiB + for (u64 offset = 0; offset + size <= region_size; offset += Alignment) { + const Common::ProcessAddress address{region_start + offset}; + if (R_SUCCEEDED(page_table.MapPageGroup(address, *pg, Kernel::KMemoryState::IoMemory, + Kernel::KMemoryPermission::UserReadWrite))) { + m_gpu_only_regions[gpu_address] = {GetInteger(address), size, std::move(pg)}; + return GetInteger(address); + } + } + LOG_ERROR(Service, "nx2:gpu 214 no room for {:#x} bytes in the alias code region " + "[{:#x}, +{:#x})", size, region_start, region_size); + pg->Close(kernel_core); + return std::nullopt; + } + + struct GpuOnlyRegion { + u64 cpu_address; + u64 size; + std::unique_ptr<Kernel::KPageGroup> pages; + }; + struct ReleasedRegion { + GpuOnlyRegion region; + std::shared_ptr<std::atomic<bool>> done; ///< Set by the GPU thread past the region's last use + }; + std::map<u64, GpuOnlyRegion> m_gpu_only_regions; ///< By GPU address + std::vector<ReleasedRegion> m_released_regions; + + struct GpuMemory { + u64 address; + u64 size; + }; + + KernelHelpers::ServiceContext m_context; + std::array<Kernel::KEvent*, NumIds> m_events{}; + std::map<u32, GpuMemory> m_memory; + u32 m_next_memory_handle{1}; +}; + +// Native memory and images handed out by the Switch 2 graphics server (gsv 111/114/423/429/315), +// shared with the vi2 session so presents can find an image's memory. Command meanings follow +// the shared changes.patch (IGraphicsSupervisor), re-checked against title 4's SDK. +struct Nx2NativeObjects { + struct Memory { + u64 size{}; + u64 address{}; // CPU (== GPU) address once bound by 114/423 + }; + struct Image { + u32 memory{}; + std::array<u32, 16> preference{}; + u64 size{}; + u64 offset{}; // into its memory (images of a decoder pool share one) + bool window_pool{}; + bool decoder{}; // a video decoder output image (gsv 911) + }; + std::mutex mutex; + std::map<u32, Memory> memories; + std::map<u32, Image> images; + u32 next_memory{1}; + u32 next_image{1}; +}; + +// Block-linear layout of an InterModuleUserImagePreference (16 words): word 2 holds +// (width - 1) | (height - 1) << 16, word 4 the block height (log2 of GOB rows), word 5 the row +// pitch in 64-byte GOBs. Formula from changes.patch (native_image_layout.h); preferences outside +// it fall back to 4 bytes per pixel and 128-row blocks. +struct Nx2ImageLayout { + u32 width; + u32 height; + u64 pitch; + u32 block_height_log2; + u64 size; + u64 alignment; +}; + +inline Nx2ImageLayout Nx2CalculateImageLayout(const u32* preference) { + Nx2ImageLayout layout{}; + layout.width = (preference[2] & 0xFFFF) + 1; + layout.height = (preference[2] >> 16) + 1; + const u64 pitch_gobs = preference[5]; + const u32 block_height_log2 = preference[4]; + if (block_height_log2 <= 8 && pitch_gobs >= (u64{layout.width} * 4 + 63) / 64 && + pitch_gobs <= 4096) { + layout.pitch = pitch_gobs * 64; + layout.block_height_log2 = block_height_log2; + } else { + layout.pitch = Common::AlignUp(u64{layout.width} * 4, 64); + layout.block_height_log2 = 7; + } + const u64 block_rows = u64{1} << layout.block_height_log2; + layout.size = layout.pitch * Common::AlignUp(u64{layout.height}, block_rows); + layout.alignment = std::max<u64>(64 * block_rows, 0x1000); + return layout; +} + +// Video decoder output images (nn::codec image pools, gsv 991/911/961). The preference +// (MakeDecodeImagePreference) holds an output layout code at word 1 and (width - 1) | +// (height - 1) << 16 at word 2. Codes come in pairs, one per CodecMode: 1/4 NV12 (format 3, what +// the movie players use), 2/5 BGRA, 3/6 RGBA; the HDR codes (10-13) are written as RGBA. The +// server picks the layout and describes it to the client with gsv 961: here pitch-linear rows, +// NV12's half-height chroma plane below the luma plane, each plane padded to 128 rows. +struct Nx2DecoderImageLayout { + u32 code; + u32 width; + u32 height; + Tegra::Host1x::Nx2PictureFormat format; + u64 pitch; + u64 chroma_offset; + u64 size; +}; + +inline Nx2DecoderImageLayout Nx2CalculateDecoderImageLayout(const u32* preference) { + using Tegra::Host1x::Nx2PictureFormat; + Nx2DecoderImageLayout layout{}; + layout.code = preference[1]; + layout.width = (preference[2] & 0xFFFF) + 1; + layout.height = (preference[2] >> 16) + 1; + switch (layout.code) { + case 1: + case 4: + layout.format = Nx2PictureFormat::Nv12; + break; + case 2: + case 5: + layout.format = Nx2PictureFormat::Bgra8; + break; + default: + layout.format = Nx2PictureFormat::Rgba8; + break; + } + const bool nv12 = layout.format == Nx2PictureFormat::Nv12; + constexpr u64 BlockRows = 128; + layout.pitch = Common::AlignUp(u64{layout.width} * (nv12 ? 1 : 4), 64); + layout.chroma_offset = layout.pitch * Common::AlignUp(u64{layout.height}, BlockRows); + layout.size = layout.chroma_offset + + (nv12 ? layout.pitch * Common::AlignUp((u64{layout.height} + 1) / 2, BlockRows) : 0); + return layout; +} + +// Placeholder for the Switch 2 display session (nn::vi2::sdk), handed out by gsv 305. Reply +// shapes read from the SDK: TIPC 1 -> u32 < 0x400, TIPC 3 -> u32 < 8, TIPC 11/31(u32) and +// 32(handle, u32) -> one handle at word 3 then the result. Handles returned here are events (a +// guess); everything else is auto-stubbed through the nx2: prefix. +class INx2Vi2Session final : public ServiceFramework<INx2Vi2Session> { +public: + explicit INx2Vi2Session(Core::System& system_, std::shared_ptr<Nx2NativeObjects> objects_) + : ServiceFramework{system_, "nx2:vi2"}, m_context{system_, "nx2:vi2"}, + m_objects{std::move(objects_)} { + RegisterHandlersTipc({ + {1, &INx2Vi2Session::ReturnOne, "Unknown1"}, + {3, &INx2Vi2Session::ReturnOne, "GetDisplayCount (guessed)"}, + {5, &INx2Vi2Session::GetTimestampRecorderCount, "GetTimestampRecorderCount (guessed)"}, + {11, &INx2Vi2Session::ReturnEvent, "Unknown11"}, + {31, &INx2Vi2Session::GetVsyncEvent, "GetVsyncEvent (guessed)"}, + {32, &INx2Vi2Session::GetVsyncEvent, "GetDisplayVsyncEvent (guessed)"}, + // Title 1 calls these ~75 times a second with the display name "default"; the + // auto stub's empty success reply works, but logged two lines per call (a 50 MB log + // in an 18 minute session). + {104, &INx2Vi2Session::SucceedQuietly, "Unknown104 (display \"default\")"}, + {105, &INx2Vi2Session::SucceedQuietly, "Unknown105 (display \"default\")"}, + {228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"}, + {229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"}, + }); + } + + ~INx2Vi2Session() override { + // The vsync callback may be running on the HostTiming thread: empty its event list under + // its lock first (it holds the state, not this session), then close the events. + { + std::scoped_lock lk{m_vsync->mutex}; + m_vsync->events.clear(); + } + if (m_vsync_timer) { + system.CoreTiming().UnscheduleEvent(m_vsync_timer); + } + for (auto* event : m_events) { + m_context.CloseEvent(event); + } + } + +private: + // 229(u64 client token, u32 slot, u32 image, ...fences): result, u64 mask of buffer slots the + // client may acquire next, four zero words. Nothing is shown yet; every slot is handed back + // at once (changes.patch's default mask is 0x7fff). + void PresentClientBuffer(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + ++m_presents; + Core::g_nx2_present_count.fetch_add(1, std::memory_order_relaxed); + if (m_presents <= 4 || (m_presents % 600) == 0) { + LOG_INFO(Service, "nx2:vi2 229 present #{} token={:#x} slot={} image={}", m_presents, + u64{w[2]} | (u64{w[3]} << 32), w[4], w[5]); + } + // Wait for the previous frame's fence instead of this one, so the game prepares frame N+1 + // while the GPU thread runs frame N (the console keeps several frames in flight). Waiting + // for this frame serialized the two: 8-56 ms of every 34-105 ms frame on title 1. + // EDEN_NX2_FRAMES_AHEAD=0 restores the wait for the current frame. + static const bool frames_ahead = [] { + const char* value = std::getenv("EDEN_NX2_FRAMES_AHEAD"); + return value == nullptr || value[0] != '0'; + }(); + const auto entered = std::chrono::steady_clock::now(); + if (frames_ahead) { + LOG_DEBUG(Service, "nx2:vi2 present #{} fence ({}, {:#x}), waiting for ({}, {:#x}); now {:#x}", + m_presents, ~w[22], w[24], m_previous_fence_id, m_previous_fence_value, + ~w[22] < 192 ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(~w[22]) : 0U); + WaitForFrame(m_previous_fence_id, m_previous_fence_value); + m_previous_fence_id = ~w[22]; + m_previous_fence_value = w[24]; + } else { + WaitForFrame(~w[22], w[24]); + } + const auto waited = std::chrono::steady_clock::now(); + Composite(w[5], ~w[22], w[24]); + const auto composited = std::chrono::steady_clock::now(); + PaceToVblank(); + const auto paced = std::chrono::steady_clock::now(); + // Where each frame's time goes on the present thread, averaged per 600 presents: the + // game is blocked here while Eden's GPU thread catches up (fence) and composites. + { + using us = std::chrono::microseconds; + // long long like the stats: microseconds::rep is long on Linux (std::max needs one type). + const auto to_us = [](auto d) -> long long { + return std::chrono::duration_cast<us>(d).count(); + }; + m_stats.fence_us += to_us(waited - entered); + m_stats.composite_us += to_us(composited - waited); + m_stats.pace_us += to_us(paced - composited); + m_stats.max_fence_us = (std::max)(m_stats.max_fence_us, to_us(waited - entered)); + if (m_stats.last_present != std::chrono::steady_clock::time_point{}) { + const long long frame = to_us(entered - m_stats.last_present); + m_stats.frame_us += frame; + m_stats.max_frame_us = (std::max)(m_stats.max_frame_us, frame); + m_stats.hitches += frame > 100'000 ? 1 : 0; + } + m_stats.last_present = entered; + if (++m_stats.presents == 600) { + LOG_INFO(Service, + "nx2:vi2 last 600 presents: frame {} us (max {}, {} over 100 ms), " + "blocked on the GPU fence {} us (max {}), composite {} us, vblank " + "pacing {} us", + m_stats.frame_us / 599, m_stats.max_frame_us, m_stats.hitches, + m_stats.fence_us / 600, m_stats.max_fence_us, m_stats.composite_us / 600, + m_stats.pace_us / 600); + m_stats = {.last_present = entered}; + } + } + constexpr u64 AvailableSlots = 0x7fff; + TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots), + static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0}); + } + + // The console's compositor frees a buffer only once its frame's GPU fence has passed, which + // is what keeps a game from running ahead of the GPU. This reply always reports every buffer + // free, so pace the game here instead: wait (bounded, in case the GPU waits on the CPU) until + // Eden's GPU thread has processed the frame. Otherwise the game laps the GPU thread and + // rewrites command memory it hasn't read yet. + u32 m_wait_timeouts{}; + struct PresentStats { + long long frame_us{}, fence_us{}, composite_us{}, pace_us{}, max_fence_us{}; + long long max_frame_us{}, hitches{}; + u32 presents{}; + std::chrono::steady_clock::time_point last_present{}; + } m_stats{}; + u32 m_previous_fence_id{~0U}; // none yet (WaitForFrame ignores ids >= 192) + u32 m_previous_fence_value{}; + + // The console flips one frame per 60 Hz vblank and keeps the rest queued, so a game can't + // present faster than that. Every slot is reported free here: hold each present until the + // next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it). + std::chrono::steady_clock::time_point m_next_vblank{}; + + void PaceToVblank() { + if (!Settings::values.use_speed_limit.GetValue() || Settings::SpeedLimit() == 0) { + return; + } + const auto interval = std::chrono::nanoseconds{100'000'000'000LL / 60 / Settings::SpeedLimit()}; + const auto now = std::chrono::steady_clock::now(); + if (now > m_next_vblank + interval) { + // Running slower than 60 fps: show at once and don't bank the lost time. + m_next_vblank = now; + } else if (now < m_next_vblank) { + std::this_thread::sleep_until(m_next_vblank); + } + m_next_vblank += interval; + } + + void WaitForFrame(u32 fence_id, u32 fence_value) { + if (fence_id >= 192) { + return; + } + auto& syncpoints = system.Host1x().GetSyncpointManager(); + const auto start = std::chrono::steady_clock::now(); + // Long enough to cover shader/pipeline compile stalls: a game that gets more than a few + // command buffers ahead misses the equal-value gates its other channels acquire. + const auto deadline = start + std::chrono::milliseconds{4000}; + if (!syncpoints.IsReadyGuest(fence_id, fence_value)) { + // Woken by the GPU thread as it reaches the fence. Polling with 200 us sleeps slept + // about 1 ms per check on Windows, added to every frame. An action left registered + // after a timeout fires later on a state nobody waits on. + struct Wake { + std::mutex mutex; + std::condition_variable cv; + bool done{}; + }; + auto wake = std::make_shared<Wake>(); + syncpoints.RegisterGuestAction(fence_id, fence_value, [wake] { + { + std::scoped_lock lk{wake->mutex}; + wake->done = true; + } + wake->cv.notify_all(); + }); + std::unique_lock lk{wake->mutex}; + while (!wake->done && system.IsPoweredOn() && + std::chrono::steady_clock::now() < deadline) { + wake->cv.wait_for(lk, std::chrono::milliseconds{50}); + } + } + if (!syncpoints.IsReadyGuest(fence_id, fence_value) && m_wait_timeouts++ < 64) { + LOG_WARNING(Service, "nx2:vi2 present #{} fence ({}, {}) not reached in 4 s (now {})", + m_presents, fence_id, fence_value, syncpoints.GetGuestSyncpointValue(fence_id)); + } + if (m_presents <= 8 || (m_presents % 600) == 0) { + LOG_INFO(Service, "nx2:vi2 present #{} fence ({}, {}) now {}, waited {} us", + m_presents, fence_id, fence_value, syncpoints.GetGuestSyncpointValue(fence_id), + std::chrono::duration_cast<std::chrono::microseconds>( + std::chrono::steady_clock::now() - start) + .count()); + } + } + + // Shows a presented native image: word 5 is the image id, ~word 22 / word 24 the GPU fence + // (syncpoint, value) to wait for (layout from changes.patch). The image is a block-linear + // RGBA8 surface in the console's native GOB layout at GPU address == CPU address. + void Composite(u32 image_id, u32 fence_id, u32 fence_value) { + u64 address{}; + Nx2ImageLayout layout{}; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(image_id); + if (image == m_objects->images.end()) { + return; + } + const auto memory = m_objects->memories.find(image->second.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return; + } + address = memory->second.address; + layout = Nx2CalculateImageLayout(image->second.preference.data()); + } + const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); + if (!device) { + LOG_WARNING(Service, "nx2:vi2 present: image {} at {:#x} has no device mapping", + image_id, address); + return; + } + std::vector<Tegra::FramebufferConfig> layers{Tegra::FramebufferConfig{ + .address = *device, + .offset = 0, + .width = layout.width, + .height = layout.height, + .stride = static_cast<u32>(layout.pitch / 4), + .pixel_format = android::PixelFormat::Rgba8888, + .transform_flags = android::BufferTransformFlags::Unset, + .crop_rect = {0, 0, static_cast<int>(layout.width), static_cast<int>(layout.height)}, + .blending = Tegra::BlendMode::Opaque, + .native_tiling = true, + }}; + std::vector<Nvidia::NvFence> fences; + if (fence_id < 192) { + fences.push_back({static_cast<s32>(fence_id), fence_value}); + } + system.GPU().RequestComposite(std::move(layers), std::move(fences)); + CaptureFrame(); + DumpRanges(); + system.SpeedLimiter().DoSpeedLimiting(system.CoreTiming().GetGlobalTimeUs()); + system.GetPerfStats().EndSystemFrame(); + system.GetPerfStats().BeginSystemFrame(); + } + + // EDEN_NX2_FRAME_CAPTURE=<dir>: every 120th present, save what the renderer shows as + // nx2-frame-<n>.tga in <dir> (lets hidden test runs check the output; idea from changes.patch). + void CaptureFrame() { + static const char* const directory = std::getenv("EDEN_NX2_FRAME_CAPTURE"); + if (directory == nullptr || (m_presents % 120) != 1 || m_capture_pending->exchange(true)) { + return; + } + const auto layout = Layout::DefaultFrameLayout(1280, 720); + auto pixels = std::make_shared<std::vector<u8>>(layout.width * layout.height * 4); + const std::string path = + fmt::format("{}/nx2-frame-{:05}.tga", directory, m_presents); + auto pending = m_capture_pending; // the screenshot callback can outlive the session + system.GPU().Renderer().RequestScreenshot( + pixels->data(), + [pixels, layout, path, pending](bool invert_y) { + std::array<u8, 18> header{}; + header[2] = 2; // uncompressed true colour + header[12] = static_cast<u8>(layout.width); + header[13] = static_cast<u8>(layout.width >> 8); + header[14] = static_cast<u8>(layout.height); + header[15] = static_cast<u8>(layout.height >> 8); + header[16] = 32; + // Descriptor bit 5: rows top to bottom. invert_y means the data is bottom-up. + header[17] = invert_y ? 0x08 : 0x28; + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fwrite(header.data(), 1, header.size(), file); + std::fwrite(pixels->data(), 1, pixels->size(), file); + std::fclose(file); + } + pending->store(false); + }, + layout); + } + + // EDEN_NX2_DUMP=<present>:<hex address>:<hex size>[,<hex address>:<hex size>...]: at that + // present, flush those GPU ranges back from the host GPU and save their bytes as + // nx2-dump-<address>.bin in the EDEN_NX2_FRAME_CAPTURE directory (to inspect what a pass + // rendered, e.g. a 3D volume, offline). + void DumpRanges() { + static const std::string spec = [] { + const char* text = std::getenv("EDEN_NX2_DUMP"); + return text ? std::string{text} : std::string{}; + }(); + if (spec.empty() || m_presents != std::strtoull(spec.c_str(), nullptr, 10)) { + return; + } + const char* const directory = std::getenv("EDEN_NX2_FRAME_CAPTURE"); + size_t position = spec.find(':'); + while (position != std::string::npos) { + char* end{}; + const u64 address = std::strtoull(spec.c_str() + position + 1, &end, 16); + const u64 size = *end == ':' ? std::strtoull(end + 1, &end, 16) : 0; + position = spec.find(',', position + 1); // the next "address:size" + const std::string path = fmt::format("{}/nx2-dump-{:x}.bin", directory ? directory : ".", address); + // On the GPU thread: waiting for it here can deadlock with a GPU wait on this present. + static_cast<void>(system.GPU().RunOnGpuThread([&system = system, address, size, path] { + std::vector<u8> bytes(size); + for (u64 current = address; current < address + size;) { + const u64 chunk = std::min((current | 0xfff) + 1, address + size) - current; + if (const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(current)) { + system.GPU().Renderer().ReadRasterizer()->FlushRegion(*device, chunk); + system.Host1x().MemoryManager().ReadBlock(*device, bytes.data() + (current - address), chunk); + } + current += chunk; + } + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fwrite(bytes.data(), 1, bytes.size(), file); + std::fclose(file); + } + LOG_INFO(Service, "nx2: dumped {:#x} bytes at {:#x} to {}", size, address, path); + })); + } + } + + // 228(u64 token, u32 slot): return a buffer without presenting it. + void CancelClientBuffer(HLERequestContext& ctx) { + constexpr u64 AvailableSlots = 0x7fff; + TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots), + static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0}); + } + + void ReturnOne(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {}, returning 1", ctx.GetCommand()); + TipcReplyWords(ctx, {1}); + } + + // The same reply as the auto stub (success, no data), without its log lines. + void SucceedQuietly(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Must be < 8; nn::vi::detail::LayerHistory creates 7 timestamp recorders per layer. + void GetTimestampRecorderCount(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC 5, returning 7"); + TipcReplyWords(ctx, {7}); + } + + void ReturnEvent(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {} [2]={:#x} [3]={:#x}, returning an event", + ctx.GetCommand(), ctx.CommandBuffer()[2], ctx.CommandBuffer()[3]); + auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Event{}", m_events.size())); + m_events.push_back(event); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + // 31(u32 kind) and 32(u32 display): events signalled on every vblank. Title 1's main thread + // waits on the 32 event until it has fired twice before it submits any GPU work, and hung + // there while it never fired. + void GetVsyncEvent(HLERequestContext& ctx) { + LOG_INFO(Service, "nx2:vi2 {} vsync event, [2]={:#x}", ctx.GetCommand(), ctx.CommandBuffer()[2]); + auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Vsync{}", m_events.size())); + m_events.push_back(event); + { + std::scoped_lock lk{m_vsync->mutex}; + m_vsync->events.push_back(event); + } + if (!m_vsync_timer) { + m_vsync_timer = Core::Timing::CreateEvent( + "nx2:vi2 vsync", + [vsync_state = m_vsync, &kernel = system.Kernel()]( + s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + std::scoped_lock lk{vsync_state->mutex}; + for (auto* vsync : vsync_state->events) { + vsync->Signal(kernel); + } + return std::nullopt; + }); + // EDEN_NX2_VSYNC_HZ=<rate> (experiment): the game starts frames on this event, so a + // frame that misses a 60 Hz tick idles until the next one. + static const std::chrono::nanoseconds VblankInterval{[] { + const char* text = std::getenv("EDEN_NX2_VSYNC_HZ"); + const long long hz = text ? (std::max)(std::strtoll(text, nullptr, 10), 1LL) : 60LL; + return 1'000'000'000LL / hz; + }()}; + system.CoreTiming().ScheduleLoopingEvent(VblankInterval, VblankInterval, m_vsync_timer); + } + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + KernelHelpers::ServiceContext m_context; + std::shared_ptr<Nx2NativeObjects> m_objects; + std::vector<Kernel::KEvent*> m_events; + // Shared with the vsync timer callback, which can outlive the session by one call. + struct VsyncState { + std::mutex mutex; + std::vector<Kernel::KEvent*> events; + }; + std::shared_ptr<VsyncState> m_vsync = std::make_shared<VsyncState>(); + std::shared_ptr<Core::Timing::EventType> m_vsync_timer; + u64 m_presents{}; + std::shared_ptr<std::atomic<bool>> m_capture_pending = std::make_shared<std::atomic<bool>>(false); +}; + +// True the first time a Switch 2 video command is seen, so each one is logged once in full and new +// or unexpected commands stand out. +inline bool Nx2FirstVideoCommand(std::string_view service, u32 command) { + static std::mutex mutex; + static std::set<std::pair<std::string, u32>> seen; + std::scoped_lock lk{mutex}; + return seen.emplace(std::string{service}, command).second; +} + +// The Switch 2 video decoder (nn::codec::video::detail::ipc). gsv 902/906/1002 hand out a session +// whose TIPC 1 (CreateVideoDecoderService: handle at word 3, result at word 4) creates a decoder +// service, another session of this class. Layouts read from the SDK's nn::codec VP9 decoder: +// - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize. +// - 101(A: codec name, "video/vp9\0"; u32 width, u32 height, u32 rate, u32): configure. +// - 3(): the submission-ready event (copy handle at word 3, result at word 4). +// - 6 / 207 / 8: start / stop / flush; 115: finalize. +// - 311(u32 image id): register an output image (gsv 911 creates it); reply its NativeSyncHandle, +// signalled once a frame has been decoded into the image. The client acquires output images in +// registration/release order from a ring of its own (no IPC), so frames fill them in that order. +// - 314(u32 image id, u32, NativeSyncHandle): give an output image back; the image may be +// rewritten once that handle (the client's reads) has passed. Reply the image's next handle. +// - 212(u32 image id): unregister it. +// - 109(A: one compressed frame, u64 pts in us, u8 end of stream): submit input; reply a u64. +// - 213(): the next finished frame (16 words): u32 image id, u32 width, u32 height, u32 width, +// u32 height, u32 colour space, TimeSpan pts (ns), u8 end of stream. +// A NativeSyncHandle is {u32 SyncCounterId (~syncpoint), u32 mode, u64 value}; clients wait on it +// with nx2:syncpt 103/201. Each decoder service owns a syncpoint that counts its finished frames. +// FFmpeg decodes on a thread of the session's own (a 1440p VP9 frame takes milliseconds, and the +// service thread also serves presents and syncpoint waits) into the layout gsv 961 describes. +class INx2VideoSession final : public ServiceFramework<INx2VideoSession> { +public: + explicit INx2VideoSession(Core::System& system_, std::shared_ptr<Nx2NativeObjects> objects_, + const char* name) + : ServiceFramework{system_, name}, m_context{system_, name}, + m_objects{std::move(objects_)} { + RegisterHandlersTipc({ + {1, &INx2VideoSession::CreateDecoderService, "CreateVideoDecoderService"}, + {3, &INx2VideoSession::GetEvent, "GetEvent"}, + {6, &INx2VideoSession::Succeed, "Start"}, + {8, &INx2VideoSession::Flush, "Flush"}, + {101, &INx2VideoSession::Configure, "Configure"}, + {109, &INx2VideoSession::SubmitInput, "SubmitInputBuffer"}, + {115, &INx2VideoSession::Succeed, "Finalize"}, + {200, &INx2VideoSession::Succeed, "Initialize"}, + {207, &INx2VideoSession::Flush, "Stop"}, + {212, &INx2VideoSession::UnregisterBuffer, "UnregisterBuffer"}, + {213, &INx2VideoSession::GetOutputInfo, "GetOutputInfo"}, + {311, &INx2VideoSession::RegisterBuffer, "RegisterBuffer"}, + {314, &INx2VideoSession::ReleaseBuffer, "ReleaseBuffer"}, + }); + } + + ~INx2VideoSession() override { + if (m_worker.joinable()) { + { + std::scoped_lock lk{m_mutex}; + m_quit = true; + } + m_cv.notify_all(); + m_worker.join(); + } + // The ready timer runs on the HostTiming thread: clear its event under its lock first. + { + std::scoped_lock lk{m_ready->mutex}; + m_ready->event = nullptr; + } + if (m_ready_timer) { + system.CoreTiming().UnscheduleEvent(m_ready_timer); + } + if (m_event != nullptr) { + m_context.CloseEvent(m_event); + } + if (m_syncpoint != NoSyncpoint) { + NvdrvSyncpoints().FreeSyncpoint(m_syncpoint); + } + } + +private: + struct Input { + std::vector<u8> data; + u64 pts; // us + bool end; + }; + struct Decoded { + std::optional<Tegra::Host1x::Nx2Picture> picture; // none: an empty end-of-stream frame + u64 pts; + bool end; + }; + struct Output { + u32 image; + u32 width; + u32 height; + u64 pts; + bool end; + }; + struct Fence { + u32 syncpoint; + u32 value; + }; + + static constexpr u32 NoSyncpoint = ~0U; + static constexpr size_t MaxQueuedInputs = 8; + static constexpr size_t MaxDecodedAhead = 2; // decoded frames held while no image is free + static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager + + void Log(HLERequestContext& ctx, std::string_view what) { + const auto* w = ctx.CommandBuffer(); + if (Nx2FirstVideoCommand(GetServiceName(), ctx.GetCommand())) { + LOG_INFO(Service, "{} TIPC {} {} (first use): words {:08x}", GetServiceName(), + ctx.GetCommand(), what, fmt::join(std::span<const u32>{w, 12}, " ")); + } else { + LOG_DEBUG(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}", + GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5]); + } + } + + Nvidia::NvCore::SyncpointManager& NvdrvSyncpoints() { + return system.ServiceManager() + .GetService<Nvidia::NVDRV>("nvdrv", true) + ->GetModule() + ->GetContainer() + .GetSyncpointManager(); + } + + void CreateDecoderService(HLERequestContext& ctx) { + Log(ctx, "create decoder service"); + ReturnRawSession(system, ctx, + std::make_shared<INx2VideoSession>(system, m_objects, "nx2:vdecsvc")); + } + + void Succeed(HLERequestContext& ctx) { + Log(ctx, ""); + TipcReplyWords(ctx, {}); + } + + void Configure(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + std::string codec; + if (!ctx.BufferDescriptorA().empty()) { + const auto name = ctx.ReadBufferA(0); + codec.assign(reinterpret_cast<const char*>(name.data()), + strnlen(reinterpret_cast<const char*>(name.data()), name.size())); + } + Log(ctx, "configure"); + LOG_INFO(Service, "{} configure codec \"{}\" {}x{} rate {}", GetServiceName(), codec, w[5], + w[6], w[7]); + { + std::scoped_lock lk{m_mutex}; + m_codec = std::move(codec); + m_width = w[5]; + m_height = w[6]; + } + if (!m_worker.joinable()) { + m_worker = std::thread{[this] { DecodeLoop(); }}; + } + TipcReplyWords(ctx, {}); + } + + void GetEvent(HLERequestContext& ctx) { + Log(ctx, "event"); + if (m_event == nullptr) { + m_event = m_context.CreateEvent(fmt::format("{}:Ready", GetServiceName())); + // The client attaches it auto-clearing, so one signal is consumed by the first wait: + // while the input queue has room, re-signal every 4 ms; a spurious wake only makes it + // check again. + { + std::scoped_lock lk{m_ready->mutex}; + m_ready->event = m_event; + } + m_ready_timer = Core::Timing::CreateEvent( + "nx2:vdec ready", + [ready = m_ready, &kernel = system.Kernel()]( + s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + std::scoped_lock lk{ready->mutex}; + if (ready->event != nullptr && ready->room.load(std::memory_order_relaxed)) { + ready->event->Signal(kernel); + } + return std::nullopt; + }); + constexpr std::chrono::nanoseconds ReadyInterval{std::chrono::milliseconds{4}}; + system.CoreTiming().ScheduleLoopingEvent(ReadyInterval, ReadyInterval, m_ready_timer); + } + m_event->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, m_event->GetReadableEvent()); + } + + // Queues an output image for the next decoded frame; returns the NativeSyncHandle words that + // signal once that frame is in it. + std::array<u32, 4> HandOut(u32 image, std::optional<Fence> release_fence) { + if (m_syncpoint == NoSyncpoint) { + m_syncpoint = NvdrvSyncpoints().AllocateSyncpoint(false); + m_sync_base = system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(m_syncpoint); + LOG_INFO(Service, "{} counts decoded frames on syncpoint {} from {}", + GetServiceName(), m_syncpoint, m_sync_base); + } + u32 value; + { + std::scoped_lock lk{m_mutex}; + m_free.push_back(image); + if (release_fence) { + m_release_fences[image] = *release_fence; + } + value = m_sync_base + ++m_handed; + } + m_cv.notify_one(); + return {~m_syncpoint, 0, value, 0}; + } + + void RegisterBuffer(HLERequestContext& ctx) { + Log(ctx, "register output"); + const auto handle = HandOut(ctx.CommandBuffer()[2], std::nullopt); + TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3]}); + } + + void ReleaseBuffer(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + Log(ctx, "release output"); + const auto handle = HandOut(w[2], Fence{~w[4], w[6]}); + TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3]}); + } + + void UnregisterBuffer(HLERequestContext& ctx) { + Log(ctx, "unregister output"); + const u32 image = ctx.CommandBuffer()[2]; + { + std::scoped_lock lk{m_mutex}; + std::erase(m_free, image); + m_release_fences.erase(image); + } + TipcReplyWords(ctx, {}); + } + + void SubmitInput(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + // Words 2-4 are the A descriptor of the compressed frame. + const u64 pts = u64{w[5]} | (u64{w[6]} << 32); + const bool end = (w[7] & 0xFF) != 0; + Log(ctx, "submit"); + Input input{{}, pts, end}; + if (!ctx.BufferDescriptorA().empty()) { + const auto data = ctx.ReadBufferA(0); + input.data.assign(data.begin(), data.end()); + } + { + std::scoped_lock lk{m_mutex}; + m_inputs.push_back(std::move(input)); + m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed); + } + m_cv.notify_one(); + const u64 id = ++m_submitted; + TipcReplyWords(ctx, {static_cast<u32>(id), static_cast<u32>(id >> 32)}); + } + + // Flush (8) and Stop (207): drop queued input and frames not yet in an image. Images already + // handed out keep their place, so the sync values stay in step with the client's ring. + void Flush(HLERequestContext& ctx) { + Log(ctx, ctx.GetCommand() == 8 ? "flush" : "stop"); + { + std::scoped_lock lk{m_mutex}; + m_inputs.clear(); + m_flush = true; + m_ready->room.store(true, std::memory_order_relaxed); + } + m_cv.notify_one(); + TipcReplyWords(ctx, {}); + } + + void GetOutputInfo(HLERequestContext& ctx) { + Log(ctx, "output info"); + std::optional<Output> output; + { + std::scoped_lock lk{m_mutex}; + if (!m_outputs.empty()) { + output = m_outputs.front(); + m_outputs.pop_front(); + } + } + if (!output) { + // Nothing finished: the code the client's own Acquire uses for an empty ring. + IPC::ResponseBuilder rb{ctx, 2}; + rb.PushRaw<u32>(0x1c286f); + return; + } + const u64 pts_ns = output->pts * 1000; + TipcReplyWords(ctx, {output->image, output->width, output->height, output->width, + output->height, 0, static_cast<u32>(pts_ns), + static_cast<u32>(pts_ns >> 32), output->end ? 1U : 0U, 0, 0, 0, 0, + 0, 0, 0}); + } + + // The decode thread: turns queued input into decoded frames and decoded frames into output + // images, in order. + void DecodeLoop() { + Common::SetCurrentThreadName("nx2:vdec"); + std::unique_ptr<Tegra::Host1x::Nx2VideoDecoder> decoder; + std::vector<Tegra::Host1x::Nx2Picture> pictures; + std::vector<u8> bytes; + std::unique_lock lk{m_mutex}; + while (true) { + m_cv.wait(lk, [&] { + return m_quit || m_flush || (!m_decoded.empty() && !m_free.empty()) || + (!m_inputs.empty() && m_decoded.size() < MaxDecodedAhead); + }); + if (m_quit) { + return; + } + if (m_flush) { + m_flush = false; + m_decoded.clear(); + if (decoder) { + lk.unlock(); + decoder->Reset(); + lk.lock(); + } + continue; + } + if (!m_decoded.empty() && !m_free.empty()) { + Decoded frame = std::move(m_decoded.front()); + m_decoded.pop_front(); + const u32 image = m_free.front(); + m_free.pop_front(); + std::optional<Fence> fence; + if (const auto it = m_release_fences.find(image); it != m_release_fences.end()) { + fence = it->second; + m_release_fences.erase(it); + } + lk.unlock(); + if (fence) { + WaitForRelease(*fence); + } + const auto [width, height] = WriteImage(decoder.get(), image, frame, bytes); + lk.lock(); + // Even an image written during a flush counts: the client waits for its value. + m_outputs.push_back({image, width, height, frame.pts, frame.end}); + system.Host1x().GetSyncpointManager().IncrementGuest(m_syncpoint); + ++m_frames_out; + if (m_frames_out == 1 || (m_frames_out % 600) == 0 || frame.end) { + LOG_INFO(Service, "{} frame {} ({}x{}, pts {} us{}) in image {}", + GetServiceName(), m_frames_out, width, height, frame.pts, + frame.end ? ", end of stream" : "", image); + } + continue; + } + Input input = std::move(m_inputs.front()); + m_inputs.pop_front(); + m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed); + const std::string codec = m_codec; + lk.unlock(); + if (!decoder || decoder->Name() != codec) { + decoder = std::make_unique<Tegra::Host1x::Nx2VideoDecoder>(codec); + } + pictures.clear(); + decoder->Decode(input.data, input.pts, pictures); + if (input.end) { + decoder->Drain(pictures); + } + lk.lock(); + if (m_flush) { + continue; + } + for (auto& picture : pictures) { + const u64 pts = picture.pts; + m_decoded.push_back({std::move(picture), pts, false}); + } + if (input.end) { + // The client stops at the frame marked end of stream; give it one even when the + // last input held no picture. + if (m_decoded.empty()) { + m_decoded.push_back({std::nullopt, input.pts, true}); + } else { + m_decoded.back().end = true; + } + } + } + } + + // Waits (bounded) until the client's reads of an image it gave back have passed. + void WaitForRelease(const Fence& fence) { + if (fence.syncpoint >= HostSyncpoints) { + return; + } + auto& syncpoints = system.Host1x().GetSyncpointManager(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds{100}; + while (!syncpoints.IsReadyGuest(fence.syncpoint, fence.value) && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + } + + // Writes a decoded frame into an output image; returns the size the frame reports. + std::pair<u32, u32> WriteImage(Tegra::Host1x::Nx2VideoDecoder* decoder, u32 image_id, + const Decoded& frame, std::vector<u8>& bytes) { + u64 address{}; + Nx2DecoderImageLayout layout{}; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(image_id); + if (image == m_objects->images.end()) { + return {m_width, m_height}; + } + layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); + const auto memory = m_objects->memories.find(image->second.memory); + if (memory != m_objects->memories.end() && memory->second.address != 0) { + address = memory->second.address + image->second.offset; + } + } + if (!frame.picture || decoder == nullptr || address == 0 || + system.ApplicationProcess() == nullptr) { + return {std::min(m_width, layout.width), std::min(m_height, layout.height)}; + } + // A frame larger than the image is scaled down to it. + const u32 width = std::min(frame.picture->width, layout.width); + const u32 height = std::min(frame.picture->height, layout.height); + bytes.resize(layout.size); + decoder->Render(*frame.picture, layout.format, width, height, layout.pitch, + layout.chroma_offset, bytes); + DumpFrame(bytes, layout, width, height); + auto& memory = system.ApplicationMemory(); + memory.WriteBlock(address, bytes.data(), layout.pitch * height); + if (layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12) { + memory.WriteBlock(address + layout.chroma_offset, bytes.data() + layout.chroma_offset, + layout.pitch * ((height + 1) / 2)); + } + return {width, height}; + } + + // EDEN_NX2_VDEC_DUMP=<dir>: save the luma of every 120th decoded frame, a quarter of its size, + // as nx2-vdec-<frame>.pgm (to check what the decoder wrote, apart from how the game shows it). + void DumpFrame(const std::vector<u8>& bytes, const Nx2DecoderImageLayout& layout, u32 width, + u32 height) { + static const char* const directory = std::getenv("EDEN_NX2_VDEC_DUMP"); + if (directory == nullptr || (m_frames_out % 120) != 0) { + return; + } + const u32 bpp = layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12 ? 1 : 4; + const u32 w = width / 4; + const u32 h = height / 4; + const std::string path = fmt::format("{}/nx2-vdec-{:05}.pgm", directory, m_frames_out + 1); + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fprintf(file, "P5 %u %u 255\n", w, h); + for (u32 y = 0; y < h; ++y) { + for (u32 x = 0; x < w; ++x) { + std::fputc(bytes[y * 4 * layout.pitch + x * 4 * bpp], file); + } + } + std::fclose(file); + } + } + + struct ReadyState { + std::mutex mutex; + Kernel::KEvent* event{}; + std::atomic<bool> room{true}; ///< The input queue takes more frames + }; + + KernelHelpers::ServiceContext m_context; + std::shared_ptr<Nx2NativeObjects> m_objects; + Kernel::KEvent* m_event{}; + std::shared_ptr<ReadyState> m_ready = std::make_shared<ReadyState>(); + std::shared_ptr<Core::Timing::EventType> m_ready_timer; + u64 m_submitted{}; + u32 m_syncpoint{NoSyncpoint}; ///< Counts finished frames (allocated at the first register) + u32 m_sync_base{}; ///< Its value before the first frame + + // Shared with the decode thread. + std::mutex m_mutex; + std::condition_variable m_cv; + std::string m_codec; + u32 m_width{}; + u32 m_height{}; + u32 m_handed{}; ///< Output images handed out (registered or released) so far + u64 m_frames_out{}; + std::deque<Input> m_inputs; + std::deque<Decoded> m_decoded; ///< Waiting for a free output image + std::deque<u32> m_free; ///< Output images, in the order the client acquires them + std::map<u32, Fence> m_release_fences; + std::deque<Output> m_outputs; ///< Finished, for 213 + bool m_flush{}; + bool m_quit{}; + std::thread m_worker; +}; + +// Placeholder for the Switch 2 graphics-system server that nn::gsv::BootWithRemoteMemory talks +// to over TIPC. Nothing behind it is emulated: commands succeed and are logged, so boot can show +// what comes next. Commands 202 and 1402 hand out the syncpt and nvsrv sessions the SDK uses. +class INx2GraphicsSystemSession final : public ServiceFramework<INx2GraphicsSystemSession> { +public: + explicit INx2GraphicsSystemSession(Core::System& system_) + : ServiceFramework{system_, "nx2:gsv"}, m_context{system_, "nx2:gsv"} { + RegisterHandlersTipc({ + {4, &INx2GraphicsSystemSession::Stub, "Boot (Switch 2, guessed)"}, + {5, &INx2GraphicsSystemSession::Stub, "Unknown5"}, + {101, &INx2GraphicsSystemSession::Stub, "Unknown101"}, + {202, &INx2GraphicsSystemSession::OpenSyncptSession, "OpenSyncptSession (guessed)"}, + {204, &INx2GraphicsSystemSession::AttachSyncptMemory, "AttachSyncptMemory (guessed)"}, + {305, &INx2GraphicsSystemSession::OpenViSession, "OpenViSession (guessed)"}, + {497, &INx2GraphicsSystemSession::OpenGpuSession, "OpenGpuSession (guessed)"}, + {111, &INx2GraphicsSystemSession::CreateNativeMemory, "CreateNativeMemory"}, + {112, &INx2GraphicsSystemSession::DestroyNativeMemory, "DestroyNativeMemory"}, + {114, &INx2GraphicsSystemSession::BindNativeMemory, "BindNativeMemory"}, + {314, &INx2GraphicsSystemSession::ConfigureWindowImagePool, "DestroyNativeWindowAliasImagePool"}, + {315, &INx2GraphicsSystemSession::ConfigureWindowImagePool, "CreateNativeWindowAliasImagePool"}, + {423, &INx2GraphicsSystemSession::BindNativeGpuMemory, "BindNativeGpuMemory"}, + {429, &INx2GraphicsSystemSession::CreateNativeImage, "CreateNativeImage"}, + {430, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyNativeImage"}, + {435, &INx2GraphicsSystemSession::CalculateNativeImageSize, "CalculateNativeImageSize"}, + {1402, &INx2GraphicsSystemSession::OpenNvsrvSession, "OpenNvsrvSession (guessed)"}, + {1510, &INx2GraphicsSystemSession::CalculateRequirements, "CalculateRequirements"}, + {991, &INx2GraphicsSystemSession::CalculateDecoderImagePoolBuffer, + "CalculateDecoderImagePoolBuffer"}, + {901, &INx2GraphicsSystemSession::Stub, "VideoInitialize (guessed)"}, + {902, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession (guessed)"}, + {906, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSessionWithVersion"}, + {1001, &INx2GraphicsSystemSession::Stub, "VideoInitialize2 (guessed)"}, + {1002, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession2 (guessed)"}, + {911, &INx2GraphicsSystemSession::CreateDecoderImage, "CreateDecoderImage"}, + {961, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetCpuUserImageDescriptor"}, + }); + } + + ~INx2GraphicsSystemSession() override { + if (m_syncpt_poll) { + system.CoreTiming().UnscheduleEvent(m_syncpt_poll); + } + for (auto* event : m_syncpt_events) { + if (event != nullptr) { + m_context.CloseEvent(event); + } + } + } + +private: + // Index of the first raw data word of a TIPC request: header (2 words), then an optional + // special header with pid and copy/move handle words. + static size_t TipcDataOffset(const u32* cmd_buf) { + if ((cmd_buf[1] & 0x80000000) == 0) { + return 2; + } + const u32 desc = cmd_buf[2]; + return 3 + ((desc & 1) ? 2 : 0) + ((desc >> 1) & 0xF) + ((desc >> 5) & 0xF); + } + + void Stub(HLERequestContext& ctx) { + const auto* cmd_buf = ctx.CommandBuffer(); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC {} [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}", + ctx.GetCommand(), cmd_buf[2], cmd_buf[3], cmd_buf[4], cmd_buf[5]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // nn::codec::video::detail::ipc::Initialize sends 901, 902, 1001, 1002. 902 and 1002 return + // a session handle at word 3 and the result at word 4; left to the auto stub, the handle was 0 + // and InitializeVp9Decoder failed with 2001-0114 (title 1 aborts when a movie starts). + // Newer SDKs (nn::codec::video::detail::InitializeServerContext) send 906(u8, u64 decoder + // library version) instead of 902; title 4 sends version 0x3'00030000 and aborted with + // InvalidHandle at its first movie while 906 was auto-stubbed. + void OpenVideoSession(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const bool second = ctx.GetCommand() == 1002; + LOG_INFO(Service, "nx2:gsv TIPC {} [2]={:#x} [3]={:#x} [4]={:#x}, returning a video decoder session", + ctx.GetCommand(), w[2], w[3], w[4]); + ReturnRawSession(system, ctx, + std::make_shared<INx2VideoSession>(system, m_objects, + second ? "nx2:vdec2" : "nx2:vdec")); + } + + void OpenSyncptSession(HLERequestContext& ctx) { + // Used by nn::syncpt::sdk::InitializeEx; handle at word 3, result at word 4. + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 202, returning a nx2:syncpt session"); + ReturnRawSession(system, ctx, std::make_shared<INx2SyncptSession>(system)); + } + + // nn::syncpt::sdk::InitializeEx sends two transfer memories (0x1000 bytes RW, then 0x2000 + // bytes the client maps read-only) and expects the server to have written a header into the + // second one: u32 version (must be 1) at +0, u32 base id at +4, u32 count at +8 (the client + // requires ((count + 0x40) >> 6) - 1 < 0x40), then offsets of: the u64 syncpoint values (RO, + // +0xc), the RW "armed" summary word (+0x10), the RO summary word (+0x14), the RW armed bits + // (+0x18) and the RO acknowledge bits (+0x1c). + // To wait, the client makes its RW bit differ from the RO bit, signals the first returned + // event (it attaches that handle as writable), and waits on the second until the server makes + // the bits equal again; it then rereads the value and rearms if the threshold isn't reached. + // Reply: the two handles (words 3 and 4), then the result. + static constexpr u32 SyncptValuesOffset = 0x40; // RO, u64 per syncpoint + static constexpr u32 SyncptRoSummaryOffset = 0x1040; + static constexpr u32 SyncptRoBitsOffset = 0x1100; // RO, up to 0x40 u64 words + static constexpr u32 SyncptRwSummaryOffset = 0x0; + static constexpr u32 SyncptRwBitsOffset = 0x100; // RW, up to 0x40 u64 words + static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager + + void AttachSyncptMemory(HLERequestContext& ctx) { + // RequestParser doesn't skip TIPC handle words, so read the raw data words. + const auto* cmd_buf = ctx.CommandBuffer(); + const size_t data = TipcDataOffset(cmd_buf); + const u32 version_requested = cmd_buf[data]; + const u32 max_syncpoints = std::min<u32>(cmd_buf[data + 1], (0x1000 - 0x40) / 8); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 204, version={} max_syncpoints={:#x}", + version_requested, max_syncpoints); + + auto rw = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(0)); + auto ro = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(1)); + if (rw.IsNotNull() && ro.IsNotNull() && ro->GetSize() >= 0x2000) { + m_syncpt_memory = &ro->GetOwner()->GetMemory(); + m_syncpt_rw = GetInteger(rw->GetSourceAddress()); + m_syncpt_ro = GetInteger(ro->GetSourceAddress()); + m_syncpt_count = max_syncpoints; + auto& memory = *m_syncpt_memory; + memory.Write32(m_syncpt_ro + 0x00, 1); // version + memory.Write32(m_syncpt_ro + 0x04, 0); // base id (guess) + memory.Write32(m_syncpt_ro + 0x08, max_syncpoints); + memory.Write32(m_syncpt_ro + 0x0c, SyncptValuesOffset); + memory.Write32(m_syncpt_ro + 0x10, SyncptRwSummaryOffset); + memory.Write32(m_syncpt_ro + 0x14, SyncptRoSummaryOffset); + memory.Write32(m_syncpt_ro + 0x18, SyncptRwBitsOffset); + memory.Write32(m_syncpt_ro + 0x1c, SyncptRoBitsOffset); + UpdateSyncptValues(); + if (!m_syncpt_poll) { + m_syncpt_poll = Core::Timing::CreateEvent( + "nx2:gsv:SyncptPoll", + [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + PollSyncpts(); + return std::nullopt; + }); + system.CoreTiming().ScheduleLoopingEvent(SyncptPollInterval, SyncptPollInterval, + m_syncpt_poll); + } + } else { + LOG_ERROR(Service, "nx2:gsv TIPC 204: missing syncpt transfer memory"); + } + + if (m_syncpt_events[0] == nullptr) { + m_syncpt_events[0] = m_context.CreateEvent("nx2:gsv:SyncptRequest"); + m_syncpt_events[1] = m_context.CreateEvent("nx2:gsv:SyncptNotify"); + } + IPC::ResponseBuilder rb{ctx, 2, 2, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, *m_syncpt_events[0]); // signalled by the client + rb.PushCopyObjects(ctx, m_syncpt_events[1]->GetReadableEvent()); + } + + // Mirrors Eden's guest syncpoint values into the shared RO page. Returns true if any changed. + bool UpdateSyncptValues() { + auto& manager = system.Host1x().GetSyncpointManager(); + bool changed = false; + for (u32 id = 0; id < std::min(m_syncpt_count, HostSyncpoints); ++id) { + const u64 value = manager.GetGuestSyncpointValue(id); + const u64 address = m_syncpt_ro + SyncptValuesOffset + id * 8; + if (m_syncpt_memory->Read64(address) != value) { + m_syncpt_memory->Write64(address, value); + changed = true; + } + } + return changed; + } + + // Answers every armed wait: the value page is current, so the client rechecks its threshold + // and rearms if it still has to wait. Acknowledging without a value change costs the client a + // wakeup per poll, which keeps waits whose syncpoint we never see from hanging. + void PollSyncpts() { + if (m_syncpt_memory == nullptr) { + return; + } + UpdateSyncptValues(); + bool acked = false; + const u32 words = (m_syncpt_count + 63) / 64; + for (u32 w = 0; w < words; ++w) { + const u64 rw_bits = m_syncpt_memory->Read64(m_syncpt_rw + SyncptRwBitsOffset + w * 8); + const u64 ro_address = m_syncpt_ro + SyncptRoBitsOffset + w * 8; + if (m_syncpt_memory->Read64(ro_address) != rw_bits) { + m_syncpt_memory->Write64(ro_address, rw_bits); + acked = true; + } + } + m_syncpt_memory->Write64(m_syncpt_ro + SyncptRoSummaryOffset, + m_syncpt_memory->Read64(m_syncpt_rw + SyncptRwSummaryOffset)); + if (acked) { + m_syncpt_events[1]->Signal(system.Kernel()); + } + } + + // nn::gsv::shim::CalculateRequirements(u64* size, u64* alignment, preference, u32): the 0x40-byte + // InterModuleUserImagePreference (words 2..17) and a u32 (word 18). A block-linear preference + // (MakeInterModuleBlockUserImagePreference) holds kind 2 at word 3, (width - 1) | (height - 1) + // << 16 at word 4, two u32s at words 6-7 and the pixel format at word 8. Reply: result, u64 + // size, u64 alignment; the client raises the alignment to at least 4 KiB and rounds the size. + void CalculateRequirements(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const auto layout = Nx2CalculateImageLayout(w + 2); + LOG_DEBUG(Service, "nx2:gsv 1510 kind={} {}x{} block={} pitch={:#x} format={} -> size={:#x} align={:#x}", + w[3], layout.width, layout.height, layout.block_height_log2, layout.pitch, w[8], + layout.size, layout.alignment); + TipcReplyWords(ctx, {static_cast<u32>(layout.size), static_cast<u32>(layout.size >> 32), + static_cast<u32>(layout.alignment), + static_cast<u32>(layout.alignment >> 32)}); + } + + // nn::codec::CalculateDecoderImagePoolBuffer(size*, alignment*, VideoDecoderImagePreference, + // u32 count) sends 991(preference[16], u32 count) and reads result, u64 size, u64 alignment; + // the client raises the alignment to at least 4 KiB and rounds the size up to it. The + // preference (MakeDecodeImagePreference) holds an output layout code at word 1 (1/4 for + // format 3, which the movie player uses, read here as NV12) and (width - 1) | (height - 1) << 16 + // at word 2. Left to the auto stub, the reply words keep the request's, so the size became + // 0x167027f'00000001 and title 1's movie player aborted (main+0x19e5b58) when its first movie + // started. Sizes are block-linear NV12 (luma, then half-height chroma, 128-row blocks) or 4 + // bytes per pixel for the other codes; nothing is decoded into them yet. + void CalculateDecoderImagePoolBuffer(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const auto layout = Nx2CalculateDecoderImageLayout(w + 2); + const u32 count = std::max<u32>(w[18], 1); + const u64 size = layout.size * count; + const u64 alignment = 64 * 128; + LOG_INFO(Service, "nx2:gsv 991 decoder image pool code={} {}x{} count={} -> size={:#x}", + layout.code, layout.width, layout.height, count, size); + TipcReplyWords(ctx, {static_cast<u32>(size), static_cast<u32>(size >> 32), + static_cast<u32>(alignment), static_cast<u32>(alignment >> 32)}); + } + + // 435(preference[16], u32 count): result, u64 size of a GPU native image. + void CalculateNativeImageSize(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 size = Nx2CalculateImageLayout(w + 2).size * std::max<u32>(w[18], 1); + LOG_DEBUG(Service, "nx2:gsv 435 count={} -> size={:#x}", w[18], size); + TipcReplyWords(ctx, {static_cast<u32>(size), static_cast<u32>(size >> 32)}); + } + + static u64 Pair(const u32* w, size_t i) { + return u64{w[i]} | (u64{w[i + 1]} << 32); + } + + // 111(u8 mode at word 2, u64 size): result, u32 native memory id. + void CreateNativeMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 size = Pair(w, 3); + std::scoped_lock lk{m_objects->mutex}; + const u32 id = m_objects->next_memory++; + m_objects->memories[id] = {size, 0}; + LOG_INFO(Service, "nx2:gsv 111 CreateNativeMemory mode={} size={:#x} -> id {}", w[2] & 0xFF, + size, id); + TipcReplyWords(ctx, {id}); + } + + void DestroyNativeMemory(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + std::scoped_lock lk{m_objects->mutex}; + m_objects->memories.erase(id); + LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id); + TipcReplyWords(ctx, {}); + } + + // 114(u32 id, u64 address, u64 size, u64 flags, u64 offset): back native memory with process + // memory. + void BindNativeMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + BindMemory(w[2], Pair(w, 3), Pair(w, 5), "114"); + TipcReplyWords(ctx, {}); + } + + // 423(u32 id, u64 address, u64 size, u32 parent GpuMemoryHandle, u64 offset, ...): back + // native memory with part of GPU-accessible memory. + void BindNativeGpuMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_DEBUG(Service, "nx2:gsv 423 parent={} offset={:#x}", w[7], Pair(w, 8)); + BindMemory(w[2], Pair(w, 3), Pair(w, 5), "423"); + TipcReplyWords(ctx, {}); + } + + void BindMemory(u32 id, u64 address, u64 size, const char* command) { + std::scoped_lock lk{m_objects->mutex}; + auto& memory = m_objects->memories[id]; + memory.address = address; + if (memory.size == 0) { + memory.size = size; + } + LOG_INFO(Service, "nx2:gsv {} bind native memory {} at {:#x}+{:#x}", command, id, address, + size); + } + + // 429(preference[16], u32 count, u32 memory id): result, u32 native image id. + void CreateNativeImage(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + std::scoped_lock lk{m_objects->mutex}; + const u32 id = m_objects->next_image++; + auto& image = m_objects->images[id]; + image.memory = w[19]; + std::copy_n(w + 2, image.preference.size(), image.preference.begin()); + image.size = Nx2CalculateImageLayout(w + 2).size; + LOG_INFO(Service, "nx2:gsv 429 CreateNativeImage memory={} count={} size={:#x} -> id {}", + w[19], w[18], image.size, id); + TipcReplyWords(ctx, {id}); + } + + // 911(preference[16], u32 count, u32 memory id) (nn::codec::InitializeDecoderImagePool): + // result, the id of the first of `count` decoder output images; the client names image i of + // the pool id + i. They share the pool's memory, one image size apart. + void CreateDecoderImage(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 count = std::max<u32>(w[18], 1); + const auto layout = Nx2CalculateDecoderImageLayout(w + 2); + std::scoped_lock lk{m_objects->mutex}; + const u32 first = m_objects->next_image; + m_objects->next_image += count; + for (u32 i = 0; i < count; ++i) { + auto& image = m_objects->images[first + i]; + image.memory = w[19]; + std::copy_n(w + 2, image.preference.size(), image.preference.begin()); + image.size = layout.size; + image.offset = layout.size * i; + image.decoder = true; + } + LOG_INFO(Service, "nx2:gsv 911 decoder images {}..{} code={} {}x{} in memory {}", first, + first + count - 1, layout.code, layout.width, layout.height, w[19]); + TipcReplyWords(ctx, {first}); + } + + // 961(u32 image id): result, the 128-byte nn::gsv::CpuUserImageDescriptor that + // nn::gs::sdk::ReadUserImage* (behind nn::codec::ReadDecoderOutputImage*) copy an image out + // with: u8 kind at +0; for the pitch-linear kinds (8 RGBA, 9 BGRA, 10 NV12) the plane's + // offset into the pool memory (u32 +0x8) and row pitch (u64 +0x10), NV12's chroma plane at + // +0x18/+0x20. 11-13 are the block-linear kinds, not used here. + void GetCpuImageDescriptor(HLERequestContext& ctx) { + using Tegra::Host1x::Nx2PictureFormat; + const u32 id = ctx.CommandBuffer()[2]; + std::array<u32, 32> words{}; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(id); + if (image != m_objects->images.end() && image->second.decoder) { + const auto layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); + const u64 offset = image->second.offset; + words[0] = layout.format == Nx2PictureFormat::Nv12 ? 0xa + : layout.format == Nx2PictureFormat::Bgra8 ? 0x9 + : 0x8; + words[2] = static_cast<u32>(offset); + words[4] = static_cast<u32>(layout.pitch); + if (layout.format == Nx2PictureFormat::Nv12) { + words[6] = static_cast<u32>(offset + layout.chroma_offset); + words[8] = static_cast<u32>(layout.pitch); + } + } + } + if (Nx2FirstVideoCommand("nx2:gsv", 961) || words[0] == 0) { + LOG_INFO(Service, "nx2:gsv 961 image {} -> kind {:#x} offset {:#x} pitch {:#x}", id, + words[0], words[2], words[4]); + } + TipcReplyWords(ctx, words); + } + + void DestroyNativeImage(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + std::scoped_lock lk{m_objects->mutex}; + m_objects->images.erase(id); + TipcReplyWords(ctx, {}); + } + + // 315(u32 image, u32 count) / 314(u32 image): add/remove a window alias image pool. + void ConfigureWindowImagePool(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + const bool create = ctx.GetCommand() == 315; + std::scoped_lock lk{m_objects->mutex}; + if (auto it = m_objects->images.find(id); it != m_objects->images.end()) { + it->second.window_pool = create; + } + LOG_INFO(Service, "nx2:gsv {} window image pool image {}", ctx.GetCommand(), id); + TipcReplyWords(ctx, {}); + } + + void OpenViSession(HLERequestContext& ctx) { + // Used by nn::vi2::sdk::Initialize; takes one u64, returns a session (handle at word 3, + // result at word 4). + const auto* cmd_buf = ctx.CommandBuffer(); + const size_t data = TipcDataOffset(cmd_buf); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 305 arg={:#x}, returning a nx2:vi2 session", + u64{cmd_buf[data]} | (u64{cmd_buf[data + 1]} << 32)); + ReturnRawSession(system, ctx, std::make_shared<INx2Vi2Session>(system, m_objects)); + } + + void OpenGpuSession(HLERequestContext& ctx) { + // Used by nn::gpu::sdk::BootWithRemoteMemory; takes one u64, returns a session (handle at + // word 3, result at word 4) that the GPU SDK keeps. + const auto* cmd_buf = ctx.CommandBuffer(); + const size_t data = TipcDataOffset(cmd_buf); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 497 arg={:#x}, returning a nx2:gpu session", + u64{cmd_buf[data]} | (u64{cmd_buf[data + 1]} << 32)); + ReturnRawSession(system, ctx, std::make_shared<INx2GpuSession>(system)); + } + + void OpenNvsrvSession(HLERequestContext& ctx) { + // The SDK reads the handle at word 3 and the result at word 4. + // nvsrv speaks the nvdrv INvDrvServices protocol (Open/Ioctl/Close/..., + // 13 = SetGraphicsFirmwareMemoryMarginEnabled), so serve it with Eden's nvdrv on the shared + // module. The nx2: name keeps unknown commands auto-stubbed and logged. + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 1402, returning an nvdrv-backed nx2:nvsrv session"); + auto module = system.ServiceManager().GetService<Nvidia::NVDRV>("nvdrv", true)->GetModule(); + auto nvsrv = std::make_shared<Nvidia::NVDRV>(system, module, "nx2:nvsrv"); + nvsrv->InitializeForProcess(ctx.GetThread().GetOwnerProcess()); + ReturnRawSession(system, ctx, std::move(nvsrv)); + } + + static constexpr std::chrono::nanoseconds SyncptPollInterval{std::chrono::microseconds{500}}; + + std::shared_ptr<Nx2NativeObjects> m_objects{std::make_shared<Nx2NativeObjects>()}; + KernelHelpers::ServiceContext m_context; + std::array<Kernel::KEvent*, 2> m_syncpt_events{}; + Core::Memory::Memory* m_syncpt_memory{}; + u64 m_syncpt_rw{}; + u64 m_syncpt_ro{}; + u32 m_syncpt_count{}; + std::shared_ptr<Core::Timing::EventType> m_syncpt_poll; +}; + +} // namespace Service diff --git a/src/core/hle/service/nx2_stub_interface.h b/src/core/hle/service/nx2_stub_interface.h new file mode 100644 index 0000000000..6890298d1b --- /dev/null +++ b/src/core/hle/service/nx2_stub_interface.h @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <deque> +#include <mutex> +#include <string> +#include <string_view> + +#include <fmt/format.h> + +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/service.h" + +namespace Service { + +// Prefix of placeholder interfaces for Switch 2-only IPC. ReportUnimplementedFunction answers +// every call on such an interface with an empty success, so boot can go on while the log shows +// which commands are used. +inline constexpr std::string_view Nx2StubPrefix = "nx2:"; + +class INx2StubInterface final : public ServiceFramework<INx2StubInterface> { +public: + explicit INx2StubInterface(Core::System& system_, const char* name) + : ServiceFramework{system_, name} {} + + // Names must outlive the interface, so they are interned here. + static const char* InternName(std::string name) { + static std::mutex mutex; + static std::deque<std::string> names; + std::scoped_lock lk{mutex}; + for (const auto& n : names) { + if (n == name) { + return n.c_str(); + } + } + return names.emplace_back(std::move(name)).c_str(); + } +}; + +// Handler for an unknown Switch 2 command that returns an interface: hands back a stub named +// after the owning service and the command id. +inline void ReturnNx2StubInterface(Core::System& system, HLERequestContext& ctx, + std::string_view owner) { + const char* name = + INx2StubInterface::InternName(fmt::format("{}{}::{}", Nx2StubPrefix, owner, ctx.GetCommand())); + LOG_WARNING(Service, "(STUBBED) Switch 2 {}::{} -> stub interface {}", owner, ctx.GetCommand(), + name); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<INx2StubInterface>(ctx, system, name); +} + +} // namespace Service diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp index b4fe2b3ed5..1490af0328 100644 --- a/src/core/hle/service/service.cpp +++ b/src/core/hle/service/service.cpp @@ -4,6 +4,14 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include "common/scope_exit.h" +#include <vector> +#include <tuple> +#include <mutex> +#include <map> +#include <chrono> +#include <algorithm> +#include <cstdlib> #include <fmt/ranges.h> #include "common/assert.h" #include "common/logging.h" @@ -13,6 +21,7 @@ #include "core/hle/kernel/kernel.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/service.h" +#include "core/hle/service/nx2_stub_interface.h" #include "core/hle/service/sm/sm.h" #include "core/reporter.h" @@ -63,13 +72,21 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx, fmt::memory_buffer buf; fmt::format_to(std::back_inserter(buf), "function '{}({})': port='{}' cmd_buf={{[0]={:#x}", ctx.GetCommand(), function_name, service_name, cmd_buf[0]); - for (int i = 1; i <= 8; ++i) + // The Switch 2 video decoder sessions are being reverse engineered: dump whole requests. + const int last_word = std::string_view{service_name}.starts_with("nx2:vdec") ? 47 : 8; + for (int i = 1; i <= last_word; ++i) fmt::format_to(std::back_inserter(buf), ", [{}]={:#x}", i, cmd_buf[i]); buf.push_back('}'); system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name); UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf)); - if (Settings::values.use_auto_stub) { + // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config. Switch 2 + // programs are always auto-stubbed: they still call many commands nobody has implemented. + static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; + const auto* const caller = ctx.GetThread().GetOwnerProcess(); + if (Settings::values.use_auto_stub || env_auto_stub || + std::string_view{service_name}.starts_with(Nx2StubPrefix) || + (caller != nullptr && caller->IsSwitch2())) { LOG_WARNING(Service, "Using auto stub fallback!"); IPC::ResponseBuilder rb{ctx, 2}; rb.Push(ResultSuccess); @@ -83,7 +100,25 @@ void ServiceFrameworkBase::InvokeRequest(HLERequestContext& ctx) { return ReportUnimplementedFunction(ctx, info); LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); + // EDEN_NX2_IPC_TRACE=1 logs every dispatched command and the result word it returned. + static const bool ipc_trace = std::getenv("EDEN_NX2_IPC_TRACE") != nullptr; + if (ipc_trace) { + LOG_INFO(Service, "ipc {}::{} ({})", service_name, ctx.GetCommand(), info->name); + } handler_invoker(this, info->handler_callback, ctx); + if (ipc_trace) { + // Response layout: [0..1] header, then 'SFCO', version, result. + const auto* cmd_buf = ctx.CommandBuffer(); + for (int i = 0; i < 12; ++i) { + if (cmd_buf[i] == 0x4F434653) { // "SFCO" + if (cmd_buf[i + 2] != 0) { + LOG_INFO(Service, "ipc {}::{} -> result {:#x}", service_name, ctx.GetCommand(), + cmd_buf[i + 2]); + } + break; + } + } + } } void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) { @@ -93,12 +128,68 @@ void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) { return ReportUnimplementedFunction(ctx, info); LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); + static const bool ipc_trace = std::getenv("EDEN_NX2_IPC_TRACE") != nullptr; + if (ipc_trace) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "tipc {}::{} ({}) [2]={:#x} [3]={:#x} [4]={:#x}", service_name, + ctx.GetCommand(), info->name, w[2], w[3], w[4]); + } handler_invoker(this, info->handler_callback, ctx); } +namespace { +// EDEN_NX2_IPC_SUMMARY: every 5 s, log how often each service command ran and with which result +// since the last summary (finds a game polling something that never becomes ready). +void CountRequest(const std::string& service, u32 command, u32 type, u32 result) { + static const bool enabled = std::getenv("EDEN_NX2_IPC_SUMMARY") != nullptr; + if (!enabled) { + return; + } + static std::mutex mutex; + static std::map<std::tuple<std::string, u32, u32, u32>, u32> counts; + static auto last = std::chrono::steady_clock::now(); + std::scoped_lock lock{mutex}; + ++counts[{service, command, type, result}]; + const auto now = std::chrono::steady_clock::now(); + if (now - last < std::chrono::seconds{5}) { + return; + } + last = now; + std::vector<std::pair<u32, std::string>> lines; + for (const auto& [key, count] : counts) { + lines.emplace_back(count, fmt::format("{}::{}{} -> {:#x}", std::get<0>(key), std::get<1>(key), + std::get<2>(key) >= 16 ? " (tipc)" : "", + std::get<3>(key))); + } + std::sort(lines.begin(), lines.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + std::string text; + for (size_t i = 0; i < std::min<size_t>(lines.size(), 24); ++i) { + text += fmt::format(" | {}x {}", lines[i].first, lines[i].second); + } + LOG_INFO(Service, "ipc summary ({} kinds){}", lines.size(), text); + counts.clear(); +} +} // Anonymous namespace + Result ServiceFrameworkBase::HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) { const auto guard = LockService(); + SCOPE_EXIT { + const auto* cmd_buf = ctx.CommandBuffer(); + u32 result_word = 0; + const u32 type = static_cast<u32>(ctx.GetCommandType()); + if (type >= 16) { + result_word = cmd_buf[2]; + } else { + for (int i = 0; i < 12; ++i) { + if (cmd_buf[i] == 0x4F434653) { // "SFCO" + result_word = cmd_buf[i + 2]; + break; + } + } + } + CountRequest(service_name, ctx.GetCommand(), type, result_word); + }; Result result = ResultSuccess; diff --git a/src/core/launch_params.cpp b/src/core/launch_params.cpp index a0483226ec..25fdd45fdb 100644 --- a/src/core/launch_params.cpp +++ b/src/core/launch_params.cpp @@ -94,7 +94,7 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch break; case 'u': { // Launch game with a specific user - bool argument_ok = isdigit(optarg[0]); + bool argument_ok = isdigit(optarg[0]) != 0; p.selected_user = atoi(optarg); if (!argument_ok) { // try to look it up by username, only finds the first username that matches. diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index 5e2ec7e0b9..458e0210a0 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -12,6 +12,7 @@ #include "core/file_sys/content_archive.h" #include "core/file_sys/control_metadata.h" #include "core/file_sys/patch_manager.h" +#include "core/file_sys/romfs.h" #include "core/file_sys/romfs_factory.h" #include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_process.h" @@ -71,6 +72,30 @@ struct PatchCollection { std::array<s32, 13> module_patcher_indices{}; }; +namespace { + +// RomFS for a loose ExeFS: the first .romfs file next to main, else a romfs/ directory next to +// the exefs/ directory (the usual layout of an extracted title). +FileSys::VirtualFile FindLooseRomFS(const FileSys::VirtualDir& exefs_dir) { + if (exefs_dir == nullptr) { + return nullptr; + } + for (const auto& f : exefs_dir->GetFiles()) { + if (f->GetExtension() == "romfs") { + return f; + } + } + if (const auto parent = exefs_dir->GetParentDirectory()) { + if (const auto romfs_dir = parent->GetSubdirectory("romfs")) { + LOG_INFO(Loader, "Using RomFS directory next to the ExeFS"); + return FileSys::CreateRomFS(romfs_dir); + } + } + return nullptr; +} + +} // namespace + AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys::VirtualFile file_, bool override_update_) : AppLoader(std::move(file_)) , override_update(override_update_) @@ -122,6 +147,8 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys FileSys::NACP nacp(nacp_file); name = nacp.GetApplicationName(); } + + romfs = FindLooseRomFS(file_dir); } AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( @@ -129,7 +156,9 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( : AppLoader(directory->GetFile("main")) , dir(std::move(directory)) , override_update(override_update_) -{} +{ + romfs = FindLooseRomFS(dir); +} FileType AppLoader_DeconstructedRomDirectory::IdentifyType(const FileSys::VirtualFile& dir_file) { if (FileSys::IsDirectoryExeFS(dir_file->GetContainingDirectory())) { @@ -145,6 +174,9 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect return {ResultStatus::ErrorAlreadyLoaded, {}}; } + // Loaded directly from a loose ExeFS (not wrapped by the NCA/NSP loaders, which register + // the process with the filesystem controller themselves). + const bool standalone = dir == nullptr; if (dir == nullptr) { if (file == nullptr) { return {ResultStatus::ErrorNullFile, {}}; @@ -266,6 +298,14 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect LOG_DEBUG(Loader, "loaded module {} @ {:#x}", module, load_addr); } + if (standalone) { + // Without this, fsp-srv SetCurrentProcess fails with ResultTargetNotFound. + system.GetFileSystemController().RegisterProcess( + process.GetProcessId(), metadata.GetTitleID(), + std::make_shared<FileSys::RomFSFactory>(*this, system.GetContentProvider(), + system.GetFileSystemController())); + } + is_loaded = true; return {ResultStatus::Success, LoadParameters{metadata.GetMainThreadPriority(), metadata.GetMainThreadStackSize()}}; diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index 92e2e2f602..5871ac412f 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -393,6 +393,13 @@ std::unique_ptr<AppLoader> GetLoader(Core::System& system, FileSys::VirtualFile FileType type = IdentifyFile(file); const FileType filename_type = GuessFromFilename(file->GetName()); + // A "main" inside an ExeFS directory is also a valid NSO, but loading it alone skips + // main.npdm, rtld and the sdk/subsdk modules. Load the whole directory instead. + if (type == FileType::NSO && filename_type == FileType::DeconstructedRomDirectory && + IdentifyFileLoader<AppLoader_DeconstructedRomDirectory>(file)) { + type = FileType::DeconstructedRomDirectory; + } + // Special case: 00 is either a NCA or NAX. if (type != filename_type && !(file->GetName() == "00" && type == FileType::NAX)) { LOG_WARNING(Loader, "File {} has a different type ({}) than its extension.", diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp index ea1d1c9ecd..7e3b9a54c2 100644 --- a/src/core/loader/nso.cpp +++ b/src/core/loader/nso.cpp @@ -247,6 +247,7 @@ std::optional<VAddr> AppLoader_NSO::LoadModule(Kernel::KProcess& process, Core:: } // Load codeset for current process + LOG_INFO(Loader, "Module {} loaded at {:#x} (size {:#x})", nso_file.GetName(), load_base, image_size); process.LoadModule(system.Kernel(), std::move(codeset), load_base); return load_base + image_size; } diff --git a/src/core/memory.cpp b/src/core/memory.cpp index eabc6dfccb..6e0ab0ee6d 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -5,6 +5,7 @@ // SPDX-FileCopyrightText: 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <atomic> #include <algorithm> #include <cstring> #include <mutex> @@ -36,6 +37,11 @@ namespace Core::Memory { +u32& UnmappedReadFlag() { + thread_local u32 flag{}; + return flag; +} + static inline bool AddressSpaceContains(const Common::PageTable& table, const Common::ProcessAddress addr, const std::size_t size) { const Common::ProcessAddress max_addr = 1ULL << table.GetAddressSpaceBits(); return addr + size >= addr && addr + size <= max_addr; @@ -623,7 +629,13 @@ struct Memory::Impl { inline T Read(Common::ProcessAddress vaddr) noexcept requires(std::is_trivially_copyable_v<T>) { const u64 addr = GetInteger(vaddr); if (auto const ptr = GetPointerImpl(addr, [addr]() { - LOG_ERROR(HW_Memory, "Unmapped Read{} @ {:#016x}", sizeof(T) * 8, addr); + // A guest spinning on a bad pointer reads unmapped memory tens of thousands of times + // a second: keep the first ones and then a sample. + static std::atomic<u64> unmapped_reads{}; + if (const u64 count = unmapped_reads.fetch_add(1); count < 1000 || count % 100000 == 0) { + LOG_ERROR(HW_Memory, "Unmapped Read{} @ {:#016x} (#{})", sizeof(T) * 8, addr, count); + } + ++UnmappedReadFlag(); }, [&]() { HandleRasterizerDownload(addr, sizeof(T)); }); ptr) [[likely]] { diff --git a/src/core/memory.h b/src/core/memory.h index 7167efbb84..204b7514f6 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -38,6 +38,9 @@ class MemoryManager; namespace Core::Memory { +/// Counts unmapped reads on this host thread since the CPU callbacks last checked (debug aid). +u32& UnmappedReadFlag(); + /** * Page size used by the ARM architecture. This is the smallest granularity with which memory can * be mapped. diff --git a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc index 8a3fab7ac1..ee9f0faa18 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc +++ b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc @@ -2,6 +2,24 @@ // SPDX-License-Identifier: GPL-3.0-or-later // DO NOT REORDER +// ARMv8.1+ encodings used by Switch 2 code. Kept first so they win over broader patterns. +INST(LDAPR_sz, "LDAPRB, LDAPRH, LDAPR", "zz11100010111111110000nnnnnttttt") +INST(AtomicMemOp, "LDADD/LDCLR/LDEOR/LDSET/LDSMAX/LDSMIN/LDUMAX/LDUMIN/SWP", "zz111000AR1sssssoooo00nnnnnttttt") +INST(CompareAndSwap, "CASB, CASH, CAS", "zz0010001L1sssssO11111nnnnnttttt") +INST(CompareAndSwapPair, "CASP", "0z0010000L1sssssO11111nnnnnttttt") +INST(PACAUT_reg, "PACIA/PACIB/PACDA/PACDB/AUTIA/AUTIB/AUTDA/AUTDB", "110110101100000100Zooonnnnnddddd") +INST(XPAC_1, "XPACI, XPACD", "110110101100000101000D11111ddddd") +INST(PACGA, "PACGA", "10011010110mmmmm001100nnnnnddddd") +INST(RETA, "RETAA, RETAB", "110101100101111100001M1111111111") +INST(BRA_BLRA, "BRAA/BRAB/BLRAA/BLRAB (+Z)", "1101011Z00L1111100001Mnnnnnmmmmm") +INST(LDRA, "LDRAA, LDRAB", "11111000MS1iiiiiiiiiW1nnnnnttttt") +INST(STG_family, "STG/STZG/ST2G/STZ2G/LDG", "11011001oo1iiiiiiiiixxnnnnnttttt") +INST(STGP, "STGP", "0110100xx0iiiiiiiuuuuunnnnnttttt") +INST(IRG, "IRG", "10011010110mmmmm000100nnnnnddddd") +INST(GMI, "GMI", "10011010110mmmmm000101nnnnnddddd") +INST(ADDG, "ADDG", "1001000110iiiiii00ttttnnnnnddddd") +INST(SUBG, "SUBG", "1101000110iiiiii00ttttnnnnnddddd") + INST(FMOV_3, "FMOV (vector, immediate)", "0Q00111100000abc111111defghddddd") INST(FMOV_2, "FMOV (vector, immediate)", "0Qo0111100000abc111101defghddddd") INST(MOVI, "MOVI, MVNI, ORR, BIC (vector, immediate)", "0Qo0111100000abcmmmm01defghddddd") diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp index 7840c63d79..70d5098a65 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp @@ -82,6 +82,9 @@ bool TranslateSingleInstruction(IR::Block& block, LocationDescriptor descriptor, #include "dynarmic/frontend/A64/translate/impl/impl.cpp" #include "dynarmic/frontend/A64/translate/impl/impl.h" #include "dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp" +#include "dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp" +#include "dynarmic/frontend/A64/translate/impl/memory_tagging.cpp" +#include "dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp" #include "dynarmic/frontend/A64/translate/impl/load_store_load_literal.cpp" #include "dynarmic/frontend/A64/translate/impl/load_store_multiple_structures.cpp" #include "dynarmic/frontend/A64/translate/impl/load_store_no_allocate_pair.cpp" diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h index c8afef3283..d8e6f2b739 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h @@ -326,6 +326,17 @@ struct TranslatorVisitor final { bool SWP(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDAPR(Reg Rn, Reg Rt); + // ARMv8.1+ forms decoded for Switch 2 code (load_store_atomic.cpp, pointer_authentication.cpp, + // memory_tagging.cpp) + bool AtomicMemOp(Imm<2> size, bool A, bool R, Reg Rs, Imm<4> op, Reg Rn, Reg Rt); + bool LDAPR_sz(Imm<2> size, Reg Rn, Reg Rt); + bool CompareAndSwap(Imm<2> size, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); + bool CompareAndSwapPair(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); + bool PACAUT_reg(bool Z, Imm<3> op, Reg Rn, Reg Rd); + bool BRA_BLRA(bool Z, bool L, bool M, Reg Rn, Reg Rm); + bool STG_family(Imm<2> opc, Imm<9> imm9, Imm<2> mode, Reg Rn, Reg Rt); + bool STGP(Imm<2> mode, Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt); + // Loads and stores - Load/Store register (register offset) bool STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt); bool LDRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt); diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp new file mode 100644 index 0000000000..eafc993988 --- /dev/null +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// ARMv8.1 LSE atomics (LDADD/LDCLR/LDEOR/LDSET/LD{S,U}{MAX,MIN}/SWP, CAS, CASP) and +// ARMv8.3 RCpc LDAPR. +// +// Each read-modify-write is built from an exclusive load and an exclusive store. If the +// store fails (another core touched the monitor in between) no register is changed and the +// same instruction is executed again, so the operation stays atomic with multiple cores. + +#include "dynarmic/frontend/A64/translate/impl/impl.h" + +namespace Dynarmic::A64 { + +static Reg AtomicNextReg(Reg r) { + return static_cast<Reg>(static_cast<size_t>(r) + 1); +} + +static IR::U64 AtomicAddress(TranslatorVisitor& v, Reg Rn) { + return Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)}; +} + +// All-ones when value == 0, otherwise zero. +static IR::U64 AtomicMaskIfZero(TranslatorVisitor& v, IR::U64 value) { + const IR::U64 nonzero = v.ir.LogicalShiftRight(v.ir.Or(value, v.ir.Sub(v.ir.Imm64(0), value)), v.ir.Imm8(63)); + return v.ir.Sub(nonzero, v.ir.Imm64(1)); +} + +static IR::U64 AtomicSelect(TranslatorVisitor& v, IR::U64 mask, IR::U64 if_set, IR::U64 if_clear) { + return v.ir.Or(v.ir.And(if_set, mask), v.ir.AndNot(if_clear, mask)); +} + +static IR::UAny AtomicTruncate(TranslatorVisitor& v, IR::U64 value, size_t bytes) { + switch (bytes) { + case 1: + return v.ir.LeastSignificantByte(value); + case 2: + return v.ir.LeastSignificantHalf(value); + case 4: + return v.ir.LeastSignificantWord(value); + default: + return value; + } +} + +static IR::U64 AtomicLoadZext(TranslatorVisitor& v, IR::U64 address, size_t bytes, IR::AccType acctype) { + const IR::UAny value = v.ExclusiveMem(address, bytes, acctype); + return bytes == 8 ? IR::U64{value} : v.ir.ZeroExtendToLong(value); +} + +static IR::U64 AtomicSignExtend(TranslatorVisitor& v, IR::U64 value, size_t bytes) { + return bytes == 8 ? value : v.ir.SignExtendToLong(AtomicTruncate(v, value, bytes)); +} + +static IR::U64 AtomicZeroExtendReg(TranslatorVisitor& v, Reg r, size_t bytes) { + const IR::U64 value = v.X(64, r); + return bytes == 8 ? value : v.ir.ZeroExtendToLong(AtomicTruncate(v, value, bytes)); +} + +// Writes `value` to `r` only when the exclusive store succeeded (ok_mask all-ones). +static void AtomicCommitReg(TranslatorVisitor& v, Reg r, IR::U64 ok_mask, IR::U64 value) { + if (r == Reg::ZR) { + return; + } + v.X(64, r, AtomicSelect(v, ok_mask, value, v.X(64, r))); +} + +// Ends the block: continue with the next instruction on success, retry this one on failure. +static bool AtomicFinish(TranslatorVisitor& v, IR::U32 status) { + v.ir.SetCheckBit(v.ir.IsZero(status)); + v.ir.SetTerm(IR::Term::CheckBit{IR::Term::LinkBlock{v.ir.current_location->AdvancePC(4)}, + IR::Term::LinkBlock{*v.ir.current_location}}); + return false; +} + +static IR::AccType AtomicAccType(bool acquire, bool release) { + return acquire || release ? IR::AccType::ORDERED : IR::AccType::ATOMIC; +} + +bool TranslatorVisitor::AtomicMemOp(Imm<2> size, bool A, bool R, Reg Rs, Imm<4> op, Reg Rn, Reg Rt) { + const size_t bytes = size_t{1} << size.ZeroExtend<size_t>(); + const size_t opv = op.ZeroExtend<size_t>(); + if (opv > 0b1000) { + return UnallocatedEncoding(); + } + + const IR::U64 address = AtomicAddress(*this, Rn); + const IR::AccType acctype = AtomicAccType(A, R); + const IR::U64 old = AtomicLoadZext(*this, address, bytes, acctype); + const IR::U64 s = AtomicZeroExtendReg(*this, Rs, bytes); + + IR::U64 result; + switch (opv) { + case 0b0000: // LDADD + result = ir.Add(old, s); + break; + case 0b0001: // LDCLR + result = ir.AndNot(old, s); + break; + case 0b0010: // LDEOR + result = ir.Eor(old, s); + break; + case 0b0011: // LDSET + result = ir.Or(old, s); + break; + case 0b0100: // LDSMAX + result = ir.MaxSigned(AtomicSignExtend(*this, old, bytes), AtomicSignExtend(*this, s, bytes)); + break; + case 0b0101: // LDSMIN + result = ir.MinSigned(AtomicSignExtend(*this, old, bytes), AtomicSignExtend(*this, s, bytes)); + break; + case 0b0110: // LDUMAX + result = ir.MaxUnsigned(old, s); + break; + case 0b0111: // LDUMIN + result = ir.MinUnsigned(old, s); + break; + default: // SWP + result = s; + break; + } + + const IR::U32 status = ExclusiveMem(address, bytes, acctype, AtomicTruncate(*this, result, bytes)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rt, ok, old); + return AtomicFinish(*this, status); +} + +bool TranslatorVisitor::LDAPR_sz(Imm<2> size, Reg Rn, Reg Rt) { + const size_t bytes = size_t{1} << size.ZeroExtend<size_t>(); + const size_t regsize = bytes == 8 ? 64 : 32; + const IR::UAny data = Mem(AtomicAddress(*this, Rn), bytes, IR::AccType::ORDERED); + X(regsize, Rt, ZeroExtend(data, regsize)); + return true; +} + +bool TranslatorVisitor::CompareAndSwap(Imm<2> size, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { + const size_t bytes = size_t{1} << size.ZeroExtend<size_t>(); + const IR::U64 address = AtomicAddress(*this, Rn); + const IR::AccType acctype = AtomicAccType(L, o0); + + const IR::U64 old = AtomicLoadZext(*this, address, bytes, acctype); + const IR::U64 expected = AtomicZeroExtendReg(*this, Rs, bytes); + const IR::U64 desired = AtomicZeroExtendReg(*this, Rt, bytes); + const IR::U64 equal = AtomicMaskIfZero(*this, ir.Eor(old, expected)); + const IR::U64 result = AtomicSelect(*this, equal, desired, old); + + const IR::U32 status = ExclusiveMem(address, bytes, acctype, AtomicTruncate(*this, result, bytes)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rs, ok, old); + return AtomicFinish(*this, status); +} + +bool TranslatorVisitor::CompareAndSwapPair(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { + if ((static_cast<size_t>(Rs) & 1) != 0 || (static_cast<size_t>(Rt) & 1) != 0) { + return UnallocatedEncoding(); + } + const Reg Rs2 = AtomicNextReg(Rs); + const Reg Rt2 = AtomicNextReg(Rt); + const IR::U64 address = AtomicAddress(*this, Rn); + const IR::AccType acctype = AtomicAccType(L, o0); + + if (!sz) { + // Two 32-bit words, accessed as one 64-bit exclusive. + const IR::U64 old = ExclusiveMem(address, 8, acctype); + const IR::U64 expected = ir.Pack2x32To1x64(X(32, Rs), X(32, Rs2)); + const IR::U64 desired = ir.Pack2x32To1x64(X(32, Rt), X(32, Rt2)); + const IR::U64 equal = AtomicMaskIfZero(*this, ir.Eor(old, expected)); + const IR::U32 status = ExclusiveMem(address, 8, acctype, AtomicSelect(*this, equal, desired, old)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rs, ok, ir.ZeroExtendToLong(ir.LeastSignificantWord(old))); + AtomicCommitReg(*this, Rs2, ok, ir.LogicalShiftRight(old, ir.Imm8(32))); + return AtomicFinish(*this, status); + } + + const IR::U128 old = ExclusiveMem(address, 16, acctype); + const IR::U64 old_lo = ir.VectorGetElement(64, old, 0); + const IR::U64 old_hi = ir.VectorGetElement(64, old, 1); + const IR::U64 diff = ir.Or(ir.Eor(old_lo, X(64, Rs)), ir.Eor(old_hi, X(64, Rs2))); + const IR::U64 equal = AtomicMaskIfZero(*this, diff); + const IR::U64 new_lo = AtomicSelect(*this, equal, X(64, Rt), old_lo); + const IR::U64 new_hi = AtomicSelect(*this, equal, X(64, Rt2), old_hi); + const IR::U32 status = ExclusiveMem(address, 16, acctype, ir.Pack2x64To1x128(new_lo, new_hi)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rs, ok, old_lo); + AtomicCommitReg(*this, Rs2, ok, old_hi); + return AtomicFinish(*this, status); +} + +} // namespace Dynarmic::A64 diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp new file mode 100644 index 0000000000..9c6bcb7872 --- /dev/null +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// ARMv8.5 memory tagging, with tags ignored: every allocation tag is 0, tag stores only do +// their address writeback, and the STZG forms still zero the granules they cover. + +#include "dynarmic/frontend/A64/translate/impl/impl.h" + +namespace Dynarmic::A64 { + +static IR::U64 TagBase(TranslatorVisitor& v, Reg Rn) { + return Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)}; +} + +static void TagSetBase(TranslatorVisitor& v, Reg Rn, IR::U64 value) { + if (Rn == Reg::SP) { + v.SP(64, value); + } else { + v.X(64, Rn, value); + } +} + +bool TranslatorVisitor::STG_family(Imm<2> opc, Imm<9> imm9, Imm<2> mode, Reg Rn, Reg Rt) { + const size_t opcv = opc.ZeroExtend<size_t>(); + const size_t modev = mode.ZeroExtend<size_t>(); + if (modev == 0b00) { + // LDG (opc 01) leaves Xt as-is since tags are 0; STZGM/STGM/LDGM are EL1-only. + if (opcv == 0b01) { + return true; + } + return UnallocatedEncoding(); + } + + const s64 offset = imm9.SignExtend<s64>() << 4; + const IR::U64 base = TagBase(*this, Rn); + const IR::U64 offset_address = ir.Add(base, ir.Imm64(static_cast<u64>(offset))); + const IR::U64 address = modev == 0b01 ? base : offset_address; // 01 = post-index + + const bool zero = (opcv & 1) != 0; // STZG, STZ2G + const size_t granules = (opcv & 2) ? 2 : 1; // ST2G, STZ2G + if (zero) { + const IR::U64 zero64 = ir.Imm64(0); + for (size_t i = 0; i < granules * 2; ++i) { + Mem(ir.Add(address, ir.Imm64(i * 8)), 8, IR::AccType::NORMAL, zero64); + } + } + (void)Rt; + if (modev != 0b10) { + TagSetBase(*this, Rn, offset_address); + } + return true; +} + +bool TranslatorVisitor::STGP(Imm<2> mode, Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt) { + const size_t modev = mode.ZeroExtend<size_t>(); + if (modev == 0b00) { + return UnallocatedEncoding(); + } + const s64 offset = imm7.SignExtend<s64>() << 4; + const IR::U64 base = TagBase(*this, Rn); + const IR::U64 offset_address = ir.Add(base, ir.Imm64(static_cast<u64>(offset))); + const IR::U64 address = modev == 0b01 ? base : offset_address; + Mem(address, 8, IR::AccType::NORMAL, X(64, Rt)); + Mem(ir.Add(address, ir.Imm64(8)), 8, IR::AccType::NORMAL, X(64, Rt2)); + if (modev != 0b10) { + TagSetBase(*this, Rn, offset_address); + } + return true; +} + +bool TranslatorVisitor::IRG(Reg Rm, Reg Rn, Reg Rd) { + (void)Rm; + TagSetBase(*this, Rd, TagBase(*this, Rn)); + return true; +} + +bool TranslatorVisitor::GMI(Reg Rm, Reg Rn, Reg Rd) { + (void)Rn; + // Tag of Xn is always 0, so the exclusion mask gains bit 0. + X(64, Rd, ir.Or(X(64, Rm), ir.Imm64(1))); + return true; +} + +bool TranslatorVisitor::ADDG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) { + (void)tag_offset; + const u64 offset = offset_imm.ZeroExtend<u64>() << 4; + TagSetBase(*this, Rd, ir.Add(TagBase(*this, Rn), ir.Imm64(offset))); + return true; +} + +bool TranslatorVisitor::SUBG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) { + (void)tag_offset; + const u64 offset = offset_imm.ZeroExtend<u64>() << 4; + TagSetBase(*this, Rd, ir.Sub(TagBase(*this, Rn), ir.Imm64(offset))); + return true; +} + +} // namespace Dynarmic::A64 diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp new file mode 100644 index 0000000000..e36942c942 --- /dev/null +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// ARMv8.3 pointer authentication, implemented as the identity: PAC* never adds a code and +// AUT*/XPAC* have nothing to strip, so signed pointers are just plain pointers. The HINT-space +// forms (PACIASP, AUTIBSP, ...) already decode as HINT/NOP. + +#include "dynarmic/frontend/A64/translate/impl/impl.h" + +namespace Dynarmic::A64 { + +bool TranslatorVisitor::PACAUT_reg(bool Z, Imm<3> op, Reg Rn, Reg Rd) { + (void)op; + (void)Rd; + if (Z && Rn != Reg::R31) { + return UnallocatedEncoding(); + } + return true; +} + +bool TranslatorVisitor::XPAC_1(bool D, Reg Rd) { + (void)D; + (void)Rd; + return true; +} + +bool TranslatorVisitor::PACGA(Reg Rm, Reg Rn, Reg Rd) { + // Any deterministic value works, since the code is only ever compared to a recomputation. + const IR::U64 m = Rm == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rm)}; + X(64, Rd, ir.And(ir.Eor(X(64, Rn), m), ir.Imm64(0xFFFFFFFF00000000ULL))); + return true; +} + +bool TranslatorVisitor::RETA(bool M) { + (void)M; + ir.SetPC(X(64, Reg::R30)); + ir.SetTerm(IR::Term::PopRSBHint{}); + return false; +} + +bool TranslatorVisitor::BRA_BLRA(bool Z, bool L, bool M, Reg Rn, Reg Rm) { + (void)M; + if (!Z && Rm != Reg::R31) { + return UnallocatedEncoding(); + } + const auto target = X(64, Rn); + if (L) { + X(64, Reg::R30, ir.Imm64(ir.PC() + 4)); + ir.PushRSB(ir.current_location->AdvancePC(4)); + } + ir.SetPC(target); + ir.SetTerm(IR::Term::FastDispatchHint{}); + return false; +} + +bool TranslatorVisitor::LDRA(bool M, bool S, Imm<9> imm9, bool W, Reg Rn, Reg Rt) { + (void)M; + const s64 offset = concatenate(Imm<1>{S ? 1u : 0u}, imm9).SignExtend<s64>() << 3; + const IR::U64 base = Rn == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rn)}; + const IR::U64 address = ir.Add(base, ir.Imm64(static_cast<u64>(offset))); + const IR::U64 data = Mem(address, 8, IR::AccType::NORMAL); + if (W) { + if (Rn == Reg::SP) { + SP(64, address); + } else { + X(64, Rn, address); + } + } + X(64, Rt, data); + return true; +} + +} // namespace Dynarmic::A64 diff --git a/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h b/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h index 05b067bdd8..46fce86bda 100644 --- a/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h +++ b/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h @@ -82,7 +82,7 @@ private: static constexpr VAddr RESERVATION_GRANULE_MASK = 0xFFFF'FFFF'FFFF'FFFFull; static constexpr VAddr INVALID_EXCLUSIVE_ADDRESS = 0xDEAD'DEAD'DEAD'DEADull; - static constexpr size_t MAX_NUM_CPU_CORES = 4; // Sync with src/core/hardware_properties + static constexpr size_t MAX_NUM_CPU_CORES = 6; // Sync with src/core/hardware_properties boost::container::static_vector<VAddr, MAX_NUM_CPU_CORES> exclusive_addresses; boost::container::static_vector<Vector, MAX_NUM_CPU_CORES> exclusive_values; SpinLock lock; diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index cc9ce95d92..39484a1bd5 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -58,6 +58,9 @@ add_library(shader_recompiler STATIC frontend/ir/type.h frontend/ir/value.cpp frontend/ir/value.h + frontend/ampere/lower_to_maxwell.h + frontend/ampere/program_override.h + frontend/ampere/program_address_map.h frontend/maxwell/control_flow.cpp frontend/maxwell/control_flow.h frontend/maxwell/decode.cpp diff --git a/src/shader_recompiler/environment.h b/src/shader_recompiler/environment.h index 184b56ebf3..67486430c8 100644 --- a/src/shader_recompiler/environment.h +++ b/src/shader_recompiler/environment.h @@ -74,6 +74,8 @@ public: [[nodiscard]] virtual u64 ReadInstruction(u32 address) = 0; + [[nodiscard]] virtual u32 TranslateProgramAddress(u32 address) const { return address; } + [[nodiscard]] virtual u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) = 0; [[nodiscard]] virtual TextureType ReadTextureType(u32 raw_handle) = 0; diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h new file mode 100644 index 0000000000..6d17044341 --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -0,0 +1,2249 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +// Encoding reference: Mesa NAK sm70_encode.rs (MIT), Copyright 2022 Collabora, Ltd. +#pragma once + +#include <cstdlib> +#include <algorithm> +#include <array> +#include <bit> +#include <span> +#include <optional> +#include <string_view> +#include <vector> +#include "common/common_types.h" + +namespace Shader::Ampere { + +struct Instruction { + std::array<u32, 4> words; + + u32 Bits(u32 offset, u32 count) const { + u64 bits = words[offset / 32]; + if (offset / 32 + 1 < words.size()) { + bits |= u64{words[offset / 32 + 1]} << 32; + } + return static_cast<u32>((bits >> (offset % 32)) & ((u64{1} << count) - 1)); + } +}; +static_assert(sizeof(Instruction) == 16); + +struct LoweredProgram { + std::vector<u64> instructions; + std::vector<u32> source_instruction_offsets; + u32 source_size{}; + u32 required_texture_buffer = ~0U; + u32 error_offset{}; + std::string_view error; + + explicit operator bool() const { return error.empty(); } +}; + +// Lower only operations whose semantics are representable by the existing +// Maxwell frontend. These are new instructions, not reinterpretations of the +// native bytes. Scheduling/control words are added by the environment adapter. +inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { + LoweredProgram result; + struct Relocation { u32 output_index; s64 target; }; + std::vector<Relocation> relocations; + std::vector<u32> barrier_tokens; + std::vector<u32> call_targets; + std::vector<u32> call_links; + bool main_exited = false; + bool computed_branch = false; // a jump table: the program ends at its self-branch, not an EXIT + bool uses_uniform_bank = false; + for (const auto& instruction : code) { + const u32 op = instruction.Bits(0, 12); + uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 || + op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)); + } + const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; }; + const auto fail = [&](u32 index, std::string_view reason) { + result.instructions.clear(); + result.source_instruction_offsets.clear(); + result.error_offset = index * sizeof(Instruction); + result.error = reason; + return result; + }; + const auto finish = [&](u32 index) { + for (const auto& relocation : relocations) { + if (relocation.target > index) return fail(index, "native branch or call leaves captured program"); + const u32 target = result.source_instruction_offsets[static_cast<size_t>(relocation.target)]; + const s64 relative = s64{address_of(target)} - address_of(relocation.output_index) - 8; + if (relative < -0x800000 || relative > 0x7fffff) return fail(index, "lowered branch distance"); + result.instructions[relocation.output_index] |= (static_cast<u64>(relative) & 0xffffff) << 20; + } + result.source_size = (index + 1) * sizeof(Instruction); + return result; + }; + if (code.empty() || code.size() > 65536) return fail(0, "invalid native program size"); + for (u32 index = 0; index < code.size(); ++index) { + auto insn = code[index]; + result.source_instruction_offsets.push_back(static_cast<u32>(result.instructions.size())); + u32 opcode = insn.Bits(0, 12); + if (insn.Bits(12, 4) == 15) continue; // @!PT: never executes (compiler padding) + std::optional<u32> active_link; + u32 active_entry = 0; + if (main_exited) { + for (size_t call = 0; call < call_targets.size(); ++call) { + if (call_targets[call] <= index && call_targets[call] >= active_entry) { + active_entry = call_targets[call]; + active_link = call_links[call]; + } + } + if (active_link && insn.Bits(16, 8) == *active_link) + return fail(index, "native callee overwrites link register"); + } + // Scratch values and emulated uniform state occupy reserved high GPRs. + // Reject native accesses that could alias that state. + for (u32 bit : {16U, 24U, 32U, 64U}) { + if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue; + const u32 reg = insn.Bits(bit, 8); + if (reg >= (uses_uniform_bank ? 216U : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); + } + // UR0-UR21 live in R216-R237 and UR22-UR27 in R240-R245 (R239 holds the uniform + // predicates), so even/odd pairs stay adjacent. Sites that still check reg < 16 rely on + // UR0-UR15 being R216-R231. + const auto uniform_register = [&](u32 reg) -> u32 { + if (reg < 22) return 216 + reg; + if (reg < 28) return 218 + reg; + if (reg == 63) return 255; + return ~0U; + }; + bool rewritten_uniform = false; + bool uniform_guard = false; + u32 uniform_output = 7; // ULOP3's UPn output: computed in P(6-n), then merged into R239 + if (opcode == 0x2a4 || opcode == 0x8a4 || opcode == 0x891 || opcode == 0x890 || opcode == 0x290 || + opcode == 0x292 || opcode == 0x892 || opcode == 0x291) { + if (insn.Bits(12, 4) != 7 && opcode != 0x292 && opcode != 0x892) + return fail(index, "native uniform arithmetic predication"); + uniform_guard = insn.Bits(12, 4) != 7; + rewritten_uniform = true; + const u32 dst = uniform_register(insn.Bits(16, 8)); + const u32 a = uniform_register(insn.Bits(24, 8)); + const u32 c = uniform_register(insn.Bits(64, 8)); + if (dst == ~0U || a == ~0U || c == ~0U) return fail(index, "native uniform arithmetic register"); + insn.words[0] = (insn.words[0] & 0xffff) | (dst << 16) | (a << 24); + insn.words[2] = (insn.words[2] & ~0x080000ffU) | c; + if (opcode == 0x2a4 || opcode == 0x290 || opcode == 0x292 || opcode == 0x291) { + const u32 b = uniform_register(insn.Bits(32, 8)); + if (b == ~0U) return fail(index, "native uniform arithmetic second register"); + insn.words[1] = (insn.words[1] & ~0xffU) | b; + } + if ((opcode == 0x292 || opcode == 0x892) && insn.Bits(80, 4) != 0xe) { + if (insn.Bits(80, 1)) return fail(index, "native uniform logic predicate output"); + uniform_output = insn.Bits(81, 3); + insn.words[2] = (insn.words[2] & ~(7U << 17)) | ((6 - uniform_output) << 17); + } + opcode = opcode == 0x2a4 ? 0x224 : opcode == 0x8a4 ? 0x824 : opcode == 0x891 ? 0x811 : opcode == 0x890 ? 0x810 + : opcode == 0x292 ? 0x212 : opcode == 0x892 ? 0x812 : opcode == 0x291 ? 0x211 : 0x210; + } + const bool uniform_predication = opcode == 0xabb || opcode == 0xab9 || opcode == 0xc82 || opcode == 0x882; + const u32 native_predicate = insn.Bits(12, 4); + const bool guarded_uniform = (uniform_predication || uniform_guard) && native_predicate != 7; + // A uniform guard UPn lives in R239 bit 6-n and is swapped into P(6-n) for the op. + const u32 guard_slot = 6 - (native_predicate & 7); + u64 predicate = u64{guarded_uniform ? ((native_predicate & 8) | guard_slot) : native_predicate} << 16; + const u64 dest = insn.Bits(16, 8); + const auto base = [&](u64 opcode_bits) { return opcode_bits | predicate | dest; }; + // NVN2 fullscreen vertex shaders read bank 24, which the game never binds (the console reads + // zeros from an unbound bank); Maxwell has 18, so read it from bank 17, also never bound. + const auto bank = [&]() -> u32 { + const u32 value = insn.Bits(54, 5); + return value == 24 ? 17U : value; + }; + const auto cbuf = [&]() -> u64 { + return (u64{insn.Bits(38, 16) / 4} << 20) | (u64{bank()} << 34); + }; + const auto valid_cbuf = [&] { + return !insn.Bits(91, 1) && bank() < 18 && + (insn.Bits(38, 16) % 4) == 0; + }; + std::string_view operand_error; + const auto emit = [&](u64 word) { result.instructions.push_back(word); }; + // P(guard slot) while a uniform guard is swapped in; P6 for lowerings that use it as + // scratch (R2UR, MATCH, HFMA2.RELU), which save and restore it with the defaults below. + const u32 guard_mask = guard_slot < 7 ? 1U << guard_slot : 0U; + const auto save_predicate = [&](u32 mask = 1U << 6) { + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + }; + const auto restore_predicate = [&](u32 mask = 1U << 6) { + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + }; + if (guarded_uniform) { + save_predicate(guard_mask); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{guard_mask} << 20)); + } + const u32 output_mask = uniform_output == 7 ? 0U : 1U << (6 - uniform_output); + if (output_mask) { + // Keep the guest P(6-n) in R254 and start from the current UPn (a false guard keeps it). + emit(0x38e8000000070000ULL | 254 | (u64{255} << 8) | (u64{output_mask} << 20)); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{output_mask} << 20)); + } + // Pd = LUT(a, b, c) over predicate operands (index | negate << 3): each input becomes + // 0 or ~0 (SEL), and LOP3 sets Pd when the result is nonzero. + const auto predicate_lut = [&](u32 lut, u32 a, u32 b, u32 c, u32 target, bool uniform_c = false) { + const u32 inputs[3]{a, b, c}; + for (u32 i = 0; i < 3; ++i) { + if (i == 2 && uniform_c) { + // UPn from R239 bit 6-n as 0 or ~0 (UPT is ~0); a negation flips the table. + if ((c & 7) == 7) { + emit(0x0100000000000000ULL | predicate | 252 | (u64{15} << 12) | (u64{~0U} << 20)); + } else { + emit(0x3800000000000000ULL | predicate | 252 | (u64{239} << 8) | + (u64{(6 - (c & 7)) | (1U << 8)} << 20)); + emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | + (u64{252} << 20) | (u64{1} << 48)); + } + if (c & 8) lut = ((lut & 0x55) << 1) | ((lut & 0xaa) >> 1); + continue; + } + emit(0x38a0000000000000ULL | predicate | (250 + i) | (u64{255} << 8) | + (u64{0x7ffff} << 20) | (u64{1} << 56) | (u64{inputs[i] ^ 8U} << 39)); + } + emit(0x5be0000000000000ULL | predicate | 255 | (u64{250} << 8) | (u64{251} << 20) | + (u64{lut} << 28) | (u64{3} << 36) | (u64{252} << 39) | (u64{target} << 48)); + }; + const auto mov_imm = [&](u32 reg, u32 value) { + emit(0x0100000000000000ULL | predicate | reg | (u64{15} << 12) | (u64{value} << 20)); + }; + const auto materialize = [&](u32 kind, u32 reg, u32 scratch, bool absolute, bool negate) -> u32 { + if (kind == 3) { + const u32 mapped = uniform_register(reg); + if (mapped == ~0U) { operand_error = "native uniform source register"; return 255; } + reg = mapped; + } else if (kind == 1) { + mov_imm(scratch, insn.words[1]); + reg = scratch; + } else if (kind == 2) { + if (!valid_cbuf()) { operand_error = "native constant buffer address"; return 255; } + emit(0x4c98000000000000ULL | predicate | scratch | (u64{15} << 39) | cbuf()); + reg = scratch; + } + if (absolute || negate) { + if (reg != scratch) emit(0x5c98000000000000ULL | predicate | scratch | (u64{reg} << 20) | (u64{15} << 39)); + if (absolute) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x7fffffff} << 20)); + if (negate) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x80000000} << 20) | (u64{2} << 53)); + reg = scratch; + } + return reg; + }; + const auto operand = [&](u32 which, bool floating = true) -> u32 { + const u32 form = insn.Bits(9, 3); + u32 reg{}, kind{}, abs{}, neg{}; + if (which == 0) { + reg = insn.Bits(24, 8); abs = insn.Bits(73, 1); neg = insn.Bits(72, 1); + } else { + if (form != 1 && form != 2 && form != 3 && form != 4 && form != 5 && form != 6 && form != 7) { + operand_error = "native ALU source form"; return 255; + } + const bool wide = which == 1 ? (form == 4 || form == 5 || form == 6) : (form == 2 || form == 3 || form == 7); + if (wide) { + kind = form == 6 || form == 7 ? 3 : (form == 4 || form == 2) ? 1 : 2; + if (kind == 3) reg = insn.Bits(32, 8); + if (kind == 2 || kind == 3) { abs = insn.Bits(62, 1); neg = insn.Bits(63, 1); } + } else { + const bool upper = which == 1 ? (form == 2 || form == 3 || form == 7) : true; + reg = insn.Bits(upper ? 64 : 32, 8); + abs = insn.Bits(upper ? 74 : 62, 1); neg = insn.Bits(upper ? 75 : 63, 1); + } + } + return materialize(kind, reg, 250 + which, floating && abs, floating && neg); + }; + struct HalfSource { u32 reg, swizzle; bool absolute, negate; }; + const auto half_source = [&](u32 which) -> HalfSource { + if (which == 0) return {insn.Bits(24, 8), insn.Bits(74, 2), + insn.Bits(73, 1) != 0, insn.Bits(72, 1) != 0}; + const u32 form = insn.Bits(9, 3); + const bool wide = which == 1 ? (form == 4 || form == 5) : (form == 2 || form == 3); + if (wide) { + const bool immediate = form == 2 || form == 4; + const u32 reg = materialize(immediate ? 1 : 2, 255, 250 + which, false, false); + return {reg, immediate ? 0U : insn.Bits(60, 2), + !immediate && insn.Bits(62, 1), !immediate && insn.Bits(63, 1)}; + } + const bool upper = which == 2 || form == 2 || form == 3; + return {insn.Bits(upper ? 64 : 32, 8), insn.Bits(upper ? 81 : 60, 2), + insn.Bits(upper ? 83 : 62, 1) != 0, insn.Bits(upper ? 84 : 63, 1) != 0}; + }; + const auto extract_half = [&](HalfSource source, u32 lane, u32 target) { + const bool high = source.swizzle == 3 || (source.swizzle == 0 && lane == 1); + emit(0x5ca8000000000000ULL | predicate | target | (u64{2} << 8) | + (u64{source.swizzle == 1 ? 2U : 1U} << 10) | (u64{source.reg} << 20) | + (u64{high} << 41) | (u64{source.negate} << 45) | (u64{source.absolute} << 49)); + }; + const auto xmad = [&](u32 target, u32 a, u32 b, u32 c, bool a_high, + bool b_high, u32 mode, bool immediate) { + emit((immediate ? 0x3600000000000000ULL : 0x5b00000000000000ULL) | + predicate | target | (u64{a} << 8) | + (u64{immediate ? ((b >> (b_high ? 16 : 0)) & 0xffff) : b} << 20) | + (u64{!immediate && b_high} << 35) | (u64{c} << 39) | + (u64{mode} << 50) | (u64{a_high} << 53)); + }; + const auto multiply_low = [&](u32 a, u32 b, u32 c, bool immediate) { + xmad(250, a, b, c, false, false, 0, immediate); + xmad(251, a, b, 255, true, false, 0, immediate); + xmad(251, a, b, 251, false, true, 0, immediate); + emit(base(0x5c18000000000000ULL) | (u64{251} << 8) | + (u64{250} << 20) | (u64{16} << 39)); + }; + switch (opcode) { + case 0x304: { // F2F.F16.F32, scalar round-to-nearest conversion. + if (insn.words[2] != 0x00200800 || insn.Bits(24, 8) || + (insn.words[1] & ~0xc00000ffU)) + return fail(index, "native scalar float conversion mode"); + emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(62, 1)} << 49) | + (u64{insn.Bits(63, 1)} << 45)); + break; + } + case 0x23e: case 0xa3e: { // F2FP.PACK_AB / MERGE_C, round-to-nearest + if (insn.Bits(62, 2) || insn.Bits(73, 5) || insn.Bits(79, 2) || + insn.Bits(81, 10) || insn.Bits(91, 1)) { + return fail(index, "native float packing modifiers"); + } + const bool merge = insn.Bits(78, 1); + if (insn.Bits(72, 1)) return fail(index, "native float packing selector"); + const u32 a = merge ? operand(2, false) : operand(0, false); + const u32 b = operand(1, false); + // Canonical F2F writes the converted half in its low 16 bits. + if (merge) { + emit(0x5c98000000000000ULL | predicate | 250 | (u64{a} << 20) | (u64{15} << 39)); + } else { + emit(0x5ca8000000000000ULL | predicate | 250 | (u64{1} << 8) | + (u64{2} << 10) | (u64{a} << 20)); + } + emit(0x5ca8000000000000ULL | predicate | 251 | (u64{1} << 8) | + (u64{2} << 10) | (u64{b} << 20)); + emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{251} << 20) | (u64{16} << 39)); + break; + } + case 0x232: case 0x832: case 0xa32: { // HMUL2 + if (insn.Bits(78, 2) || insn.Bits(81, 10) || insn.Bits(91, 1)) + return fail(index, "native half multiply mode"); + const auto a = half_source(0), b = half_source(1); + const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1); + emit(base(0x5d08000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{a.negate != b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{precision} << 39) | + (u64{a.absolute} << 44) | (u64{a.swizzle} << 47)); + break; + } + case 0x230: case 0x430: case 0x630: { // HADD2 + if (insn.Bits(76, 1) || insn.Bits(79, 1) || insn.Bits(81, 10) || insn.Bits(91, 1)) + return fail(index, "native half add mode"); + const auto a = half_source(0); + const auto b = half_source(opcode == 0x230 ? 1 : 2); + if (insn.Bits(78, 1)) { + if (a.reg != 255 || !a.negate || a.absolute || a.swizzle != 0 || + b.swizzle < 2 || insn.Bits(77, 1) || insn.Bits(80, 1)) + return fail(index, "native float-output half add mode"); + extract_half(b, 0, static_cast<u32>(dest)); + } else { + emit(base(0x5d10000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{insn.Bits(80, 1)} << 39) | + (u64{a.negate} << 43) | (u64{a.absolute} << 44) | (u64{a.swizzle} << 47)); + } + break; + } + case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: { // HFMA2 + const bool relu = insn.Bits(79, 1) != 0; + if (insn.Bits(78, 1) || insn.Bits(85, 1) || insn.Bits(91, 1) || + (relu ? (insn.Bits(87, 4) != 7 || + insn.Bits(77, 1)) : insn.Bits(87, 4) != 0)) + return fail(index, "native half fused multiply-add mode"); + if (relu && native_predicate != 7) { + // Guard the whole expansion before its temporary P6 updates. + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16)); + predicate = u64{7} << 16; + } + auto a = half_source(0); + auto b = half_source(1); + const auto c = half_source(2); + if (insn.Bits(86, 1)) { // H0_NH1: negate only the upper packed component. + if (b.swizzle != 0 || b.absolute || b.reg >= 250) + return fail(index, "native half mixed-sign source mode"); + emit(0x0400000000000000ULL | predicate | 251 | (u64{b.reg} << 8) | + (u64{0x80000000} << 20) | (u64{2} << 53)); + b.reg = 251; + } + if (b.absolute || c.absolute || (a.absolute && a.reg >= 248)) + return fail(index, "native half fused multiply-add absolute source"); + if (a.absolute) { + emit(0x0400000000000000ULL | predicate | 250 | (u64{a.reg} << 8) | + (u64{a.swizzle == 1 ? 0x7fffffffU : 0x7fff7fffU} << 20)); + a.reg = 250; + a.absolute = false; + } + const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1); + emit(0x5d00000000000000ULL | predicate | (relu ? 254U : dest) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{c.negate} << 30) | (u64{a.negate != b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{c.swizzle} << 35) | + (u64{precision} << 37) | (u64{c.reg} << 39) | (u64{a.swizzle} << 47)); + if (relu) { + // Clamp the rounded binary16 lanes, preserving positive infinity + // and canonicalizing either sign of NaN without saturating to one. + save_predicate(); + for (u32 lane = 0; lane < 2; ++lane) { + emit(0x3828000000070000ULL | 251 | (u64{254} << 8) | (u64{lane * 16} << 20)); + emit(0x0400000000070000ULL | 251 | (u64{251} << 8) | (u64{0xffff} << 20)); + emit(0x0400000000070000ULL | 252 | (u64{251} << 8) | (u64{0x7fff} << 20)); + mov_imm(250, 0x7c00); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39) | (u64{1} << 49)); + emit(0x3828000000070000ULL | 252 | (u64{251} << 8) | (u64{15} << 20)); + mov_imm(250, 1); + emit(0x5c10000000070000ULL | 252 | (u64{252} << 8) | (u64{250} << 20) | (u64{1} << 48)); + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); + mov_imm(250, 0x7fff); + emit(0x5ca0000000070000ULL | 251 | (u64{250} << 8) | (u64{251} << 20) | (u64{6} << 39)); + if (lane == 0) { + emit(base(0x5c98000000000000ULL) | (u64{251} << 20) | (u64{15} << 39)); + } else { + emit(0x3848000000070000ULL | 251 | (u64{251} << 8) | (u64{16} << 20)); + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); + } + } + restore_predicate(); + } + break; + } + case 0x240: case 0x840: { // HMNMX2, exact half promotion for min/max + if (insn.Bits(76, 11) || insn.Bits(91, 1)) + return fail(index, "native half min/max mode"); + const auto a = half_source(0); + const auto b = opcode == 0x840 ? HalfSource{255, 0, false, false} : half_source(1); + if (a.swizzle == 1 || b.swizzle == 1 || (b.reg >= 250 && b.reg != 255)) + return fail(index, "native half min/max source mode"); + for (u32 lane = 0; lane < 2; ++lane) { + extract_half(a, lane, 250); + if (opcode == 0x840) { + const u32 half = (insn.words[1] >> (lane * 16)) & 0xffff; + mov_imm(251, half); + extract_half(HalfSource{251, 2, false, false}, 0, 251); + } else { + extract_half(b, lane, 251); + } + const u32 target = lane == 0 ? 252 : 250; + emit(0x5c60000000000000ULL | predicate | target | (u64{250} << 8) | (u64{251} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x5ca8000000000000ULL | predicate | target | (u64{1} << 8) | + (u64{2} << 10) | (u64{target} << 20)); + } + emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{252} << 20) | (u64{16} << 39)); + break; + } + case 0xc24: case 0xe24: { // IMAD low word with a uniform multiplier or addend. + // 0xe24 keeps the uniform addend in bits 32-37; bit 63 negates it ("RZ, RZ, -UR4"). + const bool negate_c = opcode == 0xe24 && insn.Bits(63, 1) != 0; + if ((insn.words[2] & ~0x080002ffU) != 0x078e0000 || + (insn.words[1] & ~(0xffU | (negate_c ? 0x80000000U : 0U)))) + return fail(index, "native uniform-addend multiply mode"); + const u32 a = operand(0, false), b = operand(1, false); + u32 c = operand(2, false); + if (negate_c) { + emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | (u64{c} << 20) | + (u64{1} << 48)); + c = 252; + } + multiply_low(a, b, c, false); + break; + } + case 0x224: { // IMAD register / zero-product move + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && insn.Bits(75, 1) && + (insn.words[2] & ~0xaffU) == 0x078e0000) { + emit(base(0x5c10000000000000ULL) | (u64{255} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{1} << 48)); + break; + } + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && + (insn.words[2] & ~0xffU) == 0x078e0000) { + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20)); + break; + } + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU)) + return fail(index, "native integer register multiply mode"); + multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false); + break; + } + case 0x2a5: { // UIMAD.WIDE.U32 UR(d):UR(d+1) = URa * URb + URc:UR(c+1) + const u32 a = uniform_register(insn.Bits(24, 8)), b = uniform_register(insn.Bits(32, 8)); + const u32 c = insn.Bits(64, 8); + const u32 c_low = uniform_register(c), c_high = c == 63 ? 255U : uniform_register(c + 1); + const u32 d_low = uniform_register(insn.Bits(16, 8)), d_high = uniform_register(insn.Bits(16, 8) + 1); + if (native_predicate != 7 || (insn.words[1] & ~0xffU) || (insn.words[2] & ~0xffU) != 0x0f8e0000 || + (insn.Bits(16, 8) & 1) || (c != 63 && (c & 1)) || a == ~0U || b == ~0U || c_low == ~0U || + c_high == ~0U || d_low == ~0U || d_high == ~0U || d_low == 255) + return fail(index, "native uniform wide multiply mode"); + // High word in R253 (base-2^16 long multiplication, as IMAD.HI), low word in R250. + xmad(250, a, b, 255, false, false, 0, false); + xmad(251, a, b, 250, true, false, 2, false); + xmad(250, a, b, 251, false, true, 1, false); + xmad(252, a, b, 251, true, true, 2, false); + emit(0x3828000000070000ULL | 250 | (u64{250} << 8) | (u64{16} << 20)); + emit(0x5c10000000070000ULL | 253 | (u64{250} << 8) | (u64{252} << 20)); + xmad(250, a, b, 255, false, false, 0, false); + xmad(251, a, b, 255, true, false, 0, false); + xmad(251, a, b, 251, false, true, 0, false); + emit(0x5c18000000070000ULL | 250 | (u64{251} << 8) | (u64{250} << 20) | (u64{16} << 39)); + emit(0x5c10000000070000ULL | 250 | (u64{250} << 8) | (u64{c_low} << 20) | (u64{1} << 47)); + emit(0x5c10000000070000ULL | d_high | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 43)); + emit(0x5c10000000070000ULL | d_low | (u64{250} << 8) | (u64{255} << 20)); + break; + } + case 0x827: { // IMAD.HI.U32, captured zero addend + if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode"); + const u32 a = insn.Bits(24, 8), b = insn.words[1]; + // Base-2^16 long multiplication. Each intermediate sum fits U32. + xmad(250, a, b, 255, false, false, 0, true); + xmad(251, a, b, 250, true, false, 2, true); + xmad(250, a, b, 251, false, true, 1, true); + xmad(252, a, b, 251, true, true, 2, true); + emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } + case 0xa24: { // IMAD, constant-buffer multiplier + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || !valid_cbuf() || insn.Bits(62, 2)) + return fail(index, "native integer constant-buffer multiply mode"); + emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf()); + multiply_low(insn.Bits(24, 8), 252, insn.Bits(64, 8), false); + break; + } + case 0x424: case 0x624: { // IMAD.MOV.U32: zero product plus immediate/cbuf + if (opcode == 0x624 && insn.Bits(24, 8) == 255 && + (insn.words[2] & ~0x07800000U) == 0x000e06ffU) { + const u32 source = operand(2, false); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(base(0x5c10000000000000ULL) | (u64{source} << 8) | (u64{250} << 20)); + break; + } + if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || insn.words[2] != 0x078e00ff) { + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (opcode == 0x624 && insn.Bits(62, 2))) + return fail(index, "native multiply wide addend mode"); + const u32 a = operand(0, false), b = operand(1, false), c = operand(2, false); + multiply_low(a, b, c, false); + break; + } + if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || + insn.words[2] != 0x078e00ff) + return fail(index, "native integer move mode"); + if (opcode == 0x624) { + if (!valid_cbuf() || insn.Bits(62, 2)) + return fail(index, "native integer move constant buffer"); + emit(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf()); + } else { + emit(base(0x0100000000000000ULL) | (u64{15} << 12) | + (u64{insn.words[1]} << 20)); + } + break; + } + case 0x824: { // Captured low-word IMAD forms; bit 75 negates the addend. + const bool negate_c = insn.Bits(75, 1) != 0; + if (insn.words[1] == 1 && (insn.words[2] & 0xfffff700U) == 0x078e0200) { + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{negate_c} << 48)); + break; + } + const u32 multiplier = insn.words[1]; + if (insn.Bits(64, 8) == 255 && !negate_c && multiplier > 1 && !(multiplier & (multiplier - 1)) && + (insn.words[2] & ~0x2ffU) == 0x078e0000) { + // IMAD.SHL by a power of two: one shift (also the form Eden's jump-table tracking + // expects between the clamp and the table load). + u32 shift = 0; + while ((1U << shift) != multiplier) ++shift; + emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{shift} << 20)); + break; + } + if (insn.words[1] != 48 || insn.words[2] != 0x078e02ff) { + if ((insn.words[2] & ~0xaffU) != 0x078e0000) + return fail(index, "native integer multiply mode"); + u32 c = insn.Bits(64, 8); + if (negate_c && c != 255) { + emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | (u64{c} << 20) | + (u64{1} << 48)); + c = 252; + } + multiply_low(insn.Bits(24, 8), insn.words[1], c, true); + break; + } + const u64 source = insn.Bits(24, 8); + emit(0x5c18000000000000ULL | predicate | 250 | (source << 8) | (source << 20) | (u64{1} << 39)); + emit(base(0x3848000000000000ULL) | (u64{250} << 8) | (u64{4} << 20)); + break; + } + case 0x806: { // VOTE: register ballot, ordinary predicate input and output. + if (insn.words[1] || insn.Bits(72, 2) > 2 || (insn.words[2] & ~0x078e0300U)) + return fail(index, "native vote mode"); + emit(base(0x50d8000000000000ULL) | (u64{insn.Bits(87, 4)} << 39) | + (u64{insn.Bits(81, 3)} << 45) | (u64{insn.Bits(72, 2)} << 48)); + break; + } + case 0x886: { // VOTEU ballot and independent uniform predicate output. + const u32 output = insn.Bits(81, 3), target = insn.Bits(16, 8); + if ((target >= 16 && target != 63) || insn.words[1] || + insn.Bits(72, 2) > 2 || native_predicate != 7) + return fail(index, "native uniform vote mode"); + const u32 temporary = output == 7 ? 7 : 6 - output; + const u32 mask = 1U << temporary; + if (output != 7) emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x50d8000000070000ULL | uniform_register(target) | + (u64{insn.Bits(87, 4)} << 39) | (u64{temporary} << 45) | (u64{insn.Bits(72, 2)} << 48)); + if (output != 7) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + } + break; + } + case 0x3c2: { // R2UR: broadcast the first lane satisfying the execution predicate. + const u32 target = uniform_register(static_cast<u32>(dest)); + if (target == ~0U || target == 255 || insn.words[1] || insn.words[2] != 0x000e0000) + return fail(index, "native register-to-uniform mode"); + save_predicate(); + emit(0x50d8000000070000ULL | 250 | (u64{native_predicate} << 39) | + (u64{6} << 45) | (u64{1} << 48)); + emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); + emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); + emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); + emit(0xef10000000070000ULL | 251 | (u64{insn.Bits(24, 8)} << 8) | + (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5ca0000000070000ULL | target | (u64{251} << 8) | + (u64{target} << 20) | (u64{6} << 39)); + restore_predicate(); + break; + } + case 0x81c: { // PLOP3.LUT; bit 67 makes the third source a uniform predicate. + const u32 target = insn.Bits(81, 3); + const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3); + if (target == 7 || insn.Bits(84, 3) != 7 || + insn.Bits(16, 8) || insn.words[1]) + return fail(index, "native predicate LUT mode"); + if (!insn.Bits(67, 1) && (lut == 0xfe || lut == 0xa8)) { + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | + (u64{insn.Bits(87, 4)} << 12) | (u64{1} << 24) | + (u64{insn.Bits(77, 4)} << 29) | (u64{insn.Bits(68, 4)} << 39) | + (u64{lut == 0xfe} << 45)); + break; + } + if (!insn.Bits(67, 1) && (lut == 0x80 || + (lut == 8 && insn.Bits(68, 4) == 7 && insn.Bits(77, 4) == 7))) { + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | + (u64{insn.Bits(87, 4) ^ (lut == 8 ? 8U : 0U)} << 12) | (u64{insn.Bits(77, 4)} << 29) | + (u64{insn.Bits(68, 4)} << 39)); + break; + } + predicate_lut(lut, insn.Bits(87, 4), insn.Bits(77, 4), insn.Bits(68, 4), target, + insn.Bits(67, 1) != 0); + break; + } + case 0x89c: { // UPLOP3.LUT: load the uniform predicates into P0-P6, combine, merge back. + const u32 target = insn.Bits(81, 3); + const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3); + if (insn.Bits(84, 3) != 7 || insn.Bits(67, 1) || insn.Bits(16, 8) || insn.words[1]) + return fail(index, "native uniform predicate LUT mode"); + if (target == 7) break; + const auto slot = [](u32 operand) { return (operand & 7) == 7 ? operand : (operand & 8) | (6 - (operand & 7)); }; + predicate = u64{slot(native_predicate)} << 16; // a UPn guard is loaded as well + emit(0x38e8000000070000ULL | 254 | (u64{255} << 8) | (u64{0x7f} << 20)); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{0x7f} << 20)); + predicate_lut(lut, slot(insn.Bits(87, 4)), slot(insn.Bits(77, 4)), slot(insn.Bits(68, 4)), 6 - target); + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{1U << (6 - target)} << 20)); + emit(0x38f0000000070000ULL | (u64{254} << 8) | (u64{0x7f} << 20)); + break; + } + case 0x883: { // UP2UR UR, UPR, URZ, mask: bit k of the result is UPk (R239 bit 6-k). + const u32 target = uniform_register(insn.Bits(16, 8)), mask = insn.words[1]; + if (target == ~0U || insn.Bits(24, 8) != 63 || native_predicate != 7 || + insn.words[2] != 0x08000000 || mask > 0x7f) + return fail(index, "native uniform predicate move mode"); + if (target == 255) break; + mov_imm(target, 0); + for (u32 k = 0; k < 7; ++k) { + if (!((mask >> k) & 1)) continue; + emit(0x3800000000070000ULL | 250 | (u64{239} << 8) | (u64{(6 - k) | (1U << 8)} << 20)); + emit(0x36f0000000070000ULL | target | (u64{250} << 8) | (u64{k | (1U << 8)} << 20) | + (u64{target} << 39)); + } + break; + } + case 0x882: { // UMOV immediate. + if (dest >= 16 || insn.words[2]) return fail(index, "native uniform immediate move mode"); + mov_imm(static_cast<u32>(dest) + 216, insn.words[1]); + break; + } + case 0xc82: { // UMOV UR, UR (or URZ), including uniform execution predicates. + const u32 source = insn.words[1] < 64 ? uniform_register(insn.words[1]) : ~0U; + if (dest >= 16 || source == ~0U || insn.words[2] != 0x08000000) + return fail(index, "native uniform move mode"); + emit(0x5c98000000000000ULL | predicate | (dest + 216) | + (u64{source} << 20) | (u64{15} << 39)); + break; + } + case 0xab9: case 0xabb: { // ULDC U32/U64, direct or indexed. + const u32 uniform_dest = insn.Bits(16, 8), index_reg = insn.Bits(24, 8); + const u32 width = insn.Bits(73, 3); + const u32 target = uniform_register(uniform_dest); + const u32 index_target = opcode == 0xabb ? uniform_register(index_reg) : 255U; + if (target == ~0U || target == 255 || (width != 4 && width != 5) || + (width == 5 && ((uniform_dest & 1) || uniform_register(uniform_dest + 1) != target + 1)) || + bank() >= 18 || insn.Bits(38, 16) % 4 || insn.Bits(62, 2) || + (insn.words[2] & ~0xe00U) != (opcode == 0xab9 ? 0U : 0x08000000U) || + index_target == ~0U) + return fail(index, "native uniform constant load mode"); + if (opcode == 0xab9 && width == 4) { + emit(0x4c98000000000000ULL | predicate | target | (u64{15} << 39) | cbuf()); + } else { + emit(0xef90000000000000ULL | predicate | target | + (u64{index_target} << 8) | (u64{insn.Bits(38, 16)} << 20) | + (u64{bank()} << 36) | (u64{width} << 48)); + } + break; + } + case 0x899: { // USHF.L.U32; accept a zero word outside the selected half. + const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8); + if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 && + dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | (dst + 216) | + (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); + break; + } + // Logical right shifts of one register (bit 76 right, bit 80 high word, bits 73-74 + // U32): USHF.R.U32 URd, URa, n, URZ and USHF.R.U32.HI URd, URZ, n, URb. + const bool right_low = insn.words[2] == 0x0800163f && low < 16; + const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 16; + if ((right_low || right_high) && dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | (dst + 216) | + (u64{(right_high ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + break; + } + const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16; + const bool low_result = insn.words[2] == 0x0800063f && low < 16; + if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) + return fail(index, "native uniform shift mode"); + emit(0x3848000000000000ULL | predicate | (dst + 216) | + (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + break; + } + case 0xb82: { // LDC.64 indexed byte offset + const u32 width = insn.Bits(73, 3); + if ((width != 4 && width != 5) || insn.Bits(78, 2) || insn.Bits(91, 1) || + bank() >= 18 || + (width == 5 && ((dest & 1) || dest >= 254))) return fail(index, "native indexed constant load mode"); + emit(base(0xef90000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(38, 16)} << 20) | (u64{bank()} << 36) | (u64{width} << 48)); + break; + } + case 0xf89: { // SHFL with immediate lane and segmentation/clamp mask, any mode. + const u32 lane = insn.Bits(53, 5), mask = insn.Bits(40, 13), mode = insn.Bits(58, 2); + if (insn.words[2] != 0x000e0000 || insn.Bits(32, 8) || insn.Bits(60, 4)) + return fail(index, "native immediate shuffle mode"); + emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{lane} << 20) | (u64{1} << 28) | (u64{1} << 29) | + (u64{mode} << 30) | (u64{mask} << 34) | (u64{7} << 48)); + break; + } + case 0x822: { // FSWZADD: per lane 0 = a+b, 1 = a-b, 2 = b-a, 3 = b (nvdisasm PP/PN/NP/ZP). + const u32 swizzle = insn.words[1]; + if (swizzle > 0xff || insn.words[2] > 255) + return fail(index, "native swizzled add mode"); + // Maxwell's lane codes 1 and 2 are the other way round (1 = b-a, 2 = a-b). + const u32 maxwell = ((swizzle & 0x55) << 1) | ((swizzle & 0xaa) >> 1); + emit(base(0x50f8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{maxwell} << 28)); + break; + } + case 0xb6c: { // TXD ARRAY_2D, four channels and consecutive split destinations. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8); + if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) || + insn.Bits(61, 3) != 5 || binding >= 18 || offset > 0x1fff || + (dest & 1) || dest >= 248 || insn.Bits(64, 8) != dest + 2 || + coordinates >= 248 || derivatives >= 247) + return fail(index, "native array gradient texture mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(base(0xde00000000000000ULL) | (u64{coordinates} << 8) | + (u64{derivatives} << 20) | (u64{3} << 28) | (u64{15} << 31) | + (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); + break; + } + case 0x989: { // SHFL, immediate lane and register clamp/segmentation mask, any mode. + if ((insn.words[1] & 0xf01fffffU) || (insn.words[2] & ~0x000e00ffU)) + return fail(index, "native register-mask shuffle mode"); + emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(53, 5)} << 20) | (u64{1} << 28) | (u64{insn.Bits(58, 2)} << 30) | + (u64{insn.Bits(64, 8)} << 39) | (u64{insn.Bits(81, 3)} << 48)); + break; + } + case 0x589: { // SHFL.IDX, register lane and immediate segmentation mask. + if ((insn.words[2] & ~0x000e0000U) || insn.Bits(53, 11)) + return fail(index, "native shuffle mode"); + emit(0xef10000000000000ULL | predicate | dest | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | + (u64{1} << 29) | (u64{insn.Bits(40, 13)} << 34) | (u64{insn.Bits(81, 3)} << 48)); + break; + } + case 0x98e: { // RED.E.{ADD..XOR}.STRONG.GPU U32 (op in bits 87-89), register address. + if ((insn.words[2] & ~0x0380003fU) == 0x0c10e100 && insn.Bits(24, 8) == 255) { + // [URa + offset]: a uniform 64-bit address with a signed byte offset. + const u32 address = uniform_register(insn.Bits(64, 6)); + const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8; + if (address == ~0U || (address & 1) || offset < -0x80000 || offset > 0x7ffff) + return fail(index, "native global reduction uniform address"); + emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) | (u64{address} << 8) | + (u64{insn.Bits(87, 3)} << 23) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) | + (u64{1} << 48)); + break; + } + if ((insn.words[2] & ~0x03800000U) != 0x0010e100 || insn.Bits(40, 24) || (insn.Bits(24, 8) & 1)) + return fail(index, "native global reduction mode"); + emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(87, 3)} << 23) | (u64{1} << 48)); + break; + } + case 0x2bd: // UFLO.U32, the highest set-bit index (.SH, bit 74: its shift amount). + if (dest >= 16 || insn.words[1] >= 16 || (insn.words[2] & ~0x400U) != 0x080e0000 || native_predicate != 7) + return fail(index, "native uniform leading bit mode"); + emit(0x5c30000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 20) | + (u64{insn.Bits(74, 1)} << 41)); + break; + case 0x2be: // UBREV, as a bit-reversing zero shift like BREV. + if (dest >= 16 || insn.words[1] >= 16 || insn.words[2] != 0x08000000 || native_predicate != 7) + return fail(index, "native uniform bit reverse mode"); + emit(0x3828000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 8) | + (u64{1} << 39) | (u64{1} << 40)); + break; + case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained. + const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3); + if (dest != 255 || source >= 254 || (source & 1) || target >= 6 || + native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200) + return fail(index, "native warp match mode"); + save_predicate(); + emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); + emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); + emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); + emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); + for (u32 word = 0; word < 2; ++word) + emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) | + (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) | + (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49)); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) | + (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49)); + emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45)); + restore_predicate(); + break; + } + case 0x3c4: { // REDUX.SUM (modulo 2^32) or .MAX (unsigned) over the actual active lanes. + const u32 source = insn.Bits(24, 8), target = uniform_register(static_cast<u32>(dest)); + const bool maximum = insn.words[2] == 0x14000; + if (target == ~0U || target == 255 || insn.words[1] || + (insn.words[2] != 0xc000 && !maximum) || native_predicate != 7) + return fail(index, "native warp sum mode"); + emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); + mov_imm(target, 0); + for (u32 lane = 0; lane < 32; ++lane) { + emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) | + (u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20)); + emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20)); + emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48)); + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); + if (maximum) { + emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{15} << 39)); + } else { + emit(0x5c10000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20)); + } + } + break; + } + case 0xf8c: { // ATOMS.POPC.INC.32 with no returned value: one increment per active lane. + const u32 stride = insn.Bits(78, 2); + if (dest != 255 || insn.words[1] || (insn.words[2] & ~0xc000U) != 0x0d00003f) + return fail(index, "native shared increment mode"); + u32 address = insn.Bits(24, 8); + if (stride && address != 255) { + emit(0x3848000000000000ULL | predicate | 251 | (u64{address} << 8) | + (u64{stride + 1} << 20)); + address = 251; + } + mov_imm(250, 1); + emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20)); + break; + } + case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset. + const u32 offset = insn.Bits(40, 24); + if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode"); + emit(0xec00000000000000ULL | predicate | dest | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | + (u64{offset >> 2} << 30)); + break; + } + case 0x3a8: { // ATOMG.E.ADD.STRONG.GPU U32 at a 64-bit register address plus an offset. + const u32 address = insn.Bits(24, 8); + const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8; + if (insn.words[2] != 0x001ee100 || (address & 1) || address >= 246 || + offset < -0x80000 || offset > 0x7ffff) + return fail(index, "native global atomic register mode"); + emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) | + (u64{1} << 48)); + break; + } + case 0x9a8: { // ATOMG.E.ADD.STRONG.GPU U32 through a uniform pointer plus an offset. + const u32 address = uniform_register(insn.Bits(64, 6)); + const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8; + if (insn.Bits(24, 8) != 255 || offset < -0x80000 || offset > 0x7ffff || + (insn.words[2] & ~0x3fU) != 0x081ee140 || address == ~0U || (address & 1)) + return fail(index, "native global atomic mode"); + emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) | + (u64{1} << 48)); + break; + } + case 0xb97: { // SULD.P.2D/2D_ARRAY.IGN, direct descriptor, channel mask at 72-75. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 coordinate = insn.Bits(24, 8); + const u32 channels = insn.Bits(72, 4); + const u32 dimension = insn.Bits(61, 3); // Maxwell's codes: 3 = 2D, 4 = 2D_ARRAY + if ((insn.words[2] & ~0xf00U) != 0x001e0000 || channels == 0 || (dimension != 3 && dimension != 4) || + binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || dest == 255 || + coordinate > (dimension == 4 ? 252U : 253U)) + return fail(index, "native formatted surface load mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native surface descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(base(0xeb00000000000000ULL) | (u64{coordinate} << 8) | + (u64{channels} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + break; + } + case 0xb9b: { // SUST.P.2D/3D.IGN, direct descriptor and all four channels. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8); + const u32 dimension = insn.Bits(61, 3); + if (insn.words[2] != 0x00100f00 || dimension < 3 || dimension > 5 || + binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension >= 4 ? 252U : 253U)) + return fail(index, "native formatted surface store mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native surface descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) | + (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + break; + } + case 0x301: // BREV register. + if (insn.words[2] || insn.words[1] > 255) return fail(index, "native bit reverse mode"); + emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) | + (u64{1} << 39) | (u64{1} << 40)); + break; + case 0x219: case 0xc19: { // U32 register (or uniform, 0xc19) shifts with a zero word outside the selected result. + const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8); + const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0; + // SHF.R.U32 of the low word with a zero high word: a plain logical right shift. + if (right && !high_result && high == 255 && insn.Bits(73, 2) == 3 && shift != ~0U && + insn.words[1] <= 255 && + (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) == 0x00000600U) { + emit(base(0x5c28000000000000ULL) | (u64{low} << 8) | (u64{shift} << 20)); + break; + } + if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 || shift == ~0U || + (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) != 0x00000600U || + (right ? (!high_result || low != 255) : (high_result || high != 255))) + return fail(index, "native register shift mode"); + emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) | + (u64{right ? high : 255U} << 8) | (u64{shift} << 20) | + (u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50)); + break; + } + case 0x300: case 0xd00: { // FLO, including a uniform source. + if ((insn.words[2] & ~(0x600U | (opcode == 0xd00 ? 0x08000000U : 0U))) != 0x000e0000U) + return fail(index, "native leading bit mode"); + const u32 source = operand(1, false); + emit(base(0x5c30000000000000ULL) | (u64{source} << 20) | + (u64{insn.Bits(63, 1)} << 40) | (u64{insn.Bits(74, 1)} << 41) | + (u64{insn.Bits(73, 1)} << 48)); + break; + } + case 0x309: case 0xd09: { // POPC, including a uniform source. + if (insn.words[2] != (opcode == 0xd09 ? 0x08000000U : 0U)) + return fail(index, "native population count mode"); + const u32 source = operand(1, false); + emit(base(0x5c08000000000000ULL) | (u64{source} << 20) | (u64{insn.Bits(63, 1)} << 40)); + break; + } + case 0x918: + if (insn.words[1] || insn.words[2]) return fail(index, "native NOP mode"); + emit(0x50b0000000000f00ULL | predicate); + break; + case 0x992: { // MEMBAR.SC: preserve workgroup/device visibility. + const u32 scope = insn.Bits(76, 3); + if (insn.words[1] || insn.words[2] != (scope << 12) || + (scope != 0 && scope != 2 && scope != 3 && scope != 5)) + return fail(index, "native memory barrier scope"); + const u32 legacy_scope = scope == 0 ? 0 : scope == 2 ? 1 : scope == 3 ? 2 : 3; + emit(0xef98000000000000ULL | predicate | (u64{legacy_scope} << 8)); + break; + } + case 0x9ab: // ERRBAR: wait for prior memory operations on the host backend. + if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode"); + emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); + break; + case 0xb1d: // BAR.SYNC 0, full workgroup. + if (insn.words[1] || insn.words[2] || native_predicate != 7) + return fail(index, "native workgroup barrier mode"); + emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) | + (u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39)); + break; + case 0x388: case 0x984: { // Shared memory load/store with scaled register offsets. + const u32 width = insn.Bits(73, 3), stride = insn.Bits(78, 2); + // LDS [Ra.Xn + URb + offset]: bit 91 adds a uniform register (bits 32-39). + const bool uniform_base = opcode == 0x984 && insn.Bits(91, 1); + const u32 base_register = uniform_base ? uniform_register(insn.Bits(32, 8)) : 255U; + if (width > 6 || (insn.words[2] & ~(uniform_base ? 0x08000000U : 0U)) != ((width << 9) | (stride << 14)) || + (opcode == 0x984 && !uniform_base && insn.Bits(32, 8)) || base_register == ~0U) + return fail(index, "native shared memory addressing mode"); + u32 address = insn.Bits(24, 8); + if (stride && address != 255) { + emit(0x3848000000000000ULL | predicate | 250 | + (u64{address} << 8) | (u64{stride + 1} << 20)); + address = 250; + } + if (base_register != 255) { + emit(0x5c10000000000000ULL | predicate | 250 | (u64{address} << 8) | (u64{base_register} << 20)); + address = 250; + } + const u32 data = opcode == 0x388 ? insn.Bits(32, 8) : static_cast<u32>(dest); + emit((opcode == 0x388 ? 0xef58000000000000ULL : 0xef48000000000000ULL) | + predicate | data | (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{width} << 48)); + break; + } + case 0x23a: { // MOVM.16.MT88: transpose an 8x8 16-bit matrix held one row pair per lane. + // Lane t holds row t/4, columns 2(t%4)..+1; the result is the transposed matrix in the + // same layout, so each half comes from lane 8(t%4) + t/8 (+4) at half (t/4)%2. + const u32 source = insn.Bits(24, 8); + if (insn.words[1] || insn.words[2] || source >= 246 || dest >= 246) + return fail(index, "native matrix move mode"); + emit(0xf0c8000000000000ULL | predicate | 246); // laneid + emit(0x0400000000000000ULL | predicate | 249 | (u64{246} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{3} << 20)); // 8(t%4) + emit(0x3828000000000000ULL | predicate | 247 | (u64{246} << 8) | (u64{3} << 20)); // t/8 + emit(0x5c10000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{247} << 20)); + emit(0xef10000000000000ULL | predicate | 247 | (u64{source} << 8) | (u64{249} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x1c00000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{4} << 20)); + emit(0xef10000000000000ULL | predicate | 248 | (u64{source} << 8) | (u64{249} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x3828000000000000ULL | predicate | 249 | (u64{246} << 8) | (u64{2} << 20)); + emit(0x0400000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{1} << 20)); + emit(0x3848000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{4} << 20)); // 0 or 16 + emit(0x5c28000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{249} << 20)); + emit(0x0400000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{0xffff} << 20)); + emit(0x5c28000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{249} << 20)); + emit(0x5c18000000000000ULL | predicate | dest | (u64{248} << 8) | (u64{247} << 20) | (u64{16} << 39)); + break; + } + case 0x23c: { // HMMA.16816.F16: D(16x8) = A(16x16) * B(16x8) + C, f16 fragments. + // Lane t (group g = t/4, i = t%4): A regs hold rows g / g+8 at k 2i..2i+1 (a0, a1) and + // 8+2i.. (a2, a3); B regs hold k 2i.. (b0) / 8+2i.. (b1) of column g; D holds rows + // g / g+8 at columns 2i, 2i+1. Rebuilt from shuffles with HFMA2, accumulating in R252/R253. + // The expansion is correct but large: title 4's 372-HMMA network takes the host driver + // minutes to compile, so it stays opt-in (the QMD still completes when rejected). + // EDEN_NX2_TENSOR=stub keeps the rest of the program but makes every HMMA D = C. + static const char* const mode = std::getenv("EDEN_NX2_TENSOR"); + if (!mode) return fail(index, "native matrix multiply (set EDEN_NX2_TENSOR to emulate)"); + const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8), c = insn.Bits(64, 8); + if ((insn.words[1] & ~0xffU) || (insn.words[2] & ~0xffU) != 0x800 || dest + 2 > 246 || + a + 4 > 246 || b + 2 > 246 || (c != 255 && c + 2 > 246)) + return fail(index, "native matrix multiply mode"); + if (std::string_view{mode} == "stub") { + for (u32 row = 0; row < 2; ++row) { + if (c == 255) mov_imm(static_cast<u32>(dest) + row, 0); + else if (c != dest) emit(0x5c98000000000000ULL | predicate | (dest + row) | (u64{15} << 39) | (u64{c + row} << 20)); + } + break; + } + const auto shuffle = [&](u32 target, u32 source_reg) { + emit(0xef10000000000000ULL | predicate | target | (u64{source_reg} << 8) | (u64{246} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + }; + const auto lane_a = [&](u32 i) { // 4g + i + emit(0xf0c8000000000000ULL | predicate | 246); + emit(0x0400000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{0x1c} << 20)); + if (i) emit(0x1c00000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{i} << 20)); + }; + const auto lane_b = [&](u32 add) { // 8(t%4) + add: column 2(t%4) (+1 when add >= 4) + emit(0xf0c8000000000000ULL | predicate | 246); + emit(0x0400000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{3} << 20)); + if (add) emit(0x1c00000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{add} << 20)); + }; + const auto hfma2 = [&](u32 accumulator, u32 swizzle_a, u32 pair) { + emit(0x5d00000000000000ULL | predicate | accumulator | (u64{251} << 8) | (u64{pair} << 20) | + (u64{accumulator} << 39) | (u64{swizzle_a} << 47)); + }; + for (u32 row = 0; row < 2; ++row) { + if (c == 255) mov_imm(252 + row, 0); + else emit(0x5c98000000000000ULL | predicate | (252 + row) | (u64{15} << 39) | (u64{c + row} << 20)); + } + for (u32 i = 0; i < 4; ++i) { + for (u32 k_block = 0; k_block < 2; ++k_block) { + lane_b(i); + shuffle(247, b + k_block); + lane_b(4 + i); + shuffle(248, b + k_block); + // P0 = (B[2i][2t], B[2i][2t+1]), P1 = (B[2i+1][2t], B[2i+1][2t+1]) + emit(0x0400000000000000ULL | predicate | 249 | (u64{247} << 8) | (u64{0xffff} << 20)); + emit(0x5c18000000000000ULL | predicate | 249 | (u64{248} << 8) | (u64{249} << 20) | (u64{16} << 39)); + emit(0x3828000000000000ULL | predicate | 250 | (u64{247} << 8) | (u64{16} << 20)); + emit(0x0400000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{0xffff0000} << 20)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{248} << 20)); + for (u32 row = 0; row < 2; ++row) { + lane_a(i); + shuffle(251, a + k_block * 2 + row); + hfma2(252 + row, 2, 249); // A.H0 broadcast * P0 + hfma2(252 + row, 3, 250); // A.H1 broadcast * P1 + } + } + } + for (u32 row = 0; row < 2; ++row) + emit(0x5c98000000000000ULL | predicate | (dest + row) | (u64{15} << 39) | (u64{252 + row} << 20)); + break; + } + case 0x83b: { // LDSM.16.M88.{1,2,4} (ldmatrix): 8x8 16-bit matrices from shared memory. + // Lanes 8m..8m+7 give the row addresses of matrix m; lane t receives the 32 bits at + // row t/4, byte (t%4)*4 of each matrix in Rd+m. Emulated with SHFL.IDX + LDS.32. + const u32 count = 1U << insn.Bits(72, 2); + const bool transpose = insn.Bits(78, 1) != 0; + if (insn.Bits(72, 2) == 3 || (insn.words[2] & ~0x4300U) || insn.Bits(32, 8) || + dest + count > 246) + return fail(index, "native matrix load mode"); + u32 address = insn.Bits(24, 8); + const u64 offset = insn.Bits(40, 24); + if (address != 255 && address >= dest && address < dest + count) { + emit(0x5c10000000000000ULL | predicate | 253 | (u64{address} << 8) | (u64{255} << 20)); + address = 253; + } + if (transpose) { + // .MT88: lane t receives stored rows 2(t%4) and 2(t%4)+1 at column t/4 (16 bits each). + emit(0xf0c8000000000000ULL | predicate | 246); // laneid + emit(0x0400000000000000ULL | predicate | 247 | (u64{246} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{1} << 20)); // row 2(t%4) + emit(0x3828000000000000ULL | predicate | 248 | (u64{246} << 8) | (u64{2} << 20)); + emit(0x3848000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{1} << 20)); // byte 2(t/4) + for (u32 matrix = 0; matrix < count; ++matrix) { + for (u32 half = 0; half < 2; ++half) { + emit(0x1c00000000000000ULL | predicate | 249 | (u64{247} << 8) | + (u64{matrix * 8 + half} << 20)); + emit(0xef10000000000000ULL | predicate | 250 | (u64{address} << 8) | (u64{249} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{248} << 20)); + emit(0xef48000000000000ULL | predicate | (251 + half) | (u64{250} << 8) | + (offset << 20) | (u64{2} << 48)); // LDS.U16 + } + emit(0x5c18000000000000ULL | predicate | (dest + matrix) | (u64{252} << 8) | + (u64{251} << 20) | (u64{16} << 39)); + } + break; + } + emit(0xf0c8000000000000ULL | predicate | 250); // S2R laneid + emit(0x3828000000000000ULL | predicate | 251 | (u64{250} << 8) | (u64{2} << 20)); // row + emit(0x0400000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{2} << 20)); // column bytes + for (u32 matrix = 0; matrix < count; ++matrix) { + u32 lane = 251; + if (matrix) { + emit(0x1c00000000000000ULL | predicate | 252 | (u64{251} << 8) | (u64{matrix * 8} << 20)); + lane = 252; + } + emit(0xef10000000000000ULL | predicate | 253 | (u64{address} << 8) | (u64{lane} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{250} << 20)); + emit(0xef48000000000000ULL | predicate | (dest + matrix) | (u64{253} << 8) | + (offset << 20) | (u64{4} << 48)); + } + break; + } + case 0x919: case 0x9c3: { // S2R / S2UR thread and workgroup identities. + const u32 special = insn.Bits(72, 8); + if (insn.words[1] || insn.words[2] != (special << 8) || + (special != 0 && special != 33 && special != 34 && special != 35 && + special != 37 && special != 38 && special != 39 && + special != 56 && special != 57 && special != 58 && special != 59 && special != 60)) + return fail(index, "native special register"); + const u32 target = opcode == 0x9c3 ? uniform_register(static_cast<u32>(dest)) : static_cast<u32>(dest); + if (target == ~0U) return fail(index, "native uniform special destination"); + emit(0xf0c8000000000000ULL | predicate | target | (u64{special} << 20)); + break; + } + case 0x28c: case 0x88c: { // UISETP, signed (bit 73) or unsigned, with UPT accumulation. + const u32 output = insn.Bits(81, 3); + const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x88c ? 252U : uniform_register(insn.Bits(32, 8)); + if (output > 6 || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 || + (insn.Bits(72, 4) & ~2U) || insn.Bits(12, 4) != 7 || a == ~0U || b == ~0U) + return fail(index, "native uniform comparison mode"); + const u32 temporary = 6 - output, mask = 1U << temporary; + if (opcode == 0x88c) mov_imm(252, insn.words[1]); + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x5b60000000070000ULL | 7 | (u64{temporary} << 3) | + (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{insn.Bits(73, 1)} << 48) | + (u64{insn.Bits(76, 3)} << 49)); + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + break; + } + case 0x287: case 0x887: { // USEL, UP0/UP1; preserve ordinary predicates. + const u32 condition = insn.Bits(87, 3); + const u32 target = uniform_register(static_cast<u32>(dest)); + const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x887 ? 252U : uniform_register(insn.Bits(32, 8)); + if (condition > 6 || target == ~0U || a == ~0U || b == ~0U || + insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x07800000U) != 0x08000000U) + return fail(index, "native uniform select mode"); + if (opcode == 0x887 && index > 0 && condition == 0 && insn.words[1] <= 0x7ffff && + code[index - 1].Bits(0, 12) == 0x88c && + code[index - 1].Bits(24, 8) == insn.Bits(24, 8) && + code[index - 1].words[1] == insn.words[1] && code[index - 1].words[2] == 0x0bf01070 && + insn.words[2] == 0x08000000) { + // Preserve the unsigned clamp idiom for indirect-table analysis. + emit(0x3820000000070000ULL | target | (u64{a} << 8) | + (u64{insn.words[1]} << 20) | (u64{7} << 39)); + break; + } + if (opcode == 0x887) mov_imm(252, insn.words[1]); + const u32 temporary = 6 - condition, mask = 1U << temporary; + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x5ca0000000070000ULL | target | (u64{a} << 8) | (u64{b} << 20) | + (u64{temporary} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + break; + } + case 0x381: case 0x981: { // LDG.E of 8 to 128 bits, register or uniform 64-bit address. + // Sizes as Maxwell's: U8, S8, U16, S16, 32, 64, 128. + const u32 size = insn.Bits(73, 3); + u32 address = insn.Bits(24, 8); + if (size > 6 || (opcode == 0x381 && insn.Bits(32, 8)) || + (size == 5 && (dest & 1)) || (size == 6 && (dest & 3))) + return fail(index, "native global load mode"); + if (opcode == 0x381) { + // Bit 77 (strong/scoped ordering) needs nothing extra from a Maxwell LDG. + if ((insn.words[2] & ~0x2e00U) != 0x001e8100 || (address & 1) || address >= 246) + return fail(index, "native global load mode"); + } else { + // The uniform base sits where a register source would (bits 32-37). + const u32 uniform_address = + insn.Bits(38, 2) ? ~0U : uniform_register(insn.Bits(32, 6)); + if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100 || + uniform_address == ~0U || (uniform_address & 1)) + return fail(index, "native global load uniform addressing mode"); + if (address == 255) { + address = uniform_address; + } else { + if (insn.Bits(90, 1) && ((address & 1) || address >= 246)) + return fail(index, "native global load register address pair"); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) | + (u64{address} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) | + (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43)); + address = 250; + } + } + emit(base(0xeed0000000000000ULL) | (u64{address} << 8) | + (u64{insn.Bits(40, 24)} << 20) | (u64{1} << 45) | (u64{size} << 48)); + break; + } + case 0x386: case 0x986: { // Global store of 32/64/128 bits, 64-bit address. + u32 data = insn.Bits(32, 8); + const u32 size = insn.Bits(73, 3); + if (size > 6 || + (data != 255 && ((size == 5 && (data & 1)) || (size == 6 && (data & 3)) || + data + (size < 4 ? 1U : 1U << (size - 4)) > 246))) + return fail(index, "native global store size"); + if (data == 255 && size > 4) { + // Zeros wider than one register: store from zeroed scratch registers (aligned to the + // access: R248-R251 for 128 bits, R250-R251 for 64). + data = size == 6 ? 248 : 250; + for (u32 i = 0; i < (1U << (size - 4)); ++i) mov_imm(data + i, 0); + } + u32 address = insn.Bits(24, 8); + bool extended = insn.Bits(72, 1) != 0; + if (opcode == 0x986) { + const u32 uniform_address = uniform_register(insn.Bits(64, 6)); + if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 || + uniform_address == ~0U || (uniform_address & 1) || !extended) + return fail(index, "native global store uniform addressing mode"); + if (address == 255) address = uniform_address; + else { + if (insn.Bits(90, 1) && ((address & 1) || address >= 254)) + return fail(index, "native global store register address pair"); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) | + (u64{address} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) | + (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43)); + address = 250; + } + } else if ((insn.words[2] & ~0xe00U) != 0x0010a100) { + return fail(index, "native global store register addressing mode"); + } + emit(0xeed8000000000000ULL | predicate | data | + (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{extended} << 45) | (u64{size} << 48)); + break; + } + case 0x95b: // Fragment discard; execution predicate is preserved. + if (insn.words[1] || insn.words[2] != 0x03800000) + return fail(index, "native fragment discard mode"); + emit(0xe33000000000000fULL | predicate); + break; + case 0x305: case 0xb05: case 0xd05: { // F2I NTZ, register/constant-buffer/uniform source + const u32 mode = insn.words[2] & ~(opcode == 0xd05 ? 0x08000000U : 0U); + const bool half = opcode == 0x305 && mode == 0x0010f000; + const bool small = mode == 0x00212800; + const bool floor = (mode & ~0x10100U) == 0x00207000; + const bool nearest = (mode & ~0x10100U) == 0x00203000; + if ((!half && !small && !floor && !nearest && (mode & ~0x100U) != 0x0021f000) || + (opcode != 0xb05 && (insn.words[1] & 0xffffff00U)) || + (opcode == 0xd05 && uniform_register(insn.Bits(32, 8)) == ~0U)) { + return fail(index, "native float-to-integer conversion mode"); + } + const bool is_signed = insn.Bits(72, 1); + u32 source = opcode == 0xb05 ? materialize(2, 0, 252, false, false) + : opcode == 0xd05 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (half) { + extract_half({source, 2, false, false}, 0, 252); + source = 252; + } + if (!is_signed) { + // Clear NaNs without consuming a guest predicate. Clamp the + // conversion below 2^32, retaining an overflow mask for saturation. + emit(0x5800000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{source} << 20) | (u64{7} << 39) | (u64{7} << 48)); + emit(0x5c40000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{250} << 20) | (u64{7} << 48)); + if (!small) { + mov_imm(252, 0x4f800000); + emit(0x5800000000000000ULL | predicate | 251 | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39) | (u64{6} << 48)); + mov_imm(252, 0x4f7fffff); + emit(0x5c60000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39)); + } + source = 250; + } + emit(base(0x5cb0000000000000ULL) | (u64{small ? 1U : 2U} << 8) | (u64{2} << 10) | + (u64{is_signed} << 12) | (u64{source} << 20) | + (u64{small || nearest ? 0U : floor ? 1U : 3U} << 39) | (u64{insn.Bits(80, 1)} << 44)); + if (!is_signed && !small) { + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); + } + break; + } + case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs. + if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) == 0 || insn.Bits(72, 4) > 3 || + insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 || + insn.Bits(40, 14) > 0x1fff || (insn.Bits(72, 4) == 3 && (dest & 1)) || dest >= 254) + return fail(index, "native texture dimension query mode"); + result.required_texture_buffer = 2; + emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | + (u64{1} << 22) | (u64{insn.Bits(72, 4)} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49)); + break; + } + case 0xb66: { // TLD.SCR.LZ, direct texel fetch + const u32 mask = insn.Bits(72, 4); + const u32 dimension = insn.Bits(61, 3); + const bool array = dimension == 5; + const bool volume = dimension == 2; + u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 second_dest = insn.Bits(64, 8); + const bool split = second_dest != 255; + constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; + constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15}; + u32 swizzle = ~0U; + if (split) { + for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i; + } else { + for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; + } + if (!insn.Bits(60, 1) || (dimension != 0 && dimension != 1 && !array && !volume) || + swizzle == ~0U || insn.Bits(76, 5) || + insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 || + insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || + dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || + (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || + (array && ((source_a & 1) || source_a >= 249)) || + (volume && ((source_a & 1) || source_a >= 254))) { + return fail(index, "native texel fetch mode or descriptor binding"); + } + const u32 binding = insn.Bits(54, 5); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + if (array) { + emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20)); + source_b = 250; + } + emit(0xda00000000000000ULL | predicate | dest | (u64{source_a} << 8) | + (u64{source_b} << 20) | (u64{insn.Bits(64, 8)} << 28) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{swizzle} << 50) | (u64{array ? 8U : volume ? 7U : dimension == 0 ? 0U : 2U} << 53)); + break; + } + case 0x91a: // DEPBAR.LE SB5, 1: scoreboard wait, represented by IR data dependencies. + if (insn.words[0] != 0x791a || (insn.words[1] & ~0xfc0U) != 0xd000 || insn.words[2]) { + return fail(index, "native dependency barrier mode"); + } + break; + case 0xb69: { // TMML (LOD query), 2D. Results are 8.8 fixed point, as on Maxwell. + const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 binding = insn.Bits(54, 5); + if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(80, 10) || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || binding >= 18 || mask == 0 || source_a >= 246 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + return fail(index, "native texture LOD query mode"); + result.required_texture_buffer = binding; + // Coordinates: (Ra, Ra+1) when Rb is RZ, else (Ra, Rb). + emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | + (u64{source_b == 255 ? source_a + 1 : source_b} << 20)); + emit(0xdf58000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49)); + const u32 components = static_cast<u32>(std::popcount(mask)); + const bool split = second_dest != 255; + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20)); + } + break; + } + case 0xb63: { // TLD4 (gather), 2D without offsets or depth compare; component at bits 87-88. + const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 binding = insn.Bits(54, 5); + if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || binding >= 18 || insn.Bits(89, 1) || mask == 0 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + return fail(index, "native texture gather mode"); + result.required_texture_buffer = binding; + // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them). + emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | + (u64{source_b == 255 ? 255U : source_b} << 20)); + emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{insn.Bits(87, 2)} << 56)); + const u32 components = static_cast<u32>(std::popcount(mask)); + const bool split = second_dest != 255; + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20)); + } + break; + } + case 0xb60: { // TEX, scalar 2D implicit-LOD sample + const u32 mask = insn.Bits(72, 4); + const u32 second_dest = insn.Bits(64, 8); + constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; + constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15}; + u32 swizzle = ~0U; + const bool split = second_dest != 255; + const bool depth_compare = insn.Bits(78, 1); + const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { + if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || + insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248) + return fail(index, "native array depth comparison sample mode"); + result.required_texture_buffer = 2; + for (u32 i = 0; i < 3; ++i) { + const u32 source = i < 2 ? source_a + i : source_b; + emit(0x5c98000000000000ULL | predicate | (250 + i) | + (u64{15} << 39) | (u64{source} << 20)); + } + emit(0xc000000000000000ULL | predicate | dest | (u64{250} << 8) | + (u64{source_b + 1} << 20) | (u64{3} << 28) | (u64{1} << 31) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{1} << 50) | (u64{7} << 51)); + break; + } + // General form: gather the native operand list (array index first, coordinates, + // then LOD/bias and the depth reference) into Maxwell TEX's coordinate and meta + // registers, sample into scratch and copy the packed components out. Native packs + // up to four values as halves of Ra and Rb (1+1, 2+1, 2+2), five or more as 4+n. + const auto general_tex = [&]() -> bool { + constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 3, 2, 3, 0, 4}; + constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 6, 1, 3, 0, 7}; + const bool offsets = insn.Bits(76, 1) != 0; + if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || (lod > 3 && lod != 6 && lod != 7) || + (depth_compare && mask != 1) || mask == 0) + return false; + const u32 coordinates = coordinate_count[dimension]; + const bool has_lod = lod == 2 || lod == 3 || lod == 6 || lod == 7; + const u32 count = coordinates + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + (depth_compare ? 1 : 0); + const u32 in_a = count <= 4 ? (count + 1) / 2 : 4; + const auto value = [&](u32 i) -> u32 { + if (i < in_a) return source_a == 255 ? 255U : source_a + i; + return source_b == 255 ? 255U : source_b + (i - in_a); + }; + if ((source_a != 255 && source_a + in_a > 246) || + (count > in_a && source_b != 255 && source_b + (count - in_a) > 246)) + return false; + const u32 binding = insn.Bits(54, 5); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return false; + result.required_texture_buffer = binding; + const auto copy = [&](u32 target, u32 source) { + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); + }; + // Coordinates to R250.., then LOD/bias, offsets and reference to R246... + for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i)); + u32 meta = 246; + u32 next = coordinates; + if (has_lod) copy(meta++, value(next++)); + if (offsets) copy(meta++, value(next++)); + if (depth_compare) copy(meta++, value(next++)); + emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{meta == 246 ? 255U : 246U} << 20) | (u64{maxwell_type[dimension]} << 28) | + (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | (u64{7} << 51) | + (u64{offsets} << 54) | (u64{lod} << 55)); + const u32 components = static_cast<u32>(std::popcount(mask)); + if (insn.Bits(79, 1)) { + // FP16 results: (c0, c1) packed into the first destination, (c2, c3) into + // the second (F2F writes the half in the low 16 bits; ISCADD merges). + const auto pack = [&](u32 target, u32 low, u32 high) { + if (target == 255) return; + emit(0x5ca8000000000000ULL | predicate | 246 | (u64{1} << 8) | (u64{2} << 10) | (u64{low} << 20)); + emit(0x5ca8000000000000ULL | predicate | 247 | (u64{1} << 8) | (u64{2} << 10) | (u64{high} << 20)); + emit(0x5c18000000000000ULL | predicate | target | (u64{246} << 8) | (u64{247} << 20) | (u64{16} << 39)); + }; + pack(static_cast<u32>(dest), 250, components > 1 ? 251U : 255U); + if (components > 2) { + pack(split ? second_dest : (dest == 255 ? 255U : static_cast<u32>(dest) + 1), + 252, components > 3 ? 253U : 255U); + } + return true; + } + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else if (i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) copy(target, 250 + i); + } + return true; + }; + if (split) { + for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i; + } else { + for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; + } + // SM70 scalar coordinates and split RG/BA destinations match TEXS. + // Descriptor offsets are words in both encodings, but only native + // explicitly encodes the constant-buffer binding. + if (!insn.Bits(60, 1) || (dimension != 1 && dimension != 5) || + swizzle == ~0U || insn.Bits(76, 2) || + insn.Bits(81, 3) != 7 || (lod > 1 && (lod != 3 || dimension != 1 || depth_compare)) || + (lod == 3 && ((source_a & 1) || source_a >= 249)) || + insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || + (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || + (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) || + (dimension == 5 && ((source_a & 1) || source_a >= 249))) { + if (!general_tex()) return fail(index, "native texture mode or descriptor binding"); + break; + } + result.required_texture_buffer = 2; + emit((0xd800000000000000ULL & ~(u64{insn.Bits(79, 1)} << 59)) | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{swizzle} << 50) | (u64{dimension == 5 ? (lod ? 8U : 7U) : depth_compare ? (lod ? 6U : 4U) : (lod == 3 ? 3U : lod ? 2U : 1U)} << 53)); + break; + } + case 0x816: { // Captured PRMT half-word concatenation + if (insn.words[1] > 0xffff || (insn.words[2] & ~0xffU)) return fail(index, "native permutation mode"); + if (insn.words[1] != 0x5410 && insn.words[1] != 0x5432 && insn.words[1] != 0x7610 && insn.words[1] != 0x7632) { + // Byte i of the result is byte (s & 7) of c:a, or its sign bit repeated when s & 8. + const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8); + if (a >= 250 && a != 255) return fail(index, "native permutation source"); + mov_imm(250, 0); + for (u32 i = 0; i < 4; ++i) { + const u32 select = (insn.words[1] >> (i * 4)) & 0xf; + const u32 source = select & 4 ? c : a, position = (select & 3) * 8; + emit(0x3800000000000000ULL | predicate | 251 | (u64{source} << 8) | + (u64{select & 8 ? (position + 7) | (1U << 8) : position | (8U << 8)} << 20) | + (u64{(select >> 3) & 1} << 48)); + emit(0x36f0000000000000ULL | predicate | 250 | (u64{251} << 8) | + (u64{(i * 8) | (8U << 8)} << 20) | (u64{250} << 39)); + } + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{255} << 20)); + break; + } + const u32 a = insn.Bits(24, 8), b = insn.Bits(64, 8); + if (insn.words[1] != 0x5432 && insn.words[1] != 0x7632) { + emit(0x0400000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{0xffff} << 20)); + } else { + emit(0x3828000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{16} << 20)); + } + if (insn.words[1] == 0x7610 || insn.words[1] == 0x7632) { + emit(0x0400000000000000ULL | predicate | 251 | (u64{b} << 8) | (u64{0xffff0000} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20)); + } else { + emit(base(0x5c18000000000000ULL) | (u64{b} << 8) | (u64{250} << 20) | (u64{16} << 39)); + } + break; + } + case 0x307: { // FRND.F32, round to an integral floating-point value + if (insn.Bits(75, 2) != 2 || insn.Bits(84, 2) != 2 || insn.Bits(81, 3) || + insn.Bits(86, 6)) return fail(index, "native floating round format"); + emit(base(0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) | + (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(63, 1)} << 45) | + (u64{insn.Bits(62, 1)} << 49)); + break; + } + case 0x321: // ALD + case 0x322: { // AST + if (insn.Bits(76, 2) || insn.Bits(79, 1)) return fail(index, "native indexed/patch attribute access"); + const u32 address = insn.Bits(40, 10); + const u32 components = insn.Bits(74, 2) + 1; + if (address % 4 || address + components * 4 > 0x400 || insn.Bits(24, 8) != 255) { + return fail(index, "native attribute address or indexing"); + } + // a[0x5c] has no Maxwell meaning (Ampere uses it for a per-primitive hint such as the + // shading rate). Nothing reads it back; dropping the store keeps full-rate shading. + if (opcode == 0x322 && address == 0x5c && components == 1) break; + u32 value_reg = opcode == 0x321 ? static_cast<u32>(dest) : insn.Bits(32, 8); + if (value_reg == 255) { + if (opcode == 0x321) break; // load into RZ: nothing to do + // AST of RZ writes zeros: store them from zeroed scratch registers. + for (u32 i = 0; i < components; ++i) mov_imm(250 + i, 0); + value_reg = 250; + } + if (value_reg + components > 255) return fail(index, "native attribute register range"); + const u32 vertex = opcode == 0x321 ? insn.Bits(32, 8) : insn.Bits(64, 8); + const u64 translated = (opcode == 0x321 ? 0xefd8000000000000ULL : 0xeff0000000000000ULL) | + predicate | value_reg | (u64{255} << 8) | (u64{address} << 20) | + (u64{vertex} << 39) | (u64{components - 1} << 47); + result.instructions.push_back(translated); + break; + } + case 0x387: case 0x983: { // Captured scalar local spill at an immediate byte offset. + const bool store = opcode == 0x387; + const u32 data = insn.Bits(32, 8), offset = insn.Bits(40, 24); + if (insn.Bits(24, 8) != 255 || (store && (insn.Bits(16, 8) || data > 247)) || + (!store && (data || dest >= 248)) || (offset & 3) || offset > 0x7fffff || + insn.words[2] != (store ? 0x00100800U : 0x00300800U)) + return fail(index, "native local scalar spill mode"); + emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate | + (store ? data : dest) | (u64{255} << 8) | (u64{offset} << 20) | (u64{4} << 48)); + break; + } + case 0x803: { // P2R PR, register merge source, immediate mask, byte select (76-77) + if (insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U)) + return fail(index, "native predicate-to-register mode"); + emit(base(0x38e8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41)); + break; + } + case 0x804: { // R2P PR, register, immediate mask, byte select (76-77) + if (insn.Bits(16, 8) || insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U)) + return fail(index, "native register-to-predicate mode"); + emit(0x38f0000000000000ULL | predicate | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41)); + break; + } + case 0xc02: // MOV uniform register + case 0x202: // MOV register + case 0x802: // MOV immediate + case 0xa02: { // MOV constant buffer + if (insn.Bits(72, 4) != 15 || (insn.Bits(76, 16) & ~(opcode == 0xc02 ? 0x8000U : 0U))) return fail(index, "native MOV modifiers"); + if (opcode == 0x802) { + result.instructions.push_back(base(0x0100000000000000ULL) | (u64{15} << 12) | + (u64{insn.words[1]} << 20)); + } else if (opcode == 0xc02) { + if (insn.words[1] >= 16 || insn.words[2] != 0x08000f00) return fail(index, "native uniform move source"); + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.words[1] + 216} << 20)); + } else if (opcode == 0x202) { + if (insn.Bits(40, 24)) return fail(index, "native MOV register modifiers"); + result.instructions.push_back(base(0x5c98000000000000ULL) | (u64{15} << 39) | + (u64{insn.Bits(32, 8)} << 20)); + } else { + if (!valid_cbuf() || insn.Bits(62, 2)) return fail(index, "native MOV constant buffer"); + result.instructions.push_back(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf()); + } + break; + } + case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: { // FFMA + if (insn.Bits(76, 1) || (insn.Bits(81, 11) & ~(opcode == 0xe23 || opcode == 0xc23 ? 0x400U : 0U))) return fail(index, "native FFMA denormal/source mode"); + const u32 a = operand(0), b = operand(1), c = operand(2); + emit(base(0x5980000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{c} << 39) | + (u64{insn.Bits(77, 1)} << 50) | (u64{insn.Bits(78, 2)} << 51) | (u64{insn.Bits(80, 1)} << 53)); + break; + } + case 0x220: case 0x820: case 0xa20: case 0xc20: { // FMUL + const u32 scale = insn.Bits(84, 3); + if (insn.Bits(76, 1) || scale == 0 || insn.Bits(81, 3) || + insn.Bits(87, 4) || insn.Bits(91, 1) != (opcode == 0xc20) || + (scale != 4 && (!insn.Bits(80, 1) || insn.Bits(78, 2)))) + return fail(index, "native FMUL scale/mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5c68000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(78, 2)} << 39) | (u64{std::array<u32, 8>{0, 3, 2, 1, 0, 6, 5, 4}[scale]} << 41) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50)); + break; + } + case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xe21: { // FADD + if (insn.Bits(81, 10) || insn.Bits(91, 1) != (opcode == 0xe21)) return fail(index, "native FADD mode"); + const u32 a = operand(0), b = operand(insn.Bits(9, 3) == 2 || insn.Bits(9, 3) == 3 || insn.Bits(9, 3) == 7 ? 2 : 1); + emit(base(0x5c58000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50)); + break; + } + case 0x209: case 0x809: case 0xa09: case 0xc09: { // FMNMX, including a uniform source + if (insn.Bits(74, 6) || insn.Bits(81, 6) || (opcode != 0xc09 && insn.Bits(91, 1))) + return fail(index, "native floating min/max mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | + (u64{insn.Bits(80, 1)} << 44)); + break; + } + case 0x308: case 0xb08: { // MUFU + const u32 operation = insn.Bits(74, 6); + if (operation > 8 || operation == 6 || operation == 7 || insn.Bits(80, 12)) return fail(index, "native MUFU mode"); + // SM70+ MUFU.COS/SIN take the angle in turns (compilers multiply by 1/2pi first); + // Eden's Maxwell MUFU takes radians (its RRO is a plain move), so scale by 2pi. + const bool turns = operation == 0 || operation == 1; + const auto to_radians = [&](u32 source) { + emit(0x1e00000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{0x40c90fdbU} << 20)); + return 250U; + }; + if (insn.Bits(73, 1)) { + if (opcode != 0x308 || insn.Bits(60, 2) > 1) + return fail(index, "native half special-function source"); + emit(0x5ca8000000000000ULL | predicate | 250 | (u64{2} << 8) | (u64{1} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(60, 1)} << 41) | + (u64{insn.Bits(63, 1)} << 45) | (u64{insn.Bits(62, 1)} << 49)); + if (turns) to_radians(250); + emit(0x5080000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{operation} << 20)); + emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | (u64{250} << 20)); + } else { + u32 src = operand(1); + if (turns) src = to_radians(src); + emit(base(0x5080000000000000ULL) | (u64{src} << 8) | (u64{operation} << 20)); + } + break; + } + case 0x526: { // IPA.PASS / CONSTANT with the attribute indexed by a uniform register + const u32 index_register = uniform_register(insn.Bits(24, 8)); + if (index_register == ~0U || insn.Bits(32, 8) != 255 || + (insn.Bits(76, 4) != 0 && insn.Bits(76, 4) != 4) || insn.Bits(81, 3) != 7) + return fail(index, "native indexed IPA mode"); + if (index_register == 255) { + emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) | + (u64{insn.Bits(64, 8)} << 30) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + // Maxwell IPA.IDX reads the whole byte address from the index register. + emit(0x1c00000000000000ULL | predicate | 250 | (u64{index_register} << 8) | + (u64{insn.Bits(64, 8) * 4} << 20)); + emit(base(0xe000000000000000ULL) | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(64, 8)} << 30) | (u64{1} << 38) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + case 0x326: { // IPA.PASS / CONSTANT, default or centroid sample location + // Bits 76-77 are the sample mode (title 1 interpolates most varyings at the centroid), + // bit 78 the constant mode; Maxwell keeps them at bits 52-53 and 55. + if (insn.Bits(32, 8) != 255 || insn.Bits(76, 2) > 1 || insn.Bits(79, 1) || + insn.Bits(81, 3) != 7) + return fail(index, "native IPA interpolation mode"); + emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) | (u64{insn.Bits(64, 8)} << 30) | + (u64{insn.Bits(76, 2)} << 52) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + case 0x81a: { // SGXT.U32 Rd, Ra, n: keep the low n bits (BFE.U32 at position 0) + if (insn.words[2] || insn.words[1] > 32 || dest >= 255) + return fail(index, "native sign extend mode"); + emit(base(0x3800000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1] << 8} << 20)); + break; + } + case 0x299: { // USHF by a uniform amount, one register shifted (SHL/SHR, bit 75 wraps). + const u32 dst = uniform_register(insn.Bits(16, 8)), low = uniform_register(insn.Bits(24, 8)); + const u32 shift = uniform_register(insn.Bits(32, 8)), high = uniform_register(insn.Bits(64, 8)); + const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0; + if (dst == ~0U || dst == 255 || low == ~0U || shift == ~0U || high == ~0U || + native_predicate != 7 || insn.Bits(73, 2) != 3 || + (insn.words[2] & ~0x000118ffU) != 0x08000600U) + return fail(index, "native uniform register shift mode"); + u32 source; + if (high_result && low == 255) source = high; // (high:0) >> or << s, high word + else if (!high_result && high == 255) source = low; // (0:low) >> or << s, low word + else return fail(index, "native uniform funnel shift"); + emit((right ? 0x5c28000000000000ULL : 0x5c48000000000000ULL) | predicate | dst | + (u64{source} << 8) | (u64{shift} << 20) | (u64{insn.Bits(75, 1)} << 39)); + break; + } + case 0x419: { // SHF.L.W.U32.HI Rd, RZ, Rb, imm (high word of imm:0 << Rb) = imm << (Rb & 31) + if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0xffU) != 0x00010e00) + return fail(index, "native funnel shift immediate mode"); + mov_imm(250, insn.words[1]); + emit(base(0x5c48000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20) | + (u64{1} << 39)); + break; + } + case 0x805: { // CS2R SRZ + if (insn.Bits(72, 8) != 255 || !insn.Bits(80, 1) || dest >= 249) return fail(index, "native special-register read"); + mov_imm(static_cast<u32>(dest), 0); + mov_imm(static_cast<u32>(dest) + 1, 0); + break; + } + case 0x306: case 0xd06: { // I2F.F32.U8, low byte, round-to-nearest (0xd06: uniform source) + const u32 source = opcode == 0xd06 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (opcode == 0x306 && insn.words[2] == 0x00101000 && !(insn.words[1] & 0xcfffff00U)) { + // I2F.U16 Rd, Rs(.H1): Maxwell's 16-bit source format, byte selector 0 or 2. + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{1} << 10) | (u64{source} << 20) | + (u64{insn.Bits(60, 2) * 2} << 41)); + break; + } + if (insn.words[2] != (opcode == 0xd06 ? 0x08001000U : 0x1000U) || + (insn.words[1] & 0xffffff00U) || source == ~0U) + return fail(index, "native byte-to-float conversion mode"); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{source} << 20)); + break; + } + case 0x245: case 0xa45: case 0xc45: { // I2FP.F32.U32 + // Bits 78-79: rounding (RN/RM/RP/RZ), as Maxwell I2F's bits 39-40. + if ((insn.words[2] & ~(0x400U | 0xc000U | (opcode == 0xc45 ? 0x08000000U : 0U))) != 0x00201000) return fail(index, "native packed integer-to-float format"); + const u32 src = operand(1, false); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | (u64{insn.Bits(74, 1)} << 13) | (u64{src} << 20) | + (u64{insn.Bits(78, 2)} << 39)); + break; + } + case 0x207: case 0x807: case 0xa07: + case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL + const u32 a = operand(0, false), b = operand(1, false); + emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + break; + } + case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75 + if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || insn.Bits(91, 1)) + return fail(index, "native floating comparison output mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(74, 2)} << 45) | + (u64{insn.Bits(76, 4)} << 48) | (u64{1} << 52) | (u64{insn.Bits(80, 1)} << 55)); + break; + } + case 0x434: { // HSETP2 with an immediate half pair: A swizzle 74-75 (neg 72, abs 73), + // combine 69-70, H_AND 71, compare 76-79, FTZ 80, P/Q 81-86, input 87-90. + mov_imm(250, insn.words[1]); + emit(0x5d20000000000000ULL | predicate | insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | + (u64{insn.Bits(80, 1)} << 6) | (u64{insn.Bits(24, 8)} << 8) | (u64{250} << 20) | + (u64{insn.Bits(76, 4)} << 35) | (u64{insn.Bits(87, 3)} << 39) | + (u64{insn.Bits(90, 1)} << 42) | (u64{insn.Bits(72, 1)} << 43) | + (u64{insn.Bits(73, 1)} << 44) | (u64{insn.Bits(69, 2)} << 45) | + (u64{insn.Bits(74, 2)} << 47) | (u64{insn.Bits(71, 1)} << 49)); + break; + } + case 0x20b: case 0x80b: case 0xa0b: // FSETP + case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP + const bool floating = (opcode & 0x1ff) == 0xb; + if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison"); + const u32 a = operand(0, floating), b = operand(1, floating); + const u32 combine = insn.Bits(74, 2); + if (combine > 2) return fail(index, "native predicate combination"); + u64 out = (floating ? 0x5bb0000000000000ULL : 0x5b60000000000000ULL) | predicate | + insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{combine} << 45); + if (floating) out |= (u64{insn.Bits(80, 1)} << 47) | (u64{insn.Bits(76, 4)} << 48); + else out |= (u64{insn.Bits(73, 1)} << 48) | (u64{insn.Bits(76, 3)} << 49); + emit(out); + break; + } + case 0x212: case 0x812: case 0xa12: case 0xc12: { // LOP3.LUT, any table, optional P = (result != 0) + const u32 lut = insn.Bits(72, 8); + const u32 pred_target = insn.Bits(81, 3); + if (insn.Bits(80, 1) || insn.Bits(87, 4) != 15 || + (opcode == 0x212 && insn.Bits(40, 24)) || (opcode == 0xa12 && insn.Bits(62, 2))) + return fail(index, "native LUT/predicate mode"); + u32 b = 255; + if (opcode == 0x212) { + b = insn.Bits(32, 8); + } else if (opcode == 0x812) { + mov_imm(251, insn.words[1]); + b = 251; + } else if (opcode == 0xc12) { + b = uniform_register(insn.Bits(32, 8)); + if (b == ~0U) return fail(index, "native uniform logic source"); + } else { + b = operand(1, false); + } + const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8); + emit(0x5be0000000000000ULL | predicate | dest | (u64{a} << 8) | (u64{b} << 20) | + (u64{lut} << 28) | (u64{pred_target == 7 ? 0U : 3U} << 36) | (u64{c} << 39) | + (u64{pred_target} << 48)); + break; + } + case 0x213: case 0xa13: case 0xc13: { // IABS: max(x, 0 - x), signed. + if (insn.words[2] & ~(opcode == 0xc13 ? 0x08000000U : 0U)) + return fail(index, "native integer absolute mode"); + const u32 source = operand(1, false); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{255} << 8) | (u64{source} << 20) | + (u64{1} << 48)); + emit(base(0x5c20000000000000ULL) | (u64{source} << 8) | (u64{250} << 20) | + (u64{15} << 39) | (u64{1} << 48)); + break; + } + case 0x217: case 0x817: case 0xa17: case 0xc17: { // IMNMX register, immediate, cbuf or uniform operand. + if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 || + (opcode != 0xa17 && insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U)) || + (opcode == 0xa17 && (!valid_cbuf() || insn.Bits(62, 2)))) + return fail(index, "native integer min/max mode"); + const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1]) + : opcode == 0xa17 ? operand(1, false) + : insn.words[1]; + if (source == ~0U) return fail(index, "native uniform min/max operand"); + emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) | + (u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) | + (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48)); + break; + } + case 0x819: { // SHF.HI, immediate shift + const bool is_signed = insn.Bits(73, 2) == 2; + if (!insn.Bits(80, 1) && !insn.Bits(76, 1) && insn.Bits(64, 8) == 255 && + insn.Bits(73, 2) == 3 && !insn.Bits(75, 1) && insn.words[1] <= 32) { + emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20)); + break; + } + if (!insn.Bits(80, 1) && insn.Bits(76, 1) && insn.Bits(64, 8) == 255 && + (insn.Bits(73, 2) == 3 || is_signed) && !insn.Bits(75, 1) && insn.words[1] <= 32) { + // SHF.R low word with a zero high word: a logical right shift. + emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20)); + break; + } + if ((insn.Bits(73, 2) != 3 && !is_signed) || insn.Bits(75, 1) || !insn.Bits(80, 1) || insn.words[1] > 32) return fail(index, "native funnel-shift mode"); + const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8), shift = insn.words[1]; + if (insn.Bits(76, 1)) { + if (low != 255) return fail(index, "native nonzero low right-shift word"); + emit(base(0x3828000000000000ULL) | (u64{high} << 8) | (u64{shift} << 20) | + (u64{is_signed} << 48)); + } else { + if (is_signed) return fail(index, "native signed left funnel shift"); + emit(base(0x36f8000000000000ULL) | (u64{low} << 8) | (u64{high} << 39) | (u64{shift} << 20)); + } + break; + } + case 0xa11: { // LEA with constant base and carry predicate. + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || + insn.Bits(81, 3) == 7 || insn.Bits(62, 2)) + return fail(index, "native constant LEA carry mode"); + const u32 b = operand(1, false); + emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | + (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + break; + } + case 0x211: case 0x811: { // LEA low word, no carry result + if (insn.Bits(80, 1)) { + // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), RZ, s: Rb + high word of (Ra << s); + // bit 73 sign-extends Ra. + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 || + insn.Bits(64, 8) != 255 || (opcode == 0x211 && insn.Bits(62, 2))) + return fail(index, "native high effective address mode"); + u32 b = insn.Bits(32, 8); + if (opcode == 0x811) { + b = insn.words[1] ? 251U : 255U; + if (b == 251) mov_imm(251, insn.words[1]); + } + const u64 shift = (u64{32 - insn.Bits(75, 5)} << 20) | (u64{insn.Bits(73, 1)} << 48); + if (b == 255) { + emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | shift); + } else { + emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | shift); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{b} << 20)); + } + break; + } + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 || + (opcode == 0x211 && insn.Bits(62, 1))) return fail(index, "native LEA carry/high mode"); + // 0x211 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48. + u32 b = insn.Bits(32, 8); + if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; } + emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39) | + (u64{opcode == 0x211 ? insn.Bits(63, 1) : 0U} << 48)); + break; + } + case 0x210: case 0x810: case 0xa10: case 0xc10: { // IADD3 with zero third source and no carry outputs + // 64-bit adds: IADD3 with a carry-out predicate (bits 81-83), then IADD3.X (bit 74) with + // a carry-in predicate (bits 87-89, negate 90). Uniform forms keep UPn in R239 bit 6-n. + if ((opcode == 0x210 || opcode == 0x810) && insn.Bits(64, 8) == 255 && !insn.Bits(73, 1) && + !insn.Bits(75, 2) && insn.Bits(84, 3) == 7 && insn.Bits(77, 4) == 15 && !insn.Bits(72, 1) && + (insn.Bits(81, 3) != 7 || insn.Bits(74, 1))) { + const bool carry_in = insn.Bits(74, 1) != 0; + const u32 carry_out = insn.Bits(81, 3); + const u32 a = insn.Bits(24, 8); + if ((carry_in && carry_out != 7) || (opcode == 0x210 && (insn.words[1] & ~0xffU)) || + (rewritten_uniform && carry_out != 7 && carry_out > 1) || + (rewritten_uniform && carry_in && insn.Bits(87, 3) > 1 && insn.Bits(87, 3) != 7)) + return fail(index, "native integer carry mode"); + if (opcode == 0x810) { + emit(0x1c00000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{insn.words[1]} << 20)); + } else { + emit(0x5c10000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{insn.Bits(32, 8)} << 20)); + } + const auto save = [&](u32 mask) { emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); }; + const auto restore = [&](u32 mask) { emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); }; + if (!carry_in) { + // carry = (a + b) < a, unsigned + const u32 target = rewritten_uniform ? 6 - carry_out : carry_out; + const u32 mask = 1U << target; + if (rewritten_uniform) save(mask); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{250} << 8) | + (u64{a} << 20) | (u64{7} << 39) | (u64{1} << 49)); + if (rewritten_uniform) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + restore(mask); + } + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{250} << 20)); + break; + } + const u32 condition = insn.Bits(87, 3); + const u32 source = rewritten_uniform && condition != 7 ? 6 - condition : condition; + const u32 mask = 1U << source; + mov_imm(252, 1); + if (rewritten_uniform && condition != 7) { + save(mask); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20)); + } + emit(0x5ca0000000000000ULL | predicate | 251 | (u64{252} << 8) | (u64{255} << 20) | + (u64{source} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (rewritten_uniform && condition != 7) restore(mask); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20)); + break; + } + if (opcode == 0xa10 && insn.Bits(81, 3) != 7) { + if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f || + insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1)) + return fail(index, "native constant add carry mode"); + const u32 b = operand(1, false); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | + (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + break; + } + if (opcode == 0xa10 && insn.Bits(74, 1)) { + if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU) + return fail(index, "native extended constant add mode"); + const u32 b = operand(1, false); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20)); + break; + } + // Third source, optionally negated (bit 75; 63 and 72 negate b and a). + if (insn.Bits(64, 8) != 255 && (insn.Bits(73, 4) & ~4U) == 0 && insn.Bits(77, 7) == 0x7f && insn.Bits(84, 7) == 0x7f) { + const u32 b = operand(1, false); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20) | + (u64{insn.Bits(75, 1)} << 48)); + break; + } + if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(77, 7) != 0x7f || insn.Bits(84, 7) != 0x7f) return fail(index, "native integer carry/third source"); + if (opcode == 0xa10 || opcode == 0xc10) { + if (opcode == 0xc10 && (insn.words[1] & ~0x8000003fU)) return fail(index, "native uniform add operand"); + const u32 b = operand(1, false); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20) | + (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + } else if (opcode == 0x210) { + if (insn.words[1] & ~0x800000ffU) return fail(index, "native integer add register modifiers"); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + } else { + emit(base(0x1c00000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20) | (u64{insn.Bits(72, 1)} << 56)); + } + break; + } + case 0x949: case 0x958: { // Captured BRX / BRXU with absolute native-PC table values + if (insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x3ffffU) != (opcode == 0x958 ? 0x0b800000U : 0x03800000U)) + return fail(index, "native computed branch mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + if (s64{index} * 16 + 16 + relative != 0) + return fail(index, "native computed branch relative base"); + const s64 offset = -s64{address_of(static_cast<u32>(result.instructions.size()))} - 8; + if (offset < -0x800000) return fail(index, "native computed branch range"); + const u32 source = opcode == 0x958 ? uniform_register(insn.Bits(24, 8)) : insn.Bits(24, 8); + if (source == ~0U) return fail(index, "native uniform computed branch source"); + emit(0xe25000000000000fULL | predicate | (u64{source} << 8) | + ((static_cast<u64>(offset) & 0xffffff) << 20)); + computed_branch = true; + break; + } + case 0x944: { // Verified CALL.REL.NOINC with R55 holding the return PC + // A predicated NOINC forward transfer can terminate a loop at its + // reconvergence point without a return-link setup or matching RET. + if (native_predicate != 7 && insn.words[1] == 16 && insn.words[2] == 0x03c00000 && + index + 2 < code.size() && code[index + 1].Bits(0, 12) == 0x947) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 2}); + emit(0xe24000000000000fULL | predicate); + break; + } + if (main_exited || index == 0 || insn.words[0] != 0x7944 || + (insn.words[2] & ~0x3ffffU) != 0x03c00000 || + (code[index - 1].words[0] & ~0x00ff0000U) != 0x00007802 || + code[index - 1].Bits(16, 8) >= 250 || + code[index - 1].words[1] != (index + 1) * 16 || + code[index - 1].words[2] != 0xf00) + return fail(index, "native call linkage mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16) return fail(index, "native call destination alignment"); + const u32 link = code[index - 1].Bits(16, 8); + for (size_t call = 0; call < call_targets.size(); ++call) { + if (call_targets[call] == target / 16 && call_links[call] != link) + return fail(index, "native inconsistent call linkage"); + } + call_targets.push_back(static_cast<u32>(target / 16)); + call_links.push_back(link); + relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16}); + emit(0xe26000000007000fULL); + break; + } + case 0x950: { // Verified RET.REL.NODEC epilogue restoring the call link + const u32 return_reg = insn.Bits(24, 8); + // The return register is the link itself, or the last write to it in this function + // is a copy of the link (MOV or IMAD.MOV.U32 Rd, RZ, RZ, Rlink). + const auto copies_link = [&](const Instruction& prior) { + if (prior.Bits(16, 8) != return_reg || prior.Bits(12, 4) != 7) return false; + if ((prior.words[0] & 0xff00ffffU) == 0x00007202U) + return prior.words[1] == *active_link && prior.words[2] == 0xf00; + if ((prior.words[0] & 0xff00ffffU) == 0xff007224U) + return prior.words[1] == 0xff && prior.words[2] == (0x078e0000U | *active_link); + return false; + }; + bool linked = active_link && return_reg == *active_link; + for (u32 prior = index; active_link && !linked && prior-- > active_entry;) { + if (copies_link(code[prior])) linked = true; + else if (code[prior].Bits(16, 8) == return_reg) break; + } + if (!main_exited || !linked || + (insn.words[0] & 0x00ffffffU) != 0x00007950 || return_reg >= 249 || + (insn.words[2] & ~0x3ffffU) != 0x03c00000 || !barrier_tokens.empty()) + return fail(index, "native return linkage mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + if (s64{index} * 16 + 16 + relative != 0) + return fail(index, "native return relative base"); + emit(0xe32000000007000fULL); + bool complete = !computed_branch; + for (u32 target : call_targets) complete &= target <= index; + if (complete) return finish(index); + break; + } + case 0x945: case 0x941: case 0x942: case 0x948: // (+ WARPSYNC) + // BSSY / BSYNC / BREAK only manage warp reconvergence: BSYNC waits for the other + // threads and falls through, BREAK drops the thread from that wait. Eden runs threads + // independently, so all three are no-ops (a SYNC would wrongly jump, and a branch + // out of a lowered SSY region would leave its token on Eden's flow stack). + break; + case 0x947: { // BRA + u64 branch_predicate = predicate; + if (opcode == 0x947 && insn.Bits(87, 4) != 7) { + if (insn.Bits(12, 4) != 7) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{insn.Bits(12, 4) ^ 8U} << 16)); + } + branch_predicate = u64{insn.Bits(87, 4)} << 16; + } else if (insn.Bits(87, 4) != 7) { + return fail(index, "native extra branch condition"); + } + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16) return fail(index, "native branch destination alignment"); + relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16}); + emit(0xe24000000000000fULL | branch_predicate); + if (computed_branch && target == s64{index} * 16 && branch_predicate == (u64{7} << 16)) + return finish(index); // the closing self-branch + break; + } + case 0x94d: { // EXIT + if (insn.Bits(84, 2)) { + return fail(index, "native conditional/special EXIT"); + } + if (insn.Bits(87, 4) != 7) { + // @Pg EXIT Pc: exits when both hold; skip over a Pc-predicated EXIT unless Pg. + if (native_predicate != 7) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16)); + } + emit(0xe30000000000000fULL | (u64{insn.Bits(87, 4)} << 16)); + break; + } + result.instructions.push_back(0xe30000000000000fULL | predicate); + if (insn.Bits(12, 4) != 7) break; + if (!barrier_tokens.empty()) return fail(index, "native unclosed reconvergence scope"); + if (call_targets.empty()) { + const bool pending_branch = std::any_of(relocations.begin(), relocations.end(), + [&](const Relocation& relocation) { return relocation.target > index; }); + if (!pending_branch && !computed_branch) return finish(index); + break; + } + if (main_exited) { + // A library function may end the shader instead of returning (Eden inlines + // calls, so this is an ordinary exit at that point). + bool complete = std::none_of(relocations.begin(), relocations.end(), + [&](const Relocation& relocation) { return relocation.target > index; }); + for (u32 target : call_targets) complete &= target <= index; + if (complete && !computed_branch) return finish(index); + break; + } + for (u32 target : call_targets) { + if (target <= index) return fail(index, "native call target overlaps main program"); + } + main_exited = true; + break; + } + default: + return fail(index, "unsupported native opcode"); + } + if (!operand_error.empty()) return fail(index, operand_error); + if (output_mask) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{output_mask} << 20)); + emit(0x38f0000000070000ULL | (u64{254} << 8) | (u64{output_mask} << 20)); + } + if (guarded_uniform) restore_predicate(guard_mask); + } + return fail(static_cast<u32>(code.size()), main_exited ? "native program lacks captured library return" : "native program has no unconditional EXIT"); +} + +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_address_map.h b/src/shader_recompiler/frontend/ampere/program_address_map.h new file mode 100644 index 0000000000..bee32a821c --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_address_map.h @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once +#include <optional> +#include <span> +#include <vector> +#include "common/common_types.h" + +namespace Shader::Ampere { +constexpr u64 NativeAddressMapMagic = 0x50414d4e4544454eULL; +inline bool HasNativeAddressMap(std::span<const u64> code) { + return code.size() >= 2 && code.back() == NativeAddressMapMagic; +} +inline void AppendNativeAddressMap(std::vector<u64>& code, std::span<const u32> offsets) { + if (offsets.empty()) return; + for (u32 index : offsets) code.push_back(0x50U + index * 8 + ((index + 1) / 3) * 8); + code.push_back(offsets.size()); + code.push_back(NativeAddressMapMagic); +} +inline std::optional<u32> NativeProgramAddress(std::span<const u64> code, u32 address) { + if (!HasNativeAddressMap(code) || address % 16) return {}; + const u64 count = code[code.size() - 2]; + if (!count || count > 65536 || count + 2 > code.size() || address / 16 >= count) return {}; + const size_t start = code.size() - static_cast<size_t>(count) - 2; + const u64 target = code[start + address / 16]; + if (target < 0x50 || target >= start * 8 || target % 8 || target % 32 == 0) return {}; + return static_cast<u32>(target); +} +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_capture.h b/src/shader_recompiler/frontend/ampere/program_capture.h new file mode 100644 index 0000000000..7e308de4ab --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_capture.h @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" + +namespace Shader::Ampere { +class ProgramCaptureState { +public: + enum class Result { Continue, Complete, InvalidTarget }; + static constexpr u32 MaximumInstructions = 65536; + + Result Observe(const Instruction& instruction, u32 index) { + const u32 opcode = instruction.Bits(0, 12); + if (opcode == 0x944 || opcode == 0x947 || opcode == 0x945) { + const u64 raw = instruction.words[1] | (u64{instruction.words[2] & 0x3ffff} << 32); + const s64 relative = opcode == 0x945 ? static_cast<s32>(instruction.words[1]) + : static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16 || target >= MaximumInstructions * 16) + return Result::InvalidTarget; + // CALL.REL.NOINC (bit 86) pushes no return address: the compiler uses it as a jump + // (the lowering treats it as one), so it never pairs with a RET. + const bool noinc_call = opcode == 0x944 && (instruction.words[2] & 0x400000) != 0; + if (opcode == 0x944 && !noinc_call) { + last_call_target = std::max(last_call_target, static_cast<u32>(target / 16)); + } else { + last_forward_target = std::max(last_forward_target, static_cast<u32>(target / 16)); + } + // The compiler ends every program with an unconditional branch to itself; with a jump + // table (whose targets live in a constant buffer) that is the only reliable end. + if (opcode == 0x947 && computed_branch && instruction.Bits(12, 4) == 7 && + target == s64{index} * 16) + return Result::Complete; + } + if (opcode == 0x949 || opcode == 0x958) computed_branch = true; + if (computed_branch) return Result::Continue; + if (opcode == 0x94d && instruction.Bits(12, 4) == 7 && index >= last_forward_target) { + if (!last_call_target) return Result::Complete; + main_exited = true; + } + if (opcode == 0x950 && instruction.Bits(12, 4) == 7 && main_exited && + index >= last_call_target && index >= last_forward_target) + return Result::Complete; + return Result::Continue; + } + +private: + u32 last_call_target{}; + u32 last_forward_target{}; + bool main_exited{}; + bool computed_branch{}; +}; +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_override.h b/src/shader_recompiler/frontend/ampere/program_override.h new file mode 100644 index 0000000000..0943028b0c --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_override.h @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <cstring> +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" + +namespace Shader::Ampere { + +inline u64 NativeProgramKey(std::span<const Instruction> source) { + u64 hash = 14695981039346656037ULL; + for (const auto& instruction : source) { + for (u32 word : instruction.words) { + for (u32 byte = 0; byte < 4; ++byte) hash = (hash ^ ((word >> (byte * 8)) & 255)) * 1099511628211ULL; + } + } + return hash; +} + +// Private iteration format: version, lengths, texture bank, exact original +// instructions, then the newly generated canonical instructions. Source bytes +// are compared directly; the filename hash is only an index. +inline std::vector<u8> EncodeProgramOverride(std::span<const Instruction> source, + const LoweredProgram& program) { + if (!program || program.source_size != source.size_bytes() || program.source_instruction_offsets.size() != source.size()) return {}; + const std::array<u32, 6> header{0x4e454445, 0x3230564f, + static_cast<u32>(source.size_bytes()), static_cast<u32>(program.instructions.size()), + program.required_texture_buffer, static_cast<u32>(program.source_instruction_offsets.size())}; + std::vector<u8> bytes(sizeof(header) + source.size_bytes() + program.instructions.size() * 8 + program.source_instruction_offsets.size() * 4); + std::memcpy(bytes.data(), header.data(), sizeof(header)); + std::memcpy(bytes.data() + sizeof(header), source.data(), source.size_bytes()); + std::memcpy(bytes.data() + sizeof(header) + source.size_bytes(), program.instructions.data(), program.instructions.size() * 8); + std::memcpy(bytes.data() + sizeof(header) + source.size_bytes() + program.instructions.size() * 8, + program.source_instruction_offsets.data(), program.source_instruction_offsets.size() * 4); + return bytes; +} + +inline u32 ProgramOverrideSourceSize(std::span<const u8> bytes) { + std::array<u32, 6> header{}; + if (bytes.size() < sizeof(header)) return 0; + std::memcpy(header.data(), bytes.data(), sizeof(header)); + const bool mapped = header[1] == 0x3230564f; + if (header[0] != 0x4e454445 || (!mapped && header[1] != 0x3130564f) || + (mapped ? header[5] != header[2] / 16 : header[5] != 0) || + header[2] == 0 || header[2] % sizeof(Instruction) || header[2] > 1048576 || + header[3] == 0 || header[3] > 1048576 || (header[4] != ~0U && header[4] >= 18)) return 0; + if (bytes.size() != sizeof(header) + size_t{header[2]} + size_t{header[3]} * 8 + size_t{header[5]} * 4) return 0; + return header[2]; +} + +inline LoweredProgram DecodeProgramOverride(std::span<const u8> bytes, + std::span<const Instruction> source) { + LoweredProgram result; + const auto fail = [&result](std::string_view message) { + result.instructions.clear(); + result.source_instruction_offsets.clear(); + result.error = message; + return result; + }; + std::array<u32, 6> header{}; + if (!ProgramOverrideSourceSize(bytes)) return fail("invalid native override header or length"); + std::memcpy(header.data(), bytes.data(), sizeof(header)); + if (header[2] != source.size_bytes()) return fail("native override source length mismatch"); + if (std::memcmp(bytes.data() + sizeof(header), source.data(), source.size_bytes())) + return fail("native override source mismatch"); + result.instructions.resize(header[3]); + std::memcpy(result.instructions.data(), bytes.data() + sizeof(header) + source.size_bytes(), header[3] * 8); + if (header[5]) { + result.source_instruction_offsets.resize(header[5]); + std::memcpy(result.source_instruction_offsets.data(), + bytes.data() + sizeof(header) + source.size_bytes() + size_t{header[3]} * 8, + size_t{header[5]} * 4); + u32 previous = 0; + for (u32 offset : result.source_instruction_offsets) { + if (offset >= result.instructions.size() || offset < previous) + return fail("native override address map mismatch"); + previous = offset; + } + } + result.source_size = header[2]; + result.required_texture_buffer = header[4]; + return result; +} + +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ir/breadth_first_search.h b/src/shader_recompiler/frontend/ir/breadth_first_search.h index afe153f180..ad19917781 100644 --- a/src/shader_recompiler/frontend/ir/breadth_first_search.h +++ b/src/shader_recompiler/frontend/ir/breadth_first_search.h @@ -6,10 +6,12 @@ #pragma once +#include <algorithm> #include <functional> #include <optional> #include <type_traits> #include <queue> +#include <unordered_set> #include <boost/container/small_vector.hpp> @@ -28,6 +30,23 @@ auto BreadthFirstSearch(Value value, Pred&& pred, Resolve&& resolve = {}) // Breadth-first search visiting the right most arguments first // Small vector has been determined from shaders in Super Smash Bros. Ultimate boost::container::small_vector<const Inst*, 2> visited; + // Once a search grows, a linear scan of `visited` per argument made it quadratic: a 28,000 + // instruction Switch 2 compute shader took 3.4 s to translate, most of it here. Same visit + // order, so the same result. + std::unordered_set<const Inst*> visited_set; + const auto first_visit = [&](const Inst* inst) { + if (!visited_set.empty()) { + return visited_set.insert(inst).second; + } + if (std::ranges::find(visited, inst) != visited.end()) { + return false; + } + visited.push_back(inst); + if (visited.size() > 64) { + visited_set.insert(visited.begin(), visited.end()); + } + return true; + }; std::queue<const Inst*> queue; queue.push(value.InstRecursive()); @@ -47,8 +66,7 @@ auto BreadthFirstSearch(Value value, Pred&& pred, Resolve&& resolve = {}) } // Queue instruction if it hasn't been visited const Inst* const arg_inst{arg_value.InstRecursive()}; - if (std::ranges::find(visited, arg_inst) == visited.end()) { - visited.push_back(arg_inst); + if (first_visit(arg_inst)) { queue.push(arg_inst); } } diff --git a/src/shader_recompiler/frontend/maxwell/control_flow.cpp b/src/shader_recompiler/frontend/maxwell/control_flow.cpp index 7dabfbf16e..9fb30df664 100644 --- a/src/shader_recompiler/frontend/maxwell/control_flow.cpp +++ b/src/shader_recompiler/frontend/maxwell/control_flow.cpp @@ -457,7 +457,7 @@ CFG::AnalysisState CFG::AnalyzeBRX(Block* block, Location pc, Instruction inst, block->indirect_branches.reserve(targets.size()); for (const u32 target : targets) { - Block* const branch{AddLabel(block, block->stack, target, function_id)}; + Block* const branch{AddLabel(block, block->stack, env.TranslateProgramAddress(target), function_id)}; block->indirect_branches.push_back({ .block = branch, .address = target, diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp index 0cc4b3b844..cd4678cef6 100644 --- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp @@ -391,6 +391,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo Optimization::ConstantPropagationPass(env, program); Optimization::PositionPass(env, program); + Optimization::BlitConstantsPass(env, program); Optimization::GlobalMemoryToStorageBufferPass(program, normalized_host_info); Optimization::GeometryCompactionPass(program, normalized_host_info); diff --git a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp index f49d54e7ab..bdb4494a15 100644 --- a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp +++ b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <optional> +#include <unordered_map> #include <boost/container/flat_set.hpp> #include <boost/container/small_vector.hpp> @@ -54,6 +55,11 @@ struct StorageInfo { StorageBufferSet set; StorageInstVector to_replace; StorageWritesSet writes; + /// Track results by the instruction the search starts from, with and without the NVN bias: + /// accesses through the same address share them, and a search can walk thousands of + /// instructions (the IR does not change while storage buffers are collected). + std::unordered_map<const IR::Inst*, std::optional<StorageBufferAddr>> biased_tracks; + std::unordered_map<const IR::Inst*, std::optional<StorageBufferAddr>> unbiased_tracks; }; /// Returns true when the instruction is a global memory instruction @@ -387,10 +393,22 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info) } // First try to find storage buffers in the NVN address const IR::U32 low_addr{low_addr_info->value}; - std::optional<StorageBufferAddr> storage_buffer{Track(low_addr, &nvn_bias)}; + const auto cached_track = [&](const Bias* bias, auto& cache) { + if (low_addr.IsImmediate()) { + return Track(low_addr, bias); + } + const IR::Inst* const start{low_addr.InstRecursive()}; + if (const auto it = cache.find(start); it != cache.end()) { + return it->second; + } + const std::optional<StorageBufferAddr> result{Track(low_addr, bias)}; + cache.emplace(start, result); + return result; + }; + std::optional<StorageBufferAddr> storage_buffer{cached_track(&nvn_bias, info.biased_tracks)}; if (!storage_buffer) { // If it fails, track without a bias - storage_buffer = Track(low_addr, nullptr); + storage_buffer = cached_track(nullptr, info.unbiased_tracks); if (!storage_buffer) { // If that also fails, use NVN fallbacks LOG_WARNING(Shader, "Storage buffer failed to track, using global memory fallbacks"); diff --git a/src/shader_recompiler/ir_opt/passes.h b/src/shader_recompiler/ir_opt/passes.h index bf325a4cd9..e3422deb9d 100644 --- a/src/shader_recompiler/ir_opt/passes.h +++ b/src/shader_recompiler/ir_opt/passes.h @@ -28,6 +28,7 @@ void LowerInt64ToInt32(IR::Program& program); void RescalingPass(IR::Program& program); void SsaRewritePass(IR::Program& program); void PositionPass(Environment& env, IR::Program& program); +void BlitConstantsPass(Environment& env, IR::Program& program); void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo& host_info); void LayerPass(IR::Program& program, const HostTranslateInfo& host_info); void VendorWorkaroundPass(IR::Program& program); diff --git a/src/shader_recompiler/ir_opt/position_pass.cpp b/src/shader_recompiler/ir_opt/position_pass.cpp index 3c20b71897..4890346944 100644 --- a/src/shader_recompiler/ir_opt/position_pass.cpp +++ b/src/shader_recompiler/ir_opt/position_pass.cpp @@ -74,4 +74,70 @@ void PositionPass(Environment& env, IR::Program& program) { } } } +// Switch 2 fullscreen blit vertex shaders read a parameter block from constant bank 24 (lowered to +// bank 17, which games never bind) that the console provides itself. For a blit covering the whole +// render target it holds: texel size (0x0/0x4, again at 0x10/0x14), size (0x8/0xc), a UV scale of +// 1 (0x18/0x1c), the UV clamp rectangle 0,0..1,1 (0x30-0x3c, read by the fragment shader), a zero +// dither bias (0x40-0x48) and the integer source rectangle 0,0..W-1,H-1 (0x50-0x5c). +void BlitConstantsPass(Environment& env, IR::Program& program) { + if (env.ShaderStage() != Stage::VertexB && env.ShaderStage() != Stage::Fragment) { + return; + } + bool used{}; + for (IR::Block* const block : program.post_order_blocks) { + for (IR::Inst& inst : block->Instructions()) { + const IR::Opcode opcode{inst.GetOpcode()}; + if (opcode != IR::Opcode::GetCbufU32 && opcode != IR::Opcode::GetCbufF32 && + opcode != IR::Opcode::GetCbufU32x2) { + continue; + } + if (!inst.Arg(0).IsImmediate() || inst.Arg(0).U32() != 17 || !inst.Arg(1).IsImmediate()) { + continue; + } + IR::IREmitter ir{*block, IR::Block::InstructionList::s_iterator_to(inst)}; + const IR::F32 width{ir.RenderAreaWidth()}; + const IR::F32 height{ir.RenderAreaHeight()}; + const auto word = [&](u32 offset) -> IR::U32 { + switch (offset) { + case 0x0: + case 0x10: + return ir.BitCast<IR::U32>(IR::F32{ir.FPRecip(width)}); + case 0x4: + case 0x14: + return ir.BitCast<IR::U32>(IR::F32{ir.FPRecip(height)}); + case 0x8: + return ir.BitCast<IR::U32>(width); + case 0xc: + return ir.BitCast<IR::U32>(height); + case 0x18: + case 0x1c: + case 0x38: + case 0x3c: + return ir.Imm32(0x3f800000U); + case 0x58: + return IR::U32{ir.ConvertFToU(32, ir.FPAdd(width, ir.Imm32(-1.0f)))}; + case 0x5c: + return IR::U32{ir.ConvertFToU(32, ir.FPAdd(height, ir.Imm32(-1.0f)))}; + default: + return ir.Imm32(0U); + } + }; + const u32 offset{inst.Arg(1).U32()}; + IR::Value value; + if (opcode == IR::Opcode::GetCbufU32x2) { + value = ir.CompositeConstruct(word(offset), word(offset + 4)); + } else if (opcode == IR::Opcode::GetCbufF32) { + value = ir.BitCast<IR::F32>(word(offset)); + } else { + value = word(offset); + } + inst.ReplaceUsesWith(value); + used = true; + } + } + if (used) { + program.info.uses_render_area = true; + } +} + } // namespace Shader::Optimization diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 64d14ab2a6..7f83e0d3c2 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -84,21 +84,31 @@ add_library(video_core STATIC host1x/nvdec.cpp host1x/nvdec.h host1x/nvdec_common.h + host1x/nx2_video_decoder.cpp + host1x/nx2_video_decoder.h host1x/syncpoint_manager.cpp host1x/syncpoint_manager.h host1x/vic.cpp host1x/vic.h macro.cpp macro.h + macro_memo.h + macro_turing.h fence_manager.h gpu.cpp gpu.h + gpu_split.cpp + gpu_split.h gpu_thread.cpp gpu_thread.h guest_memory.h invalidation_accumulator.h memory_manager.cpp memory_manager.h + native_qmd.h + native_query_trace.h + nx2.h + nx2_profile.h present.h pte_kind.h query_cache/bank_base.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f30c85911f..42d7ba066d 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -9,11 +9,15 @@ #include <algorithm> #include <memory> #include <numeric> +#include <stdexcept> +#include <vector> +#include <fmt/ranges.h> #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/nx2_profile.h" #include "video_core/texture_cache/util.h" namespace VideoCommon { @@ -54,21 +58,39 @@ template <class P> BufferCache<P>::~BufferCache() = default; template <class P> -void BufferCache<P>::RunGarbageCollector() { - const bool aggressive_gc = total_used_memory >= critical_memory; +void BufferCache<P>::RunGarbageCollector(bool aggressive_gc) { const u64 ticks_to_destroy = aggressive_gc ? 60 : 120; int num_iterations = aggressive_gc ? 64 : 32; - const auto clean_up = [this, &num_iterations](BufferId buffer_id) { + // Their GPU-written data is downloaded first, all with one wait for the GPU: one wait per + // buffer stalled the GPU thread every frame on title 1 (an 8 GB GPU near its budget, no + // per-fence downloads keeping the buffers clean), ~2.5% of its time. + boost::container::small_vector<BufferId, 64> evicted; + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, [&](BufferId buffer_id) { if (num_iterations == 0) { return true; } --num_iterations; - auto& buffer = slot_buffers[buffer_id]; - DownloadBufferMemory(buffer); - DeleteBuffer(buffer_id); + // Below the critical threshold, buffers holding GPU-written data stay: evicting one + // means downloading it first. The thresholds compare the whole device's use (mostly + // textures) with fixed headroom, so on an 8 GB GPU the collector ran every frame once a + // title 1 course was loaded, and the downloads were ~2.7% of the GPU thread. + const Buffer& buffer = slot_buffers[buffer_id]; + if (!aggressive_gc && gpu_modified_ranges.Intersects(buffer.CpuAddr(), buffer.SizeBytes())) { + return false; + } + evicted.push_back(buffer_id); return false; - }; - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up); + }); + if constexpr (USE_MEMORY_MAPS) { + DownloadBuffersMemory(evicted); + } else { + for (const BufferId buffer_id : evicted) { + DownloadBufferMemory(slot_buffers[buffer_id]); + } + } + for (const BufferId buffer_id : evicted) { + DeleteBuffer(buffer_id); + } } template <class P> @@ -78,6 +100,20 @@ void BufferCache<P>::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (static u64 frames{}; ++frames % 600 == 0) { + u64 total{}; + size_t count{}; + std::vector<std::pair<u64, VAddr>> largest; + for (const auto [buffer_id, buffer] : slot_buffers) { + total += buffer->SizeBytes(); + ++count; + largest.emplace_back(buffer->SizeBytes(), buffer->CpuAddr()); + } + std::partial_sort(largest.begin(), largest.begin() + std::min<size_t>(6, largest.size()), + largest.end(), std::greater<>{}); + LOG_INFO(HW_GPU, "Buffer cache: {} buffers {} MiB; largest {}", count, total >> 20, + fmt::join(largest.begin(), largest.begin() + std::min<size_t>(6, largest.size()), ", ")); + } // Calculate hits and shots and move hit bits to the right const u32 hits = std::reduce(channel_state->uniform_cache_hits.begin(), @@ -100,8 +136,17 @@ void BufferCache<P>::TickFrame() { if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); } - if (total_used_memory >= minimum_memory) { - RunGarbageCollector(); + if (gpu_memory != nullptr && gpu_memory->UsesNativeTiling()) { + // Switch 2 titles fill the device with textures (their collector watches the budget); + // title 1's buffers were ~200 MiB of an 8 GB GPU's ~6.8 GB in use, yet this collector + // ran aggressively every frame on the device-wide figure, evicting up to 64 buffers (and + // downloading the GPU-written ones, ~5% of the GPU thread) that were recreated later. + // Collect when the buffers themselves are large. + if (buffer_bytes >= 1_GiB) { + RunGarbageCollector(buffer_bytes >= 2_GiB); + } + } else if (total_used_memory >= minimum_memory) { + RunGarbageCollector(total_used_memory >= critical_memory); } ++frame_tick; delayed_destruction_ring.Tick(); @@ -234,7 +279,9 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const DAddr diff = base_address - *cpu_src_address; const DAddr new_base_address = *cpu_dest_address + diff; tmp_intervals.push_back({new_base_address, size}); - uncommitted_gpu_modified_ranges.Add(new_base_address, size); + if (AsyncBufferDownloads()) { + uncommitted_gpu_modified_ranges.Add(new_base_address, size); + } }; gpu_modified_ranges.ForEachInRange(*cpu_src_address, amount, mirror); // This subtraction in this order is important for overlapping copies. @@ -649,6 +696,8 @@ void BufferCache<P>::CommitAsyncFlushesHigh() { async_buffers.emplace_back(std::optional<Async_Buffer>{}); return; } + Nx2Profile::Add(Nx2Profile::Counter::DownloadBytes, total_size_bytes); + Nx2Profile::Add(Nx2Profile::Counter::DownloadCopies, downloads.size()); auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes, true); boost::container::small_vector<BufferCopy, 4> normalized_copies; runtime.PreCopyBarrier(); @@ -659,6 +708,19 @@ void BufferCache<P>::CommitAsyncFlushesHigh() { Buffer& buffer = slot_buffers[buffer_id]; second_copy.src_offset = static_cast<size_t>(buffer.CpuAddr()) + copy.src_offset; const DAddr orig_device_addr = static_cast<DAddr>(second_copy.src_offset); + if (Nx2Profile::Enabled()) { + u64 on_dma_pages = 0; + const auto& pages = Nx2Profile::DmaReadPages(); + for (DAddr page = orig_device_addr >> 12; page <= (orig_device_addr + copy.size - 1) >> 12; + ++page) { + if (pages.contains(page)) { + const DAddr begin = (std::max)(orig_device_addr, page << 12); + const DAddr end = (std::min)(orig_device_addr + copy.size, (page + 1) << 12); + on_dma_pages += end - begin; + } + } + Nx2Profile::Add(Nx2Profile::Counter::DownloadBytesOnDmaPages, on_dma_pages); + } async_downloads.Add(orig_device_addr, copy.size); buffer.MarkUsage(copy.src_offset, copy.size); runtime.CopyBuffer(download_staging.buffer, buffer, copies, false); @@ -711,9 +773,9 @@ void BufferCache<P>::PopAsyncBuffers() { template <class P> bool BufferCache<P>::IsRegionGpuModified(DAddr addr, size_t size) { - bool is_dirty = false; - gpu_modified_ranges.ForEachInRange(addr, size, [&](DAddr, DAddr) { is_dirty = true; }); - return is_dirty; + // One lookup instead of visiting every range inside (~4% of the GPU thread for title 1's + // inline uploads once GPU-written ranges are no longer cleared by downloads). + return gpu_modified_ranges.Intersects(addr, size); } template <class P> @@ -1194,8 +1256,15 @@ void BufferCache<P>::BindHostComputeUniformBuffers() { return alignment > 1 && (offset % alignment) != 0; } }(); + // Small buffers the GPU has not written are streamed, as graphics ones are: synchronizing + // them made every dispatch re-protect pages an inline upload had just unprotected (two + // VirtualProtect calls per dispatch, ~4,000 dispatches a second in title 1). + const bool use_fast_buffer = + needs_alignment_stream || + (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size && + !memory_tracker.IsRegionGpuModified(binding.device_addr, size)); if constexpr (!IS_OPENGL) { - if (needs_alignment_stream) { + if (use_fast_buffer) { const std::span<u8> span = runtime.BindMappedUniformBuffer(0, binding_index, size); device_memory.ReadBlockUnsafe(binding.device_addr, span.data(), size); @@ -1372,7 +1441,12 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) { const auto& array = maxwell3d->regs.vertex_streams[index]; const auto& limit = maxwell3d->regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); - const GPUVAddr gpu_addr_end = limit.Address() + 1; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; + const u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0; + if (native && array.enable != 0 && native_size > (std::numeric_limits<u32>::max)()) { + throw std::runtime_error("Native vertex stream exceeds supported 32-bit buffer size"); + } + const GPUVAddr gpu_addr_end = native ? gpu_addr_begin + native_size : limit.Address() + 1; const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin); u32 size = address_size; // TODO: Analyze stride and number of vertices @@ -1529,7 +1603,9 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } memory_tracker.MarkRegionAsGpuModified(device_addr, size); gpu_modified_ranges.Add(device_addr, size); - uncommitted_gpu_modified_ranges.Add(device_addr, size); + if (AsyncBufferDownloads()) { + uncommitted_gpu_modified_ranges.Add(device_addr, size); + } } template <class P> @@ -1710,9 +1786,11 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) { Buffer& buffer = slot_buffers[buffer_id]; const auto size = buffer.SizeBytes(); if (insert) { + buffer_bytes += Common::AlignUp(size, 1024); total_used_memory += Common::AlignUp(size, 1024); buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick)); } else { + buffer_bytes -= Common::AlignUp(size, 1024); total_used_memory -= Common::AlignUp(size, 1024); lru_cache.Free(buffer.getLRUID()); } @@ -1902,6 +1980,52 @@ void BufferCache<P>::DownloadBufferMemory(Buffer& buffer) { DownloadBufferMemory(buffer, buffer.CpuAddr(), buffer.SizeBytes()); } +template <class P> +void BufferCache<P>::DownloadBuffersMemory(std::span<const BufferId> buffer_ids) { + struct Download { + BufferId buffer_id; + BufferCopy copy; + }; + std::vector<Download> downloads; + u64 total_size_bytes = 0; + for (const BufferId buffer_id : buffer_ids) { + Buffer& buffer = slot_buffers[buffer_id]; + const DAddr buffer_addr = buffer.CpuAddr(); + memory_tracker.ForEachDownloadRangeAndClear( + buffer_addr, buffer.SizeBytes(), [&](u64 device_addr_out, u64 range_size) { + gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, + [&](DAddr start, DAddr end) { + downloads.push_back({buffer_id, BufferCopy{ + .src_offset = start - buffer_addr, + .dst_offset = total_size_bytes, + .size = end - start, + }}); + total_size_bytes += Common::AlignUp(end - start, 64); + }); + ClearDownload(device_addr_out, range_size); + gpu_modified_ranges.Subtract(device_addr_out, range_size); + }); + } + if (total_size_bytes == 0) { + return; + } + auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); + for (Download& download : downloads) { + download.copy.dst_offset += download_staging.offset; + Buffer& buffer = slot_buffers[download.buffer_id]; + buffer.MarkUsage(download.copy.src_offset, download.copy.size); + runtime.CopyBuffer(download_staging.buffer, buffer, std::span{&download.copy, 1}, true); + } + runtime.Finish(); + const u8* const mapped_memory = download_staging.mapped_span.data(); + for (const Download& download : downloads) { + const DAddr device_addr = slot_buffers[download.buffer_id].CpuAddr() + download.copy.src_offset; + device_memory.WriteBlockUnsafe( + device_addr, mapped_memory + (download.copy.dst_offset - download_staging.offset), + download.copy.size); + } +} + template <class P> void BufferCache<P>::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 size) { boost::container::small_vector<BufferCopy, 1> copies; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 26bf3dd9ed..45a1836341 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -7,6 +7,7 @@ #pragma once #include <algorithm> +#include <cstdlib> #include <array> #include <bit> #include <functional> @@ -21,6 +22,7 @@ #include <boost/container/small_vector.hpp> #include "common/common_types.h" +#include "common/spin_recursive_mutex.h" #include "common/div_ceil.h" #include "common/literals.h" #include "common/lru_cache.h" @@ -339,7 +341,7 @@ public: } while (channel_state->has_deleted_buffers); } - std::recursive_mutex mutex; + Common::SpinRecursiveMutex mutex; Runtime& runtime; bool any_buffer_uploaded = false; @@ -376,7 +378,7 @@ private: ((device_addr + size) & ~Core::DEVICE_PAGEMASK); } - void RunGarbageCollector(); + void RunGarbageCollector(bool aggressive_gc); void BindHostIndexBuffer(); @@ -466,6 +468,9 @@ private: void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size); + /// Downloads the GPU-written data of several buffers with one wait for the GPU. + void DownloadBuffersMemory(std::span<const BufferId> buffer_ids); + void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false); [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, @@ -513,6 +518,24 @@ private: MemoryTracker memory_tracker; Common::RangeSet<DAddr> uncommitted_gpu_modified_ranges; + + /// Whether GPU-written buffer ranges are copied back to guest memory at the next fence; + /// otherwise only when something flushes them (as upstream yuzu below High GPU accuracy). + /// Not for Switch 2 titles: title 1 had 250-330 MB/s copied back (a fence per compute + /// dispatch, the fencing thread holding the buffer cache while it wrote them), while its CPU + /// read GPU-written memory at most 64 times in 200 s (counted with reactive flushing) and + /// the native MME's DMA reads never touched those pages. Without them a fence costs ~1 us + /// instead of ~5 and draw setup ~7% less. EDEN_NX2_BUFFER_DOWNLOADS=0/1 forces it. + bool AsyncBufferDownloads() const { + static const int forced = [] { + const char* value = std::getenv("EDEN_NX2_BUFFER_DOWNLOADS"); + return value == nullptr ? -1 : (value[0] == '0' ? 0 : 1); + }(); + if (forced >= 0) { + return forced == 1; + } + return gpu_memory == nullptr || !gpu_memory->UsesNativeTiling(); + } Common::RangeSet<DAddr> gpu_modified_ranges; std::deque<Common::RangeSet<DAddr>> committed_gpu_modified_ranges; @@ -537,6 +560,7 @@ private: std::vector<MultiRangeSegment> compute_segments; u64 frame_tick = 0; u64 total_used_memory = 0; + u64 buffer_bytes = 0; ///< The buffers' own size (total_used_memory may be the device's) u64 minimum_memory = 0; u64 critical_memory = 0; BufferId inline_buffer_id; @@ -544,7 +568,8 @@ private: bool immediately_free = false; #endif - std::array<BufferId, ((1ULL << 34) >> CACHING_PAGEBITS)> page_table; + // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits). + std::array<BufferId, ((1ULL << 36) >> CACHING_PAGEBITS)> page_table; Common::ScratchBuffer<u8> tmp_buffer; }; diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h index b8547f3149..8936da023f 100644 --- a/src/video_core/buffer_cache/memory_tracker_base.h +++ b/src/video_core/buffer_cache/memory_tracker_base.h @@ -22,7 +22,7 @@ namespace VideoCommon { template <typename DeviceTracker> class MemoryTrackerBase { - static constexpr size_t MAX_CPU_PAGE_BITS = 34; + static constexpr size_t MAX_CPU_PAGE_BITS = 36; // device addresses (MaxwellDeviceTraits) static constexpr size_t HIGHER_PAGE_BITS = 22; static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS; static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL; diff --git a/src/video_core/buffer_cache/usage_tracker.h b/src/video_core/buffer_cache/usage_tracker.h index 6f84e463e3..3e2da8bc0b 100644 --- a/src/video_core/buffer_cache/usage_tracker.h +++ b/src/video_core/buffer_cache/usage_tracker.h @@ -6,6 +6,10 @@ #pragma once +#include <array> +#include <utility> +#include <vector> + #include "common/alignment.h" #include "common/common_types.h" @@ -24,9 +28,19 @@ public: void Reset() noexcept { std::ranges::fill(pages, 0ULL); + recent.fill({~u64{0}, 0}); } void Track(u64 offset, u64 size) noexcept { + // Tracking is idempotent until Reset: skip ranges tracked just before. Draws bind the + // same (often multi-megabyte) storage buffers again and again, and each Track wrote a + // word per 4 KiB of the range. + for (const auto& [recent_offset, recent_size] : recent) { + if (recent_offset == offset && recent_size == size) { + return; + } + } + recent[recent_cursor++ % recent.size()] = {offset, size}; const size_t page = offset >> BUFFER_PAGE_SHIFT; const size_t page_end = (offset + size) >> BUFFER_PAGE_SHIFT; if (page_end < page || page_end >= pages.size()) { @@ -83,6 +97,9 @@ private: private: std::vector<u64> pages; + /// Ranges tracked last, (offset, size). + std::array<std::pair<u64, u64>, 4> recent{{{~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}}}; + size_t recent_cursor{}; }; } // namespace VideoCommon diff --git a/src/video_core/control/channel_state.cpp b/src/video_core/control/channel_state.cpp index 12a452dfc0..8e0b6c1b9e 100644 --- a/src/video_core/control/channel_state.cpp +++ b/src/video_core/control/channel_state.cpp @@ -13,7 +13,9 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_dma.h" #include "video_core/engines/puller.h" +#include "video_core/macro.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" namespace Tegra::Control { namespace { @@ -21,15 +23,30 @@ namespace { // Match NVK/Nouveau's initial pushbuffer subchannel layout. constexpr u32 Nvk3DSubchannel = 0; constexpr u32 NvkComputeSubchannel = 1; +constexpr u32 NvkM2mfSubchannel = 2; constexpr u32 Nvk2DSubchannel = 3; constexpr u32 NvkCopySubchannel = 4; +// Switch 2 channels of a process share one TSG, whose graphics context holds the macro RAM. +std::shared_ptr<NativeMacroContext> Nx2MacroContext() { + static const auto context = [] { + auto created = std::make_shared<NativeMacroContext>(); + created->engine.native_turing = true; + created->compute_engine.native_turing = true; + return created; + }(); + return context; +} + void BindNvkDefaultSubchannels(ChannelState::Payload& payload) { auto& dma_pusher = payload.dma_pusher; dma_pusher.BindSubchannel(&payload.maxwell_3d, Nvk3DSubchannel, Engines::EngineTypes::Maxwell3D); dma_pusher.BindSubchannel(&payload.kepler_compute, NvkComputeSubchannel, Engines::EngineTypes::KeplerCompute); - // Subchannel 2 is M2MF there; Eden does not expose a 0x9039 engine yet. + // Subchannel 2 is M2MF/inline-to-memory there (NVN uses it without binding a class); + // KeplerMemory has the same LINE_LENGTH_IN/OFFSET_OUT/LAUNCH_DMA/LOAD_INLINE_DATA layout. + dma_pusher.BindSubchannel(&payload.kepler_memory, NvkM2mfSubchannel, + Engines::EngineTypes::KeplerMemory); dma_pusher.BindSubchannel(&payload.fermi_2d, Nvk2DSubchannel, Engines::EngineTypes::Fermi2D); dma_pusher.BindSubchannel(&payload.maxwell_dma, NvkCopySubchannel, Engines::EngineTypes::MaxwellDMA); @@ -56,6 +73,17 @@ void ChannelState::Init(Core::System& system, u64 program_id_) { program_id = program_id_; payload.emplace(system, *memory_manager, *this); BindNvkDefaultSubchannels(*payload); + if (g_nx2_ampere) { + // Switch 2 command streams use Ampere classes with Turing+ (MME64) macros, and its + // block-linear surfaces the console's GOB swizzle. + memory_manager->EnableNativeTiling(); + const auto macros = Nx2MacroContext(); + payload->maxwell_3d.SetNativeMacroContext(macros); + payload->maxwell_3d.SetNativeTuringMacroMode(true); + payload->kepler_compute.SetNativeMacroMode(¯os->engine, macros->positions, + macros.get()); + payload->kepler_compute.SetNativeDataRam(&payload->maxwell_3d.GetNativeDataRam()); + } initialized = true; } diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index 8af13e415c..dc42906752 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -4,15 +4,33 @@ // SPDX-FileCopyrightText: 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include <algorithm> #include <memory> +#include <vector> #include "common/assert.h" +#include "core/core.h" #include "video_core/control/channel_state.h" #include "video_core/control/scheduler.h" #include "video_core/gpu.h" +#include "video_core/gpu_split.h" namespace Tegra::Control { +void Scheduler::BindChannel(GPU& gpu, s32 bind_id) { + // With the GPU thread split the rasterizer's channel switches with the methods around it, on + // the back thread (deferred outside the lock: the back takes it to bind). + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + split.Defer([this, gpu_ptr = &gpu, bind_id] { + std::scoped_lock lk{scheduling_guard}; + gpu_ptr->BindChannel(bind_id); + }); + return; + } + std::scoped_lock lk{scheduling_guard}; + gpu.BindChannel(bind_id); +} + void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) { std::shared_ptr<ChannelState> channel_state; { @@ -20,12 +38,45 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) { auto it = channels.find(channel); ASSERT(it != channels.end()); channel_state = it->second; - gpu.BindChannel(channel_state->bind_id); } + BindChannel(gpu, channel_state->bind_id); // Process commands outside the lock to reduce contention. // Multiple channels can prepare their commands in parallel. channel_state->payload->dma_pusher.Push(std::move(entries)); channel_state->payload->dma_pusher.DispatchCalls(); + if (channel_state->payload->dma_pusher.HasPendingAcquire() && + std::find(pending_channels.begin(), pending_channels.end(), channel_state) == + pending_channels.end()) { + pending_channels.push_back(channel_state); + } + if (!pending_channels.empty()) { + ResumePending(gpu); + } +} + +void Scheduler::ResumePending(GPU& gpu) { + // Running one channel can release the semaphore another waits for: retry until none moves. + for (bool progress = true; progress;) { + progress = false; + for (const auto& state : pending_channels) { + auto& dma_pusher = state->payload->dma_pusher; + // Powered off (shutting down), DispatchCalls returns at once and the acquire stays + // pending: resuming it again counted as progress forever and the GPU thread never + // reached its stop request (closing the game hung). + if (!dma_pusher.system.IsPoweredOn()) { + return; + } + if (!dma_pusher.HasPendingAcquire() || !dma_pusher.CanResumeAcquire()) { + continue; + } + BindChannel(gpu, state->bind_id); + dma_pusher.DispatchCalls(); + progress = true; + } + } + std::erase_if(pending_channels, [](const auto& state) { + return !state->payload->dma_pusher.HasPendingAcquire(); + }); } void Scheduler::DeclareChannel(std::shared_ptr<ChannelState> new_channel) { diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h index 342bea5025..9a0f1b919f 100644 --- a/src/video_core/control/scheduler.h +++ b/src/video_core/control/scheduler.h @@ -8,6 +8,7 @@ #include <memory> #include <mutex> +#include <vector> #include "common/container/unordered_map.h" #include "video_core/dma_pusher.h" @@ -24,11 +25,20 @@ class Scheduler { public: void Push(GPU& gpu, s32 channel, CommandList&& entries); + /// Runs channels whose pending semaphore acquire is now met. + void ResumePending(GPU& gpu); + [[nodiscard]] bool HasPendingChannels() const { + return !pending_channels.empty(); + } + void DeclareChannel(std::shared_ptr<ChannelState> new_channel); private: + void BindChannel(GPU& gpu, s32 bind_id); + ::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels; std::mutex scheduling_guard; + std::vector<std::shared_ptr<ChannelState>> pending_channels; ///< Suspended on an acquire }; } // namespace Control diff --git a/src/video_core/dirty_flags.cpp b/src/video_core/dirty_flags.cpp index c2ecc12f56..e6856aaca3 100644 --- a/src/video_core/dirty_flags.cpp +++ b/src/video_core/dirty_flags.cpp @@ -22,6 +22,7 @@ void SetupDirtyVertexBuffers(Maxwell3D::DirtyState::Tables& tables) { OFF(vertex_stream_limits) + i * NUM(vertex_stream_limits[0]); FillBlock(tables, array_offset, num_array, VertexBuffer0 + i, VertexBuffers); + FillBlock(tables, 0x600 / sizeof(u32) + i * 2, 2, VertexBuffer0 + i, VertexBuffers); FillBlock(tables, limit_offset, NUM(vertex_stream_limits), VertexBuffer0 + i, VertexBuffers); } diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 2c0ba007ef..62f7e6bfdb 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -4,13 +4,24 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> +#include <atomic> +#include <cstdlib> +#include <cstdio> + #include "common/settings.h" #include "core/core.h" +#include "core/memory.h" +#include "video_core/nx2_profile.h" #include "video_core/dma_pusher.h" +#include "video_core/control/channel_state.h" +#include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu.h" +#include "video_core/gpu_split.h" #include "video_core/guest_memory.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/rasterizer_interface.h" namespace Tegra { @@ -29,13 +40,21 @@ DmaPusher::DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Cont DmaPusher::~DmaPusher() = default; void DmaPusher::DispatchCalls() { - dma_pushbuffer_subindex = 0; + // The sub-index is kept: a channel suspended on a semaphore resumes inside its command list. dma_state.is_last_call = true; while (system.IsPoweredOn()) { if (!Step()) { break; } } + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + split.Defer([gpu = &system.GPU()] { + gpu->FlushCommands(); + gpu->OnCommandListEnd(); + }); + split.Publish(); + return; + } system.GPU().FlushCommands(); system.GPU().OnCommandListEnd(); } @@ -44,6 +63,9 @@ bool DmaPusher::Step() { if (!ib_enable || dma_pushbuffer.empty()) { return false; } + if (!puller.ResumeAcquire(*this)) { + return false; + } CommandList& command_list = dma_pushbuffer.front(); @@ -57,7 +79,13 @@ bool DmaPusher::Step() { } if (prefetch_size > 0) { - ProcessCommands(command_list.prefetch_command_list); + const std::span<const CommandHeader> commands{command_list.prefetch_command_list}; + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + if (puller.HasPendingAcquire()) { + return false; + } + command_resume_offset = 0; dma_pushbuffer.pop(); return true; } @@ -80,20 +108,86 @@ bool DmaPusher::Step() { const bool macro_payload = engine == Engines::EngineTypes::Maxwell3D && dma_state.method >= MacroRegistersStart; if (kepler_payload || macro_payload) { const size_t words = std::min<size_t>(dma_state.method_count, header.size); - subchannels[dma_state.subchannel]->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32)); + auto* const engine_state = subchannels[dma_state.subchannel]; + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + // The caches behind IsMemoryDirty belong to the split's back thread. + split.Defer([engine_state, memory = &memory_manager, address = GPUVAddr{dma_state.dma_get}, + bytes = words * sizeof(u32)] { + engine_state->current_dirty = memory->IsMemoryDirty(address, bytes); + }); + } else { + engine_state->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32)); + } } } + if (command_resume_offset == 0) { + // A method's data continues across GP entries, whatever their level: title 4 ends a + // main entry after a macro's first word and supplies the remaining parameters in a + // 5-word subroutine entry. EDEN_NX2_PARK_SPLIT_METHODS restores the old guess (an + // entry elsewhere is a call that starts clean; resume where the split one ended). + static const bool park_split_methods = std::getenv("EDEN_NX2_PARK_SPLIT_METHODS") != nullptr; + if (g_nx2_ampere) { + if (static u32 split_logs{}; dma_state.method_count != 0 && split_logs < 64) { + ++split_logs; + LOG_DEBUG(HW_GPU, "Split method {:#x} ({} left) on ch {}: next entry {:#x} size {} raw {:#x} (level bit {}, priv bit {}), expected {:#x}, parked {}", + dma_state.method, dma_state.method_count, channel_state.bind_id, + u64{header.addr}, u64{header.size}, header.raw, u64{header.is_push_buffer}, + u64{header.allow_flush}, split_method_end, parked_methods.size()); + std::array<u32, 8> next_words{}; + std::array<u32, 4> prev_words{}; + memory_manager.ReadBlockUnsafe(header.addr, next_words.data(), + std::min<size_t>(header.size, 8) * sizeof(u32)); + memory_manager.ReadBlockUnsafe(split_method_end - sizeof(prev_words), prev_words.data(), + sizeof(prev_words)); + LOG_DEBUG(HW_GPU, " previous entry ends {:08x}; next entry starts {:08x}", + fmt::join(prev_words, " "), fmt::join(next_words, " ")); + } + if (park_split_methods) { + ResolveSplitMethod(header.addr); + } + } + if (dma_pushbuffer_subindex < command_list.submit_hashes.size()) { + CheckSubmitHash(header, command_list.submit_hashes[dma_pushbuffer_subindex]); + } + } + // A resumed entry is read again: the words after the acquire may be new by now. const bool use_safe = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe(); if (use_safe) { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::SafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); - ProcessCommands(headers); + const std::span<const CommandHeader> commands{headers.data(), headers.size()}; + RecordParsedEntry(header, commands); + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); - ProcessCommands(headers); + const std::span<const CommandHeader> commands{headers.data(), headers.size()}; + RecordParsedEntry(header, commands); + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + } + if (puller.HasPendingAcquire()) { + return false; + } + command_resume_offset = 0; + if (dma_state.method_count != 0) { + split_method_end = header.addr + header.size * sizeof(u32); } } if (++dma_pushbuffer_subindex >= command_list_size) { + // A split method resumes within the submission that split it; ring addresses repeat + // every frame, so a stale one could otherwise swallow the headers of a later entry. + for (const auto& parked : parked_methods) { + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 16) { + LOG_WARNING(HW_GPU, + "Submission ended with parked method {:#x} (sub {}, {} words left, " + "resume {:#x})", + parked.method, parked.subchannel, parked.method_count, + parked.resume_address); + } + } + parked_methods.clear(); dma_pushbuffer.pop(); dma_pushbuffer_subindex = 0; } else { @@ -101,6 +195,7 @@ bool DmaPusher::Step() { } if (signal_sync) { + VideoCommon::GpuSplit::Get().Drain(); rasterizer->SignalFence([this]() { std::scoped_lock lk(sync_mutex); synced = true; @@ -111,20 +206,23 @@ bool DmaPusher::Step() { return true; } -void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) { +size_t DmaPusher::ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset) { for (size_t index = 0; index < commands.size();) { // Data word of methods command if (dma_state.method_count && dma_state.non_incrementing) { auto const& command_header = commands[index]; //must ref (MUltiMethod re) - dma_state.dma_word_offset = u32(index * sizeof(u32)); - const u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index); + dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32)); + u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index); + if (g_nx2_ampere && dma_state.method < non_puller_methods) { + max_write = 1; // a semaphore acquire may suspend the channel after any word + } CallMultiMethod(&command_header.argument, max_write); dma_state.method_count -= max_write; dma_state.is_last_call = true; index += max_write; } else if (dma_state.method_count) { auto const command_header = commands[index]; //can copy - dma_state.dma_word_offset = u32(index * sizeof(u32)); + dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32)); dma_state.is_last_call = dma_state.method_count <= 1; CallMethod(command_header.argument); dma_state.method += !dma_state.non_incrementing ? 1 : 0; @@ -163,6 +261,54 @@ void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) { } index++; } + if (puller.HasPendingAcquire()) { + return index; + } + } + return commands.size(); +} + +// NVN on Switch 2 ends GPFIFO entries in the middle of a method's data and calls another command +// buffer (e.g. a compute dispatch) before the entry that carries the rest, which starts right +// where the first one ended. An entry starting anywhere else is such a call: run it with a clean +// method state and resume the unfinished method when the stream continues at that address. +void DmaPusher::ResolveSplitMethod(GPUVAddr entry_address) { + const auto continued = std::ranges::find(parked_methods, entry_address, + &ParkedMethod::resume_address); + std::optional<ParkedMethod> resume; + if (continued != parked_methods.end()) { + resume = *continued; + parked_methods.erase(continued); + } + if (dma_state.method_count != 0 && entry_address != split_method_end) { + if (parked_methods.size() == MaxParkedMethods) { + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 8) { + const auto& oldest = parked_methods.front(); + LOG_WARNING(HW_GPU, + "Dropping parked method {:#x} (sub {}, {} words left, resume {:#x})", + oldest.method, oldest.subchannel, oldest.method_count, + oldest.resume_address); + } + parked_methods.erase(parked_methods.begin()); + } + parked_methods.push_back(ParkedMethod{ + .method = dma_state.method, + .subchannel = dma_state.subchannel, + .method_count = dma_state.method_count, + .non_incrementing = dma_state.non_incrementing, + .increment_once = dma_increment_once, + .resume_address = split_method_end, + }); + dma_state.method_count = 0; + dma_increment_once = false; + } + if (resume && dma_state.method_count == 0) { + dma_state.method = resume->method; + dma_state.subchannel = resume->subchannel; + dma_state.method_count = resume->method_count; + dma_state.non_incrementing = resume->non_incrementing; + dma_increment_once = resume->increment_once; } } @@ -172,7 +318,36 @@ void DmaPusher::SetState(const CommandHeader& command_header) { dma_state.method_count = command_header.method_count; } +// EDEN_NX2_SEM_WATCH=<gpu address>: log every method whose argument is the low word of an +// address in the 16 bytes there (finds the engine and method that release a semaphore). +static const u32 g_semaphore_watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? static_cast<u32>(std::strtoull(value, nullptr, 16)) : 0U; +}(); + +// EDEN_NX2_PB_CHECK also keeps the last parsed entries, for DumpCurrentEntry. +static const bool g_record_parsed_entries = std::getenv("EDEN_NX2_PB_CHECK") != nullptr; + +void DmaPusher::WatchArgument(u32 method, u32 argument) const { + const u32 watch = g_semaphore_watch; + if ((argument & ~0xfU) == watch) { + LOG_INFO(HW_GPU, "Semaphore watch: channel {} subchannel {} method {:#x} argument {:#x}", + channel_state.bind_id, dma_state.subchannel, method, argument); + } +} + +bool DmaPusher::WantsSegment() const { + // Only macro parameters and compute uploads read the engine's current_dma_segment. + const auto engine = subchannel_type[dma_state.subchannel]; + return engine == Engines::EngineTypes::KeplerCompute || + (engine == Engines::EngineTypes::Maxwell3D && dma_state.method >= MacroRegistersStart); +} + void DmaPusher::CallMethod(u32 argument) { + if (g_semaphore_watch != 0) [[unlikely]] { + WatchArgument(dma_state.method, argument); + } + auto& split = VideoCommon::GpuSplit::Get(); if (dma_state.method < non_puller_methods) { puller.CallPullerMethod(*this, Engines::Puller::MethodCall{ dma_state.method, @@ -182,9 +357,38 @@ void DmaPusher::CallMethod(u32 argument) { }); } else { auto subchannel = subchannels[dma_state.subchannel]; + if (subchannel == nullptr) { + ReportUnboundSubchannel(); + return; + } + if (split.Active()) { + const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset; + if (split.FrontMacros()) { + const auto type = subchannel_type[dma_state.subchannel]; + if (type == Engines::EngineTypes::Maxwell3D) { + auto* const maxwell3d = static_cast<Engines::Maxwell3D*>(subchannel); + if (maxwell3d->GetNativeMacroEngine().native_turing) { + maxwell3d->FrontCallMethod(system, split, dma_state.method, argument, + dma_state.is_last_call); + return; + } + } else if (type == Engines::EngineTypes::KeplerCompute) { + auto* const compute = static_cast<Engines::KeplerCompute*>(subchannel); + if (compute->FrontRunsMacros()) { + compute->FrontCallMethod(system, split, dma_state.method, argument, + dma_state.is_last_call, &segment); + return; + } + } + } + split.Method(subchannel, dma_state.method, argument, dma_state.is_last_call, + WantsSegment() ? &segment : nullptr); + return; + } if (!subchannel->execution_mask[dma_state.method]) { subchannel->method_sink.emplace_back(dma_state.method, argument); } else { + VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method}; subchannel->ConsumeSink(system); subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset; subchannel->CallMethod(system, dma_state.method, argument, dma_state.is_last_call); @@ -193,16 +397,154 @@ void DmaPusher::CallMethod(u32 argument) { } void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { + if (g_semaphore_watch != 0) [[unlikely]] { + for (u32 i = 0; i < num_methods; ++i) { + WatchArgument(dma_state.method + (dma_state.non_incrementing ? 0 : i), base_start[i]); + } + } + auto& split = VideoCommon::GpuSplit::Get(); if (dma_state.method < non_puller_methods) { puller.CallMultiMethod(*this, dma_state.method, dma_state.subchannel, base_start, num_methods, dma_state.method_count); } else { auto subchannel = subchannels[dma_state.subchannel]; + if (subchannel == nullptr) { + ReportUnboundSubchannel(); + return; + } + if (split.Active()) { + const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset; + if (split.FrontMacros()) { + const auto type = subchannel_type[dma_state.subchannel]; + if (type == Engines::EngineTypes::Maxwell3D) { + auto* const maxwell3d = static_cast<Engines::Maxwell3D*>(subchannel); + if (maxwell3d->GetNativeMacroEngine().native_turing) { + maxwell3d->FrontCallMultiMethod(system, split, dma_state.method, base_start, + num_methods, dma_state.method_count); + return; + } + } else if (type == Engines::EngineTypes::KeplerCompute) { + auto* const compute = static_cast<Engines::KeplerCompute*>(subchannel); + if (compute->FrontRunsMacros()) { + compute->FrontCallMultiMethod(system, split, dma_state.method, base_start, + num_methods, dma_state.method_count, &segment); + return; + } + } + } + split.MultiMethod(subchannel, dma_state.method, base_start, num_methods, + dma_state.method_count, WantsSegment() ? &segment : nullptr); + return; + } + VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method}; subchannel->ConsumeSink(system); subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset; subchannel->CallMultiMethod(system, dma_state.method, base_start, num_methods, dma_state.method_count); } } +void DmaPusher::ReportUnboundSubchannel() { + // Methods sent to a subchannel with no engine (no SetObject yet) are dropped. + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 8) { + LOG_WARNING(HW_GPU, "Dropping method {:#x} on unbound subchannel {}", dma_state.method, + dma_state.subchannel); + } +} + +void DmaPusher::CheckSubmitHash(const CommandListHeader& header, u64 submit_hash) { + std::vector<u32> words(header.size); + memory_manager.ReadBlockUnsafe(header.addr, words.data(), words.size() * sizeof(u32)); + static std::atomic<u64> checked{}; + static std::atomic<u64> changed{}; + ++checked; + if (HashPushbufferWords(words) != submit_hash) { + if (changed.fetch_add(1) < 16) { + LOG_WARNING(HW_GPU, "pushbuffer entry {:#x}+{:#x} changed between submit and execution", + header.addr.Value(), header.size.Value()); + } + } + if ((checked % 20000) == 0) { + LOG_WARNING(HW_GPU, "pushbuffer check: {} of {} entries changed after submit", + changed.load(), checked.load()); + } +} + +void DmaPusher::RecordParsedEntry(const CommandListHeader& header, + std::span<const CommandHeader> commands) { + if (!g_record_parsed_entries) { + return; + } + u64 hash = 0xcbf29ce484222325ULL; + for (const CommandHeader& command : commands) { + hash = (hash ^ command.argument) * 0x100000001b3ULL; + } + parsed_entries[parsed_entry_count++ % parsed_entries.size()] = {header.addr, static_cast<u32>(header.size), hash}; +} + +void DmaPusher::DumpCurrentEntry(const char* reason) { + // Entries whose memory changed after they were parsed (the CPU rewrote them meanwhile). + for (size_t i = 0; i < std::min(parsed_entry_count, parsed_entries.size()); ++i) { + const auto& entry = parsed_entries[(parsed_entry_count - 1 - i) % parsed_entries.size()]; + std::vector<u32> words(entry.size); + memory_manager.ReadBlockUnsafe(entry.address, words.data(), words.size() * sizeof(u32)); + u64 hash = 0xcbf29ce484222325ULL; + for (const u32 word : words) { + hash = (hash ^ word) * 0x100000001b3ULL; + } + LOG_WARNING(HW_GPU, " parsed -{}: {:#x}+{:#x} {}", i, entry.address, entry.size, + hash == entry.hash ? "unchanged" : "CHANGED since parsed"); + } + // EDEN_NX2_PB_DUMP=<file>: also write the whole command list (u64 address, u32 size and the + // current words of every entry, then the current entry index) for offline replay. + if (static const char* const path = std::getenv("EDEN_NX2_PB_DUMP"); + path != nullptr && !dma_pushbuffer.empty()) { + if (std::FILE* file = std::fopen(path, "wb")) { + const auto& lists = dma_pushbuffer.front().command_lists; + const u64 current = dma_pushbuffer_subindex; + const u64 count = lists.size(); + std::fwrite(&count, sizeof(count), 1, file); + std::fwrite(¤t, sizeof(current), 1, file); + for (const CommandListHeader& header : lists) { + const u64 address = header.addr; + const u32 size = static_cast<u32>(header.size); + std::vector<u32> words(size); + memory_manager.ReadBlockUnsafe(address, words.data(), words.size() * sizeof(u32)); + std::fwrite(&address, sizeof(address), 1, file); + std::fwrite(&size, sizeof(size), 1, file); + std::fwrite(words.data(), sizeof(u32), words.size(), file); + } + std::fclose(file); + } + } + LOG_WARNING(HW_GPU, "pushbuffer dump ({}): method={:#x} sub={} count={} non_inc={} get={:#x}", + reason, dma_state.method, dma_state.subchannel, dma_state.method_count, + dma_state.non_incrementing, dma_state.dma_get); + if (dma_pushbuffer.empty()) { + return; + } + const auto& lists = dma_pushbuffer.front().command_lists; + const size_t first = dma_pushbuffer_subindex >= 2 ? dma_pushbuffer_subindex - 2 : 0; + for (size_t i = first; i <= dma_pushbuffer_subindex && i < lists.size(); ++i) { + const CommandListHeader& header = lists[i]; + LOG_WARNING(HW_GPU, " entry {} of {}: addr={:#x} size={:#x}", i, lists.size(), + header.addr.Value(), header.size.Value()); + for (u64 word = 0; word < std::min<u64>(header.size, 0x80); word += 8) { + std::string gpu_line; + std::string cpu_line; + for (u64 j = word; j < std::min<u64>(header.size, word + 8); ++j) { + const GPUVAddr address = header.addr + j * sizeof(u32); + const u32 gpu_value = memory_manager.Read<u32>(address); + const u32 cpu_value = system.ApplicationMemory().Read32(address); + gpu_line += fmt::format(" {:08x}", gpu_value); + cpu_line += gpu_value == cpu_value ? std::string(" ........") + : fmt::format(" {:08x}", cpu_value); + } + LOG_WARNING(HW_GPU, " +{:03x}:{}", word * 4, gpu_line); + LOG_WARNING(HW_GPU, " cpu :{}", cpu_line); + } + } +} + void DmaPusher::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { rasterizer = rasterizer_; } diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 62af628511..82d29edb4b 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -8,6 +8,7 @@ #include <array> #include <condition_variable> +#include <optional> #include <span> #include <vector> #include <boost/container/small_vector.hpp> @@ -114,8 +115,19 @@ struct CommandList final { boost::container::small_vector<CommandListHeader, 512> command_lists; boost::container::small_vector<CommandHeader, 512> prefetch_command_list; + /// EDEN_NX2_PB_CHECK: hash of each entry's words when it was submitted. + std::vector<u64> submit_hashes; }; +/// FNV-1a over pushbuffer words (EDEN_NX2_PB_CHECK). +inline u64 HashPushbufferWords(std::span<const u32> words) { + u64 hash = 0xcbf29ce484222325ULL; + for (const u32 word : words) { + hash = (hash ^ word) * 0x100000001b3ULL; + } + return hash; +} + /// @brief The DmaPusher class implements DMA submission to FIFOs, providing an area of memory that the /// emulated app fills with commands and tells PFIFO to process. The pushbuffers are then assembled /// into a "command stream" consisting of 32-bit words that make up "commands". @@ -132,6 +144,14 @@ public: void DispatchCalls(); + /// Switch 2: true while the channel is suspended on a semaphore acquire. + [[nodiscard]] bool HasPendingAcquire() const { + return puller.HasPendingAcquire(); + } + [[nodiscard]] bool CanResumeAcquire() { + return puller.CanResumeAcquire(*this); + } + void BindSubchannel(Engines::EngineInterface* engine, u32 subchannel_id, Engines::EngineTypes engine_type) { subchannels[subchannel_id] = engine; subchannel_type[subchannel_id] = engine_type; @@ -143,12 +163,27 @@ private: static constexpr u32 non_puller_methods = 0x40; static constexpr u32 max_subchannels = 8; bool Step(); - void ProcessCommands(std::span<const CommandHeader> commands); + /// Returns how many words were processed: fewer than given when a semaphore acquire + /// suspended the channel. + size_t ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset = 0); + size_t command_resume_offset{}; ///< Words of the current entry done before a suspension void SetState(const CommandHeader& command_header); void CallMethod(u32 argument); void CallMultiMethod(const u32* base_start, u32 num_methods); + [[nodiscard]] bool WantsSegment() const; + void ReportUnboundSubchannel(); + void WatchArgument(u32 method, u32 argument) const; + +public: + /// Logs the GPFIFO entry being processed (and the one before it), read through the GPU MMU + /// and straight from guest memory, to find pushbuffer desyncs. Debug aid for Switch 2. + void DumpCurrentEntry(const char* reason); + void CheckSubmitHash(const CommandListHeader& header, u64 submit_hash); + void RecordParsedEntry(const CommandListHeader& header, std::span<const CommandHeader> commands); + +private: public: Common::ScratchBuffer<CommandHeader> command_headers; ///< Buffer for list of commands fetched at once @@ -172,6 +207,31 @@ public: Control::ChannelState& channel_state; DmaState dma_state{}; + + // Switch 2: methods whose data continues in a later GPFIFO entry, set aside while other + // command buffers run (NVN interleaves several, so more than one can be pending). + struct ParkedMethod { + u32 method; + u32 subchannel; + u32 method_count; + bool non_incrementing; + bool increment_once; + GPUVAddr resume_address; ///< Where the entry that continues the method starts + }; + void ResolveSplitMethod(GPUVAddr entry_address); + static constexpr size_t MaxParkedMethods = 16; + + // Debug aid: hashes of the words of the last entries as they were parsed. + struct ParsedEntry { + GPUVAddr address; + u32 size; + u64 hash; + }; + std::array<ParsedEntry, 64> parsed_entries{}; + size_t parsed_entry_count{}; + std::vector<ParkedMethod> parked_methods; + GPUVAddr split_method_end{}; ///< End of the last entry that ended inside a method + std::array<Engines::EngineInterface*, max_subchannels> subchannels{}; std::array<Engines::EngineTypes, max_subchannels> subchannel_type; diff --git a/src/video_core/engines/engine_upload.cpp b/src/video_core/engines/engine_upload.cpp index ba21eec4c4..0ffdae8036 100644 --- a/src/video_core/engines/engine_upload.cpp +++ b/src/video_core/engines/engine_upload.cpp @@ -27,6 +27,7 @@ void State::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { void State::ProcessExec(const bool is_linear_) { write_offset = 0; + flushed_offset = 0; copy_size = regs.line_length_in * regs.line_count; inner_buffer.resize_destructive(copy_size); is_linear = is_linear_; @@ -39,10 +40,29 @@ void State::ProcessData(const u32 data, const bool is_last_call) { if (!is_last_call) { return; } + if (native_partial && is_linear && regs.line_count == 1) { + const u32 count = write_offset - flushed_offset; + if (count) { + rasterizer->AccelerateInlineToMemory(regs.dest.Address() + flushed_offset, count, + std::span<const u8>{inner_buffer.data() + flushed_offset, count}); + flushed_offset = write_offset; + } + return; + } ProcessData(inner_buffer); } void State::ProcessData(const u32* data, size_t num_data) { + if (native_partial && is_linear && regs.line_count == 1) { + const auto count = std::min<size_t>(num_data * sizeof(u32), copy_size - write_offset); + if (count) { + rasterizer->AccelerateInlineToMemory(regs.dest.Address() + write_offset, count, + std::span<const u8>{reinterpret_cast<const u8*>(data), count}); + write_offset += static_cast<u32>(count); + flushed_offset = write_offset; + } + return; + } std::span<const u8> read_buffer(reinterpret_cast<const u8*>(data), num_data * sizeof(u32)); ProcessData(read_buffer); } diff --git a/src/video_core/engines/engine_upload.h b/src/video_core/engines/engine_upload.h index 21bf8aeb46..55a82f938b 100644 --- a/src/video_core/engines/engine_upload.h +++ b/src/video_core/engines/engine_upload.h @@ -62,6 +62,7 @@ public: explicit State(MemoryManager& memory_manager_, Registers& regs_); ~State(); + void SetNativePartialTransfers(bool enabled) { native_partial = enabled; } void ProcessExec(bool is_linear_); void ProcessData(u32 data, bool is_last_call); void ProcessData(const u32* data, size_t num_data); @@ -80,6 +81,8 @@ public: private: void ProcessData(std::span<const u8> read_buffer); + bool native_partial{}; + u32 flushed_offset{}; u32 write_offset = 0; u32 copy_size = 0; Common::ScratchBuffer<u8> inner_buffer; diff --git a/src/video_core/engines/fermi_2d.cpp b/src/video_core/engines/fermi_2d.cpp index aeff5e156f..35b2c69801 100644 --- a/src/video_core/engines/fermi_2d.cpp +++ b/src/video_core/engines/fermi_2d.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> #include "common/assert.h" #include "common/logging.h" #include "video_core/engines/fermi_2d.h" @@ -59,6 +60,27 @@ void Fermi2D::ConsumeSinkImpl(Core::System& system) { } void Fermi2D::Blit() { + if (static const u64 trace_destination = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_DST"); + return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + trace_destination != 0 && regs.dst.Address() >= trace_destination && + regs.dst.Address() < trace_destination + (32ULL << 20)) { + static u32 traced_destination{}; + if (traced_destination++ < 40) { + LOG_INFO(HW_GPU, "nx2 2D blit to traced dst: {:#x} {}x{} -> {:#x} {}x{}", + regs.src.Address(), regs.src.width, regs.src.height, regs.dst.Address(), + regs.dst.width, regs.dst.height); + } + } + if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) { + static u32 traced{}; + if (traced++ < 64) { + LOG_INFO(HW_GPU, "nx2 2D blit {:#x} {}x{} -> {:#x} {}x{}", regs.src.Address(), + regs.src.width, regs.src.height, regs.dst.Address(), regs.dst.width, + regs.dst.height); + } + } LOG_DEBUG(HW_GPU, "called. source address={:#x}, destination address={:#x}", regs.src.Address(), regs.dst.Address()); UNIMPLEMENTED_IF_MSG(regs.operation != Operation::SrcCopy, "Operation is not copy"); diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 96a12539a2..ab2ba10162 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -4,13 +4,30 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> +#include <atomic> #include <bitset> +#include <cstdlib> +#include <fstream> +#include <optional> +#include <string> +#include <unordered_map> +#include <unordered_set> #include "common/assert.h" #include "common/logging.h" +#include "shader_recompiler/exception.h" #include "core/core.h" +#include "video_core/gpu_split.h" +#include "video_core/nx2_profile.h" +#include "video_core/nx2.h" +#include "video_core/engines/fermi_2d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/memory_manager.h" +#include "video_core/native_query_trace.h" +#ifdef ARCHITECTURE_x86_64 +#include "video_core/macro_turing_jit.h" +#endif #include "video_core/rasterizer_interface.h" #include "video_core/textures/decoders.h" @@ -20,10 +37,31 @@ KeplerCompute::KeplerCompute(MemoryManager& memory_manager_) : memory_manager{memory_manager_} , upload_state{memory_manager, regs.upload} { + // EDEN_NX2_QMD_TRACE: log the QMD scheduler's loads, dependency counts and launches. + if (std::getenv("EDEN_NX2_QMD_TRACE")) { + native_qmd.trace = [this](std::string_view event, u64 address, s32 value) { + LOG_INFO(HW_GPU, "QMD trace {} {:#x} {} (engine {} as {})", event, address, value, + fmt::ptr(this), memory_manager.GetID()); + }; + } execution_mask.reset(); execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true; execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true; execution_mask[KEPLER_COMPUTE_REG_INDEX(launch)] = true; + // Everything ProcessNativeMethod acts on besides storing the value; the QMD and 0x5a + // trace range too while native draw diagnostics trace it. + native_plain_methods.set(); + for (const u32 method : {0x45U, 0x46U, 0x47U, 0x48U, 0xaeU, 0xafU, 0xb0U, 0x107U, 0x6c3U, + static_cast<u32>(KEPLER_COMPUTE_REG_INDEX(exec_upload)), + static_cast<u32>(KEPLER_COMPUTE_REG_INDEX(data_upload))}) { + native_plain_methods[method] = false; + } + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr) { + native_plain_methods[0x5a] = false; + for (u32 method = 0xc6; method <= 0x108; ++method) { + native_plain_methods[method] = false; + } + } } KeplerCompute::~KeplerCompute() = default; @@ -35,13 +73,389 @@ void KeplerCompute::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) void KeplerCompute::ConsumeSinkImpl(Core::System& system) { for (auto [method, value] : method_sink) { - regs.reg_array[method] = value; + if (method < Regs::NUM_REGS) { + regs.reg_array[method] = value; + } } method_sink.clear(); } +// SET_REPORT_SEMAPHORE_A-D (0x6c0-0x6c3): address high/low, payload, then operation in bits 1:0 +// (release / acquire / report only / trap), reduction enable bit 3 with the op in bits 11:9, and +// a one-word (no timestamp) structure in bit 28. NVN2 releases queue syncs from compute this way. +void KeplerCompute::ProcessNativeReportSemaphore(u32 argument) { + const GPUVAddr address = (GPUVAddr{native_registers[0x6c0] & 0x1ffffff} << 32) | + native_registers[0x6c1]; + const u32 payload = native_registers[0x6c2]; + const u32 operation = argument & 3; + const bool one_word = (argument >> 28) & 1; + static u32 logged{}; + if (logged++ < 32) { + LOG_INFO(HW_GPU, "Native compute report semaphore {:#x} = {:#x}, control {:#x}", address, + payload, argument); + } + if (operation != 0 && operation != 2) { + LOG_WARNING(HW_GPU, "Native compute report semaphore operation {} not supported", operation); + return; + } + if ((argument >> 3) & 1) { + const u32 reduction = (argument >> 9) & 7; + rasterizer->SignalFence([this, address, payload, reduction] { + const u32 old_value = memory_manager.Read<u32>(address); + const u32 value = reduction == 0 ? old_value + payload + : reduction == 1 ? (std::min)(old_value, payload) + : reduction == 2 ? (std::max)(old_value, payload) + : reduction == 3 ? (old_value >= payload ? 0 : old_value + 1) + : reduction == 4 ? (old_value == 0 || old_value > payload ? payload : old_value - 1) + : reduction == 5 ? (old_value & payload) + : reduction == 6 ? (old_value | payload) + : (old_value ^ payload); + memory_manager.Write<u32>(address, value); + }); + return; + } + rasterizer->Query(address, VideoCommon::QueryType::Payload, + one_word ? VideoCommon::QueryPropertiesFlags::IsAFence + : VideoCommon::QueryPropertiesFlags::HasTimeout, + payload, 0); +} + +bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call) { + if (native_context && !front_macros) { + if (method == 0x45 || method == 0x47) native_compute_uploaded = true; + native_macro = native_compute_uploaded ? &native_context->compute_engine : &native_context->engine; + native_macro_positions = native_compute_uploaded ? std::span<u32>{native_context->compute_positions} : std::span<u32>{native_context->positions}; + } + if (method >= native_registers.size()) { + LOG_ERROR(HW_GPU, "Native compute method out of range: {:#x}", method); + return true; + } + if (method >= 0xe00) { + if (!native_executing_macro) { + if (method & 1) { LOG_ERROR(HW_GPU, "Native compute macro starts on argument method"); return true; } + native_executing_macro = method; + } + native_parameters.push_back(argument); + if (!is_last_call) return true; + const u32 entry = ((*native_executing_macro - 0xe00) / 2) % native_macro_positions.size(); + native_executing_macro_logged = native_executing_macro; + native_executing_macro.reset(); + static const u32 watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? static_cast<u32>(std::strtoull(value, nullptr, 16)) : 0U; + }(); + const bool watched = watch != 0 && std::ranges::any_of(native_parameters, + [](u32 parameter) { return (parameter & ~0xfU) == watch; }); + if (watched) { + LOG_INFO(HW_GPU, "Semaphore watch: compute macro start={:#x} parameters={:08x}", + native_macro_positions[entry], fmt::join(native_parameters, ",")); + } + // EDEN_NX2_MME_DUMP: write the compute macro code once per engine, and log each macro + // call with its entry and parameters (diagnostics). + if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr; mme_dump) { + if (!native_mme_dumped && native_macro_positions[entry] == 0xda) { + native_mme_dumped = true; + std::ofstream dump(fmt::format("native-compute-mme-{}.bin", fmt::ptr(this)), + std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()), + sizeof(native_macro->native_code)); + } + LOG_INFO(HW_GPU, "MME call engine {} method {:#x} start={:#x} parameters={:08x}", + fmt::ptr(this), *native_executing_macro_logged, native_macro_positions[entry], + fmt::join(native_parameters, ",")); + } + if (native_data_ram) native_macro->native_interpreter.active_dram = native_data_ram; + const u32 dram0_before = (*native_macro->native_interpreter.active_dram)[0x27]; + const u32 start = native_macro_positions[entry]; + const auto emit = [&](u32 address, u32 value) { + static u32 diagnostic_writes{}; + static const bool log_emits = std::getenv("EDEN_NX2_SEM_WATCH_EMITS") != nullptr; + if (watched && log_emits) LOG_INFO(HW_GPU, "Semaphore watch: compute macro emit {:#x} = {:#x}", address, value); + if (diagnostic_writes++ < 4096) LOG_DEBUG(HW_GPU, "Native compute MME emit method={:#x}, value={:#x}", address, value); + if (address >= 0xe00) return false; + VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method}; + CallMethod(system, address, value, true); + return true; + }; +#ifdef ARCHITECTURE_x86_64 + // Compiled macro code, as for 3D macros (macro_turing_jit.h); EDEN_NX2_MME_JIT=0 off. + static const bool use_jit = [] { + const char* value = std::getenv("EDEN_NX2_MME_JIT"); + return value == nullptr || value[0] != '0'; + }(); + if (use_jit && !watched) { + if (!native_macro->native_jit) { + native_macro->native_jit = std::make_shared<TuringMacroJit>(); + } + native_macro->native_jit->Prepare(native_macro->native_code, + native_macro->native_code_generation); + const auto result = native_macro->native_jit->Execute( + start, native_parameters, + std::span<u32>{*native_macro->native_interpreter.active_dram}, + [&](u32 address) { return address < native_registers.size() ? native_registers[address] : 0U; }, + emit); + if (!result.success) { + native_macro->native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native MME (JIT) failed: compute start={:#x}, consumed={}/{}, error={}", + start, result.consumed, native_parameters.size(), result.error); + } + native_parameters.clear(); + return true; + } +#endif + // Repeated calls replay their writes, as for 3D macros (macro_memo.h). + static const bool use_memo = [] { + const char* value = std::getenv("EDEN_NX2_MME_MEMO"); + return value == nullptr || value[0] != '0'; + }(); + auto& memo = native_macro->native_memo; + if (use_memo && !watched) { + memo.Synchronize(native_macro->native_code_generation); + if (std::remove_reference_t<decltype(memo)>::Effects effects; + memo.Find(start, native_parameters, *native_macro->native_interpreter.active_dram, + effects)) { + auto& data_ram = *native_macro->native_interpreter.active_dram; + for (size_t i = 0; i < effects.data_ram_writes.size(); i += 2) { + data_ram[effects.data_ram_writes[i]] = effects.data_ram_writes[i + 1]; + } + for (size_t i = 0; i < effects.writes.size(); i += 2) { + emit(effects.writes[i], effects.writes[i + 1]); + } + native_parameters.clear(); + return true; + } + native_macro->native_writes.clear(); + } + // The interpreter consumes native_parameters as its FIFO: keep the call's own parameters. + native_memo_parameters.assign(native_parameters.begin(), native_parameters.end()); + const u64 code_generation = native_macro->native_code_generation; + const auto result = native_macro->native_interpreter.Execute(native_macro->native_code, + start, native_parameters, + [&](u32 address) { return address < native_registers.size() ? native_registers[address] : 0U; }, + [&](u32 address, u32 value) { + if (use_memo) { + native_macro->native_writes.emplace_back(address, value); + } + return emit(address, value); + }); + const auto& interpreter = native_macro->native_interpreter; + if (use_memo && !watched && result.success && !interpreter.read_registers && + !interpreter.reread_data_ram && code_generation == native_macro->native_code_generation) { + memo.Store(start, native_memo_parameters, interpreter.data_ram_reads, + interpreter.data_ram_writes, native_macro->native_writes); + } + if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr; + mme_dump && (*native_macro->native_interpreter.active_dram)[0x27] != dram0_before) { + LOG_INFO(HW_GPU, "MME data RAM[0x27] {:#x} -> {:#x} by engine {} macro start={:#x}", + dram0_before, (*native_macro->native_interpreter.active_dram)[0x27], fmt::ptr(this), + native_macro_positions[entry]); + } + if (!result.success) { + native_macro->native_execution_failed = true; + if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + diagnostics) { + std::ofstream dump("native-compute-mme.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code)); + } + LOG_ERROR(HW_GPU, "Native MME failed: compute start={:#x}, pc={:#x}, consumed={}/{}, error={}, parameters={:08x}", + native_macro_positions[entry], result.pc, result.consumed, native_parameters.size(), result.error, + fmt::join(native_parameters, ",")); + } + native_parameters.clear(); + return true; + } + native_registers[method] = argument; + if (method == 0x6c3) { // SET_REPORT_SEMAPHORE_D: the compute class's semaphore release + ProcessNativeReportSemaphore(argument); + return true; + } + if (method == 0x5a || (method >= 0xc6 && method <= 0x108)) { + VideoCommon::TraceNativeQuery(12, memory_manager.GetID(), method, 0, 0, argument, 0); + } + if (method == 0x107) { + const GPUVAddr address = ((GPUVAddr{native_registers[0xc6]} << 32) | + native_registers[0xc7]) << 8; + const GPUVAddr program = (GPUVAddr{native_registers[0xc8 + 49] & 0x1ffff} << 32) | + native_registers[0xc8 + 48]; + const u32 control = native_registers[0xa7]; + LOG_DEBUG(HW_GPU, "Native inline QMD: address={:#x}, program={:#x}, control={:#x}, grid={}/{}/{}", + address, program, control, native_registers[0xc8 + 12], + native_registers[0xc8 + 13] & 0xffff, native_registers[0xc8 + 14] & 0xffff); + if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + diagnostics) { + std::ofstream dump("native-inline-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_registers.data() + 0xc8), 256); + std::ofstream registers("native-inline-compute-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers)); + if (memory_manager.IsFullyMappedRange(program, 0x10000)) { + std::vector<u8> code(0x10000); + memory_manager.ReadBlockUnsafe(program, code.data(), code.size()); + std::ofstream shader("native-inline-compute-shader.bin", std::ios::binary); + shader.write(reinterpret_cast<const char*>(code.data()), code.size()); + } + } + if (!memory_manager.IsFullyMappedRange(address, 256)) { + native_macro->native_execution_failed = true; + LOG_ERROR(HW_GPU, "Unmapped native inline QMD destination"); + return true; + } + rasterizer->AccelerateInlineToMemory(address, 256, + std::span<const u8>{reinterpret_cast<const u8*>(native_registers.data() + 0xc8), 256}); + if ((control & (1U << 24)) || !(native_registers[0xcc] & (1U << 12))) { + const u32 previous = native_registers[0xad]; + native_registers[0xad] = static_cast<u32>(address >> 8); + ProcessNativeMethod(system, 0xb0, 3, true); + native_registers[0xad] = previous; + } + return true; + } + if (front_macros && method >= 0x45 && method <= 0x48) { + return true; // Loaded and bound by the split GPU thread's front (FrontApply). + } + switch (method) { + case 0x45: native_macro->ClearCode(argument); return true; + case 0x46: native_macro->AddCode(0, argument); return true; + case 0x47: native_bind_pointer = argument; return true; + case 0x48: + if (native_bind_pointer < native_macro_positions.size()) native_macro_positions[native_bind_pointer++] = argument; + else LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds"); + return true; + case 0xae: case 0xaf: case 0xb0: { + const GPUVAddr qmd_address = GPUVAddr{native_registers[0xad]} << 8; + VideoCommon::TraceNativeQuery(9, memory_manager.GetID(), qmd_address, + memory_manager.GpuToCpuAddress(qmd_address).value_or(0), method, argument, 0); + if (method == 0xae || (method == 0xaf && !(argument & 2)) || + (method == 0xb0 && ((argument & 15) == 0 || (argument & 15) == 1))) return true; + static const bool qmd_diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + if ((argument & 15) == 12 && qmd_diagnostics) { + static u32 captured{}; + if (captured++ == 0 && memory_manager.IsFullyMappedRange(qmd_address, 256)) { + std::array<u32, 64> descriptor{}; + memory_manager.ReadBlockUnsafe(qmd_address, descriptor.data(), sizeof(descriptor)); + std::ofstream dump("native-increment-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(descriptor.data()), sizeof(descriptor)); + LOG_WARNING(HW_GPU, "Native increment dependency descriptor: address={:#x}, words={:08x}", + qmd_address, fmt::join(descriptor, ",")); + } + } + const auto error = native_qmd.Signal(qmd_address, argument, + [&](u64 address, NativeQmdScheduler::Descriptor& descriptor) { + if (!memory_manager.IsFullyMappedRange(address, sizeof(descriptor))) return false; + memory_manager.ReadBlockUnsafe(address, descriptor.data(), sizeof(descriptor)); + if (static u32 logged{}; (descriptor[53] & (1U << 23)) && logged < 8) { + ++logged; + LOG_INFO(HW_GPU, "Native QMD {:#x} releases: {:08x} | {:08x} | {:08x}", address, + fmt::join(std::span{descriptor}.subspan(24, 4), ","), + fmt::join(std::span{descriptor}.subspan(28, 4), ","), + fmt::join(std::span{descriptor}.subspan(52, 4), ",")); + } + for (u32 slot = 0; slot < 2; ++slot) { + const u32 flags = descriptor[25 + slot * 4]; + if (!(flags & (1U << 23))) continue; + const GPUVAddr destination = (GPUVAddr{flags & 0xff} << 32) | descriptor[24 + slot * 4]; + VideoCommon::TraceNativeQuery(7 + slot, memory_manager.GetID(), destination, + memory_manager.GpuToCpuAddress(destination).value_or(0), + descriptor[4], descriptor[26 + slot * 4], flags); + } + return true; + }, + [&](u64 address, u32 payload, u32 flags) { + if (!memory_manager.IsFullyMappedRange(address, sizeof(payload))) return false; + if (flags & (1U << 26)) { + rasterizer->SignalFence([this, address, payload] { + const u32 value = memory_manager.Read<u32>(address) + payload; + memory_manager.Write<u32>(address, value); + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value); + } + static const u64 watch = [] { + const char* text = std::getenv("EDEN_NX2_SEM_WATCH"); + return text ? u64{std::strtoull(text, nullptr, 16)} : u64{}; + }(); + if (watch != 0 && (address & ~u64{0xf}) == watch) { + LOG_INFO(HW_GPU, "Semaphore watch: QMD release add {:#x} -> {:#x}", address, value); + } + VideoCommon::TraceNativeQuery(1, memory_manager.GetID(), address, + memory_manager.GpuToCpuAddress(address).value_or(0), 0, value, 1U << 26); + }); + } else if (flags & (1U << 19)) { + // NVN2 tracks QMD completion with a max release (bit 19) into a word other + // releases also advance: a plain write would move it backwards and leave + // GEQ acquires waiting forever. + rasterizer->SignalFence([this, address, payload] { + const u32 value = (std::max)(memory_manager.Read<u32>(address), payload); + memory_manager.Write<u32>(address, value); + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value); + } + }); + } else { + static const u64 watch = [] { + const char* text = std::getenv("EDEN_NX2_SEM_WATCH"); + return text ? u64{std::strtoull(text, nullptr, 16)} : u64{}; + }(); + if (watch != 0 && (address & ~u64{0xf}) == (watch & ~u64{0xf})) { + LOG_INFO(HW_GPU, "Semaphore watch: QMD release {:#x} = {:#x} flags {:#x}", address, payload, flags); + } + rasterizer->Query(address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + } + return true; + }, [&](const auto& descriptor) { return DispatchNativeCompute(descriptor); }); + if (error.empty()) { + LOG_DEBUG(HW_GPU, "Native QMD control action={:#x}, QMD={:#x}", argument, qmd_address); + return true; + } + native_macro->native_execution_failed = true; + std::array<u32, 128> qmd{}; + if (memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) { + memory_manager.ReadBlockUnsafe(qmd_address, qmd.data(), sizeof(qmd)); + } + // Log a failing program's first few dispatches only (QMD words 48-49 hold its code + // address); it fails the same way every frame. + static std::unordered_map<u64, u32> failures; + const u64 program = (u64{qmd[49]} << 32) | qmd[48]; + if (failures[program ^ std::hash<std::string>{}(std::string{error})]++ >= 2) { + return true; + } + LOG_ERROR(HW_GPU, "Native QMD failed: {}", error); + if (qmd_diagnostics && memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) { + std::ofstream dump("native-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(qmd.data()), sizeof(qmd)); + std::ofstream macros("native-dispatch-mme.bin", std::ios::binary); + macros.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code)); + std::ofstream registers("native-dispatch-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers)); + } + LOG_ERROR(HW_GPU, "Native compute dispatch unsupported: method={:#x}, value={:#x}, QMD={:#x}, words={:08x}", method, argument, + qmd_address, fmt::join(qmd, ",")); + return true; + } + default: + // The inline upload register block is shared with the older class. + if (method < Regs::NUM_REGS) regs.reg_array[method] = argument; + return method != KEPLER_COMPUTE_REG_INDEX(exec_upload) && method != KEPLER_COMPUTE_REG_INDEX(data_upload); + } +} + void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - ASSERT_MSG(method < Regs::NUM_REGS, "Invalid KeplerCompute register, increase the size of the Regs structure"); + if (native_mode) { + // Most of the compute class's register writes (QMD fields replayed by its macros for + // each of thousands of dispatches a second) only store the value: ~4% of title 1's GPU + // thread went through ProcessNativeMethod's checks for them. + if (method < native_plain_methods.size() && native_plain_methods[method]) { + native_registers[method] = method_argument; + if (method < Regs::NUM_REGS) { + regs.reg_array[method] = method_argument; + } + return; + } + if (ProcessNativeMethod(system, method, method_argument, is_last_call)) return; + } + if (method >= Regs::NUM_REGS) { + LOG_ERROR(HW_GPU, "Compute register out of range: {:#x}", method); + return; + } regs.reg_array[method] = method_argument; @@ -85,6 +499,10 @@ void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argu } void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) { + if (native_mode) { + for (u32 i = 0; i < amount; ++i) CallMethod(system, method, base_start[i], methods_pending - i <= 1); + return; + } switch (method) { case KEPLER_COMPUTE_REG_INDEX(data_upload): upload_address = current_dma_segment; @@ -100,6 +518,223 @@ void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* } } +bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& qmd) { + launch_description = {}; + launch_description.grid_dim_x.Assign(qmd[12]); + launch_description.grid_dim_y.Assign(qmd[13] & 0xffff); + launch_description.grid_dim_z.Assign(qmd[14] & 0xffff); + launch_description.shared_alloc.Assign(qmd[17] & 0x3ffff); + launch_description.block_dim_x.Assign(qmd[18] >> 16); + launch_description.block_dim_y.Assign(qmd[19] & 0xffff); + launch_description.block_dim_z.Assign(qmd[19] >> 16); + launch_description.const_buffer_enable_mask.Assign(qmd[20] & 0xff); + launch_description.local_pos_alloc.Assign(qmd[23] & 0xffffff); + launch_description.local_crs_alloc.Assign(qmd[50] & 0xffffff); + regs.code_loc.address_low = qmd[48]; + regs.code_loc.address_high = qmd[49] & 0x1ffff; + for (u32 slot = 0; slot < NumConstBuffers; ++slot) { + const u32 high = qmd[33 + slot * 2]; + if (high & 0x1ff00) { + LOG_ERROR(HW_GPU, "Native compute constant address exceeds supported GPU width"); + return false; + } + auto& buffer = launch_description.const_buffer_config[slot]; + buffer.address_low = qmd[32 + slot * 2]; + buffer.address_high.Assign(high & 0xff); + buffer.size.Assign(((high >> 19) & 0x1fff) << 4); + } + if (!launch_description.block_dim_x || !launch_description.block_dim_y || !launch_description.block_dim_z) + return false; + try { + rasterizer->DispatchCompute(); + } catch (const Shader::Exception& error) { + static std::unordered_set<std::string> logged; + if (logged.insert(error.what()).second) { + LOG_ERROR(HW_GPU, "Native compute shader rejected: {}", error.what()); + } + if (std::string_view{error.what()}.find("matrix multiply") != std::string_view::npos) { + DescribeNativeResources(qmd); + // The console runs this on tensor cores; stand in with a plain upscale of its input. + return UpscaleInsteadOfNativeProgram(); + } + return false; + } + return true; +} + +// Logs the images a rejected program's handles (constant bank 0, bytes 0x80-0x11c) point to, so a +// substitute (e.g. an upscale blit for a tensor-core pass) can find its inputs and output. +void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd) { + static u32 described{}; + if (described++ >= 2) return; + const auto& cbuf = launch_description.const_buffer_config[0]; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + LOG_INFO(HW_GPU, "Native resources: cbuf0 {:#x} size {:#x}, grid {}x{}x{}, block {}x{}x{}, tic pool {:#x}", + cbuf_address, cbuf.size.Value(), qmd[12], qmd[13] & 0xffff, qmd[14] & 0xffff, + launch_description.block_dim_x.Value(), launch_description.block_dim_y.Value(), + launch_description.block_dim_z.Value(), regs.tic.Address()); + for (u32 offset = 0x80; offset < 0x120; offset += 4) { + const u32 handle = memory_manager.Read<u32>(cbuf_address + offset); + const u32 tic_index = handle & 0xfffff; + if (handle == 0 || tic_index > regs.tic.limit) continue; + const auto tic = GetTICEntry(tic_index); + LOG_INFO(HW_GPU, " handle @{:#x} = {:#x}: tic {} address {:#x} {}x{} format {:#x} types {}/{}/{}/{}", + offset, handle, tic_index, tic.Address(), tic.Width(), tic.Height(), + static_cast<u32>(tic.format.Value()), static_cast<u32>(tic.r_type.Value()), + static_cast<u32>(tic.g_type.Value()), static_cast<u32>(tic.b_type.Value()), + static_cast<u32>(tic.a_type.Value())); + } +} + +std::atomic<u32> g_nx2_trace_draws_after_upscale{}; + +// Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and +// writes the output resolution image through the surface handle at cbuf0+0xa0 (from its +// TLD/SUST operands). Blit one to the other with bilinear filtering when both are plain 2D images +// of the same format and the output is larger. +bool KeplerCompute::UpscaleInsteadOfNativeProgram() { + const auto& cbuf = launch_description.const_buffer_config[0]; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + const auto image = [&](u32 offset) -> std::optional<Texture::TICEntry> { + const u32 tic_index = memory_manager.Read<u32>(cbuf_address + offset) & 0xfffff; + if (tic_index > regs.tic.limit) return std::nullopt; + return GetTICEntry(tic_index); + }; + const auto input = image(0x90), output = image(0xa0); + if (!input || !output || input->format != output->format || input->Address() == 0 || + output->Address() == 0 || output->Width() <= input->Width() || !input->IsBlockLinear() || + !output->IsBlockLinear()) + return false; + using RTFormat = Tegra::RenderTargetFormat; + RTFormat format{}; + switch (input->format) { + case Texture::TextureFormat::A2B10G10R10: format = RTFormat::A2B10G10R10_UNORM; break; + case Texture::TextureFormat::A8B8G8R8: format = RTFormat::A8B8G8R8_UNORM; break; + case Texture::TextureFormat::B10G11R11: format = RTFormat::B10G11R11_FLOAT; break; + case Texture::TextureFormat::R16G16B16A16: format = RTFormat::R16G16B16A16_FLOAT; break; + default: return false; + } + const auto surface = [&](const Texture::TICEntry& tic) { + Fermi2D::Surface result{}; + result.format = format; + result.linear = Fermi2D::MemoryLayout::BlockLinear; + result.block_width.Assign(tic.block_width); + result.block_height.Assign(tic.block_height); + result.block_depth.Assign(tic.block_depth); + result.depth = 1; + result.width = tic.Width(); + result.height = tic.Height(); + result.addr_upper = static_cast<u32>(tic.Address() >> 32); + result.addr_lower = static_cast<u32>(tic.Address()); + return result; + }; + const Fermi2D::Config config{ + .operation = Fermi2D::Operation::SrcCopy, + .filter = Fermi2D::Filter::Bilinear, + .must_accelerate = true, + .dst_x0 = 0, + .dst_y0 = 0, + .dst_x1 = static_cast<s32>(output->Width()), + .dst_y1 = static_cast<s32>(output->Height()), + .src_x0 = 0, + .src_y0 = 0, + .src_x1 = static_cast<s32>(input->Width()), + .src_y1 = static_cast<s32>(input->Height()), + }; + // Debug: does the input hold anything? Download it once and count non-zero words. + static const u32 sample_at = [] { + const char* value = std::getenv("EDEN_NX2_SAMPLE_AT"); + return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 20U; + }(); + if (static u32 sampled{}; sampled++ == sample_at) { + const auto sample = [&](const Texture::TICEntry& tic) { + const u64 size = u64{tic.Width()} * tic.Height() * 4; + if (const auto device = memory_manager.GpuToCpuAddress(tic.Address())) { + rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All); + } + u32 nonzero{}, samples{}; + u64 sum{}; + for (u64 offset = 0; offset < size; offset += 4096 + 4) { + const u32 word = memory_manager.Read<u32>(tic.Address() + offset); + ++samples; + nonzero += word != 0; + sum += (word & 0x3ff) + ((word >> 10) & 0x3ff) + ((word >> 20) & 0x3ff); + } + LOG_INFO(HW_GPU, "Upscale fallback image {:#x}: {}/{} sampled words non-zero, mean channel {}", + tic.Address(), nonzero, samples, samples ? sum / (samples * 3) : 0); + }; + sample(*input); + sample(*output); + // Every 2560x1440 target the in-game passes draw to (from EDEN_NX2_RT_TRACE). + for (const GPUVAddr address : {0x8116a03000ULL, 0x8117ae5000ULL, 0x8114b62000ULL, + 0x8115a63000ULL, 0x8117904000ULL, 0x8114a02000ULL}) { + Texture::TICEntry fake = *input; + const u64 size = 2560ULL * 1440 * 4; + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All); + } + u32 nonzero{}, samples{}, distinct{}; + u32 last{}; + for (u64 offset = 0; offset < size; offset += 4096 + 4) { + const u32 word = memory_manager.Read<u32>(address + offset); + ++samples; + nonzero += word != 0; + distinct += word != last; + last = word; + } + LOG_INFO(HW_GPU, "Upscale fallback target {:#x}: {}/{} non-zero, {} changes", address, nonzero, + samples, distinct); + (void)fake; + } + } + auto source = surface(*input); + // EDEN_NX2_UPSCALE_SOURCE=<gpu address>: blit from another same-sized image (debug aid). + if (static const u64 override_source = [] { + const char* value = std::getenv("EDEN_NX2_UPSCALE_SOURCE"); + return value ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + override_source != 0) { + source.addr_upper = static_cast<u32>(override_source >> 32); + source.addr_lower = static_cast<u32>(override_source); + if (const char* value = std::getenv("EDEN_NX2_UPSCALE_SOURCE_FORMAT")) { + source.format = static_cast<RTFormat>(std::strtoul(value, nullptr, 16)); + } + } + const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); + if (static u32 checked{}; checked++ == sample_at) { + const u64 size = u64{output->Width()} * output->Height() * 4; + if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) { + rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All); + } + u32 nonzero{}, samples{}, changes{}, last{}; + for (u64 offset = 0; offset < size; offset += 4096 + 4) { + const u32 word = memory_manager.Read<u32>(output->Address() + offset); + ++samples; + nonzero += word != 0; + changes += word != last; + last = word; + } + LOG_INFO(HW_GPU, "Upscale fallback output after blit: {}/{} non-zero, {} changes", nonzero, samples, + changes); + } + // Debug: trace the draws that follow one upscale (EDEN_NX2_RT_TRACE_AFTER_UPSCALE). + static const u32 trace_at = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_AFTER_UPSCALE"); + return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 30U; + }(); + if (static u32 count{}; ++count == trace_at) g_nx2_trace_draws_after_upscale = 40; + static u32 logged{}; + if (logged++ < 4) { + LOG_INFO(HW_GPU, "Upscale fallback {:#x} {}x{} block {}/{}/{} -> {:#x} {}x{} block {}/{}/{} levels {}: {}", + input->Address(), input->Width(), input->Height(), input->block_width.Value(), + input->block_height.Value(), input->block_depth.Value(), output->Address(), + output->Width(), output->Height(), output->block_width.Value(), + output->block_height.Value(), output->block_depth.Value(), output->max_mip_level.Value(), + done ? "blitted" : "not accelerated"); + } + return done; +} + void KeplerCompute::ProcessLaunch() { const GPUVAddr launch_desc_loc = regs.launch_desc_loc.Address(); memory_manager.ReadBlockUnsafe(launch_desc_loc, &launch_description, @@ -123,4 +758,162 @@ Texture::TSCEntry KeplerCompute::GetTSCEntry(u32 tsc_index) const { return tsc_entry; } +// Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2): the front +// runs compute macros against its own copy of the native registers (which hold the last value +// written to each, on the back too) and loads and binds the compute macro code itself. +struct KeplerCompute::FrontState { + std::array<u32, 0x1000> registers{}; + std::optional<u32> executing_macro; + std::vector<u32> parameters; + bool compute_uploaded{}; + u32 bind_pointer{}; + MacroEngine* macro{}; ///< Without a native context: the one macro engine +}; + +KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) { + if (!front) [[unlikely]] { + // Copy the back's state and take the macros over while it is idle. + split.Drain(); + front = std::make_unique<FrontState>(); + front->registers = native_registers; + front->executing_macro = native_executing_macro; + front->parameters = std::move(native_parameters); + native_parameters.clear(); + native_executing_macro.reset(); + front->compute_uploaded = native_compute_uploaded; + front->bind_pointer = native_bind_pointer; + front->macro = native_macro; + front_macros = true; + } + return *front; +} + +void KeplerCompute::FrontApply(FrontState& fs, u32 method, u32 argument) { + // As ProcessNativeMethod does with the macro code and bindings. + if (method == 0x45 || method == 0x47) { + fs.compute_uploaded = true; + } + fs.registers[method] = argument; + switch (method) { + case 0x45: + FrontMacroEngine(fs).ClearCode(argument); + break; + case 0x46: + FrontMacroEngine(fs).AddCode(0, argument); + break; + case 0x47: + fs.bind_pointer = argument; + break; + case 0x48: { + const auto positions = FrontMacroPositions(fs); + if (fs.bind_pointer < positions.size()) { + positions[fs.bind_pointer++] = argument; + } else { + LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds"); + } + break; + } + default: + break; + } +} + +MacroEngine& KeplerCompute::FrontMacroEngine(FrontState& fs) { + if (native_context) { + return fs.compute_uploaded ? native_context->compute_engine : native_context->engine; + } + return *fs.macro; +} + +std::span<u32> KeplerCompute::FrontMacroPositions(FrontState& fs) { + if (native_context) { + return fs.compute_uploaded ? std::span<u32>{native_context->compute_positions} + : std::span<u32>{native_context->positions}; + } + return native_macro_positions; +} + +void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, + u32 method, u32 argument, bool is_last_call, + const GPUVAddr* segment) { + auto& fs = Front(split); + if (method >= fs.registers.size()) { + split.Method(this, method, argument, is_last_call, segment); + return; + } + if (method >= 0xe00) { + if (!fs.executing_macro) { + if (method & 1) { + LOG_ERROR(HW_GPU, "Native compute macro starts on argument method"); + return; + } + fs.executing_macro = method; + } + fs.parameters.push_back(argument); + if (is_last_call) { + FrontRunMacro(system, split, fs); + } + return; + } + FrontApply(fs, method, argument); + split.Method(this, method, argument, is_last_call, segment); +} + +void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, + u32 method, const u32* base_start, u32 amount, + u32 methods_pending, const GPUVAddr* segment) { + auto& fs = Front(split); + if (method >= 0xe00 && method < fs.registers.size()) { + for (u32 i = 0; i < amount; ++i) { + FrontCallMethod(system, split, method, base_start[i], methods_pending - i <= 1, segment); + } + return; + } + if (method < fs.registers.size()) { + for (u32 i = 0; i < amount; ++i) { + FrontApply(fs, method, base_start[i]); + } + } + split.MultiMethod(this, method, base_start, amount, methods_pending, segment); +} + +void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, + FrontState& fs) { +#ifdef ARCHITECTURE_x86_64 + const auto positions = FrontMacroPositions(fs); + const u32 start = positions[((*fs.executing_macro - 0xe00) / 2) % positions.size()]; + fs.executing_macro.reset(); + auto& macro = FrontMacroEngine(fs); + if (native_data_ram) { + macro.native_interpreter.active_dram = native_data_ram; + } + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro}; + if (!macro.native_jit) { + macro.native_jit = std::make_shared<TuringMacroJit>(); + } + macro.native_jit->Prepare(macro.native_code, macro.native_code_generation); + const auto result = macro.native_jit->Execute( + start, fs.parameters, std::span<u32>{*macro.native_interpreter.active_dram}, + [&fs](u32 address) { + return address < fs.registers.size() ? fs.registers[address] : 0U; + }, + [&](u32 address, u32 value) { + if (address >= 0xe00) { + return false; + } + FrontApply(fs, address, value); + split.MacroWrite(this, address, value); + return true; + }); + if (!result.success) { + macro.native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: compute start={:#x}, consumed={}, error={}", + start, result.consumed, result.error); + } +#else + fs.executing_macro.reset(); +#endif + fs.parameters.clear(); +} + } // namespace Tegra::Engines diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 078d3d45be..b525636dde 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -7,20 +7,29 @@ #pragma once #include <array> +#include <atomic> #include <cstddef> +#include <memory> #include <optional> +#include <span> #include <vector> #include "common/bit_field.h" #include "common/common_funcs.h" #include "common/common_types.h" #include "video_core/engines/engine_interface.h" #include "video_core/engines/engine_upload.h" +#include "video_core/macro.h" +#include "video_core/native_qmd.h" #include "video_core/textures/texture.h" namespace Core { class System; } +namespace VideoCommon { +class GpuSplit; +} + namespace Tegra { class MemoryManager; } @@ -31,6 +40,9 @@ class RasterizerInterface; namespace Tegra::Engines { +/// Draws left to trace after an upscale fallback (debug aid). +extern std::atomic<u32> g_nx2_trace_draws_after_upscale; + /** * This Engine is known as GK104_Compute. Documentation can be found in: * https://github.com/envytools/envytools/blob/master/rnndb/graph/gk104_compute.xml @@ -51,6 +63,24 @@ public: /// Binds a rasterizer to this engine. void BindRasterizer(VideoCore::RasterizerInterface* rasterizer); + /// The channel's MME data RAM, shared with its 3D engine. + void SetNativeDataRam(std::array<u32, 0xc00>* data_ram) { native_data_ram = data_ram; } + void SetNativeMacroMode(MacroEngine* shared_macro, std::span<u32> positions, NativeMacroContext* context = nullptr) { + if (native_context != context) native_compute_uploaded = false; + native_context = context; + native_mode = shared_macro != nullptr; + upload_state.SetNativePartialTransfers(native_mode); + native_macro = shared_macro; + native_macro_positions = positions; + if (native_mode) execution_mask.set(); + else { + execution_mask.reset(); + execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true; + execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true; + execution_mask[KEPLER_COMPUTE_REG_INDEX(launch)] = true; + } + } + static constexpr std::size_t NumConstBuffers = 8; struct Regs { @@ -211,7 +241,51 @@ public: return indirect_compute; } + bool IsNativeMode() const { return native_mode; } + + /// GPU thread split with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2), called by + /// the front instead of CallMethod for native compute engines: runs their macros and macro + /// uploads against a copy of the registers and sends the register writes to the back. + bool FrontRunsMacros() const { return native_mode; } + void FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, + u32 argument, bool is_last_call, const GPUVAddr* segment); + void FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, + const u32* base_start, u32 amount, u32 methods_pending, + const GPUVAddr* segment); + private: + struct FrontState; + FrontState& Front(VideoCommon::GpuSplit& split); + void FrontApply(FrontState& state, u32 method, u32 argument); + MacroEngine& FrontMacroEngine(FrontState& state); + std::span<u32> FrontMacroPositions(FrontState& state); + void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); + std::unique_ptr<FrontState> front; + bool front_macros{}; ///< The front owns the macro code and bindings (the back skips them) + + bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor); + void ProcessNativeReportSemaphore(u32 argument); + void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd); + bool UpscaleInsteadOfNativeProgram(); + bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call); + NativeQmdScheduler native_qmd; + std::array<u32, 0xc00>* native_data_ram{}; + bool native_mme_dumped{}; + std::optional<u32> native_executing_macro_logged; + bool native_mode{}; + NativeMacroContext* native_context{}; + bool native_compute_uploaded{}; + MacroEngine* native_macro{}; + std::array<u32, 0x1000> native_registers{}; + /// Native methods that only store their value (no macro, QMD, upload, semaphore or code + /// side effect): CallMethod writes those directly. + std::bitset<0xe00> native_plain_methods; + std::span<u32> native_macro_positions; + std::vector<u32> native_parameters; + std::vector<u32> native_memo_parameters; ///< The current call's parameters, for the memo + std::optional<u32> native_executing_macro; + u32 native_bind_pointer{}; + void ProcessLaunch(); void ConsumeSinkImpl(Core::System& system) override; diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index e7da9d9138..6adfcf89b8 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -4,7 +4,10 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> +#include <array> #include <cstring> +#include <limits> #include <optional> #include "common/assert.h" #include "common/bit_util.h" @@ -16,9 +19,14 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu.h" +#include "video_core/gpu_split.h" #include "video_core/memory_manager.h" +#include "video_core/nx2_profile.h" #include "video_core/rasterizer_interface.h" #include "video_core/textures/texture.h" +#ifdef ARCHITECTURE_x86_64 +#include "video_core/macro_turing_jit.h" +#endif namespace Tegra::Engines { @@ -207,11 +215,29 @@ void Maxwell3D::ProcessMacro(Core::System& system, u32 method, const u32* base_s executing_macro = method; } - macro_params.insert(macro_params.end(), base_start, base_start + amount); macro_segments.emplace_back(current_dma_segment, amount); current_macro_dirty |= current_dirty; current_dirty = false; + // Nearly every call arrives whole in one method run (title 1 makes ~1,000,000 a second): + // copy it to the stack instead of growing the parameter vector. Still a copy: the command + // list is often guest memory read in place, which the guest may reuse while the macro runs + // (and the macro memo stores the parameters after it ran). Only accurate GPU mode rewrites + // the parameters in place (RefreshParameters), so it keeps the vector. + // EDEN_NX2_MACRO_COPY=1: always use the vector (for comparisons). + static const bool always_copy = std::getenv("EDEN_NX2_MACRO_COPY") != nullptr; + if (is_last_call && macro_params.empty() && amount <= 64 && !always_copy && + !Settings::IsGPULevelHigh()) { + std::array<u32, 64> parameters; + std::copy_n(base_start, amount, parameters.begin()); + ConsumeSink(system); + CallMacroMethod(system, executing_macro, std::span<const u32>{parameters.data(), amount}); + macro_segments.clear(); + current_macro_dirty = false; + return; + } + macro_params.insert(macro_params.end(), base_start, base_start + amount); + // Call the macro when there are no more parameters in the command buffer if (is_last_call) { ConsumeSink(system); @@ -253,7 +279,10 @@ u32 Maxwell3D::GetMaxCurrentVertices() { const auto& limit = regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); const GPUVAddr gpu_addr_end = limit.Address() + 1; - const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin); + const u64 stream_size = GetNativeMacroEngine().native_turing + ? regs.NativeVertexStreamSize(index) + : gpu_addr_end - gpu_addr_begin; + const u32 address_size = static_cast<u32>((std::min<u64>)(stream_size, (std::numeric_limits<u32>::max)())); num_vertices = (std::max)( num_vertices, address_size / (std::max)(attribute.SizeInBytes(), array.stride.Value())); break; @@ -283,17 +312,25 @@ u32 Maxwell3D::ProcessShadowRam(u32 method, u32 argument) { void Maxwell3D::ConsumeSinkImpl(Core::System& system) { const auto control = shadow_state.shadow_ram_control; + // Sunk methods are state writes only (not in execution_mask): skip unchanged values, as + // CallMethod does. + const bool skip_same = SkipSameValueWrites(); + const auto process = [&](u32 method, u32 value) { + if (!skip_same || regs.reg_array[method] != value) { + ProcessDirtyRegisters(method, value); + } + }; if (control == Regs::ShadowRamControl::Track || control == Regs::ShadowRamControl::TrackWithFilter) { for (auto [method, value] : method_sink) { shadow_state.reg_array[method] = value; - ProcessDirtyRegisters(method, value); + process(method, value); } } else if (control == Regs::ShadowRamControl::Replay) { for (auto [method, value] : method_sink) - ProcessDirtyRegisters(method, shadow_state.reg_array[method]); + process(method, shadow_state.reg_array[method]); } else { for (auto [method, value] : method_sink) - ProcessDirtyRegisters(method, value); + process(method, value); } method_sink.clear(); } @@ -305,17 +342,27 @@ void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) { } void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) { + if (GetNativeMacroEngine().native_turing && method == MAXWELL3D_REG_INDEX(clear_surface)) { + LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument); + } switch (method) { case MAXWELL3D_REG_INDEX(wait_for_idle): return rasterizer->WaitForIdle(); case MAXWELL3D_REG_INDEX(shadow_ram_control): shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument); return; + // With macros on the split GPU thread's front (FrontWrite), the front loads and binds them. case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): - return macro_engine.ClearCode(regs.load_mme.instruction_ptr); + if (front_macros) return; + return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr); case MAXWELL3D_REG_INDEX(load_mme.instruction): - return macro_engine.AddCode(regs.load_mme.instruction_ptr, argument); + if (front_macros) return; + return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument); case MAXWELL3D_REG_INDEX(load_mme.start_address): + if (front_macros) { + ++regs.load_mme.start_address_ptr; + return; + } return ProcessMacroBind(argument); case MAXWELL3D_REG_INDEX(falcon[4]): return ProcessFirmwareCall4(); @@ -372,25 +419,40 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume } } -void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, const std::vector<u32>& parameters) { +void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, std::span<const u32> parameters) { // Reset the current macro. executing_macro = 0; // Lookup the macro offset const u32 entry = - ((method - MacroRegistersStart) >> 1) % static_cast<u32>(macro_positions.size()); + ((method - MacroRegistersStart) >> 1) % static_cast<u32>(GetNativeMacroPositions().size()); // Execute the current macro. - macro_engine.Execute(system, *this, macro_positions[entry], parameters); + GetNativeMacroEngine().Execute(system, *this, GetNativeMacroPositions()[entry], parameters); draw_manager.DrawDeferred(*this); } +// EDEN_NX2_3D_METHODS: log the first write to each 3D register (finds methods Eden doesn't model). +static const bool g_trace_first_method_writes = std::getenv("EDEN_NX2_3D_METHODS") != nullptr; + +void Maxwell3D::TraceFirstMethodWrite(u32 method, u32 argument) { + static std::array<bool, Regs::NUM_REGS> seen{}; + if (method < seen.size() && !seen[method]) { + seen[method] = true; + LOG_INFO(HW_GPU, "3D method first write: {:#x} (byte {:#x}) = {:#x}", method, method * 4, argument); + } +} + void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { // It is an error to write to a register other than the current macro's ARG register before // it has finished execution. + if (executing_macro != 0 && method != executing_macro + 1) { + DumpRecentWrites(method); + } if (executing_macro != 0) { ASSERT(method == executing_macro + 1); } + RecordWrite(method, method_argument, 1, is_last_call ? 1 : 2); // Methods after 0xE00 are special, they're actually triggers for some microcode that was // uploaded to the GPU during initialization. @@ -398,14 +460,56 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument ProcessMacro(system, method, &method_argument, 1, is_last_call); return; } + if (g_trace_first_method_writes) [[unlikely]] { + TraceFirstMethodWrite(method, method_argument); + } ASSERT(method < Regs::NUM_REGS && "Invalid Maxwell3D register, increase the size of the Regs structure"); + if (VideoCommon::Nx2Profile::Enabled()) [[unlikely]] { + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::MethodWrites, 1); + if (regs.reg_array[method] == method_argument) { + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::SameValueWrites, 1); + } + } + if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && front_macros) [[unlikely]] { + back_control_writes[back_control_count++ % back_control_writes.size()] = { + method_argument, static_cast<u32>(shadow_state.shadow_ram_control), is_last_call ? 1U : 0U}; + } const u32 argument = ProcessShadowRam(method, method_argument); + // A state register written with the value it holds changes nothing: no dirty flags (they + // would make the next draw re-check that state), no method processing. Title 1 makes ~6 + // million 3D register writes a second, ~60% of them same-value, most from macro replays. + // Methods with side effects (execution_mask: draws, clears, constant buffer data and + // binds, semaphores, ...) always run. EDEN_NX2_SKIP_SAME=0 turns it off. + if (!execution_mask[method]) { + // ProcessMethodCall (and the draw manager behind it) acts only on executable methods. + if (!SkipSameValueWrites() || regs.reg_array[method] != argument) { + ProcessDirtyRegisters(method, argument); + } + return; + } ProcessDirtyRegisters(method, argument); ProcessMethodCall(method, argument, method_argument, is_last_call); } +void Maxwell3D::DumpRecentWrites(u32 method) { + static u32 dumps{}; + if (dumps++ >= 4) { + return; + } + LOG_ERROR(HW_GPU, "Write to {:#x} while macro {:#x} collects parameters ({} so far); last writes:", + method, executing_macro, macro_params.size()); + for (size_t i = 0; i < recent_writes.size(); ++i) { + const auto& w = recent_writes[(recent_write_index + i) % recent_writes.size()]; + if (w.amount != 0) { + LOG_ERROR(HW_GPU, " method {:#x} arg {:#x} amount {} pending {}", w.method, w.argument, + w.amount, w.pending); + } + } +} + void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) { + RecordWrite(method, amount ? base_start[0] : 0, amount, methods_pending); // Methods after 0xE00 are special, they're actually triggers for some microcode that was // uploaded to the GPU during initialization. if (method >= MacroRegistersStart) { @@ -453,11 +557,11 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas } void Maxwell3D::ProcessMacroUpload(u32 data) { - macro_engine.AddCode(regs.load_mme.instruction_ptr++, data); + GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr++, data); } void Maxwell3D::ProcessMacroBind(u32 data) { - macro_positions[regs.load_mme.start_address_ptr++] = data; + GetNativeMacroPositions()[regs.load_mme.start_address_ptr++] = data; } void Maxwell3D::ProcessFirmwareCall4() { @@ -586,6 +690,9 @@ void Maxwell3D::ProcessCounterReset() { } void Maxwell3D::ProcessSyncPoint() { + if (GetNativeMacroEngine().native_turing) { + LOG_DEBUG(HW_GPU, "Native 3D syncpoint increment: id={}, raw={:#x}", regs.sync_info.sync_point.Value(), regs.reg_array[MAXWELL3D_REG_INDEX(sync_info)]); + } const u32 sync_point = regs.sync_info.sync_point.Value(); [[maybe_unused]] const u32 cache_flush = regs.sync_info.clean_l2.Value(); rasterizer->SignalSyncPoint(sync_point); @@ -595,6 +702,15 @@ void Maxwell3D::ProcessCBBind(size_t stage_index) { // Bind the buffer currently in CB_ADDRESS to the specified index in the desired shader // stage. const auto& bind_data = regs.bind_groups[stage_index]; + static std::array<std::array<bool, 32>, 5> seen{}; + if (!seen[stage_index][bind_data.shader_slot]) { + seen[stage_index][bind_data.shader_slot] = true; + LOG_INFO(HW_GPU, "Constant buffer bind: stage {} slot {} address {:#x} size {:#x}", stage_index, + bind_data.shader_slot.Value(), regs.const_buffer.Address(), regs.const_buffer.size); + } + if (bind_data.shader_slot >= Regs::MaxConstBuffers) { + return; // slot field is 5 bits; the state array holds 18 + } auto& buffer = state.shader_stages[stage_index].const_buffers[bind_data.shader_slot]; buffer.enabled = bind_data.valid.Value() != 0; buffer.address = regs.const_buffer.Address(); @@ -666,4 +782,416 @@ void Maxwell3D::SetHLEReplacementAttributeType(u32 bank, u32 offset, replace_table.emplace(key, name); } +// Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2). The front keeps +// what macros read: the registers as the back thread will hold them when it reaches the same +// point of the method stream. That is every write the front sends, with the shadow RAM applied, +// plus the few changes the back makes on its own (constant buffer offset, shadow_scratch[0] on +// falcon[4], the macro bind pointer); the front loads and binds the macro code itself. +struct Maxwell3D::FrontState { + std::array<ControlWrite, 16> control_writes{}; + u32 control_count{}; + std::array<u32, Regs::NUM_REGS> regs{}; + std::array<u32, Regs::NUM_REGS> shadow{}; + Regs::ShadowRamControl control{}; + u32 executing_macro{}; + std::vector<u32> parameters; + std::vector<u32> fifo; + u64 calls{}; + /// A method the draw manager handles was sent since the back last ran DrawDeferred: only + /// then can it have a deferred instanced draw to issue at the end of a macro. + bool draw_pending{true}; + bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once +}; + +namespace { + +// EDEN_NX2_SPLIT_VERIFY=1: every 4096 front macro calls, wait for the back and compare the +// front's registers with the back's (logs and resynchronizes differences). +bool FrontVerifyEnabled() { + static const bool verify = std::getenv("EDEN_NX2_SPLIT_VERIFY") != nullptr; + return verify; +} + +// Executable methods ProcessMethodCall handles itself; the rest go to the draw manager. +bool FrontHandledOutsideDrawManager(u32 method) { + constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); + if (method >= cb_data && method < cb_data + 16) { + return true; + } + switch (method) { + case MAXWELL3D_REG_INDEX(wait_for_idle): + case MAXWELL3D_REG_INDEX(shadow_ram_control): + case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): + case MAXWELL3D_REG_INDEX(load_mme.instruction): + case MAXWELL3D_REG_INDEX(load_mme.start_address): + case MAXWELL3D_REG_INDEX(falcon[4]): + case MAXWELL3D_REG_INDEX(bind_groups[0].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[1].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[2].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[3].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config): + case MAXWELL3D_REG_INDEX(report_semaphore.query): + case MAXWELL3D_REG_INDEX(render_enable.mode): + case MAXWELL3D_REG_INDEX(clear_report_value): + case MAXWELL3D_REG_INDEX(sync_info): + case MAXWELL3D_REG_INDEX(launch_dma): + case MAXWELL3D_REG_INDEX(inline_data): + case MAXWELL3D_REG_INDEX(fragment_barrier): + case MAXWELL3D_REG_INDEX(invalidate_texture_data_cache): + case MAXWELL3D_REG_INDEX(tiled_cache_barrier): + return true; + default: + return false; + } +} + +// EDEN_NX2_SPLIT_SKIP=0: send state writes that keep the register's value too. +bool FrontSkipSame() { + static const bool skip = [] { + const char* value = std::getenv("EDEN_NX2_SPLIT_SKIP"); + return value == nullptr || value[0] != '0'; + }(); + return skip; +} + +} // Anonymous namespace + +Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSplit& split) { + if (!front) [[unlikely]] { + // Copy the back's state and take the macros over while it is idle (state writes it + // still holds in its method sink first). + split.Drain(); + ConsumeSink(system); + front = std::make_unique<FrontState>(); + std::memcpy(front->regs.data(), regs.reg_array.data(), sizeof(front->regs)); + std::memcpy(front->shadow.data(), shadow_state.reg_array.data(), sizeof(front->shadow)); + front->control = shadow_state.shadow_ram_control; + front->skip_same = SkipSameValueWrites() && FrontSkipSame(); + front->executing_macro = executing_macro; + front->parameters = std::move(macro_params); + macro_params.clear(); + executing_macro = 0; + front_macros = true; + } + return *front; +} + +void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 method, + u32 argument, bool is_last_call, bool from_macro) { + namespace Profile = VideoCommon::Nx2Profile; + // As ProcessShadowRam does on the back. + u32 value = argument; + bool shadow_kept = true; + if (fs.control == Regs::ShadowRamControl::Track || + fs.control == Regs::ShadowRamControl::TrackWithFilter) { + shadow_kept = fs.shadow[method] == argument; + fs.shadow[method] = argument; + } else if (fs.control == Regs::ShadowRamControl::Replay) { + value = fs.shadow[method]; + } + if (Profile::Enabled()) [[unlikely]] { + Profile::Add(Profile::Counter::MethodWrites, 1); + } + if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] { + fs.control_writes[fs.control_count++ % fs.control_writes.size()] = { + argument, static_cast<u32>(fs.control), from_macro ? 2U : 1U}; + } + if (!execution_mask[method]) { + // Nothing would change on the back: CallMethod and ConsumeSinkImpl skip same-value state + // writes, and the shadow RAM (tracked by default) holds the value already. + if (fs.regs[method] == value && shadow_kept && fs.skip_same) { + if (Profile::Enabled()) [[unlikely]] { + Profile::Add(Profile::Counter::SameValueWrites, 1); + } + return; + } + fs.regs[method] = value; + } else { + fs.regs[method] = value; + constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); + switch (method) { + case MAXWELL3D_REG_INDEX(shadow_ram_control): + // The back's mode is shadow_state.shadow_ram_control: the shadow copy of this very + // register, which a later Replay write reads back. + fs.control = static_cast<Regs::ShadowRamControl>(argument); + fs.shadow[method] = argument; + break; + case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): + GetNativeMacroEngine().ClearCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)]); + break; + case MAXWELL3D_REG_INDEX(load_mme.instruction): + GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value); + break; + case MAXWELL3D_REG_INDEX(load_mme.start_address): { + const auto positions = GetNativeMacroPositions(); + u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)]; + if (pointer < positions.size()) { + positions[pointer] = value; + } + ++pointer; + break; + } + case MAXWELL3D_REG_INDEX(falcon[4]): + fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1; + break; + default: + if (method >= cb_data && method < cb_data + 16) { + fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += sizeof(u32); + } else if (!FrontHandledOutsideDrawManager(method)) { + fs.draw_pending = true; + } + break; + } + } + if (from_macro) { + split.MacroWrite(this, method, argument); + } else { + split.Method(this, method, argument, is_last_call, nullptr); + } +} + +void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, + u32 argument, bool is_last_call) { + auto& fs = Front(system, split); + if (method >= MacroRegistersStart) { + if (fs.executing_macro == 0) { + fs.executing_macro = method; + } + fs.parameters.push_back(argument); + if (is_last_call) { + FrontRunMacro(system, split, fs); + } + return; + } + FrontWrite(split, fs, method, argument, is_last_call, false); +} + +void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, + u32 method, const u32* base_start, u32 amount, + u32 methods_pending) { + auto& fs = Front(system, split); + if (method >= MacroRegistersStart) { + if (fs.executing_macro == 0) { + fs.executing_macro = method; + } + fs.parameters.insert(fs.parameters.end(), base_start, base_start + amount); + if (amount == methods_pending) { + FrontRunMacro(system, split, fs); + } + return; + } + // What CallMultiMethod does to the registers on the back, then one entry for all the words. + constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); + const auto tracked = [&fs] { + return fs.control == Regs::ShadowRamControl::Track || + fs.control == Regs::ShadowRamControl::TrackWithFilter; + }; + if (method >= cb_data && method < cb_data + 16) { + fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += amount * sizeof(u32); + } else if (method == MAXWELL3D_REG_INDEX(inline_data)) { + // Upload data only. + } else if ((method == MAXWELL3D_REG_INDEX(draw_inline_index) || + method == MAXWELL3D_REG_INDEX(inline_index_2x16.even) || + method == MAXWELL3D_REG_INDEX(inline_index_4x8.index0)) && + fs.control != Regs::ShadowRamControl::Replay) { + if (amount != 0) { + const u32 last = base_start[amount - 1]; + if (tracked()) { + fs.shadow[method] = last; + } + fs.regs[method] = last; + } + fs.draw_pending = true; + } else { + // CallMethod for each word, macro uploads among them (load_mme.instruction). A word may + // change the shadow RAM mode for the next (a run of SET_SHADOW_RAM_CONTROL writes). + for (u32 i = 0; i < amount; ++i) { + const u32 argument = base_start[i]; + u32 value = argument; + if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] { + fs.control_writes[fs.control_count++ % fs.control_writes.size()] = { + argument, static_cast<u32>(fs.control), 3U}; + } + if (tracked()) { + fs.shadow[method] = argument; + } else if (fs.control == Regs::ShadowRamControl::Replay) { + value = fs.shadow[method]; + } + fs.regs[method] = value; + if (!execution_mask[method]) { + continue; + } + switch (method) { + case MAXWELL3D_REG_INDEX(shadow_ram_control): + fs.control = static_cast<Regs::ShadowRamControl>(argument); + fs.shadow[method] = argument; // The same word on the back (see FrontWrite) + break; + case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): + GetNativeMacroEngine().ClearCode(value); + break; + case MAXWELL3D_REG_INDEX(load_mme.instruction): + GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value); + break; + case MAXWELL3D_REG_INDEX(load_mme.start_address): { + const auto positions = GetNativeMacroPositions(); + u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)]; + if (pointer < positions.size()) { + positions[pointer] = value; + } + ++pointer; + break; + } + case MAXWELL3D_REG_INDEX(falcon[4]): + fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1; + break; + default: + if (!FrontHandledOutsideDrawManager(method)) { + fs.draw_pending = true; + } + break; + } + } + } + split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr); +} + +void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, + FrontState& fs) { +#ifdef ARCHITECTURE_x86_64 + const u32 method = fs.executing_macro; + fs.executing_macro = 0; + auto& engine = GetNativeMacroEngine(); + const auto positions = GetNativeMacroPositions(); + const u32 start = positions[((method - MacroRegistersStart) >> 1) % positions.size()]; + { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro}; + engine.native_interpreter.active_dram = &native_data_ram; + if (!engine.native_jit) { + engine.native_jit = std::make_shared<TuringMacroJit>(); + } + engine.native_jit->Prepare(engine.native_code, engine.native_code_generation); + fs.fifo.assign(fs.parameters.begin(), fs.parameters.end()); + const auto result = engine.native_jit->Execute( + start, fs.fifo, std::span<u32>{native_data_ram}, + [&fs](u32 address) { return address < Regs::NUM_REGS ? fs.regs[address] : 0U; }, + [&](u32 address, u32 value) { + if (address >= MacroRegistersStart) { + return false; + } + FrontWrite(split, fs, address, value, true, true); + // MME_DMA_READ (into data RAM) and MME_DMA_READ_FIFOED (into the load FIFO). + if (address == 0x157 || address == 0x158) { + FrontDmaRead(split, fs, address, value); + } + return true; + }); + if (!result.success) { + engine.native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: start={:#x}, consumed={}/{}, error={}", + start, result.consumed, fs.parameters.size(), result.error); + } + } + fs.parameters.clear(); + if (fs.draw_pending) { + fs.draw_pending = false; + split.Defer([this, system_ptr = &system] { FinishFrontMacro(*system_ptr); }); + } + if (FrontVerifyEnabled() && (++fs.calls & 4095) == 0) [[unlikely]] { + FrontVerify(split, fs); + } +#else + fs.executing_macro = 0; + fs.parameters.clear(); +#endif +} + +void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 method, + u32 count) { + // The words may be written by methods still queued for the back (indirect draw arguments + // from a culling dispatch), and the caches that may hold newer ones belong to it. + split.Drain(); + const GPUVAddr source = (GPUVAddr{fs.regs[0x154] & 0x1ffffff} << 32) | fs.regs[0x155]; + const u32 ram = fs.regs[0x156]; + const u32 words = std::min<u32>(count, 0x1000); + const size_t read_bytes = size_t{words} * sizeof(u32); + if (VideoCommon::Nx2Profile::Enabled()) { + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes); + } + if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + memory_manager.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); + } + for (u32 i = 0; i < words; ++i) { + const u32 word = memory_manager.Read<u32>(source + u64{i} * sizeof(u32)); + if (method == 0x158) { + fs.fifo.push_back(word); + } else if (ram + i < native_data_ram.size()) { + native_data_ram[ram + i] = word; + } + } +} + +void Maxwell3D::FrontVerify(VideoCommon::GpuSplit& split, FrontState& fs) { + split.Drain(); + static u64 checks{}; + static u64 differences{}; + ++checks; + // State writes the back still holds in its method sink count as done. + std::array<u32, Regs::NUM_REGS> expected; + std::memcpy(expected.data(), regs.reg_array.data(), sizeof(expected)); + for (const auto& [method, value] : method_sink) { + expected[method] = shadow_state.shadow_ram_control == Regs::ShadowRamControl::Replay + ? shadow_state.reg_array[method] + : value; + } + static u64 control_differences{}; + if ((fs.control != shadow_state.shadow_ram_control || + fs.shadow[MAXWELL3D_REG_INDEX(shadow_ram_control)] != shadow_state.reg_array[MAXWELL3D_REG_INDEX(shadow_ram_control)] || + fs.regs[MAXWELL3D_REG_INDEX(shadow_ram_control)] != expected[MAXWELL3D_REG_INDEX(shadow_ram_control)]) && + control_differences < 4) { + LOG_WARNING(HW_GPU, "Split shadow RAM control writes, front ({} total) / back ({} total):", + fs.control_count, back_control_count); + for (u32 i = 0; i < 16; ++i) { + const auto& f = fs.control_writes[(fs.control_count + i) % 16]; + const auto& b = back_control_writes[(back_control_count + i) % 16]; + LOG_WARNING(HW_GPU, " front {:#x} (mode {}, path {}) | back {:#x} (mode {}, last {})", + f.argument, f.before, f.path, b.argument, b.before, b.path); + } + } + for (u32 i = 0; i < Regs::NUM_REGS; ++i) { + if (fs.regs[i] != expected[i]) { + if (differences++ < 64) { + LOG_WARNING(HW_GPU, "Split front register {:#x}: front {:#x}, back {:#x}", i, + fs.regs[i], expected[i]); + } + fs.regs[i] = expected[i]; + } + } + if (fs.control != shadow_state.shadow_ram_control) { + if (control_differences++ < 16) { + LOG_WARNING(HW_GPU, "Split front shadow RAM control {} vs back {}", + static_cast<u32>(fs.control), + static_cast<u32>(shadow_state.shadow_ram_control)); + } + fs.control = shadow_state.shadow_ram_control; + } + for (u32 i = 0; i < Regs::NUM_REGS; ++i) { + if (fs.shadow[i] != shadow_state.reg_array[i]) { + if (control_differences++ < 16) { + LOG_WARNING(HW_GPU, "Split front shadow register {:#x}: front {:#x}, back {:#x}", i, + fs.shadow[i], shadow_state.reg_array[i]); + } + fs.shadow[i] = shadow_state.reg_array[i]; + } + } + if ((checks & 63) == 1) { + LOG_INFO(HW_GPU, "Split front register checks: {} done, {} differences", checks, differences); + } +} + +void Maxwell3D::FinishFrontMacro(Core::System& system) { + // What ProcessMacro and CallMacroMethod do around the macro. + ConsumeSink(system); + draw_manager.DrawDeferred(*this); +} + } // namespace Tegra::Engines diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0cab2ae986..0121f1a01c 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -9,8 +9,10 @@ #include <array> #include <bitset> #include <cmath> +#include <cstdlib> #include <limits> #include <optional> +#include <span> #include <type_traits> #include <vector> @@ -35,6 +37,10 @@ namespace Tegra { class MemoryManager; } +namespace VideoCommon { +class GpuSplit; +} + namespace VideoCore { class RasterizerInterface; } @@ -2605,6 +2611,12 @@ public: u32 stop_shader_mask; }; + // Native C597/C797 use byte counts at 0x600 rather than legacy end addresses. + u64 NativeVertexStreamSize(std::size_t index) const { + const std::size_t method = 0x600 / sizeof(u32) + index * 2; + return (u64{reg_array[method] & 0xffU} << 32) | reg_array[method + 1]; + } + // clang-format off union { struct { @@ -3129,12 +3141,40 @@ public: /// Reads a register value located at the input method address u32 GetRegisterValue(u32 method) const; + void TraceFirstMethodWrite(u32 method, u32 argument); + + /// GPU memory, for Turing macros that read memory into their FIFO or data RAM. + MemoryManager& GetMemoryManager() { + return memory_manager; + } + + void SetNativeTuringMacroMode(bool enabled) { + GetNativeMacroEngine().native_turing = enabled; + upload_state.SetNativePartialTransfers(enabled); + } + void SetNativeMacroContext(std::shared_ptr<NativeMacroContext> context) { + native_macro_context = std::move(context); + } + MacroEngine& GetNativeMacroEngine() { return native_macro_context ? native_macro_context->engine : macro_engine; } + /// This channel's MME data RAM (shared with its compute engine). + std::array<u32, 0xc00>& GetNativeDataRam() { return native_data_ram; } + std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; } + + /// Write the value to the register identified by method. void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override; /// Write multiple values to the register identified by method. void CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) override; + /// GPU thread split with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2), called by + /// the front instead of CallMethod: runs macros and macro uploads against a copy of the + /// registers and sends the register writes to the back thread. + void FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, + u32 argument, bool is_last_call); + void FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, + const u32* base_start, u32 amount, u32 methods_pending); + bool ShouldExecute() const { return execute_on; } @@ -3221,7 +3261,16 @@ private: * @param method Method to call * @param parameters Arguments to the method call */ - void CallMacroMethod(Core::System& system, u32 method, const std::vector<u32>& parameters); + void CallMacroMethod(Core::System& system, u32 method, std::span<const u32> parameters); + + /// Whether state register writes that keep the register's value are dropped (CallMethod). + static bool SkipSameValueWrites() { + static const bool skip = [] { + const char* value = std::getenv("EDEN_NX2_SKIP_SAME"); + return value == nullptr || value[0] != '0'; + }(); + return skip; + } /// Handles writes to the macro uploading register. void ProcessMacroUpload(u32 data); @@ -3260,10 +3309,25 @@ private: /// Macro method that is currently being executed / being fed parameters. u32 executing_macro = 0; + + struct RecentWrite { + u32 method; + u32 argument; + u32 amount; + u32 pending; + }; + std::array<RecentWrite, 48> recent_writes{}; + size_t recent_write_index = 0; + void RecordWrite(u32 method, u32 argument, u32 amount, u32 pending) { + recent_writes[recent_write_index++ % recent_writes.size()] = {method, argument, amount, pending}; + } + void DumpRecentWrites(u32 method); /// Parameters that have been submitted to the macro call so far. std::vector<u32> macro_params; /// Interpreter for the macro codes uploaded to the GPU. + std::shared_ptr<NativeMacroContext> native_macro_context; + std::array<u32, 0xc00> native_data_ram{}; MacroEngine macro_engine; Upload::State upload_state; @@ -3272,6 +3336,28 @@ private: std::vector<std::pair<GPUVAddr, size_t>> macro_segments; bool current_macro_dirty{}; + + // Split GPU thread, macros on the front (FrontCallMethod): the front's state, and whether + // the front owns the macro code and bindings (the back thread then leaves them alone). + struct FrontState; + FrontState& Front(Core::System& system, VideoCommon::GpuSplit& split); + void FrontWrite(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 argument, + bool is_last_call, bool from_macro); + void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); + void FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 count); + void FrontVerify(VideoCommon::GpuSplit& split, FrontState& state); + void FinishFrontMacro(Core::System& system); + std::unique_ptr<FrontState> front; + bool front_macros{}; + // EDEN_NX2_SPLIT_VERIFY: the last SET_SHADOW_RAM_CONTROL writes each side saw (argument, + // mode before, path), printed when the sides disagree. + struct ControlWrite { + u32 argument; + u32 before; + u32 path; + }; + std::array<ControlWrite, 16> back_control_writes{}; + u32 back_control_count{}; }; #define ASSERT_REG_POSITION(field_name, position) \ diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index ed4675ed12..0b0541799b 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> #include "common/algorithm.h" #include "common/assert.h" #include "common/logging.h" @@ -42,7 +43,11 @@ void MaxwellDMA::ConsumeSinkImpl(Core::System& system) { } void MaxwellDMA::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - ASSERT_MSG(method < NUM_REGS, "Invalid MaxwellDMA register"); + if (method >= NUM_REGS) { + // Ampere copy (Switch 2) can use methods past this register file; don't write past it. + LOG_DEBUG(HW_GPU, "Ignoring MaxwellDMA method {:#x} past the register file", method); + return; + } regs.reg_array[method] = method_argument; @@ -59,6 +64,32 @@ void MaxwellDMA::CallMultiMethod(Core::System& system, u32 method, const u32* ba void MaxwellDMA::Launch() { LOG_TRACE(Render_OpenGL, "DMA copy {:#x} -> {:#x}", static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out)); + // EDEN_NX2_TRACE_DST=<hex>: also log copies writing within 32 MiB above that GPU address. + static const u64 trace_destination = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_DST"); + return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + if (trace_destination != 0 && GPUVAddr(regs.offset_out) >= trace_destination && + GPUVAddr(regs.offset_out) < trace_destination + (32ULL << 20)) { + static u32 traced_destination{}; + if (traced_destination++ < 40) { + LOG_INFO(HW_GPU, + "nx2 DMA to traced dst: {:#x} -> {:#x} lines={} length={} pitch in={:#x} " + "out={:#x} launch={:#x} src_layout={} dst_layout={}", + static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out), + regs.line_count, regs.line_length_in, regs.pitch_in, regs.pitch_out, + regs.reg_array[0xc0], regs.launch_dma.src_memory_layout.Value() == LaunchDMA::MemoryLayout::PITCH ? "pitch" : "block", + regs.launch_dma.dst_memory_layout.Value() == LaunchDMA::MemoryLayout::PITCH ? "pitch" : "block"); + } + } + if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) { + static u32 traced{}; + if (traced++ < 64) { + LOG_INFO(HW_GPU, "nx2 DMA {:#x} -> {:#x} lines={} length={} launch={:#x}", + static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out), + regs.line_count, regs.line_length_in, regs.reg_array[0xc0]); + } + } // TODO(Subv): Perform more research and implement all features of this engine. const LaunchDMA& launch = regs.launch_dma; @@ -219,7 +250,7 @@ void MaxwellDMA::CopyBlockLinearToPitch() { UnswizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth, x_offset, src_params.origin.y, x_elements, regs.line_count, block_height, - block_depth, dst_operand.pitch); + block_depth, dst_operand.pitch, memory_manager.UsesNativeTiling()); } void MaxwellDMA::CopyPitchToBlockLinear() { @@ -284,7 +315,7 @@ void MaxwellDMA::CopyPitchToBlockLinear() { // If the input is linear and the output is tiled, swizzle the input and copy it over. SwizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth, x_offset, dst_params.origin.y, x_elements, regs.line_count, block_height, - block_depth, regs.pitch_in); + block_depth, regs.pitch_in, memory_manager.UsesNativeTiling()); } void MaxwellDMA::CopyBlockLinearToBlockLinear() { @@ -337,11 +368,11 @@ void MaxwellDMA::CopyBlockLinearToBlockLinear() { UnswizzleSubrect(intermediate_buffer, tmp_read_buffer, bytes_per_pixel, src_width, src.height, src.depth, src_x_offset, src.origin.y, x_elements, regs.line_count, - src.block_size.height, src.block_size.depth, pitch); + src.block_size.height, src.block_size.depth, pitch, memory_manager.UsesNativeTiling()); SwizzleSubrect(tmp_write_buffer, intermediate_buffer, bytes_per_pixel, dst_width, dst.height, dst.depth, dst_x_offset, dst.origin.y, x_elements, regs.line_count, - dst.block_size.height, dst.block_size.depth, pitch); + dst.block_size.height, dst.block_size.depth, pitch, memory_manager.UsesNativeTiling()); } void MaxwellDMA::ReleaseSemaphore() { diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 0b05d31718..0bb1809aa3 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -4,10 +4,14 @@ // SPDX-FileCopyrightText: 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include <atomic> +#include <cstdlib> + #include "common/assert.h" #include "common/logging.h" #include "common/settings.h" #include "core/core.h" +#include "video_core/nx2_profile.h" #include "video_core/control/channel_state.h" #include "video_core/dma_pusher.h" #include "video_core/engines/fermi_2d.h" @@ -17,7 +21,9 @@ #include "video_core/engines/maxwell_dma.h" #include "video_core/engines/puller.h" #include "video_core/gpu.h" +#include "video_core/gpu_split.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/rasterizer_interface.h" namespace Tegra::Engines { @@ -27,14 +33,120 @@ namespace { constexpr u32 Gf100BindClassMask = 0xffff; constexpr u32 Gf100BindValidMask = 0x1f0000 | Gf100BindClassMask; +// Reads a semaphore word, flushing a GPU-written copy out of the buffer cache first. +u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) { + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + device && dma_pusher.rasterizer->MustFlushRegion(*device, sizeof(u32), + VideoCommon::CacheType::BufferCache)) { + dma_pusher.rasterizer->FlushRegion(*device, sizeof(u32), + VideoCommon::CacheType::BufferCache); + } + return dma_pusher.memory_manager.Read<u32>(address); +} + +// With the GPU thread split (gpu_split.h) the rasterizer belongs to the back thread: work that +// only feeds it runs there in order; reads of what it wrote first wait for it (DrainBack). +template <typename Func> +void OnBack(const Func& func) { + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + split.Defer(func); + } else { + func(); + } +} + +void DrainBack() { + VideoCommon::GpuSplit::Get().Drain(); +} + } // Anonymous namespace +bool Puller::PendingAcquire::Matches(u32 current) const { + switch (static_cast<GpuSemaphoreOperation>(operation)) { + case GpuSemaphoreOperation::AcquireEqual: + return current == value; + case GpuSemaphoreOperation::AcquireGequal: + return current >= value; + case GpuSemaphoreOperation::AcquireMask: + return (current & value) != 0; + default: + return true; + } +} + +bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { + if (!pending_acquire) { + return true; + } + // The release may still be queued for the split GPU thread's back half (and the caches + // ReadSemaphoreWord asks are its state). + DrainBack(); + const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address); + if (Nx2SemaphoreWaiters::Get().Latched(this)) { + if (static std::atomic<u32> rescued{}; + !pending_acquire->Matches(current) && rescued.fetch_add(1) < 64) { + LOG_INFO(HW_GPU, "Channel {} met semaphore {:#x} value {:#x} at its release (now {:#x})", + dma_pusher.channel_state.bind_id, pending_acquire->address, + pending_acquire->value, current); + } + return true; + } + if (pending_acquire->Matches(current)) { + return true; + } + if (!pending_acquire->reported && + std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) { + pending_acquire->reported = true; + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); + LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; " + "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}", + dma_pusher.channel_state.bind_id, pending_acquire->address, + pending_acquire->operation, pending_acquire->value, current, device.value_or(0), + device && dma_pusher.rasterizer->MustFlushRegion(*device, 16, VideoCommon::CacheType::All), + dma_pusher.memory_manager.Read<u32>(pending_acquire->address & ~GPUVAddr{0xf}), + dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 4), + dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 8), + dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 12)); + } + return false; +} + +bool Puller::ResumeAcquire(DmaPusher& dma_pusher) { + if (!CanResumeAcquire(dma_pusher)) { + return false; + } + pending_acquire.reset(); + Nx2SemaphoreWaiters::Get().Remove(this); + return true; +} + void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { LOG_DEBUG(HW_GPU, "Binding subchannel {} to engine {:#x}", method_call.subchannel, method_call.argument); u32 engine = method_call.argument; if ((engine & ~Gf100BindClassMask) != 0 && (engine & ~Gf100BindValidMask) == 0) { engine &= Gf100BindClassMask; } + // Switch 2 binds Ampere classes; run them on the Maxwell engines (same method layout for + // most state). Class 0 unbinds the subchannel. + switch (engine & 0xFFFF) { + case 0xC797: // AMPERE_B + engine = static_cast<u32>(EngineID::MAXWELL_B); + break; + case 0xC7C0: // AMPERE_COMPUTE_B + engine = static_cast<u32>(EngineID::KEPLER_COMPUTE_B); + break; + case 0xC7B5: // AMPERE_DMA_COPY_B + engine = static_cast<u32>(EngineID::MAXWELL_DMA_COPY_A); + break; + case 0x0: + if (g_nx2_ampere) { + LOG_DEBUG(HW_GPU, "Switch 2 unbinds subchannel {}", method_call.subchannel); + return; + } + break; + default: + break; + } const auto engine_id = static_cast<EngineID>(engine); bound_engines[method_call.subchannel] = engine_id; @@ -68,10 +180,12 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { case Puller::FenceOperation::Acquire: // UNIMPLEMENTED_MSG("Channel Scheduling pending."); // WaitFence(regs.fence_action.syncpoint_id, regs.fence_value); - dma_pusher.rasterizer->ReleaseFences(); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }); break; case Puller::FenceOperation::Increment: - dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id); + OnBack([rasterizer = dma_pusher.rasterizer, id = static_cast<u32>(regs.fence_action.syncpoint_id.Value())] { + rasterizer->SignalSyncPoint(id); + }); break; default: UNIMPLEMENTED_MSG("Unimplemented operation {}", regs.fence_action.op.Value()); @@ -82,11 +196,73 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); + // EDEN_NX2_SEM_WATCH=<gpu address>: log every release/acquire on the 16 bytes there. + static const GPUVAddr watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{}; + }(); + if (watch != 0 && (regs.semaphore_address.SemaphoreAddress() & ~GPUVAddr{0xf}) == watch) { + LOG_INFO(HW_GPU, "Semaphore watch: channel {} as {} device {:#x} op {} {:#x} value {:#x} (now {:#x})", + dma_pusher.channel_state.bind_id, dma_pusher.memory_manager.GetID(), + dma_pusher.memory_manager.GpuToCpuAddress(regs.semaphore_address.SemaphoreAddress()).value_or(0), + static_cast<u32>(op), + regs.semaphore_address.SemaphoreAddress(), regs.semaphore_sequence, + dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress())); + } if (op == GpuSemaphoreOperation::WriteLong) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_sequence; - dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0); + // RELEASE_SIZE (bit 24) set means a 4-byte release without the timestamp; writing 16 + // bytes there clobbers the words next to it (Switch 2 packs semaphores tightly). + const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0 + ? VideoCommon::QueryPropertiesFlags::IsAFence + : VideoCommon::QueryPropertiesFlags::HasTimeout; + const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0; + if (mirror != 0) { + LOG_DEBUG(HW_GPU, "Switch 2 semaphore {:#x}={:#x} mirrored to {:#x}", sequence_address, payload, mirror); + } + OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, mirror, properties, payload] { + rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); + if (mirror != 0) { + rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0); + } + }); + } else if (g_nx2_ampere) { + // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has + // written the commands this channel runs next: suspend the channel until the value is + // met instead of reading ahead (approach from changes.patch). + PendingAcquire acquire{ + .address = regs.semaphore_address.SemaphoreAddress(), + .value = regs.semaphore_sequence, + .operation = static_cast<u32>(op), + .since = std::chrono::steady_clock::now(), + .reported = false, + }; + if (op != GpuSemaphoreOperation::AcquireEqual && op != GpuSemaphoreOperation::AcquireGequal && + op != GpuSemaphoreOperation::AcquireMask) { + LOG_ERROR(HW_GPU, "Invalid semaphore operation {}", static_cast<u32>(op)); + return; + } + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); + DrainBack(); + if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + return; + } + // Registered before the fences run, so a release among them that meets the value counts + // even if a later one overwrites it. + auto& waiters = Nx2SemaphoreWaiters::Get(); + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address)) { + waiters.Add(this, *device, acquire.value, acquire.operation); + } + dma_pusher.rasterizer->ReleaseFences(); + if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + pending_acquire = acquire; + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireWaits, 1); + } else { + waiters.Remove(this); + } } else { + DrainBack(); do { const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress()); regs.acquire_source = true; @@ -120,10 +296,14 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_release; - dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, payload] { + rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + }); } void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { + DrainBack(); u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress()); const auto value = regs.semaphore_acquire; while (word != value) { @@ -137,8 +317,46 @@ void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { } } +// Ampere SEM_EXECUTE (method 0x1B): operation in bits 2:0 on the semaphore at +// SEM_ADDR_HI:SEM_ADDR_LO (methods 0x18/0x17) with payload SEM_PAYLOAD_LO (0x19). +void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { + const GPUVAddr address = (GPUVAddr{regs.reg_array[0x18] & 0x1FFFFFF} << 32) | + (regs.reg_array[0x17] & ~3U); + const u32 payload = regs.reg_array[0x19]; + const u32 operation = argument & 7; + if (operation == 1) { // RELEASE + OnBack([rasterizer = dma_pusher.rasterizer, address, payload] { + rasterizer->Query(address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + }); + return; + } + // Acquires: check once after flushing pending fences; never block the GPU thread on them. + DrainBack(); + dma_pusher.rasterizer->ReleaseFences(); + const u32 word = dma_pusher.memory_manager.Read<u32>(address); + const bool ready = operation == 0 ? word == payload + : operation == 2 ? word >= payload + : operation == 3 ? static_cast<s32>(word - payload) >= 0 + : operation == 4 ? (word & payload) != 0 + : operation == 5 ? (word & payload) == 0 + : true; + if (!ready) { + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 8) { + LOG_WARNING(HW_GPU, "SEM_EXECUTE op {} at {:#x}: {:#x} vs {:#x} not met, continuing", + operation, address, word, payload); + } + } +} + /// Calls a GPU puller method. void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { + static const bool nx2_trace = std::getenv("EDEN_NX2_PULLER_TRACE") != nullptr; + if (nx2_trace) { + LOG_INFO(HW_GPU, "nx2 puller sub={} method={:#x} arg={:#x}", method_call.subchannel, + method_call.method, method_call.argument); + } regs.reg_array[method_call.method] = method_call.argument; const auto method = static_cast<BufferMethods>(method_call.method); @@ -155,13 +373,13 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca case BufferMethods::WrcacheFlush: break; case BufferMethods::RefCnt: - dma_pusher.rasterizer->SignalReference(); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->SignalReference(); }); break; case BufferMethods::SyncpointOperation: ProcessFenceActionMethod(dma_pusher); break; case BufferMethods::WaitForIdle: - dma_pusher.rasterizer->WaitForIdle(); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->WaitForIdle(); }); break; case BufferMethods::SemaphoreOperation: { ProcessSemaphoreTriggerMethod(dma_pusher); @@ -172,25 +390,32 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca break; } case BufferMethods::MemOpA: { - LOG_ERROR(HW_GPU, "Memory Operation A"); + LOG_TRACE(HW_GPU, "Memory Operation A"); break; } case BufferMethods::MemOpB: { // Implement this better. - dma_pusher.rasterizer->InvalidateGPUCache(); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->InvalidateGPUCache(); }); break; } case BufferMethods::MemOpC: case BufferMethods::MemOpD: { - LOG_ERROR(HW_GPU, "Memory Operation C,D"); + LOG_TRACE(HW_GPU, "Memory Operation C,D"); break; } case BufferMethods::SemaphoreAcquire: { - ProcessSemaphoreAcquire(dma_pusher); + // On Ampere host classes 0x17..0x1A are SEM_ADDR_LO/HI and SEM_PAYLOAD_LO/HI. + if (!g_nx2_ampere) { + ProcessSemaphoreAcquire(dma_pusher); + } break; } case BufferMethods::SemaphoreRelease: { - ProcessSemaphoreRelease(dma_pusher); + if (g_nx2_ampere) { + ProcessSemaphoreExecute(dma_pusher, method_call.argument); + } else { + ProcessSemaphoreRelease(dma_pusher); + } break; } case BufferMethods::Yield: { @@ -200,13 +425,27 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca } default: LOG_ERROR(HW_GPU, "Special puller engine method {:X} not implemented", method); + if (static bool dumped = false; g_nx2_ampere && !dumped) { + dumped = true; + dma_pusher.DumpCurrentEntry("unknown host method"); + } break; } } /// Calls a GPU engine method. +static bool Nx2MethodTrace() { + static const bool trace = std::getenv("EDEN_NX2_METHOD_TRACE") != nullptr; + return trace; +} + void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { const EngineID engine = bound_engines[method_call.subchannel]; + if (Nx2MethodTrace()) { + LOG_INFO(HW_GPU, "nx2 method sub={} engine={:04X} method={:#x} arg={:#x}", + method_call.subchannel, static_cast<u32>(engine), method_call.method, + method_call.argument); + } switch (engine) { case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); @@ -235,6 +474,10 @@ void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_ca /// Calls a GPU engine multivalue method. void Puller::CallEngineMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending) { const EngineID engine = bound_engines[subchannel]; + if (Nx2MethodTrace()) { + LOG_INFO(HW_GPU, "nx2 multimethod sub={} engine={:04X} method={:#x} amount={} first={:#x}", + subchannel, static_cast<u32>(engine), method, amount, base_start[0]); + } switch (engine) { case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index e8994f5640..2b9e2c5df3 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -7,7 +7,9 @@ #pragma once #include <array> +#include <chrono> #include <cstddef> +#include <optional> #include <vector> #include "common/bit_field.h" #include "common/common_funcs.h" @@ -70,6 +72,24 @@ public: BitField<8, 24, u32> syncpoint_id; }; + /// Switch 2: a semaphore acquire this channel is suspended on. + struct PendingAcquire { + GPUVAddr address; + u32 value; + u32 operation; + std::chrono::steady_clock::time_point since; + bool reported; + + [[nodiscard]] bool Matches(u32 current) const; + }; + + [[nodiscard]] bool HasPendingAcquire() const { + return pending_acquire.has_value(); + } + [[nodiscard]] bool CanResumeAcquire(DmaPusher& dma_pusher); + /// Clears the pending acquire once it is met; false while the channel has to keep waiting. + bool ResumeAcquire(DmaPusher& dma_pusher); + void CallMethod(DmaPusher& dma_pusher, const MethodCall& method_call); void CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending); void BindRasterizer(DmaPusher& dma_pusher, VideoCore::RasterizerInterface* rasterizer); @@ -125,8 +145,11 @@ private: void ProcessSemaphoreAcquire(DmaPusher& dma_pusher); void ProcessSemaphoreRelease(DmaPusher& dma_pusher); void ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher); + void ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument); [[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method); + std::optional<PendingAcquire> pending_acquire; + /// Mapping of command subchannels to their bound engine ids std::array<EngineID, 8> bound_engines{}; diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 776c01e3fa..61c9844cf2 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -8,11 +8,13 @@ #include <algorithm> #include <condition_variable> +#include <cstdlib> #include <cstring> #include <deque> #include <functional> #include <mutex> #include <thread> +#include <vector> #include <queue> #include "common/common_types.h" @@ -22,6 +24,7 @@ #include "video_core/gpu.h" #include "video_core/host1x/host1x.h" #include "video_core/host1x/syncpoint_manager.h" +#include "video_core/nx2_profile.h" #include "video_core/rasterizer_interface.h" namespace VideoCommon { @@ -68,24 +71,42 @@ public: } void SyncOperation(std::function<void()>&& func) { + // Not only the GPU thread queues these (query cache work runs elsewhere too), while + // SignalFence moves the list into pending_operations: unguarded, the list was corrupted + // and the fencing thread crashed walking it. Its own lock, held for no callbacks. + std::scoped_lock lock{uncommitted_guard}; uncommitted_operations.emplace_back(std::move(func)); } void SignalFence(std::function<void()>&& func) { + Nx2Profile::Scope profile_scope{Nx2Profile::Zone::Fence}; const bool delay_fence = Settings::IsGPUFenceBehaviorDefault() ? Settings::IsGPULevelHigh() : Settings::IsGPUFenceBehaviorBalanced() || Settings::IsGPUFenceBehaviorAccurate(); - const bool should_flush = ShouldFlush(); if constexpr (!can_async_check) { TryReleasePendingFences<false>(); } - CommitAsyncFlushes(); + auto [should_flush, committed_flushes] = CommitCacheFlushes(); + if (!delay_fence && committed_flushes == 0 && SkipEmptyFence()) { + Nx2Profile::Scope skipped_scope{Nx2Profile::Zone::FenceSkipped}; + func(); + rasterizer.InvalidateGPUCache(); + return; + } + skipped_fences = 0; + should_flush |= query_cache.HasUncommittedFlushes(); + query_cache.CommitAsyncFlushes(); TFence new_fence = CreateFence(!should_flush); if constexpr (can_async_check) { guard.lock(); } - if (delay_fence) { - uncommitted_operations.emplace_back(std::move(func)); + { + std::scoped_lock lock{uncommitted_guard}; + if (delay_fence) { + uncommitted_operations.emplace_back(std::move(func)); + } + pending_operations.emplace_back(std::move(uncommitted_operations)); + uncommitted_operations.clear(); // moved-from: make it empty explicitly } - pending_operations.emplace_back(std::move(uncommitted_operations)); + pending_flushes.push_back(committed_flushes); QueueFence(new_fence); if (!delay_fence) { func(); @@ -178,7 +199,8 @@ private: return; } } - PopAsyncFlushes(); + PopAsyncFlushes(pending_flushes.front()); + pending_flushes.pop_front(); auto operations = std::move(pending_operations.front()); pending_operations.pop_front(); for (auto& operation : operations) { @@ -197,8 +219,29 @@ private: Common::SetCurrentThreadPriority(Common::ThreadPriority::High); Common::SetCurrentThreadToPerformanceCores(); - TFence current_fence; - std::deque<std::function<void()>> current_operations; + // Every fence the host GPU has already passed is released together: Switch 2 titles + // signal thousands a second and the host GPU completes them in bursts, so each cache's + // lock is taken once per burst instead of once per fence (the GPU thread, which holds + // them for nearly every command, spun, slept and woke this thread for each one). + // EDEN_NX2_FENCE_BATCH=0: one fence at a time (for comparisons). + static const size_t MaxBatch = [] { + const char* value = std::getenv("EDEN_NX2_FENCE_BATCH"); + return value != nullptr && value[0] == '0' ? size_t{1} : size_t{64}; + }(); + struct Released { + TFence fence; + std::deque<std::function<void()>> operations; + u8 flushes; + }; + std::vector<Released> batch; + batch.reserve(MaxBatch); + const auto take_front = [&] { + batch.push_back({std::move(fences.front()), std::move(pending_operations.front()), + pending_flushes.front()}); + fences.pop(); + pending_operations.pop_front(); + pending_flushes.pop_front(); + }; while (!stop_token.stop_requested()) { { std::unique_lock lock(guard); @@ -206,22 +249,50 @@ private: if (stop_token.stop_requested()) [[unlikely]] { return; } - current_fence = std::move(fences.front()); - current_operations = std::move(pending_operations.front()); - fences.pop(); - pending_operations.pop_front(); + take_front(); } - if (!current_fence->IsStubbed()) { - WaitFence(current_fence); + if (!batch.front().fence->IsStubbed()) { + WaitFence(batch.front().fence); } - PopAsyncFlushes(); - for (auto& operation : current_operations) { - operation(); + { + std::unique_lock lock(guard); + while (batch.size() < MaxBatch && !fences.empty() && + (fences.front()->IsStubbed() || IsFenceSignaled(fences.front()))) { + take_front(); + } + } + size_t texture_pops = 0; + size_t buffer_pops = 0; + for (const Released& released : batch) { + texture_pops += (released.flushes & TextureFlushes) != 0 ? 1 : 0; + buffer_pops += (released.flushes & BufferFlushes) != 0 ? 1 : 0; + } + if (texture_pops != 0) { + std::scoped_lock lock{texture_cache.mutex}; + for (size_t i = 0; i < texture_pops; ++i) { + texture_cache.PopAsyncFlushes(); + } + } + if (buffer_pops != 0) { + std::scoped_lock lock{buffer_cache.mutex}; + for (size_t i = 0; i < buffer_pops; ++i) { + buffer_cache.PopAsyncFlushes(); + } + } + // Queries and operations keep their per-fence order. + for (Released& released : batch) { + query_cache.PopAsyncFlushes(); + for (auto& operation : released.operations) { + operation(); + } } { std::unique_lock lock(ring_guard); - delayed_destruction_ring.Push(std::move(current_fence)); + for (Released& released : batch) { + delayed_destruction_ring.Push(std::move(released.fence)); + } } + batch.clear(); } } @@ -231,36 +302,86 @@ private: query_cache.ShouldWaitAsyncFlushes(); } - bool ShouldFlush() const { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; - return texture_cache.HasUncommittedFlushes() || buffer_cache.HasUncommittedFlushes() || - query_cache.HasUncommittedFlushes(); - } + static constexpr u8 TextureFlushes = 1; + static constexpr u8 BufferFlushes = 2; - void PopAsyncFlushes() { - { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + /// committed: what CommitAsyncFlushes returned for this fence. + void PopAsyncFlushes(u8 committed) { + if (committed & TextureFlushes) { + std::scoped_lock lock{texture_cache.mutex}; texture_cache.PopAsyncFlushes(); + } + if (committed & BufferFlushes) { + std::scoped_lock lock{buffer_cache.mutex}; buffer_cache.PopAsyncFlushes(); } query_cache.PopAsyncFlushes(); } - void CommitAsyncFlushes() { + struct CacheFlushes { + bool should_flush; ///< Downloads to wait for: the fence must reach the host GPU + u8 committed; ///< TextureFlushes, BufferFlushes: the caches that committed downloads + }; + + /// Commits the texture and buffer caches' downloads for a new fence (the query cache's are + /// committed once a fence is made). Switch 2 titles signal a fence per compute dispatch + /// (thousands a second), mostly with nothing or only buffer data to download: each cache's + /// lock is taken once, alone, and the fencing thread later takes only the lock of the cache + /// with work, instead of holding both while it copies (the GPU thread's draws and uploads + /// waited for it). + CacheFlushes CommitCacheFlushes() { + CacheFlushes flushes{}; { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; - texture_cache.CommitAsyncFlushes(); - buffer_cache.CommitAsyncFlushes(); + std::scoped_lock lock{texture_cache.mutex}; + flushes.should_flush = texture_cache.HasUncommittedFlushes(); + if (texture_cache.HasAsyncFlushWork()) { + texture_cache.CommitAsyncFlushes(); + flushes.committed |= TextureFlushes; + } + } + { + std::scoped_lock lock{buffer_cache.mutex}; + if (buffer_cache.HasUncommittedFlushes()) { + buffer_cache.CommitAsyncFlushes(); + flushes.should_flush = true; + flushes.committed |= BufferFlushes; + } + } + return flushes; + } + + /// Whether a fence with no downloads would do nothing, so need not be made: with fast GPU + /// fences its callback runs at once anyway, and most of the per-dispatch fences of Switch 2 + /// titles have no queries or queued operations either. Making, queueing and retiring them + /// cost the GPU thread ~5% and woke the fencing thread for each. One in 64 is still made, so + /// the operations fences carry along (query slot cleanup) keep running. + bool SkipEmptyFence() { + if constexpr (requires { query_cache.HasPendingFenceWork(); }) { + if (skipped_fences >= 64 || query_cache.HasPendingFenceWork()) { + return false; + } + { + std::scoped_lock lock{uncommitted_guard}; + if (!uncommitted_operations.empty()) { + return false; + } + } + ++skipped_fences; + return true; + } else { + return false; } - query_cache.CommitAsyncFlushes(); } std::queue<TFence> fences; std::deque<std::function<void()>> uncommitted_operations; std::deque<std::deque<std::function<void()>>> pending_operations; + std::deque<u8> pending_flushes; ///< Per fence: the caches that committed downloads + u32 skipped_fences{}; ///< Empty fences skipped since the last one made (GPU thread only) std::mutex guard; std::mutex ring_guard; + std::mutex uncommitted_guard; ///< Guards uncommitted_operations std::condition_variable cv; std::jthread fence_thread; diff --git a/src/video_core/framebuffer_config.h b/src/video_core/framebuffer_config.h index 1fc70583f9..bb041cf12f 100644 --- a/src/video_core/framebuffer_config.h +++ b/src/video_core/framebuffer_config.h @@ -37,6 +37,7 @@ struct FramebufferConfig { Service::android::BufferTransformFlags transform_flags{}; Common::Rectangle<int> crop_rect{}; BlendMode blending{}; + bool native_tiling{}; u32 layer_stack_mask{Service::Nvnflinger::DefaultLayerStackMask}; }; diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 1621b24125..53f8f063f7 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -206,6 +206,10 @@ struct GPU::Impl { gpu_thread.SubmitList(channel, std::move(entries), is_async); } + void RunInOrder(std::function<void()>&& func) { + gpu_thread.Run(std::move(func), is_async); + } + /// Notify rasterizer that any caches of the specified region should be flushed to Switch memory void FlushRegion(DAddr addr, u64 size) { gpu_thread.FlushRegion(addr, size, is_async); @@ -217,6 +221,13 @@ struct GPU::Impl { return raster_area; } raster_area.preemtive = true; + // A guest CPU read of GPU-written memory (reactive flushing): count them, they decide + // whether buffer data must be copied back at every fence (EDEN_NX2_BUFFER_DOWNLOADS). + if (static std::atomic<u64> reads{}; (reads.fetch_add(1, std::memory_order_relaxed) & 63) == 0) { + LOG_INFO(HW_GPU, "CPU read of GPU-written memory [{:#x}, {:#x}) ({} so far)", + raster_area.start_address, raster_area.end_address, + reads.load(std::memory_order_relaxed)); + } const u64 fence = RequestSyncOperation([this, &raster_area]() { renderer->ReadRasterizer()->FlushRegion(raster_area.start_address, raster_area.end_address - raster_area.start_address); }); @@ -407,6 +418,10 @@ u64 GPU::RequestFlush(DAddr addr, std::size_t size) { }); } +u64 GPU::RunOnGpuThread(std::function<void()>&& action) { + return impl->RequestSyncOperation(std::move(action)); +} + u64 GPU::CurrentSyncRequestFence() const { return impl->CurrentSyncRequestFence(); } @@ -530,6 +545,10 @@ void GPU::FlushRegion(DAddr addr, u64 size) { impl->FlushRegion(addr, size); } +void GPU::RunInOrder(std::function<void()>&& func) { + impl->RunInOrder(std::move(func)); +} + void GPU::InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes) { impl->InvalidateRegion(addr, size, preserve_gpu_writes); } diff --git a/src/video_core/gpu.h b/src/video_core/gpu.h index 974d214e10..3a6546b41b 100644 --- a/src/video_core/gpu.h +++ b/src/video_core/gpu.h @@ -6,6 +6,7 @@ #pragma once +#include <functional> #include <memory> #include "common/bit_field.h" @@ -163,6 +164,9 @@ public: /// Request a host GPU memory flush from the CPU. [[nodiscard]] u64 RequestFlush(DAddr addr, std::size_t size); + /// Runs an action on the GPU thread between command lists (returns its sync fence). + u64 RunOnGpuThread(std::function<void()>&& action); + /// Obtains current flush request fence id. [[nodiscard]] u64 CurrentSyncRequestFence() const; @@ -247,6 +251,9 @@ public: /// Notify rasterizer that any caches of the specified region should be flushed to Switch memory void FlushRegion(DAddr addr, u64 size); + /// Runs func on the GPU thread after the command lists submitted so far. + void RunInOrder(std::function<void()>&& func); + /// Notify rasterizer that any caches of the specified region should be invalidated void InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes = false); diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp new file mode 100644 index 0000000000..07fda589ff --- /dev/null +++ b/src/video_core/gpu_split.cpp @@ -0,0 +1,335 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <algorithm> +#include <cstdlib> +#include <thread> + +#ifdef _MSC_VER +#include <intrin.h> +#endif +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> +#endif +#include <cstdio> + +#include "common/logging.h" +#include "common/settings.h" +#include "common/thread.h" +#include "core/core.h" +#include "video_core/engines/engine_interface.h" +#include "video_core/gpu_split.h" +#include "video_core/nx2.h" +#include "video_core/nx2_profile.h" + +namespace VideoCommon { + +namespace { + +void Pause() { +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + _mm_pause(); +#elif defined(__x86_64__) || defined(__i386__) + __builtin_ia32_pause(); +#else + std::this_thread::yield(); +#endif +} + +std::atomic<u32> g_back_thread_id{}; +std::atomic<bool> g_back_busy{}; + +} // Anonymous namespace + +void GpuSplit::CheckOwner(const char* where) { +#ifdef _WIN32 + if (!g_back_busy.load(std::memory_order_acquire)) { + return; + } + const DWORD thread = GetCurrentThreadId(); + if (thread == g_back_thread_id.load(std::memory_order_relaxed)) { + return; + } + static std::atomic<u32> reports{}; + if (reports.fetch_add(1) >= 12) { + return; + } + std::fprintf(stderr, "GPU split owner check: %s on thread %lu while the back runs\n", where, + thread); + void* frames[40]; + const USHORT count = RtlCaptureStackBackTrace(1, 40, frames, nullptr); + for (USHORT i = 0; i < count; ++i) { + HMODULE module{}; + char name[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + static_cast<LPCSTR>(frames[i]), &module)) { + GetModuleFileNameA(module, name, sizeof(name)); + } + const char* base_name = std::strrchr(name, '\\'); + std::fprintf(stderr, " frame %s+%#llx\n", base_name ? base_name + 1 : name, + static_cast<unsigned long long>(reinterpret_cast<uintptr_t>(frames[i]) - + reinterpret_cast<uintptr_t>(module))); + } + std::fflush(stderr); +#else + (void)where; +#endif +} + +GpuSplit GpuSplit::instance; + +void GpuSplit::MaybeStart(Core::System& system_) { + static const char mode = [] { + const char* value = std::getenv("EDEN_NX2_SPLIT"); + return value != nullptr ? value[0] : '2'; + }(); + const bool wanted = mode == '1' || mode == '2'; + if (active || !wanted || !Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + return; + } + // The front reads command lists straight from guest memory; a read that flushes the caches + // first (safe DMA, accurate GPU) would touch back-owned state. + const bool safe_reads = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() + : Settings::IsDMALevelSafe(); + if (safe_reads || Settings::IsGPULevelHigh() || + Settings::values.renderer_backend.GetValue() != Settings::RendererBackend::Vulkan) { + return; + } + system = &system_; + check_owner = std::getenv("EDEN_NX2_SPLIT_CHECK") != nullptr; + ring = std::make_unique<u64[]>(RingWords); + back_thread = std::jthread([this](std::stop_token stop_token) { BackLoop(stop_token); }); + active = true; +#ifdef ARCHITECTURE_x86_64 + // Front macros run compiled (macro_turing_jit.h) only. + const char* jit = std::getenv("EDEN_NX2_MME_JIT"); + front_macros = mode == '2' && (jit == nullptr || jit[0] != '0'); +#endif + LOG_INFO(HW_GPU, "GPU thread split: engine methods run on a second thread{}", + front_macros ? ", macros on the first" : ""); +} + +void GpuSplit::Stop() { + if (!back_thread.joinable()) { + return; + } + back_thread.request_stop(); + wake.fetch_add(1, std::memory_order_seq_cst); + wake.notify_one(); + back_thread.join(); + active = false; +} + +u64* GpuSplit::ReserveSlow(size_t words) { + size_t index = write_position & RingMask; + if (index + words > RingWords) { + // Entries never wrap: skip to the start of the ring. + const size_t rest = RingWords - index; + WaitForSpace(rest); + ring[index] = static_cast<u64>(Kind::Wrap); + write_position += rest; + index = 0; + } + WaitForSpace(words); + return &ring[index]; +} + +void GpuSplit::WaitForSpace(size_t words) { + if (write_position + words - consumed_cache <= RingWords) { + return; + } + Publish(); + Nx2Profile::Scope scope{Nx2Profile::Zone::SplitDrain}; + for (u32 spins = 0;; ++spins) { + consumed_cache = consumed.load(std::memory_order_acquire); + if (write_position + words - consumed_cache <= RingWords) { + return; + } + if (spins < 4096) { + Pause(); + } else { + std::this_thread::yield(); + } + } +} + +void GpuSplit::Publish() { + if (published_local == write_position) { + return; + } + published_local = write_position; + publish_every = std::min<size_t>(publish_every * 2, PublishEvery); + published.store(write_position, std::memory_order_seq_cst); + if (back_waiting.load(std::memory_order_seq_cst)) { + wake.fetch_add(1, std::memory_order_seq_cst); + wake.notify_one(); + } +} + +void GpuSplit::Drain() { + consumed_cache = consumed.load(std::memory_order_acquire); + if (consumed_cache == write_position) { + return; + } + Publish(); + Nx2Profile::Scope scope{Nx2Profile::Zone::SplitDrain}; + for (u32 spins = 0;; ++spins) { + consumed_cache = consumed.load(std::memory_order_acquire); + if (consumed_cache == write_position) { + publish_every = 8; + return; + } + if (spins < 16384) { + Pause(); + } else { + std::this_thread::yield(); + } + } +} + +void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) { + front_engine = engine; + u64* const entry = Reserve(2); + entry[0] = static_cast<u64>(Kind::Engine); + entry[1] = reinterpret_cast<u64>(engine); + Commit(2); +} + +void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, + u32 amount, u32 methods_pending, const GPUVAddr* segment) { + SelectEngine(engine); + // Long runs (constant buffer and inline uploads) go in pieces, as when a method's data + // continues in the next GPFIFO entry. + while (amount > 0) { + const u32 piece = std::min(amount, MaxMultiWords); + const size_t head = segment != nullptr ? 3 : 2; + const size_t size = head + (piece + 1) / 2; + u64* const entry = Reserve(size); + entry[0] = static_cast<u64>(Kind::Multi) | (segment != nullptr ? HasSegment : 0) | + (u64{method & 0xffff} << 8) | (u64{piece} << 32); + entry[1] = methods_pending; + if (segment != nullptr) { + entry[2] = *segment; + } + entry[size - 1] = 0; + std::memcpy(entry + head, words, size_t{piece} * sizeof(u32)); + Commit(size); + words += piece; + amount -= piece; + methods_pending -= piece; + } +} + +void GpuSplit::DeferFunction(std::function<void()>&& func) { + auto* const heap = new std::function<void()>(std::move(func)); + Defer([heap] { + (*heap)(); + delete heap; + }); +} + +size_t GpuSplit::Execute(const u64* entry) { + const u64 head = entry[0]; + switch (static_cast<Kind>(head & 0xf)) { + case Kind::Wrap: + return 0; + case Kind::Engine: + back_engine = reinterpret_cast<Tegra::Engines::EngineInterface*>(entry[1]); + return 2; + case Kind::Method: { + const u32 method = static_cast<u32>((head >> 8) & 0xffff); + const u32 argument = static_cast<u32>(head >> 32); + auto* const engine = back_engine; + if ((head & Direct) != 0) { + Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method}; + engine->ConsumeSink(*system); + engine->CallMethod(*system, method, argument, true); + } else if (!engine->execution_mask[method]) { + engine->method_sink.emplace_back(method, argument); + } else { + Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method}; + engine->ConsumeSink(*system); + if ((head & HasSegment) != 0) { + engine->current_dma_segment = entry[1]; + } + engine->CallMethod(*system, method, argument, (head & IsLast) != 0); + } + return (head & HasSegment) != 0 ? 2 : 1; + } + case Kind::Multi: { + const u32 method = static_cast<u32>((head >> 8) & 0xffff); + const u32 amount = static_cast<u32>(head >> 32); + const size_t data = (head & HasSegment) != 0 ? 3 : 2; + auto* const engine = back_engine; + Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method}; + engine->ConsumeSink(*system); + if ((head & HasSegment) != 0) { + engine->current_dma_segment = entry[2]; + } + engine->CallMultiMethod(*system, method, reinterpret_cast<const u32*>(entry + data), + amount, static_cast<u32>(entry[1])); + return data + (amount + 1) / 2; + } + case Kind::Call: { + const size_t payload = static_cast<size_t>((head >> 8) & 0xffffff); + reinterpret_cast<void (*)(const u64*)>(entry[1])(entry + 2); + return 2 + payload; + } + } + return 1; +} + +void GpuSplit::BackLoop(std::stop_token stop_token) { + Common::SetCurrentThreadName("GPUBack"); + Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical); + Common::SetCurrentThreadToPerformanceCores(); + system->RegisterHostThread(); +#ifdef _WIN32 + g_back_thread_id.store(GetCurrentThreadId(), std::memory_order_relaxed); +#endif + u64 read_position = 0; + while (!stop_token.stop_requested()) { + Nx2Profile::Report("GPU back thread"); + u64 available = published.load(std::memory_order_acquire); + if (available == read_position) { + Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWait}; + // The front usually sends more within microseconds: spin briefly before sleeping. + for (u32 spins = 0; spins < 8192 && available == read_position; ++spins) { + Pause(); + available = published.load(std::memory_order_acquire); + } + if (available == read_position) { + // Publish bumps `wake` when it sees back_waiting: read it before the last look + // at `published`, so a publish in between ends the wait at once. + back_waiting.store(true, std::memory_order_seq_cst); + const u32 seen = wake.load(std::memory_order_seq_cst); + available = published.load(std::memory_order_seq_cst); + if (available == read_position && !stop_token.stop_requested()) { + wake.wait(seen, std::memory_order_seq_cst); + } + back_waiting.store(false, std::memory_order_relaxed); + continue; + } + } + u64 last_store = read_position; + g_back_busy.store(true, std::memory_order_relaxed); + while (read_position != available) { + const size_t index = read_position & RingMask; + const size_t size = Execute(&ring[index]); + read_position += size != 0 ? size : RingWords - index; + // Hand ring space back now and then while working through a long batch. + if (read_position - last_store >= 4096 && read_position != available) { + consumed.store(read_position, std::memory_order_release); + last_store = read_position; + } + } + g_back_busy.store(false, std::memory_order_release); + consumed.store(read_position, std::memory_order_release); + } +} + +} // namespace VideoCommon diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h new file mode 100644 index 0000000000..d6f91e4141 --- /dev/null +++ b/src/video_core/gpu_split.h @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <atomic> +#include <cstring> +#include <functional> +#include <memory> +#include <thread> +#include <type_traits> + +#include "common/common_types.h" + +namespace Core { +class System; +} + +namespace Tegra::Engines { +class EngineInterface; +} + +namespace VideoCommon { + +// Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the +// CPU's commands, parses command lists and runs the puller; engine method calls and everything +// behind them (register state, macros, draws, the rasterizer and its caches) run in order on a +// second thread (the back), fed through a single-producer single-consumer ring. Work the front +// would do on back-owned state is deferred into the same ring; the front waits for the back to +// catch up (Drain) only where it needs a result from that state, e.g. a semaphore it acquires. +// On for Switch 2 titles with the 3D and compute engines' macros on the front, against copies of +// the registers they read (Maxwell3D and KeplerCompute Front*), which sends the writes they make. +// EDEN_NX2_SPLIT=1 keeps the macros on the back; EDEN_NX2_SPLIT=0 runs everything on one thread. +class GpuSplit { +public: + static GpuSplit& Get() { + return instance; + } + + /// Whether the front sends work to the back (front thread only). + [[nodiscard]] bool Active() const { + return active; + } + /// Whether the front runs the engines' macros (EDEN_NX2_SPLIT=2). + [[nodiscard]] bool FrontMacros() const { + return front_macros; + } + + /// Starts the back thread once the split is wanted (EDEN_NX2_SPLIT=1, a Switch 2 title, + /// commands read without cache flushes). Called by the front between commands. + void MaybeStart(Core::System& system); + void Stop(); + + /// An engine method write, executed as DmaPusher would (method sink, ConsumeSink, call). + /// `segment`: the engine's current_dma_segment for the call, when the engine reads it. + void Method(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument, bool is_last, + const GPUVAddr* segment) { + SelectEngine(engine); + const size_t words = segment != nullptr ? 2 : 1; + u64* const entry = Reserve(words); + entry[0] = static_cast<u64>(Kind::Method) | (is_last ? IsLast : 0) | + (segment != nullptr ? HasSegment : 0) | (u64{method & 0xffff} << 8) | + (u64{argument} << 32); + if (segment != nullptr) { + entry[1] = *segment; + } + Commit(words); + } + void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, + u32 amount, u32 methods_pending, const GPUVAddr* segment); + /// A write made by a macro: CallMethod without the method sink, as the engines' macros do. + void MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument) { + SelectEngine(engine); + u64* const entry = Reserve(1); + entry[0] = static_cast<u64>(Kind::Method) | IsLast | Direct | (u64{method & 0xffff} << 8) | + (u64{argument} << 32); + Commit(1); + } + + /// Runs `func` on the back after everything sent before it. Must be trivially copyable. + template <typename Func> + void Defer(const Func& func) { + static_assert(std::is_trivially_copyable_v<Func> && alignof(Func) <= alignof(u64)); + constexpr size_t payload = (sizeof(Func) + sizeof(u64) - 1) / sizeof(u64); + u64* const entry = Reserve(2 + payload); + entry[0] = static_cast<u64>(Kind::Call) | (u64{payload} << 8); + entry[1] = reinterpret_cast<u64>(&CallThunk<Func>); + std::memcpy(entry + 2, &func, sizeof(Func)); + Commit(2 + payload); + } + void DeferFunction(std::function<void()>&& func); + + /// EDEN_NX2_SPLIT_CHECK=1 (debug): reports, with its stack, a thread other than the back + /// using back-owned state (the Vulkan scheduler) while the back runs entries. + static inline bool check_owner{}; + static void CheckOwner(const char* where); + + /// Makes what was sent visible to the back. + void Publish(); + /// Waits until the back executed everything sent so far. + void Drain(); + +private: + enum class Kind : u64 { + Wrap, ///< The rest of the ring is unused: continue at its start + Engine, ///< The engine the next methods go to: word 1 + Method, ///< Bits 8-23 method, 32-63 argument; flags IsLast, HasSegment (word 1) + Multi, ///< Bits 8-23 method, 32-63 amount; word 1 methods pending, [segment], words + Call, ///< Bits 8-31 payload size in words; word 1 thunk, then the payload + }; + static constexpr u64 IsLast = 1 << 4; + static constexpr u64 HasSegment = 1 << 5; + static constexpr u64 Direct = 1 << 6; ///< Method: call it even if not executable (macros) + + static constexpr size_t RingWords = size_t{1} << 20; // 8 MiB + static constexpr size_t RingMask = RingWords - 1; + static constexpr u32 MaxMultiWords = 8192; + static constexpr size_t PublishEvery = 256; + + template <typename Func> + static void CallThunk(const u64* payload) { + (*reinterpret_cast<const Func*>(payload))(); + } + + /// Room for an entry of `words` at the write position (never across the ring's end). + u64* Reserve(size_t words) { + const size_t index = write_position & RingMask; + if (index + words <= RingWords && write_position + words - consumed_cache <= RingWords) + [[likely]] { + return &ring[index]; + } + return ReserveSlow(words); + } + u64* ReserveSlow(size_t words); + void Commit(size_t words) { + write_position += words; + if (write_position - published_local >= publish_every) { + Publish(); + } + } + void WaitForSpace(size_t words); + void SelectEngine(Tegra::Engines::EngineInterface* engine) { + if (engine != front_engine) [[unlikely]] { + SwitchEngine(engine); + } + } + void SwitchEngine(Tegra::Engines::EngineInterface* engine); + + void BackLoop(std::stop_token stop_token); + size_t Execute(const u64* entry); + + Core::System* system{}; + bool active{}; + bool front_macros{}; + std::unique_ptr<u64[]> ring; + + // Front side. + u64 write_position{}; + u64 published_local{}; + u64 consumed_cache{}; + /// Words between publishes: small right after a drain, while the back has nothing to do. + size_t publish_every{PublishEvery}; + Tegra::Engines::EngineInterface* front_engine{}; + + // Back side. + Tegra::Engines::EngineInterface* back_engine{}; + + static GpuSplit instance; + + alignas(64) std::atomic<u64> published{}; + alignas(64) std::atomic<u64> consumed{}; + alignas(64) std::atomic<bool> back_waiting{}; + std::atomic<u32> wake{}; + std::jthread back_thread; +}; + +} // namespace VideoCommon diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index a01375e7be..55200b0647 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <chrono> +#include <thread> + #include "common/assert.h" #include "common/scope_exit.h" #include "common/settings.h" @@ -14,11 +17,28 @@ #include "video_core/dma_pusher.h" #include "video_core/gpu.h" #include "video_core/gpu_thread.h" +#include "video_core/gpu_split.h" +#include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" #include "video_core/renderer_base.h" namespace VideoCommon::GPUThread { +namespace { + +// Tells a CPU thread waiting in PushCommand that its command ran. +void SignalCommand(SynchState& state, u64 fence, bool block) { + state.signaled_fence.store(fence); + if (block) { + // We have to lock the write_lock to ensure that the condition_variable wait not get a + // race between the check and the lock itself. + std::scoped_lock lk{state.write_lock}; + state.cv.notify_all(); + } +} + +} // Anonymous namespace + ThreadManager::ThreadManager(Core::System& system_) : system{system_} {} @@ -35,29 +55,87 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto current_context = context.Acquire(); CommandDataContainer next; + // When a pending channel last moved or a command arrived (see the polling below). + auto last_progress = std::chrono::steady_clock::now(); + namespace Profile = VideoCommon::Nx2Profile; + auto& split = VideoCommon::GpuSplit::Get(); while (!stop_token.stop_requested()) { - state.queue.PopWait(next, stop_token); + Profile::Report(); + split.MaybeStart(system); + if (scheduler.HasPendingChannels()) { + // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. + { + Profile::Scope scope{Profile::Zone::ResumePending}; + scheduler.ResumePending(system.GPU()); + } + if (!state.queue.TryPop(next)) { + // The CPU usually releases the semaphore soon after: poll for half a + // millisecond before sleeping, or every wait also waits out a host sleep + // (title 1 waits ~25 times a second). + Profile::Scope scope{Profile::Zone::PendingSleep}; + if (std::chrono::steady_clock::now() - last_progress < + std::chrono::microseconds{500}) { + std::this_thread::yield(); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + continue; + } + } else { + Profile::Scope scope{Profile::Zone::Idle}; + state.queue.PopWait(next, stop_token); + } if (stop_token.stop_requested()) { break; } + last_progress = std::chrono::steady_clock::now(); + if (split.Active()) { + // Only command lists are parsed here; everything else runs on the back thread + // in order, and so does the signal that the command ran. + auto* const rasterizer_ptr = renderer.ReadRasterizer(); + if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::Push}; + scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); + } else if (std::holds_alternative<GPUTickCommand>(next.data)) { + split.Defer([gpu = &system.GPU()] { gpu->TickWork(); }); + } else if (auto* run = std::get_if<RunCommand>(&next.data)) { + split.DeferFunction(std::move(run->func)); + } else if (const auto* flush = std::get_if<FlushRegionCommand>(&next.data)) { + split.Defer([rasterizer_ptr, addr = flush->addr, size = flush->size] { + rasterizer_ptr->FlushRegion(addr, size); + }); + } else if (const auto* invalidate = std::get_if<InvalidateRegionCommand>(&next.data)) { + split.Defer([rasterizer_ptr, addr = invalidate->addr, size = invalidate->size] { + rasterizer_ptr->OnCacheInvalidation(addr, size); + }); + } else { + ASSERT(false); + } + split.Defer([state_ptr = &state, fence = next.fence, block = next.block] { + SignalCommand(*state_ptr, fence, block); + }); + split.Publish(); + continue; + } if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::Push}; scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); } else if (std::holds_alternative<GPUTickCommand>(next.data)) { + Profile::Scope scope{Profile::Zone::Tick}; system.GPU().TickWork(); + } else if (auto* run = std::get_if<RunCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::Run}; + run->func(); } else if (const auto* flush = std::get_if<FlushRegionCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::FlushInvalidate}; renderer.ReadRasterizer()->FlushRegion(flush->addr, flush->size); } else if (const auto* invalidate = std::get_if<InvalidateRegionCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::FlushInvalidate}; renderer.ReadRasterizer()->OnCacheInvalidation(invalidate->addr, invalidate->size); } else { ASSERT(false); } - state.signaled_fence.store(next.fence); - if (next.block) { - // We have to lock the write_lock to ensure that the condition_variable wait not get a - // race between the check and the lock itself. - std::scoped_lock lk{state.write_lock}; - state.cv.notify_all(); - } + SignalCommand(state, next.fence, next.block); } }); } @@ -77,6 +155,10 @@ void ThreadManager::TickGPU(bool is_async) { PushCommand(GPUTickCommand(), false, is_async); } +void ThreadManager::Run(std::function<void()>&& func, bool is_async) { + PushCommand(RunCommand{std::move(func)}, false, is_async); +} + void ThreadManager::InvalidateRegion(DAddr addr, u64 size) { rasterizer->OnCacheInvalidation(addr, size); } @@ -119,6 +201,7 @@ void ThreadManager::NotifyShutdown() { thread.request_stop(); thread.join(); } + VideoCommon::GpuSplit::Get().Stop(); } } // namespace VideoCommon::GPUThread diff --git a/src/video_core/gpu_thread.h b/src/video_core/gpu_thread.h index 631a680db4..796c0d89f6 100644 --- a/src/video_core/gpu_thread.h +++ b/src/video_core/gpu_thread.h @@ -6,6 +6,7 @@ #pragma once +#include <functional> #include <atomic> #include <condition_variable> #include <mutex> @@ -76,9 +77,14 @@ struct FlushAndInvalidateRegionCommand final { /// Command to make the gpu look into pending requests struct GPUTickCommand final {}; +/// Command to run a function on the GPU thread after the commands queued before it +struct RunCommand final { + std::function<void()> func; +}; + using CommandData = std::variant<std::monostate, SubmitListCommand, FlushRegionCommand, InvalidateRegionCommand, - FlushAndInvalidateRegionCommand, GPUTickCommand>; + FlushAndInvalidateRegionCommand, GPUTickCommand, RunCommand>; struct CommandDataContainer { CommandDataContainer() = default; @@ -125,6 +131,9 @@ public: void TickGPU(bool is_async); + /// Runs func on the GPU thread once the commands queued before it have been processed. + void Run(std::function<void()>&& func, bool is_async); + void NotifyShutdown(); private: diff --git a/src/video_core/host1x/gpu_device_memory_manager.h b/src/video_core/host1x/gpu_device_memory_manager.h index a9f249991f..1b4f57fe24 100644 --- a/src/video_core/host1x/gpu_device_memory_manager.h +++ b/src/video_core/host1x/gpu_device_memory_manager.h @@ -14,7 +14,9 @@ namespace Tegra { struct MaxwellDeviceMethods; struct MaxwellDeviceTraits { - static constexpr size_t device_virtual_bits = 34; + // 64 GiB: Switch 2 titles map most of their memory twice plus GPU-only regions (title 1 + // ran out of 16 GiB). + static constexpr size_t device_virtual_bits = 36; using DeviceInterface = typename VideoCore::RasterizerInterface; using DeviceMethods = MaxwellDeviceMethods; }; diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp new file mode 100644 index 0000000000..7d185ca573 --- /dev/null +++ b/src/video_core/host1x/nx2_video_decoder.cpp @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <array> +#include <cstring> +#include <optional> +#include <set> + +#include "common/logging.h" +#include "common/settings.h" +#include "video_core/host1x/ffmpeg.h" +#include "video_core/host1x/nx2_video_decoder.h" + +extern "C" { +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + +#include <libswscale/swscale.h> + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +} + +namespace Tegra::Host1x { + +namespace { + +std::optional<NvdecCommon::VideoCodec> CodecFromMime(std::string_view mime) { + if (mime.find("vp9") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP9; + } + if (mime.find("vp8") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP8; + } + if (mime.find("avc") != std::string_view::npos || mime.find("264") != std::string_view::npos) { + return NvdecCommon::VideoCodec::H264; + } + return std::nullopt; +} + +} // namespace + +struct Nx2VideoDecoder::Impl { + std::string name; + std::optional<FFmpeg::Decoder> decoder; + std::optional<FFmpeg::HardwareContext> hardware; + std::optional<FFmpeg::DecoderContext> context; + SwsContext* scaler{}; + bool open{}; + /// Timestamps of submitted frames not yet output. Frames copied back from a host GPU decoder + /// lose theirs and take the earliest one instead. + std::multiset<u64> pending_pts; + std::vector<u8> packet; ///< FFmpeg reads up to AV_INPUT_BUFFER_PADDING_SIZE past the data + + ~Impl() { + sws_freeContext(scaler); + context.reset(); + hardware.reset(); + } + + void Receive(std::vector<Nx2Picture>& pictures) { + while (auto frame = context->ReceiveFrame()) { + const AVFrame* f = frame->GetFrame(); + const s64 stamp = f->best_effort_timestamp != AV_NOPTS_VALUE ? f->best_effort_timestamp + : f->pts; + u64 pts{}; + if (stamp != AV_NOPTS_VALUE) { + pts = static_cast<u64>(stamp); + if (const auto it = pending_pts.find(pts); it != pending_pts.end()) { + pending_pts.erase(it); + } + } else if (!pending_pts.empty()) { + pts = *pending_pts.begin(); + pending_pts.erase(pending_pts.begin()); + } + pictures.push_back({static_cast<u32>(f->width), static_cast<u32>(f->height), pts, + std::move(frame)}); + } + } +}; + +Nx2VideoDecoder::Nx2VideoDecoder(std::string_view mime) : impl{std::make_unique<Impl>()} { + impl->name = std::string{mime}; + const auto codec = CodecFromMime(mime); + if (!codec) { + LOG_ERROR(HW_GPU, "nx2 video: no decoder for codec \"{}\"", mime); + return; + } + impl->decoder.emplace(*codec); + if (impl->decoder->GetCodec() == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: FFmpeg has no decoder for \"{}\"", mime); + return; + } + impl->context.emplace(*impl->decoder); + if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { + impl->hardware.emplace(); + impl->hardware->InitializeForDecoder(*impl->context, *impl->decoder); + } + impl->open = impl->context->OpenContext(*impl->decoder); + LOG_INFO(HW_GPU, "nx2 video: decoder for \"{}\" {}", mime, impl->open ? "open" : "failed to open"); +} + +Nx2VideoDecoder::~Nx2VideoDecoder() = default; + +bool Nx2VideoDecoder::IsOpen() const { + return impl->open; +} + +const std::string& Nx2VideoDecoder::Name() const { + return impl->name; +} + +void Nx2VideoDecoder::Decode(std::span<const u8> data, u64 pts, std::vector<Nx2Picture>& pictures) { + if (!impl->open || data.empty()) { + return; + } + impl->packet.assign(data.size() + AV_INPUT_BUFFER_PADDING_SIZE, 0); + std::memcpy(impl->packet.data(), data.data(), data.size()); + FFmpeg::Packet packet{std::span<const u8>{impl->packet.data(), data.size()}}; + packet.GetPacket()->pts = static_cast<s64>(pts); + // Hidden VP9 frames never come out: keep the set from growing over a long movie. + if (impl->pending_pts.size() >= 64) { + impl->pending_pts.erase(impl->pending_pts.begin()); + } + impl->pending_pts.insert(pts); + impl->context->SendPacket(packet); + impl->Receive(pictures); +} + +void Nx2VideoDecoder::Drain(std::vector<Nx2Picture>& pictures) { + if (!impl->open) { + return; + } + avcodec_send_packet(impl->context->GetCodecContext(), nullptr); + impl->Receive(pictures); + Reset(); +} + +void Nx2VideoDecoder::Reset() { + if (impl->open) { + avcodec_flush_buffers(impl->context->GetCodecContext()); + } + impl->pending_pts.clear(); +} + +void Nx2VideoDecoder::Render(const Nx2Picture& picture, Nx2PictureFormat format, u32 width, + u32 height, u64 pitch, u64 chroma_offset, std::span<u8> out) { + const AVFrame* f = picture.frame->GetFrame(); + const u64 needed = format == Nx2PictureFormat::Nv12 + ? chroma_offset + pitch * ((u64{height} + 1) / 2) + : pitch * height; + if (width == 0 || height == 0 || out.size() < needed) { + return; + } + const AVPixelFormat target = format == Nx2PictureFormat::Nv12 ? AV_PIX_FMT_NV12 + : format == Nx2PictureFormat::Rgba8 ? AV_PIX_FMT_RGBA + : AV_PIX_FMT_BGRA; + impl->scaler = sws_getCachedContext(impl->scaler, f->width, f->height, + static_cast<AVPixelFormat>(f->format), + static_cast<int>(width), static_cast<int>(height), target, + SWS_BILINEAR, nullptr, nullptr, nullptr); + if (impl->scaler == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: cannot convert {} to {}", + av_get_pix_fmt_name(static_cast<AVPixelFormat>(f->format)), + av_get_pix_fmt_name(target)); + return; + } + if (format != Nx2PictureFormat::Nv12) { + // The stream's matrix and range; untagged HD video is BT.709. + int space = SWS_CS_ITU601; + if (f->colorspace == AVCOL_SPC_BT709 || + (f->colorspace == AVCOL_SPC_UNSPECIFIED && f->height >= 720)) { + space = SWS_CS_ITU709; + } else if (f->colorspace == AVCOL_SPC_BT2020_NCL || f->colorspace == AVCOL_SPC_BT2020_CL) { + space = SWS_CS_BT2020; + } + const int* coefficients = sws_getCoefficients(space); + sws_setColorspaceDetails(impl->scaler, coefficients, f->color_range == AVCOL_RANGE_JPEG, + coefficients, 1, 0, 1 << 16, 1 << 16); + } + std::array<u8*, 4> planes{out.data(), nullptr, nullptr, nullptr}; + std::array<int, 4> strides{static_cast<int>(pitch), 0, 0, 0}; + if (format == Nx2PictureFormat::Nv12) { + planes[1] = out.data() + chroma_offset; + strides[1] = static_cast<int>(pitch); + } + sws_scale(impl->scaler, f->data, f->linesize, 0, f->height, planes.data(), strides.data()); +} + +} // namespace Tegra::Host1x diff --git a/src/video_core/host1x/nx2_video_decoder.h b/src/video_core/host1x/nx2_video_decoder.h new file mode 100644 index 0000000000..c979a2c801 --- /dev/null +++ b/src/video_core/host1x/nx2_video_decoder.h @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <memory> +#include <span> +#include <string> +#include <string_view> +#include <vector> + +#include "common/common_types.h" + +namespace FFmpeg { +class Frame; +} + +namespace Tegra::Host1x { + +/// A picture decoded by Nx2VideoDecoder, in display order. +struct Nx2Picture { + u32 width{}; + u32 height{}; + u64 pts{}; ///< As submitted with the compressed frame + std::shared_ptr<FFmpeg::Frame> frame; +}; + +/// Pixel layouts of a Switch 2 decoder output image. +enum class Nx2PictureFormat : u32 { + Nv12, ///< Luma rows, then interleaved Cb/Cr rows at chroma_offset + Rgba8, + Bgra8, +}; + +/// The decoder behind the Switch 2 video decoder service (nn::codec::video): FFmpeg decodes the +/// elementary stream the game demuxed, in software or on the host GPU (NVDEC emulation setting). +/// Used from one thread at a time. +class Nx2VideoDecoder { +public: + /// mime: the codec name nn::codec sends with Configure ("video/vp9", "video/vp8", ...). + explicit Nx2VideoDecoder(std::string_view mime); + ~Nx2VideoDecoder(); + + bool IsOpen() const; + + const std::string& Name() const; + + /// Decodes one compressed frame and appends the pictures it completes. + void Decode(std::span<const u8> data, u64 pts, std::vector<Nx2Picture>& pictures); + + /// End of stream: appends the pictures the decoder still holds, then starts over. + void Drain(std::vector<Nx2Picture>& pictures); + + /// Drops everything queued in the decoder (the game flushed it to seek). + void Reset(); + + /// Converts a picture to `format` in `out`: rows `pitch` bytes apart, NV12's chroma plane at + /// `chroma_offset`, scaled to width x height. + void Render(const Nx2Picture& picture, Nx2PictureFormat format, u32 width, u32 height, + u64 pitch, u64 chroma_offset, std::span<u8> out); + +private: + struct Impl; + std::unique_ptr<Impl> impl; +}; + +} // namespace Tegra::Host1x diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index c4e7ac3f6f..8082e830d6 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -15,6 +15,7 @@ set(GLSL_INCLUDES set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp + ${CMAKE_CURRENT_SOURCE_DIR}/bc_encoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag diff --git a/src/video_core/host_shaders/astc_decoder.comp b/src/video_core/host_shaders/astc_decoder.comp index e059c683ff..01bec11f17 100644 --- a/src/video_core/host_shaders/astc_decoder.comp +++ b/src/video_core/host_shaders/astc_decoder.comp @@ -33,6 +33,7 @@ UNIFORM(3) uint block_size; UNIFORM(4) uint x_shift; UNIFORM(5) uint block_height; UNIFORM(6) uint block_height_mask; +UNIFORM(7) uint native_tiling; END_PUSH_CONSTANTS struct EncodingData { @@ -1404,7 +1405,12 @@ uint SwizzleOffset(uvec2 pos) { void main() { uvec3 pos = gl_GlobalInvocationID; pos.x <<= BYTES_PER_BLOCK_LOG2; - const uint swizzle = SwizzleOffset(pos.xy); + uint swizzle = SwizzleOffset(pos.xy); + if (native_tiling != 0u) { + // Switch 2 GOBs swap offset bits 5 and 6 (see block_linear_unswizzle_2d.comp). + const uint difference = ((swizzle >> 5u) ^ (swizzle >> 6u)) & 1u; + swizzle ^= (difference << 5u) | (difference << 6u); + } const uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; uint offset = 0; diff --git a/src/video_core/host_shaders/bc_encoder.comp b/src/video_core/host_shaders/bc_encoder.comp new file mode 100644 index 0000000000..3a87e92dcb --- /dev/null +++ b/src/video_core/host_shaders/bc_encoder.comp @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Compresses one level of an RGBA8 image (the ASTC decoder's output) to BC1 or BC3, one 4x4 block +// per invocation, so re-compressed ASTC never round-trips through the CPU. Colour endpoints are +// the block's bounding box, oriented along the colours' covariance and inset by 1/16; each texel +// takes the nearest palette entry. Values are encoded as stored (sRGB data stays sRGB, like the +// CPU path). + +#version 460 core + +layout (local_size_x = 8, local_size_y = 8, local_size_z = 1) in; + +layout (set = 0, binding = 0, rgba8) uniform readonly image2DArray source; + +layout (std430, set = 0, binding = 1) writeonly buffer Output { + uvec2 blocks[]; +}; + +layout (push_constant) uniform PushConstants { + uint blocks_x; + uint blocks_y; + uint width; + uint height; + uint bc3; + uint out_offset; // in 8-byte units +}; + +uint Pack565(vec3 color) { + const uvec3 q = uvec3(round(clamp(color, 0.0, 1.0) * vec3(31.0, 63.0, 31.0))); + return (q.r << 11) | (q.g << 5) | q.b; +} + +vec3 Unpack565(uint value) { + const uvec3 q = uvec3((value >> 11) & 31u, (value >> 5) & 63u, value & 31u); + const uvec3 e = uvec3((q.r << 3) | (q.r >> 2), (q.g << 2) | (q.g >> 4), (q.b << 3) | (q.b >> 2)); + return vec3(e) / 255.0; +} + +float Distance2(vec3 a, vec3 b) { + const vec3 d = a - b; + return dot(d, d); +} + +// BC1 colour block. With punch_through (BC1 only), texels with alpha < 0.5 use the transparent +// index of the three-colour mode; otherwise the four-colour mode is used (BC3 always decodes so). +uvec2 EncodeColor(in vec4 texels[16], bool punch_through) { + vec3 lo = vec3(1.0); + vec3 hi = vec3(0.0); + for (int i = 0; i < 16; ++i) { + if (punch_through && texels[i].a < 0.5) { + continue; + } + lo = min(lo, texels[i].rgb); + hi = max(hi, texels[i].rgb); + } + if (any(greaterThan(lo, hi))) { + // Every texel transparent: colour0 == colour1 selects three-colour mode, index 3. + return uvec2(0u, 0xFFFFFFFFu); + } + const vec3 center = (lo + hi) * 0.5; + const vec3 extent = hi - lo; + const int major = extent.g >= extent.r && extent.g >= extent.b ? 1 : (extent.r >= extent.b ? 0 : 2); + vec3 covariance = vec3(0.0); + for (int i = 0; i < 16; ++i) { + if (punch_through && texels[i].a < 0.5) { + continue; + } + const vec3 d = texels[i].rgb - center; + covariance += d * d[major]; + } + vec3 e0 = hi; + vec3 e1 = lo; + for (int c = 0; c < 3; ++c) { + if (covariance[c] < 0.0) { + e0[c] = lo[c]; + e1[c] = hi[c]; + } + } + const vec3 inset = (e0 - e1) / 16.0; + e0 -= inset; + e1 += inset; + + uint q0 = Pack565(e0); + uint q1 = Pack565(e1); + // Four-colour mode needs colour0 > colour1, three-colour mode colour0 <= colour1. + if (punch_through ? q0 > q1 : q0 < q1) { + const uint t = q0; + q0 = q1; + q1 = t; + } + if (!punch_through && q0 == q1) { + return uvec2(q0 | (q1 << 16), 0u); + } + vec3 palette[4]; + palette[0] = Unpack565(q0); + palette[1] = Unpack565(q1); + if (punch_through) { + palette[2] = (palette[0] + palette[1]) * 0.5; + palette[3] = vec3(0.0); + } else { + palette[2] = (2.0 * palette[0] + palette[1]) / 3.0; + palette[3] = (palette[0] + 2.0 * palette[1]) / 3.0; + } + const int entries = punch_through ? 3 : 4; + uint indices = 0u; + for (int i = 0; i < 16; ++i) { + uint best = 0u; + if (punch_through && texels[i].a < 0.5) { + best = 3u; + } else { + float best_distance = Distance2(texels[i].rgb, palette[0]); + for (int p = 1; p < entries; ++p) { + const float distance = Distance2(texels[i].rgb, palette[p]); + if (distance < best_distance) { + best_distance = distance; + best = uint(p); + } + } + } + indices |= best << (2 * i); + } + return uvec2(q0 | (q1 << 16), indices); +} + +// BC3 alpha block: eight-value mode (alpha0 > alpha1), indices 0 = alpha0, 1 = alpha1, 2..7 the +// ramp from alpha0 to alpha1. +uvec2 EncodeAlpha(in vec4 texels[16]) { + float lo = 1.0; + float hi = 0.0; + for (int i = 0; i < 16; ++i) { + lo = min(lo, texels[i].a); + hi = max(hi, texels[i].a); + } + const uint a0 = uint(round(hi * 255.0)); + const uint a1 = uint(round(lo * 255.0)); + uint bits_lo = a0 | (a1 << 8); + uint bits_hi = 0u; + if (a0 == a1) { + return uvec2(bits_lo, bits_hi); + } + for (uint i = 0u; i < 16u; ++i) { + const float t = (texels[i].a * 255.0 - float(a1)) / float(a0 - a1); + const uint q = uint(clamp(round(t * 7.0), 0.0, 7.0)); + const uint index = q == 7u ? 0u : (q == 0u ? 1u : 8u - q); + const uint bit = 16u + 3u * i; + if (bit < 32u) { + bits_lo |= index << bit; + if (bit > 29u) { + bits_hi |= index >> (32u - bit); + } + } else { + bits_hi |= index << (bit - 32u); + } + } + return uvec2(bits_lo, bits_hi); +} + +void main() { + const uvec3 id = gl_GlobalInvocationID; + if (id.x >= blocks_x || id.y >= blocks_y) { + return; + } + vec4 texels[16]; + const ivec2 base = ivec2(id.xy) * 4; + const ivec2 last = ivec2(width, height) - 1; + bool any_transparent = false; + for (int y = 0; y < 4; ++y) { + for (int x = 0; x < 4; ++x) { + // Edge blocks repeat the last row/column so padding doesn't widen the endpoints. + const ivec2 coord = min(base + ivec2(x, y), last); + texels[y * 4 + x] = imageLoad(source, ivec3(coord, int(id.z))); + any_transparent = any_transparent || texels[y * 4 + x].a < 0.5; + } + } + const uint block = (id.z * blocks_y + id.y) * blocks_x + id.x; + if (bc3 != 0u) { + blocks[out_offset + block * 2u] = EncodeAlpha(texels); + blocks[out_offset + block * 2u + 1u] = EncodeColor(texels, false); + } else { + blocks[out_offset + block] = EncodeColor(texels, any_transparent); + } +} diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp index 2a3446ca66..bbe73acd1a 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp @@ -39,6 +39,7 @@ UNIFORM(4) uint block_size; UNIFORM(5) uint x_shift; UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; +UNIFORM(8) uint native_tiling; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -77,7 +78,12 @@ uint SwizzleTable(uint pos) { uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp index 34e4f93307..8dfaf0cd1a 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp @@ -41,6 +41,7 @@ UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; UNIFORM(8) uint block_depth; UNIFORM(9) uint block_depth_mask; +UNIFORM(10) uint native_tiling; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -79,7 +80,12 @@ uint SwizzleTable(uint pos) { uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp index e084837f25..28b1dcf599 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp @@ -42,7 +42,7 @@ layout(push_constant) uniform PushConstants { uint block_depth; // Offset 48 uint block_depth_mask; // Offset 52 - int _pad; // Offset 56 + uint native_tiling; // Offset 56 ivec3 destination; // Offset 60 } pc; @@ -59,6 +59,7 @@ BEGIN_PUSH_CONSTANTS UNIFORM(8) uint block_depth; UNIFORM(9) uint block_depth_mask; UNIFORM(10) uvec3 blocks_dim; + UNIFORM(11) uint native_tiling; END_PUSH_CONSTANTS #define pc // Map pc prefix to nothing for OpenGL compatibility #endif @@ -103,7 +104,12 @@ uint SwizzleTable(uint pos) { // --- Helpers --- uint SwizzleOffset(uvec2 pos) { pos &= SWIZZLE_MASK; - return SwizzleTable(pos.y * 64u + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (pc.native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp index 86f528af15..485b8eaf10 100644 --- a/src/video_core/macro.cpp +++ b/src/video_core/macro.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <array> +#include <cstdlib> +#include <unordered_set> #include <cstring> #include <fstream> #include <optional> @@ -14,6 +17,7 @@ #ifdef ARCHITECTURE_x86_64 #include "common/x64/xbyak.h" +#include "video_core/macro_turing_jit.h" #endif #include "common/assert.h" @@ -25,6 +29,7 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/dirty_flags.h" +#include "video_core/memory_manager.h" #include "video_core/rasterizer_interface.h" #include "video_core/macro.h" @@ -32,6 +37,7 @@ #include "common/bit_field.h" #include "common/logging.h" #include "video_core/engines/maxwell_3d.h" +#include "video_core/nx2_profile.h" namespace Tegra { @@ -1339,6 +1345,212 @@ static void Dump(u64 hash, std::span<const u32> code, bool decompiled = false) { void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u32 method, std::span<const u32> parameters) { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro}; + if (native_turing) { + maxwell3d.RefreshParameters(); + const auto context = reinterpret_cast<std::uintptr_t>(this); + static std::unordered_set<std::uintptr_t> captured_contexts; + static const bool capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE") != nullptr; + if (capture && captured_contexts.insert(context).second) { + std::ofstream dump("native-graphics-mme-" + std::to_string(context) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + } + // EDEN_NX2_MME_LOG: log every macro call (runs ~750,000 times a second in title 1). + if (static const bool log_calls = std::getenv("EDEN_NX2_MME_LOG") != nullptr; log_calls) { + LOG_DEBUG(HW_GPU, "Native MME context={:#x}", context); + LOG_DEBUG(HW_GPU, "Native MME start={:#x}, parameters={}, data={:08x}", method, parameters.size(), fmt::join(parameters, ",")); + } + // Reused between calls (a macro never calls another: methods >= 0xe00 are refused below); + // filled only for calls that are interpreted. + auto& fifo = native_fifo; + native_interpreter.active_dram = &maxwell3d.GetNativeDataRam(); + // EDEN_NX2_MME_RECORD=<file>: write the first 200000 calls (u32 start, u32 count, the + // parameters) and then the macro code to <file>.code, to check interpreter changes + // offline against the previous interpreter. EDEN_NX2_MME_RECORD_SKIP=<n>: skip the first + // n calls (title 1 makes ~1,000,000 a second in 3D scenes). + if (static const char* const record_path = std::getenv("EDEN_NX2_MME_RECORD"); record_path != nullptr) { + static std::ofstream record{record_path, std::ios::binary}; + static u32 recorded{}; + static u64 to_skip = [] { + const char* value = std::getenv("EDEN_NX2_MME_RECORD_SKIP"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL; + }(); + if (to_skip > 0) { + --to_skip; + } else if (recorded < 200000) { + const u32 count = static_cast<u32>(parameters.size()); + record.write(reinterpret_cast<const char*>(&method), sizeof(method)); + record.write(reinterpret_cast<const char*>(&count), sizeof(count)); + record.write(reinterpret_cast<const char*>(parameters.data()), parameters.size_bytes()); + if (++recorded == 200000) { + record.flush(); + std::ofstream code{std::string{record_path} + ".code", std::ios::binary}; + code.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + LOG_INFO(HW_GPU, "Recorded 200000 macro calls to {}", record_path); + } + } + } + // A method write of the macro (also what a remembered call replays). + bool read_memory = false; + const auto emit = [&](u32 address, u32 value) { + if (address >= 0xe00) return false; + if (address == 0x556) { + // SET_RENDER_ENABLE_OVERRIDE: 0 use the render-enable condition, 1 always, + // 2 never. Log a histogram every 20000 macro writes (pop-in diagnostics). + static std::array<u64, 4> modes{}; + static u64 enable_writes{}; + ++modes[value & 3]; + if (++enable_writes % 20000 == 0) { + LOG_INFO(HW_GPU, "Native MME render enable modes: use {} always {} never {} other {}", + modes[0], modes[1], modes[2], modes[3]); + } + } + { + VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method}; + maxwell3d.CallMethod(system, address, value, true); + } + // Turing MME memory reads (methods 0x550 SET_MME_MEM_ADDRESS_A/B, 0x558 data RAM + // address, 0x55c MME_DMA_READ into data RAM, 0x560 MME_DMA_READ_FIFOED into + // the load FIFO, which the macro then reads after a load barrier). + constexpr u32 MemAddressA = 0x154, MemAddressB = 0x155, DataRamAddress = 0x156; + constexpr u32 DmaRead = 0x157, DmaReadFifoed = 0x158; + if (address == DmaRead || address == DmaReadFifoed) { + read_memory = true; + const GPUVAddr source = (GPUVAddr{maxwell3d.GetRegisterValue(MemAddressA) & 0x1ffffff} << 32) | + maxwell3d.GetRegisterValue(MemAddressB); + auto& memory = maxwell3d.GetMemoryManager(); + const u32 ram = maxwell3d.GetRegisterValue(DataRamAddress); + // The words may have just been written by the GPU (e.g. indirect draw arguments + // from a culling dispatch) and still be only in host buffers: flush them first, + // or the macro draws with an older frame's arguments (objects pop in and out). + const size_t read_bytes = size_t{std::min<u32>(value, 0x1000)} * 4; + if (VideoCommon::Nx2Profile::Enabled()) { + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes); + for (GPUVAddr page = source & ~GPUVAddr{0xfff}; page < source + read_bytes; page += 0x1000) { + if (const auto device = memory.GpuToCpuAddress(page)) { + VideoCommon::Nx2Profile::DmaReadPages().insert(*device >> 12); + } + } + } + if (memory.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + memory.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); + static u64 flushes{}; + if (++flushes % 2000 == 1) { + LOG_INFO(HW_GPU, "Native MME DMA read flushed GPU-written memory ({} times)", flushes); + } + } + static u32 dma_logs{}; + if (dma_logs < 400) { + ++dma_logs; + std::array<u32, 8> words{}; + for (u32 i = 0; i < std::min<u32>(value, 8); ++i) words[i] = memory.Read<u32>(source + u64{i} * 4); + LOG_INFO(HW_GPU, "Native MME DMA read{} start={:#x} source={:#x} count={} words={:08x}", + address == DmaReadFifoed ? " fifoed" : "", method, source, value, fmt::join(words, ",")); + } + for (u32 i = 0; i < std::min<u32>(value, 0x1000); ++i) { + const u32 word = memory.Read<u32>(source + u64{i} * 4); + if (address == DmaReadFifoed) { + fifo.push_back(word); + } else if (ram + i < native_interpreter.active_dram->size()) { + if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr; + mme_dump && ram + i == 0x27 && (*native_interpreter.active_dram)[0x27] != word) { + LOG_INFO(HW_GPU, "MME data RAM[0x27] {:#x} -> {:#x} by 3D engine {} (DMA from {:#x})", + (*native_interpreter.active_dram)[0x27], word, fmt::ptr(&maxwell3d), source); + } + (*native_interpreter.active_dram)[ram + i] = word; + } + } + } + return true; + }; +#ifdef ARCHITECTURE_x86_64 + // Macro code runs compiled to x86-64 (macro_turing_jit.h) instead of through the memo + // and the interpreter; EDEN_NX2_MME_JIT=0 goes back to those. + static const bool use_jit = [] { + const char* value = std::getenv("EDEN_NX2_MME_JIT"); + return value == nullptr || value[0] != '0'; + }(); + if (use_jit) { + if (!native_jit) { + native_jit = std::make_shared<TuringMacroJit>(); + } + native_jit->Prepare(native_code, native_code_generation); + fifo.assign(parameters.begin(), parameters.end()); + const auto result = native_jit->Execute( + method, fifo, std::span<u32>{*native_interpreter.active_dram}, + [&](u32 address) { + return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U; + }, + emit); + if (!result.success) { + native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native MME (JIT) failed: start={:#x}, consumed={}/{}, error={}", + method, result.consumed, parameters.size(), result.error); + } + return; + } +#endif + // EDEN_NX2_MME_MEMO=0: interpret every call (see macro_memo.h). + static const bool use_memo = [] { + const char* value = std::getenv("EDEN_NX2_MME_MEMO"); + return value == nullptr || value[0] != '0'; + }(); + if (use_memo) { + // EDEN_NX2_MME_MEMO_STATS: log the memo's counters every 2^24 calls. + if (static const bool memo_stats = std::getenv("EDEN_NX2_MME_MEMO_STATS") != nullptr; + memo_stats && (++native_calls & 0xffffff) == 0) { + const auto& stats = native_memo.GetStatistics(); + LOG_INFO(HW_GPU, + "Macro memo: {} hits ({} on the last call's slot), {} misses, {} guard " + "failures, {} stores ({} in place), {} compactions, {} clears; {} entries, " + "{} KiB", + stats.hits, stats.slot_hits, stats.misses, stats.guard_failures, stats.stores, + stats.overwrites, stats.compactions, stats.clears, native_memo.Size(), + native_memo.ArenaWords() * 4 / 1024); + } + native_memo.Synchronize(native_code_generation); + if (decltype(native_memo)::Effects effects; + native_memo.Find(method, parameters, *native_interpreter.active_dram, effects)) { + auto& data_ram = *native_interpreter.active_dram; + for (size_t i = 0; i < effects.data_ram_writes.size(); i += 2) { + data_ram[effects.data_ram_writes[i]] = effects.data_ram_writes[i + 1]; + } + for (size_t i = 0; i < effects.writes.size(); i += 2) { + emit(effects.writes[i], effects.writes[i + 1]); + } + return; + } + native_writes.clear(); + } + fifo.assign(parameters.begin(), parameters.end()); + const u64 code_generation = native_code_generation; + const auto result = native_interpreter.Execute(native_code, method, fifo, + [&](u32 address) { return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U; }, + [&](u32 address, u32 value) { + if (use_memo) { + native_writes.emplace_back(address, value); + } + return emit(address, value); + }); + // Remember the call if its effects depend only on its parameters and the data RAM words + // it read (and the macro code did not change under it). + if (use_memo && result.success && !read_memory && !native_interpreter.read_registers && + !native_interpreter.reread_data_ram && code_generation == native_code_generation) { + native_memo.Store(method, parameters, native_interpreter.data_ram_reads, + native_interpreter.data_ram_writes, native_writes); + } + if (!result.success) { + native_execution_failed = true; + if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + diagnostics) { + std::ofstream dump("native-mme-" + std::to_string(method) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + } + LOG_ERROR(HW_GPU, "Native MME failed: start={:#x}, pc={:#x}, consumed={}/{}, error={}", + method, result.pc, result.consumed, parameters.size(), result.error); + } + return; + } const auto execute_variant = [&system, &maxwell3d, ¶meters, method](AnyCachedMacro& cached) { if (std::holds_alternative<MacroInterpreterImpl>(cached) || diff --git a/src/video_core/macro.h b/src/video_core/macro.h index 90519a5429..0d574636e9 100644 --- a/src/video_core/macro.h +++ b/src/video_core/macro.h @@ -7,6 +7,12 @@ #pragma once #include <memory> +#include "video_core/macro_memo.h" +#include "video_core/macro_turing.h" + +namespace Tegra { +class TuringMacroJit; +} #include <span> #include <variant> #include <vector> @@ -223,10 +229,16 @@ struct MacroEngine { MacroEngine(bool is_interpreted_) noexcept : is_interpreted{is_interpreted_} {} // Store the uploaded macro code to compile them when they're called. inline void AddCode(u32 method, u32 data) noexcept { + if (native_turing) { + if (native_write_offset < native_code.size()) native_code[native_write_offset++] = data; + ++native_code_generation; + return; + } uploaded_macro_code[method].push_back(data); } // Clear the code associated with a method. inline void ClearCode(u32 method) noexcept { + if (native_turing) { native_write_offset = method < 0x1000 ? method * 3 : static_cast<u32>(native_code.size()); return; } macro_cache.erase(method); uploaded_macro_code.erase(method); } @@ -240,6 +252,27 @@ struct MacroEngine { ::Common::unordered_map<u32, CacheInfo> macro_cache; ::Common::unordered_map<u32, std::vector<u32>> uploaded_macro_code; bool is_interpreted; + bool native_turing{}; + bool native_execution_failed{}; + u32 native_write_offset{}; + std::array<u32, 0x3000> native_code{}; + TuringMacroInterpreter native_interpreter; + std::vector<u32> native_fifo; + // Calls replayed instead of interpreted (macro_memo.h), and what the current call wrote. + u64 native_code_generation{}; + BasicMacroMemo<::Common::unordered_map<u64, u32, MacroMemoHash>> native_memo; + /// The macro code compiled to x86-64 (EDEN_NX2_MME_JIT=0 turns it off), created on first use. + std::shared_ptr<TuringMacroJit> native_jit; + MacroMemoPairs native_writes; + u64 native_calls{}; + +}; + +struct NativeMacroContext { + MacroEngine engine{true}; + std::array<u32, 0x80> positions{}; + MacroEngine compute_engine{true}; + std::array<u32, 0x80> compute_positions{}; }; } // namespace Tegra diff --git a/src/video_core/macro_memo.h b/src/video_core/macro_memo.h new file mode 100644 index 0000000000..7fe914b4f2 --- /dev/null +++ b/src/video_core/macro_memo.h @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <cstddef> +#include <cstdint> +#include <span> +#include <type_traits> +#include <utility> +#include <vector> + +namespace Tegra { + +using MacroMemoPairs = std::vector<std::pair<std::uint32_t, std::uint32_t>>; + +/// The memo's keys are already well mixed hashes: use them as they are (std::hash<u64> is a +/// byte-wise FNV-1a on MSVC, ~20% of a lookup). +struct MacroMemoHash { + using is_avalanching = std::true_type; + std::size_t operator()(std::uint64_t key) const noexcept { + return static_cast<std::size_t>(key); + } +}; + +// The effects of native macro calls that depend on nothing but their start address, their +// parameters and data RAM words they read (no register or memory reads), so that a repeated +// call replays its data RAM and method writes instead of being interpreted. In title 1's 3D +// scenes ~96% of the ~1,000,000 calls a second repeat an earlier one exactly, and they run 7 +// instructions on average: the per-call interpreter cost, not the instructions, was the time. +// `Map` maps a u64 (the call's hash) to a u32 (its record's offset in the arena). +template <typename Map> +class BasicMacroMemo { +public: + /// A remembered call's effects: (address, value) pairs, flattened. + struct Effects { + std::span<const std::uint32_t> data_ram_writes; + std::span<const std::uint32_t> writes; ///< (method, value) in order + }; + + struct Statistics { + std::uint64_t hits{}, slot_hits{}, misses{}, guard_failures{}, stores{}, overwrites{}, + compactions{}, clears{}; + }; + + BasicMacroMemo() { + last_by_start.fill({0, NoRecord}); + } + + /// An earlier call with this start and these parameters whose data RAM reads still hold. + bool Find(std::uint32_t start, std::span<const std::uint32_t> parameters, + std::span<const std::uint32_t> data_ram, Effects& effects) { + // Nearly half the calls repeat the previous call of the same macro: try its record before + // hashing (no probe, and the record is still in the cache). + auto& last = last_by_start[start % last_by_start.size()]; + if (last.first == start && last.second != NoRecord && + Match(last.second, start, parameters, data_ram, effects)) { + ++statistics.hits; + ++statistics.slot_hits; + return true; + } + const auto it = entries.find(Hash(start, parameters)); + if (it == entries.end()) { + ++statistics.misses; + return false; + } + if (!Match(it->second, start, parameters, data_ram, effects)) { + ++statistics.guard_failures; // or another call with the same hash + return false; + } + last = {start, it->second}; + ++statistics.hits; + return true; + } + +private: + static constexpr std::uint32_t NoRecord = ~0U; + + /// Whether the record at `offset` is this call with its data RAM reads holding; fills + /// `effects` if so. + bool Match(std::uint32_t offset, std::uint32_t start, std::span<const std::uint32_t> parameters, + std::span<const std::uint32_t> data_ram, Effects& effects) { + // Record: start, parameter count, read count, data RAM write count, write count, then + // the parameters and the (address, value) pairs. + const std::uint32_t* record = arena.data() + offset; + const std::uint32_t num_parameters = record[1], num_reads = record[2]; + const std::uint32_t num_data_ram_writes = record[3], num_writes = record[4]; + if (record[0] != start || num_parameters != parameters.size() || + !std::equal(parameters.begin(), parameters.end(), record + HeaderWords)) { + return false; // another call + } + const std::uint32_t* reads = record + HeaderWords + num_parameters; + for (std::uint32_t i = 0; i < num_reads; ++i) { + if (data_ram[reads[i * 2]] != reads[i * 2 + 1]) { + return false; + } + } + const std::uint32_t* data_ram_writes = reads + num_reads * 2; + effects.data_ram_writes = {data_ram_writes, num_data_ram_writes * 2}; + effects.writes = {data_ram_writes + num_data_ram_writes * 2, num_writes * 2}; + return true; + } + +public: + void Store(std::uint32_t start, std::span<const std::uint32_t> parameters, + const MacroMemoPairs& data_ram_reads, const MacroMemoPairs& data_ram_writes, + const MacroMemoPairs& writes) { + ++statistics.stores; + const std::uint64_t hash = Hash(start, parameters); + // A call stored again (its data RAM reads no longer held) usually has the same shape: + // rewrite its record in place, or the arena fills with stale copies. + if (const auto it = entries.find(hash); it != entries.end()) { + std::uint32_t* record = arena.data() + it->second; + if (record[0] == start && record[1] == parameters.size() && + record[2] == data_ram_reads.size() && record[3] == data_ram_writes.size() && + record[4] == writes.size() && + std::equal(parameters.begin(), parameters.end(), record + HeaderWords)) { + std::uint32_t* out = record + HeaderWords + parameters.size(); + for (const MacroMemoPairs* pairs : {&data_ram_reads, &data_ram_writes, &writes}) { + for (const auto& [address, value] : *pairs) { + *out++ = address; + *out++ = value; + } + } + ++statistics.overwrites; + return; + } + } + if (arena.size() >= MaxArenaWords) { + Compact(); + } + const auto offset = static_cast<std::uint32_t>(arena.size()); + arena.push_back(start); + arena.push_back(static_cast<std::uint32_t>(parameters.size())); + arena.push_back(static_cast<std::uint32_t>(data_ram_reads.size())); + arena.push_back(static_cast<std::uint32_t>(data_ram_writes.size())); + arena.push_back(static_cast<std::uint32_t>(writes.size())); + arena.insert(arena.end(), parameters.begin(), parameters.end()); + for (const MacroMemoPairs* pairs : {&data_ram_reads, &data_ram_writes, &writes}) { + for (const auto& [address, value] : *pairs) { + arena.push_back(address); + arena.push_back(value); + } + } + entries.insert_or_assign(hash, offset); + last_by_start[start % last_by_start.size()] = {start, offset}; + } + + const Statistics& GetStatistics() const { + return statistics; + } + + std::size_t Size() const { + return entries.size(); + } + + std::size_t ArenaWords() const { + return arena.size(); + } + + /// Forgets every call when the macro code is no longer the code they ran. + void Synchronize(std::uint64_t code_generation) { + if (code_generation != generation) { + generation = code_generation; + Clear(); + } + } + +private: + static constexpr std::size_t HeaderWords = 5; + static constexpr std::size_t MaxArenaWords = std::size_t{1} << 22; // 16 MiB + + static std::uint64_t Hash(std::uint32_t start, std::span<const std::uint32_t> parameters) { + std::uint64_t hash = 0x9e3779b97f4a7c15ULL ^ (std::uint64_t{start} << 32) ^ parameters.size(); + for (const std::uint32_t word : parameters) { + hash = (hash ^ word) * 0xff51afd7ed558ccdULL; + hash ^= hash >> 32; + } + return hash; + } + + void Clear() { + arena.clear(); + entries.clear(); + last_by_start.fill({0, NoRecord}); + } + + /// Copies the records still in the map to a new arena. A call stored again with another + /// shape leaves its old record behind: title 1 filled the arena with them every ~2 s, and + /// clearing it made the next ~20,000 calls interpret again. Clears only when the live + /// records alone would fill most of it. + void Compact() { + compacted.clear(); + for (auto& entry : entries) { + const std::uint32_t* record = arena.data() + entry.second; + const std::size_t words = + HeaderWords + record[1] + std::size_t{2} * (record[2] + record[3] + record[4]); + entry.second = static_cast<std::uint32_t>(compacted.size()); + compacted.insert(compacted.end(), record, record + words); + } + arena.swap(compacted); + last_by_start.fill({0, NoRecord}); + ++statistics.compactions; + if (arena.size() >= MaxArenaWords / 4 * 3) { + Clear(); + ++statistics.clears; + } + } + + Map entries; + std::vector<std::uint32_t> arena; + std::vector<std::uint32_t> compacted; ///< Compact's new arena (kept for its capacity) + /// The last record found or stored per macro start (direct mapped). + std::array<std::pair<std::uint32_t, std::uint32_t>, 256> last_by_start{}; + std::uint64_t generation{}; + Statistics statistics; +}; + +} // namespace Tegra diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h new file mode 100644 index 0000000000..600db48119 --- /dev/null +++ b/src/video_core/macro_turing.h @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +// Instruction semantics adapted from Mesa mme_tu104_sim.c and mme_tu104.c: +// Copyright © 2022 Collabora Ltd. SPDX-License-Identifier: MIT +#pragma once + +#include <algorithm> +#include <array> +#include <bit> +#include <cstdint> +#include <span> +#include <string_view> +#include <vector> + +namespace Tegra { + +// TU104/GA10x MME has two ALUs and two outputs per 96-bit instruction. +class TuringMacroInterpreter { +public: + using Word = std::uint32_t; + struct Result { + bool success; + Word pc; + std::size_t consumed; + std::string_view error; + }; + + // One instruction's fields, decoded once and reused while its words stay the same (macro + // code runs thousands of times a frame; decoding it every time was most of the interpreter). + struct Decoded { + std::array<Word, 3> raw{~0U, ~0U, ~0U}; + std::array<Word, 2> imm, op, dst, src_a, src_b, out_method, out_emit; + Word pred_reg, pred_mode; + // Lane/output enable masks when the predicate register is non-zero / zero. + Word mask_true, mask_false; + bool end; + std::array<bool, 2> load; + }; + + static Decoded Decode(const std::array<Word, 3>& bits) { + const auto field = [&](Word low, Word count) { + std::uint64_t value = bits[low / 32]; + if (low / 32 < 2) value |= std::uint64_t{bits[low / 32 + 1]} << 32; + return Word((value >> (low % 32)) & ((std::uint64_t{1} << count) - 1)); + }; + Decoded d; + d.raw = bits; + d.imm = {field(30, 16), field(66, 16)}; + d.op = {field(10, 5), field(46, 5)}; + d.dst = {field(15, 5), field(51, 5)}; + d.src_a = {field(20, 5), field(56, 5)}; + d.src_b = {field(25, 5), field(61, 5)}; + d.out_method = {field(82, 3), field(89, 3)}; + d.out_emit = {field(85, 4), field(92, 4)}; + d.pred_reg = field(5, 5); + d.pred_mode = field(1, 4); + d.end = field(0, 1) != 0; + constexpr std::array<std::string_view, 16> predicates{ + "UUUU", "TTTT", "FFFF", "TTUU", "FFUU", "TFUU", "TUUU", "FUUU", + "UUTT", "UUTF", "UUTU", "UUFT", "UUFF", "UUFU", "UUUT", "UUUF"}; + d.mask_true = d.mask_false = 15; + if (d.pred_mode) { + d.mask_true = d.mask_false = 0; + for (Word i = 0; i < 4; ++i) { + if (predicates[d.pred_mode][i] != 'T') d.mask_true |= 1U << i; + if (predicates[d.pred_mode][i] != 'F') d.mask_false |= 1U << i; + } + } + for (Word i = 0; i < 2; ++i) { + const Word reg = 28 + i, out = 3 + i; + d.load[i] = d.pred_reg == reg || d.src_a[0] == reg || d.src_a[1] == reg || + d.src_b[0] == reg || d.src_b[1] == reg || d.out_method[0] == out || + d.out_method[1] == out || d.out_emit[0] == out || d.out_emit[1] == out; + } + return d; + } + + template <typename Read, typename Emit> + // `parameters` is the load FIFO; `emit` may append to it (MME_DMA_READ_FIFOED). + Result Execute(std::span<const Word> code, Word start, + std::vector<Word>& parameters, Read read, Emit emit) { + std::array<Word, 24> regs{}; + Word initialized{}, carry{}, method{}, increment{}; + Word pc = start, loop_count{}, loop_start{}, loop_end{}; + std::size_t consumed{}; + bool end_next{}, stopped{}; + std::string_view error; + const auto fail = [&](std::string_view message) { if (error.empty()) error = message; }; + if (decoded.size() < code.size() / 3) decoded.resize(code.size() / 3); + read_registers = false; + reread_data_ram = false; + data_ram_reads.clear(); + data_ram_writes.clear(); + for (Word steps = 0; steps < 1000000; ++steps) { + if (static_cast<std::size_t>(pc) * 3 + 2 >= code.size()) { + fail("instruction outside uploaded RAM"); + break; + } + const std::array<Word, 3> bits{code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]}; + Decoded& d = decoded[pc]; + if (d.raw != bits) d = Decode(bits); + std::array<Word, 2> load{}, alu{}; + for (Word i = 0; i < 2; ++i) { + if (d.load[i]) { + if (consumed >= parameters.size()) { fail("parameter FIFO underflow"); break; } + load[i] = parameters[consumed++]; + } + } + // Both ALUs read the register file before either destination is committed: their + // operands are read up front. Carry chaining remains explicit between the lanes. + const auto get = [&](Word reg, Word lane) -> Word { + if (reg < 24) { + if (!(initialized & (1U << reg))) fail("uninitialized register read"); + return regs[reg]; + } + switch (reg) { + case 24: return 0; + case 25: return Word(std::int32_t(std::int16_t(d.imm[lane]))); + case 26: return Word(std::int32_t(std::int16_t(d.imm[1 - lane]))); + case 27: return (d.imm[0] << 16) | d.imm[1]; + case 28: return load[0]; + case 29: return load[1]; + default: fail("invalid source register"); return 0; + } + }; + const Word mask = d.pred_mode ? (get(d.pred_reg, 0) ? d.mask_true : d.mask_false) : 15; + std::array<Word, 2> xs{}, ys{}; + for (Word lane = 0; lane < 2; ++lane) { + if (mask & (1U << lane)) { + xs[lane] = get(d.src_a[lane], lane); + ys[lane] = get(d.src_b[lane], lane); + } + } + Word next_pc = pc + 1; + bool branch_cancels_end{}; + for (Word lane = 0; lane < 2 && error.empty(); ++lane) { + if (!(mask & (1U << lane))) continue; + const Word x = xs[lane], y = ys[lane]; + const Word imm = d.imm[lane]; + Word value{}; + const auto condition = [&](Word comparison) { + switch (comparison) { + case 17: case 25: return std::int32_t(x) < std::int32_t(y); + case 18: case 26: return x < y; + case 19: case 27: return std::int32_t(x) <= std::int32_t(y); + case 20: case 28: return x <= y; + default: return x == y; + } + }; + switch (d.op[lane]) { + case 0: value = x + y; carry = value < x; break; + case 1: value = x + y + carry; break; + case 2: value = x - y; carry = value > x; break; + case 3: value = x - y - carry; break; + case 4: case 6: { + const std::uint64_t a = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x; + const std::uint64_t b = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y; + const auto product = a * b; + value = Word(product); carry = Word(product >> 32); break; + } + case 5: value = carry; break; + case 7: + // EXTENDED: x = 0x1000 is the load barrier (Mesa mme_tu104_load_barrier), + // nothing to do in a sequential interpreter. + if (x != 0x1000) fail("unsupported extended operation"); + break; + case 8: value = Word(std::countl_zero(x)); break; + case 9: value = x << (y & 31); break; + case 10: value = x >> (y & 31); break; + case 11: value = Word(std::int32_t(x) >> (y & 31)); break; + case 12: value = x & y; break; + case 13: value = ~(x & y); break; + case 14: value = x | y; break; + case 15: value = x ^ y; break; + case 16: { + const Word dest = (imm >> 10) & 63, count = (imm >> 5) & 31; + const Word source = imm & 31; + if (dest >= 32) { fail("invalid merge destination"); break; } + const Word bits_mask = (1U << count) - 1; + value = (x & ~(bits_mask << dest)) | (((y >> source) & bits_mask) << dest); + break; + } + case 17: case 18: case 19: case 20: case 21: + value = condition(d.op[lane]) ? ~0U : 0; break; + case 22: value = read((x + y) & 0xffff); read_registers = true; break; + case 23: + if (loop_count || imm < 2) { fail("invalid loop"); break; } + // The native SDK emits zero-count loops around FIFO reads. + // Unlike Mesa's test simulator, these must skip the body. + if (x == 0) { next_pc = pc + imm; break; } + loop_count = x - 1; loop_start = pc; loop_end = pc + imm - 1; + break; + case 24: next_pc = pc + (imm & 0x7fff); break; + case 25: case 26: case 27: case 28: case 29: + if (condition(d.op[lane]) == bool(imm & 0x8000)) { + const auto offset = std::int32_t(imm << 19) >> 19; + if (offset == -4096) stopped = true; + else { + next_pc = Word(std::int32_t(pc) + offset); + // Native back edges use bit 14 to suppress the + // fallthrough termination when the branch is taken. + branch_cancels_end = (imm & 0x4000) != 0; + } + } + break; + case 30: + if (x >= active_dram->size()) { fail("data RAM read out of bounds"); break; } + value = (*active_dram)[x]; + if (std::ranges::find(data_ram_writes, x, &DataRamAccess::first) != data_ram_writes.end()) { + reread_data_ram = true; + } + data_ram_reads.emplace_back(x, value); + break; + case 31: + if (x >= active_dram->size()) { fail("data RAM write out of bounds"); break; } + (*active_dram)[x] = y; + data_ram_writes.emplace_back(x, y); + break; + } + alu[lane] = value; + if (d.dst[lane] < 24) { regs[d.dst[lane]] = value; initialized |= 1U << d.dst[lane]; } + else if (d.dst[lane] != 24) fail("invalid destination register"); + } + const auto output = [&](Word selector) -> Word { + switch (selector) { + case 1: return alu[0]; case 2: return alu[1]; + case 3: return load[0]; case 4: return load[1]; + case 5: return d.imm[0]; case 6: return d.imm[1]; + case 8: return d.imm[0] >> 12; case 9: return d.imm[1] >> 12; + case 10: return (d.imm[0] << 16) | d.imm[1]; + default: fail("invalid output selector"); return 0; + } + }; + for (Word lane = 0; lane < 2 && error.empty(); ++lane) { + if (!(mask & (4U << lane))) continue; + if (d.out_method[lane]) { + const Word address = output(d.out_method[lane]); + method = address & 0xfff; increment = (address >> 12) & 15; + } + if (d.out_emit[lane]) { + const Word value = output(d.out_emit[lane]); + if (error.empty() && !emit(method, value)) fail("unsupported emitted method"); + method += increment; + } + } + if (!error.empty()) break; + if (end_next || stopped) return {true, pc, consumed, {}}; + end_next = d.end && !branch_cancels_end; + if (loop_count && pc == loop_end) { --loop_count; next_pc = loop_start + 1; } + pc = next_pc; + } + if (error.empty()) error = "instruction limit exceeded"; + return {false, pc, consumed, error}; + } + + using DataRam = std::array<Word, 0xc00>; + DataRam dram{}; + // The data RAM DREAD/DWRITE use: a channel's own copy when the caller sets one. Macro code is + // shared by the channels of a process, but each channel keeps its own data RAM (title 1's + // channels store their own QMD ring base in word 0x27). + DataRam* active_dram{&dram}; + // Decoded instructions by PC, re-decoded whenever their words differ. + std::vector<Decoded> decoded; + // What the last Execute read besides its parameters, for the macro memo: whether it read a + // register (MME_READ), the data RAM words it read and wrote (address, value) in order, and + // whether it read a word it had written itself. + using DataRamAccess = std::pair<Word, Word>; + bool read_registers{}; + bool reread_data_ram{}; + std::vector<DataRamAccess> data_ram_reads; + std::vector<DataRamAccess> data_ram_writes; +}; + +} // namespace Tegra diff --git a/src/video_core/macro_turing_jit.h b/src/video_core/macro_turing_jit.h new file mode 100644 index 0000000000..d0f24080a3 --- /dev/null +++ b/src/video_core/macro_turing_jit.h @@ -0,0 +1,662 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <array> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <span> +#include <string_view> +#include <vector> + +// Eden's Xbyak configuration (its hash maps): every translation unit must see the same one. +#include "common/x64/xbyak.h" + +#include "video_core/macro_turing.h" + +namespace Tegra { + +// x86-64 code for the TU104/GA10x MME: every instruction of the uploaded macro memory compiled +// once (each a block that falls through to the next one, branches and loops through a jump +// table), with the semantics of TuringMacroInterpreter::Execute. Title 1 calls macros ~1.3 +// million times a second; the interpreter and the macro memo's lookups were ~10% of its GPU +// thread. Emits (method writes) and register reads call back into C++. +class TuringMacroJit { +public: + using Word = std::uint32_t; + using Result = TuringMacroInterpreter::Result; + + struct State { + std::array<Word, 24> regs; + Word carry; + Word method; + Word increment; + Word loop_count; + Word loop_start; + Word loop_end; + Word next_pc; + Word steps; + Word start; + Word consumed; + Word params_size; + Word fail_pc; + std::uint8_t end_next; + std::uint8_t stopped; + std::uint8_t cancel_end; + std::uint8_t pad; + Word scratch_x; + Word scratch_y; + const Word* params; + Word* dram; + void* user; + bool (*emit)(State*, Word method, Word value); + Word (*read)(State*, Word address); + const std::vector<Word>* fifo; + }; + + enum Error : Word { + None = 0, + FifoUnderflow, + InvalidSource, + InvalidDestination, + UnsupportedExtended, + InvalidMerge, + InvalidLoop, + DataRamReadOutOfBounds, + DataRamWriteOutOfBounds, + InvalidOutput, + UnsupportedEmit, + OutsideCode, + StepLimit, + }; + + static std::string_view ErrorText(Word error) { + constexpr std::array<std::string_view, 13> texts{ + "", + "parameter FIFO underflow", + "invalid source register", + "invalid destination register", + "unsupported extended operation", + "invalid merge destination", + "invalid loop", + "data RAM read out of bounds", + "data RAM write out of bounds", + "invalid output selector", + "unsupported emitted method", + "instruction outside uploaded RAM", + "instruction limit exceeded", + }; + return error < texts.size() ? texts[error] : "unknown"; + } + + /// Compiles `code` (3 words per instruction) when it differs from what was compiled. + void Prepare(std::span<const Word> code, std::uint64_t generation) { + if (program && generation == compiled_generation) { + return; + } + program = std::make_unique<Program>(code); + compiled_generation = generation; + } + + template <typename Read, typename Emit> + Result Execute(Word start, std::vector<Word>& parameters, std::span<Word> data_ram, Read read, + Emit emit) { + State state; + state.regs.fill(0); + state.carry = state.method = state.increment = 0; + state.loop_count = state.loop_start = state.loop_end = 0; + state.consumed = 0; + state.fail_pc = 0; + state.end_next = state.stopped = state.cancel_end = 0; + state.start = start; + state.steps = 1000000; + state.params = parameters.data(); + state.params_size = static_cast<Word>(parameters.size()); + state.dram = data_ram.data(); + state.fifo = ¶meters; + Thunks<Read, Emit> thunks{read, emit}; + state.user = &thunks; + state.emit = &Thunks<Read, Emit>::CallEmit; + state.read = &Thunks<Read, Emit>::CallRead; + const Word error = program->entry(&state); + if (error == None) { + return {true, state.fail_pc, state.consumed, {}}; + } + return {false, state.fail_pc, state.consumed, ErrorText(error)}; + } + +private: + template <typename Read, typename Emit> + struct Thunks { + Read& read; + Emit& emit; + static bool CallEmit(State* state, Word method, Word value) { + const bool ok = static_cast<Thunks*>(state->user)->emit(method, value); + // MME_DMA_READ_FIFOED appends to the FIFO from inside the emit. + state->params = state->fifo->data(); + state->params_size = static_cast<Word>(state->fifo->size()); + return ok; + } + static Word CallRead(State* state, Word address) { + return static_cast<Thunks*>(state->user)->read(address); + } + }; + + struct Program : Xbyak::CodeGenerator { + explicit Program(std::span<const Word> code) + : Xbyak::CodeGenerator(1 << 20, Xbyak::AutoGrow) { + Generate(code); + ready(); + entry = getCode<Word (*)(State*)>(); + } + + Word (*entry)(State*) = nullptr; + + private: + using Decoded = TuringMacroInterpreter::Decoded; +#ifdef _WIN32 + const Xbyak::Reg64 arg0 = rcx, arg1 = rdx, arg2 = r8; +#else + const Xbyak::Reg64 arg0 = rdi, arg1 = rsi, arg2 = rdx; +#endif + // rbx: State*; r12d/r13d: the instruction's FIFO loads; r14d/r15d: its ALU results. + const Xbyak::Reg64 st = rbx; + + static constexpr std::size_t RegOffset(Word reg) { + return offsetof(State, regs) + reg * sizeof(Word); + } + + void Generate(std::span<const Word> code) { + const Word count = static_cast<Word>(code.size() / 3); + labels.resize(count); + std::vector<Decoded> decoded(count); + for (Word pc = 0; pc < count; ++pc) { + decoded[pc] = TuringMacroInterpreter::Decode( + {code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]}); + } + Analyze(decoded); + push(rbx); push(rbp); push(r12); push(r13); push(r14); push(r15); + sub(rsp, 40); + mov(st, arg0); + mov(eax, dword[st + offsetof(State, start)]); + cmp(eax, count); + jae(fail_outside, T_NEAR); + lea(rcx, ptr[rip + table]); + jmp(ptr[rcx + rax * 8]); + + for (Word pc = 0; pc < count; ++pc) { + L(labels[pc]); + Instruction(pc, count, decoded[pc]); + } + + // Dispatch on eax (the next pc). Every backward edge comes through here, so the + // instruction limit is counted here. + L(dispatch); + sub(dword[st + offsetof(State, steps)], 1); + jz(fail_steps, T_NEAR); + cmp(eax, count); + jae(fail_outside, T_NEAR); + lea(rcx, ptr[rip + table]); + jmp(ptr[rcx + rax * 8]); + + L(success); + xor_(eax, eax); + L(leave); + add(rsp, 40); + pop(r15); pop(r14); pop(r13); pop(r12); pop(rbp); pop(rbx); + ret(); + const auto fail_with = [&](Xbyak::Label& label, Word error) { + L(label); + mov(eax, error); + jmp(leave, T_NEAR); + }; + fail_with(fail_underflow, FifoUnderflow); + fail_with(fail_source, InvalidSource); + fail_with(fail_destination, InvalidDestination); + fail_with(fail_extended, UnsupportedExtended); + fail_with(fail_merge, InvalidMerge); + fail_with(fail_loop, InvalidLoop); + fail_with(fail_dread, DataRamReadOutOfBounds); + fail_with(fail_dwrite, DataRamWriteOutOfBounds); + fail_with(fail_output, InvalidOutput); + fail_with(fail_emit, UnsupportedEmit); + fail_with(fail_outside, OutsideCode); + fail_with(fail_steps, StepLimit); + + align(8); + L(table); + for (Word pc = 0; pc < count; ++pc) { + putL(labels[pc]); + } + } + + // Loads MME register `reg` (as an operand of `lane`) into `dst`. + void Get(const Xbyak::Reg32& dst, const Decoded& d, Word reg, Word lane) { + if (reg < 24) { + mov(dst, dword[st + RegOffset(reg)]); + return; + } + switch (reg) { + case 24: xor_(dst, dst); return; + case 25: mov(dst, Word(std::int32_t(std::int16_t(d.imm[lane])))); return; + case 26: mov(dst, Word(std::int32_t(std::int16_t(d.imm[1 - lane])))); return; + case 27: mov(dst, (d.imm[0] << 16) | d.imm[1]); return; + case 28: mov(dst, r12d); return; + case 29: mov(dst, r13d); return; + default: jmp(fail_source, T_NEAR); return; + } + } + + static bool IsFlow(Word op) { + return op >= 23 && op <= 29; + } + + // Where a loop can end (its back edge is checked only there) and which instructions can + // run right after an END (only those check for the end of the macro). + void Analyze(const std::vector<Decoded>& decoded) { + const Word count = static_cast<Word>(decoded.size()); + loop_end.assign(count, false); + after_end.assign(count, false); + std::vector<Word> loop_starts_of_end(count, ~0U); + const auto mark = [&](std::vector<bool>& set, std::int64_t pc) { + if (pc >= 0 && pc < count) set[static_cast<std::size_t>(pc)] = true; + }; + for (Word pc = 0; pc < count; ++pc) { + for (Word lane = 0; lane < 2; ++lane) { + const Decoded& d = decoded[pc]; + if (d.op[lane] == 23 && d.imm[lane] >= 2 && pc + d.imm[lane] - 1 < count) { + loop_end[pc + d.imm[lane] - 1] = true; + } + } + } + for (Word pc = 0; pc < count; ++pc) { + const Decoded& d = decoded[pc]; + if (!d.end) continue; + mark(after_end, std::int64_t{pc} + 1); + for (Word lane = 0; lane < 2; ++lane) { + const Word imm = d.imm[lane]; + switch (d.op[lane]) { + case 23: mark(after_end, std::int64_t{pc} + imm); break; + case 24: mark(after_end, std::int64_t{pc} + (imm & 0x7fff)); break; + case 25: case 26: case 27: case 28: case 29: + mark(after_end, std::int64_t{pc} + (std::int32_t(imm << 19) >> 19)); + break; + default: break; + } + } + } + // An END at a loop's last instruction can be followed by the loop's second one. + for (Word pc = 0; pc < count; ++pc) { + for (Word lane = 0; lane < 2; ++lane) { + const Decoded& d = decoded[pc]; + if (d.op[lane] == 23 && d.imm[lane] >= 2) { + const Word end = pc + d.imm[lane] - 1; + if (end < count && decoded[end].end) mark(after_end, std::int64_t{pc} + 1); + } + } + } + } + + void Instruction(Word pc, Word count, const Decoded& d) { + // FIFO loads, in lane order. + for (Word i = 0; i < 2; ++i) { + const Xbyak::Reg32 load = i == 0 ? r12d : r13d; + if (!d.load[i]) { + xor_(load, load); + continue; + } + mov(eax, dword[st + offsetof(State, consumed)]); + cmp(eax, dword[st + offsetof(State, params_size)]); + jae(fail_underflow, T_NEAR); + mov(rcx, qword[st + offsetof(State, params)]); + mov(load, dword[rcx + rax * 4]); + inc(eax); + mov(dword[st + offsetof(State, consumed)], eax); + } + if (!d.pred_mode) { + Body(pc, count, d, 15); + return; + } + Xbyak::Label when_false; + Get(eax, d, d.pred_reg, 0); + test(eax, eax); + jz(when_false, T_NEAR); + Body(pc, count, d, d.mask_true); + L(when_false); + Body(pc, count, d, d.mask_false); + } + + void Body(Word pc, Word count, const Decoded& d, Word mask) { + // An inactive lane's result reads as 0 when an output selects it. + for (Word lane = 0; lane < 2; ++lane) { + if (mask & (1U << lane)) continue; + bool selected = false; + for (Word out = 0; out < 2; ++out) { + if (!(mask & (4U << out))) continue; + selected |= d.out_method[out] == lane + 1 || d.out_emit[out] == lane + 1; + } + if (selected) { + xor_(lane == 0 ? r14d : r15d, lane == 0 ? r14d : r15d); + } + } + const bool flow = ((mask & 1) && IsFlow(d.op[0])) || ((mask & 2) && IsFlow(d.op[1])); + if (flow) { + mov(dword[st + offsetof(State, next_pc)], pc + 1); + mov(byte[st + offsetof(State, cancel_end)], 0); + } + // Lane 1's operands are read before lane 0 commits its destination (or calls out). + const bool lane0_active = (mask & 1) != 0; + const bool early_operands = + (mask & 2) && lane0_active && + (d.op[0] == 22 || (d.dst[0] < 24 && (d.dst[0] == d.src_a[1] || d.dst[0] == d.src_b[1]))); + if (early_operands) { + Get(eax, d, d.src_a[1], 1); + mov(dword[st + offsetof(State, scratch_x)], eax); + Get(eax, d, d.src_b[1], 1); + mov(dword[st + offsetof(State, scratch_y)], eax); + } + for (Word lane = 0; lane < 2; ++lane) { + if (!(mask & (1U << lane))) { + continue; + } + if (lane == 1 && early_operands) { + mov(ecx, dword[st + offsetof(State, scratch_x)]); + mov(edx, dword[st + offsetof(State, scratch_y)]); + } else { + Get(ecx, d, d.src_a[lane], lane); + Get(edx, d, d.src_b[lane], lane); + } + Lane(pc, d, lane); + // eax holds the lane's value. + mov(lane == 0 ? r14d : r15d, eax); + if (d.dst[lane] < 24) { + mov(dword[st + RegOffset(d.dst[lane])], eax); + } else if (d.dst[lane] != 24) { + jmp(fail_destination, T_NEAR); + } + } + for (Word lane = 0; lane < 2; ++lane) { + if (!(mask & (4U << lane))) { + continue; + } + if (d.out_method[lane]) { + Output(eax, d, d.out_method[lane]); + mov(ecx, eax); + and_(ecx, 0xfff); + mov(dword[st + offsetof(State, method)], ecx); + shr(eax, 12); + and_(eax, 15); + mov(dword[st + offsetof(State, increment)], eax); + } + if (d.out_emit[lane]) { + Output(eax, d, d.out_emit[lane]); + mov(arg2.cvt32(), eax); + mov(arg1.cvt32(), dword[st + offsetof(State, method)]); + mov(arg0, st); + call(ptr[st + offsetof(State, emit)]); + test(al, al); + jz(fail_emit, T_NEAR); + mov(eax, dword[st + offsetof(State, increment)]); + add(dword[st + offsetof(State, method)], eax); + } + } + // An END in the previous instruction ends the macro after this one. + if (after_end[pc]) { + cmp(byte[st + offsetof(State, end_next)], 0); + jne(success, T_NEAR); + } + if (flow) { + cmp(byte[st + offsetof(State, stopped)], 0); + jne(success, T_NEAR); + } + if (d.end) { + if (flow) { + mov(al, byte[st + offsetof(State, cancel_end)]); + xor_(al, 1); + mov(byte[st + offsetof(State, end_next)], al); + } else { + mov(byte[st + offsetof(State, end_next)], 1); + } + } + if (loop_end[pc]) { + Xbyak::Label no_loop; + mov(eax, dword[st + offsetof(State, loop_count)]); + test(eax, eax); + jz(no_loop); + cmp(dword[st + offsetof(State, loop_end)], pc); + jne(no_loop); + dec(dword[st + offsetof(State, loop_count)]); + mov(eax, dword[st + offsetof(State, loop_start)]); + inc(eax); + jmp(dispatch, T_NEAR); + L(no_loop); + } + if (flow) { + mov(eax, dword[st + offsetof(State, next_pc)]); + if (pc + 1 < count) { + cmp(eax, pc + 1); + jne(dispatch, T_NEAR); + jmp(labels[pc + 1], T_NEAR); + } else { + jmp(dispatch, T_NEAR); + } + } else if (pc + 1 < count) { + jmp(labels[pc + 1], T_NEAR); + } else { + jmp(fail_outside, T_NEAR); + } + } + + void Output(const Xbyak::Reg32& dst, const Decoded& d, Word selector) { + switch (selector) { + case 1: mov(dst, r14d); return; + case 2: mov(dst, r15d); return; + case 3: mov(dst, r12d); return; + case 4: mov(dst, r13d); return; + case 5: mov(dst, d.imm[0]); return; + case 6: mov(dst, d.imm[1]); return; + case 8: mov(dst, d.imm[0] >> 12); return; + case 9: mov(dst, d.imm[1] >> 12); return; + case 10: mov(dst, (d.imm[0] << 16) | d.imm[1]); return; + default: jmp(fail_output, T_NEAR); return; + } + } + + // ecx = x, edx = y; leaves the value in eax. + void Compare(Word op) { + cmp(ecx, edx); + switch (op) { + case 17: case 25: setl(al); break; + case 18: case 26: setb(al); break; + case 19: case 27: setle(al); break; + case 20: case 28: setbe(al); break; + default: sete(al); break; + } + movzx(eax, al); + } + + void Lane(Word pc, const Decoded& d, Word lane) { + const Word imm = d.imm[lane]; + const Word op = d.op[lane]; + switch (op) { + case 0: + mov(eax, ecx); + add(eax, edx); + setc(r8b); + movzx(r8d, r8b); + mov(dword[st + offsetof(State, carry)], r8d); + return; + case 1: + lea(eax, ptr[rcx + rdx]); + add(eax, dword[st + offsetof(State, carry)]); + return; + case 2: + mov(eax, ecx); + sub(eax, edx); + setc(r8b); + movzx(r8d, r8b); + mov(dword[st + offsetof(State, carry)], r8d); + return; + case 3: + mov(eax, ecx); + sub(eax, edx); + sub(eax, dword[st + offsetof(State, carry)]); + return; + case 4: + movsxd(rax, ecx); + movsxd(rdx, edx); + imul(rax, rdx); + mov(rcx, rax); + shr(rcx, 32); + mov(dword[st + offsetof(State, carry)], ecx); + return; + case 6: + mov(eax, ecx); + mov(edx, edx); + imul(rax, rdx); + mov(rcx, rax); + shr(rcx, 32); + mov(dword[st + offsetof(State, carry)], ecx); + return; + case 5: + mov(eax, dword[st + offsetof(State, carry)]); + return; + case 7: + cmp(ecx, 0x1000); + jne(fail_extended, T_NEAR); + xor_(eax, eax); + return; + case 8: { + Xbyak::Label zero, done; + test(ecx, ecx); + jz(zero); + bsr(eax, ecx); + xor_(eax, 31); + jmp(done); + L(zero); + mov(eax, 32); + L(done); + return; + } + case 9: + mov(eax, ecx); + mov(ecx, edx); + and_(ecx, 31); + shl(eax, cl); + return; + case 10: + mov(eax, ecx); + mov(ecx, edx); + and_(ecx, 31); + shr(eax, cl); + return; + case 11: + mov(eax, ecx); + mov(ecx, edx); + and_(ecx, 31); + sar(eax, cl); + return; + case 12: mov(eax, ecx); and_(eax, edx); return; + case 13: mov(eax, ecx); and_(eax, edx); not_(eax); return; + case 14: mov(eax, ecx); or_(eax, edx); return; + case 15: mov(eax, ecx); xor_(eax, edx); return; + case 16: { + const Word dest = (imm >> 10) & 63, bit_count = (imm >> 5) & 31; + const Word source = imm & 31; + if (dest >= 32) { + jmp(fail_merge, T_NEAR); + return; + } + const Word bits_mask = (1U << bit_count) - 1; + mov(eax, ecx); + and_(eax, ~(bits_mask << dest)); + shr(edx, source); + and_(edx, bits_mask); + shl(edx, dest); + or_(eax, edx); + return; + } + case 17: case 18: case 19: case 20: case 21: + Compare(op); + neg(eax); + return; + case 22: + lea(eax, ptr[rcx + rdx]); + and_(eax, 0xffff); + mov(arg1.cvt32(), eax); + mov(arg0, st); + call(ptr[st + offsetof(State, read)]); + return; + case 23: { + if (imm < 2) { + jmp(fail_loop, T_NEAR); + return; + } + Xbyak::Label skip, done; + cmp(dword[st + offsetof(State, loop_count)], 0); + jne(fail_loop, T_NEAR); + test(ecx, ecx); + jz(skip); + dec(ecx); + mov(dword[st + offsetof(State, loop_count)], ecx); + mov(dword[st + offsetof(State, loop_start)], pc); + mov(dword[st + offsetof(State, loop_end)], pc + imm - 1); + jmp(done); + L(skip); + mov(dword[st + offsetof(State, next_pc)], pc + imm); + L(done); + xor_(eax, eax); + return; + } + case 24: + mov(dword[st + offsetof(State, next_pc)], pc + (imm & 0x7fff)); + xor_(eax, eax); + return; + case 25: case 26: case 27: case 28: case 29: { + Xbyak::Label not_taken; + Compare(op); + cmp(eax, (imm & 0x8000) ? 1 : 0); + jne(not_taken); + const auto offset = std::int32_t(imm << 19) >> 19; + if (offset == -4096) { + mov(byte[st + offsetof(State, stopped)], 1); + } else { + mov(dword[st + offsetof(State, next_pc)], Word(std::int32_t(pc) + offset)); + mov(byte[st + offsetof(State, cancel_end)], (imm & 0x4000) ? 1 : 0); + } + L(not_taken); + xor_(eax, eax); + return; + } + case 30: + cmp(ecx, 0xc00); + jae(fail_dread, T_NEAR); + mov(rax, qword[st + offsetof(State, dram)]); + mov(eax, dword[rax + rcx * 4]); + return; + case 31: + cmp(ecx, 0xc00); + jae(fail_dwrite, T_NEAR); + mov(rax, qword[st + offsetof(State, dram)]); + mov(dword[rax + rcx * 4], edx); + xor_(eax, eax); + return; + default: + xor_(eax, eax); + return; + } + } + + std::vector<Xbyak::Label> labels; + std::vector<bool> loop_end, after_end; + Xbyak::Label table, dispatch, success, leave; + Xbyak::Label fail_underflow, fail_source, fail_destination, fail_extended, fail_merge, + fail_loop, fail_dread, fail_dwrite, fail_output, fail_emit, fail_outside, fail_steps; + }; + + std::unique_ptr<Program> program; + std::uint64_t compiled_generation{~std::uint64_t{0}}; +}; + +} // namespace Tegra diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 8ffdcb6276..0eae8fac24 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -22,6 +22,10 @@ #include "video_core/texture_cache/util.h" namespace Tegra { +bool MemoryManager::IsShuttingDown() const { + return system.IsShuttingDown(); +} + using Tegra::Memory::GuestMemoryFlags; std::atomic<size_t> MemoryManager::unique_identifier_generator{}; @@ -56,6 +60,10 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, GPU : MemoryManager(system_, system_.Host1x().MemoryManager(), address_space_bits_, split_address_, big_page_bits_, page_bits_) {} +void MemoryManager::SetNativeSyncpointShim(GPUVAddr address) { + native_syncpoint_shim.store(address, std::memory_order_release); +} + MemoryManager::~MemoryManager() = default; MemoryManager::EntryType MemoryManager::GetEntry(size_t position, bool is_big_page) const { @@ -231,6 +239,15 @@ std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t s template <typename T> T MemoryManager::Read(GPUVAddr addr) const { + if constexpr (sizeof(T) == sizeof(u32)) { + const auto shim = native_syncpoint_shim.load(std::memory_order_acquire); + if (shim && addr >= shim && addr - shim < NumShimSyncpoints * sizeof(u32) && + Common::IsAligned(addr - shim, sizeof(u32))) { + return static_cast<T>(system.Host1x().GetSyncpointManager().GetGuestSyncpointValue( + static_cast<u32>((addr - shim) / sizeof(u32)))); + } + } + if (auto page_pointer{GetPointer(addr)}; page_pointer) { // NOTE: Avoid adding any extra logic to this fast-path block T value; @@ -238,7 +255,10 @@ T MemoryManager::Read(GPUVAddr addr) const { return value; } - ASSERT(false); + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 16) { + LOG_ERROR(HW_GPU, "Read{} of unmapped GPU address {:#x}", sizeof(T) * 8, addr); + } return {}; } @@ -390,6 +410,16 @@ void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::siz } void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, const std::size_t size) const { + // Small reads within one 4 KiB page (texture and sampler descriptors, several per draw): one + // translation and a copy, as Read<T> does, instead of the page walk (~3% of the GPU thread + // in title 1). Anything else, unmapped or sparse pages included, takes the full path. + constexpr std::size_t SmallPage = 0x1000; + if (size <= SmallPage && (gpu_src_addr & (SmallPage - 1)) + size <= SmallPage) { + if (const u8* const pointer = GetPointer(gpu_src_addr)) { + std::memcpy(dest_buffer, pointer, size); + return; + } + } ReadBlockImpl(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None, true); } diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 08978567d2..bd5025e9fd 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -43,6 +43,19 @@ public: u64 big_page_bits_ = 16, u64 page_bits_ = 12); ~MemoryManager(); + // Switch 2: GPU address of the read-only syncpoint shim (u32 per syncpoint); 0 = none. + void SetNativeSyncpointShim(GPUVAddr address); + static constexpr u32 NumShimSyncpoints = 192; // Tegra::Host1x::SyncpointManager + + /// Switch 2 lays out block-linear surfaces with the console's GOB swizzle. + void EnableNativeTiling() { + native_tiling = true; + } + bool UsesNativeTiling() const { + return native_tiling; + } + bool IsShuttingDown() const; + static constexpr bool HAS_FLUSH_INVALIDATION = true; inline size_t GetID() const noexcept { @@ -152,6 +165,12 @@ public: void FlushCaching(); + /// Leaves the caches' invalidation of a range already written for the next FlushCaching + /// (before every draw, dispatch, clear and copy), merged with neighbouring ranges. + void AccumulateInvalidation(GPUVAddr gpu_addr, std::size_t size) { + accumulator.Add(gpu_addr, size); + } + const u8* GetSpan(const GPUVAddr src_addr, const std::size_t size) const; u8* GetSpan(const GPUVAddr src_addr, const std::size_t size); @@ -178,6 +197,7 @@ private: GPUVAddr gpu_addr, std::size_t size, boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result) const; + std::atomic<GPUVAddr> native_syncpoint_shim{}; Core::System& system; MaxwellDeviceMemoryManager& memory; @@ -220,6 +240,7 @@ private: boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash{}; boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash2{}; + bool native_tiling{}; mutable std::mutex guard; static constexpr size_t continuous_bits = 64; diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h new file mode 100644 index 0000000000..15c59cb7c6 --- /dev/null +++ b/src/video_core/native_qmd.h @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once +#include <array> +#include <cstdint> +#include <functional> +#include <string_view> +#include <type_traits> +#include <unordered_map> + +namespace Tegra { + +// Native QMD 3.0 dependency scheduling; nonempty grids require a dispatch backend. +class NativeQmdScheduler { +public: + using Word = std::uint32_t; + using Address = std::uint64_t; + using Descriptor = std::array<Word, 64>; + struct RejectShaderDispatch { bool operator()(const Descriptor&) const { return false; } }; + // Optional diagnostics: called with "load"/"count"/"defer"/"run" events. + std::function<void(std::string_view, Address, std::int32_t)> trace; + template <typename Read, typename Release> + std::string_view Signal(Address address, Word argument, Read read, Release release) { + return Signal(address, argument, read, release, RejectShaderDispatch{}); + } + template <typename Read, typename Release, typename Dispatch> + std::string_view Signal(Address address, Word argument, Read read, Release release, Dispatch dispatch) { + struct Pending { Address address; Word action; Word count; }; + std::string_view error; + std::array<Pending, 256> pending{}; + std::size_t count = 1; + pending[0] = {address, argument & 15, ((argument >> 14) & 0x3ff) + 1}; + for (std::size_t i = 0; i < count; ++i) { + const auto request = pending[i]; + if (states.size() >= 4096 && !states.contains(request.address)) return "QMD cache limit"; + auto& state = states[request.address]; + if (request.action == 3 || request.action == 2) { + if (request.action == 3 || !state.loaded) { + if (!read(request.address, state.descriptor)) return "unmapped QMD"; + const auto& qmd = state.descriptor; + if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; + if constexpr (std::is_same_v<Dispatch, RejectShaderDispatch>) { + if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff)) + return "native shader dispatch required"; + } + state.dependencies = std::int32_t((qmd[4] >> 14) & 0xffff) + state.deferred; + if (trace) trace("base", request.address, std::int32_t((qmd[4] >> 14) & 0xffff)); + if (trace) trace("load", request.address, state.dependencies); + state.deferred = 0; + state.loaded = true; + state.completed = false; + } + state.scheduled = true; + } else if (request.action == 7 || request.action == 12) { + const auto delta = request.action == 7 ? -std::int32_t(request.count) : std::int32_t(request.count); + if (!state.loaded || state.completed) { + state.deferred += delta; + if (trace) trace("defer", request.address, state.deferred); + continue; + } + state.dependencies += delta; + if (trace) trace("count", request.address, state.dependencies); + } else if (request.action == 0 || request.action == 1) { + continue; + } else { + return "unsupported PCAS action"; + } + if (!state.loaded || !state.scheduled || state.completed || state.dependencies > 0) continue; + if (state.dependencies < 0) return "QMD dependency underflow"; + // The SDK patches release payloads while a scheduled QMD is + // waiting for its remaining dependencies. Fetch them at execution. + if (!read(request.address, state.descriptor)) return "unmapped QMD"; + const auto qmd = state.descriptor; + if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; + // Validate the release slots before making any visible. + for (const Word base : ReleaseSlots) { + const Word flags = qmd[base + 1]; + if (!(flags & (1U << 23))) continue; + const bool add = (flags & (1U << 26)) != 0; + if ((flags & (0x3bU << 24)) || (flags >> 30) != 1 || + (add && (flags & (7U << 20)))) return "unsupported QMD release format"; + if (add) { + if constexpr (!std::is_invocable_r_v<bool, Release, Address, Word, Word>) + return "unsupported QMD reduction callback"; + } + } + // A program Eden can't run is skipped, but the QMD still completes: its releases + // are what other queues and the CPU wait on (the console never fails a launch). + const bool dispatched = !(qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff)) || dispatch(qmd); + if (!dispatched) error = "native shader dispatch required"; + if (trace) trace("run", request.address, dispatched ? 1 : 0); + state.completed = true; + for (const Word base : ReleaseSlots) { + const Word flags = qmd[base + 1]; + if (!(flags & (1U << 23))) continue; + const Address destination = (Address{flags & 0xff} << 32) | qmd[base]; + bool released{}; + if constexpr (std::is_invocable_r_v<bool, Release, Address, Word, Word>) { + released = release(destination, qmd[base + 2], flags); + } else { + released = release(destination, qmd[base + 2]); + } + if (!released) return "unmapped QMD release"; + } + if (qmd[16] & 1) { + const Word action = (qmd[16] >> 1) & 7; + if (action != 4 && action != 3 && action != 1) return "unsupported dependent QMD action"; + if (count == pending.size()) return "QMD chain limit"; + pending[count++] = {Address{qmd[15]} << 8, action == 4 ? 7U : action == 3 ? 3U : 2U, 1}; + } + if (qmd[16] & (1U << 5)) return "second dependent QMD unsupported"; + } + return error; + } +private: + // Release slots (address low, flags | address high, payload). NVN2 patches a third one at + // word 52 for queue syncs (e.g. a compute queue releasing a semaphore another queue waits on). + static constexpr std::array<Word, 3> ReleaseSlots{24, 28, 52}; + + struct State { + Descriptor descriptor{}; + std::int32_t dependencies{}; + std::int32_t deferred{}; + bool loaded{}; + bool scheduled{}; + bool completed{}; + }; + // One table for every channel: a QMD's dependency counter is state of the QMD itself, and + // title 1 schedules a QMD on one channel while another decrements it (a decrement that + // arrived first was kept by the other channel's scheduler and the QMD never launched). + static inline std::unordered_map<Address, State> states; +}; +} // namespace Tegra diff --git a/src/video_core/native_query_trace.h b/src/video_core/native_query_trace.h new file mode 100644 index 0000000000..036c3de936 --- /dev/null +++ b/src/video_core/native_query_trace.h @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <cstdlib> +#include <fstream> +#include <iomanip> +#include <mutex> +#include "common/common_types.h" + +namespace VideoCommon { +namespace NativeQueryDiagnostics { +struct State { + State() : file{"native-query-trace.csv", std::ios::trunc} { + file << "event,asid,gpu,cpu,type,value,flags\n"; + } + std::mutex mutex; + std::ofstream file; + size_t records{}; +}; +inline State& GetState() { + static State state; + return state; +} +} // namespace NativeQueryDiagnostics + +// Events: 0=request, 1=written, 2=invalidated, 3=unsynchronized, +// 4=channel suspended, 5=channel resumed, 6=buffer readback, +// 7/8=QMD release slots observed, 9=PCAS request, +// 10=DMA destination (value=source, flags=line bytes), 11=inline write. +// Keep this opt-in and bounded. +inline void TraceNativeQuery(u32 event, u64 asid, u64 gpu, u64 cpu, u32 type, u64 value, u32 flags) { + static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + if (!diagnostics) return; + auto& state = NativeQueryDiagnostics::GetState(); + const std::scoped_lock lock{state.mutex}; + if (state.records++ < 1000000 || event == 4 || event == 5) { + state.file << event << ',' << asid << ',' << std::hex << gpu << ',' << cpu << ',' + << type << ',' << value << ',' << flags << std::dec << '\n'; + } + if (event == 4 || event == 5) state.file.flush(); +} +} // namespace VideoCommon diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h new file mode 100644 index 0000000000..5b6d89abdd --- /dev/null +++ b/src/video_core/nx2.h @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <atomic> +#include <mutex> +#include <vector> + +#include "common/common_types.h" + +namespace Tegra { + +// Set once a Switch 2 (Ampere, NVN2) driver is talking to the GPU. Its command streams use +// Ampere classes and the Turing+ macro ISA, which Eden's Maxwell engines don't implement. +inline std::atomic<bool> g_nx2_ampere{false}; + +// NVN2 queues release their progress values into the second page of a two-page GPU-accessible +// region and GPU acquires read the same offsets in the first page (the CPU polls the second). +// Something on the console copies one into the other; until that is known, the puller mirrors +// releases that land in a second page. Every queue has such a region, so keep a small table. +inline constexpr size_t Nx2MaxSemaphoreRegions = 32; +inline std::array<std::atomic<u64>, Nx2MaxSemaphoreRegions> g_nx2_semaphore_regions{}; // starts + +inline void Nx2AddSemaphoreRegion(u64 start) { + for (auto& region : g_nx2_semaphore_regions) { + if (region.load() == start) { + return; + } + } + for (auto& region : g_nx2_semaphore_regions) { + u64 expected = 0; + if (region.compare_exchange_strong(expected, start)) { + return; + } + } +} + +/// The region's memory was destroyed: stop mirroring into it. +inline void Nx2RemoveSemaphoreRegion(u64 start) { + for (auto& region : g_nx2_semaphore_regions) { + u64 expected = start; + region.compare_exchange_strong(expected, 0); + } +} + +/// GPU VA in the first page that mirrors a release at `address`, or 0. +inline u64 Nx2SemaphoreMirror(u64 address) { + for (const auto& region : g_nx2_semaphore_regions) { + const u64 start = region.load(std::memory_order_relaxed); + if (start == 0) { + continue; + } + if (address >= start + 0x1000 && address < start + 0x2000) { + return address - 0x1000; + } + } + return 0; +} + +/// Channels suspended on a semaphore acquire. A console channel polls and sees every release; +/// Eden writes releases in bursts when fences signal and resumes waiters only between command +/// lists, so an acquire-equal could miss its value. Writes latch the waiters they satisfy. +class Nx2SemaphoreWaiters { +public: + // Puller GpuSemaphoreOperation values. + static constexpr u32 AcquireEqual = 0x1; + static constexpr u32 AcquireGequal = 0x4; + static constexpr u32 AcquireMask = 0x8; + + static Nx2SemaphoreWaiters& Get() { + static Nx2SemaphoreWaiters waiters; + return waiters; + } + + void Add(const void* owner, u64 device_address, u32 value, u32 operation) { + std::scoped_lock lk{mutex}; + std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; }); + waiters.push_back({owner, device_address, value, operation, false}); + count.store(waiters.size(), std::memory_order_release); + } + + void Remove(const void* owner) { + if (count.load(std::memory_order_acquire) == 0) { + return; + } + std::scoped_lock lk{mutex}; + std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; }); + count.store(waiters.size(), std::memory_order_release); + } + + [[nodiscard]] bool Latched(const void* owner) { + if (count.load(std::memory_order_acquire) == 0) { + return false; + } + std::scoped_lock lk{mutex}; + return std::ranges::any_of(waiters, + [owner](const Waiter& w) { return w.owner == owner && w.latched; }); + } + + /// A GPU write of `value` to the u32 at `device_address`. + void OnWrite(u64 device_address, u32 value) { + if (count.load(std::memory_order_acquire) == 0) { + return; + } + std::scoped_lock lk{mutex}; + for (auto& w : waiters) { + if (w.address == device_address && !w.latched && Matches(w, value)) { + w.latched = true; + } + } + } + +private: + struct Waiter { + const void* owner; + u64 address; + u32 value; + u32 operation; + bool latched; + }; + + static bool Matches(const Waiter& w, u32 current) { + switch (w.operation) { + case AcquireEqual: + return current == w.value; + case AcquireGequal: + return current >= w.value; + case AcquireMask: + return (current & w.value) != 0; + default: + return false; + } + } + + std::mutex mutex; + std::vector<Waiter> waiters; + std::atomic<size_t> count{}; +}; + +} // namespace Tegra diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h new file mode 100644 index 0000000000..a921b72082 --- /dev/null +++ b/src/video_core/nx2_profile.h @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <chrono> +#include <cstdlib> +#include <string> +#include <unordered_set> + +#include "common/common_types.h" +#include "common/logging.h" +#include "common/cpu_features.h" + +// EDEN_NX2_GPU_PROFILE: time where the GPU thread spends its time. Each zone counts its own +// (exclusive) time, nested zones excluded; every 10 s the thread logs each zone's share. +namespace VideoCommon::Nx2Profile { + +enum class Zone : u32 { + Other, ///< GPU thread time outside every zone below (register writes, command parsing) + Idle, + PendingSleep, + ResumePending, + Push, + Tick, + Run, + FlushInvalidate, + Macro, + Draw, + Clear, + Compute, + FlushWork, + PipelineLookup, + Configure, + SyncDescriptors, + ReadHandles, + FillImageViews, + TextureBuffers, + UpdateBuffers, + DescriptorAcquire, + BindStageBuffers, + PushDescriptors, + RenderTargets, + ConfigureDraw, + DynamicState, + DrawRecord, + TextureUpload, + Fence, ///< Fences made (SignalFence) + FenceSkipped, ///< Empty fences not made + Method, ///< Engine method calls (what a split GPU thread's back half would run) + SplitDrain, ///< Split GPU thread: the front waits for the back (gpu_split.h) + SplitWait, ///< Split GPU thread: the back waits for work + Count, +}; + +inline constexpr std::array<const char*, static_cast<size_t>(Zone::Count)> zone_names{ + "other", "idle", "pending sleep", "resume pending", "push", + "tick", "run", "flush/inval", "macro", "draw", + "clear", "compute", "flush work", "pipeline lookup", "configure", + "sync descriptors", "read handles", "fill image views", "texture buffers", "update buffers", + "descriptor acquire", "bind stage buffers", "push descriptors", "render targets", + "configure draw", "dynamic state", "draw record", "texture upload", "fence", + "fence skipped", "method", "split drain", "split wait", +}; + +// Read once at startup: Enabled() runs several times per method write (no guard to check). +inline const bool g_enabled = std::getenv("EDEN_NX2_GPU_PROFILE") != nullptr; + +inline bool Enabled() { + return g_enabled; +} + +// Nanoseconds; from the host's own counter where the wall clock reads it directly (the zones +// run ~150,000 times a second, and QueryPerformanceCounter cost ~5% of the GPU thread). +inline u64 Now() { + return static_cast<u64>(Common::g_wall_clock.GetTimeNS().count()); +} + +struct ThreadData { + std::array<u64, static_cast<size_t>(Zone::Count)> exclusive_ns{}; + std::array<u64, static_cast<size_t>(Zone::Count)> count{}; + std::array<u64, 8> counters{}; ///< See Counter + u64 child_ns{}; ///< Time of the zones nested in the current one + u64 window_start{}; +}; + +inline thread_local ThreadData t_data; + +class Scope { +public: + explicit Scope(Zone zone_) : zone{zone_}, enabled{Enabled()} { + if (!enabled) { + return; + } + parent_child_ns = t_data.child_ns; + t_data.child_ns = 0; + start = Now(); + } + + ~Scope() { + if (!enabled) { + return; + } + const u64 elapsed = Now() - start; + auto& data = t_data; + const auto index = static_cast<size_t>(zone); + data.exclusive_ns[index] += elapsed - data.child_ns; + ++data.count[index]; + data.child_ns = parent_child_ns + elapsed; + } + + Scope(const Scope&) = delete; + Scope& operator=(const Scope&) = delete; + +private: + Zone zone; + bool enabled; + u64 start{}; + u64 parent_child_ns{}; +}; + +/// Amounts counted on the GPU thread, logged per second with the zones. +enum class Counter : u32 { + DownloadBytes, ///< Buffer data copied back from the host GPU at fences + DownloadCopies, ///< Buffer ranges copied back + DmaReadBytes, ///< Guest memory the native MME read with DMA + DownloadBytesOnDmaPages, ///< Of DownloadBytes: on pages the MME has DMA-read + MethodWrites, ///< 3D register writes (Maxwell3D::CallMethod below the macro range) + SameValueWrites, ///< Of MethodWrites: the register already held the value + AcquireChecks, ///< Semaphore acquires evaluated + AcquireWaits, ///< Of AcquireChecks: not met yet (the channel waits) +}; + +/// Device pages (4 KiB) the native MME has DMA-read since start (GPU thread only; filled only +/// while profiling): which buffer downloads anything on the GPU thread reads back. +inline std::unordered_set<u64>& DmaReadPages() { + static std::unordered_set<u64> pages; + return pages; +} + +inline void Add(Counter counter, u64 amount) { + if (Enabled()) { + t_data.counters[static_cast<size_t>(counter)] += amount; + } +} + +/// Called by the GPU thread (and the split's back thread) between commands: logs and resets +/// the totals every 10 s. +inline void Report(const char* thread_name = "GPU thread") { + if (!Enabled()) { + return; + } + auto& data = t_data; + const u64 now = Now(); + if (data.window_start == 0) { + data.window_start = now; + data.child_ns = 0; + return; + } + const u64 window = now - data.window_start; + if (window < 10'000'000'000ULL) { + return; + } + // Time in no zone at all is the wall time minus the zones' totals. + u64 zoned{}; + for (size_t i = 1; i < data.exclusive_ns.size(); ++i) { + zoned += data.exclusive_ns[i]; + } + data.exclusive_ns[0] = window > zoned ? window - zoned : 0; + std::string line; + for (size_t i = 0; i < data.exclusive_ns.size(); ++i) { + if (data.exclusive_ns[i] == 0 && data.count[i] == 0) { + continue; + } + // Per second of wall time: milliseconds spent and calls made. + const double ms = static_cast<double>(data.exclusive_ns[i]) * 1000.0 / window; + const double calls = static_cast<double>(data.count[i]) * 1e9 / window; + line += fmt::format(" | {} {:.1f} ms ({:.0f}/s)", zone_names[i], ms, calls); + } + line += fmt::format(" | buffer downloads {:.0f} KiB in {:.0f} copies ({:.0f} KiB on pages " + "MME DMA reads) | MME DMA reads {:.0f} KiB", + static_cast<double>(data.counters[0]) * 1e9 / 1024.0 / window, + static_cast<double>(data.counters[1]) * 1e9 / window, + static_cast<double>(data.counters[3]) * 1e9 / 1024.0 / window, + static_cast<double>(data.counters[2]) * 1e9 / 1024.0 / window); + line += fmt::format(" | 3D register writes {:.0f} ({:.0f} same value)", + static_cast<double>(data.counters[4]) * 1e9 / window, + static_cast<double>(data.counters[5]) * 1e9 / window); + line += fmt::format(" | acquires {:.0f} ({:.0f} waited)", + static_cast<double>(data.counters[6]) * 1e9 / window, + static_cast<double>(data.counters[7]) * 1e9 / window); + LOG_INFO(HW_GPU, "{} per second:{}", thread_name, line); + data = {}; + data.window_start = now; +} + +} // namespace VideoCommon::Nx2Profile diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index b8c1bc4f45..60ca50eb1f 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -7,6 +7,7 @@ #pragma once #include <array> +#include <cstdlib> #include <deque> #include <memory> #include <mutex> @@ -22,6 +23,7 @@ #include "video_core/gpu.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/query_cache/bank_base.h" #include "video_core/query_cache/query_base.h" #include "video_core/query_cache/query_cache_base.h" @@ -233,6 +235,16 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type QueryPropertiesFlags flags, u32 payload, u32 subreport) { const bool has_timestamp = True(flags & QueryPropertiesFlags::HasTimeout); const bool is_fence = True(flags & QueryPropertiesFlags::IsAFence); + static const GPUVAddr watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{}; + }(); + if (static const bool trace = std::getenv("EDEN_NX2_QUERY_TRACE") != nullptr; + trace || (watch != 0 && (addr & ~GPUVAddr{0xf}) == watch)) { + LOG_INFO(HW_GPU, "nx2 report addr={:#x} type={} flags={:#x} payload={:#x} as={} device={:#x}", addr, + static_cast<u32>(counter_type), static_cast<u32>(flags), payload, gpu_memory->GetID(), + gpu_memory->GpuToCpuAddress(addr).value_or(0)); + } size_t streamer_id = static_cast<size_t>(counter_type); auto* streamer = impl->streamers[streamer_id]; if (streamer == nullptr) [[unlikely]] { @@ -262,7 +274,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type u8* pointer_timestamp = impl->device_memory.template GetPointer<u8>(cpu_addr + 8); bool is_synced = (Settings::IsGPUFenceBehaviorDefault() ? !Settings::IsGPULevelHigh() : !Settings::IsGPUFenceBehaviorBalanced() && !Settings::IsGPUFenceBehaviorAccurate()) && is_fence; std::function<void()> operation([this, is_synced, streamer, query_base = query, query_location, - pointer, pointer_timestamp] { + pointer, pointer_timestamp, cpu_addr] { if (True(query_base->flags & QueryFlagBits::IsInvalidated)) { if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); @@ -287,6 +299,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type u32 value = static_cast<u32>(query_base->value); std::memcpy(pointer, &value, sizeof(value)); } + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast<u32>(query_base->value)); if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); } @@ -303,6 +316,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type } else { std::memcpy(pointer, &payload, sizeof(payload)); } + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload); streamer->Free(new_query_id); return; } @@ -505,6 +519,16 @@ bool QueryCacheBase<Traits>::HasUncommittedFlushes() const { return result; } +template <typename Traits> +bool QueryCacheBase<Traits>::HasPendingFenceWork() const { + bool result = false; + impl->ForEachStreamer([&result](StreamerInterface* streamer) { + result |= streamer->HasUnsyncedQueries() || streamer->HasPendingSync(); + return result; + }); + return result; +} + template <typename Traits> bool QueryCacheBase<Traits>::ShouldWaitAsyncFlushes() { std::scoped_lock lk(impl->flush_guard); diff --git a/src/video_core/query_cache/query_cache_base.h b/src/video_core/query_cache/query_cache_base.h index d73e9039f7..5a60e42986 100644 --- a/src/video_core/query_cache/query_cache_base.h +++ b/src/video_core/query_cache/query_cache_base.h @@ -109,6 +109,9 @@ public: bool HasUncommittedFlushes() const; + /// Whether a new fence has queries to flush or host writes to sync for this cache. + bool HasPendingFenceWork() const; + bool ShouldWaitAsyncFlushes(); void PopAsyncFlushes(); diff --git a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp index 83545463ac..ecd6544b10 100644 --- a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp +++ b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp @@ -20,7 +20,9 @@ #include "video_core/shader_notify.h" #include "video_core/texture_cache/texture_cache.h" -#if defined(_MSC_VER) && defined(NDEBUG) +#if defined(__clang__) && defined(NDEBUG) +#define LAMBDA_FORCEINLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) && defined(NDEBUG) #define LAMBDA_FORCEINLINE [[msvc::forceinline]] #else #define LAMBDA_FORCEINLINE diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 1d77d28c46..4e0972d1d2 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -716,7 +716,7 @@ void RasterizerOpenGL::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si } gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size); { - std::unique_lock<std::recursive_mutex> lock{buffer_cache.mutex}; + std::unique_lock lock{buffer_cache.mutex}; if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) { buffer_cache.WriteMemory(*cpu_addr, copy_size); } diff --git a/src/video_core/renderer_opengl/util_shaders.cpp b/src/video_core/renderer_opengl/util_shaders.cpp index 7ced8c77b4..5ed96a2037 100644 --- a/src/video_core/renderer_opengl/util_shaders.cpp +++ b/src/video_core/renderer_opengl/util_shaders.cpp @@ -96,6 +96,7 @@ void UtilShaders::ASTCDecode(Image& image, const StagingBufferMap& map, glUniform1ui(4, params.x_shift); glUniform1ui(5, params.block_height); glUniform1ui(6, params.block_height_mask); + glUniform1ui(7, params.native_tiling); // ASTC texture data glBindBufferRange(GL_SHADER_STORAGE_BUFFER, BINDING_INPUT_BUFFER, map.buffer, input_offset, diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp index 0ced890929..9dac98c830 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp @@ -179,7 +179,14 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe for (size_t i = 0; i < regs.rt.size(); ++i) { color_formats[i] = static_cast<u8>(regs.rt[i].format); } - alpha_test_ref = std::bit_cast<u32>(regs.alpha_test_ref); + // The reference is part of the pipeline only while the test compares with it: title 1 + // animates the register with the test off (text fading in and out), which made a new + // pipeline, translated on the GPU thread, for every value (~84% of its runtime pipelines). + const u32 packed_alpha_test = alpha_test_func.Value(); + alpha_test_ref = packed_alpha_test != PackComparisonOp(Maxwell::ComparisonOp::Never_GL) && + packed_alpha_test != PackComparisonOp(Maxwell::ComparisonOp::Always_GL) + ? std::bit_cast<u32>(regs.alpha_test_ref) + : 0; point_size = std::bit_cast<u32>(regs.point_size); if (maxwell3d.dirty.flags[Dirty::VertexInput]) { diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp index 9f4aaf390f..8a11c356ff 100644 --- a/src/video_core/renderer_vulkan/present/layer.cpp +++ b/src/video_core/renderer_vulkan/present/layer.cpp @@ -349,7 +349,7 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i if (host_ptr) { Tegra::Texture::UnswizzleTexture( mapped_span.subspan(image_offset, linear_size), std::span(host_ptr, tiled_size), - bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0); + bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0, 1, framebuffer.native_tiling); buffer.Flush(); // Ensure host writes are visible before the GPU copy. } diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index ab58560170..b3ebf7b3f6 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -5,6 +5,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <array> +#include <chrono> +#include <cstdlib> #include <memory> #include <numeric> #include <optional> @@ -14,9 +16,11 @@ #include "common/assert.h" #include "common/common_types.h" +#include "common/logging.h" #include "common/div_ceil.h" #include "common/vector_math.h" #include "video_core/host_shaders/astc_decoder_comp_spv.h" +#include "video_core/host_shaders/bc_encoder_comp_spv.h" #include "video_core/host_shaders/queries_prefix_scan_sum_comp_spv.h" #include "video_core/host_shaders/queries_prefix_scan_sum_nosubgroups_comp_spv.h" #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" @@ -31,6 +35,7 @@ #include "video_core/renderer_vulkan/vk_update_descriptor.h" #include "video_core/texture_cache/accelerated_swizzle.h" #include "video_core/texture_cache/types.h" +#include "video_core/texture_cache/util.h" #include "video_core/textures/decoders.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -183,6 +188,52 @@ struct AstcPushConstants { u32 x_shift; u32 block_height; u32 block_height_mask; + u32 native_tiling; +}; + +constexpr std::array<VkDescriptorSetLayoutBinding, 2> BC_ENCODER_DESCRIPTOR_SET_BINDINGS{{ + { + .binding = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + { + .binding = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, +}}; + +constexpr std::array<VkDescriptorUpdateTemplateEntry, 2> BC_ENCODER_DESCRIPTOR_UPDATE_TEMPLATE{{ + { + .dstBinding = 0, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .offset = 0, + .stride = sizeof(DescriptorUpdateEntry), + }, + { + .dstBinding = 1, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, +}}; + +struct BcEncoderPushConstants { + u32 blocks_x; + u32 blocks_y; + u32 width; + u32 height; + u32 bc3; + u32 out_offset; // in 8-byte units }; struct QueriesPrefixScanPushConstants { @@ -521,16 +572,31 @@ ASTCDecoderPass::~ASTCDecoderPass() = default; void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, std::span<const VideoCommon::SwizzleParameters> swizzles) { + const Target target{ + .image = image.Handle(), + .aspect_mask = image.AspectMask(), + .initialized = image.ExchangeInitialization(), + .level_view = [&image](s32 host_level) { return image.StorageImageView(host_level); }, + .wait = true, + }; + AssembleInto(image.info, image.host_level_offset, image.guest_size_bytes, target, map, + swizzles); +} + +void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, + size_t guest_size_bytes, const Target& target, + const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles) { using namespace VideoCommon::Accelerated; const std::array<u32, 2> block_dims{ - VideoCore::Surface::DefaultBlockWidth(image.info.format), - VideoCore::Surface::DefaultBlockHeight(image.info.format), + VideoCore::Surface::DefaultBlockWidth(info.format), + VideoCore::Surface::DefaultBlockHeight(info.format), }; scheduler.RequestOutsideRenderPassOperationContext(); const VkPipeline vk_pipeline = *pipeline; - const VkImageAspectFlags aspect_mask = image.AspectMask(); - const VkImage vk_image = image.Handle(); - const bool is_initialized = image.ExchangeInitialization(); + const VkImageAspectFlags aspect_mask = target.aspect_mask; + const VkImage vk_image = target.image; + const bool is_initialized = target.initialized; scheduler.Record([vk_pipeline, vk_image, aspect_mask, is_initialized](vk::CommandBuffer cmdbuf) { const VkImageMemoryBarrier image_barrier{ @@ -557,20 +623,30 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, image_barrier); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); }); + // With trimmed host levels the input holds each layer's mip chain from host_level_offset on + // (see TextureCache::UploadImageContents), so its layer stride is that tail's size. + const u32 input_layer_stride = + skip != 0 ? info.layer_stride - + static_cast<u32>(VideoCommon::FullUploadSwizzles(info)[skip].buffer_offset) + : info.layer_stride; + const size_t input_size = skip != 0 ? size_t{input_layer_stride} * info.resources.layers + : guest_size_bytes; for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { const size_t input_offset = swizzle.buffer_offset + map.offset; const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 8U); const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U); - const u32 num_dispatches_z = image.info.resources.layers; + const u32 num_dispatches_z = info.resources.layers; compute_pass_descriptor_queue.Acquire(scheduler, 2); compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, - image.guest_size_bytes - swizzle.buffer_offset); - compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + input_size - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage( + target.level_view(swizzle.level - static_cast<s32>(skip))); const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; // To unswizzle the ASTC data - const auto params = MakeBlockLinearSwizzle2DParams(swizzle, image.info); + auto params = MakeBlockLinearSwizzle2DParams(swizzle, info); + params.layer_stride = input_layer_stride; ASSERT(params.origin == (std::array<u32, 3>{0, 0, 0})); ASSERT(params.destination == (std::array<s32, 3>{0, 0, 0})); ASSERT(params.bytes_per_block_log2 == 4); @@ -583,6 +659,7 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, .x_shift = params.x_shift, .block_height = params.block_height, .block_height_mask = params.block_height_mask, + .native_tiling = params.native_tiling, }; const VkDescriptorSet set = descriptor_allocator.Commit(); device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); @@ -613,7 +690,74 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); }); - scheduler.Finish(); + // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here + // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead. + static const bool flush_only = [] { + const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH"); + return value != nullptr && value[0] == '1'; + }(); + if (!target.wait) { + return; + } + const auto start = std::chrono::steady_clock::now(); + if (flush_only) { + scheduler.Flush(); + } else { + scheduler.Finish(); + } + const auto end = std::chrono::steady_clock::now(); + static u32 decodes{}; + static std::chrono::nanoseconds waited{}; + static auto window_start = end; + ++decodes; + waited += end - start; + if (end - window_start >= std::chrono::seconds{5}) { + LOG_INFO(Render_Vulkan, "ASTC GPU decodes: {} in {} ms, {} ms waiting ({})", decodes, + std::chrono::duration_cast<std::chrono::milliseconds>(end - window_start).count(), + std::chrono::duration_cast<std::chrono::milliseconds>(waited).count(), + flush_only ? "flush" : "finish"); + decodes = 0; + waited = {}; + window_start = end; + } +} + +BcEncoderPass::BcEncoderPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, BC_ENCODER_DESCRIPTOR_SET_BINDINGS, + BC_ENCODER_DESCRIPTOR_UPDATE_TEMPLATE, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE<sizeof(BcEncoderPushConstants)>, BC_ENCODER_COMP_SPV), + scheduler{scheduler_}, compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BcEncoderPass::~BcEncoderPass() = default; + +void BcEncoderPass::Encode(VkImageView source, u32 width, u32 height, u32 layers, bool bc3, + VkBuffer buffer, VkDeviceSize buffer_offset, VkDeviceSize buffer_size, + VkDeviceSize out_offset) { + scheduler.RequestOutsideRenderPassOperationContext(); + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddImage(source); + compute_pass_descriptor_queue.AddBuffer(buffer, buffer_offset, buffer_size); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + const BcEncoderPushConstants uniforms{ + .blocks_x = Common::DivCeil(width, 4U), + .blocks_y = Common::DivCeil(height, 4U), + .width = width, + .height = height, + .bc3 = bc3 ? 1U : 0U, + .out_offset = static_cast<u32>(out_offset / 8), + }; + const VkPipeline vk_pipeline = *pipeline; + scheduler.Record([this, vk_pipeline, uniforms, layers, descriptor_data](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms); + cmdbuf.Dispatch(Common::DivCeil(uniforms.blocks_x, 8U), Common::DivCeil(uniforms.blocks_y, 8U), + layers); + }); } constexpr u32 BL3D_BINDING_INPUT_BUFFER = 0; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 3d30aa6bdc..5daf4eb3de 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -6,6 +6,7 @@ #pragma once +#include <functional> #include <optional> #include <span> #include <utility> @@ -130,6 +131,21 @@ public: void Assemble(Image& image, const StagingBufferRef& map, std::span<const VideoCommon::SwizzleParameters> swizzles); + /// Where a decode writes: an RGBA8 image with a storage view per (host) level. + struct Target { + VkImage image; + VkImageAspectFlags aspect_mask; + bool initialized; + std::function<VkImageView(s32 host_level)> level_view; + bool wait; ///< Wait for the GPU afterwards (the decoded image itself is sampled). + }; + + /// Decodes `info`'s ASTC data in `map` into `target`. `skip` leading levels are absent on the + /// host (TrimmedLevels) and `guest_size_bytes` is the full guest size. + void AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, size_t guest_size_bytes, + const Target& target, const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles); + private: Scheduler& scheduler; StagingBufferPool& staging_buffer_pool; @@ -137,6 +153,24 @@ private: MemoryAllocator& memory_allocator; }; +/// Compresses an RGBA8 image level to BC1 or BC3 blocks in a buffer (bc_encoder.comp). +class BcEncoderPass final : public ComputePass { +public: + explicit BcEncoderPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BcEncoderPass(); + + /// Encodes every layer of one level (`source` is its RGBA8 storage view) into `buffer`, + /// `out_offset` bytes past `buffer_offset`; blocks are row-major, layer after layer. + void Encode(VkImageView source, u32 width, u32 height, u32 layers, bool bc3, VkBuffer buffer, + VkDeviceSize buffer_offset, VkDeviceSize buffer_size, VkDeviceSize out_offset); + +private: + Scheduler& scheduler; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + class BlockLinearUnswizzle3DPass final : public ComputePass { public: explicit BlockLinearUnswizzle3DPass(const Device& device_, Scheduler& scheduler_, diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 8b5b0bc9c5..94bace432c 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -29,8 +29,11 @@ #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/gpu_logging/gpu_logging.h" #include "common/settings.h" +#include "video_core/nx2_profile.h" -#if defined(_MSC_VER) && defined(NDEBUG) +#if defined(__clang__) && defined(NDEBUG) +#define LAMBDA_FORCEINLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) && defined(NDEBUG) #define LAMBDA_FORCEINLINE [[msvc::forceinline]] #else #define LAMBDA_FORCEINLINE @@ -355,7 +358,11 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { views.reserve(num_image_elements); samplers.reserve(num_textures); - texture_cache.SynchronizeDescriptors(false); + namespace Profile = VideoCommon::Nx2Profile; + { + Profile::Scope scope{Profile::Zone::SyncDescriptors}; + texture_cache.SynchronizeDescriptors(false); + } buffer_cache.SetUniformBuffersState(enabled_uniform_buffer_masks, &uniform_buffer_sizes); @@ -432,24 +439,31 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { return true; }}; - if constexpr (Spec::enabled_stages[0]) { - config_stage(0); - } - if constexpr (Spec::enabled_stages[1]) { - config_stage(1); - } - if constexpr (Spec::enabled_stages[2]) { - config_stage(2); - } - if constexpr (Spec::enabled_stages[3]) { - config_stage(3); - } - if constexpr (Spec::enabled_stages[4]) { - config_stage(4); + { + Profile::Scope scope{Profile::Zone::ReadHandles}; + if constexpr (Spec::enabled_stages[0]) { + config_stage(0); + } + if constexpr (Spec::enabled_stages[1]) { + config_stage(1); + } + if constexpr (Spec::enabled_stages[2]) { + config_stage(2); + } + if constexpr (Spec::enabled_stages[3]) { + config_stage(3); + } + if constexpr (Spec::enabled_stages[4]) { + config_stage(4); + } } ASSERT(views.size() == num_image_elements); ASSERT(samplers.size() == num_textures); - texture_cache.FillImageViews(std::span(views.data(), views.size()), false, Spec::has_images); + { + Profile::Scope scope{Profile::Zone::FillImageViews}; + texture_cache.FillImageViews(std::span(views.data(), views.size()), false, + Spec::has_images); + } VideoCommon::ImageViewInOut* texture_buffer_it{views.data()}; const auto bind_stage_info{[&](size_t stage) LAMBDA_FORCEINLINE { @@ -493,44 +507,62 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { texture_buffer_it += Shader::NumDescriptors(info.image_descriptors); } }}; - if constexpr (Spec::enabled_stages[0]) { - bind_stage_info(0); - } - if constexpr (Spec::enabled_stages[1]) { - bind_stage_info(1); - } - if constexpr (Spec::enabled_stages[2]) { - bind_stage_info(2); - } - if constexpr (Spec::enabled_stages[3]) { - bind_stage_info(3); - } - if constexpr (Spec::enabled_stages[4]) { - bind_stage_info(4); + { + Profile::Scope scope{Profile::Zone::TextureBuffers}; + if constexpr (Spec::enabled_stages[0]) { + bind_stage_info(0); + } + if constexpr (Spec::enabled_stages[1]) { + bind_stage_info(1); + } + if constexpr (Spec::enabled_stages[2]) { + bind_stage_info(2); + } + if constexpr (Spec::enabled_stages[3]) { + bind_stage_info(3); + } + if constexpr (Spec::enabled_stages[4]) { + bind_stage_info(4); + } } if (regs.transform_feedback_enabled != 0) { scheduler.RequestOutsideRenderPassOperationContext(); } - buffer_cache.UpdateGraphicsBuffers(is_indexed); - buffer_cache.BindHostGeometryBuffers(is_indexed); + { + Profile::Scope scope{Profile::Zone::UpdateBuffers}; + buffer_cache.UpdateGraphicsBuffers(is_indexed); + buffer_cache.BindHostGeometryBuffers(is_indexed); + } - guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); + { + Profile::Scope scope{Profile::Zone::DescriptorAcquire}; + guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); + } RescalingPushConstant rescaling; RenderAreaPushConstant render_area; const VideoCommon::SamplerId* samplers_it{samplers.data()}; const VideoCommon::ImageViewInOut* views_it{views.data()}; const auto prepare_stage{[&](size_t stage) LAMBDA_FORCEINLINE { - buffer_cache.BindHostStageBuffers(stage); - PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage], rescaling, - samplers_it, views_it); + { + Profile::Scope scope{Profile::Zone::BindStageBuffers}; + buffer_cache.BindHostStageBuffers(stage); + } + { + Profile::Scope scope{Profile::Zone::PushDescriptors}; + PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage], + rescaling, samplers_it, views_it); + } const auto& info{stage_infos[stage]}; if (info.uses_render_area) { render_area.uses_render_area = true; - render_area.words = {static_cast<float>(regs.surface_clip.width), - static_cast<float>(regs.surface_clip.height)}; + // Switch 2 titles may never program the surface clip; use colour target 0 then. + const u32 clip_width = regs.surface_clip.width, clip_height = regs.surface_clip.height; + const bool clip_set = clip_width != 0 && clip_height != 0; + render_area.words = {static_cast<float>(clip_set ? clip_width : u32{regs.rt[0].width}), + static_cast<float>(clip_set ? clip_height : u32{regs.rt[0].height})}; } }}; if constexpr (Spec::enabled_stages[0]) { @@ -552,12 +584,16 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { buffer_cache.runtime.PostCopyBarrier(); buffer_cache.any_buffer_uploaded = false; } - texture_cache.UpdateRenderTargets(false); - texture_cache.CheckFeedbackLoop(std::span<const VideoCommon::ImageViewInOut>{views.data(), - views.size()}); + { + Profile::Scope scope{Profile::Zone::RenderTargets}; + texture_cache.UpdateRenderTargets(false); + texture_cache.CheckFeedbackLoop( + std::span<const VideoCommon::ImageViewInOut>{views.data(), views.size()}); + } if (IsBuilt() && !pipeline) { return false; } + Profile::Scope scope{Profile::Zone::ConfigureDraw}; return ConfigureDraw(rescaling, render_area); } diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index d4ffcd8317..17f57983e3 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <chrono> #include <cstddef> #include <fstream> #include <iostream> @@ -15,6 +16,7 @@ #include <bit> #include <numeric> #include "common/cityhash.h" +#include "common/scope_exit.h" #include "common/fs/fs.h" #include "common/fs/path_util.h" #include "common/thread_worker.h" @@ -766,10 +768,66 @@ void PipelineCache::QueueVulkanPipelineCacheFlush() { }); } +namespace { +// EDEN_NX2_PIPELINE_DIFF=1: for a new graphics pipeline whose shaders an earlier one uses too, +// log which parts of the fixed state differ from that one (finds state that splits the same +// draws into many pipelines, each translated on the GPU thread). +void ReportNewPipelineKey(const GraphicsPipelineCacheKey& key) { + static const bool enabled = std::getenv("EDEN_NX2_PIPELINE_DIFF") != nullptr; + if (!enabled) { + return; + } + static std::unordered_map<u64, GraphicsPipelineCacheKey> last_by_shaders; + static u32 logged{}; + const u64 shaders = Common::CityHash64(reinterpret_cast<const char*>(key.unique_hashes.data()), + sizeof(key.unique_hashes)); + const auto [it, first] = last_by_shaders.try_emplace(shaders, key); + if (first) { + return; + } + const FixedPipelineState& now = key.state; + const FixedPipelineState& before = it->second.state; + std::string diffs; + if (now.raw1 != before.raw1) diffs += fmt::format(" raw1^{:#x}", now.raw1 ^ before.raw1); + if (now.raw2 != before.raw2) diffs += fmt::format(" raw2^{:#x}", now.raw2 ^ before.raw2); + if (now.color_formats != before.color_formats) { + diffs += fmt::format(" color_formats {:02x} -> {:02x}", fmt::join(before.color_formats, ""), + fmt::join(now.color_formats, "")); + } + if (now.alpha_test_ref != before.alpha_test_ref) { + diffs += fmt::format(" alpha_test_ref {:#x} -> {:#x} (func {})", before.alpha_test_ref, + now.alpha_test_ref, now.alpha_test_func.Value()); + } + if (now.point_size != before.point_size) { + diffs += fmt::format(" point_size {:#x} -> {:#x}", before.point_size, now.point_size); + } + if (now.viewport_swizzles != before.viewport_swizzles) diffs += " viewport_swizzles"; + if (now.attribute_types != before.attribute_types) { + diffs += fmt::format(" attribute_types^{:#x}", now.attribute_types ^ before.attribute_types); + } + if (std::memcmp(&now.dynamic_state, &before.dynamic_state, sizeof(now.dynamic_state)) != 0) { + diffs += " dynamic_state"; + } + if (now.driver_id != before.driver_id || now.driver_version != before.driver_version) { + diffs += " driver"; + } + const size_t size = now.Size(); + if (diffs.empty() && size > offsetof(FixedPipelineState, attachments)) { + diffs = fmt::format(" past attachments (key size {})", size); + } + it->second = key; + if (logged++ < 400) { + LOG_INFO(Render_Vulkan, "New pipeline for known shaders {:016x} differs in:{}", shaders, + diffs.empty() ? std::string{" nothing in the key prefix"} : diffs); + } +} +} // Anonymous namespace + GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() { const auto [pair, is_new]{graphics_cache.try_emplace(graphics_key)}; auto& pipeline{pair->second}; if (is_new) { + ReportNewPipelineKey(graphics_key); pipeline = CreateGraphicsPipeline(); } if (!pipeline) { @@ -930,7 +988,38 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline( return nullptr; } +namespace { +// Pipelines built on the GPU thread at run time (shader translation; the driver compile is on +// workers): every one over 2 ms is logged, and the total every 10 s, to tell how much of the +// stutter on new content is translation. +void ReportRuntimePipeline(const char* kind, std::chrono::steady_clock::time_point begin) { + static std::chrono::steady_clock::duration total{}; + static u32 count{}, logged{}; + static auto window = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + const auto elapsed = now - begin; + total += elapsed; + ++count; + const auto ms = [](auto d) { return std::chrono::duration<double, std::milli>(d).count(); }; + if (elapsed > std::chrono::milliseconds{2} && logged < 400) { + ++logged; + LOG_INFO(Render_Vulkan, "{} pipeline built on the GPU thread in {:.1f} ms", kind, ms(elapsed)); + } + if (now - window > std::chrono::seconds{10}) { + LOG_INFO(Render_Vulkan, "Runtime pipelines in the last 10 s: {} built, {:.1f} ms on the GPU thread", + count, ms(total)); + total = {}; + count = 0; + window = now; + } +} +} // Anonymous namespace + std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() { + const auto begin = std::chrono::steady_clock::now(); + SCOPE_EXIT { + ReportRuntimePipeline("Graphics", begin); + }; GraphicsEnvironments environments; GetGraphicsEnvironments(environments, graphics_key.unique_hashes); @@ -956,6 +1045,10 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() { std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline( const ComputePipelineCacheKey& key, const ShaderInfo* shader) { + const auto begin = std::chrono::steady_clock::now(); + SCOPE_EXIT { + ReportRuntimePipeline("Compute", begin); + }; const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()}; const auto& qmd{kepler_compute->launch_description}; ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; diff --git a/src/video_core/renderer_vulkan/vk_present_manager.cpp b/src/video_core/renderer_vulkan/vk_present_manager.cpp index 9298764546..c1724e1ab4 100644 --- a/src/video_core/renderer_vulkan/vk_present_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_present_manager.cpp @@ -434,7 +434,7 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) { RecreateSwapchain(frame); } - while (swapchain.AcquireNextImage()) { + while (swapchain.AcquireNextImage(!use_present_thread)) { RecreateSwapchain(frame); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 264ffb28b5..0826820714 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -7,6 +7,11 @@ #include <algorithm> #include <array> #include <memory> +#include <cstdio> +#include <cstdlib> +#include <tuple> +#include <set> +#include <fstream> #include <mutex> #include <fmt/format.h> @@ -14,12 +19,14 @@ #include "video_core/renderer_vulkan/renderer_vulkan.h" #include "common/assert.h" +#include "shader_recompiler/exception.h" #include "common/logging.h" #include "common/scope_exit.h" #include "common/settings.h" #include "video_core/buffer_cache/buffer_cache.h" #include "video_core/gpu_logging/gpu_logging.h" #include "video_core/control/channel_state.h" +#include "video_core/native_query_trace.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" @@ -42,6 +49,7 @@ #include "video_core/texture_cache/texture_cache_base.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" +#include "video_core/nx2_profile.h" namespace Vulkan { @@ -238,28 +246,296 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { SCOPE_EXIT { gpu.TickWork(); }; - FlushWork(); - gpu_memory->FlushCaching(); + bool submitted = false; + SCOPE_EXIT { + auto& native = maxwell3d->GetNativeMacroEngine(); + if (native.native_turing && !submitted) { + native.native_execution_failed = true; + LOG_DEBUG(HW_GPU, "Native graphics draw failed; refusing submission completion"); + } + }; + namespace Profile = VideoCommon::Nx2Profile; + { + Profile::Scope scope{Profile::Zone::FlushWork}; + FlushWork(); + gpu_memory->FlushCaching(); + } - GraphicsPipeline* const pipeline{pipeline_cache.CurrentGraphicsPipeline()}; + const auto report_failure = [&](std::string_view reason) { + static std::set<std::tuple<u64, u64, std::string_view>> seen; + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + if (seen.size() < 64 && seen.emplace(vs, ps, reason).second) { + LOG_WARNING(HW_GPU, "Draw skipped ({}): vs={:#x} ps={:#x} rt0={:#x} {}x{}", reason, vs, ps, + regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height); + } + }; + GraphicsPipeline* const pipeline{[&] { + Profile::Scope scope{Profile::Zone::PipelineLookup}; + return pipeline_cache.CurrentGraphicsPipeline(); + }()}; if (!pipeline) { + report_failure("no pipeline"); return; } std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; // update engine as channel may be different. pipeline->SetEngine(maxwell3d, gpu_memory); - if (!pipeline->Configure(is_indexed)) + if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->Configure(is_indexed)) { + report_failure("configure failed"); return; + } - UpdateDynamicStates(); + { + Profile::Scope scope{Profile::Zone::DynamicState}; + UpdateDynamicStates(); + } query_cache.NotifySegment(true); HandleTransformFeedback(); query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); - draw_func(); + { + Profile::Scope scope{Profile::Zone::DrawRecord}; + draw_func(); + } + submitted = true; + static u32 native_capture_count{}; + if (maxwell3d->GetNativeMacroEngine().native_turing && + native_capture_count < 16 && [] { + static const bool capture = std::getenv("EDEN_NATIVE_DRAW_CAPTURE") != nullptr; + return capture; + }()) { + const u32 capture = native_capture_count++; + const auto prefix = fmt::format("native-draw-{}", capture); + const auto dump = [&](const std::string& name, GPUVAddr address, size_t size) { + if (!size || !gpu_memory->IsFullyMappedRange(address, size)) return; + std::vector<u8> bytes(size); + gpu_memory->ReadBlockUnsafe(address, bytes.data(), bytes.size()); + std::ofstream file(prefix + name + ".bin", std::ios::binary); + file.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); + LOG_INFO(HW_GPU, "Native draw capture {}{}: address={:#x}, bytes={}, nonzero={}", + capture, name, address, size, + std::count_if(bytes.begin(), bytes.end(), [](u8 byte) { return byte != 0; })); + }; + const auto& regs = maxwell3d->regs; + const auto& rt = regs.rt[0]; + const size_t size = static_cast<size_t>(rt.array_pitch) * 4; + if (size && size <= 16 * 1024 * 1024) { + if (const auto address = gpu_memory->GpuToCpuAddress(rt.Address(), size)) { + LOG_INFO(HW_GPU, "Native draw {} render target physical address={:#x}", capture, *address); + texture_cache.DownloadMemory(*address, size); + dump("-rt0", rt.Address(), size); + } + } + std::ofstream registers(prefix + "-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(®s), sizeof(regs)); + for (size_t stream = 0; stream < 4; ++stream) { + if (regs.vertex_streams[stream].IsEnabled()) + dump(fmt::format("-vertex-{}", stream), regs.vertex_streams[stream].Address(), 128); + } + for (size_t stage : {0U, 4U}) { + for (size_t bank : {2U, 3U}) { + const auto& cb = maxwell3d->state.shader_stages[stage].const_buffers[bank]; + if (cb.enabled) dump(fmt::format("-cbuf-{}-{}", stage, bank), cb.address, std::min<size_t>(cb.size, 128)); + } + } + } } + +namespace { +// EDEN_NX2_WINDOW_TRACE: describe clears and draws into screen-sized colour targets (debug aid). +void TraceWindowTarget(const Tegra::Engines::Maxwell3D& maxwell3d, Tegra::MemoryManager& memory, + const char* kind, u32 count) { + static const bool trace = std::getenv("EDEN_NX2_WINDOW_TRACE") != nullptr; + static u32 traced{}; + const auto& regs = maxwell3d.regs; + static const u64 skip = [] { + const char* value = std::getenv("EDEN_NX2_WINDOW_TRACE_SKIP"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL; + }(); + static u64 seen_events{}; + static const u64 only_ps = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_PS"); + return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + const u64 pixel_shader = + (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + if (!trace || regs.rt[0].width != 3840 || traced >= 400 || seen_events++ < skip || + (only_ps != 0 && pixel_shader != only_ps)) { + return; + } + ++traced; + const auto& vp = regs.viewport_transform[0]; + const auto& sc = regs.scissor_test[0]; + LOG_INFO(HW_GPU, + "nx2 window {} rt0={:#x} count={} clear={:#x} color=({},{},{},{}) mask_common={} " + "mask0={:#x} rt_count={} vp scale=({},{}) translate=({},{}) scissor en={} x={}..{} " + "y={}..{} blend0={} vs={:#x} ps={:#x} rasterize={}", + kind, regs.rt[0].Address(), count, regs.clear_surface.raw, regs.clear_color[0], + regs.clear_color[1], regs.clear_color[2], regs.clear_color[3], regs.color_mask_common, + regs.color_mask[0].raw, regs.rt_control.count.Value(), vp.scale_x, vp.scale_y, + vp.translate_x, vp.translate_y, sc.enable, sc.min_x.Value(), sc.max_x.Value(), + sc.min_y.Value(), sc.max_y.Value(), regs.blend.enable[0], + (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1], + (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1], + regs.rasterize_enable); + if (kind[0] != 'd') { + return; + } + // Pixel-stage constant buffer 3: colour constants of the loading-screen passes. + if (const auto& cb3 = maxwell3d.state.shader_stages[4].const_buffers[3]; cb3.enabled) { + std::array<f32, 8> values{}; + memory.ReadBlockUnsafe(cb3.address, values.data(), sizeof(values)); + LOG_INFO(HW_GPU, "nx2 window ps c[3]@{:#x} size={}: {}", cb3.address, cb3.size, + fmt::join(values, " ")); + } + static u32 texture_traces{}; + if (texture_traces++ > 200) { + return; + } + // Texture used by the pixel shader's first bound TEX (handle in c[2][0x20]) and its TIC. + const auto& cb = maxwell3d.state.shader_stages[4].const_buffers[2]; + const u32 handle = cb.enabled ? memory.Read<u32>(cb.address + 0x20) : 0; + const u32 tic_index = handle & 0xfffff; + const GPUVAddr tic_address = regs.tex_header.Address() + u64{tic_index} * 32; + std::array<u32, 8> tic{}; + memory.ReadBlockUnsafe(tic_address, tic.data(), sizeof(tic)); + LOG_INFO(HW_GPU, + "nx2 window texture: cb2={:#x} en={} handle={:#x} tic pool={:#x} limit={} " + "sampler pool={:#x} tic[{}]={:08x}", + cb.address, cb.enabled, handle, regs.tex_header.Address(), regs.tex_header.limit, + regs.tex_sampler.Address(), tic_index, fmt::join(tic, " ")); + // First bytes of the texture's memory (block-linear, so just a sample). + const GPUVAddr texture = (GPUVAddr{tic[2] & 0xffff} << 32) | tic[1]; + std::array<u32, 16> texels{}; + memory.ReadBlockUnsafe(texture, texels.data(), sizeof(texels)); + LOG_INFO(HW_GPU, "nx2 window texture {:#x} {}x{}: {:08x}", texture, (tic[4] & 0xffff) + 1, + (tic[5] & 0xffff) + 1, fmt::join(texels, " ")); + // EDEN_NX2_TEXTURE_DUMP=<file>: save the first 4 MiB of the last traced texture's memory. + if (const char* path = std::getenv("EDEN_NX2_TEXTURE_DUMP"); path != nullptr) { + std::vector<u8> bytes(2048u * 10240u); + memory.ReadBlockUnsafe(texture, bytes.data(), bytes.size()); + if (std::FILE* file = std::fopen(path, "wb")) { + std::fwrite(bytes.data(), 1, bytes.size(), file); + std::fclose(file); + } + } +} +} // Anonymous namespace + void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Draw}; + TraceWindowTarget(*maxwell3d, *gpu_memory, is_indexed ? "draw-indexed" : "draw", + is_indexed ? maxwell3d->draw_manager.draw_state.index_buffer.count + : maxwell3d->draw_manager.draw_state.vertex_buffer.count); + // EDEN_NX2_CB24_TRACE: for draws whose vertex shader reads bank 24, show the inline + // constant buffer upload target (CB_ADDRESS) and its first words. + if (static const bool trace = std::getenv("EDEN_NX2_CB24_TRACE") != nullptr; trace) { + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + static u32 logged{}; + if ((vs == 0x11471c9a00 || vs == 0x11471ca600 || vs == 0x11471cd300) && logged++ < 3) { + std::array<u32, 24> words{}; + gpu_memory->ReadBlockUnsafe(regs.const_buffer.Address(), words.data(), sizeof(words)); + LOG_INFO(HW_GPU, "bank24 draw vs={:#x}: CB_ADDRESS {:#x} size {:#x} offset {:#x}: {:08x}", vs, + regs.const_buffer.Address(), regs.const_buffer.size, regs.const_buffer.offset, + fmt::join(words, " ")); + // Search every bound vertex-stage buffer for render-target-size values. + const auto& stage = maxwell3d->state.shader_stages[0]; + for (u32 slot = 0; slot < Tegra::Engines::Maxwell3D::Regs::MaxConstBuffers; ++slot) { + const auto& buffer = stage.const_buffers[slot]; + if (!buffer.enabled || buffer.size == 0) continue; + std::vector<u32> data(std::min<u32>(buffer.size, 0x10000) / 4); + gpu_memory->ReadBlockUnsafe(buffer.address, data.data(), data.size() * 4); + for (size_t i = 0; i < data.size(); ++i) { + const u32 w = data[i]; + if (w == 0x45200000 || w == 0x44b40000 || w == 0x39cccccd || w == 0x3a360b61 || + w == 0x44a00000 || w == 0x44340000 || w == 2559 || w == 1439) { + LOG_INFO(HW_GPU, " slot {} +{:#x} = {:#x}", slot, i * 4, w); + } + } + } + } + } + // EDEN_NX2_TRACE_VS=<address>: describe the next few draws using that vertex shader like the + // after-upscale trace does (render target, blend, fragment textures). + static const u64 trace_vs = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_VS"); + return value ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + if (trace_vs != 0) { + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + static u32 traced{}; + if (vs == trace_vs && traced++ % 200 == 0 && traced < 1000) { + Tegra::Engines::g_nx2_trace_draws_after_upscale = 1; + } + } + if (Tegra::Engines::g_nx2_trace_draws_after_upscale > 0) { + --Tegra::Engines::g_nx2_trace_draws_after_upscale; + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + LOG_INFO(HW_GPU, "after upscale: draw rt0 {:#x} {}x{} fmt {:#x} block {}/{}/{} pitch_linear {} depth {} array_pitch {:#x} ctrl {:#x} vs {:#x} ps {:#x}", + regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height, + static_cast<u32>(regs.rt[0].format), regs.rt[0].tile_mode.block_width.Value(), + regs.rt[0].tile_mode.block_height.Value(), regs.rt[0].tile_mode.block_depth.Value(), + regs.rt[0].tile_mode.is_pitch_linear.Value(), regs.rt[0].depth.Value(), + regs.rt[0].array_pitch, regs.rt_control.count.Value(), vs, ps); + // Fragment-stage texture handles (bound buffer 2) and the images they name. + const auto& handles = maxwell3d->state.shader_stages[4].const_buffers[2]; + if (handles.enabled) { + for (u32 i = 0; i < 24; ++i) { + const u32 handle = gpu_memory->Read<u32>(handles.address + i * 4); + if (handle == 0 || handle == 0xffffffff) continue; + Tegra::Texture::TICEntry tic{}; + gpu_memory->ReadBlockUnsafe(regs.tex_header.Address() + u64{handle & 0xfffff} * sizeof(tic), + &tic, sizeof(tic)); + LOG_INFO(HW_GPU, " fs handle[{}] = {:#x}: {:#x} {}x{} format {:#x}", i, handle, + tic.Address(), tic.Width(), tic.Height(), static_cast<u32>(tic.format.Value())); + } + } + const auto& bt = regs.blend_per_target[0]; + LOG_INFO(HW_GPU, " blend enable {} per-target {} global op {} src {:#x} dst {:#x} | rt0 op {} src {:#x} dst {:#x} alpha {:#x}/{:#x} mask {:#x}", + regs.blend.enable[0], regs.blend_per_target_enabled, + static_cast<u32>(regs.blend.color_op), static_cast<u32>(regs.blend.color_source), + static_cast<u32>(regs.blend.color_dest), static_cast<u32>(bt.color_op), + static_cast<u32>(bt.color_source), static_cast<u32>(bt.color_dest), + static_cast<u32>(bt.alpha_source), static_cast<u32>(bt.alpha_dest), + regs.color_mask[0].raw); + } + // EDEN_NX2_SKIP_VS=<address>[,<address>...]: drop draws using those vertex shaders (debug aid). + static const std::vector<u64> skip_list = [] { + std::vector<u64> list; + if (const char* value = std::getenv("EDEN_NX2_SKIP_VS")) { + for (char* end = nullptr; *value; value = *end ? end + 1 : end) { + list.push_back(std::strtoull(value, &end, 16)); + if (end == value) break; + } + } + return list; + }(); + if (!skip_list.empty()) { + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + if (std::find(skip_list.begin(), skip_list.end(), vs) != skip_list.end()) return; + } + // EDEN_NX2_RT_TRACE: log the first draw to every distinct colour target (debug aid). + if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) { + static std::set<std::tuple<GPUVAddr, u32, u32, u64, u64>> seen; + const auto& regs = maxwell3d->regs; + const auto& rt = regs.rt[0]; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + const auto key = std::make_tuple(rt.Address(), rt.width, static_cast<u32>(rt.format), vs, ps); + if (rt.width >= 640 && seen.size() < 600 && seen.insert(key).second) { + LOG_INFO(HW_GPU, "nx2 draw target {:#x} {}x{} format={:#x} vs={:#x} ps={:#x} blend={} (pairs so far {})", + rt.Address(), rt.width, rt.height, static_cast<u32>(rt.format), vs, ps, + regs.blend.enable[0], seen.size()); + } + } PrepareDraw(is_indexed, [this, is_indexed, instance_count] { const auto& draw_state = maxwell3d->draw_manager.draw_state; const u32 num_instances{instance_count}; @@ -293,6 +569,7 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { } void RasterizerVulkan::DrawIndirect() { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Draw}; const auto& params = maxwell3d->draw_manager.indirect_state; buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms] { @@ -398,6 +675,15 @@ void RasterizerVulkan::DrawTexture() { } void RasterizerVulkan::Clear(u32 layer_count) { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Clear}; + if (Tegra::Engines::g_nx2_trace_draws_after_upscale > 0) { + const auto& regs = maxwell3d->regs; + LOG_INFO(HW_GPU, "after upscale: clear rt0 {:#x} {}x{} fmt {:#x} surface {:#x} colour {} {} {} {}", + regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height, + static_cast<u32>(regs.rt[0].format), regs.clear_surface.raw, regs.clear_color[0], + regs.clear_color[1], regs.clear_color[2], regs.clear_color[3]); + } + TraceWindowTarget(*maxwell3d, *gpu_memory, "clear", layer_count); FlushWork(); gpu_memory->FlushCaching(); @@ -583,16 +869,19 @@ void RasterizerVulkan::Clear(u32 layer_count) { } void RasterizerVulkan::DispatchCompute() { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Compute}; FlushWork(); gpu_memory->FlushCaching(); ComputePipeline* const pipeline{pipeline_cache.CurrentComputePipeline()}; if (!pipeline) { + if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute pipeline unavailable"); return; } std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex}; if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache, texture_cache)) { + if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable"); return; } @@ -942,23 +1231,56 @@ Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, std::span<const u8> memory) { auto cpu_addr = gpu_memory->GpuToCpuAddress(address); + if (gpu_memory->UsesNativeTiling()) { + u32 first_word{}; + if (memory.size() >= sizeof(first_word)) std::memcpy(&first_word, memory.data(), sizeof(first_word)); + VideoCommon::TraceNativeQuery(11, gpu_memory->GetID(), address, cpu_addr.value_or(0), + 0, first_word, static_cast<u32>(copy_size)); + static const GPUVAddr watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{}; + }(); + if (watch != 0 && address < watch + 16 && address + copy_size > watch) { + LOG_INFO(Render_Vulkan, "Semaphore watch: inline write {:#x} size {} first word {:#x}", + address, copy_size, first_word); + } + } if (!cpu_addr) [[unlikely]] { gpu_memory->WriteBlock(address, memory.data(), copy_size); return; } gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size); + // EDEN_NX2_INLINE_DEFER=0: invalidate every cache at once (for comparisons). + static const bool defer = [] { + const char* value = std::getenv("EDEN_NX2_INLINE_DEFER"); + return value == nullptr || value[0] != '0'; + }(); + // One cache at a time: holding both while waiting for either (std::lock) waited longer, the + // fencing thread and CPU write traps holding one of them most of the time. + bool uploaded; { - std::unique_lock<std::recursive_mutex> lock{buffer_cache.mutex}; - if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) { + std::scoped_lock lock{buffer_cache.mutex}; + uploaded = buffer_cache.InlineMemory(*cpu_addr, copy_size, memory); + if (!uploaded && !defer) { buffer_cache.WriteMemory(*cpu_addr, copy_size); } } + // Pending query results must not land over the new words later: drop them now. + query_cache.InvalidateRegion(*cpu_addr, copy_size); + if (!uploaded && defer) { + // Title 1 makes thousands of small inline writes between draws (~10% of the GPU + // thread invalidating every cache for each). Guest memory is written; the buffer, + // texture and shader caches see the merged ranges at the next FlushCaching, which runs + // before anything of theirs is used. Writes into GPU-written buffer data were uploaded + // in place above, as before. + gpu_memory->AccumulateInvalidation(address, copy_size); + return; + } { - std::scoped_lock lock_texture{texture_cache.mutex}; + std::scoped_lock lock{texture_cache.mutex}; texture_cache.WriteMemory(*cpu_addr, copy_size); } pipeline_cache.InvalidateRegion(*cpu_addr, copy_size); - query_cache.InvalidateRegion(*cpu_addr, copy_size); } std::optional<FramebufferTextureInfo> RasterizerVulkan::AccelerateDisplay( @@ -969,6 +1291,13 @@ std::optional<FramebufferTextureInfo> RasterizerVulkan::AccelerateDisplay( std::scoped_lock lock{texture_cache.mutex}; const auto [image_view, scaled] = texture_cache.TryFindFramebufferImageView(config, framebuffer_addr); + static u32 display_trace_count{}; + static const bool frame_capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE") != nullptr; + if (frame_capture && display_trace_count++ < 64) { + LOG_INFO(HW_GPU, "Native display source: address={:#x}, offset={:#x}, dimensions={}x{}, stride={}, accelerated={}, mask={:#x}", + framebuffer_addr, config.offset, config.width, config.height, pixel_stride, + image_view != nullptr, config.layer_stack_mask); + } if (!image_view) { return {}; } @@ -997,8 +1326,10 @@ void RasterizerVulkan::FlushWork() { static constexpr u32 DRAWS_TO_DISPATCH = 512; static constexpr u32 CHECK_MASK = 3; #else + // Every 32 draws (was 8): each dispatch woke the worker thread, and Switch 2 titles make + // ~55,000 draws a second. A full chunk is dispatched on its own (Scheduler::Record). static constexpr u32 DRAWS_TO_DISPATCH = 4096; - static constexpr u32 CHECK_MASK = 7; + static constexpr u32 CHECK_MASK = 31; #endif // __ANDROID__ static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 4a7a7d5e78..e1e989c79f 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -55,6 +55,9 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) Scheduler::~Scheduler() = default; u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) { + if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { + VideoCommon::GpuSplit::CheckOwner("Scheduler::Flush"); + } // When flushing, we only send data to the worker thread; no waiting is necessary. const u64 signal_value = SubmitExecution(signal_semaphore, wait_semaphore); AllocateNewContext(); @@ -62,6 +65,9 @@ u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) { } void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) { + if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { + VideoCommon::GpuSplit::CheckOwner("Scheduler::Finish"); + } // When finishing, we need to wait for the submission to have executed on the device. const u64 presubmit_tick = CurrentTick(); SubmitExecution(signal_semaphore, wait_semaphore); @@ -83,12 +89,21 @@ void Scheduler::WaitWorker() { } void Scheduler::DispatchWork() { + if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { + VideoCommon::GpuSplit::CheckOwner("Scheduler::DispatchWork"); + } if (chunk && !chunk->Empty()) { + // Waking the worker is a system call (~2% of the GPU thread at one dispatch per 8 + // draws): skip it while the worker is still busy, it pops the queue before waiting. + bool wake; { std::scoped_lock ql{queue_mutex}; work_queue.push(std::move(chunk)); + wake = worker_waiting; + } + if (wake) { + event_cv.notify_all(); } - event_cv.notify_all(); AcquireNewChunk(); } } @@ -290,7 +305,11 @@ void Scheduler::WorkerThread(std::stop_token stop_token) { std::unique_lock lk{queue_mutex}; // Wait for work. - event_cv.wait(lk, stop_token, [&] { return TryPopQueue(work); }); + event_cv.wait(lk, stop_token, [&] { + worker_waiting = !TryPopQueue(work); + return !worker_waiting; + }); + worker_waiting = false; // If we've been asked to stop, we're done. if (stop_token.stop_requested()) { diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index d4bc8d0ec5..568063ddbd 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -18,6 +18,7 @@ #include "common/common_types.h" #include "common/polyfill_thread.h" #include "common/settings.h" +#include "video_core/gpu_split.h" #include "video_core/renderer_vulkan/vk_master_semaphore.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -103,6 +104,9 @@ public: template <typename T> requires std::is_invocable_v<T, vk::CommandBuffer, vk::CommandBuffer> void RecordWithUploadBuffer(T&& command) { + if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { + VideoCommon::GpuSplit::CheckOwner("Scheduler::Record"); + } if (chunk->Record(command)) { return; } @@ -119,6 +123,18 @@ public: }); } + /// Whether the recording thread submitted `tick` within 20 ms. + [[nodiscard]] bool WaitForSubmission(u64 tick) const { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds{20}; + while (tick >= master_semaphore->CurrentTick()) { + if (std::chrono::steady_clock::now() >= deadline) { + return false; + } + std::this_thread::sleep_for(std::chrono::microseconds{100}); + } + return true; + } + /// Returns the current command buffer tick. [[nodiscard]] u64 CurrentTick() const noexcept { return master_semaphore->CurrentTick(); @@ -129,9 +145,13 @@ public: return master_semaphore->IsFree(tick); } - /// Waits for the given GPU tick, optionally pacing frames. - void Wait(u64 tick, double target_fps = 0.0) { - if (tick > 0) { + /// Waits for the given GPU tick, optionally pacing frames. Threads other than the one that + /// records (the async present thread) pass may_flush = false: flushing from there records + /// into the current chunk while that thread does (corrupt command buffers, crashes). They + /// wait for the recording thread to submit the tick instead, and give up after 20 ms (it may + /// be waiting for them). + void Wait(u64 tick, double target_fps = 0.0, bool may_flush = true) { + if (tick > 0 && (may_flush || WaitForSubmission(tick))) { if (tick >= master_semaphore->CurrentTick()) { Flush(); } @@ -323,6 +343,7 @@ private: std::mutex reserve_mutex; std::mutex queue_mutex; std::condition_variable_any event_cv; + bool worker_waiting{}; ///< The worker is waiting for work (guarded by queue_mutex) std::jthread worker_thread; std::chrono::steady_clock::duration frame_interval{}; diff --git a/src/video_core/renderer_vulkan/vk_swapchain.cpp b/src/video_core/renderer_vulkan/vk_swapchain.cpp index fea5398557..a0ed5a4493 100644 --- a/src/video_core/renderer_vulkan/vk_swapchain.cpp +++ b/src/video_core/renderer_vulkan/vk_swapchain.cpp @@ -158,7 +158,7 @@ void Swapchain::Create( resource_ticks.resize(image_count); } -bool Swapchain::AcquireNextImage() { +bool Swapchain::AcquireNextImage(bool may_flush) { const VkResult result = device.GetLogical().AcquireNextImageKHR( *swapchain, (std::numeric_limits<u64>::max)(), *present_semaphores[frame_index], VK_NULL_HANDLE, &image_index); @@ -179,29 +179,29 @@ bool Swapchain::AcquireNextImage() { break; } - const auto wait_with_frame_pacing = [this] { + const auto wait_with_frame_pacing = [this, may_flush] { switch (Settings::values.frame_pacing_mode.GetValue()) { case Settings::FramePacingMode::Target_Auto: - scheduler.Wait(resource_ticks[image_index]); + scheduler.Wait(resource_ticks[image_index], 0.0, may_flush); break; case Settings::FramePacingMode::Target_30: - scheduler.Wait(resource_ticks[image_index], 30.0); + scheduler.Wait(resource_ticks[image_index], 30.0, may_flush); break; case Settings::FramePacingMode::Target_60: - scheduler.Wait(resource_ticks[image_index], 60.0); + scheduler.Wait(resource_ticks[image_index], 60.0, may_flush); break; case Settings::FramePacingMode::Target_90: - scheduler.Wait(resource_ticks[image_index], 90.0); + scheduler.Wait(resource_ticks[image_index], 90.0, may_flush); break; case Settings::FramePacingMode::Target_120: - scheduler.Wait(resource_ticks[image_index], 120.0); + scheduler.Wait(resource_ticks[image_index], 120.0, may_flush); break; } }; #ifdef __ANDROID__ if (android_get_device_api_level() >= 30) { - scheduler.Wait(resource_ticks[image_index]); + scheduler.Wait(resource_ticks[image_index], 0.0, may_flush); } else { wait_with_frame_pacing(); } diff --git a/src/video_core/renderer_vulkan/vk_swapchain.h b/src/video_core/renderer_vulkan/vk_swapchain.h index d926cc118a..415160046c 100644 --- a/src/video_core/renderer_vulkan/vk_swapchain.h +++ b/src/video_core/renderer_vulkan/vk_swapchain.h @@ -37,7 +37,8 @@ public: u32 height); /// Acquires the next image in the swapchain, waits as needed. - bool AcquireNextImage(); + /// `may_flush`: false on the async present thread (see Scheduler::Wait). + bool AcquireNextImage(bool may_flush = true); /// Presents the rendered image to the swapchain. void Present(VkSemaphore render_semaphore); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 583cb6edaf..f5fcdd1380 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -7,6 +7,7 @@ #include <algorithm> #include <limits> #include <array> +#include <cstdlib> #include <optional> #include <span> #include <memory> @@ -148,10 +149,68 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) { return usage; } +// An ASTC 2D array whose RGBA8 decode exceeds 1 GiB (title 1's 8192x8192x68 terrain textures). +// These stay on the GPU decoder as RGBA8 with trimmed levels (TrimmedHostInfo) even when other +// ASTC images are re-compressed to BC1/BC3 on the CPU: re-compressing them would not fit either. +[[nodiscard]] bool IsHugeAstcArray(const Device& device, const ImageInfo& info) { + if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported() || + info.type != ImageType::e2D || info.size.depth != 1) { + return false; + } + u64 bytes = 0; + for (s32 level = 0; level < info.resources.levels; ++level) { + bytes += u64{(std::max)(info.size.width >> level, 1U)} * + (std::max)(info.size.height >> level, 1U) * 4; + } + return bytes * info.resources.layers > (u64{1} << 30); +} + +// The host format of a huge ASTC array: the GPU decoder's RGBA8 (sRGB kept for sampling). +[[nodiscard]] std::optional<VkFormat> HugeAstcHostFormat(const Device& device, const ImageInfo& info) { + if (!IsHugeAstcArray(device, info)) { + return std::nullopt; + } + return VideoCore::Surface::IsPixelFormatSRGB(info.format) ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; +} + +constexpr std::array<VkFormat, 2> HugeAstcViewFormats{VK_FORMAT_A8B8G8R8_UNORM_PACK32, + VK_FORMAT_A8B8G8R8_SRGB_PACK32}; + +// Opaque ASTC (ImageInfo::astc_opaque) re-compressed to BC1 instead of BC3. +[[nodiscard]] bool IsOpaqueBc1Astc(const Device& device, const ImageInfo& info) { + return info.astc_opaque && IsPixelFormatASTC(info.format) && !IsHugeAstcArray(device, info); +} + +constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM_BLOCK, + VK_FORMAT_BC1_RGBA_SRGB_BLOCK}; + +[[nodiscard]] std::optional<VkFormat> AstcHostFormat(const Device& device, const ImageInfo& info) { + if (IsOpaqueBc1Astc(device, info)) { + return VideoCore::Surface::IsPixelFormatSRGB(info.format) ? VK_FORMAT_BC1_RGBA_SRGB_BLOCK + : VK_FORMAT_BC1_RGBA_UNORM_BLOCK; + } + return HugeAstcHostFormat(device, info); +} + +// ASTC that the GPU decodes and re-compresses to BC1/BC3 itself (TranscodeAstcToBcn): no CPU +// decode, so streamed textures are ready the frame they are first used instead of seconds later. +[[nodiscard]] bool WillUseGpuBcnTranscode(const Device& device, const ImageInfo& info) { + if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported() || + IsHugeAstcArray(device, info) || info.type != ImageType::e2D || info.size.depth != 1) { + return false; + } + return Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu && + Settings::values.astc_recompression.GetValue() != Settings::AstcRecompression::Uncompressed; +} + [[nodiscard]] bool WillUseAcceleratedAstcDecode(const Device& device, const ImageInfo& info) { if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported()) { return false; } + if (IsHugeAstcArray(device, info)) { + return true; + } if (Settings::values.accelerate_astc.GetValue() != Settings::AstcDecodeMode::Gpu) { return false; } @@ -160,6 +219,40 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) { info.size.depth == 1; } +// The host shape of an image whose decoded copy would not fit in VRAM: a GPU-decoded ASTC array +// over 1 GiB as RGBA8 (title 1's 8192x8192x68 ASTC 8x8 virtual textures are 24 GB each) keeps only +// its mip levels from `level_offset` on, so host mip 0 is guest mip `level_offset`. Normalized +// coordinates and derivative LOD still address it correctly; explicit LODs sample one level lower. +[[nodiscard]] ImageInfo TrimmedHostInfo(const Device& device, const ImageInfo& info, u32& level_offset) { + level_offset = 0; + if (!WillUseAcceleratedAstcDecode(device, info) || info.type != ImageType::e2D) { + return info; + } + const auto decoded_bytes = [&](u32 skip) { + u64 bytes = 0; + for (s32 level = static_cast<s32>(skip); level < info.resources.levels; ++level) { + bytes += u64{(std::max)(info.size.width >> level, 1U)} * + (std::max)(info.size.height >> level, 1U) * 4; + } + return bytes * info.resources.layers; + }; + if (decoded_bytes(0) <= (u64{1} << 30)) { + return info; + } + while (level_offset + 1 < static_cast<u32>(info.resources.levels) && + decoded_bytes(level_offset) > (u64{512} << 20)) { + ++level_offset; + } + ImageInfo host = info; + host.size.width = (std::max)(info.size.width >> level_offset, 1U); + host.size.height = (std::max)(info.size.height >> level_offset, 1U); + host.resources.levels -= static_cast<s32>(level_offset); + LOG_WARNING(Render_Vulkan, "{}x{}x{} ASTC image would be {} MiB decoded; keeping mip levels {}+ ({}x{})", + info.size.width, info.size.height, info.resources.layers, decoded_bytes(0) >> 20, + level_offset, host.size.width, host.size.height); + return host; +} + [[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info, std::optional<VkFormat> format_override = {}) { auto format_info = @@ -527,6 +620,20 @@ TransformBufferCopies(std::span<const VideoCommon::BufferCopy> copies, size_t bu }; } +// A guest view's levels on a host image that keeps only levels >= host_level_offset. +[[nodiscard]] SubresourceRange HostRange(const Image& image, SubresourceRange range) { + const s32 skip = static_cast<s32>(image.host_level_offset); + if (skip == 0) { + return range; + } + const s32 host_levels = image.info.resources.levels - skip; + const s32 base = std::clamp(range.base.level - skip, 0, host_levels - 1); + const s32 end = std::clamp(range.base.level + range.extent.levels - skip, base + 1, host_levels); + range.base.level = base; + range.extent.levels = end - base; + return range; +} + [[nodiscard]] VkImageSubresourceRange MakeSubresourceRange(const ImageView* image_view) { SubresourceRange range = image_view->range; if (True(image_view->flags & VideoCommon::ImageViewFlagBits::Slice)) { @@ -942,9 +1049,16 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched : device{device_}, scheduler{scheduler_}, memory_allocator{memory_allocator_}, staging_buffer_pool{staging_buffer_pool_}, blit_image_helper{blit_image_helper_}, render_pass_cache{render_pass_cache_}, resolution{Settings::values.resolution_info} { - if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu) { + // Huge ASTC arrays always decode on the GPU (IsHugeAstcArray), whatever the decode setting. + if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu || + !device.IsOptimalAstcSupported()) { astc_decoder_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, compute_pass_descriptor_queue, memory_allocator); + if (Settings::values.astc_recompression.GetValue() != + Settings::AstcRecompression::Uncompressed) { + bc_encoder_pass.emplace(device, scheduler, descriptor_pool, + compute_pass_descriptor_queue); + } } if (!device.IsKhrImageFormatListSupported()) { return; @@ -1796,6 +1910,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { + return device.GetDeviceMemoryBudget(); +} + bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const { const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format); if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) { @@ -1814,6 +1932,20 @@ bool TextureCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } +void TextureCacheRuntime::LogMemoryStatistics() const { + Vulkan::LogMemoryStatistics(device); +} + +bool TextureCacheRuntime::UseOpaqueAstcFormat() const { + // EDEN_NX2_ASTC_BC1=0 keeps every re-compressed ASTC image in BC3. + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_ASTC_BC1"); + return value == nullptr || value[0] != '0'; + }(); + return enabled && !device.IsOptimalAstcSupported() && + Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc3; +} + void TextureCacheRuntime::FlushDeferredClear() { scheduler.FlushDeferredClear(); } @@ -1859,6 +1991,9 @@ void TextureCacheRuntime::ReleaseMsaaScratchImage(VkImage image) { } void TextureCacheRuntime::TickFrame() { + while (!pending_transcodes.empty() && scheduler.IsFree(pending_transcodes.front().tick)) { + pending_transcodes.pop_front(); + } static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60; std::erase_if(msaa_scratch_images, [this](MsaaScratchImage& scratch) { if (!scheduler.IsFree(scratch.tick)) { @@ -1876,13 +2011,30 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu VAddr cpu_addr_) : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler}, runtime{&runtime_}, - original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info, - runtime->ViewFormats(info.format))), + original_image(MakeImage(runtime_.device, runtime_.memory_allocator, + TrimmedHostInfo(runtime_.device, info, host_level_offset), + IsHugeAstcArray(runtime_.device, info) + ? std::span<const VkFormat>{HugeAstcViewFormats} + : IsOpaqueBc1Astc(runtime_.device, info) + ? std::span<const VkFormat>{OpaqueAstcViewFormats} + : runtime->ViewFormats(info.format), + AstcHostFormat(runtime_.device, info))), aspect_mask(ImageAspectMask(info.format)) { - if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { + if (original_image.IsUnbacked()) { + flags |= VideoCommon::ImageFlagBits::Unbacked; + } + if (host_level_offset != 0) { + flags |= VideoCommon::ImageFlagBits::TrimmedLevels; + } + if (IsHugeAstcArray(runtime->device, info)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + flags |= VideoCommon::ImageFlagBits::Converted; + flags |= VideoCommon::ImageFlagBits::CostlyLoad; + } else if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { switch (Settings::values.accelerate_astc.GetValue()) { case Settings::AstcDecodeMode::Gpu: - if (WillUseAcceleratedAstcDecode(runtime->device, info)) { + if (WillUseAcceleratedAstcDecode(runtime->device, info) || + WillUseGpuBcnTranscode(runtime->device, info)) { flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; } break; @@ -1903,11 +2055,12 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu original_image.SetObjectNameEXT(VideoCommon::Name(*this).c_str()); } current_image = &Image::original_image; - storage_image_views.resize(info.resources.levels); + const s32 host_levels = info.resources.levels - static_cast<s32>(host_level_offset); + storage_image_views.resize(host_levels); if (WillUseAcceleratedAstcDecode(runtime->device, info)) { const auto& device = runtime->device.GetLogical(); const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; - for (s32 level = 0; level < info.resources.levels; ++level) { + for (s32 level = 0; level < host_levels; ++level) { storage_image_views[level] = MakeStorageView(device, level, *original_image, storage_format); } @@ -2085,6 +2238,11 @@ void Image::DownloadMemory(VkBuffer buffer, size_t offset, void Image::DownloadMemory(std::span<VkBuffer> buffers_span, std::span<size_t> offsets_span, std::span<const VideoCommon::BufferImageCopy> copies) { + if (!original_image) { + LOG_CRITICAL(Render_Vulkan, "Missing image at download: gpu={:#x}, cpu={:#x}, type={}, format={}, flags={:#x}, size={}, dimensions={}x{}", + gpu_addr, cpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format), + static_cast<u32>(flags), unswizzled_size_bytes, info.size.width, info.size.height); + } const bool is_rescaled = True(flags & ImageFlagBits::Rescaled); if (is_rescaled) { ScaleDown(); @@ -2519,6 +2677,15 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI } } auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format); + if (IsHugeAstcArray(*device, image.info)) { + format_info.format = VideoCore::Surface::IsPixelFormatSRGB(format) + ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + } else if (IsOpaqueBc1Astc(*device, image.info) && IsPixelFormatASTC(format)) { + format_info.format = VideoCore::Surface::IsPixelFormatSRGB(format) + ? VK_FORMAT_BC1_RGBA_SRGB_BLOCK + : VK_FORMAT_BC1_RGBA_UNORM_BLOCK; + } if (device->ApiVersion() >= VK_API_VERSION_1_3) { const VkFormatProperties3 properties3 = device->GetPhysical().GetFormatProperties3(format_info.format); @@ -2567,7 +2734,7 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI .viewType = VkImageViewType{}, .format = format_info.format, .components = swizzle_mapping, - .subresourceRange = MakeSubresourceRange(aspect_mask, info.range), + .subresourceRange = MakeSubresourceRange(aspect_mask, HostRange(image, info.range)), }; const auto create = [&](TextureType tex_type, std::optional<u32> num_layers) { VkImageViewCreateInfo ci{create_info}; @@ -2821,6 +2988,11 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t if (tsc.mipmap_filter != TextureMipmapFilter::None) { min_lod = tsc.MinLod(); max_lod = tsc.MaxLod(); + // EDEN_NX2_FORCE_LOD0: sample only the top mip level (debug aid for mip layout problems). + if (static const bool force_lod0 = std::getenv("EDEN_NX2_FORCE_LOD0") != nullptr; force_lod0) { + min_lod = 0.0f; + max_lod = 0.25f; + } } base_ci = VkSamplerCreateInfo{ .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, @@ -3133,6 +3305,9 @@ void TextureCacheRuntime::AccelerateImageUpload( u32 z_start, u32 z_count) { if (IsPixelFormatASTC(image.info.format)) { + if (bc_encoder_pass && WillUseGpuBcnTranscode(device, image.info)) { + return TranscodeAstcToBcn(image, map, swizzles); + } return astc_decoder_pass->Assemble(image, map, swizzles); } @@ -3151,6 +3326,100 @@ void TextureCacheRuntime::AccelerateImageUpload( ASSERT(false); } +void TextureCacheRuntime::TranscodeAstcToBcn( + Image& image, const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles) { + const ImageInfo& info = image.info; + const u32 width = info.size.width; + const u32 height = info.size.height; + const u32 levels = static_cast<u32>(info.resources.levels); + const u32 layers = static_cast<u32>(info.resources.layers); + constexpr VkFormat DecodeFormat = VK_FORMAT_A8B8G8R8_UNORM_PACK32; + PendingTranscode pending{ + .tick = 0, + .image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = DecodeFormat, + .extent = {width, height, 1}, + .mipLevels = levels, + .arrayLayers = layers, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_STORAGE_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }, true), + .views = {}, + }; + for (u32 level = 0; level < levels; ++level) { + pending.views.push_back( + MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat)); + } + astc_decoder_pass->AssembleInto( + info, 0, image.guest_size_bytes, + ASTCDecoderPass::Target{ + .image = *pending.image, + .aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT, + .initialized = false, + .level_view = [&pending](s32 level) { return *pending.views[level]; }, + .wait = false, + }, + map, swizzles); + + // Encode every level into one buffer, then copy it into the image. + const bool bc3 = !IsOpaqueBc1Astc(device, info) && + Settings::values.astc_recompression.GetValue() != + Settings::AstcRecompression::Bc1; + const u64 block_bytes = bc3 ? 16 : 8; + boost::container::small_vector<VideoCommon::BufferImageCopy, 16> copies; + u64 total = 0; + for (u32 level = 0; level < levels; ++level) { + const u32 level_width = (std::max)(width >> level, 1U); + const u32 level_height = (std::max)(height >> level, 1U); + const u32 blocks_x = Common::DivCeil(level_width, 4U); + const u32 blocks_y = Common::DivCeil(level_height, 4U); + const u64 size = u64{blocks_x} * blocks_y * layers * block_bytes; + copies.push_back({ + .buffer_offset = total, + .buffer_size = size, + .buffer_row_length = blocks_x * 4, + .buffer_image_height = blocks_y * 4, + .image_subresource{ + .base_level = static_cast<s32>(level), + .base_layer = 0, + .num_layers = static_cast<s32>(layers), + }, + .image_offset{0, 0, 0}, + .image_extent{level_width, level_height, 1}, + }); + total += size; + } + const auto output = staging_buffer_pool.Request(total, MemoryUsage::DeviceLocal); + for (u32 level = 0; level < levels; ++level) { + bc_encoder_pass->Encode(*pending.views[level], copies[level].image_extent.width, + copies[level].image_extent.height, layers, bc3, output.buffer, + output.offset, total, copies[level].buffer_offset); + } + scheduler.Record([](vk::CommandBuffer cmdbuf) { + const VkMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, barrier); + }); + image.UploadMemory(output.buffer, output.offset, copies); + pending.tick = scheduler.CurrentTick(); + pending_transcodes.push_back(std::move(pending)); +} + void TextureCacheRuntime::TransitionImageLayout(Image& image) { if (!image.ExchangeInitialization()) { VkImageMemoryBarrier barrier{ diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 2e9736f707..1d92d439f4 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -6,6 +6,7 @@ #pragma once +#include <deque> #include <span> #include "video_core/texture_cache/texture_cache_base.h" @@ -66,8 +67,15 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceMemoryBudget() const; + bool CanReportMemoryUsage() const; + void LogMemoryStatistics() const; + + /// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead. + bool UseOpaqueAstcFormat() const; + bool CanDownloadMsaa(const VideoCommon::ImageInfo& info) const; [[nodiscard]] VkImage AcquireMsaaScratchImage(const VkImageCreateInfo& image_ci); @@ -157,6 +165,20 @@ public: BlitImageHelper& blit_image_helper; RenderPassCache& render_pass_cache; std::optional<ASTCDecoderPass> astc_decoder_pass; + std::optional<BcEncoderPass> bc_encoder_pass; + + /// ASTC decoded on the GPU and re-compressed to BC1/BC3 there (accelerate_astc=Gpu with + /// astc_recompression=Bc1/Bc3): decode into a temporary RGBA8 image, encode, copy. + void TranscodeAstcToBcn(Image& image, const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles); + + /// Temporary decode images, freed once the GPU is past `tick`. + struct PendingTranscode { + u64 tick; + vk::Image image; + std::vector<vk::ImageView> views; + }; + std::deque<PendingTranscode> pending_transcodes; std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass; const Settings::ResolutionScalingInfo& resolution; diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp index f1aff2810f..dfe80abac3 100644 --- a/src/video_core/shader_cache.cpp +++ b/src/video_core/shader_cache.cpp @@ -5,12 +5,17 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <cstring> +#include <string> +#include <unordered_set> +#include <fmt/ranges.h> #include <array> #include <vector> #include <boost/container/small_vector.hpp> #include "common/assert.h" #include "shader_recompiler/frontend/maxwell/control_flow.h" +#include "shader_recompiler/exception.h" #include "shader_recompiler/object_pool.h" #include "video_core/control/channel_state.h" #include "video_core/dirty_flags.h" @@ -43,7 +48,7 @@ void ShaderCache::SyncGuestHost() { ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_) : device_memory{device_memory_} {} -bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { +bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { auto& dirty{maxwell3d->dirty.flags}; if (!dirty[VideoCommon::Dirty::Shaders]) { return last_shaders_valid; @@ -51,6 +56,7 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { dirty[VideoCommon::Dirty::Shaders] = false; const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) { if (!maxwell3d->regs.IsShaderConfigEnabled(index)) { unique_hashes[index] = 0; @@ -63,7 +69,9 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { unique_hashes[index] = 0; continue; } - const GPUVAddr shader_addr{base_addr + shader_config.offset}; + const GPUVAddr shader_addr{native + ? (static_cast<u64>(shader_config.reserved[0] & 0xff) << 32) | shader_config.reserved[1] + : base_addr + shader_config.offset}; const std::optional<VAddr> cpu_shader_addr{gpu_memory->GpuToCpuAddress(shader_addr)}; if (!cpu_shader_addr) { LOG_ERROR(HW_GPU, "Invalid GPU address for shader {:#016x}", shader_addr); @@ -71,16 +79,68 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { return false; } const ShaderInfo* shader_info{TryGet(*cpu_shader_addr)}; + if (!shader_info && native && IsRejected(*cpu_shader_addr)) { + // A rejected program fails the same way every time it is bound: skip its draws + // without rethrowing (two C++ throws each time cost ~1-2% of the GPU thread). + maxwell3d->GetNativeMacroEngine().native_execution_failed = true; + last_shaders_valid = false; + return false; + } if (!shader_info) { - const u32 start_address{shader_config.offset}; - GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, base_addr, start_address}; - shader_info = MakeShaderInfo(env, *cpu_shader_addr); + const u32 start_address{native ? 0 : shader_config.offset}; + try { + // Native programs are lowered while the environment is built. + RethrowIfRejected(*cpu_shader_addr); + GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, native ? shader_addr : base_addr, start_address}; + shader_info = MakeShaderInfo(env, *cpu_shader_addr); + } catch (const Shader::Exception&) { + RememberRejection(*cpu_shader_addr); + // Once per program: which stage slot it is bound to, its header and the slot's + // registers (to tell how a pipeline with task/mesh programs is set up). + static std::unordered_set<GPUVAddr> described; + if (native && described.insert(shader_addr).second) { + std::array<u32, 5> header{}; + gpu_memory->ReadBlockUnsafe(shader_addr, header.data(), sizeof(header)); + std::array<u32, sizeof(shader_config) / sizeof(u32)> config{}; + std::memcpy(config.data(), &shader_config, sizeof(config)); + LOG_ERROR(HW_GPU, "Rejected program {:#x} is bound to stage {}: header {:08x}, slot {:08x}", + shader_addr, index, fmt::join(header, " "), fmt::join(config, " ")); + } + throw; + } } shader_infos[index] = shader_info; unique_hashes[index] = shader_info->unique_hash; } last_shaders_valid = true; return true; +} catch (const Shader::Exception& error) { + auto& native = maxwell3d->GetNativeMacroEngine(); + if (!native.native_turing) throw; + native.native_execution_failed = true; + last_shaders_valid = false; + // Each rejected program is logged once (the draws that use it keep failing quietly). + static std::unordered_set<std::string> logged; + if (logged.insert(error.what()).second) { + LOG_ERROR(HW_GPU, "Native graphics shader rejected: {}", error.what()); + } + // Once: show what the vertex stage has bound, to find where an unbound bank's data lives. + static std::vector<std::string> dumped_errors; + if (const std::string what{error.what()}; + what.find("constant buffer") != std::string::npos && dumped_errors.size() < 12 && + std::count(dumped_errors.begin(), dumped_errors.end(), what) < 3) { + dumped_errors.push_back(what); + LOG_INFO(HW_GPU, "VS cbuf dump for: {}", what.substr(0, 40)); + for (u32 slot = 3; slot < 8; ++slot) { + const auto& buffer = maxwell3d->state.shader_stages[0].const_buffers[slot]; + if (!buffer.enabled) continue; + std::array<u32, 24> words{}; + gpu_memory->ReadBlockUnsafe(buffer.address, words.data(), sizeof(words)); + LOG_INFO(HW_GPU, "VS cbuf {} at {:#x} size {:#x}: {:08x}", slot, buffer.address, buffer.size, + fmt::join(words, " ")); + } + } + return false; } const ShaderInfo* ShaderCache::ComputeShader() { @@ -95,22 +155,54 @@ const ShaderInfo* ShaderCache::ComputeShader() { if (const ShaderInfo* const shader = TryGet(*cpu_shader_addr)) { return shader; } - ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; - return MakeShaderInfo(env, *cpu_shader_addr); + try { + RethrowIfRejected(*cpu_shader_addr); + ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; + return MakeShaderInfo(env, *cpu_shader_addr); + } catch (const Shader::Exception&) { + RememberRejection(*cpu_shader_addr); + throw; + } +} + +void ShaderCache::RethrowIfRejected(VAddr addr) { + std::scoped_lock lock{rejected_mutex}; + if (const auto it = rejected_native.find(addr); it != rejected_native.end()) { + std::rethrow_exception(it->second); + } +} + +bool ShaderCache::IsRejected(VAddr addr) { + std::scoped_lock lock{rejected_mutex}; + return rejected_native.contains(addr); +} + +void ShaderCache::RememberRejection(VAddr addr) { + std::scoped_lock lock{rejected_mutex}; + rejected_native.emplace(addr, std::current_exception()); + rejected_lowest.store((std::min)(rejected_lowest.load(std::memory_order_relaxed), addr), + std::memory_order_relaxed); + rejected_highest.store((std::max)(rejected_highest.load(std::memory_order_relaxed), addr), + std::memory_order_relaxed); } void ShaderCache::GetGraphicsEnvironments(GraphicsEnvironments& result, const std::array<u64, NUM_PROGRAMS>& unique_hashes) { size_t env_index{}; const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; for (size_t index = 0; index < NUM_PROGRAMS; ++index) { if (unique_hashes[index] == 0) { continue; } const auto program{static_cast<Tegra::Engines::Maxwell3D::Regs::ShaderType>(index)}; auto& env{result.envs[index]}; - const u32 start_address{maxwell3d->regs.pipelines[index].offset}; - env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, base_addr, start_address}; + const auto& config = maxwell3d->regs.pipelines[index]; + const u32 start_address{native ? 0 : config.offset}; + const GPUVAddr address = native + ? (static_cast<u64>(config.reserved[0] & 0xff) << 32) | config.reserved[1] + : base_addr; + env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address}; env.SetCachedSize(shader_infos[index]->size_bytes); result.env_ptrs[env_index++] = &env; } @@ -144,6 +236,15 @@ void ShaderCache::Register(std::unique_ptr<ShaderInfo> data, VAddr addr, size_t void ShaderCache::InvalidatePagesInRegion(VAddr addr, size_t size) { const VAddr addr_end = addr + size; + // Rejected programs are few and in a few places: look at them only for writes that can + // reach them (this runs for every inline upload and pre-draw invalidation). + if (addr_end > rejected_lowest.load(std::memory_order_relaxed) && + addr <= rejected_highest.load(std::memory_order_relaxed)) { + std::scoped_lock lock{rejected_mutex}; + boost::unordered::erase_if(rejected_native, [&](const auto& rejected) { + return rejected.first >= addr && rejected.first < addr_end; + }); + } const u64 page_end = (addr_end + YUZU_PAGESIZE - 1) >> YUZU_PAGEBITS; for (u64 page = addr >> YUZU_PAGEBITS; page < page_end; ++page) { auto it = invalidation_cache.find(page); @@ -241,7 +342,7 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu info->unique_hash = env.CalculateHash(); info->size_bytes = env.ReadSizeBytes(); } - const size_t size_bytes{info->size_bytes}; + const size_t size_bytes{env.SourceSizeBytes() ? env.SourceSizeBytes() : info->size_bytes}; const ShaderInfo* const result{info.get()}; Register(std::move(info), cpu_addr, size_bytes); return result; diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h index 997f2b0371..f4179dd435 100644 --- a/src/video_core/shader_cache.h +++ b/src/video_core/shader_cache.h @@ -7,6 +7,8 @@ #pragma once #include <array> +#include <atomic> +#include <exception> #include <memory> #include <mutex> #include <span> @@ -143,6 +145,13 @@ private: /// @brief Create a new shader entry and register it const ShaderInfo* MakeShaderInfo(GenericEnvironment& env, VAddr cpu_addr); + /// Rethrows the error a native program at this address was rejected with, if any. + void RethrowIfRejected(VAddr addr); + /// Whether a native program at this address was rejected. + bool IsRejected(VAddr addr); + /// Remembers the exception being handled as this address's rejection. + void RememberRejection(VAddr addr); + Tegra::MaxwellDeviceMemoryManager& device_memory; mutable std::mutex lookup_mutex; @@ -152,6 +161,14 @@ private: ::Common::unordered_map<u64, std::vector<Entry*>> invalidation_cache; std::vector<std::unique_ptr<ShaderInfo>> storage; std::vector<Entry*> marked_for_removal; + + /// Native programs the lowering rejected, by address, so each draw or dispatch rethrows the + /// first error instead of lowering again; dropped when guest memory there is written. + std::mutex rejected_mutex; + ::Common::unordered_map<u64, std::exception_ptr> rejected_native; + /// Bounds of every address ever rejected (never shrinks): a quick test for invalidations. + std::atomic<VAddr> rejected_lowest{~VAddr{}}; + std::atomic<VAddr> rejected_highest{}; }; } // namespace VideoCommon diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 4d5db7e511..96306d9a27 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -5,7 +5,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <chrono> +#include <cstdlib> +#include <thread> #include <filesystem> +#include <cstring> +#include <string> #include <fstream> #include <memory> #include <optional> @@ -21,6 +26,11 @@ #include "common/settings.h" #include <ranges> #include "shader_recompiler/environment.h" +#include "shader_recompiler/exception.h" +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" +#include "shader_recompiler/frontend/ampere/program_override.h" +#include "shader_recompiler/frontend/ampere/program_capture.h" +#include "shader_recompiler/frontend/ampere/program_address_map.h" #include "video_core/engines/kepler_compute.h" #include "video_core/memory_manager.h" #include "video_core/shader_environment.h" @@ -149,11 +159,26 @@ u64 GenericEnvironment::ReadInstruction(u32 address) { if (address >= cached_lowest && address < cached_highest) { return code[(address - cached_lowest) / INST_SIZE]; } + if (native_source_size) { + throw Shader::RuntimeError("Lowered native instruction address out of range: {:#x}", address); + } has_unbound_instructions = true; return gpu_memory->Read<u64>(program_base + address); } +u32 GenericEnvironment::TranslateProgramAddress(u32 address) const { + if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated; + if (native_source_size || Shader::Ampere::HasNativeAddressMap(code)) + throw Shader::RuntimeError("Invalid native indirect branch address {:#x}", address); + return address; +} + std::optional<u64> GenericEnvironment::Analyze() { + if (native_source_size) { + // Include the converted header and a version tag to isolate this experimental ABI. + return Common::CityHash64(reinterpret_cast<const char*>(code.data()), + code.size() * sizeof(u64)) ^ 0x4f554e4345000002ULL; + } const std::optional<u64> size{TryFindSize()}; if (!size) { return std::nullopt; @@ -164,6 +189,12 @@ std::optional<u64> GenericEnvironment::Analyze() { } void GenericEnvironment::SetCachedSize(size_t size_bytes) { + if (native_source_size) { + if (size_bytes != CachedSizeBytes()) { + throw Shader::RuntimeError("Native shader changed while constructing pipeline"); + } + return; + } cached_lowest = start_address; cached_highest = start_address + static_cast<u32>(size_bytes); code.resize(CachedSizeWords()); @@ -291,6 +322,146 @@ Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, return entry; } +void GenericEnvironment::InitializeNativeCode(u32 source_header_size) { + using Shader::Ampere::Instruction; + const u32 native_header_size = source_header_size; + constexpr u32 maximum_instructions = Shader::Ampere::ProgramCaptureState::MaximumInstructions; + std::vector<Instruction> native; + Shader::Ampere::ProgramCaptureState capture; + for (u32 index = 0; index < maximum_instructions; ++index) { + const GPUVAddr address = program_base + native_header_size + index * sizeof(Instruction); + if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) { + // The program may end before a branch target past its last EXIT (the capture can't + // tell); stop at the mapping's end and let the lowering judge what was captured. + if (!native.empty()) break; + throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address); + } + auto& instruction = native.emplace_back(); + gpu_memory->ReadBlock(address, &instruction, sizeof(instruction)); + const auto status = capture.Observe(instruction, index); + if (status == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget) { + if (const char* dir = std::getenv("EDEN_NX2_SHADER_DUMP")) { + const auto path = fmt::format("{}/rejected-{:x}.bin", dir, program_base + native_header_size); + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fwrite(native.data(), sizeof(Instruction), std::min<size_t>(native.size(), 4096), file); + std::fclose(file); + } + } + throw Shader::RuntimeError("Native shader branch outside capture bounds at {:#x} +{:#x} [{:08x} {:08x} {:08x} {:08x}]", + program_base + native_header_size, index * 16, + instruction.words[0], instruction.words[1], + instruction.words[2], instruction.words[3]); + } + if (status == Shader::Ampere::ProgramCaptureState::Result::Complete) break; + } + auto lowered = Shader::Ampere::LowerToMaxwell(native); + // EDEN_NX2_REJECT=<hex address>,...: treat these programs as rejected (to find which newly + // lowered program changes a game's behaviour without rebuilding). + if (static const std::string reject_list = [] { + const char* text = std::getenv("EDEN_NX2_REJECT"); + return text ? "," + std::string{text} + "," : std::string{}; + }(); + !reject_list.empty() && + reject_list.find(fmt::format(",{:x},", program_base)) != std::string::npos) { + throw Shader::RuntimeError("Native shader at {:#x} rejected by EDEN_NX2_REJECT", program_base); + } + // EDEN_NX2_SHADER_DUMP=<dir>: keep every captured native program (named by its key) so the + // lowering can be tested offline against all of a game's shaders. + if (static const char* const dump_directory = std::getenv("EDEN_NX2_SHADER_DUMP"); + dump_directory != nullptr) { + const auto folder = std::filesystem::path{dump_directory}; + const auto path = folder / fmt::format("{:016x}-{}-{:x}.bin", Shader::Ampere::NativeProgramKey(native), + source_header_size == 0 ? "cs" : "gfx", program_base); + if (!std::filesystem::exists(path)) { + std::filesystem::create_directories(folder); + std::ofstream dump(path, std::ios::binary); + dump.write(reinterpret_cast<const char*>(native.data()), + native.size() * sizeof(Shader::Ampere::Instruction)); + } + } + if (!lowered) { + if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_OVERRIDE_DIR")) { + const auto key = fmt::format("{:016x}", Shader::Ampere::NativeProgramKey(native)); + const auto folder = std::filesystem::path{directory}; + std::filesystem::create_directories(folder); + const auto source_path = folder / (key + ".source.bin"); + const auto override_path = folder / (key + ".override.bin"); + { + std::ofstream dump(source_path, std::ios::binary); + dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Shader::Ampere::Instruction)); + } + { + // Retain mapped bytes beyond the first EXIT for investigating + // native CALL targets in compiler library subroutines. + std::ofstream mapped(folder / (key + ".mapped.bin"), std::ios::binary); + std::array<u8, 0x1000> block{}; + for (u32 offset = 0; offset < 0x10000; offset += 0x1000) { + if (!gpu_memory->IsFullyMappedRange(program_base + offset, block.size())) break; + gpu_memory->ReadBlock(program_base + offset, block.data(), block.size()); + mapped.write(reinterpret_cast<const char*>(block.data()), block.size()); + } + } + LOG_WARNING(Shader, "Native shader paused: key={}, offset={:#x}, reason={}, source={}", + key, lowered.error_offset, lowered.error, source_path.string()); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::minutes{30}; + while (!gpu_memory->IsShuttingDown() && std::chrono::steady_clock::now() < deadline) { + if (std::filesystem::exists(override_path)) { + std::ifstream file(override_path, std::ios::binary | std::ios::ate); + const auto length = file.tellg(); + if (length <= 0 || length > 16 * 1024 * 1024) { + throw Shader::RuntimeError("Invalid native shader override size"); + } + std::vector<u8> bytes(static_cast<size_t>(length)); + file.seekg(0); + file.read(reinterpret_cast<char*>(bytes.data()), bytes.size()); + if (!file) throw Shader::RuntimeError("Native shader override read failed"); + const u32 source_bytes = Shader::Ampere::ProgramOverrideSourceSize(bytes); + if (source_bytes > native.size() * sizeof(Shader::Ampere::Instruction)) { + if (!gpu_memory->IsFullyMappedRange(program_base + native_header_size, source_bytes)) + throw Shader::RuntimeError("Unmapped extended native shader source"); + native.resize(source_bytes / sizeof(Shader::Ampere::Instruction)); + gpu_memory->ReadBlock(program_base + native_header_size, native.data(), source_bytes); + } + lowered = Shader::Ampere::DecodeProgramOverride(bytes, native); + if (!lowered) throw Shader::RuntimeError("Native shader override rejected: {}", lowered.error); + LOG_WARNING(Shader, "Native shader resumed: key={}, instructions={}", key, lowered.instructions.size()); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds{100}); + } + } + } + if (!lowered) { + const size_t failed = lowered.error_offset / sizeof(Shader::Ampere::Instruction); + const auto words = failed < native.size() ? native[failed].words : std::array<u32, 4>{}; + throw Shader::RuntimeError( + "Native shader at {:#x}, instruction {:#x}: {} [op {:03x} {:08x} {:08x} {:08x} {:08x}]", + program_base, lowered.error_offset, lowered.error, words[0] & 0xfff, words[0], + words[1], words[2], words[3]); + } + if (lowered.required_texture_buffer != ~0U) { + // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment. + texture_bound = lowered.required_texture_buffer; + } + native_source_size = native_header_size + lowered.source_size; + code.resize(sizeof(sph) / sizeof(u64)); + std::memcpy(code.data(), &sph, sizeof(sph)); + for (const u64 instruction : lowered.instructions) { + if ((code.size() * sizeof(u64)) % 32 == 0) { + code.push_back(0); // Maxwell scheduling word, absent from native instructions. + } + code.push_back(instruction); + } + // Keep the original padding ABI, then retain the native address map as + // unreachable data. Existing disk-cache serialization carries it with code. + code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); + cached_lowest = 0; + cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64)); + LOG_INFO(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions", + program_base, native_source_size, lowered.instructions.size()); +} + GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, Tegra::MemoryManager& gpu_memory_, Maxwell::ShaderType program, GPUVAddr program_base_, @@ -298,6 +469,23 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, : GenericEnvironment{gpu_memory_, program_base_, start_address_}, maxwell3d{&maxwell3d_} { gpu_memory->ReadBlock(program_base + start_address, &sph, sizeof(sph)); initial_offset = sizeof(sph); + texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; + if (maxwell3d->GetNativeMacroEngine().native_turing) { + if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel) { + throw Shader::RuntimeError("Unsupported native graphics shader stage {}", static_cast<u32>(program)); + } + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + const u32 native_stage = program == Maxwell::ShaderType::Pixel ? 4 : 0; + const auto& table = maxwell3d->state.shader_stages[native_stage].const_buffers[1]; + if (table.enabled && table.size && table.size <= Tegra::Engines::Maxwell3D::Regs::MaxConstBufferSize) { + std::vector<u8> bytes(table.size); + gpu_memory->ReadBlock(table.address, bytes.data(), bytes.size()); + std::ofstream dump(fmt::format("native-graphics-cbuf-{}-1.bin", static_cast<u32>(program)), std::ios::binary); + dump.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); + } + } + InitializeNativeCode(); + } gp_passthrough_mask = maxwell3d->regs.post_vtg_shader_attrib_skip_mask; switch (program) { case Maxwell::ShaderType::VertexA: @@ -331,8 +519,7 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, const u64 local_size{sph.LocalMemorySize()}; ASSERT(local_size <= (std::numeric_limits<u32>::max)()); local_memory_size = static_cast<u32>(local_size) + sph.common3.shader_local_memory_crs_size; - texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; - is_proprietary_driver = texture_bound == 2; + is_proprietary_driver = maxwell3d->GetNativeMacroEngine().native_turing || texture_bound == 2; has_hle_engine_state = maxwell3d->engine_state == Tegra::Engines::Maxwell3D::EngineHint::OnHLEMacro; } @@ -417,6 +604,12 @@ ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_com is_proprietary_driver = texture_bound == 2; shared_memory_size = qmd.shared_alloc; workgroup_size = {qmd.block_dim_x, qmd.block_dim_y, qmd.block_dim_z}; + if (kepler_compute->IsNativeMode()) { + initial_offset = sizeof(sph); + is_proprietary_driver = true; + InitializeNativeCode(0); + start_address = sizeof(sph); + } } u32 ComputeEnvironment::ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) { @@ -511,7 +704,7 @@ void FileEnvironment::Deserialize(std::ifstream& file) { if (stage == Shader::Stage::Compute) { file.read(reinterpret_cast<char*>(&workgroup_size), sizeof(workgroup_size)) .read(reinterpret_cast<char*>(&shared_memory_size), sizeof(shared_memory_size)); - initial_offset = 0; + initial_offset = Shader::Ampere::HasNativeAddressMap(code) ? sizeof(sph) : 0; } else { file.read(reinterpret_cast<char*>(&sph), sizeof(sph)); initial_offset = sizeof(sph); @@ -526,6 +719,13 @@ void FileEnvironment::Dump(u64 pipeline_hash, u64 shader_hash) { DumpImpl(pipeline_hash, shader_hash, code, read_highest, read_lowest, initial_offset, stage); } +u32 FileEnvironment::TranslateProgramAddress(u32 address) const { + if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated; + if (Shader::Ampere::HasNativeAddressMap(code)) + throw Shader::RuntimeError("Invalid cached native indirect branch address {:#x}", address); + return address; +} + u64 FileEnvironment::ReadInstruction(u32 address) { if (address < read_lowest || address > read_highest) { throw Shader::LogicError("Out of bounds address {}", address); diff --git a/src/video_core/shader_environment.h b/src/video_core/shader_environment.h index 924e61ef46..e49f01d95f 100644 --- a/src/video_core/shader_environment.h +++ b/src/video_core/shader_environment.h @@ -47,6 +47,8 @@ public: [[nodiscard]] u64 ReadInstruction(u32 address) final; + [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override; + [[nodiscard]] std::optional<u64> Analyze(); void SetCachedSize(size_t size_bytes); @@ -57,6 +59,10 @@ public: [[nodiscard]] size_t ReadSizeBytes() const noexcept; + [[nodiscard]] size_t SourceSizeBytes() const noexcept { + return native_source_size; + } + [[nodiscard]] bool CanBeSerialized() const noexcept; [[nodiscard]] u64 CalculateHash() const; @@ -72,6 +78,9 @@ public: protected: std::optional<u64> TryFindSize(); + void InitializeNativeCode(u32 source_header_size = 128); + size_t native_source_size{}; + Tegra::Texture::TICEntry ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw); @@ -172,6 +181,8 @@ public: [[nodiscard]] u64 ReadInstruction(u32 address) override; + [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override; + [[nodiscard]] u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) override; [[nodiscard]] Shader::TextureType ReadTextureType(u32 handle) override; diff --git a/src/video_core/texture_cache/accelerated_swizzle.cpp b/src/video_core/texture_cache/accelerated_swizzle.cpp index 4c3f724d79..8ae0ed218d 100644 --- a/src/video_core/texture_cache/accelerated_swizzle.cpp +++ b/src/video_core/texture_cache/accelerated_swizzle.cpp @@ -37,6 +37,7 @@ BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams(const SwizzleParameter .x_shift = GOB_SIZE_SHIFT + block.height + block.depth, .block_height = block.height, .block_height_mask = (1U << block.height) - 1, + .native_tiling = info.native_tiling, }; } @@ -63,6 +64,7 @@ BlockLinearSwizzle3DParams MakeBlockLinearSwizzle3DParams(const SwizzleParameter .block_height_mask = (1U << block.height) - 1, .block_depth = block.depth, .block_depth_mask = (1U << block.depth) - 1, + .native_tiling = info.native_tiling, }; } diff --git a/src/video_core/texture_cache/accelerated_swizzle.h b/src/video_core/texture_cache/accelerated_swizzle.h index d4250da689..8296950c38 100644 --- a/src/video_core/texture_cache/accelerated_swizzle.h +++ b/src/video_core/texture_cache/accelerated_swizzle.h @@ -20,6 +20,7 @@ struct BlockLinearSwizzle2DParams { u32 x_shift; u32 block_height; u32 block_height_mask; + u32 native_tiling; }; struct BlockLinearSwizzle3DParams { @@ -33,6 +34,7 @@ struct BlockLinearSwizzle3DParams { u32 block_height_mask; u32 block_depth; u32 block_depth_mask; + u32 native_tiling; }; [[nodiscard]] BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams( diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp index d6e019cddf..20bb96aae3 100644 --- a/src/video_core/texture_cache/format_lookup_table.cpp +++ b/src/video_core/texture_cache/format_lookup_table.cpp @@ -157,6 +157,7 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red, return PixelFormat::S8_UINT_D24_UNORM; case Hash(TextureFormat::G24R8, UINT, UNORM, UNORM, UNORM, LINEAR): return PixelFormat::S8_UINT_D24_UNORM; + case Hash(TextureFormat::S8Z24, UNORM, UINT, UNORM, UNORM, LINEAR): case Hash(TextureFormat::S8Z24, UNORM, UINT, UINT, UINT, LINEAR): return PixelFormat::D24_UNORM_S8_UINT; case Hash(TextureFormat::Z32_X24S8, FLOAT, UINT, UNORM, UNORM, LINEAR): diff --git a/src/video_core/texture_cache/image_base.cpp b/src/video_core/texture_cache/image_base.cpp index 9b0f33220d..8db081debd 100644 --- a/src/video_core/texture_cache/image_base.cpp +++ b/src/video_core/texture_cache/image_base.cpp @@ -109,7 +109,9 @@ std::optional<SubresourceBase> ImageBase::TryFindBase(GPUVAddr other_addr) const } ImageViewId ImageBase::FindView(const ImageViewInfo& view_info) const noexcept { - const auto it = std::ranges::find(image_view_infos, view_info); + const auto it = std::ranges::find_if(image_view_infos, [&view_info](const ImageViewInfo& info) { + return info == view_info; + }); if (it == image_view_infos.end()) { return ImageViewId{}; } diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 79fc8505d6..47bab9617a 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -45,6 +45,11 @@ enum class ImageFlagBits : u32 { AsynchronousDecode = 1 << 16, IsDecoding = 1 << 17, ///< Is currently being decoded asynchronously. + Unbacked = 1 << 18, ///< Host image has no memory (too big to back); never upload/download. + /// The host image keeps only mip levels >= host_level_offset (its decoded copy would not fit + /// in VRAM); uploads skip the rest, downloads are never done. + TrimmedLevels = 1 << 19, + ZeroFilled = 1 << 20, ///< Cleared to zero blocks before its first asynchronous decode. }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) @@ -93,6 +98,7 @@ struct ImageBase { u32 unswizzled_size_bytes = 0; u32 converted_size_bytes = 0; u32 scale_rating = 0; + u32 host_level_offset = 0; ///< Guest mip level held by host mip 0 (see TrimmedLevels). u64 scale_tick = 0; bool has_scaled = false; diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp index 8b59eb6841..7e86bf00cf 100644 --- a/src/video_core/texture_cache/image_info.cpp +++ b/src/video_core/texture_cache/image_info.cpp @@ -4,9 +4,13 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> +#include <array> +#include <vector> #include <fmt/ranges.h> #include "common/assert.h" +#include "common/logging.h" #include "common/settings.h" #include "video_core/surface.h" #include "video_core/texture_cache/format_lookup_table.h" @@ -109,14 +113,23 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept { size.height = config.Height(); resources.layers = config.BaseLayer() + config.Depth() * 6; break; - case TextureType::Texture3D: + case TextureType::Texture3D: { ASSERT(config.BaseLayer() == 0); type = ImageType::e3D; size.width = config.Width(); size.height = config.Height(); size.depth = config.Depth(); resources.layers = 1; + // A few distinct 3D descriptors, raw (to check Switch 2 volume fields). + static std::vector<std::array<u64, 4>> logged; + if (logged.size() < 16 && std::find(logged.begin(), logged.end(), config.raw) == logged.end()) { + logged.push_back(config.raw); + LOG_INFO(HW_GPU, "3D TIC {:016x} {:016x} {:016x} {:016x}: {}x{}x{} block_depth {} levels {}", + config.raw[0], config.raw[1], config.raw[2], config.raw[3], size.width, size.height, + size.depth, config.block_depth.Value(), config.max_mip_level.Value() + 1); + } break; + } case TextureType::Texture1DBuffer: type = ImageType::Buffer; size.width = config.Width(); diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h index eb490a6424..79237aa482 100644 --- a/src/video_core/texture_cache/image_info.h +++ b/src/video_core/texture_cache/image_info.h @@ -14,6 +14,17 @@ namespace VideoCommon { using Tegra::Texture::TICEntry; using VideoCore::Surface::PixelFormat; +// Native descriptors address the selected view directly. The legacy NVN +// layer-base bookkeeping is not an additional array prefix for this ABI. +inline TICEntry TextureViewHeader(TICEntry entry, bool native) { + if (native) { + entry.layer_base_0_2.Assign(0); + entry.layer_base_3_7.Assign(0); + entry.layer_base_8_10.Assign(0); + } + return entry; +} + struct ImageInfo { ImageInfo() = default; explicit ImageInfo(const TICEntry& config) noexcept; @@ -37,11 +48,14 @@ struct ImageInfo { u32 maybe_unaligned_layer_stride = 0; u32 num_samples = 1; u32 tile_width_spacing = 0; + bool native_tiling = false; bool rescaleable = false; bool downscaleable = false; bool forced_flushed = false; bool dma_downloaded = false; bool is_sparse = false; + // ASTC data whose blocks carry no alpha: re-compressed to BC1 instead of BC3 (half the size). + bool astc_opaque = false; }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/pending_download.h b/src/video_core/texture_cache/pending_download.h new file mode 100644 index 0000000000..52a48e82d3 --- /dev/null +++ b/src/video_core/texture_cache/pending_download.h @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <algorithm> +#include <memory> +#include <unordered_set> +#include <cstddef> +#include <utility> +#include "common/slot_vector.h" + +namespace VideoCommon { + +template <class Image> +struct PendingImageDownload { + bool is_swizzle; + size_t async_buffer_id; + Common::SlotId object_id; + std::shared_ptr<Image> retired_image{}; + + Image& GetImage(Common::SlotVector<Image>& images) const { + return retired_image ? *retired_image : images[object_id]; + } +}; + +// Whole-image copies are recorded together at commit time. Repeated requests +// for the same live image therefore observe the same contents. Keep the first +// occurrence so reverse CPU writeback order remains unchanged for aliases. +template <class Image, class Downloads> +size_t DeduplicatePendingImageDownloads(Downloads& downloads, Common::SlotVector<Image>& images) { + std::unordered_set<const Image*> seen; + const size_t previous_size = downloads.size(); + const auto end = std::remove_if(downloads.begin(), downloads.end(), [&](const auto& download) { + return download.is_swizzle && !seen.insert(&download.GetImage(images)).second; + }); + downloads.erase(end, downloads.end()); + return previous_size - downloads.size(); +} + +// Cache slots may be recycled before queued GPU downloads are submitted or +// consumed. Keep the actual image, including its metadata, until those finish. +template <class Image, class Downloads, class Batches> +bool RetainPendingImageDownloads(Common::SlotId id, Image& image, Downloads& pending, + Batches& submitted) { + std::shared_ptr<Image> retained; + const auto retain = [&](auto& downloads) { + for (auto& download : downloads) { + if (!download.is_swizzle || download.object_id != id || download.retired_image) continue; + if (!retained) retained = std::make_shared<Image>(std::move(image)); + download.retired_image = retained; + } + }; + retain(pending); + for (auto& downloads : submitted) retain(downloads); + return retained != nullptr; +} + +} // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index e92e54f8ce..b95d9d8a39 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -6,7 +6,11 @@ #pragma once +#include <algorithm> +#include <cstdlib> #include <limits> +#include <map> +#include <vector> #include <optional> #include <bit> #include "common/container/unordered_map.h" @@ -24,6 +28,7 @@ #include "video_core/texture_cache/samples_helper.h" #include "video_core/texture_cache/texture_cache_base.h" #include "video_core/texture_cache/util.h" +#include "video_core/nx2_profile.h" #include "video_core/textures/decoders.h" namespace VideoCommon { @@ -76,6 +81,7 @@ TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB; minimum_memory = 0; } + ApplyThresholdOverrides(); const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); @@ -123,6 +129,12 @@ void TextureCache<P>::RunGarbageCollector() { high_priority_mode = total_used_memory >= expected_memory; aggressive_mode = allow_aggressive && total_used_memory >= critical_memory; ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL; + // EDEN_NX2_TC_MIN_AGE=<frames>: never evict images used more recently (thrash tests). + static const u64 min_age = [] { + const char* value = std::getenv("EDEN_NX2_TC_MIN_AGE"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL; + }(); + ticks_to_destroy = (std::max)(ticks_to_destroy, min_age); num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); }; const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { @@ -144,10 +156,12 @@ void TextureCache<P>::RunGarbageCollector() { image.DownloadMemory(map, copies); runtime.Finish(); SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer); + ++gc_download_deletes; } if (True(image.flags & ImageFlagBits::Tracked)) { UntrackImage(image, image_id); } + ++(True(image.flags & ImageFlagBits::CostlyLoad) ? gc_costly_deletes : gc_deletes); UnregisterImage(image_id); DeleteImage(image_id, image.scale_tick > frame_tick + 5); if (aggressive_mode && total_used_memory < critical_memory) { @@ -167,12 +181,120 @@ void TextureCache<P>::RunGarbageCollector() { } } +template <class P> +void TextureCache<P>::ApplyThresholdOverrides(bool log) { + // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds. + for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory}, + std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) { + if (const char* value = std::getenv(name)) { + *target = std::strtoull(value, nullptr, 10) << 20; + } + } + if (log) { + LOG_INFO(HW_GPU, "Texture cache thresholds: expected {} MiB, critical {} MiB, minimum {} MiB", + expected_memory >> 20, critical_memory >> 20, minimum_memory >> 20); + } +} + template <class P> void TextureCache<P>::TickFrame() { + if constexpr (HAS_DEVICE_MEMORY_INFO) { + // Switch 2 titles (native tiling) stream far more texture data than Eden's thresholds + // were tuned for: title 1's attract loop alone keeps ~5.3 GB resident, so the collector + // sat in aggressive mode evicting decoded ASTC (and decoding it again). Total use is + // measured from the driver, so collect only near the device budget instead. + if (!native_thresholds_applied && gpu_memory != nullptr && gpu_memory->UsesNativeTiling()) { + native_thresholds_applied = true; + const u64 device_local = runtime.GetDeviceLocalMemory(); + critical_memory = (std::max)(critical_memory, device_local - 256_MiB); + expected_memory = (std::max)(expected_memory, device_local - 1280_MiB); + ApplyThresholdOverrides(); + native_critical_memory = critical_memory; + native_expected_memory = expected_memory; + } + // Follow the driver's budget (re-read now and then): it shrinks when other programs use + // VRAM (another game: 7.0 -> 6.4 GB), and collecting only past it means the driver pages + // textures out. It is also the room to use: Eden's device memory figure is capped near + // 7 GB, which on a 12 GB card (11.2 GB budget) kept title 1 evicting textures with 3 GB + // unused. EDEN_NX2_TC_*_MIB overrides still win. + if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + if (native_thresholds_applied && budget_check_frames++ % 60 == 0) { + const u64 budget = runtime.GetDeviceMemoryBudget(); + if (budget > 2_GiB) { + const u64 old_critical = critical_memory; + critical_memory = budget - 768_MiB; + expected_memory = budget - 1536_MiB; + // Log only real changes (the budget moves by a few MiB all the time). + const u64 change = critical_memory > old_critical + ? critical_memory - old_critical + : old_critical - critical_memory; + ApplyThresholdOverrides(change >= 256_MiB); + } + } + } + } // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); } + // Every 600 frames: device memory in use against the collector's thresholds, and what the + // images themselves account for (VRAM pressure makes the collector thrash). + if (static u64 frames{}; ++frames % 600 == 0) { + u64 image_bytes{}; + size_t images{}; + std::vector<std::pair<u64, const Image*>> largest; + std::map<u32, std::pair<u64, u32>> by_format; + for (const auto [image_id, image] : slot_images) { + const u64 bytes = GetScaledImageSizeBytes(*image); + image_bytes += bytes; + ++images; + largest.emplace_back(bytes, image); + auto& format = by_format[static_cast<u32>(image->info.format)]; + format.first += bytes; + ++format.second; + } + LOG_INFO(HW_GPU, "Texture cache: device memory {} MiB (expected {} MiB, critical {} MiB), {} images {} MiB", + total_used_memory >> 20, expected_memory >> 20, critical_memory >> 20, images, + image_bytes >> 20); + std::partial_sort(largest.begin(), largest.begin() + std::min<size_t>(12, largest.size()), + largest.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + for (size_t i = 0; i < std::min<size_t>(12, largest.size()); ++i) { + const Image& image = *largest[i].second; + LOG_INFO(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}", + largest[i].first >> 20, image.gpu_addr, image.info.size.width, image.info.size.height, + image.info.size.depth, image.info.resources.layers, image.info.resources.levels, + static_cast<u32>(image.info.format), static_cast<u32>(image.info.type), + static_cast<u32>(image.flags)); + } + std::vector<std::pair<u64, u32>> formats; + for (const auto& [format, total] : by_format) formats.emplace_back(total.first, format); + std::sort(formats.rbegin(), formats.rend()); + for (size_t i = 0; i < std::min<size_t>(8, formats.size()); ++i) { + LOG_INFO(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20, + by_format[formats[i].second].second); + } + if constexpr (requires { runtime.LogMemoryStatistics(); }) { + runtime.LogMemoryStatistics(); + } + LOG_INFO(HW_GPU, "Texture cache last 600 frames: created {} images, deleted {} (collector {}, {} costly, {} downloaded), {} async decodes, ASTC {} opaque / {} alpha", + images_created, images_deleted, gc_deletes + gc_costly_deletes, gc_costly_deletes, + gc_download_deletes, + async_decodes_queued, astc_opaque_images, astc_alpha_images); + astc_opaque_images = astc_alpha_images = 0; + std::vector<std::pair<u32, u64>> shapes; + for (const auto& [key, value] : created_shapes) shapes.emplace_back(value.first, key); + std::sort(shapes.rbegin(), shapes.rend()); + for (size_t i = 0; i < std::min<size_t>(10, shapes.size()); ++i) { + const u64 key = shapes[i].second; + LOG_INFO(HW_GPU, " created {}x: format {} type {} levels {} layers {} {}x{} (last at {:#x})", + shapes[i].first, key >> 56, (key >> 52) & 0xF, (key >> 48) & 0xF, + (key >> 36) & 0xFFF, (key >> 18) & 0x3FFFF, key & 0x3FFFF, + created_shapes[key].second); + } + created_shapes.clear(); + gc_deletes = gc_costly_deletes = gc_download_deletes = async_decodes_queued = 0; + images_created = images_deleted = 0; + } if (total_used_memory > minimum_memory) { RunGarbageCollector(); } @@ -597,7 +719,7 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) { template <class P> bool TextureCache<P>::IsDownloadable(const ImageBase& image) const noexcept { - if (!image.IsSafeGpuCopy()) { + if (!image.IsSafeGpuCopy() || True(image.flags & (ImageFlagBits::Unbacked | ImageFlagBits::TrimmedLevels))) { return false; } if (image.info.num_samples == 1) { @@ -714,6 +836,14 @@ bool TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, Image& dst_image = slot_images[dst_id]; Image& src_image = slot_images[src_id]; + if (True(dst_image.flags & ImageFlagBits::Unbacked)) { + static u32 logged{}; + if (logged++ < 64) { + LOG_INFO(HW_GPU, "2D blit into unbacked image {:#x}: dst {:#x} {}x{} from {:#x} {}x{}", + dst_image.gpu_addr, dst.Address(), dst.width, dst.height, src.Address(), + src.width, src.height); + } + } bool is_src_rescaled = True(src_image.flags & ImageFlagBits::Rescaled); bool is_dst_rescaled = True(dst_image.flags & ImageFlagBits::Rescaled); @@ -803,6 +933,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage if (it == page_table.end()) { return {}; } + const auto view_format = [&]() { + switch (config.pixel_format) { + case Service::android::PixelFormat::Rgb565: + return PixelFormat::R5G6B5_UNORM; + case Service::android::PixelFormat::Bgra8888: + return PixelFormat::B8G8R8A8_UNORM; + default: + return PixelFormat::A8B8G8R8_UNORM; + } + }(); + const auto& image_map_ids = it->second; boost::container::small_vector<ImageId, 4> valid_image_ids; for (const ImageMapId map_id : image_map_ids) { @@ -814,20 +955,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage if (image.image_view_ids.empty()) { continue; } + if (!VideoCore::Surface::IsViewCompatible(image.info.format, view_format, false, true)) { + static u32 rejected_alias_count{}; + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && rejected_alias_count++ < 8) { + LOG_WARNING(HW_GPU, "Ignoring incompatible framebuffer alias: address={:#x}, image={}, view={}", + cpu_addr, image.info.format, view_format); + } + continue; + } valid_image_ids.push_back(map.image_id); } - const auto view_format = [&]() { - switch (config.pixel_format) { - case Service::android::PixelFormat::Rgb565: - return PixelFormat::R5G6B5_UNORM; - case Service::android::PixelFormat::Bgra8888: - return PixelFormat::B8G8R8A8_UNORM; - default: - return PixelFormat::A8B8G8R8_UNORM; - } - }(); - const auto GetImageViewForFramebuffer = [&](ImageId image_id) { ImageViewInfo info{ImageViewType::e2D, view_format}; if (config.blending == Tegra::BlendMode::Opaque) { @@ -859,6 +997,11 @@ bool TextureCache<P>::HasUncommittedFlushes() const noexcept { return !uncommitted_downloads.empty(); } +template <class P> +bool TextureCache<P>::HasAsyncFlushWork() const noexcept { + return !uncommitted_downloads.empty() || !uncommitted_async_buffers.empty(); +} + template <class P> bool TextureCache<P>::ShouldWaitAsyncFlushes() const noexcept { return !committed_downloads.empty() && !committed_downloads.front().empty(); @@ -869,6 +1012,7 @@ void TextureCache<P>::CommitAsyncFlushes() { // This is intentionally passing the value by copy if constexpr (IMPLEMENTS_ASYNC_DOWNLOADS) { auto& download_ids = uncommitted_downloads; + const size_t redundant_images = DeduplicatePendingImageDownloads(download_ids, slot_images); if (download_ids.empty()) { committed_downloads.emplace_back(std::move(uncommitted_downloads)); uncommitted_downloads.clear(); @@ -882,17 +1026,32 @@ void TextureCache<P>::CommitAsyncFlushes() { for (PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { total_size_bytes += - Common::AlignUp(slot_images[download_info.object_id].unswizzled_size_bytes, 64); + Common::AlignUp(download_info.GetImage(slot_images).unswizzled_size_bytes, 64); any_none_dma = true; download_info.async_buffer_id = last_async_buffer_id; } } if (any_none_dma) { + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && total_size_bytes >= 64_MiB) { + LOG_WARNING(HW_GPU, "Large image readback batch: bytes={}, entries={}, duplicates_removed={}, submitted_batches={}, retained_buffers={}", + total_size_bytes, download_ids.size(), redundant_images, + committed_downloads.size(), async_buffers_death_ring.size()); + size_t traced_images = 0; + for (const auto& download : download_ids) { + if (!download.is_swizzle || traced_images++ >= 8) continue; + const auto& image = download.GetImage(slot_images); + LOG_WARNING(HW_GPU, "Readback image: gpu={:#x}, cpu={:#x}, format={}, size={}x{}x{}, levels={}, layers={}, layer_stride={}, guest_bytes={}, download_bytes={}", + image.gpu_addr, image.cpu_addr, image.info.format, + image.info.size.width, image.info.size.height, image.info.size.depth, + image.info.resources.levels, image.info.resources.layers, + image.info.layer_stride, image.guest_size_bytes, image.unswizzled_size_bytes); + } + } auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true); for (const PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { - Image& image = slot_images[download_info.object_id]; + Image& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); image.DownloadMemory(download_map, copies); download_map.offset += Common::AlignUp(image.unswizzled_size_bytes, 64); @@ -925,7 +1084,7 @@ void TextureCache<P>::PopAsyncFlushes() { auto& download_info = download_ids[i - 1]; auto& download_buffer = download_map[download_info.async_buffer_id]; if (download_info.is_swizzle) { - const ImageBase& image = slot_images[download_info.object_id]; + const ImageBase& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); download_buffer.offset -= Common::AlignUp(image.unswizzled_size_bytes, 64); std::span<u8> download_span = @@ -954,7 +1113,7 @@ void TextureCache<P>::PopAsyncFlushes() { size_t total_size_bytes = 0; for (const PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { - total_size_bytes += slot_images[download_info.object_id].unswizzled_size_bytes; + total_size_bytes += download_info.GetImage(slot_images).unswizzled_size_bytes; } } auto download_map = runtime.DownloadStagingBuffer(total_size_bytes); @@ -963,7 +1122,7 @@ void TextureCache<P>::PopAsyncFlushes() { if (!download_info.is_swizzle) { continue; } - Image& image = slot_images[download_info.object_id]; + Image& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); image.DownloadMemory(download_map, copies); download_map.offset += image.unswizzled_size_bytes; @@ -976,7 +1135,7 @@ void TextureCache<P>::PopAsyncFlushes() { if (!download_info.is_swizzle) { continue; } - const ImageBase& image = slot_images[download_info.object_id]; + const ImageBase& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span, swizzle_data_buffer); download_map.offset += image.unswizzled_size_bytes; @@ -1117,11 +1276,18 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) { // Only upload modified images return; } + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::TextureUpload}; image.flags &= ~ImageFlagBits::CpuModified; TrackImage(image, image_id); + if (True(image.flags & ImageFlagBits::Unbacked)) { + // Nothing to upload into (and staging it could take tens of GB). + runtime.TransitionImageLayout(image); + return; + } + if (image.info.num_samples > 1 && !runtime.CanUploadMSAA()) { LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented"); runtime.TransitionImageLayout(image); @@ -1156,6 +1322,24 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) const std::span<u8> mapped_span = staging.mapped_span; const GPUVAddr gpu_addr = image.gpu_addr; + if (True(image.flags & ImageFlagBits::AcceleratedUpload) && image.host_level_offset != 0) { + // Only levels >= host_level_offset exist on the host: read each layer's tail of the mip + // chain into a compact buffer (layer stride = that tail's size) and decode those levels. + auto uploads = FullUploadSwizzles(image.info); + const size_t level_offset = uploads[image.host_level_offset].buffer_offset; + const size_t tail = image.info.layer_stride - level_offset; + for (s32 layer = 0; layer < image.info.resources.layers; ++layer) { + gpu_memory->ReadBlock(gpu_addr + layer * image.info.layer_stride + level_offset, + mapped_span.data() + layer * tail, tail, + VideoCommon::CacheType::NoTextureCache); + } + uploads.erase(uploads.begin(), uploads.begin() + image.host_level_offset); + for (auto& upload : uploads) { + upload.buffer_offset -= level_offset; + } + runtime.AccelerateImageUpload(image, staging, uploads, 0, 0); + return; + } if (True(image.flags & ImageFlagBits::AcceleratedUpload)) { gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(), VideoCommon::CacheType::NoTextureCache); @@ -1178,7 +1362,18 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) } template <class P> -ImageViewId TextureCache<P>::CreateImageView(const TICEntry& config) { +ImageViewId TextureCache<P>::CreateImageView(const TICEntry& raw_config) { + const bool native = gpu_memory->UsesNativeTiling(); + if (native && raw_config.BaseLayer() && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + static u32 texture_view_traces{}; + if (texture_view_traces++ < 8) { + LOG_WARNING(HW_GPU, "Native texture view header: address={:#x}, dimensions={}x{}x{}, type={}, legacy_layer={}, raw={:016x}/{:016x}/{:016x}/{:016x}", + raw_config.Address(), raw_config.Width(), raw_config.Height(), raw_config.Depth(), + static_cast<u32>(raw_config.texture_type.Value()), raw_config.BaseLayer(), + raw_config.raw[0], raw_config.raw[1], raw_config.raw[2], raw_config.raw[3]); + } + } + const TICEntry config = TextureViewHeader(raw_config, native); const ImageInfo info(config); if (info.type == ImageType::Buffer) { const ImageViewInfo view_info(config, 0); @@ -1213,12 +1408,9 @@ ImageId TextureCache<P>::FindOrInsertImage(const ImageInfo& info, GPUVAddr gpu_a template <class P> ImageId TextureCache<P>::FindImage(const ImageInfo& info, GPUVAddr gpu_addr, RelaxedOptions options) { - std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + const std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); if (!cpu_addr) { - cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); - if (!cpu_addr) { - return ImageId{}; - } + return ImageId{}; } const bool broken_views = runtime.HasBrokenTextureViewFormats() || True(options & RelaxedOptions::ForceBrokenViews); @@ -1344,9 +1536,10 @@ u64 TextureCache<P>::GetScaledImageSizeBytes(const ImageBase& image) { template <class P> void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) { UNIMPLEMENTED_IF(False(image.flags & ImageFlagBits::Converted)); - LOG_INFO(HW_GPU, "Queuing async texture decode"); + LOG_DEBUG(HW_GPU, "Queuing async texture decode {:#x}", image.gpu_addr); image.flags |= ImageFlagBits::IsDecoding; + ++async_decodes_queued; auto decode = std::make_unique<AsyncDecodeContext>(); auto* decode_ptr = decode.get(); decode->image_id = image_id; @@ -1356,6 +1549,18 @@ void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) { Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::UnsafeRead> swizzle_data(*gpu_memory, image.gpu_addr, image.guest_size_bytes, &swizzle_data_buffer); auto copies = UnswizzleImage(*gpu_memory, image.gpu_addr, image.info, swizzle_data, local_unswizzle_data_buffer); const size_t out_size = MapSizeBytes(image); + if (False(image.flags & ImageFlagBits::ZeroFilled)) { + // Until the decode lands (a second or more while title 1 streams ~40 textures a second) + // a new image showed whatever its memory last held, often other textures. Zero blocks + // read as opaque black (BC1) or transparent (BC3) instead. + image.flags |= ImageFlagBits::ZeroFilled; + auto zero_copies = copies; + ConvertedLayout(image.info, zero_copies); + auto staging = runtime.UploadStagingBuffer(out_size); + std::memset(staging.mapped_span.data(), 0, out_size); + image.UploadMemory(staging, zero_copies); + runtime.InsertUploadMemoryBarrier(); + } auto func = [out_size, copies, info = image.info, input = std::move(local_unswizzle_data_buffer), @@ -1392,12 +1597,26 @@ void TextureCache<P>::TickAsyncDecode() { auto i = async_decodes.begin(); while (i != async_decodes.end()) { auto* async_decode = i->get(); - std::unique_lock lock{async_decode->mutex}; - if (!async_decode->complete) { - ++i; + { + // Release the lock before the context is erased below: unlocking it afterwards wrote + // into freed memory (heap corruption, crashes minutes into title 1). + std::unique_lock lock{async_decode->mutex}; + if (!async_decode->complete) { + ++i; + continue; + } + } + if (async_decode->cancelled) { + i = async_decodes.erase(i); continue; } Image& image = slot_images[async_decode->image_id]; + if (async_decode->decoded_data.size() > MapSizeBytes(image)) { + LOG_ERROR(HW_GPU, "Async decode of {} bytes for a {}-byte image at {:#x}, dropped", + async_decode->decoded_data.size(), MapSizeBytes(image), image.gpu_addr); + i = async_decodes.erase(i); + continue; + } auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); std::memcpy(staging.mapped_span.data(), async_decode->decoded_data.data(), async_decode->decoded_data.size()); @@ -1512,21 +1731,43 @@ bool TextureCache<P>::ScaleDown(Image& image) { return true; } +template <class P> +std::optional<DAddr> TextureCache<P>::ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const { + if (const std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr)) { + return cpu_addr; + } + if (const std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, size)) { + return cpu_addr; + } + // No CPU backing (title 1 renders a bloom chain at GPU 0xf0000-0x15b600): a stand-in address + // fixed by the GPU address, at or above ~(1 << 40) so TrackImage skips it. A fresh address per + // lookup made every use create another image (thousands a minute, never found again). + constexpr GPUVAddr StandInRange = 1ULL << 40; + if (gpu_addr >= StandInRange || size > StandInRange - gpu_addr - 1) { + return std::nullopt; + } + return ~(1ULL << 40ULL) + gpu_addr; +} + template <class P> ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, RelaxedOptions options) { - std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); if (!cpu_addr) { - const auto size = CalculateGuestSizeInBytes(info); - cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, size); - if (!cpu_addr) { - const DAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space; - virtual_invalid_space += Common::AlignUp(size, 32); - cpu_addr = std::optional<DAddr>(fake_addr); - } + const DAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space; + virtual_invalid_space += Common::AlignUp(CalculateGuestSizeInBytes(info), 32); + cpu_addr = std::optional<DAddr>(fake_addr); } ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr={:#x}", gpu_addr); - const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr); + ImageInfo storage_info = info; + storage_info.native_tiling = gpu_memory->UsesNativeTiling(); + if (storage_info.native_tiling && info.type == ImageType::e3D) { + // Switch 2 3D images (lighting/fog volumes) are rare: log each one's shape. + LOG_INFO(HW_GPU, "3D image at {:#x}: {}x{}x{} levels {} format {} block {}x{}x{}", + gpu_addr, info.size.width, info.size.height, info.size.depth, info.resources.levels, + static_cast<u32>(info.format), info.block.width, info.block.height, info.block.depth); + } + const ImageId image_id = JoinImages(storage_info, gpu_addr, *cpu_addr); const Image& image = slot_images[image_id]; // Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr); @@ -1634,10 +1875,36 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA } } + if constexpr (requires { runtime.UseOpaqueAstcFormat(); }) { + // Opaque ASTC goes to BC1 rather than BC3 when it is re-compressed (the Vulkan runtime + // decides). The guest data is already there when an image is first used. + constexpr size_t MaxScanBytes = 64_MiB; + if (IsPixelFormatASTC(new_info.format) && runtime.UseOpaqueAstcFormat()) { + const size_t scan_bytes = CalculateGuestSizeInBytes(new_info); + if (scan_bytes <= MaxScanBytes) { + astc_scan_buffer.resize(scan_bytes); + gpu_memory->ReadBlockUnsafe(gpu_addr, astc_scan_buffer.data(), scan_bytes); + new_info.astc_opaque = AstcDataLooksOpaque(astc_scan_buffer); + ++(new_info.astc_opaque ? astc_opaque_images : astc_alpha_images); + } + } + } const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr); Image& new_image = slot_images[new_image_id]; new_image.allocation_tick = frame_tick; + { + // Which shapes are being created (periodic log): format, type, levels, layers, size. + const u64 key = (u64{static_cast<u32>(new_info.format)} << 56) | + (u64{static_cast<u32>(new_info.type)} << 52) | + (u64{static_cast<u32>(new_info.resources.levels) & 0xF} << 48) | + (u64{static_cast<u32>(new_info.resources.layers) & 0xFFF} << 36) | + (u64{new_info.size.width & 0x3FFFF} << 18) | (new_info.size.height & 0x3FFFF); + auto& shape = created_shapes[key]; + ++shape.first; + shape.second = gpu_addr; + ++images_created; + } if (!gpu_memory->IsContinuousRange(new_image.gpu_addr, new_image.guest_size_bytes) && new_info.is_sparse) { @@ -1833,12 +2100,9 @@ std::optional<typename TextureCache<P>::BlitImages> TextureCache<P>::GetBlitImag template <class P> ImageId TextureCache<P>::FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr) { - std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + const std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); if (!cpu_addr) { - cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); - if (!cpu_addr) { - return ImageId{}; - } + return ImageId{}; } ImageId image_id{}; boost::container::small_vector<ImageId, 8> image_ids; @@ -1859,6 +2123,13 @@ ImageId TextureCache<P>::FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr) return true; } } else if (IsSubCopy(info, existing_image, gpu_addr)) { + if (True(existing_image.flags & ImageFlagBits::Unbacked)) { + static u32 logged{}; + if (logged++ < 64) { + LOG_INFO(HW_GPU, "DMA copy at {:#x} ({}x{}) into unbacked image {:#x}", gpu_addr, + info.size.width, info.size.height, existing_image.gpu_addr); + } + } image_id = existing_image_id; image_ids.push_back(existing_image_id); return true; @@ -1954,6 +2225,13 @@ ImageViewId TextureCache<P>::FindRenderTargetView(const ImageInfo& info, GPUVAdd base = image.TryFindBase(gpu_addr).value(); } const s32 layers = image.info.type == ImageType::e3D ? info.size.depth : info.resources.layers; + if (True(image.flags & ImageFlagBits::Unbacked)) { + static u32 logged{}; + if (logged++ < 64) { + LOG_INFO(HW_GPU, "Render target in unbacked image {:#x} at {:#x}: {}x{} level {} layer {}", + image.gpu_addr, gpu_addr, info.size.width, info.size.height, base.level, base.layer); + } + } const SubresourceRange range{ .base = base, .extent = {.levels = 1, .layers = layers}, @@ -2136,6 +2414,13 @@ ImageViewId TextureCache<P>::FindOrEmplaceImageView(ImageId image_id, const Imag if (const ImageViewId image_view_id = image.FindView(info); image_view_id) { return image_view_id; } + if (True(image.flags & ImageFlagBits::Unbacked)) { + // How the game uses an image too large to back (e.g. a sparse virtual texture). + LOG_INFO(HW_GPU, "View of unbacked image {:#x}: type {} format {} levels {}+{} layers {}+{}", + image.gpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format), + info.range.base.level, info.range.extent.levels, info.range.base.layer, + info.range.extent.layers); + } const ImageViewId image_view_id = slot_image_views.insert(runtime, info, image_id, image, slot_images); image.InsertView(info, image_view_id); return image_view_id; @@ -2316,6 +2601,16 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) { template <class P> void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { ImageBase& image = slot_images[image_id]; + ++images_deleted; + // Drop every decode still queued for it, whatever IsDecoding says: an image re-decoded while + // a decode was in flight has two, and the first to finish clears the flag. A decode left + // behind was uploaded into the next image to take this slot (another texture's data: garbage + // textures, and a staging overrun when that image was smaller). + for (auto& async_decode : async_decodes) { + if (async_decode->image_id == image_id) { + async_decode->cancelled = true; + } + } if (image.HasScaled()) { total_used_memory -= GetScaledImageSizeBytes(image); } @@ -2385,7 +2680,11 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { } slot_image_views.erase(image_view_id); } - if (!immediate_delete) { + const bool retained = RetainPendingImageDownloads(image_id, slot_images[image_id], + uncommitted_downloads, committed_downloads); + if (retained) { + LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); + } else if (!immediate_delete) { sentenced_images.Push(std::move(slot_images[image_id])); } slot_images.erase(image_id); @@ -2537,6 +2836,11 @@ template <class P> void TextureCache<P>::CopyImage(ImageId dst_id, ImageId src_id, std::vector<ImageCopy> copies) { Image& dst = slot_images[dst_id]; Image& src = slot_images[src_id]; + if (dst.info.astc_opaque != src.info.astc_opaque) { + // BC1 and BC3 blocks differ in size, so the host can't copy between them; both hold the + // same guest data, which is uploaded from guest memory instead. + return; + } const bool is_rescaled = True(src.flags & ImageFlagBits::Rescaled); if (is_rescaled) { ASSERT(True(dst.flags & ImageFlagBits::Rescaled)); @@ -2628,7 +2932,8 @@ void TextureCache<P>::BindRenderTarget(ImageViewId* old_id, ImageViewId new_id) } if (new_id) { const ImageViewBase& old_view = slot_image_views[new_id]; - if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload)) { + if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload) && + False(slot_images[old_view.image_id].flags & (ImageFlagBits::Unbacked | ImageFlagBits::TrimmedLevels))) { const PendingDownload new_download{true, 0, old_view.image_id}; uncommitted_downloads.emplace_back(new_download); } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 8a2318cc72..2e375595de 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -9,6 +9,7 @@ #include <atomic> #include <deque> #include <limits> +#include <map> #include <mutex> #include <span> #include <type_traits> @@ -22,12 +23,14 @@ #include <queue> #include "common/common_types.h" +#include "common/spin_recursive_mutex.h" #include "common/hash.h" #include "common/literals.h" #include "common/lru_cache.h" #include <ranges> #include "common/scratch_buffer.h" #include "common/slot_vector.h" +#include "video_core/texture_cache/pending_download.h" #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" @@ -67,6 +70,7 @@ struct AsyncDecodeContext { boost::container::small_vector<BufferImageCopy, 16> copies; std::mutex mutex; std::atomic_bool complete; + bool cancelled{}; ///< The image was deleted meanwhile: drop the result (GPU thread only). }; using TextureCacheGPUMap = ::Common::unordered_map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>; @@ -118,6 +122,22 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI static constexpr s64 TARGET_THRESHOLD = 4_GiB; #endif + // Periodic statistics (TickFrame). + u64 gc_deletes{}; + u64 gc_costly_deletes{}; + u64 gc_download_deletes{}; ///< Evictions that downloaded the image first (a full GPU wait) + u64 async_decodes_queued{}; + u64 images_created{}; + bool native_thresholds_applied{}; + u64 native_critical_memory{}; + u64 native_expected_memory{}; + u64 budget_check_frames{}; + u64 astc_opaque_images{}; + u64 astc_alpha_images{}; + std::vector<u8> astc_scan_buffer; + u64 images_deleted{}; + std::map<u64, std::pair<u32, GPUVAddr>> created_shapes; + static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; static constexpr size_t GC_EMERGENCY_COUNTS = 2; @@ -226,6 +246,9 @@ public: /// Return true when there are uncommitted images to be downloaded [[nodiscard]] bool HasUncommittedFlushes() const noexcept; + /// Whether CommitAsyncFlushes would commit anything (downloads or their buffers). + [[nodiscard]] bool HasAsyncFlushWork() const noexcept; + /// Return true when the caller should wait for async downloads [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept; @@ -258,7 +281,7 @@ public: /// Prepare an image to be used void PrepareImage(ImageId image_id, bool is_modification, bool invalidate); - std::recursive_mutex mutex; + Common::SpinRecursiveMutex mutex; private: /// Iterate over all page indices in a range @@ -448,6 +471,12 @@ private: DAddr virtual_invalid_space{}; + /// Applies EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB and logs the thresholds. + void ApplyThresholdOverrides(bool log = true); + + /// The CPU address an image at gpu_addr is keyed by: its backing, or a fixed stand-in. + [[nodiscard]] std::optional<DAddr> ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const; + bool has_deleted_images = false; bool is_rescaling = false; u64 total_used_memory = 0; @@ -463,11 +492,7 @@ private: size_t size; }; - struct PendingDownload { - bool is_swizzle; - size_t async_buffer_id; - Common::SlotId object_id; - }; + using PendingDownload = PendingImageDownload<Image>; Common::SlotVector<Image> slot_images; Common::SlotVector<ImageMapView> slot_map_views; diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index e55d0752ec..ffc1d59ec3 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -11,6 +11,7 @@ #include <algorithm> #include <array> +#include <cstring> #include <numeric> #include <optional> #include <span> @@ -439,7 +440,7 @@ template <u32 GOB_EXTENT> } layers = 1; } else { - layers = (std::max)(resources.layers, info.resources.layers + base->layer); + layers = (std::max)(info.resources.layers, resources.layers + base->layer); } return OverlapResult{ .gpu_addr = overlap.gpu_addr, @@ -566,7 +567,7 @@ void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr dst(gpu_memory, gpu_addr + guest_offset, subresource_size, &tmp_buffer); SwizzleTexture(dst, src, bytes_per_block, num_tiles.width, num_tiles.height, - num_tiles.depth, block.height, block.depth); + num_tiles.depth, block.height, block.depth, 1, gpu_memory.UsesNativeTiling()); } host_offset += host_bytes_per_layer; @@ -603,6 +604,10 @@ u32 CalculateUnswizzledSizeBytes(const ImageInfo& info) noexcept { return NumBlocksPerLayer(info, tile_size) * info.resources.layers * BytesPerBlock(info.format); } +bool AstcDataLooksOpaque(std::span<const u8> data) noexcept { + return Tegra::Texture::ASTC::IsOpaque(data); +} + u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept { if (info.type == ImageType::Buffer) { return info.size.width * BytesPerBlock(info.format); @@ -611,8 +616,10 @@ u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept { if (IsPixelFormatASTC(info.format) && Settings::values.astc_recompression.GetValue() != Settings::AstcRecompression::Uncompressed) { const u32 bpp_div = - Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc1 ? 2 - : 1; + Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc1 || + info.astc_opaque + ? 2 + : 1; // NumBlocksPerLayer doesn't account for this correctly, so we have to do it manually. u32 output_size = 0; for (s32 i = 0; i < info.resources.levels; i++) { @@ -913,7 +920,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory const std::span<u8> dst = output.subspan(host_offset); const std::span<const u8> src = input.subspan(guest_offset + guest_layer_offset); UnswizzleTexture(dst, src, 1U << bpp_log2, num_tiles.width, num_tiles.height, - num_tiles.depth, block.height, block.depth, stride_alignment); + num_tiles.depth, block.height, block.depth, stride_alignment, gpu_memory.UsesNativeTiling()); guest_layer_offset += layer_stride; host_offset += host_bytes_per_layer; } @@ -922,6 +929,35 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory return copies; } +void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies) { + // Same offsets and sizes as ConvertImage below. + const auto recompression = Settings::values.astc_recompression.GetValue(); + const bool astc = IsPixelFormatASTC(info.format); + u64 output_offset = 0; + for (BufferImageCopy& copy : copies) { + const Extent3D mip_size = AdjustMipSize(info.size, copy.image_subresource.base_level); + const u64 layers = static_cast<u64>(copy.image_subresource.num_layers) * copy.image_extent.depth; + u64 size; + if (astc && recompression == Settings::AstcRecompression::Uncompressed) { + size = u64{copy.image_extent.width} * copy.image_extent.height * layers * + BytesPerBlock(PixelFormat::A8B8G8R8_UNORM); + } else if (astc) { + const u64 bpp_div = + recompression == Settings::AstcRecompression::Bc1 || info.astc_opaque ? 2 : 1; + size = u64{Common::AlignUp(copy.image_extent.width, 4U)} * + Common::AlignUp(copy.image_extent.height, 4U) * layers / bpp_div; + } else { + size = u64{copy.image_extent.width} * copy.image_extent.height * layers * + ConvertedBytesPerBlock(info.format); + } + copy.buffer_offset = output_offset; + copy.buffer_size = size; + copy.buffer_row_length = mip_size.width; + copy.buffer_image_height = mip_size.height; + output_offset += size; + } +} + void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8> output, std::span<BufferImageCopy> copies) { u32 output_offset = 0; @@ -955,10 +991,11 @@ void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8 } else if (astc) { // BC1 uses 0.5 bytes per texel // BC3 uses 1 byte per texel - const auto compress = recompression_setting == Settings::AstcRecompression::Bc1 - ? Tegra::Texture::BCN::CompressBC1 - : Tegra::Texture::BCN::CompressBC3; - const auto bpp_div = recompression_setting == Settings::AstcRecompression::Bc1 ? 2 : 1; + const bool bc1 = + recompression_setting == Settings::AstcRecompression::Bc1 || info.astc_opaque; + const auto compress = + bc1 ? Tegra::Texture::BCN::CompressBC1 : Tegra::Texture::BCN::CompressBC3; + const auto bpp_div = bc1 ? 2 : 1; const u32 plane_dim = copy.image_extent.width * copy.image_extent.height; const u32 level_size = plane_dim * copy.image_extent.depth * @@ -1298,6 +1335,13 @@ void DeduceBlitImages(ImageInfo& dst_info, ImageInfo& src_info, const ImageBase* } u32 MapSizeBytes(const ImageBase& image) { + if (True(image.flags & ImageFlagBits::AcceleratedUpload) && image.host_level_offset != 0) { + // Each layer's mip chain from host_level_offset on (see UploadImageContents). + const auto uploads = FullUploadSwizzles(image.info); + const u32 tail = image.info.layer_stride - + static_cast<u32>(uploads[image.host_level_offset].buffer_offset); + return tail * image.info.resources.layers; + } if (True(image.flags & ImageFlagBits::AcceleratedUpload)) { return image.guest_size_bytes; } else if (True(image.flags & ImageFlagBits::Converted)) { diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index 590d85b042..0381d2d24b 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -36,6 +36,9 @@ struct OverlapResult { [[nodiscard]] u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept; +/// True if no 16-byte ASTC block in `data` decodes to alpha below 1. +[[nodiscard]] bool AstcDataLooksOpaque(std::span<const u8> data) noexcept; + [[nodiscard]] u32 CalculateLayerStride(const ImageInfo& info) noexcept; [[nodiscard]] u32 CalculateLayerSize(const ImageInfo& info) noexcept; @@ -69,6 +72,9 @@ struct OverlapResult { Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info, std::span<const u8> input, std::span<u8> output); +/// Rewrites `copies` (from UnswizzleImage) to the layout ConvertImage produces, without converting. +void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies); + void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8> output, std::span<BufferImageCopy> copies); diff --git a/src/video_core/textures/astc.cpp b/src/video_core/textures/astc.cpp index 0b6d06c49a..950ce28d6b 100644 --- a/src/video_core/textures/astc.cpp +++ b/src/video_core/textures/astc.cpp @@ -2038,4 +2038,69 @@ void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, } } +bool IsOpaque(std::span<const uint8_t> data) { + // Endpoint modes with an alpha channel: LDR luminance+alpha (4, 5), RGB scale + two alpha + // (10), LDR RGBA (12, 13) and HDR RGBA (14, 15). + constexpr u32 AlphaModes = (1U << 4) | (1U << 5) | (1U << 10) | (1U << 12) | (1U << 13) | + (1U << 14) | (1U << 15); + for (size_t offset = 0; offset + 16 <= data.size(); offset += 16) { + const std::span<const u8> block = data.subspan(offset, 16); + InputBitStream strm(block); + const TexelWeightParams params = DecodeBlockInfo(strm); + if (params.m_bError) { + continue; // decodes to the opaque error colour + } + if (params.m_bVoidExtentLDR || params.m_bVoidExtentHDR) { + // One RGBA16 colour in bits 64-127: UNORM16, or FP16 for HDR. + const u32 alpha = block[14] | (u32{block[15]} << 8); + if (alpha != (params.m_bVoidExtentHDR ? 0x3C00U : 0xFFFFU)) { + return false; + } + continue; + } + const u32 partitions = strm.ReadBits<2>() + 1; + if (partitions == 4 && params.m_bDualPlane) { + continue; + } + if (partitions == 1) { + if ((AlphaModes >> strm.ReadBits<4>()) & 1) { + return false; + } + continue; + } + strm.ReadBits<10>(); // partition index + const u32 base_cem = strm.ReadBits<6>(); + const u32 base_mode = base_cem & 3; + if (base_mode == 0) { + if ((AlphaModes >> (base_cem >> 2)) & 1) { + return false; + } + continue; + } + // The rest of the per-partition modes sit just below the weights (DecompressBlock). + const u32 weight_bits = params.GetPackedBitSize(); + if (weight_bits < 24 || weight_bits > 96) { + continue; + } + const u32 extra_bits = partitions == 2 ? 2 : partitions == 3 ? 5 : 8; + const u32 extra_offset = 128 - weight_bits - extra_bits; + u32 extra = 0; + for (u32 bit = 0; bit < extra_bits; ++bit) { + const u32 position = extra_offset + bit; + extra |= ((block[position / 8] >> (position % 8)) & 1U) << bit; + } + u32 cem = ((extra << 6) | base_cem) >> 2; + const u32 classes = cem; + cem >>= partitions; + for (u32 i = 0; i < partitions; ++i) { + const u32 mode = ((base_mode - (((classes >> i) & 1) != 0 ? 0 : 1)) << 2) | + ((cem >> (2 * i)) & 3); + if ((AlphaModes >> mode) & 1) { + return false; + } + } + } + return true; +} + } // namespace Tegra::Texture::ASTC diff --git a/src/video_core/textures/astc.h b/src/video_core/textures/astc.h index afd3933c3e..95c2c7ffea 100644 --- a/src/video_core/textures/astc.h +++ b/src/video_core/textures/astc.h @@ -8,4 +8,7 @@ namespace Tegra::Texture::ASTC { void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, uint32_t depth, uint32_t block_width, uint32_t block_height, std::span<uint8_t> output); +/// True if no 16-byte block in `data` decodes to alpha below 1. +bool IsOpaque(std::span<const uint8_t> data); + } // namespace Tegra::Texture::ASTC diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 25b0860b0f..ef50be156b 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -18,6 +18,11 @@ namespace Tegra::Texture { namespace { +constexpr u32 NativeGobOffset(u32 offset, bool native_tiling) { + if (!native_tiling) return offset; + const u32 difference = ((offset >> 5) ^ (offset >> 6)) & 1; + return offset ^ (difference << 5) ^ (difference << 6); +} template <u32 mask> constexpr u32 pdep(u32 value) { u32 result = 0; @@ -38,7 +43,7 @@ void incrpdep(u32& value) { template <bool TO_LINEAR, u32 BYTES_PER_PIXEL> void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth, - u32 block_height, u32 block_depth, u32 stride) { + u32 block_height, u32 block_depth, u32 stride, bool native_tiling) { // The origin of the transformation can be configured here, leave it as zero as the current API // doesn't expose it. static constexpr u32 origin_x = 0; @@ -77,7 +82,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling); const u32 unswizzled_offset = slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; @@ -94,7 +99,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 template <bool TO_LINEAR, u32 BYTES_PER_PIXEL> void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 num_lines, - u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { // The origin of the transformation can be configured here, leave it as zero as the current API // doesn't expose it. static constexpr u32 origin_z = 0; @@ -136,7 +141,7 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling); const u32 unswizzled_offset = slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; @@ -156,12 +161,12 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid template <bool TO_LINEAR> void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, - u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment) { + u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleImpl<TO_LINEAR, x>(output, input, width, height, depth, block_height, \ - block_depth, stride_alignment); + block_depth, stride_alignment, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) @@ -181,35 +186,35 @@ void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixe void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment) { + u32 stride_alignment, bool native_tiling) { const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel; const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel))); width = (width * bytes_per_pixel) >> new_bpp; bytes_per_pixel = 1U << new_bpp; Swizzle<false>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth, - stride); + stride, native_tiling); } void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment) { + u32 stride_alignment, bool native_tiling) { const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel; const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel))); width = (width * bytes_per_pixel) >> new_bpp; bytes_per_pixel = 1U << new_bpp; Swizzle<true>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth, - stride); + stride, native_tiling); } void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y, - u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleSubrectImpl<true, x>(output, input, width, height, depth, origin_x, \ origin_y, extent_x, extent_y, block_height, \ - block_depth, pitch_linear); + block_depth, pitch_linear, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) @@ -227,13 +232,13 @@ void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_p void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, - u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleSubrectImpl<false, x>(output, input, width, height, depth, origin_x, \ origin_y, extent_x, extent_y, block_height, \ - block_depth, pitch_linear); + block_depth, pitch_linear, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) diff --git a/src/video_core/textures/decoders.h b/src/video_core/textures/decoders.h index 1e14a38753..6151967869 100644 --- a/src/video_core/textures/decoders.h +++ b/src/video_core/textures/decoders.h @@ -29,12 +29,12 @@ constexpr u32 SWIZZLE_Y_BITS = 0b011010000; /// Unswizzles a block linear texture into linear memory. void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment = 1); + u32 stride_alignment = 1, bool native_tiling = false); /// Swizzles linear memory into a block linear texture. void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment = 1); + u32 stride_alignment = 1, bool native_tiling = false); /// This function calculates the correct size of a texture depending if it's tiled or not. std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, @@ -43,12 +43,12 @@ std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height /// Copies an untiled subrectangle into a tiled surface. void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y, - u32 block_height, u32 block_depth, u32 pitch_linear); + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false); /// Copies a tiled subrectangle into a linear surface. void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, - u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear); + u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false); /// Obtains the offset of the gob for positions 'dst_x' & 'dst_y' u64 GetGOBOffset(u32 width, u32 height, u32 dst_x, u32 dst_y, u32 block_height, diff --git a/src/video_core/textures/workers.cpp b/src/video_core/textures/workers.cpp index a1953c4bf4..e3a0f26cf2 100644 --- a/src/video_core/textures/workers.cpp +++ b/src/video_core/textures/workers.cpp @@ -4,13 +4,23 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "video_core/textures/workers.h" namespace Tegra::Texture { Common::ThreadWorker& GetThreadWorkers() { - static Common::ThreadWorker workers{(std::max)(std::thread::hardware_concurrency(), 2U) / 2, - "ImageTranscode"}; + // A quarter of the hardware threads (at least 2). Half took ~3.5 cores from the emulated + // CPU cores and the GPU thread on an 8-core laptop when textures were thrashing; with an + // eighth (2) both threads stayed saturated and streamed textures arrived seconds late (shown + // as placeholders), while 4 cost no frame rate. EDEN_NX2_TRANSCODE_THREADS=<n> overrides. + static Common::ThreadWorker workers{[] { + if (const char* text = std::getenv("EDEN_NX2_TRANSCODE_THREADS")) { + return (std::max)(static_cast<u32>(std::strtoul(text, nullptr, 10)), 1U); + } + return (std::max)(std::thread::hardware_concurrency() / 4, 2U); + }(), "ImageTranscode"}; return workers; } diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index be09bca8d3..69c3ba14fa 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <array> #include <bitset> #include <chrono> #include <filesystem> @@ -752,9 +753,10 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR .flags = flags, .physicalDevice = physical, .device = *logical, - .preferredLargeHeapBlockSize = is_integrated - ? (64u * 1024u * 1024u) - : (256u * 1024u * 1024u), + // 64 MiB blocks on discrete GPUs too: with 256 MiB ones, Switch 2 titles' mix of + // texture sizes left ~38% of the image blocks empty (2 GB of an 8 GB card in title + // 1), which the texture cache counted as used and kept evicting textures for. + .preferredLargeHeapBlockSize = 64u * 1024u * 1024u, .pAllocationCallbacks = nullptr, .pDeviceMemoryCallbacks = nullptr, .pHeapSizeLimit = nullptr, @@ -1609,6 +1611,33 @@ u64 Device::GetDeviceMemoryUsage() const { for (const size_t heap : valid_heap_memory) { result += budget.heapUsage[heap]; } + // Space in VMA's blocks that no allocation uses counts as used, but freeing an image gives + // its share back only once its whole block is empty: the caches, collecting until this drops, + // evicted thousands of textures in a row without lowering it (title 1). Count only what + // allocations use. + if (allocator != nullptr) { + std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> heap_budgets{}; + vmaGetHeapBudgets(allocator, heap_budgets.data()); + for (const size_t heap : valid_heap_memory) { + const VmaStatistics& statistics = heap_budgets[heap].statistics; + const u64 unused = statistics.blockBytes - statistics.allocationBytes; + result -= (std::min)(result, unused); + } + } + return result; +} + +u64 Device::GetDeviceMemoryBudget() const { + if (!extensions.memory_budget) { + return 0; + } + VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{}; + budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; + physical.GetMemoryProperties(&budget); + u64 result{}; + for (const size_t heap : valid_heap_memory) { + result += budget.heapBudget[heap]; + } return result; } diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index bbf3754e39..b1fa8db183 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -963,6 +963,9 @@ FN_MAX_LIMIT_LIST u64 GetDeviceMemoryUsage() const; + /// The driver's current memory budget for this process (VK_EXT_memory_budget), 0 without it. + u64 GetDeviceMemoryBudget() const; + u32 GetSetsPerPool() const { return sets_per_pool; } @@ -1145,7 +1148,7 @@ private: private: VkInstance instance; ///< Vulkan instance. - VmaAllocator allocator; ///< VMA allocator. + VmaAllocator allocator{}; ///< VMA allocator. vk::DeviceDispatch dld; ///< Device function pointers. vk::PhysicalDevice physical; ///< Physical device. vk::Device logical; ///< Logical device. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index f4721543ae..27f8fa6b4c 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <array> #include <bit> #include <limits> #include <optional> @@ -205,10 +206,12 @@ MemoryAllocator::MemoryAllocator(const Device &device_) MemoryAllocator::~MemoryAllocator() = default; -vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const +vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicated) const { const VmaAllocationCreateInfo alloc_ci = { - .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, + .flags = static_cast<VmaAllocationCreateFlags>(VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) | + (dedicated ? VmaAllocationCreateFlags{VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT} + : VmaAllocationCreateFlags{0}), .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, .requiredFlags = 0, .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, @@ -221,7 +224,53 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; - vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + // Images this large (title 1's sparse texture arrays reach 4.5G texels, ~23 GB) are never + // backed: VMA would quietly place them in system memory once VRAM is full. They take the + // unbacked sparse fallback below. + const u64 texels = u64{ci.extent.width} * ci.extent.height * ci.extent.depth * ci.arrayLayers; + const bool huge = texels > (u64{1} << 28); + VkResult result = huge ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) { + // Over the device memory budget (Switch 2 titles fill 8 GB cards): let the driver page + // it out rather than failing the draw and aborting. + static u32 logged{}; + if (logged++ < 32) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) is over the memory " + "budget; allocating it anyway", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format)); + } + VmaAllocationCreateInfo relaxed = alloc_ci; + relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info); + } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) { + // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192, + // 68 layers, ~23 GB): on the console only resident tiles have memory. Create a sparse + // resident image with nothing bound, so views and copies still work; reads of it return + // zero and writes are dropped, instead of aborting. + VkImageCreateInfo sparse_ci = ci; + sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; + VkImage sparse{}; + if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr, + &sparse) == VK_SUCCESS) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) does not fit; using an " + "unbacked sparse image", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format)); + return vk::Image(sparse, ci.usage, *device.GetLogical(), allocator, VK_NULL_HANDLE, + device.GetDispatchLoader()); + } + } + if (result != VK_SUCCESS) { + LOG_CRITICAL(Render_Vulkan, "Image {}x{}x{} ({} levels, {} layers, format {}) failed: {}", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format), static_cast<int>(result)); + } + vk::Check(result); // Log GPU memory allocation for images if (GPU::Logging::IsActive() && @@ -405,4 +454,29 @@ MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage return MemoryCommit(allocator, a, info); } +void LogMemoryStatistics(const Device& device) { + const VmaAllocator allocator = device.GetAllocator(); + const VkPhysicalDeviceMemoryProperties* properties{}; + vmaGetMemoryProperties(allocator, &properties); + VmaTotalStatistics stats{}; + vmaCalculateStatistics(allocator, &stats); + for (u32 type = 0; type < properties->memoryTypeCount; ++type) { + const VmaStatistics& s = stats.memoryType[type].statistics; + if (s.blockCount == 0) { + continue; + } + LOG_INFO(Render_Vulkan, "VMA type {} (heap {}, flags {:#x}): {} blocks {} MiB, {} allocations {} MiB", + type, properties->memoryTypes[type].heapIndex, + static_cast<u32>(properties->memoryTypes[type].propertyFlags), s.blockCount, + s.blockBytes >> 20, s.allocationCount, s.allocationBytes >> 20); + } + std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> budgets{}; + vmaGetHeapBudgets(allocator, budgets.data()); + for (u32 heap = 0; heap < properties->memoryHeapCount; ++heap) { + LOG_INFO(Render_Vulkan, "VMA heap {}: usage {} MiB of budget {} MiB, VMA blocks {} MiB", + heap, budgets[heap].usage >> 20, budgets[heap].budget >> 20, + budgets[heap].statistics.blockBytes >> 20); + } +} + } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index a0f9081fc0..fa7d4b41c3 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -103,7 +103,9 @@ namespace Vulkan { MemoryAllocator(const MemoryAllocator &) = delete; - vk::Image CreateImage(const VkImageCreateInfo &ci) const; + /// `dedicated`: own device memory, returned to the driver on destruction (short-lived + /// images would otherwise leave holes in the shared blocks). + vk::Image CreateImage(const VkImageCreateInfo &ci, bool dedicated = false) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; @@ -142,4 +144,7 @@ namespace Vulkan { u32 valid_memory_types{~0u}; }; + /// Logs VMA's blocks and allocations per memory type, and each heap's budget. + void LogMemoryStatistics(const Device& device); + } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 97d2ba3700..947150648d 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -689,6 +689,11 @@ public: allocation{allocation_}, dld{&dld_} {} Image() = default; + /// True for an image created without memory (an unbacked sparse image). + bool IsUnbacked() const noexcept { + return handle != VK_NULL_HANDLE && allocation == VK_NULL_HANDLE; + } + Image(const Image&) = delete; Image& operator=(const Image&) = delete; diff --git a/src/yuzu_cmd/CMakeLists.txt b/src/yuzu_cmd/CMakeLists.txt index 17a1545167..a87b4e5288 100644 --- a/src/yuzu_cmd/CMakeLists.txt +++ b/src/yuzu_cmd/CMakeLists.txt @@ -31,6 +31,8 @@ add_executable(yuzu-cmd emu_window/emu_window_sdl3_null.h emu_window/emu_window_sdl3_vk.cpp emu_window/emu_window_sdl3_vk.h + nx2_sampler.cpp + nx2_sampler.h sdl_config.cpp sdl_config.h yuzu.cpp diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp index 950e001f81..174f8b5328 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp @@ -4,6 +4,10 @@ // SPDX-FileCopyrightText: 2016 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> +#include <cstdio> +#include <chrono> +#include <array> #include <SDL3/SDL.h> #include <SDL3/SDL_timer.h> #ifdef __EMSCRIPTEN__ @@ -39,7 +43,10 @@ EmuWindow_SDL3::EmuWindow_SDL3(InputCommon::InputSubsystem* input_subsystem_, Co }, this); mouse_timer = SDL_AddTimer(100, [](void *userdata, SDL_TimerID, Uint32) -> Uint32 { auto* this_ = (EmuWindow_SDL3*)userdata; - this_->input_subsystem->GetMouse()->NotifyChanged(); + // The timer can fire before the input subsystem has created its mouse driver. + if (auto* const mouse = this_->input_subsystem->GetMouse()) { + mouse->NotifyChanged(); + } return 100; }, this); } @@ -115,7 +122,26 @@ void EmuWindow_SDL3::OnFingerUp() { input_subsystem->GetTouchScreen()->ReleaseAllTouch(); } +// Seconds since the first frame; shared by the key script (yuzu.cpp) and key recording. +double Nx2KeyClockSeconds() { + static const auto begin = std::chrono::steady_clock::now(); + return std::chrono::duration<double>(std::chrono::steady_clock::now() - begin).count(); +} + void EmuWindow_SDL3::OnKeyEvent(int key, u8 state) { + // EDEN_NX2_KEY_RECORD=<file>: append each press/release ("seconds:code", "seconds:-code"). + static std::FILE* const record = [] { + const char* path = std::getenv("EDEN_NX2_KEY_RECORD"); + return path ? std::fopen(path, "w") : nullptr; + }(); + static std::array<bool, 512> held{}; + if (record && key >= 0 && key < static_cast<int>(held.size()) && held[key] != (state != 0)) { + held[key] = state != 0; + std::fprintf(record, "f%llu:%s%d,", + static_cast<unsigned long long>(Core::g_nx2_present_count.load()), + state != 0 ? "" : "-", key); + std::fflush(record); + } if (state != 0) { input_subsystem->GetKeyboard()->PressKey(static_cast<std::size_t>(key)); } else { @@ -201,6 +227,7 @@ void EmuWindow_SDL3::OnEvent(SDL_Event& event) { is_shown = true; return OnResize(); case SDL_EVENT_WINDOW_CLOSE_REQUESTED: + LOG_INFO(Frontend, "Window close requested"); is_open = false; return; case SDL_EVENT_KEY_DOWN: @@ -224,6 +251,7 @@ void EmuWindow_SDL3::OnEvent(SDL_Event& event) { case SDL_EVENT_FINGER_UP: return OnFingerUp(); case SDL_EVENT_QUIT: + LOG_INFO(Frontend, "Quit requested"); is_open = false; return; default: diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3.h b/src/yuzu_cmd/emu_window/emu_window_sdl3.h index ffabaeb143..0dc2a4983a 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3.h +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3.h @@ -6,6 +6,7 @@ #pragma once +#include <cstdlib> #include <tuple> #include <utility> @@ -24,6 +25,14 @@ class InputSubsystem; enum class MouseButton; } // namespace InputCommon +// EDEN_NX2_HIDE_WINDOW=1 (anything but 0) creates the render window hidden and without focus, for unattended +// test runs that must not cover or interrupt the desktop. +inline SDL_WindowFlags Nx2HiddenWindowFlags() { + const char* hide = std::getenv("EDEN_NX2_HIDE_WINDOW"); + return hide != nullptr && hide[0] != '0' ? SDL_WINDOW_HIDDEN | SDL_WINDOW_NOT_FOCUSABLE + : SDL_WindowFlags{0}; +} + class EmuWindow_SDL3 : public Core::Frontend::EmuWindow { public: explicit EmuWindow_SDL3(InputCommon::InputSubsystem* input_subsystem_, Core::System& system_); diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp index e86858a232..38cc340426 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp @@ -101,7 +101,7 @@ EmuWindow_SDL3_GL::EmuWindow_SDL3_GL(InputCommon::InputSubsystem* input_subsyste SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width, Layout::ScreenUndocked::Height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE | - SDL_WINDOW_HIGH_PIXEL_DENSITY); + SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags()); if (render_window == nullptr) { LOG_CRITICAL(Frontend, "Failed to create SDL3 window! {}", SDL_GetError()); diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp index f5e4e19a30..45ff657f13 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp @@ -31,7 +31,7 @@ EmuWindow_SDL3_Null::EmuWindow_SDL3_Null(InputCommon::InputSubsystem* input_subs render_window = SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width, Layout::ScreenUndocked::Height, - SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY); + SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags()); SetWindowIcon(); diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp index 9019cbbba0..afbe67716f 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp @@ -29,7 +29,7 @@ EmuWindow_SDL3_VK::EmuWindow_SDL3_VK(InputCommon::InputSubsystem* input_subsyste render_window = SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width, Layout::ScreenUndocked::Height, - SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY); + SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags()); const SDL_PropertiesID window_props = SDL_GetWindowProperties(render_window); diff --git a/src/yuzu_cmd/nx2_sampler.cpp b/src/yuzu_cmd/nx2_sampler.cpp new file mode 100644 index 0000000000..1488a3a078 --- /dev/null +++ b/src/yuzu_cmd/nx2_sampler.cpp @@ -0,0 +1,333 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "yuzu_cmd/nx2_sampler.h" + +#ifdef _WIN32 + +#include <algorithm> +#include <array> +#include <chrono> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <map> +#include <set> +#include <string> +#include <thread> +#include <unordered_map> +#include <vector> + +#include <windows.h> +#include <tlhelp32.h> +#include <dbghelp.h> + +namespace Nx2Sampler { +namespace { + +constexpr size_t MaxDepth = 40; + +struct Sample { + unsigned thread; + unsigned depth; + std::array<DWORD64, MaxDepth> frames; +}; + +struct Target { + HANDLE handle; + std::string name; +}; + +double EnvSeconds(const char* name, double fallback) { + const char* text = std::getenv(name); + return text != nullptr ? std::strtod(text, nullptr) : fallback; +} + +std::vector<std::string> SplitNames(const char* text) { + std::vector<std::string> names; + std::string current; + for (const char* c = text; *c != '\0'; ++c) { + if (*c == ',') { + names.push_back(current); + current.clear(); + } else { + current += *c; + } + } + if (!current.empty()) { + names.push_back(current); + } + return names; +} + +bool Matches(const std::string& thread_name, const std::vector<std::string>& patterns) { + for (const auto& pattern : patterns) { + if (!pattern.empty() && pattern.back() == '*') { + if (thread_name.compare(0, pattern.size() - 1, pattern, 0, pattern.size() - 1) == 0) { + return true; + } + } else if (thread_name == pattern) { + return true; + } + } + return false; +} + +std::vector<Target> FindThreads(const std::vector<std::string>& patterns) { + using GetThreadDescriptionFn = HRESULT(WINAPI*)(HANDLE, PWSTR*); + const auto get_description = reinterpret_cast<GetThreadDescriptionFn>( + GetProcAddress(GetModuleHandleA("KernelBase.dll"), "GetThreadDescription")); + std::vector<Target> targets; + if (get_description == nullptr) { + return targets; + } + const HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (snapshot == INVALID_HANDLE_VALUE) { + return targets; + } + THREADENTRY32 entry{}; + entry.dwSize = sizeof(entry); + const DWORD self = GetCurrentThreadId(); + for (BOOL ok = Thread32First(snapshot, &entry); ok; ok = Thread32Next(snapshot, &entry)) { + if (entry.th32OwnerProcessID != GetCurrentProcessId() || entry.th32ThreadID == self) { + continue; + } + const HANDLE handle = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | + THREAD_QUERY_LIMITED_INFORMATION, + FALSE, entry.th32ThreadID); + if (handle == nullptr) { + continue; + } + PWSTR wide{}; + std::string name; + if (SUCCEEDED(get_description(handle, &wide)) && wide != nullptr) { + char buffer[256]{}; + WideCharToMultiByte(CP_UTF8, 0, wide, -1, buffer, sizeof(buffer) - 1, nullptr, + nullptr); + name = buffer; + LocalFree(wide); + } + if (Matches(name, patterns)) { + targets.push_back({handle, name}); + } else { + CloseHandle(handle); + } + } + CloseHandle(snapshot); + return targets; +} + +// Walks a suspended thread's stack from its context. Nothing here may allocate: the thread +// could be paused holding the heap lock. +unsigned Walk(CONTEXT context, std::array<DWORD64, MaxDepth>& frames) { + unsigned depth = 0; + while (depth < MaxDepth && context.Rip != 0) { + frames[depth++] = context.Rip; + DWORD64 image_base{}; + const PRUNTIME_FUNCTION function = RtlLookupFunctionEntry(context.Rip, &image_base, nullptr); + if (function == nullptr) { + // A leaf function (no unwind data): its return address is on top of the stack. JIT + // code has none either; past the first frame, stop there. + if (depth != 1) { + break; + } + context.Rip = *reinterpret_cast<const DWORD64*>(context.Rsp); + context.Rsp += 8; + continue; + } + PVOID handler_data{}; + DWORD64 establisher_frame{}; + RtlVirtualUnwind(UNW_FLAG_NHANDLER, image_base, context.Rip, function, &context, + &handler_data, &establisher_frame, nullptr); + } + return depth; +} + +struct Symbolizer { + using SymInitializeFn = BOOL(WINAPI*)(HANDLE, PCSTR, BOOL); + using SymSetOptionsFn = DWORD(WINAPI*)(DWORD); + using SymFromAddrFn = BOOL(WINAPI*)(HANDLE, DWORD64, PDWORD64, PSYMBOL_INFO); + + Symbolizer() { + const HMODULE dbghelp = LoadLibraryA("dbghelp.dll"); + if (dbghelp == nullptr) { + return; + } + const auto set_options = + reinterpret_cast<SymSetOptionsFn>(GetProcAddress(dbghelp, "SymSetOptions")); + const auto initialize = + reinterpret_cast<SymInitializeFn>(GetProcAddress(dbghelp, "SymInitialize")); + from_address = reinterpret_cast<SymFromAddrFn>(GetProcAddress(dbghelp, "SymFromAddr")); + if (set_options == nullptr || initialize == nullptr || from_address == nullptr) { + from_address = nullptr; + return; + } + set_options(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS); + if (!initialize(GetCurrentProcess(), nullptr, TRUE)) { + from_address = nullptr; + } + } + + const std::string& Name(DWORD64 address) { + const auto [it, is_new] = cache.try_emplace(address); + if (!is_new) { + return it->second; + } + if (from_address != nullptr) { + alignas(SYMBOL_INFO) char buffer[sizeof(SYMBOL_INFO) + 1024]{}; + auto* const symbol = reinterpret_cast<SYMBOL_INFO*>(buffer); + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); + symbol->MaxNameLen = 1024; + DWORD64 displacement{}; + if (from_address(GetCurrentProcess(), address, &displacement, symbol)) { + it->second.assign(symbol->Name, std::min<size_t>(symbol->NameLen, 160)); + return it->second; + } + } + HMODULE module{}; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast<LPCSTR>(address), &module)) { + char path[MAX_PATH] = "?"; + GetModuleFileNameA(module, path, sizeof(path)); + const char* base = std::strrchr(path, '\\'); + it->second = std::string{base != nullptr ? base + 1 : path} + "!?"; + } else { + it->second = "(jit)"; + } + return it->second; + } + + SymFromAddrFn from_address{}; + std::unordered_map<DWORD64, std::string> cache; +}; + +void Report(const std::vector<Target>& targets, const std::vector<Sample>& samples, + const char* path) { + Symbolizer symbolizer; + std::FILE* const file = std::fopen(path, "w"); + if (file == nullptr) { + return; + } + for (unsigned thread = 0; thread < targets.size(); ++thread) { + std::map<std::string, unsigned> self; + std::map<std::string, unsigned> inclusive; + std::map<std::string, unsigned> self_caller; + unsigned total = 0; + for (const Sample& sample : samples) { + if (sample.thread != thread || sample.depth == 0) { + continue; + } + ++total; + const std::string& top = symbolizer.Name(sample.frames[0]); + ++self[top]; + if (sample.depth > 1) { + ++self_caller[top + " <- " + symbolizer.Name(sample.frames[1])]; + } + std::set<std::string> seen; + for (unsigned i = 0; i < sample.depth; ++i) { + const std::string& name = symbolizer.Name(sample.frames[i]); + if (seen.insert(name).second) { + ++inclusive[name]; + } + } + } + const auto print = [&](const char* title, const std::map<std::string, unsigned>& counts, + size_t limit) { + std::vector<std::pair<unsigned, std::string>> sorted; + for (const auto& [name, count] : counts) { + sorted.emplace_back(count, name); + } + std::sort(sorted.rbegin(), sorted.rend()); + std::fprintf(file, "-- %s\n", title); + for (size_t i = 0; i < (std::min)(limit, sorted.size()); ++i) { + std::fprintf(file, "%6u %5.1f%% %s\n", sorted[i].first, + 100.0 * sorted[i].first / (std::max)(total, 1U), + sorted[i].second.c_str()); + } + }; + std::fprintf(file, "=== thread \"%s\": %u samples\n", targets[thread].name.c_str(), total); + print("self (where the thread is)", self, 60); + print("self <- caller", self_caller, 50); + print("inclusive (on the stack)", inclusive, 90); + std::fprintf(file, "\n"); + } + std::fclose(file); + + // <out>.folded: every distinct stack with its count, root first ("thread;a;b;c count"). + std::map<std::string, unsigned> folded; + for (const Sample& sample : samples) { + std::string line = targets[sample.thread].name; + for (unsigned i = sample.depth; i-- > 0;) { + line += ';'; + line += symbolizer.Name(sample.frames[i]); + } + ++folded[line]; + } + if (std::FILE* const out = std::fopen((std::string{path} + ".folded").c_str(), "w")) { + for (const auto& [line, count] : folded) { + std::fprintf(out, "%s %u\n", line.c_str(), count); + } + std::fclose(out); + } +} + +void Run(std::vector<std::string> patterns, double start, double seconds, std::string out) { + std::this_thread::sleep_for(std::chrono::duration<double>(start)); + const std::vector<Target> targets = FindThreads(patterns); + std::fprintf(stderr, "nx2 sampler: %zu threads match, sampling %.0f s\n", targets.size(), + seconds); + if (targets.empty()) { + return; + } + std::vector<Sample> samples; + samples.reserve(static_cast<size_t>(seconds * 1100.0) * targets.size() + 16); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::duration<double>(seconds); + while (std::chrono::steady_clock::now() < deadline && + samples.size() + targets.size() <= samples.capacity()) { + for (unsigned i = 0; i < targets.size(); ++i) { + const HANDLE handle = targets[i].handle; + if (SuspendThread(handle) == static_cast<DWORD>(-1)) { + continue; + } + CONTEXT context{}; + context.ContextFlags = CONTEXT_FULL; + if (GetThreadContext(handle, &context)) { + Sample& sample = samples.emplace_back(); + sample.thread = i; + sample.depth = Walk(context, sample.frames); + } + ResumeThread(handle); + } + Sleep(1); + } + Report(targets, samples, out.c_str()); + std::fprintf(stderr, "nx2 sampler: %zu samples written to %s\n", samples.size(), out.c_str()); + for (const auto& target : targets) { + CloseHandle(target.handle); + } +} + +} // Anonymous namespace + +void StartFromEnvironment() { + const char* names = std::getenv("EDEN_NX2_SAMPLE"); + if (names == nullptr || *names == '\0') { + return; + } + const char* out = std::getenv("EDEN_NX2_SAMPLE_OUT"); + std::thread{Run, SplitNames(names), EnvSeconds("EDEN_NX2_SAMPLE_START", 60.0), + EnvSeconds("EDEN_NX2_SAMPLE_SECONDS", 30.0), + std::string{out != nullptr ? out : "nx2_samples.txt"}} + .detach(); +} + +} // namespace Nx2Sampler + +#else + +namespace Nx2Sampler { +void StartFromEnvironment() {} +} // namespace Nx2Sampler + +#endif diff --git a/src/yuzu_cmd/nx2_sampler.h b/src/yuzu_cmd/nx2_sampler.h new file mode 100644 index 0000000000..a73c8fa302 --- /dev/null +++ b/src/yuzu_cmd/nx2_sampler.h @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +namespace Nx2Sampler { + +// EDEN_NX2_SAMPLE=<thread names>: a sampling profiler for named host threads (Windows only). +// Names are comma separated; a name ending in '*' matches by prefix (CPUCore_*). From +// EDEN_NX2_SAMPLE_START seconds (default 60) for EDEN_NX2_SAMPLE_SECONDS (default 30), each +// thread is paused about 1000 times a second and its call stack recorded; the counts per +// function (where the thread is, and everything on its stack) go to EDEN_NX2_SAMPLE_OUT +// (default nx2_samples.txt). +void StartFromEnvironment(); + +} // namespace Nx2Sampler diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp index 473a5b1014..3d9f50d4e5 100644 --- a/src/yuzu_cmd/yuzu.cpp +++ b/src/yuzu_cmd/yuzu.cpp @@ -4,6 +4,14 @@ // SPDX-FileCopyrightText: 2014 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <atomic> +#include <iterator> +#include <fstream> +#include <vector> +#include <cstdlib> +#include <chrono> +#include <cstdio> +#include <cstring> #include <iostream> #include <memory> #include <mutex> @@ -33,6 +41,7 @@ #include "core/hle/service/filesystem/filesystem.h" #include "core/loader/loader.h" #include "frontend_common/config.h" +#include "input_common/drivers/keyboard.h" #include "input_common/main.h" #include "network/network.h" #include "sdl_config.h" @@ -45,6 +54,7 @@ #endif #include "yuzu_cmd/emu_window/emu_window_sdl3_null.h" #include "yuzu_cmd/emu_window/emu_window_sdl3_vk.h" +#include "yuzu_cmd/nx2_sampler.h" #ifdef _WIN32 // windows.h needs to be included before shellapi.h @@ -60,6 +70,75 @@ extern "C" { __declspec(dllexport) unsigned long NvOptimusEnablement = 0x00000001; __declspec(dllexport) int AmdPowerXpressRequestHighPerformance = 1; } + +// EDEN_NX2_CRASH_DUMP=<dir>: on an unhandled exception (a real crash; fastmem faults are handled +// before this), write <dir>/eden-crash-<pid>.dmp. A debugger changes the timing enough to hide +// some crashes, so this catches them at full speed. dbghelp.dll is loaded only then. +// Prints a code address as module+offset (resolve with llvm-symbolizer and the module's PDB). +static void PrintCodeAddress(const char* label, const void* address) { + HMODULE module{}; + char name[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + static_cast<LPCSTR>(address), &module)) { + GetModuleFileNameA(module, name, sizeof(name)); + } + const char* base_name = std::strrchr(name, '\\'); + std::fprintf(stderr, " %s %p %s+%#llx\n", label, address, base_name ? base_name + 1 : name, + static_cast<unsigned long long>(reinterpret_cast<uintptr_t>(address) - + reinterpret_cast<uintptr_t>(module))); +} + +static LONG WINAPI WriteCrashDump(EXCEPTION_POINTERS* exception) { + static std::atomic<bool> written{}; + const char* directory = std::getenv("EDEN_NX2_CRASH_DUMP"); + if (directory == nullptr || written.exchange(true)) { + return EXCEPTION_CONTINUE_SEARCH; + } + // The faulting address and stack first: they survive even if the dump fails. + const EXCEPTION_RECORD& record = *exception->ExceptionRecord; + std::fprintf(stderr, "Unhandled exception %#lx on thread %lu, info %#llx %#llx\n", + record.ExceptionCode, GetCurrentThreadId(), + static_cast<unsigned long long>(record.NumberParameters > 0 ? record.ExceptionInformation[0] : 0), + static_cast<unsigned long long>(record.NumberParameters > 1 ? record.ExceptionInformation[1] : 0)); + PrintCodeAddress("at", record.ExceptionAddress); + void* frames[48]; + const USHORT count = RtlCaptureStackBackTrace(0, 48, frames, nullptr); + for (USHORT i = 0; i < count; ++i) { + PrintCodeAddress("frame", frames[i]); + } + std::fflush(stderr); + struct ExceptionInformation { + DWORD thread_id; + EXCEPTION_POINTERS* exception_pointers; + BOOL client_pointers; + }; + using MiniDumpWriteDumpFn = BOOL(WINAPI*)(HANDLE, DWORD, HANDLE, int, const ExceptionInformation*, + const void*, const void*); + const HMODULE dbghelp = LoadLibraryA("dbghelp.dll"); + const auto write = dbghelp ? reinterpret_cast<MiniDumpWriteDumpFn>( + GetProcAddress(dbghelp, "MiniDumpWriteDump")) + : nullptr; + if (write == nullptr) { + return EXCEPTION_CONTINUE_SEARCH; + } + char path[MAX_PATH]; + std::snprintf(path, sizeof(path), "%s\\eden-crash-%lu.dmp", directory, GetCurrentProcessId()); + const HANDLE file = CreateFileA(path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file != INVALID_HANDLE_VALUE) { + const ExceptionInformation information{GetCurrentThreadId(), exception, FALSE}; + // MiniDumpWithThreadInfo; the larger MiniDumpWithIndirectlyReferencedMemory failed on + // Eden's address space and left an empty file. + const BOOL ok = write(GetCurrentProcess(), GetCurrentProcessId(), file, 0x1000, + &information, nullptr, nullptr); + const DWORD error = ok ? 0 : GetLastError(); + CloseHandle(file); + std::fprintf(stderr, ok ? "Crash dump written to %s\n" : "Crash dump to %s failed (%#lx)\n", + path, error); + } + return EXCEPTION_CONTINUE_SEARCH; +} #endif static void OnStateChanged(const Network::RoomMember::State& state) { @@ -159,17 +238,30 @@ static void OnStatusMessageReceived(const Network::StatusMessageEntry& msg) { struct SdlState { Core::System system{}; + // Must outlive emu_window, which keeps a pointer to it. + InputCommon::InputSubsystem input_subsystem{}; std::unique_ptr<EmuWindow_SDL3> emu_window; }; extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { +#ifdef _WIN32 + SetUnhandledExceptionFilter(WriteCrashDump); +#endif + Nx2Sampler::StartFromEnvironment(); + // SDL calls SDL_AppIterate back to back by default, which kept a host core at 100% polling + // input; 240 Hz is plenty for input and the key script. + SDL_SetHint(SDL_HINT_MAIN_CALLBACK_RATE, "240"); SdlState* state = new SdlState(); + *appstate = state; #ifdef _WIN32 if (AttachConsole(ATTACH_PARENT_PROCESS)) { freopen("CONOUT$", "wb", stdout); freopen("CONOUT$", "wb", stderr); } + // Raise the CRT's open-file limit like the Qt frontend: loose RomFS folders keep thousands of + // files open through the VFS cache. + _setmaxstdio(8192); #endif Common::Log::Initialize(); @@ -202,7 +294,6 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { state->system.Initialize(); - InputCommon::InputSubsystem input_subsystem{}; // Apply the command line arguments state->system.ApplySettings(); @@ -212,14 +303,14 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { case Settings::RendererBackend::OpenGL_GLSL: case Settings::RendererBackend::OpenGL_GLASM: case Settings::RendererBackend::OpenGL_SPIRV: - state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&state->input_subsystem, state->system, lp.fullscreen); break; #endif case Settings::RendererBackend::Vulkan: - state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&state->input_subsystem, state->system, lp.fullscreen); break; case Settings::RendererBackend::Null: - state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&state->input_subsystem, state->system, lp.fullscreen); break; default: LOG_CRITICAL(Frontend, "Invalid renderer backend"); @@ -319,16 +410,73 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { void(state->system.Run()); if (state->system.DebuggerEnabled()) state->system.InitializeDebugger(); - return SDL_APP_SUCCESS; + return SDL_APP_CONTINUE; } +// EDEN_NX2_KEY_SCRIPT="seconds:code,seconds:-code,..." (or "@<file>"): press (code) or release (-code) keyboard +// keys, by SDL scancode, that many seconds after start. Lets hidden test runs drive menus. +double Nx2KeyClockSeconds(); // emu_window_sdl3.cpp + +static void RunKeyScript(InputCommon::InputSubsystem& input_subsystem) { + struct KeyEvent { + double seconds; // when frame is 0 + u64 frame; // "f<frame>:code": after that many presents + int code; + }; + static const std::vector<KeyEvent> script = [] { + std::vector<KeyEvent> events; + if (const char* text = std::getenv("EDEN_NX2_KEY_SCRIPT")) { + // "@<file>" reads the list from a file: long scripts in the environment slow down + // every getenv in the process. + std::string list{text}; + if (!list.empty() && list[0] == '@') { + std::ifstream file{list.substr(1)}; + list.assign(std::istreambuf_iterator<char>{file}, {}); + } + size_t start = 0; + while (start < list.size()) { + const size_t end = (std::min)(list.find(',', start), list.size()); + const std::string item = list.substr(start, end - start); + if (const size_t colon = item.find(':'); colon != std::string::npos) { + const std::string when = item.substr(0, colon); + const int code = std::stoi(item.substr(colon + 1)); + if (!when.empty() && when[0] == 'f') { + events.push_back({0.0, std::stoull(when.substr(1)), code}); + } else { + events.push_back({std::stod(when), 0, code}); + } + } + start = end + 1; + } + } + return events; + }(); + static size_t next = 0; + const double elapsed = Nx2KeyClockSeconds(); + const u64 frame = Core::g_nx2_present_count.load(); + while (next < script.size() && (script[next].frame != 0 ? script[next].frame <= frame + : script[next].seconds <= elapsed)) { + const int code = script[next].code; + LOG_INFO(Frontend, "Key script: {} key {} at {:.1f} s (frame {})", + code < 0 ? "release" : "press", code < 0 ? -code : code, elapsed, frame); + if (code < 0) { + input_subsystem.GetKeyboard()->ReleaseKey(static_cast<std::size_t>(-code)); + } else { + input_subsystem.GetKeyboard()->PressKey(static_cast<std::size_t>(code)); + } + ++next; + } +} + extern "C" SDL_AppResult SDL_AppIterate(void *appstate) { SdlState *state = (SdlState *)appstate; - return state->emu_window->IsOpen() ? SDL_APP_CONTINUE : SDL_APP_SUCCESS; + RunKeyScript(state->input_subsystem); + return state->emu_window && state->emu_window->IsOpen() ? SDL_APP_CONTINUE : SDL_APP_SUCCESS; } extern "C" SDL_AppResult SDL_AppEvent(void *appstate, SDL_Event *event) { SdlState *state = (SdlState *)appstate; - state->emu_window->OnEvent(*event); - return SDL_APP_SUCCESS; + if (state->emu_window) + state->emu_window->OnEvent(*event); + return SDL_APP_CONTINUE; } extern "C" void SDL_AppQuit(void *appstate, SDL_AppResult result) { SdlState *state = (SdlState *)appstate; diff --git a/tools/nx2/.gitignore b/tools/nx2/.gitignore new file mode 100644 index 0000000000..c18dd8d83c --- /dev/null +++ b/tools/nx2/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tools/nx2/a64_coverage.py b/tools/nx2/a64_coverage.py new file mode 100644 index 0000000000..2c13d8aee7 --- /dev/null +++ b/tools/nx2/a64_coverage.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Report which AArch64 encodings in NSO .text segments are not decoded by dynarmic. +# Usage: a64_coverage.py <a64.inc> <nso> [<nso> ...] +# The NSO files are only read. +import struct +import sys +from collections import Counter + +import numpy as np + + +def lz4_block_decompress(src, out_size): + dst = bytearray(out_size) + si = di = 0 + n = len(src) + while si < n: + tok = src[si] + si += 1 + lit = tok >> 4 + if lit == 15: + while True: + b = src[si] + si += 1 + lit += b + if b != 255: + break + dst[di:di + lit] = src[si:si + lit] + si += lit + di += lit + if si >= n: + break + off = src[si] | (src[si + 1] << 8) + si += 2 + ml = tok & 15 + if ml == 15: + while True: + b = src[si] + si += 1 + ml += b + if b != 255: + break + ml += 4 + start = di - off + if off >= ml: + dst[di:di + ml] = dst[start:start + ml] + else: + for k in range(ml): + dst[di + k] = dst[start + k] + di += ml + return bytes(dst) + + +def nso_text(path): + data = open(path, "rb").read() + assert data[:4] == b"NSO0", path + flags = struct.unpack_from("<I", data, 0xC)[0] + file_off, mem_off, size = struct.unpack_from("<III", data, 0x10) + csize = struct.unpack_from("<I", data, 0x60)[0] + raw = data[file_off:file_off + (csize if flags & 1 else size)] + return lz4_block_decompress(raw, size) if flags & 1 else raw + + +def load_patterns(inc): + pats = [] + for line in open(inc, encoding="utf-8"): + line = line.strip() + if not line.startswith("INST("): + continue + parts = line.split('"') + name, bits = parts[1], parts[3] + assert len(bits) == 32, line + mask = expect = 0 + for c in bits: + mask <<= 1 + expect <<= 1 + if c in "01": + mask |= 1 + expect |= int(c) + pats.append((name, mask, expect)) + return pats + + +# Named ARMv8.1+ encodings, for labelling what dynarmic rejects. +NAMED = [ + ("LSE atomic (LDADD/LDCLR/LDEOR/LDSET/LD[SU]MAX/MIN)", 0x3F208C00, 0x38200000), + ("LSE SWP", 0x3F20FC00, 0x38208000), + ("LDAPR (RCpc)", 0x3FFFFC00, 0x38BFC000), + ("CAS/CASA/CASL/CASAL", 0x3FA07C00, 0x08A07C00), + ("CASP", 0xBFA07C00, 0x08207C00), + ("LDAPUR/STLUR (RCpc2)", 0x3F200C00, 0x19000000), + ("RETAA/RETAB", 0xFFFFFBFF, 0xD65F0BFF), + ("BRAA/BRAB/BLRAA/BLRAB(+Z)", 0xFEDFF800, 0xD61F0800), + ("ERETAA/ERETAB", 0xFFFFFBFF, 0xD69F0BFF), + ("LDRAA/LDRAB", 0xFF200400, 0xF8200400), + ("PAC*/AUT* (register)", 0xFFFFC000, 0xDAC10000), + ("XPACI/XPACD", 0xFFFFFBE0, 0xDAC143E0), + ("PACGA", 0xFFE0FC00, 0x9AC03000), + ("MTE STG/STZG/ST2G/STZ2G/LDG", 0xFF200000, 0xD9200000), + ("MTE STGP", 0xFE400000, 0x68000000), + ("MTE IRG/GMI", 0xFFE0E800, 0x9AC01000), + ("MTE ADDG/SUBG", 0xBFC0C000, 0x91800000), + ("SDOT/UDOT (vector)", 0xBF20FC00, 0x0E009400), + ("SDOT/UDOT (element)", 0xBF00F400, 0x0F00E000), + ("FJCVTZS", 0xFFFFFC00, 0x1E7E0000), + ("FCMLA/FCADD", 0xBF20E400, 0x2E00C400), + ("SQRDMLAH/SQRDMLSH (vector)", 0xBF20F400, 0x2E008400), + ("SQRDMLAH/SQRDMLSH (element)", 0xBF00D400, 0x2F00D000), + ("RMIF/SETF (flagm)", 0xFFE07C10, 0xBA000400), + ("CFINV/XAFLAG/AXFLAG", 0xFFFFF0FF, 0xD500401F), + ("DC CVAP/CVADP etc", 0xFFF80000, 0xD5080000), +] + + +def main(): + pats = load_patterns(sys.argv[1]) + by_mask = {} + for _, m, e in pats: + by_mask.setdefault(m, set()).add(e) + total_unknown = Counter() + for path in sys.argv[2:]: + try: + text = nso_text(path) + except (IndexError, AssertionError) as e: + print(f"== {path}: .text does not decompress as a plain LZ4 block ({type(e).__name__})") + continue + w = np.frombuffer(text[: len(text) // 4 * 4], dtype="<u4") + known = np.zeros(w.shape, dtype=bool) + for m, es in by_mask.items(): + known |= np.isin(w & np.uint32(m), np.array(sorted(es), dtype=np.uint32)) + unk = w[~known] + named = np.zeros(unk.shape, dtype=bool) + print(f"== {path}: {len(w)} words, {len(unk)} not decoded by dynarmic") + for name, m, e in NAMED: + hit = (unk & np.uint32(m)) == np.uint32(e) + hit &= ~named + named |= hit + c = int(hit.sum()) + if c: + total_unknown[name] += c + ex = int(unk[hit][0]) + print(f" {c:9d} {name} (e.g. {ex:08x})") + rest = unk[~named] + # Group the remainder by the top 11 bits (rough instruction class). + groups = Counter((int(x) >> 21) for x in rest) + examples = {} + for x in rest: + examples.setdefault(int(x) >> 21, int(x)) + print(f" {len(rest):9d} unnamed (mostly data in .text or zero padding); top groups:") + for g, c in groups.most_common(15): + print(f" {c:8d} top11={g:03x} e.g. {examples[g]:08x}") + print("== total named:") + for name, c in total_unknown.most_common(): + print(f" {c:9d} {name}") + + +if __name__ == "__main__": + main() diff --git a/tools/nx2/gdbrsp.py b/tools/nx2/gdbrsp.py new file mode 100644 index 0000000000..83318177ac --- /dev/null +++ b/tools/nx2/gdbrsp.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +# Minimal GDB remote-serial-protocol client for Eden's gdbstub (AArch64 guests). +# Usage: gdbrsp.py <port> <script> +# Script lines: break <hexaddr> | unbreak <hexaddr> | continue | regs | x0 | mem <hexaddr|xN> <len> | bt <hexaddr|sp> <count> +# | quit +import socket +import struct +import sys + +REGS = [f"x{i}" for i in range(31)] + ["sp", "pc", "cpsr"] + + +class Rsp: + def __init__(self, port): + import time + for _ in range(240): # wait up to ~2 min for the stub to listen + try: + self.s = socket.create_connection(("127.0.0.1", port), timeout=3) + self.s.settimeout(None) # a stop can be hours of play away + break + except OSError: + time.sleep(0.5) + else: + raise SystemExit("gdbstub not reachable") + self.buf = b"" + + def _read_packet(self): + while True: + while b"$" not in self.buf or b"#" not in self.buf[self.buf.index(b"$"):]: + chunk = self.s.recv(65536) + if not chunk: + raise EOFError + self.buf += chunk + start = self.buf.index(b"$") + end = self.buf.index(b"#", start) + if len(self.buf) < end + 3: + self.buf += self.s.recv(65536) + continue + data = self.buf[start + 1:end] + self.buf = self.buf[end + 3:] + self.s.sendall(b"+") + return data.decode(errors="replace") + + def cmd(self, c): + pkt = c.encode() + self.s.sendall(b"$" + pkt + b"#" + f"{sum(pkt) & 0xff:02x}".encode()) + return self._read_packet() + + def regs(self): + r = self.cmd("g") + vals = {} + off = 0 + for name in REGS: + n = 8 if name != "cpsr" else 4 + if off + n * 2 > len(r): + break + vals[name] = int.from_bytes(bytes.fromhex(r[off:off + n * 2]), "little") + off += n * 2 + return vals + + def mem(self, addr, length): + r = self.cmd(f"m{addr:x},{length:x}") + try: + return bytes.fromhex(r) + except ValueError: + return b"" + + +def main(): + import re + import time + rsp = Rsp(int(sys.argv[1])) + # Until the guest process exists the stub answers with a bogus thread id; commands sent then + # are lost, so wait for a real one. + for _ in range(120): + reason = rsp.cmd("?") + m = re.search(r"thread:([0-9a-f]+);", reason) + if m and int(m.group(1), 16) < 0x10000: + break + time.sleep(0.5) + print("halt reason:", reason) + regs = {} + for line in open(sys.argv[2]): + parts = line.split() + if not parts or parts[0].startswith("#"): + continue + op = parts[0] + if op == "break": + print("break", parts[1], "->", rsp.cmd(f"Z0,{int(parts[1], 16):x},4")) + elif op == "breaklr": + # Break at the current return address (x30 from the last regs/x0). + lr = regs.get("x30", 0) & 0xFFFFFFFFFFFF + print("breaklr", hex(lr), "->", rsp.cmd(f"Z0,{lr:x},4")) + elif op == "unbreakpc": + pc = regs.get("pc", 0) + print("unbreakpc", hex(pc), "->", rsp.cmd(f"z0,{pc:x},4")) + elif op == "unbreak": + print("unbreak", parts[1], "->", rsp.cmd(f"z0,{int(parts[1], 16):x},4")) + elif op == "step": + print("step:", rsp.cmd("s")) + elif op == "trace": + # Single-step N instructions, printing pc and x0 (stop early at an optional pc). + n = int(parts[1]) + stop_at = int(parts[2], 16) if len(parts) > 2 else None + lo = int(parts[3], 16) if len(parts) > 4 else 0 + hi = int(parts[4], 16) if len(parts) > 4 else 1 << 64 + for _ in range(n): + rsp.cmd("s") + r = rsp.regs() + if lo <= r["pc"] < hi: + print(f" t pc={r['pc']:#x} x0={r['x0']:#x} x8={r['x8']:#x}") + if stop_at is not None and r["pc"] == stop_at: + break + elif op == "breakwhen": + # breakwhen <addr> <reg> <value> [max]: stop at addr only when reg == value. + addr, reg, want = int(parts[1], 16), parts[2], int(parts[3], 16) + for _ in range(int(parts[4]) if len(parts) > 4 else 50): + rsp.cmd(f"Z0,{addr:x},4") + rsp.cmd("c") + regs = rsp.regs() + if regs.get("pc") != addr: + print(" breakwhen: stopped elsewhere at", hex(regs.get("pc", 0))) + break + if regs.get(reg) == want: + print(f" breakwhen: hit with {reg}={want:#x}") + break + rsp.cmd(f"z0,{addr:x},4") + rsp.cmd("s") + print(" " + " ".join(f"{k}={v:#x}" for k, v in regs.items() if k in ("pc", "x0", "x1", "x2", "x19", "x20", "x21", "x22", "x24", "x30"))) + elif op == "flow": + # flow N stop_hex: single-step N instructions, print only non-sequential pcs (branches) + n, stop_at = int(parts[1]), int(parts[2], 16) + prev = rsp.regs().get("pc", 0) + for _ in range(n): + rsp.cmd("s") + r = rsp.regs() + pc = r.get("pc", 0) + if pc != prev + 4: + print(f" f {prev:#x} -> {pc:#x} x0={r.get('x0', 0):#x}") + prev = pc + if pc == stop_at: + break + elif op == "hits": + # hits <max> <addr>...: break at the addresses and, on each hit, print the registers, + # the frame-pointer chain and the 0x40 bytes at x0, then step past and continue. + # Stops after <max> hits or when the target stops anywhere else. + limit, addrs = int(parts[1]), [int(a, 16) for a in parts[2:]] + for a in addrs: + rsp.cmd(f"Z0,{a:x},4") + for hit in range(limit): + reason = rsp.cmd("c") + # Several threads can stop at once: read and step the one this reply names. + tid = re.search(r"thread:([0-9a-f]+);", reason) + if tid: + rsp.cmd(f"Hg{tid.group(1)}") + r = rsp.regs() + pc = r.get("pc", 0) + stamp = time.strftime("%H:%M:%S") + label = (f"hit {hit}" if pc in addrs else f"stopped elsewhere ({reason})") + (f" thread {tid.group(1)}" if tid else "") + print(f"[{stamp}] {label} pc={pc:#x} " + " ".join( + f"{k}={r.get(k, 0):#x}" for k in ("x0", "x1", "x2", "x19", "x20", "x21", "x30", "sp")), flush=True) + obj = rsp.mem(r.get("x0", 0), 0x40) + if len(obj) == 0x40: + print(" *x0: " + " ".join(f"{w:016x}" for w in struct.unpack("<8Q", obj)), flush=True) + fp = r.get("x29", 0) + chain = [] + for _ in range(16): + d = rsp.mem(fp, 16) + if len(d) < 16: + break + nfp, ret = struct.unpack("<QQ", d) + chain.append(f"{ret & 0xFFFFFFFFFFFF:x}") + if nfp <= fp: + break + fp = nfp + print(" bt " + " ".join(chain), flush=True) + if pc not in addrs: + break + rsp.cmd(f"z0,{pc:x},4") + rsp.cmd("s") + rsp.cmd(f"Z0,{pc:x},4") + for a in addrs: + rsp.cmd(f"z0,{a:x},4") + elif op == "continue": + print("stop:", rsp.cmd("c")) + elif op == "continue_async": + # Resume without waiting for a stop reply (pair with interrupt). + pkt = b"c" + rsp.s.sendall(b"$" + pkt + b"#" + f"{sum(pkt) & 0xff:02x}".encode()) + elif op == "x0": + r = rsp.regs() + regs = r + print(f" pc={r.get('pc', 0):#x} x0={r.get('x0', 0):#x} x1={r.get('x1', 0):#x} lr={r.get('x30', 0):#x}") + elif op == "regs": + regs = rsp.regs() + print(" ".join(f"{k}={v:#x}" for k, v in regs.items())) + elif op == "mem": + base, _, off = parts[1].partition("+") + a = (regs[base] if base in regs else int(base, 16)) + (int(off, 16) if off else 0) + data = rsp.mem(a, int(parts[2], 0)) + for i in range(0, len(data), 16): + words = [struct.unpack_from("<Q", data, i + j)[0] for j in range(0, min(16, len(data) - i) - 7, 8)] + print(f" {a + i:#x}: " + " ".join(f"{w:016x}" for w in words)) + elif op == "fp": + # Walk the frame-pointer chain: print return addresses. + fp = regs.get("x29", 0) + for _ in range(int(parts[1]) if len(parts) > 1 else 12): + d = rsp.mem(fp, 16) + if len(d) < 16: + break + nfp, ret = struct.unpack("<QQ", d) + print(f" frame fp={fp:#x} ret={ret & 0xFFFFFFFFFFFF:#x}") + if nfp <= fp: + break + fp = nfp + elif op == "interrupt": + # Stop a running target (Ctrl-C) and print the stop reply. + rsp.s.sendall(b"") + print("interrupt:", rsp._read_packet()) + elif op == "threads": + # threads [frames]: every thread's pc, lr and frame-pointer chain. + ids = [] + r = rsp.cmd("qfThreadInfo") + while r.startswith("m"): + ids += [t for t in r[1:].split(",") if t] + r = rsp.cmd("qsThreadInfo") + depth = int(parts[1]) if len(parts) > 1 else 16 + for t in ids: + rsp.cmd(f"Hg{t}") + r = rsp.regs() + chain = [] + fp = r.get("x29", 0) + for _ in range(depth): + d = rsp.mem(fp, 16) + if len(d) < 16: + break + nfp, ret = struct.unpack("<QQ", d) + chain.append(ret & 0xFFFFFFFFFFFF) + if nfp <= fp: + break + fp = nfp + print(f"thread {t} pc={r.get('pc', 0):#x} lr={r.get('x30', 0) & 0xFFFFFFFFFFFF:#x} " + f"x0={r.get('x0', 0):#x} x1={r.get('x1', 0):#x} chain=" + ",".join(f"{c:#x}" for c in chain)) + elif op == "selthread": + # selthread <lo> <hi>: select the first thread whose pc, lr or frame chain falls in + # [lo, hi); later regs/mem ops then read that thread. + lo, hi = int(parts[1], 16), int(parts[2], 16) + ids = [] + r = rsp.cmd("qfThreadInfo") + while r.startswith("m"): + ids += [t for t in r[1:].split(",") if t] + r = rsp.cmd("qsThreadInfo") + for t in ids: + rsp.cmd(f"Hg{t}") + r = rsp.regs() + cands = [r.get("pc", 0), r.get("x30", 0) & 0xFFFFFFFFFFFF] + fp = r.get("x29", 0) + for _ in range(8): + d = rsp.mem(fp, 16) + if len(d) < 16: + break + nfp, ret = struct.unpack("<QQ", d) + cands.append(ret & 0xFFFFFFFFFFFF) + if nfp <= fp: + break + fp = nfp + if any(lo <= c < hi for c in cands): + regs = r + print("selthread", t) + break + else: + print("selthread: none") + elif op == "hits": + # hits <addr> <n> [memreg]: stop at addr n times, printing x0/x1/lr (and 16 bytes at + # memreg) each time; steps off the breakpoint before resuming. + addr, n = int(parts[1], 16), int(parts[2]) + memreg = parts[3] if len(parts) > 3 else None + for i in range(n): + rsp.cmd(f"Z0,{addr:x},4") + rsp.cmd("c") + r = rsp.regs() + regs = r + extra = "" + if memreg and memreg in r: + extra = " mem=" + rsp.mem(r[memreg], 16).hex() + print(f" hit {i} pc={r.get('pc', 0):#x} x0={r.get('x0', 0):#x} x1={r.get('x1', 0):#x} " + f"x19={r.get('x19', 0):#x} x20={r.get('x20', 0):#x}" + extra) + if r.get("pc") != addr: + break + rsp.cmd(f"z0,{addr:x},4") + rsp.cmd("s") + elif op == "watch": + # watch <addr> <n> [stopval]: write watchpoint on a u64; print pc, lr and the new value + # for up to n hits, stopping early once it equals stopval. + addr, n = int(parts[1], 16), int(parts[2]) + stopval = int(parts[3], 16) if len(parts) > 3 else None + print("watch ->", rsp.cmd(f"Z2,{addr:x},8")) + for i in range(n): + reply = rsp.cmd("c") + r = rsp.regs() + regs = r + d = rsp.mem(addr, 8) + val = struct.unpack("<Q", d)[0] if len(d) == 8 else None + chain = [] + fp = r.get("x29", 0) + for _ in range(6): + fd = rsp.mem(fp, 16) + if len(fd) < 16: + break + nfp, ret = struct.unpack("<QQ", fd) + chain.append(ret & 0xFFFFFFFFFFFF) + if nfp <= fp: + break + fp = nfp + print(f" w{i} {reply[:40]} pc={r.get('pc', 0):#x} lr={r.get('x30', 0) & 0xFFFFFFFFFFFF:#x} value={val if val is None else hex(val)} chain=" + ",".join(f"{c:#x}" for c in chain)) + if stopval is not None and val == stopval: + break + # Step off the access with the watchpoint removed, then re-arm it. + rsp.cmd(f"z2,{addr:x},8") + rsp.cmd("s") + rsp.cmd(f"Z2,{addr:x},8") + elif op == "monitor": + # monitor <command...>: run a gdbstub monitor command (qRcmd) and print its text. + reply = rsp.cmd("qRcmd," + " ".join(parts[1:]).encode().hex()) + try: + print(bytes.fromhex(reply).decode(errors="replace")) + except ValueError: + print("monitor:", reply) + elif op == "raw": + # raw <packet>: send one packet and print the reply (e.g. qXfer:threads:read::0,ffff). + print("raw:", rsp.cmd(parts[1])) + elif op == "sleep": + import time + time.sleep(float(parts[1])) + elif op == "quit": + break + rsp.s.close() + + +if __name__ == "__main__": + main() diff --git a/tools/nx2/npdm_info.py b/tools/nx2/npdm_info.py new file mode 100644 index 0000000000..20fe713163 --- /dev/null +++ b/tools/nx2/npdm_info.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Print the META header and kernel capabilities of a main.npdm (read only). +import struct +import sys + +d = open(sys.argv[1], "rb").read() +assert d[:4] == b"META" +flags = d[0xC] +print(f"META flags=0x{flags:02x} is64={flags & 1} address_space_type={(flags >> 1) & 7} " + f"prio={d[0xE]} core={d[0xF]} sys_res=0x{struct.unpack_from('<I', d, 0x10)[0]:x} " + f"version=0x{struct.unpack_from('<I', d, 0x14)[0]:x} stack=0x{struct.unpack_from('<I', d, 0x1C)[0]:x} " + f"name={d[0x20:0x30].rstrip(bytes(1)).decode(errors='replace')}") +aci_off, aci_sz, acid_off, acid_sz = struct.unpack_from("<IIII", d, 0x70) +print(f"ACI0 @0x{aci_off:x}+0x{aci_sz:x} ACID @0x{acid_off:x}+0x{acid_sz:x}") +a = d[aci_off:aci_off + aci_sz] +assert a[:4] == b"ACI0", a[:4] +pid = struct.unpack_from("<Q", a, 0x10)[0] +fs_off, fs_sz, sac_off, sac_sz, kc_off, kc_sz = struct.unpack_from("<IIIIII", a, 0x20) +print(f"program_id={pid:016x} fs=0x{fs_sz:x} sac=0x{sac_sz:x} kc=0x{kc_sz:x}") +sac = a[sac_off:sac_off + sac_sz] +i, names = 0, [] +while i < len(sac): + ctl = sac[i] + n = (ctl & 7) + 1 + names.append(("S:" if ctl & 0x80 else "") + sac[i + 1:i + 1 + n].decode(errors="replace")) + i += 1 + n +print("services:", " ".join(names)) +kc = a[kc_off:kc_off + kc_sz] +svcs = set() +for (cap,) in struct.iter_unpack("<I", kc): + if cap == 0xFFFFFFFF: + continue + t = (~cap & (cap + 1)).bit_length() - 1 # number of trailing ones + if t == 3: + lo, hi = cap >> 4 & 0x3F, cap >> 10 & 0x3F + cl, ch = cap >> 16 & 0xFF, cap >> 24 + print(f" ThreadInfo prio {lo}..{hi} cores {cl}..{ch}") + elif t == 4: + idx, mask = cap >> 29, cap >> 5 & 0xFFFFFF + for b in range(24): + if mask >> b & 1: + svcs.add(idx * 24 + b) + elif t == 6: + print(f" MapRange 0x{cap:08x}") + elif t == 7: + print(f" MapIoPage 0x{cap:08x}") + elif t == 10: + print(f" MapRegion 0x{cap:08x}") + elif t == 11: + print(f" EnableInterrupts 0x{cap:08x}") + elif t == 13: + print(f" ProgramType {cap >> 14 & 7}") + elif t == 14: + print(f" KernelVersion {cap >> 19 & 0x1FFF}.{cap >> 15 & 0xF}") + elif t == 15: + print(f" HandleTableSize {cap >> 16 & 0x3FF}") + elif t == 16: + print(f" DebugFlags 0x{cap:08x}") + else: + print(f" unknown cap type {t}: 0x{cap:08x}") +print("SVCs:", " ".join(f"{s:02x}" for s in sorted(svcs))) diff --git a/tools/nx2/nso_dis.py b/tools/nx2/nso_dis.py new file mode 100644 index 0000000000..5e899b0264 --- /dev/null +++ b/tools/nx2/nso_dis.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Disassemble a range of an NSO's .text using clang + llvm-objdump. +# Usage: nso_dis.py <llvm bin dir> <nso> <text_offset_hex> <count> <work dir> +# The NSO is only read; scratch files go to <work dir>. +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from a64_coverage import nso_text # noqa: E402 + +llvm, nso, off, count, work = sys.argv[1], sys.argv[2], int(sys.argv[3], 16), int(sys.argv[4]), Path(sys.argv[5]) +work.mkdir(parents=True, exist_ok=True) +text = nso_text(nso) +words = [int.from_bytes(text[i:i + 4], "little") for i in range(off, off + count * 4, 4)] +# .inst (not .incbin) so the bytes get a code mapping symbol and objdump decodes them. +(work / "range.s").write_text(".text\n" + "".join(f".inst 0x{w:08x}\n" for w in words)) +subprocess.run([str(Path(llvm) / "clang.exe"), "--target=aarch64-none-elf", "-c", "range.s", "-o", "range.o"], + cwd=work, check=True) +out = subprocess.run([str(Path(llvm) / "llvm-objdump.exe"), "-d", "-j", ".text", "--no-show-raw-insn", + "--mattr=+v8.5a,+mte,+lse,+rcpc,+pauth,+crc,+sha2,+dotprod,+fullfp16", "range.o"], + cwd=work, capture_output=True, text=True, check=True).stdout +for line in out.splitlines(): + parts = line.split(":", 1) + if len(parts) == 2 and parts[0].strip() and all(c in "0123456789abcdef" for c in parts[0].strip()): + print(f"{off + int(parts[0], 16):08x}:{parts[1].strip()}") diff --git a/tools/nx2/nso_syms.py b/tools/nx2/nso_syms.py new file mode 100644 index 0000000000..1d20c4f2ec --- /dev/null +++ b/tools/nx2/nso_syms.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Symbol lookup for an NSO (read only). +# nso_syms.py <nso> addr <hex offset>... nearest exported symbol for module offsets +# nso_syms.py <nso> got <hex offset>... import bound to a GOT slot (e.g. from a PLT stub) +# nso_syms.py <nso> find <substring> exported symbols whose name contains substring +import struct +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from a64_coverage import lz4_block_decompress # noqa: E402 + +DT_NULL, DT_STRTAB, DT_SYMTAB, DT_RELA, DT_RELASZ, DT_JMPREL, DT_PLTRELSZ = 0, 5, 6, 7, 8, 23, 2 + + +def load_image(path): + d = open(path, "rb").read() + assert d[:4] == b"NSO0" + flags = struct.unpack_from("<I", d, 0xC)[0] + segs = [struct.unpack_from("<III", d, 0x10 + i * 0x10) for i in range(3)] + csizes = struct.unpack_from("<III", d, 0x60) + bss = struct.unpack_from("<I", d, 0x3C)[0] + end = segs[2][1] + segs[2][2] + bss + img = bytearray(end) + for i, (off, loc, size) in enumerate(segs): + raw = d[off:off + (csizes[i] if flags >> i & 1 else size)] + img[loc:loc + size] = lz4_block_decompress(raw, size) if flags >> i & 1 else raw + return bytes(img) + + +def parse(img): + mod0 = struct.unpack_from("<I", img, 4)[0] + assert img[mod0:mod0 + 4] == b"MOD0" + dyn = mod0 + struct.unpack_from("<i", img, mod0 + 4)[0] + tags = {} + i = dyn + while True: + tag, val = struct.unpack_from("<qQ", img, i) + if tag == DT_NULL: + break + tags.setdefault(tag, val) + i += 16 + strtab, symtab = tags[DT_STRTAB], tags[DT_SYMTAB] + + def name(idx): + st_name = struct.unpack_from("<I", img, symtab + idx * 24)[0] + e = img.index(b"\0", strtab + st_name) + return img[strtab + st_name:e].decode(errors="replace") + + syms = [] + idx = 1 + # The symbol table directly precedes the string table in these modules. + while symtab + (idx + 1) * 24 <= strtab: + st_name, st_info, st_other, st_shndx, st_value, st_size = struct.unpack_from("<IBBHQQ", img, symtab + idx * 24) + if st_value and st_shndx: + syms.append((st_value, st_size, name(idx))) + idx += 1 + syms.sort() + got = {} + for start, size in ((tags.get(DT_JMPREL), tags.get(DT_PLTRELSZ)), (tags.get(DT_RELA), tags.get(DT_RELASZ))): + if not start: + continue + for j in range(start, start + size, 24): + r_off, r_info, r_add = struct.unpack_from("<QQq", img, j) + sym = r_info >> 32 + if sym: + got[r_off] = name(sym) + return syms, got + + +def main(): + img = load_image(sys.argv[1]) + syms, got = parse(img) + mode, args = sys.argv[2], sys.argv[3:] + if mode == "addr": + for a in args: + a = int(a, 16) + best = None + for v, s, n in syms: + if v <= a: + best = (v, s, n) + else: + break + print(f"{a:#x}: " + (f"{best[2]}+{a - best[0]:#x}" if best else "?")) + elif mode == "got": + for a in args: + a = int(a, 16) + print(f"{a:#x}: {got.get(a, '?')}") + elif mode == "find": + for v, s, n in syms: + if args[0] in n: + print(f"{v:#x} {s:#x} {n}") + + +if __name__ == "__main__": + main()