Add opt-in S2G full wait with L2 write ACKs and WT L1 invalidation

Author: symmetryyyyy

VX_config.toml

@@ -39,6 +39,7 @@ VX_CFG_EXT_DMA_ENABLE = false
 VX_CFG_EXT_DXA_ENABLE = false
 VX_CFG_EXT_DXA_GROUP_ENABLE = false
 VX_CFG_EXT_DXA_S2G_ENABLE = false
+VX_CFG_EXT_DXA_FULL_ENABLE = false
 VX_CFG_EXT_TEX_ENABLE = false
 VX_CFG_EXT_RASTER_ENABLE = false
 VX_CFG_EXT_OM_ENABLE = false
@@ -161,6 +162,15 @@ VX_CFG_PTW_WALK_CACHE_SIZE = 16
 VX_CFG_VM_PINNED_REGION_SIZE = 0x10000000   # 256 MB
 
 [cache]
+# Opt-in response for selected stores. The unique write-back LLC responds only
+# after selected bytes commit into its data array; write-through upper levels
+# retain an MSHR context and relay that response.
+VX_CFG_CACHE_STORE_RSP_ENABLE = "expr: $VX_CFG_EXT_DXA_FULL_ENABLE"
+
+# Request attribute for a non-cascading whole-L1 invalidation at the production
+# write-through D-cache. The initial hook requires one D-cache unit per socket.
+VX_CFG_CACHE_LOCAL_INVAL_ENABLE = "expr: $VX_CFG_EXT_DXA_FULL_ENABLE"
+
 # Cache replacement-policy enums + LLC/topology helpers, consumed by the
 # per-level cache sections below and by [amo].
 __cache_repl_random = 0
@@ -298,6 +308,7 @@ VX_CFG_DXA_S2G_PIPELINED = false
 VX_CFG_DXA_S2G_PIPE_MULTI_READ = false
 VX_CFG_DXA_S2G_PIPE_SLOTS = 4
 VX_CFG_DXA_S2G_READ_CREDITS = "expr: $VX_CFG_DXA_S2G_PIPE_SLOTS"
+VX_CFG_DXA_S2G_WRITE_CREDITS = 16
 # Group slots include a reserved open group, when present.
 VX_CFG_DXA_GROUP_DEPTH = 8
 # Simultaneously source-unconsumed operations per group, not historical issues.

docs/designs/dxa_s2g_full_wait.md

@@ -0,0 +1,180 @@
+# Optional S2G destination completion and local visibility
+
+This branch extends the source-only group implementation at `b1593b3d2`. It
+does not change what `vx_dxa_wait_group_read<N>()` guarantees. Enable the new
+path with `VX_CFG_EXT_DXA_FULL_ENABLE`; it defaults off. Full wait is a separate
+instruction, `vx_dxa_wait_group<N>()` (custom DXA funct3=4, five-bit immediate).
+
+## Supported contract
+
+* Groups remain warp-owned, with independent ordered queues for each warp.
+* READ waits for source consumption, permitting SMEM reuse before destination
+  writes finish. It does not invalidate a cache or imply destination visibility.
+* FULL waits for the older committed groups to reach the configured WB L2
+  commit point, then invalidates the issuing socket's WT L1 and waits for its
+  acknowledgement. The newest N groups and the uncommitted open group remain
+  excluded from the destination wait.
+* This first implementation requires enabled write-through L1, one physical
+  D-cache per socket, write-back L2, and no L3. Unsupported configurations fail
+  assertions. It neither discards dirty L1 data nor claims global coherence.
+  The DXA store path reaches L2 directly; the optional generic WT ACK relay
+  does not support an opted-in store racing an older same-line fill and asserts
+  on that case. It is not used by this full-wait path.
+* A consuming warp on another socket needs its own refresh and explicit
+  inter-warp/CTA synchronization. A full wait is not a CTA barrier. Concurrent
+  unsynchronized accesses to an excluded or open group's destination are races.
+
+## Data and completion paths
+
+```
+                  {core, wid, gid}
+warp ISSUE ──► source + destination rings ──► fixed-mapped DXA worker
+                   │                             │
+                   │                  stable LMEM source capture
+                   │                             │
+                   ◄──── SOURCE_CONSUMED ────────┤
+                   │                             │
+                   │                  strsp-marked store requests
+                   │                             ▼
+                   │                       WB L2 array commit
+                   │                             │
+                   │                    one ACK per request beat
+                   ◄──── DESTINATION_COMPLETE ──┘
+                   │
+FULL<N> parks warp; destination depth <= N
+                   │
+                   ▼
+            zero-byte local-invalidate command
+                   │
+                   ▼
+            local WT L1 ingress serialization
+            drain older requests / MSHRs / old fills
+            invalidate clean tags; no lower-level invalidation
+                   │
+                   ▼
+             tagged invalidate ACK ──► unlock issuing warp
+```
+
+`VX_dxa_store_ack` wraps either existing S2G data engine. It does not send the
+data through LSU. The worker retains one high-level operation until its write
+ACK count drains. It may have `VX_CFG_DXA_S2G_WRITE_CREDITS` requests outstanding
+(default 16); ACK order within this operation is irrelevant to the count. The
+cache transport must deliver exactly one ACK for every opted-in write. The
+wrapper forwards SOURCE_CONSUMED independently and emits exactly one later
+DESTINATION_COMPLETE. No per-packet warp/group bookkeeping is added to L2.
+
+The L2 ACK is enqueued atomically with the actual array write, including
+write-miss fill/replay and byte-mask merge. Accepting a write into an upstream
+queue is not sufficient. Zero-length and wholly out-of-bounds operations still
+produce one event in each domain, with zero memory writes.
+
+## Why two retirement fronts are necessary
+
+```
+                   source pending     destination pending
+issue G0                1                       1
+commit G0               1                       1
+source capture G0       0                       1
+wait.read<0>           PASS                      1
+overwrite SMEM          safe                    1
+L2 ACK drain G0          0                       0
+wait.full<0>           refresh L1 -> ACK -> PASS
+```
+
+Two direct-indexed instances of the existing small per-warp ring track the
+source and destination domains. Both see the same accepted ISSUE and COMMIT;
+their tails and open group identities match, but their retirement heads may
+differ. An issue succeeds only when both have room. Thus a source-complete row
+cannot be recycled while its old destination identity is live. This is
+deliberately conservative; an independently sized destination pool is future
+work, not hidden in this patch.
+
+For every live row, destination pending is at least source pending. Every
+DESTINATION_COMPLETE follows acceptance of that operation's SOURCE_CONSUMED
+event. Assertions check these invariants. Existing exact-once transport and
+owner drain before warp reuse avoid the need for new generation tags. A reset
+must reset/flush both transport and trackers; individual warp kill cannot
+silently recycle an outstanding identity.
+
+The depth and pending bounds stay configurable (`VX_CFG_DXA_GROUP_DEPTH=8`,
+`VX_CFG_DXA_GROUP_MAX_PENDING=8`). Counts denote currently outstanding
+operations, not historical issues. Because the destination count drains later,
+enabling FULL may impose stronger issue backpressure than a source-only build.
+Commit closes an already reserved row, never allocating a new row.
+
+## Invalidation transport
+
+The endpoint injects `{rw=1, byteen=0, local_inval=1, tag=wid}` through a
+two-input arbiter on local L1 request port zero. One extra route bit keeps its
+ACK separate from LSU responses. Other L1 ports still operate normally until
+the cache takes the invalidate lock. Each endpoint allows one invalidate in
+flight and remembers other ready full waiters. Only the matching ACK unlocks
+the corresponding warp; SFU is not occupied while it waits.
+
+The cache consumes this control request locally. Its ordinary flush machinery
+drains older bank ingress and outstanding fills before clearing tags, preventing
+an old refill from restoring stale data afterward. It returns an explicit ACK
+without issuing a dummy read or zero-byte write downstream. Unrelated clean
+lines may be evicted: this loses hit rate, not architectural data. Dirty-cache
+support would require a different drain/writeback policy and is not enabled.
+
+## Validation and limitations
+
+* `hw/unittest/cache_strsp`: real WB array ACK timing, miss replay, byte masks.
+* `hw/unittest/cache_strsp_hier`: opted-in ACK transport through WT to WB.
+* `hw/unittest/cache_wt_invalidate`: actual cache data; stale primed line,
+  outstanding old fill, unrelated clean data, standalone ACK/backpressure.
+* `tests/unittest/cache_visibility`: matching SimX lifecycle/cache checks.
+* `hw/unittest/dxa_s2g`: serial and pipelined workers, delayed/reordered ACKs,
+  source poisoning, credit bounds, partial/OOB/zero-length transfers.
+* `hw/unittest/dxa_full_wait`: independent source/destination heads, retained
+  group identities on wrap, full thresholds 0/1/2, parked waiters and ACK-gated
+  wakeup.
+* `tests/regression/dxa_s2g_full_wait`: one and two issuer modes; each group has
+  a 512-byte tile plus 32-byte vector, old destination priming, source poisoning
+  after READ, FULL followed by same-kernel reload, unrelated data and canaries.
+
+SimX's functional LSU accesses can bypass cached data. Its integrated PASS
+therefore validates lifecycle/control, not by itself stale-L1 data visibility.
+Real RTL cache tests and the whole RTL regression must be reported separately.
+No performance, ASIC area, FPGA, or cross-socket coherence claim is made here.
+
+## Reproduction
+
+Configure out of tree with the installed Vortex toolchain; the regression
+Makefile supplies DXA, S2G, groups, full wait and L2 flags consistently to the
+kernel, runtime and model. Example commands from the configured build:
+
+```bash
+make -C tests/regression/dxa_s2g_full_wait -j8 THREADS=8 run-simx
+make -C tests/regression/dxa_s2g_full_wait -j8 THREADS=8 run-rtlsim
+make -C hw/unittest/dxa_full_wait THREADS=8 run
+make -C hw/unittest/cache_strsp THREADS=8 run
+make -C hw/unittest/cache_strsp_hier THREADS=8 run
+make -C hw/unittest/cache_wt_invalidate THREADS=8 run
+make -C tests/unittest/cache_visibility -j8 run
+```
+
+Use a fresh build directory for each cache latency variant
+(`PARAMS=-GLATENCY=4` or `6` on `cache_strsp`) and for serial/pipelined worker
+variants. Full serial worker flags are DXA/GROUP/S2G/FULL/L2; the pipelined
+variant additionally defines `VX_CFG_DXA_S2G_PIPELINED` and
+`VX_CFG_DXA_S2G_PIPE_MULTI_READ`. Debug builds (`DEBUG=4`) report source and
+destination completion separately, plus full-wait invalidate/wakeup events.
+The regression defaults to four warps of four lanes, one core/socket. Both one
+issuer and two issuer modes run; each issuer commits four heterogeneous groups.
+
+Negative control: keep all destination ACK tracking, replace endpoint
+`full_unlock` with `destination_unlock`, and stop scheduling cache invalidate
+requests in an isolated source/build copy. The unchanged kernel then reports
+544 mismatches in its one-issuer mode (all 4 x 136 payload words); the real
+implementation reports zero. This demonstrates a cache-visibility benefit,
+not a speedup. Never ship the negative-control modification.
+
+Checkpoint limitation: default-off L2 `vecadd_v1 -n64` passes in RTL, but the
+newer queue-based `vecadd -n64` and `async_barrier -n16` time out at CP sequence
+26 (target 27) in the tested tool environment. No untouched-baseline comparison
+has established whether those failures predate this branch. They remain open;
+the successful full-wait tests must not be presented as an all-regressions-green
+release. The existing plain `hw/unittest/cache` executable is an empty harness,
+so its successful elaboration is not counted as functional validation.

hw/rtl/VX_cluster.sv

@@ -226,6 +226,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*;
         .TAG_WIDTH      (L2_TAG_WIDTH),
         .WRITE_ENABLE   (1),
         .WRITEBACK      (`VX_CFG_L2_WRITEBACK),
+        .STORE_RSP_COMMIT (`VX_CFG_CACHE_STORE_RSP_ENABLED && L2_IS_LLC),
         .DIRTY_BYTES    (`VX_CFG_L2_DIRTYBYTES),
         .REPL_POLICY    (`VX_CFG_L2_REPL_POLICY),
         .CORE_OUT_BUF   (3),

hw/rtl/VX_gpu_pkg.sv

@@ -210,6 +210,12 @@ package VX_gpu_pkg;
     // offset (VX_cache_bank), so inserting a bit below it would silently shift
     // MEM_ATTR_AMO_OFFS and reinterpret the AMO sideband.
     typedef struct packed {
+`ifdef VX_CFG_CACHE_LOCAL_INVAL_ENABLE
+        logic                        local_inval;  // non-cascading invalidate at designated cache
+`endif
+`ifdef VX_CFG_CACHE_STORE_RSP_ENABLE
+        logic                        strsp;        // MEM_ATTR_STRSP_OFFS (store-response request)
+`endif
         amo_req_t                    amo;           // MEM_ATTR_AMO_OFFS   = 4
         logic                        is_addr_om;    // MEM_ATTR_OM_OFFS    = 3
         logic                        is_addr_local; // MEM_ATTR_LOCAL_OFFS = 2
@@ -222,6 +228,20 @@ package VX_gpu_pkg;
     localparam MEM_ATTR_LOCAL_OFFS  = 2;
     localparam MEM_ATTR_OM_OFFS     = 3;
     localparam MEM_ATTR_AMO_OFFS    = 4;
+`ifdef VX_CFG_CACHE_STORE_RSP_ENABLE
+    // Above the AMO field so every pre-existing offset is untouched; the
+    // attr width grows only when opt-in store responses are compiled in.
+    localparam MEM_ATTR_STRSP_OFFS  = MEM_ATTR_AMO_OFFS + AMO_REQ_BITS;
+`endif
+`ifdef VX_CFG_CACHE_LOCAL_INVAL_ENABLE
+    // Also appended above all pre-existing fields. When strsp is present,
+    // local_inval occupies the next bit; otherwise it directly follows AMO.
+`ifdef VX_CFG_CACHE_STORE_RSP_ENABLE
+    localparam MEM_ATTR_LOCAL_INVAL_OFFS = MEM_ATTR_STRSP_OFFS + 1;
+`else
+    localparam MEM_ATTR_LOCAL_INVAL_OFFS = MEM_ATTR_AMO_OFFS + AMO_REQ_BITS;
+`endif
+`endif
 
     // Total width of the mem-bus attr field. Use this as the parameter
     // default for VX_mem_bus_if's ATTR_WIDTH parameter and as the
@@ -947,6 +967,7 @@ package VX_gpu_pkg;
     localparam INST_DXA_ISSUE_S2G    = 3'd1;
     localparam INST_DXA_COMMIT_GROUP = 3'd2;
     localparam INST_DXA_WAIT_READ    = 3'd3;
+    localparam INST_DXA_WAIT_FULL    = 3'd4;
 
     typedef struct packed {
         logic [INST_ARGS_BITS-8-1:0] __padding;
@@ -1523,17 +1544,22 @@ package VX_gpu_pkg;
     // shared walker lives at the cluster and fetches PTEs through the L2 cache,
     // so the dcache tag is the base width in every mode.
     localparam DCACHE_TAG_WIDTH       = DCACHE_TAG_WIDTH_BASE;
+`ifdef VX_CFG_EXT_DXA_FULL_ENABLE
+    localparam DCACHE_INPUT_TAG_WIDTH = DCACHE_TAG_WIDTH + 1;
+`else
+    localparam DCACHE_INPUT_TAG_WIDTH = DCACHE_TAG_WIDTH;
+`endif
 
     // Memory request data bits (mem transacts in sectors)
     localparam DCACHE_MEM_DATA_WIDTH = (DCACHE_SECTOR_SIZE * 8);
 
-    // Memory request tag bits (computed with DCACHE_TAG_WIDTH since
+    // Memory request tag bits (computed with DCACHE_INPUT_TAG_WIDTH since
     // VX_cache_cluster sees the post-arb tag width). The bypass tag encodes the
     // in-block offset at the mem granule, so it uses the sector, not the line.
 `ifdef VX_CFG_DCACHE_ENABLE
-    localparam DCACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_NC_MEM_TAG_WIDTH(`VX_CFG_DCACHE_MSHR_SIZE, DCACHE_NUM_BANKS, DCACHE_NUM_REQS, L1_MEM_PORTS, DCACHE_SECTOR_SIZE, DCACHE_WORD_SIZE, DCACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_DCACHES, UUID_WIDTH);
+    localparam DCACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_NC_MEM_TAG_WIDTH(`VX_CFG_DCACHE_MSHR_SIZE, DCACHE_NUM_BANKS, DCACHE_NUM_REQS, L1_MEM_PORTS, DCACHE_SECTOR_SIZE, DCACHE_WORD_SIZE, DCACHE_INPUT_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_DCACHES, UUID_WIDTH);
 `else
-    localparam DCACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(DCACHE_NUM_REQS, L1_MEM_PORTS, DCACHE_SECTOR_SIZE, DCACHE_WORD_SIZE, DCACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_DCACHES);
+    localparam DCACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(DCACHE_NUM_REQS, L1_MEM_PORTS, DCACHE_SECTOR_SIZE, DCACHE_WORD_SIZE, DCACHE_INPUT_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_DCACHES);
 `endif
 
     // L1 dcache is the LLC iff neither L2 nor L3 is enabled.