eyad-silx commited on
Commit
daae1b8
·
verified ·
1 Parent(s): b9dc683

Fix FLA fused recurrent generation cache state

Browse files
Files changed (1) hide show
  1. fla/ops/gsa/fused_recurrent.py +7 -6
fla/ops/gsa/fused_recurrent.py CHANGED
@@ -101,12 +101,13 @@ def fused_recurrent_gsa_inference(
101
  else:
102
  hk0, hv0 = q.new_zeros(B, H, K, M, dtype=torch.float), q.new_zeros(B, H, M, V, dtype=torch.float)
103
 
104
- hkt, hvt = None, None
105
- if output_final_state:
106
- if NG == 1:
107
- hkt, hvt = hk0, hv0
108
- else:
109
- hkt, hvt = q.new_empty(B, H, K, M, dtype=torch.float), q.new_empty(B, H, M, V, dtype=torch.float)
 
110
 
111
  o = v.new_empty(B, T, HQ, V)
112
  grid = (B * HQ,)
 
101
  else:
102
  hk0, hv0 = q.new_zeros(B, H, K, M, dtype=torch.float), q.new_zeros(B, H, M, V, dtype=torch.float)
103
 
104
+ # The Triton inference kernel stores final recurrent state for the first
105
+ # query head in each KV group. It therefore needs valid hkt/hvt pointers
106
+ # even when the caller does not request returning the final state.
107
+ if NG == 1:
108
+ hkt, hvt = hk0, hv0
109
+ else:
110
+ hkt, hvt = q.new_empty(B, H, K, M, dtype=torch.float), q.new_empty(B, H, M, V, dtype=torch.float)
111
 
112
  o = v.new_empty(B, T, HQ, V)
113
  grid = (B * HQ,)