Merge remote-tracking branch 'origin/master' into opt-lsq

src/main/scala/xiangshan/backend/dispatch/DispatchQueue.scala

@@ -20,7 +20,6 @@ class DispatchQueueIO(enqnum: Int, deqnum: Int) extends XSBundle {
 // dispatch queue: accepts at most enqnum uops from dispatch1 and dispatches deqnum uops at every clock cycle
 class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with HasCircularQueuePtrHelper {
   val io = IO(new DispatchQueueIO(enqnum, deqnum))
-  val indexWidth = log2Ceil(size)
 
   val s_invalid :: s_valid:: Nil = Enum(2)
 
@@ -34,10 +33,12 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
   // tail: first invalid entry (free entry)
   val tailPtr = RegInit(VecInit((0 until enqnum).map(_.U.asTypeOf(new CircularQueuePtr(size)))))
   val tailPtrMask = UIntToMask(tailPtr(0).value, size)
+  // valid entries counter
+  val validCounter = RegInit(0.U(log2Ceil(size).W))
+  val allowEnqueue = RegInit(true.B)
 
-  val validEntries = distanceBetween(tailPtr(0), headPtr(0))
   val isTrueEmpty = ~Cat((0 until size).map(i => stateEntries(i) === s_valid)).orR
-  val canEnqueue = validEntries <= (size - enqnum).U
+  val canEnqueue = allowEnqueue
   val canActualEnqueue = canEnqueue && !io.redirect.valid
 
   /**
@@ -93,7 +94,8 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
     */
 
   // dequeue
-  val numDeqTry = Mux(validEntries > deqnum.U, deqnum.U, validEntries)
+  val currentValidCounter = distanceBetween(tailPtr(0), headPtr(0))
+  val numDeqTry = Mux(currentValidCounter > deqnum.U, deqnum.U, currentValidCounter)
   val numDeqFire = PriorityEncoder(io.deq.zipWithIndex.map{case (deq, i) =>
     // For dequeue, the first entry should never be s_invalid
     // Otherwise, there should be a redirect and tail walks back
@@ -146,6 +148,28 @@ class DispatchQueue(size: Int, enqnum: Int, deqnum: Int) extends XSModule with H
       )
   }
 
+  // update valid counter and allowEnqueue reg
+  validCounter := Mux(exceptionValid,
+    0.U,
+    Mux(io.redirect.valid,
+      validCounter,
+      Mux(lastLastCycleMisprediction,
+        currentValidCounter,
+        validCounter + numEnq - numDeq)
+    )
+  )
+  allowEnqueue := Mux(io.redirect.valid,
+    false.B,
+    Mux(lastLastCycleMisprediction,
+      currentValidCounter <= (size - enqnum).U,
+      // To optimize timing, we don't use numDeq here.
+      // It affects cases when validCount + numEnq - numDeq <= (size - enqnum).U.
+      // For example, there're 10 empty entries with 6 enqueue and 2 dequeue.
+      // However, since dispatch queue size > (numEnq + numDeq),
+      // even if we allow enqueue, they cannot be dispatched immediately.
+      validCounter + numEnq <= (size - enqnum).U
+    )
+  )
 
   /**
     * Part 3: set output and input

src/main/scala/xiangshan/backend/roq/Roq.scala

@@ -115,6 +115,8 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   val enqPtr = RegInit(0.U.asTypeOf(new RoqPtr))
   val deqPtrVec = RegInit(VecInit((0 until CommitWidth).map(_.U.asTypeOf(new RoqPtr))))
   val walkPtrVec = Reg(Vec(CommitWidth, new RoqPtr))
+  val validCounter = RegInit(0.U(log2Ceil(RoqSize).W))
+  val allowEnqueue = RegInit(true.B)
 
   val enqPtrVec = VecInit((0 until RenameWidth).map(i => enqPtr + PopCount(io.enq.needAlloc.take(i))))
   val deqPtr = deqPtrVec(0)
@@ -171,6 +173,7 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   when (io.commits.valid.asUInt.orR) { hasNoSpecExec:= false.B }
 
   for (i <- 0 until RenameWidth) {
+    // we don't determine whether io.redirect.valid here since redirect has higher priority
     when(io.enq.req(i).valid && io.enq.canAccept) {
       // store uop in data module and microOp Vec
       commitData.io.wen(i) := true.B
@@ -185,13 +188,10 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
     io.enq.resp(i) := enqPtrVec(i)
   }
 
-  val validEntries = distanceBetween(enqPtr, deqPtr)
   val firedDispatch = Mux(io.enq.canAccept, PopCount(Cat(io.enq.req.map(_.valid))), 0.U)
-  io.enq.canAccept := (validEntries <= (RoqSize - RenameWidth).U) && !hasBlockBackward
+  io.enq.canAccept := allowEnqueue && !hasBlockBackward
   io.enq.isEmpty   := isEmpty
   XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n")
-
-  enqPtr := enqPtr + firedDispatch
   when (firedDispatch =/= 0.U) {
     XSInfo("dispatched %d insts\n", firedDispatch)
   }
@@ -359,9 +359,6 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   }
 
   // move tail ptr
-  when (state === s_idle) {
-    deqPtrVec := VecInit(deqPtrVec.map(_ + commitCnt))
-  }
   val retireCounter = Mux(state === s_idle, commitCnt, 0.U)
   XSInfo(retireCounter > 0.U, "retired %d insts\n", retireCounter)
 
@@ -378,7 +375,6 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
     }
     val currentWalkPtr = Mux(state === s_walk || state === s_extrawalk, walkPtr, enqPtr - 1.U)
     walkCounter := distanceBetween(currentWalkPtr, io.redirect.bits.roqIdx) + io.redirect.bits.flushItself() - Mux(state === s_walk, commitCnt, 0.U)
-    enqPtr := io.redirect.bits.roqIdx + Mux(io.redirect.bits.flushItself(), 0.U, 1.U)
   }
 
   // no enough space for walk, allocate extra space
@@ -392,10 +388,44 @@ class Roq(numWbPorts: Int) extends XSModule with HasCircularQueuePtrHelper {
   // when exception occurs, cancels all
   when (io.redirectOut.valid) {
     state := s_idle
-    enqPtr := 0.U.asTypeOf(new RoqPtr)
+  }
+
+  /**
+    * pointers
+    */
+  when (io.redirectOut.valid) {
     deqPtrVec := VecInit((0 until CommitWidth).map(_.U.asTypeOf(new RoqPtr)))
+  }.elsewhen(state === s_idle) {
+    deqPtrVec := deqPtrVec.map(_ + commitCnt)
   }
 
+  when (io.redirectOut.valid) {
+    enqPtr := 0.U.asTypeOf(new RoqPtr)
+  }.elsewhen (io.redirect.valid) {
+    enqPtr := io.redirect.bits.roqIdx + Mux(io.redirect.bits.flushItself(), 0.U, 1.U)
+  }.otherwise {
+    enqPtr := enqPtr + firedDispatch
+  }
+
+  val lastCycleRedirect = RegNext(io.redirect.valid)
+  val trueValidCounter = Mux(lastCycleRedirect, distanceBetween(enqPtr, deqPtr), validCounter)
+
+  validCounter := Mux(io.redirectOut.valid,
+    0.U,
+    Mux(state === s_idle,
+      (validCounter - commitCnt) + firedDispatch,
+      trueValidCounter
+    )
+  )
+
+  allowEnqueue := Mux(io.redirectOut.valid,
+    true.B,
+    Mux(state === s_idle,
+      validCounter + firedDispatch <= (RoqSize - RenameWidth).U,
+      trueValidCounter <= (RoqSize - RenameWidth).U
+    )
+  )
+
   /**
     * States
     * We put all the stage changes here.