diff --git a/src/main/scala/utils/DebugIdentityNode.scala b/src/main/scala/utils/DebugIdentityNode.scala index 92992e478d330858e4a37ae11c266ab02cc65db1..487b2421d34c2dc63bebf02589149bf00715d912 100644 --- a/src/main/scala/utils/DebugIdentityNode.scala +++ b/src/main/scala/utils/DebugIdentityNode.scala @@ -31,7 +31,7 @@ class DebugIdentityNode()(implicit p: Parameters) extends LazyModule { } ) } - debug(in, true) + debug(in, false) } } diff --git a/src/main/scala/utils/TLDump.scala b/src/main/scala/utils/TLDump.scala index 64fa168d19801bb5c1628a2d4498347e716b62b2..3a0b0df45d73a07671e2c0ac47a767d65935c88b 100644 --- a/src/main/scala/utils/TLDump.scala +++ b/src/main/scala/utils/TLDump.scala @@ -1,35 +1,464 @@ package utils import chisel3._ +import chisel3.util._ +import freechips.rocketchip.tilelink.TLMessages._ +import freechips.rocketchip.tilelink.TLPermissions._ import freechips.rocketchip.tilelink.{TLBundle, TLBundleA, TLBundleB, TLBundleC, TLBundleD, TLBundleE, TLChannel} import xiangshan.HasXSLog -trait HasTLDump { this: HasXSLog => +trait HasTLDump { + this: HasXSLog => implicit class TLDump(channel: TLChannel) { def dump = channel match { case a: TLBundleA => + printChannelA(a) + case b: TLBundleB => + printChannelB(b) + case c: TLBundleC => + printChannelC(c) + case d: TLBundleD => + printChannelD(d) + case e: TLBundleE => + printChannelE(e) + } + } + + def printChannelA(a: TLBundleA): Unit = { + switch(a.opcode) { + is(PutFullData) { XSDebug(false, true.B, - a.channelName + " opcode: %x param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", - a.opcode, a.param, a.size, a.source, a.address, a.mask, a.data, a.corrupt + a.channelName + " PutFullData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.param, a.size, a.source, a.address, a.mask, a.data, a.corrupt ) - case b: TLBundleB => + } + + is(PutPartialData) { XSDebug(false, true.B, - b.channelName + " opcode: %x param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", - b.opcode, b.param, b.size, b.source, b.address, b.mask, b.data, b.corrupt + a.channelName + " PutPartialData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.param, a.size, a.source, a.address, a.mask, a.data, a.corrupt ) - case c: TLBundleC => + } + + is(ArithmeticData) { XSDebug(false, true.B, - c.channelName + " opcode: %x param: %x size: %x source: %d address: %x data: %x corrupt: %b\n", - c.opcode, c.param, c.size, c.source, c.address, c.data, c.corrupt + a.channelName + " ArithmeticData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.param, a.size, a.source, a.address, a.mask, a.data, a.corrupt ) - case d: TLBundleD => + } + + is(LogicalData) { XSDebug(false, true.B, - d.channelName + " opcode: %x param: %x size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", - d.opcode, d.param, d.size, d.source, d.sink, d.denied, d.data, d.corrupt + a.channelName + " LogicalData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.param, a.size, a.source, a.address, a.mask, a.data, a.corrupt ) - case e: TLBundleE => - XSDebug(false, true.B, e.channelName + " sink: %d\n", e.sink) + } + + is(Get) { + XSDebug(false, true.B, + a.channelName + " Get param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.param, a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + + is(Hint) { + XSDebug(false, true.B, + a.channelName + " Intent param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.param, a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + + is(AcquireBlock) { + switch(a.param) { + is(NtoB) { + XSDebug(false, true.B, + a.channelName + " AcquireBlock NtoB size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + is(NtoT) { + XSDebug(false, true.B, + a.channelName + " AcquireBlock NtoT size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + is(BtoT) { + XSDebug(false, true.B, + a.channelName + " AcquireBlock BtoT size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + } + } + + is(AcquirePerm) { + switch(a.param) { + is(NtoB) { + XSDebug(false, true.B, + a.channelName + " AcquirePerm NtoB size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + is(NtoT) { + XSDebug(false, true.B, + a.channelName + " AcquirePerm NtoT size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + is(BtoT) { + XSDebug(false, true.B, + a.channelName + " AcquirePerm BtoT size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + a.size, a.source, a.address, a.mask, a.data, a.corrupt + ) + } + } + } + } } + + def printChannelB(b: TLBundleB): Unit = { + switch(b.opcode) { + is(PutFullData) { + XSDebug(false, true.B, + b.channelName + " PutFullData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.param, b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + + is(PutPartialData) { + XSDebug(false, true.B, + b.channelName + " PutPartialData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.param, b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + + is(ArithmeticData) { + XSDebug(false, true.B, + b.channelName + " ArithmeticData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.param, b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + + is(LogicalData) { + XSDebug(false, true.B, + b.channelName + " LogicalData param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.param, b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + + is(Get) { + XSDebug(false, true.B, + b.channelName + " Get param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.param, b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + + is(Hint) { + XSDebug(false, true.B, + b.channelName + " Intent param: %x size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.param, b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + + is(Probe) { + switch(b.param) { + is(toN) { + XSDebug(false, true.B, + b.channelName + " Probe toN size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + is(toB) { + XSDebug(false, true.B, + b.channelName + " Probe toB size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + is(toT) { + XSDebug(false, true.B, + b.channelName + " Probe toT size: %x source: %d address: %x mask: %x data: %x corrupt: %b\n", + b.size, b.source, b.address, b.mask, b.data, b.corrupt + ) + } + } + } + + } + } + + def printChannelC(c: TLBundleC): Unit = { + switch(c.opcode) { + is(AccessAck) { + XSDebug(false, true.B, + c.channelName + " AccessAck param: %x size: %x source: %d address: %x data: %x corrupt: %b\n", + c.param, c.size, c.source, c.address, c.data, c.corrupt + ) + } + + is(AccessAckData) { + XSDebug(false, true.B, + c.channelName + " AccessAckData param: %x size: %x source: %d address: %x data: %x corrupt: %b\n", + c.param, c.size, c.source, c.address, c.data, c.corrupt + ) + } + + is(HintAck) { + XSDebug(false, true.B, + c.channelName + " HintAck param: %x size: %x source: %d address: %x data: %x corrupt: %b\n", + c.param, c.size, c.source, c.address, c.data, c.corrupt + ) + } + + is(ProbeAck) { + switch(c.param) { + is(TtoB) { + XSDebug(false, true.B, + c.channelName + " ProbeAck TtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoN) { + XSDebug(false, true.B, + c.channelName + " ProbeAck TtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoN) { + XSDebug(false, true.B, + c.channelName + " ProbeAck BtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoT) { + XSDebug(false, true.B, + c.channelName + " ProbeAck TtoT size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoB) { + XSDebug(false, true.B, + c.channelName + " ProbeAck BtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(NtoN) { + XSDebug(false, true.B, + c.channelName + " ProbeAck NtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + } + } + + is(ProbeAckData) { + switch(c.param) { + is(TtoB) { + XSDebug(false, true.B, + c.channelName + " ProbeAckData TtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoN) { + XSDebug(false, true.B, + c.channelName + " ProbeAckData TtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoN) { + XSDebug(false, true.B, + c.channelName + " ProbeAckData BtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoT) { + XSDebug(false, true.B, + c.channelName + " ProbeAckData TtoT size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoB) { + XSDebug(false, true.B, + c.channelName + " ProbeAckData BtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(NtoN) { + XSDebug(false, true.B, + c.channelName + " ProbeAckData NtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + } + } + + is(Release) { + switch(c.param) { + is(TtoB) { + XSDebug(false, true.B, + c.channelName + " Release TtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoN) { + XSDebug(false, true.B, + c.channelName + " Release TtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoN) { + XSDebug(false, true.B, + c.channelName + " Release BtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoT) { + XSDebug(false, true.B, + c.channelName + " Release TtoT size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoB) { + XSDebug(false, true.B, + c.channelName + " Release BtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(NtoN) { + XSDebug(false, true.B, + c.channelName + " Release NtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + } + } + + is(ReleaseData) { + switch(c.param) { + is(TtoB) { + XSDebug(false, true.B, + c.channelName + " ReleaseData TtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoN) { + XSDebug(false, true.B, + c.channelName + " ReleaseData TtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoN) { + XSDebug(false, true.B, + c.channelName + " ReleaseData BtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(TtoT) { + XSDebug(false, true.B, + c.channelName + " ReleaseData TtoT size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(BtoB) { + XSDebug(false, true.B, + c.channelName + " ReleaseData BtoB size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + is(NtoN) { + XSDebug(false, true.B, + c.channelName + " ReleaseData NtoN size: %x source: %d address: %x data: %x corrupt: %b\n", + c.size, c.source, c.address, c.data, c.corrupt + ) + } + } + } + + } + } + + def printChannelD(d: TLBundleD): Unit = { + switch(d.opcode) { + is(AccessAck) { + XSDebug(false, true.B, + d.channelName + " AccessAck param: %x size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.param, d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + + is(AccessAckData) { + XSDebug(false, true.B, + d.channelName + " AccessAckData param: %x size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.param, d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + + is(HintAck) { + XSDebug(false, true.B, + d.channelName + " HintAck param: %x size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.param, d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + + is(Grant) { + switch(d.param) { + is(toT) { + XSDebug(false, true.B, + d.channelName + " Grant toT size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + is(toB) { + XSDebug(false, true.B, + d.channelName + " Grant toB size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + is(toN) { + XSDebug(false, true.B, + d.channelName + " Grant toN size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + } + } + + is(GrantData) { + switch(d.param) { + is(toT) { + XSDebug(false, true.B, + d.channelName + " GrantData toT size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + is(toB) { + XSDebug(false, true.B, + d.channelName + " GrantData toB size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + is(toN) { + XSDebug(false, true.B, + d.channelName + " GrantData toN size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + } + } + + is(ReleaseAck) { + XSDebug(false, true.B, + d.channelName + " ReleaseAck param: %x size: %x source: %d sink: %d denied: %b data: %x corrupt: %b\n", + d.param, d.size, d.source, d.sink, d.denied, d.data, d.corrupt + ) + } + + } + } + + def printChannelE(e: TLBundleE): Unit = { + XSDebug(false, true.B, e.channelName + "GrantAck sink: %d\n", e.sink) + } + } diff --git a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala index ae429e78bf2b790be050f1564ff6ee5e4b342651..a35d81d1128b2608951f2de7e1e84a9a7ef04815 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/LoadQueue.scala @@ -62,6 +62,9 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new LqPtr)))) val deqPtrExt = RegInit(0.U.asTypeOf(new LqPtr)) + val validCounter = RegInit(0.U(log2Ceil(LoadQueueSize).W)) + val allowEnqueue = RegInit(true.B) + val enqPtr = enqPtrExt(0).value val deqPtr = deqPtrExt.value val sameFlag = enqPtrExt(0).flag === deqPtrExt.flag @@ -80,10 +83,8 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP * * Currently, LoadQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth) */ - val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt) - val firedDispatch = io.enq.req.map(_.valid) - io.enq.canAccept := validEntries <= (LoadQueueSize - RenameWidth).U - XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n") + io.enq.canAccept := allowEnqueue + for (i <- 0 until RenameWidth) { val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) val lqIdx = enqPtrExt(offset) @@ -100,13 +101,7 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP } io.enq.resp(i) := lqIdx } - - // when io.brqRedirect.valid, we don't allow eneuque even though it may fire. - when (Cat(firedDispatch).orR && io.enq.canAccept && io.enq.sqCanAccept && !io.brqRedirect.valid) { - val enqNumber = PopCount(firedDispatch) - enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) - XSInfo("dispatched %d insts to lq\n", enqNumber) - } + XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") /** * Writeback load from load units @@ -335,7 +330,6 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP XSDebug("load commit %d: idx %d %x\n", i.U, mcommitIdx(i), uop(mcommitIdx(i)).cf.pc) } }) - deqPtrExt := deqPtrExt + PopCount(loadCommit) def getFirstOne(mask: Vec[Bool], startMask: UInt) = { val length = mask.length @@ -549,13 +543,38 @@ class LoadQueue extends XSModule with HasDCacheParameters with HasCircularQueueP allocated(i) := false.B } } - // we recover the pointers in the next cycle after redirect - val needCancelReg = RegNext(needCancel) + + /** + * update pointers + */ + val lastCycleCancelCount = PopCount(RegNext(needCancel)) + // when io.brqRedirect.valid, we don't allow eneuque even though it may fire. + val enqNumber = Mux(io.enq.canAccept && io.enq.sqCanAccept && !io.brqRedirect.valid, PopCount(io.enq.req.map(_.valid)), 0.U) when (lastCycleRedirect.valid) { - val cancelCount = PopCount(needCancelReg) - enqPtrExt := VecInit(enqPtrExt.map(_ - cancelCount)) + // we recover the pointers in the next cycle after redirect + enqPtrExt := VecInit(enqPtrExt.map(_ - lastCycleCancelCount)) + }.otherwise { + enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) } + val commitCount = PopCount(loadCommit) + deqPtrExt := deqPtrExt + commitCount + + val lastLastCycleRedirect = RegNext(lastCycleRedirect.valid) + val trueValidCounter = distanceBetween(enqPtrExt(0), deqPtrExt) + validCounter := Mux(lastLastCycleRedirect, + trueValidCounter, + validCounter + enqNumber - commitCount + ) + + allowEnqueue := Mux(io.brqRedirect.valid, + false.B, + Mux(lastLastCycleRedirect, + trueValidCounter <= (LoadQueueSize - RenameWidth).U, + validCounter + enqNumber <= (LoadQueueSize - RenameWidth).U + ) + ) + // debug info XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt.flag, deqPtr) diff --git a/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala b/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala index c50dd763af73ee04edaf9b0fac2bf8c5b8b4da5a..78647bd6512604657751a6901e0301fe22b24c65 100644 --- a/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala +++ b/src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala @@ -58,6 +58,9 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue require(StoreQueueSize > RenameWidth) val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new SqPtr)))) val deqPtrExt = RegInit(VecInit((0 until StorePipelineWidth).map(_.U.asTypeOf(new SqPtr)))) + val validCounter = RegInit(0.U(log2Ceil(LoadQueueSize).W)) + val allowEnqueue = RegInit(true.B) + val enqPtr = enqPtrExt(0).value val deqPtr = deqPtrExt(0).value @@ -69,10 +72,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue * * Currently, StoreQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth) */ - val validEntries = distanceBetween(enqPtrExt(0), deqPtrExt(0)) - val firedDispatch = io.enq.req.map(_.valid) - io.enq.canAccept := validEntries <= (StoreQueueSize - RenameWidth).U - XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(firedDispatch))}\n") + io.enq.canAccept := allowEnqueue for (i <- 0 until RenameWidth) { val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) val sqIdx = enqPtrExt(offset) @@ -87,12 +87,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue } io.enq.resp(i) := sqIdx } - - when (Cat(firedDispatch).orR && io.enq.canAccept && io.enq.lqCanAccept && !io.brqRedirect.valid) { - val enqNumber = PopCount(firedDispatch) - enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) - XSInfo("dispatched %d insts to sq\n", enqNumber) - } + XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") /** * Writeback store from store units @@ -104,7 +99,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue * instead of pending to avoid sending them to lower level. * (2) For an mmio instruction without exceptions, we mark it as pending. * When the instruction reaches ROB's head, StoreQueue sends it to uncache channel. - * Upon receiving the response, StoreQueue writes back the instruction + * Upon receiving the response, StoreQueue writes back the instruction * through arbiter with store units. It will later commit as normal. */ for (i <- 0 until StorePipelineWidth) { @@ -246,7 +241,7 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue when (io.mmioStout.fire()) { writebacked(deqPtr) := true.B allocated(deqPtr) := false.B - deqPtrExt := VecInit(deqPtrExt.map(_ + 1.U)) + } /** @@ -284,14 +279,10 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr) } } - // note that sbuffer will not accept req(1) if req(0) is not accepted. - when (Cat(io.sbuffer.map(_.fire())).orR) { - val stepForward = Mux(io.sbuffer(1).fire(), 2.U, 1.U) - deqPtrExt := VecInit(deqPtrExt.map(_ + stepForward)) - when (io.sbuffer(1).fire()) { - assert(io.sbuffer(0).fire()) - } + when (io.sbuffer(1).fire()) { + assert(io.sbuffer(0).fire()) } + if (!env.FPGAPlatform) { val storeCommit = PopCount(io.sbuffer.map(_.fire())) val waddr = VecInit(io.sbuffer.map(req => SignExt(req.bits.addr, 64))) @@ -316,13 +307,45 @@ class StoreQueue extends XSModule with HasDCacheParameters with HasCircularQueue allocated(i) := false.B } } - // we recover the pointers in the next cycle after redirect - val lastCycleRedirectValid = RegNext(io.brqRedirect.valid) - val needCancelCount = PopCount(RegNext(needCancel)) - when (lastCycleRedirectValid) { - enqPtrExt := VecInit(enqPtrExt.map(_ - needCancelCount)) + + /** + * update pointers + */ + val lastCycleRedirect = RegNext(io.brqRedirect.valid) + val lastCycleCancelCount = PopCount(RegNext(needCancel)) + // when io.brqRedirect.valid, we don't allow eneuque even though it may fire. + val enqNumber = Mux(io.enq.canAccept && io.enq.lqCanAccept && !io.brqRedirect.valid, PopCount(io.enq.req.map(_.valid)), 0.U) + when (lastCycleRedirect) { + // we recover the pointers in the next cycle after redirect + enqPtrExt := VecInit(enqPtrExt.map(_ - lastCycleCancelCount)) + }.otherwise { + enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) } + deqPtrExt := Mux(io.sbuffer(1).fire(), + VecInit(deqPtrExt.map(_ + 2.U)), + Mux(io.sbuffer(0).fire() || io.mmioStout.fire(), + VecInit(deqPtrExt.map(_ + 1.U)), + deqPtrExt + ) + ) + + val lastLastCycleRedirect = RegNext(lastCycleRedirect) + val dequeueCount = Mux(io.sbuffer(1).fire(), 2.U, Mux(io.sbuffer(0).fire() || io.mmioStout.fire(), 1.U, 0.U)) + val trueValidCounter = distanceBetween(enqPtrExt(0), deqPtrExt(0)) + validCounter := Mux(lastLastCycleRedirect, + trueValidCounter - dequeueCount, + validCounter + enqNumber - dequeueCount + ) + + allowEnqueue := Mux(io.brqRedirect.valid, + false.B, + Mux(lastLastCycleRedirect, + trueValidCounter <= (StoreQueueSize - RenameWidth).U, + validCounter + enqNumber <= (StoreQueueSize - RenameWidth).U + ) + ) + // debug info XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr)