Backport extra sequencer metrics

execution/gethexec/sequencer.go

@@ -46,6 +46,10 @@ import (
 
 var (
 	sequencerBacklogGauge                   = metrics.NewRegisteredGauge("arb/sequencer/backlog", nil)
+	sequencerQueueNormalGauge               = metrics.NewRegisteredGauge("arb/sequencer/queue/normal", nil)
+	sequencerQueueTimeboostGauge            = metrics.NewRegisteredGauge("arb/sequencer/queue/timeboost", nil)
+	sequencerQueueNormalHistogram           = metrics.NewRegisteredHistogram("arb/sequencer/queue/histogram/normal", nil, metrics.NewBoundedHistogramSample())
+	sequencerQueueTimeboostHistogram        = metrics.NewRegisteredHistogram("arb/sequencer/queue/histogram/timeboost", nil, metrics.NewBoundedHistogramSample())
 	nonceCacheHitCounter                    = metrics.NewRegisteredCounter("arb/sequencer/noncecache/hit", nil)
 	nonceCacheMissCounter                   = metrics.NewRegisteredCounter("arb/sequencer/noncecache/miss", nil)
 	nonceCacheRejectedCounter               = metrics.NewRegisteredCounter("arb/sequencer/noncecache/rejected", nil)
@@ -54,12 +58,21 @@ var (
 	nonceFailureCacheOverflowCounter        = metrics.NewRegisteredCounter("arb/sequencer/noncefailurecache/overflow", nil)
 	blockCreationTimer                      = metrics.NewRegisteredHistogram("arb/sequencer/block/creation", nil, metrics.NewBoundedHistogramSample())
 	successfulBlocksCounter                 = metrics.NewRegisteredCounter("arb/sequencer/block/successful", nil)
+	blockTxSizeHistogram                    = metrics.NewRegisteredHistogram("arb/sequencer/block/txsize", nil, metrics.NewBoundedHistogramSample())
+	txSizeHistogram                         = metrics.NewRegisteredHistogram("arb/sequencer/transactions/txsize", nil, metrics.NewBoundedHistogramSample())
 	conditionalTxRejectedBySequencerCounter = metrics.NewRegisteredCounter("arb/sequencer/conditionaltx/rejected", nil)
 	conditionalTxAcceptedBySequencerCounter = metrics.NewRegisteredCounter("arb/sequencer/conditionaltx/accepted", nil)
 	l1GasPriceGauge                         = metrics.NewRegisteredGauge("arb/sequencer/l1gasprice", nil)
 	callDataUnitsBacklogGauge               = metrics.NewRegisteredGauge("arb/sequencer/calldataunitsbacklog", nil)
 	currentSurplusGauge                     = metrics.NewRegisteredGauge("arb/sequencer/currentsurplus", nil)
 	expectedSurplusGauge                    = metrics.NewRegisteredGauge("arb/sequencer/expectedsurplus", nil)
+	waitForTxHistogram                      = metrics.NewRegisteredHistogram("arb/sequencer/waitfortx", nil, metrics.NewBoundedHistogramSample())
+	// number of blocks ended because of block gas limit
+	gasFullBlocksCounter = metrics.NewRegisteredCounter("arb/sequencer/block/gasfull", nil)
+	// number of blocks ended because of txes data size limit
+	dataFullBlocksCounter = metrics.NewRegisteredCounter("arb/sequencer/block/datafull", nil)
+	// number of blocks ended because of exhausting the transactions to sequence
+	txExhaustedBlocksCounter = metrics.NewRegisteredCounter("arb/sequencer/block/txexhausted", nil)
 )
 
 type SequencerConfig struct {
@@ -613,6 +626,7 @@ func (s *Sequencer) PublishAuctionResolutionTransaction(ctx context.Context, tx
 		firstAppearance: time.Now(),
 		isTimeboosted:   false,
 	}
+	sequencerQueueTimeboostGauge.Update(int64(len(s.timeboostAuctionResolutionTxQueue)))
 	return nil
 }
 
@@ -716,7 +730,7 @@ func (s *Sequencer) publishTransactionToQueue(queueCtx context.Context, tx *type
 	case <-queueCtx.Done():
 		return queueCtx.Err()
 	}
-
+	sequencerQueueNormalGauge.Update(int64(len(s.txQueue)))
 	return nil
 }
 
@@ -1220,10 +1234,11 @@ func (s *Sequencer) createBlock(ctx context.Context) (returnValue bool) {
 	}()
 
 	var startOfReadingFromTxQueue time.Time
-
+	startOfBlockCreation := time.Now()
 	for {
 		if len(queueItems) == 1 {
 			startOfReadingFromTxQueue = time.Now()
+			waitForTxHistogram.Update(int64(time.Since(startOfBlockCreation)))
 		} else if len(queueItems) > 1 && time.Since(startOfReadingFromTxQueue) > config.ReadFromTxQueueTimeout {
 			break
 		}
@@ -1321,14 +1336,22 @@ func (s *Sequencer) createBlock(ctx context.Context) (returnValue bool) {
 	s.nonceCache.BeginNewBlock()
 	queueItems = s.precheckNonces(queueItems)
 	timeboostedTxs := make(map[common.Hash]struct{})
+	maxTxDataSize := s.config().MaxTxDataSize
 	hooks := MakeSequencingHooks(
 		queueItems,
-		s.config().MaxTxDataSize,
+		maxTxDataSize,
 		s.preTxFilter,
 		s.postTxFilter,
 		nil,
 	)
 
+	txQueueLen := int64(len(s.txQueue))
+	timeboostTxQueueLen := int64(len(s.timeboostAuctionResolutionTxQueue))
+	sequencerQueueNormalGauge.Update(txQueueLen)
+	sequencerQueueNormalHistogram.Update(txQueueLen)
+	sequencerQueueTimeboostGauge.Update(timeboostTxQueueLen)
+	sequencerQueueTimeboostHistogram.Update(timeboostTxQueueLen)
+
 	for _, queueItem := range queueItems {
 		if queueItem.isTimeboosted {
 			timeboostedTxs[queueItem.tx.Hash()] = struct{}{}
@@ -1430,14 +1453,27 @@ func (s *Sequencer) createBlock(ctx context.Context) (returnValue bool) {
 	}
 
 	madeBlock := false
+	blockLimitReached := false
+	var blockTxSize int64
 	for i, err := range hooks.txErrors {
+		queueItem := queueItems[i]
 		if err == nil {
 			madeBlock = true
+			blockTxSize += int64(queueItem.txSize)
+			txSizeHistogram.Update(int64(queueItem.txSize))
 		}
-		queueItem := queueItems[i]
 		if errors.Is(err, core.ErrGasLimitReached) {
 			// There's not enough gas left in the block for this tx.
 			if madeBlock {
+				// make sure that this was last tx processed (it should always be true)
+				if i == len(hooks.txErrors)-1 {
+					if blockTxSize+int64(queueItem.txSize) > int64(maxTxDataSize) {
+						dataFullBlocksCounter.Inc(1)
+					} else {
+						gasFullBlocksCounter.Inc(1)
+					}
+					blockLimitReached = true
+				}
 				// There was already an earlier tx in the block; retry in a fresh block.
 				s.txRetryQueue.Push(queueItem)
 				continue
@@ -1454,6 +1490,12 @@ func (s *Sequencer) createBlock(ctx context.Context) (returnValue bool) {
 		}
 		queueItem.returnResult(err)
 	}
+	if madeBlock {
+		blockTxSizeHistogram.Update(blockTxSize)
+		if !blockLimitReached && hooks.sequencedQueueItemsCount == len(hooks.queueItems) {
+			txExhaustedBlocksCounter.Inc(1)
+		}
+	}
 	return madeBlock
 }