From 07f705e1e237e0464bcace198fc014c0d0a3721c Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Tue, 6 Oct 2026 11:41:29 -0400 Subject: [PATCH 1/4] TxMgr - cap unkown error retries --- chains/txmgr/broadcaster.go | 57 +++++++++++++++++++++++++++++++++++- chains/txmgr/types/config.go | 6 ++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index 084781fd..4a0af488 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -113,6 +113,9 @@ type Broadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, T // Each key has its own trigger triggers map[ADDR]chan struct{} + unknownErrsMu sync.Mutex + unknownErrs map[ADDR]unknownErrState + chStop services.StopChan wg sync.WaitGroup @@ -120,6 +123,12 @@ type Broadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, T isStarted bool } +type unknownErrState struct { + txID int64 + count uint32 + firstSeen time.Time +} + func NewBroadcaster[CID chains.ID, HEAD chains.Head[BHASH], ADDR chains.Hashable, THASH chains.Hashable, BHASH chains.Hashable, SEQ chains.Sequence, FEE fees.Fee]( txStore types.TransactionStore[ADDR, CID, THASH, BHASH, SEQ, FEE], client types.TransactionClient[CID, ADDR, THASH, BHASH, SEQ, FEE], @@ -192,6 +201,11 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) startInternal(ct eb.triggersMu.Lock() eb.triggers = make(map[ADDR]chan struct{}) eb.triggersMu.Unlock() + + eb.unknownErrsMu.Lock() + eb.unknownErrs = make(map[ADDR]unkownErrState) + eb.unknownErrsMu.Unlock() + eb.wg.Add(1) go eb.loadAndMonitor() @@ -495,6 +509,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress } if errType == multinode.Fatal || errType == multinode.TerminallyStuck { + eb.clearUnknownErrState(etx.FromAddress) eb.SvcErrBuffer.Append(err) etx.Error = null.StringFrom(err.Error()) return eb.saveFatallyErroredTransaction(lgr, &etx), true @@ -550,6 +565,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress if err != nil { return err, true } + eb.clearUnknownErrState(etx.FromAddress) eb.metrics.IncrementNumBroadcastedTxs(ctx) // Increment sequence if successfully broadcasted eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) @@ -620,6 +636,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress eb.metrics.IncrementNumBroadcastedTxs(ctx) // Increment sequence if successfully broadcasted eb.sequenceTracker.GenerateNextSequence(etx.FromAddress, *etx.Sequence) + eb.clearUnknownErrState(etx.FromAddress) return err, true } // Either the unknown error prevented the transaction from being mined, or @@ -627,11 +644,49 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress // remote RPC. // // In all cases, the best thing we can do is go into a retry loop and keep - // trying to send the transaction over again. + // trying to send the transaction over again, unless the operator has opted + // into capping these retries. + if exceeded, count, elapsed := eb.recordUnknownErr(etx.FromAddress, etx.ID); exceeded { + eb.clearUnknownErrState(etx.FromAddress) + fatalError := fmt.Errorf("giving up on transaction after %d unkown errors over %s (MaxUnknownErrorRetries=%d, UnknownErrorRetryTimeout=%s): %w", + count, elapsed, eb.txConfig.MaxUnknownErrorRetries(), eb.txConfig.UnknownErrorRetryTimeout(), err) + lgr.Criticalw("Unknown error retry limit reached, marking transaction as fatally errored. The sequence will be reused by the next transaction", + "etxID", etx.ID, "attempt", attempt, "unknownErrCount", count, "elapsed", elapsed, "err", err) + eb.SvcErrBuffer.Append(fatalError) + etx.Error = null.StringFrom(fatalError.Error()) + return eb.saveFatallyErroredTransaction(lgr, &etx), true + } + return fmt.Errorf("retryable error while sending transaction %s (tx ID %d): %w", attempt.Hash.String(), etx.ID, err), true } } +func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) recordUnknownErr(addr ADDR, txID int64) (exceeded bool, count uint32, elapsed time.Duration) { + maxRetries := eb.txConfig.MaxUnknownErrorRetries() + timeout := eb.txConfig.UnknownErrorRetryTimeout() + if maxRetries == 0 && timeout <= 0 { + return false, 0, 0 + } + + eb.unknownErrsMu.Lock() + defer eb.unknownErrsMu.Unlock() + + state, ok := eb.unknownErrs[addr] + if !ok || state.txID != txID { + state = unknownErrState{txID: txID, firstSeen: time.Now()} + } + state.count++ + eb.unknownErrs[addr] = state + elapsed = time.Since(state.firstSeen) + exceeded = (maxRetries > 0 && state.count > maxRetries) || (timeout > 0 && elapsed >= timeout) + + return exceeded, state.count, elapsed +} +func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) clearUnknownErrState(addr ADDR) { + eb.unknownErrsMu.Lock() + defer eb.unknownErrsMu.Unlock() + delete(eb.unknownErrs, addr) +} func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) validateOnChainSequence(ctx context.Context, lgr logger.SugaredLogger, errType multinode.SendTxReturnCode, err error, etx types.Tx[CID, ADDR, THASH, BHASH, SEQ, FEE], retryCount int) (multinode.SendTxReturnCode, error) { // Only check if sequence was incremented if broadcast was successful, otherwise return the existing err type if errType != multinode.Successful { diff --git a/chains/txmgr/types/config.go b/chains/txmgr/types/config.go index 1ab334b3..67616ecf 100644 --- a/chains/txmgr/types/config.go +++ b/chains/txmgr/types/config.go @@ -32,6 +32,12 @@ type BroadcasterFeeConfig interface { type BroadcasterTransactionsConfig interface { MaxInFlight() uint32 + // MaxUnknownErrorRetries is the number of consecutive Unknown send errors after which an in-progress tx is + // marked as fatally errored. 0 means unlimited. + MaxUnknownErrorRetries() uint32 + // UnknownErrorRetryTimeout is how long an in-progress tx may keep failing with Unknown send errors before it is + // marked as fatally errored. 0 means unlimited. + UnknownErrorRetryTimeout() time.Duration } type BroadcasterListenerConfig interface { From 44fa26a56c6c8e527522c2b47447bcf155ca67f5 Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Tue, 6 Oct 2026 13:53:56 -0400 Subject: [PATCH 2/4] fix typo --- chains/txmgr/broadcaster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index 4a0af488..c09a9f2a 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -203,7 +203,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) startInternal(ct eb.triggersMu.Unlock() eb.unknownErrsMu.Lock() - eb.unknownErrs = make(map[ADDR]unkownErrState) + eb.unknownErrs = make(map[ADDR]unknownErrState) eb.unknownErrsMu.Unlock() eb.wg.Add(1) From 0cd3027735c2bbf738a3e8f269e95b36be9c81cd Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Tue, 6 Oct 2026 14:20:00 -0400 Subject: [PATCH 3/4] fix typo --- chains/txmgr/broadcaster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index c09a9f2a..2276db7b 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -648,7 +648,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress // into capping these retries. if exceeded, count, elapsed := eb.recordUnknownErr(etx.FromAddress, etx.ID); exceeded { eb.clearUnknownErrState(etx.FromAddress) - fatalError := fmt.Errorf("giving up on transaction after %d unkown errors over %s (MaxUnknownErrorRetries=%d, UnknownErrorRetryTimeout=%s): %w", + fatalError := fmt.Errorf("giving up on transaction after %d unknown errors over %s (MaxUnknownErrorRetries=%d, UnknownErrorRetryTimeout=%s): %w", count, elapsed, eb.txConfig.MaxUnknownErrorRetries(), eb.txConfig.UnknownErrorRetryTimeout(), err) lgr.Criticalw("Unknown error retry limit reached, marking transaction as fatally errored. The sequence will be reused by the next transaction", "etxID", etx.ID, "attempt", attempt, "unknownErrCount", count, "elapsed", elapsed, "err", err) From d5f9ab882cffd05b7d4a1cf0f990b0a3ddd9f976 Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Thu, 8 Oct 2026 15:45:15 -0400 Subject: [PATCH 4/4] rm MaxUnknownErrorRetries --- chains/txmgr/broadcaster.go | 11 +++++------ chains/txmgr/types/config.go | 3 --- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/chains/txmgr/broadcaster.go b/chains/txmgr/broadcaster.go index 2276db7b..2bacab36 100644 --- a/chains/txmgr/broadcaster.go +++ b/chains/txmgr/broadcaster.go @@ -645,11 +645,11 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress // // In all cases, the best thing we can do is go into a retry loop and keep // trying to send the transaction over again, unless the operator has opted - // into capping these retries. + // into capping the total retry duration. if exceeded, count, elapsed := eb.recordUnknownErr(etx.FromAddress, etx.ID); exceeded { eb.clearUnknownErrState(etx.FromAddress) - fatalError := fmt.Errorf("giving up on transaction after %d unknown errors over %s (MaxUnknownErrorRetries=%d, UnknownErrorRetryTimeout=%s): %w", - count, elapsed, eb.txConfig.MaxUnknownErrorRetries(), eb.txConfig.UnknownErrorRetryTimeout(), err) + fatalError := fmt.Errorf("giving up on transaction after %d unknown errors over %s (UnknownErrorRetryTimeout=%s): %w", + count, elapsed, eb.txConfig.UnknownErrorRetryTimeout(), err) lgr.Criticalw("Unknown error retry limit reached, marking transaction as fatally errored. The sequence will be reused by the next transaction", "etxID", etx.ID, "attempt", attempt, "unknownErrCount", count, "elapsed", elapsed, "err", err) eb.SvcErrBuffer.Append(fatalError) @@ -661,9 +661,8 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) handleInProgress } } func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) recordUnknownErr(addr ADDR, txID int64) (exceeded bool, count uint32, elapsed time.Duration) { - maxRetries := eb.txConfig.MaxUnknownErrorRetries() timeout := eb.txConfig.UnknownErrorRetryTimeout() - if maxRetries == 0 && timeout <= 0 { + if timeout <= 0 { return false, 0, 0 } @@ -678,7 +677,7 @@ func (eb *Broadcaster[CID, HEAD, ADDR, THASH, BHASH, SEQ, FEE]) recordUnknownErr eb.unknownErrs[addr] = state elapsed = time.Since(state.firstSeen) - exceeded = (maxRetries > 0 && state.count > maxRetries) || (timeout > 0 && elapsed >= timeout) + exceeded = elapsed >= timeout return exceeded, state.count, elapsed } diff --git a/chains/txmgr/types/config.go b/chains/txmgr/types/config.go index 67616ecf..95a962b1 100644 --- a/chains/txmgr/types/config.go +++ b/chains/txmgr/types/config.go @@ -32,9 +32,6 @@ type BroadcasterFeeConfig interface { type BroadcasterTransactionsConfig interface { MaxInFlight() uint32 - // MaxUnknownErrorRetries is the number of consecutive Unknown send errors after which an in-progress tx is - // marked as fatally errored. 0 means unlimited. - MaxUnknownErrorRetries() uint32 // UnknownErrorRetryTimeout is how long an in-progress tx may keep failing with Unknown send errors before it is // marked as fatally errored. 0 means unlimited. UnknownErrorRetryTimeout() time.Duration