Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 58 additions & 1 deletion multinode/multi_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,31 @@ func NewMultiNode[
c.lggr = c.eng.WithOptions(zap.AddStacktrace(zapcore.PanicLevel))

c.lggr.Debugf("The MultiNode is configured to use NodeSelectionMode: %s", selectionMode)
c.warnIfFinalizedBlockOutOfSyncFallbackUnsupported()

return c
}

// warnIfFinalizedBlockOutOfSyncFallbackUnsupported logs once per pool when nodes requested the fallback but some
// primary nodes (e.g. wrappers around a Node) cannot report health. Those nodes keep strict behaviour; see
// node.SetPoolChainInfoProvider.
func (c *MultiNode[CHAIN_ID, RPC]) warnIfFinalizedBlockOutOfSyncFallbackUnsupported() {
requested := false
for _, n := range c.primaryNodes {
if f, ok := n.(finalizedBlockOutOfSyncFallbackReporter); ok && f.finalizedBlockOutOfSyncFallbackEnabled() {
requested = true
break
}
}
if !requested {
return
}
if missing := c.nodesWithoutHealthSnapshot(); len(missing) > 0 {
c.lggr.Warnw("Finalized block out-of-sync fallback is enabled, but some nodes in the pool do not report health; "+
"keeping strict EnforceRepeatableRead behaviour for the whole pool", "nodes", missing)
}
}

func (c *MultiNode[CHAIN_ID, RPC]) ChainID() CHAIN_ID {
return c.chainID
}
Expand Down Expand Up @@ -275,6 +296,40 @@ func (c *MultiNode[CHAIN_ID, RPC]) LatestChainInfo(callerName string) (int, Chai
return nLiveNodes, ch
}

// nodesWithoutHealthSnapshot implements healthyPeersProvider.
func (c *MultiNode[CHAIN_ID, RPC]) nodesWithoutHealthSnapshot() []string {
var names []string
for _, n := range c.primaryNodes {
if _, ok := n.(healthSnapshotReporter); !ok {
names = append(names, n.Name())
}
}
return names
}

// HealthyPeersChainInfo implements healthyPeersProvider.
func (c *MultiNode[CHAIN_ID, RPC]) HealthyPeersChainInfo(callerName string) (int, ChainInfo) {
var nHealthy int
ch := ChainInfo{
TotalDifficulty: big.NewInt(0),
}
for _, n := range c.primaryNodes {
reporter, ok := n.(healthSnapshotReporter)
if !ok || n.Name() == callerName {
continue
}
healthy, latest := reporter.loadHealthSnapshot()
if !healthy {
continue
}
nHealthy++
ch.BlockNumber = max(ch.BlockNumber, latest.BlockNumber)
ch.FinalizedBlockNumber = max(ch.FinalizedBlockNumber, latest.FinalizedBlockNumber)
ch.TotalDifficulty = MaxTotalDifficulty(ch.TotalDifficulty, latest.TotalDifficulty)
}
return nHealthy, ch
}

// HighestUserObservations - returns highest ChainInfo ever observed by any user of the MultiNode
func (c *MultiNode[CHAIN_ID, RPC]) HighestUserObservations() ChainInfo {
ch := ChainInfo{
Expand Down Expand Up @@ -360,8 +415,10 @@ func (c *MultiNode[CHAIN_ID, RPC]) report(nodesStateInfo []nodeWithState) {
var dead int
counts := make(map[nodeState]int)
for i, n := range c.primaryNodes {
// Liveness (DeadSince, health errors) follows routing via State(); metrics use StateForMetrics(), which also
// exposes failures hidden by fallbacks. Routing can retain the last unhealthy node for optimistic retries, and
// a node serving a delayed finalized view counts as alive here but as FinalizedBlockOutOfSync in metrics.
state := n.State()
// Routing can retain the last unhealthy node for optimistic retries.
counts[n.StateForMetrics()]++
nodesStateInfo[i].State = state.String()
if state == nodeStateAlive {
Expand Down
43 changes: 42 additions & 1 deletion multinode/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net/url"
"sync"
"sync/atomic"
"time"

"github.com/smartcontractkit/chainlink-common/pkg/logger"
Expand Down Expand Up @@ -44,6 +45,13 @@ type FinalizedStateCheckConfig interface {
FinalizedStateCheckFailureThreshold() uint32
}

// FinalizedBlockOutOfSyncFallbackConfig is an optional, opt-in interface. When enabled, a node lagging on finalized
// block stays routable if no healthy peer offers a more recent finalized block, trading the EnforceRepeatableRead
// guarantee for liveness during such windows. Ignored unless EnforceRepeatableRead is enabled.
type FinalizedBlockOutOfSyncFallbackConfig interface {
FinalizedBlockOutOfSyncFallbackEnabled() bool
}

// FinalizedStateChecker is an optional interface for RPCClients that support finalized state checks.
// It is optional (not part of RPCClient) so non-EVM multinode consumers avoid boilerplate and review churn; see aliveLoop in node_lifecycle.go.
type FinalizedStateChecker interface {
Expand Down Expand Up @@ -72,6 +80,27 @@ type nodeMetrics interface {
IncrementFinalizedStateFailed(ctx context.Context, nodeName string)
}

// healthSnapshotReporter is kept out of Node so external implementations are unaffected.
type healthSnapshotReporter interface {
loadHealthSnapshot() (healthy bool, latest ChainInfo)
}

// finalizedBlockOutOfSyncFallbackReporter lets MultiNode tell whether any node requested the fallback.
type finalizedBlockOutOfSyncFallbackReporter interface {
finalizedBlockOutOfSyncFallbackEnabled() bool
}

// healthyPeersProvider is an optional extension of PoolChainInfoProvider, kept out of that exported interface so
// external implementations are unaffected. If the pool does not implement it, or cannot read the health of every
// primary node, the finalized block out-of-sync fallback stays disabled and strict EnforceRepeatableRead applies.
type healthyPeersProvider interface {
// HealthyPeersChainInfo returns the number of healthy nodes (excluding callerName), ignoring finalized-block lag,
// and the highest latest ChainInfo among them. Must not acquire node state locks: called from within Node.State().
HealthyPeersChainInfo(callerName string) (int, ChainInfo)
// nodesWithoutHealthSnapshot returns the names of primary nodes that do not implement healthSnapshotReporter.
nodesWithoutHealthSnapshot() []string
}

type Node[
CHAIN_ID ID,
RPC any,
Expand Down Expand Up @@ -129,11 +158,20 @@ type node[
finalizedStateChecker FinalizedStateChecker // set in NewNode when rpc implements FinalizedStateChecker
isLoadBalancedRPC bool

finalizedBlockOutOfSyncFallback bool // set in NewNode from FinalizedBlockOutOfSyncFallbackConfig
// healthyPeers is set in SetPoolChainInfoProvider only when the fallback is enabled and the pool supports it;
// nil means strict EnforceRepeatableRead behaviour.
healthyPeers healthyPeersProvider

stateMu sync.RWMutex // protects state* fields
state nodeState
stateHealthFailures healthCheckFailures
stateSyncIssues syncStatus

// healthSnapshot is read lock-free by peers from within their State(): taking a peer's stateMu there can deadlock.
healthSnapshot atomic.Bool
servingDelayedFinalized atomic.Bool

poolInfoProvider PoolChainInfoProvider

stopCh services.StopChan
Expand Down Expand Up @@ -190,6 +228,9 @@ func NewNode[
if fs, ok := any(rpc).(FinalizedStateChecker); ok {
n.finalizedStateChecker = fs
}
if cfg, ok := n.nodePoolCfg.(FinalizedBlockOutOfSyncFallbackConfig); ok {
n.finalizedBlockOutOfSyncFallback = cfg.FinalizedBlockOutOfSyncFallbackEnabled()
}
n.isLoadBalancedRPC = isLoadBalancedRPC
n.chainFamily = chainFamily
return n
Expand Down Expand Up @@ -244,7 +285,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) close() error {
defer n.stateMu.Unlock()

close(n.stopCh)
n.state = nodeStateClosed
n.setStateLocked(nodeStateClosed)
return nil
}

Expand Down
Loading
Loading