Skip to content

Allow witness signature collection to fail open #634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AlCutter merged 3 commits into from
May 12, 2025
Merged

Allow witness signature collection to fail open #634

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b0ddc8
Remove unnecessary getter for Witnesses.
AlCutter May 9, 2025
1aab8a7
Pass FailOpen via a new WitnessOptions arg
AlCutter May 9, 2025
db837d2
Add metric
AlCutter May 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 39 additions & 8 deletions append_lifecycle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ var (
appenderNextIndex metric.Int64Gauge
appenderSignedSize metric.Int64Gauge
appenderWitnessedSize metric.Int64Gauge
appenderWitnessRequests metric.Int64Counter

followerEntriesProcessed metric.Int64Gauge
followerLag metric.Int64Gauge
Expand Down Expand Up @@ -139,6 +140,7 @@ func init() {
if err != nil {
klog.Exitf("Failed to create followerEntriesProcessed metric: %v", err)
}

followerLag, err = meter.Int64Gauge(
"tessera.follower.lag",
metric.WithDescription("Number of unprocessed entries in the current integrated tree"),
Expand All @@ -147,6 +149,14 @@ func init() {
klog.Exitf("Failed to create followerLag metric: %v", err)
}

appenderWitnessRequests, err = meter.Int64Counter(
"tessera.appender.witness.requests",
metric.WithDescription("Number of attempts to witness a log checkpoint"),
metric.WithUnit("{call}"))
if err != nil {
klog.Exitf("Failed to create appenderWitnessRequests metric: %v", err)
}

}

// Add adds a new entry to be sequenced.
Expand Down Expand Up @@ -496,6 +506,7 @@ type AppendOptions struct {

checkpointInterval time.Duration
witnesses WitnessGroup
witnessOpts WitnessOptions

addDecorators []func(AddFn) AddFn
followers []Follower
Expand All @@ -511,7 +522,7 @@ func (o AppendOptions) valid() error {

// CheckpointPublisher returns a function which should be used to create, sign, and potentially witness a new checkpoint.
func (o AppendOptions) CheckpointPublisher(lr LogReader, httpClient *http.Client) func(context.Context, uint64, []byte) ([]byte, error) {
wg := witness.NewWitnessGateway(o.Witnesses(), httpClient, lr.ReadTile)
wg := witness.NewWitnessGateway(o.witnesses, httpClient, lr.ReadTile)
return func(ctx context.Context, size uint64, root []byte) ([]byte, error) {
ctx, span := tracer.Start(ctx, "tessera.CheckpointPublisher")
defer span.End()
Expand All @@ -522,13 +533,21 @@ func (o AppendOptions) CheckpointPublisher(lr LogReader, httpClient *http.Client
}
appenderSignedSize.Record(ctx, otel.Clamp64(size))

witAttr := []attribute.KeyValue{}
cp, err = wg.Witness(ctx, cp)
if err != nil {
return nil, err
if !o.witnessOpts.FailOpen {
Copy link
Contributor

@mhutchinson mhutchinson May 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should have metrics so that operators can safely determine when they are able to switch modes.

Copy link
Collaborator Author

@AlCutter AlCutter May 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea, done

appenderWitnessRequests.Add(ctx, 1, metric.WithAttributes(attribute.String("error.type", "failed")))
return nil, err
}
klog.Warningf("WitnessGateway: failing-open despite error: %v", err)
witAttr = append(witAttr, attribute.String("error.type", "failed_open"))
}

appenderWitnessRequests.Add(ctx, 1, metric.WithAttributes(witAttr...))
appenderWitnessedSize.Record(ctx, otel.Clamp64(size))

return cp, err
return cp, nil
}
}

Expand All @@ -552,10 +571,6 @@ func (o AppendOptions) CheckpointInterval() time.Duration {
return o.checkpointInterval
}

func (o AppendOptions) Witnesses() WitnessGroup {
return o.witnesses
}

// WithCheckpointSigner is an option for setting the note signer and verifier to use when creating and parsing checkpoints.
// This option is mandatory for creating logs where the checkpoint is signed locally, e.g. in
// the Appender mode. This does not need to be provided where the storage will be used to mirror
Expand Down Expand Up @@ -660,7 +675,23 @@ func (o *AppendOptions) WithCheckpointInterval(interval time.Duration) *AppendOp
//
// If this method is not called, then the default empty WitnessGroup will be used, which contacts zero
// witnesses and requires zero witnesses in order to publish.
func (o *AppendOptions) WithWitnesses(witnesses WitnessGroup) *AppendOptions {
func (o *AppendOptions) WithWitnesses(witnesses WitnessGroup, opts *WitnessOptions) *AppendOptions {
if opts == nil {
opts = &WitnessOptions{}
}

o.witnesses = witnesses
o.witnessOpts = *opts
return o
}

// WitnessOptions contains extra optional configuration for how Tessera should use/interact with
// a user-provided WitnessGroup policy.
type WitnessOptions struct {
// FailOpen controls whether a checkpoint, for which the witness policy was unable to be met,
// should still be published.
//
// This setting is intended only for facilitating early "non-blocking" adoption of witnessing,
// and will be disabled and/or removed in the future.
FailOpen bool
}
Loading