diff --git a/pkg/clusterservice/cluster.go b/pkg/clusterservice/cluster.go index e0e233e1abd0b..4021d9819f909 100644 --- a/pkg/clusterservice/cluster.go +++ b/pkg/clusterservice/cluster.go @@ -30,6 +30,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/stopper" logpb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) // GetMOCluster get mo cluster from process level runtime @@ -187,6 +188,16 @@ func WithDisableRefresh() Option { } } +// WithGlobalSysVarRoutingFilter makes the cluster snapshot suitable for SQL +// routing by excluding CNs that have not applied the durable global-system- +// variable watermark. It must not be enabled for the process-wide discovery +// view used by lockservice and other internal components. +func WithGlobalSysVarRoutingFilter() Option { + return func(c *cluster) { + c.options.globalSysVarRoutingFilter = true + } +} + type cluster struct { logger *log.MOLogger stopper *stopper.Stopper @@ -207,11 +218,13 @@ type cluster struct { // Correctness: readyOnce.Do guarantees that ready.Store(true) happens before // close(readyC), so if readyC is closed (i.e., <-readyC returns), ready is // guaranteed to be true. If ready.Load() returns false, we fall back to channel wait. - ready atomic.Bool - services atomic.Pointer[services] - regexpCache *regexpCache - options struct { - disableRefresh bool + ready atomic.Bool + services atomic.Pointer[services] + globalSysVarCommitTS atomic.Pointer[timestamp.Timestamp] + regexpCache *regexpCache + options struct { + disableRefresh bool + globalSysVarRoutingFilter bool } } @@ -526,6 +539,17 @@ func (c *cluster) refreshWithContext(ctx context.Context) error { new := &services{} for _, cn := range details.CNStores { + if c.options.globalSysVarRoutingFilter && + !details.GlobalSysVarCommitTS.IsEmpty() && + cn.GlobalSysVarCommitTS.Less(details.GlobalSysVarCommitTS) { + if c.logger.Enabled(zap.DebugLevel) { + c.logger.Debug("cn service fenced by global sysvar watermark", + zap.String("cn", cn.UUID), + zap.String("required", details.GlobalSysVarCommitTS.DebugString()), + zap.String("applied", cn.GlobalSysVarCommitTS.DebugString())) + } + continue + } v := newCNService(cn) new.addCN([]metadata.CNService{v}) if c.logger.Enabled(zap.DebugLevel) { @@ -549,6 +573,7 @@ func (c *cluster) refreshWithContext(ctx context.Context) error { new.tn = new.tn[:1] } c.services.Store(new) + c.publishGlobalSysVarCommitTS(details.GlobalSysVarCommitTS) c.readyOnce.Do(func() { c.ready.Store(true) close(c.readyC) @@ -556,6 +581,33 @@ func (c *cluster) refreshWithContext(ctx context.Context) error { return nil } +func (c *cluster) publishGlobalSysVarCommitTS(ts timestamp.Timestamp) { + if ts.IsEmpty() { + return + } + for { + current := c.globalSysVarCommitTS.Load() + if current != nil && current.GreaterEq(ts) { + return + } + next := ts + if c.globalSysVarCommitTS.CompareAndSwap(current, &next) { + return + } + } +} + +// GlobalSysVarCommitTS returns the durable routing watermark represented by +// service's currently published CN snapshot. +func GlobalSysVarCommitTS(service MOCluster) timestamp.Timestamp { + if c, ok := service.(*cluster); ok { + if ts := c.globalSysVarCommitTS.Load(); ts != nil { + return *ts + } + } + return timestamp.Timestamp{} +} + func (c *cluster) acquireRefresh(ctx context.Context) error { c.refreshGateOnce.Do(func() { c.refreshGateC = make(chan struct{}, 1) diff --git a/pkg/clusterservice/cluster_test.go b/pkg/clusterservice/cluster_test.go index 011eedc539b5e..2bac02d0db00c 100644 --- a/pkg/clusterservice/cluster_test.go +++ b/pkg/clusterservice/cluster_test.go @@ -28,6 +28,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/runtime" logpb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) func TestClusterReady(t *testing.T) { @@ -308,13 +309,21 @@ func TestCluster_GetTNService(t *testing.T) { func runClusterTest( refreshInterval time.Duration, fn func(*testHAKeeperClient, *cluster), +) { + runClusterTestWithOptions(refreshInterval, fn) +} + +func runClusterTestWithOptions( + refreshInterval time.Duration, + fn func(*testHAKeeperClient, *cluster), + opts ...Option, ) { sid := "" runtime.RunTest( sid, func(rt runtime.Runtime) { hc := &testHAKeeperClient{} - c := NewMOCluster(sid, hc, refreshInterval) + c := NewMOCluster(sid, hc, refreshInterval, opts...) defer c.Close() fn(hc, c.(*cluster)) }, @@ -349,6 +358,63 @@ func (c *testHAKeeperClient) addTN(tick uint64, serviceIDs ...string) { } } +func TestClusterGlobalSysVarWatermarkFilteringIsRoutingOnly(t *testing.T) { + runClusterTestWithOptions( + time.Hour, + func(hc *testHAKeeperClient, c *cluster) { + commitTS := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 1} + hc.Lock() + hc.value = logpb.ClusterDetails{ + GlobalSysVarCommitTS: commitTS, + CNStores: []logpb.CNStore{ + {UUID: "cn-a", WorkState: metadata.WorkState_Working, GlobalSysVarCommitTS: commitTS}, + {UUID: "cn-b", WorkState: metadata.WorkState_Working}, + }, + } + hc.Unlock() + + c.ForceRefresh(true) + require.Equal(t, []string{"cn-a"}, routableCNIDs(c)) + + hc.Lock() + hc.value.CNStores[1].GlobalSysVarCommitTS = commitTS + hc.Unlock() + c.ForceRefresh(true) + require.ElementsMatch(t, []string{"cn-a", "cn-b"}, routableCNIDs(c)) + }, + WithGlobalSysVarRoutingFilter(), + ) + + runClusterTest( + time.Hour, + func(hc *testHAKeeperClient, c *cluster) { + commitTS := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 1} + hc.Lock() + hc.value = logpb.ClusterDetails{ + GlobalSysVarCommitTS: commitTS, + CNStores: []logpb.CNStore{ + {UUID: "cn-a", WorkState: metadata.WorkState_Working, GlobalSysVarCommitTS: commitTS}, + {UUID: "cn-b", WorkState: metadata.WorkState_Working}, + }, + } + hc.Unlock() + + c.ForceRefresh(true) + require.ElementsMatch(t, []string{"cn-a", "cn-b"}, routableCNIDs(c), + "general service discovery must not be filtered by SQL routing admission") + }, + ) +} + +func routableCNIDs(c *cluster) []string { + var ids []string + c.GetCNService(NewSelector(), func(cn metadata.CNService) bool { + ids = append(ids, cn.ServiceID) + return true + }) + return ids +} + func TestNewTNServicePreservesAutoIncrEpochFenceCapability(t *testing.T) { service := newTNService(logpb.TNStore{ UUID: "tn-new", diff --git a/pkg/cnservice/server.go b/pkg/cnservice/server.go index 20b8e6779c405..4e67e835a79bb 100644 --- a/pkg/cnservice/server.go +++ b/pkg/cnservice/server.go @@ -155,15 +155,17 @@ func NewService( UUID: cfg.UUID, Role: metadata.MustParseCNRole(cfg.Role), }, - cfg: cfg, - logger: logutil.GetGlobalLogger().Named("cn-service"), - metadataFS: metadataFS, - etlFS: etlFS, - fileService: fileService, - sessionMgr: queryservice.NewSessionManager(), - addressMgr: address.NewAddressManager(cfg.ServiceHost, cfg.PortBase), - gossipNode: gossipNode, - } + cfg: cfg, + logger: logutil.GetGlobalLogger().Named("cn-service"), + metadataFS: metadataFS, + etlFS: etlFS, + fileService: fileService, + sessionMgr: queryservice.NewSessionManager(), + addressMgr: address.NewAddressManager(cfg.ServiceHost, cfg.PortBase), + gossipNode: gossipNode, + globalSysVarGeneration: uuid.NewString(), + } + srv.initControlChannels() srv.colexecServer = colexec.NewServer(cfg.UUID) srv.requestHandler = func(ctx context.Context, @@ -412,6 +414,15 @@ func (s *service) Start() (err error) { if err = s.bootstrap(); err != nil { return err } + admissionCtx, admissionCancel := context.WithTimeout( + context.Background(), s.cfg.HAKeeper.DiscoveryTimeout.Duration) + err = s.waitGlobalSysVarAdmission(admissionCtx) + if err != nil { + err = moerr.AttachCause(admissionCtx, err) + admissionCancel() + return err + } + admissionCancel() s.initSqlWriterFactory() diff --git a/pkg/cnservice/server_heartbeat.go b/pkg/cnservice/server_heartbeat.go index fde73e2b1eb9f..d8a6f20a47d69 100644 --- a/pkg/cnservice/server_heartbeat.go +++ b/pkg/cnservice/server_heartbeat.go @@ -22,9 +22,11 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/system" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/logservice" "github.com/matrixorigin/matrixone/pkg/logutil" logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/version" ) @@ -72,14 +74,154 @@ func (s *service) heartbeatTask(ctx context.Context) { } func (s *service) controlTask(ctx context.Context) { - s.commandPollWakeup = make(chan struct{}, 1) + s.initControlChannels() commandDone := make(chan struct{}) + watermarkDone := make(chan struct{}) go func() { defer close(commandDone) s.commandTask(ctx) }() + go func() { + defer close(watermarkDone) + s.globalSysVarWatermarkTask(ctx) + }() s.heartbeatTask(ctx) <-commandDone + <-watermarkDone +} + +func (s *service) initControlChannels() { + s.controlChannelsOnce.Do(func() { + if s.commandPollWakeup == nil { + s.commandPollWakeup = make(chan struct{}, 1) + } + if s.globalSysVarWakeup == nil { + s.globalSysVarWakeup = make(chan struct{}, 1) + } + if s.globalSysVarAppliedC == nil { + s.globalSysVarAppliedC = make(chan struct{}, 1) + } + }) +} + +func (s *service) observeGlobalSysVarCommitTS(ts timestamp.Timestamp) { + if ts.IsEmpty() { + return + } + for { + current := s.globalSysVarDesired.Load() + if current != nil && current.GreaterEq(ts) { + return + } + next := ts + if s.globalSysVarDesired.CompareAndSwap(current, &next) { + select { + case s.globalSysVarWakeup <- struct{}{}: + default: + } + return + } + } +} + +func (s *service) publishGlobalSysVarCommitTS(ts timestamp.Timestamp) { + for { + current := s.globalSysVarApplied.Load() + if current != nil && current.GreaterEq(ts) { + return + } + next := ts + if s.globalSysVarApplied.CompareAndSwap(current, &next) { + select { + case s.globalSysVarAppliedC <- struct{}{}: + default: + } + return + } + } +} + +func (s *service) renewServingLease() { + duration := s.cfg.HAKeeper.HeatbeatInterval.Duration + + s.cfg.HAKeeper.HeatbeatTimeout.Duration + deadline := time.Now().Add(duration) + s.servingLeaseDeadline.Store(&deadline) +} + +func (s *service) revokeServingLease() { + s.servingLeaseDeadline.Store(nil) +} + +func (s *service) globalSysVarCaughtUp() bool { + desired := s.globalSysVarDesired.Load() + if desired == nil || desired.IsEmpty() { + return true + } + applied := s.globalSysVarApplied.Load() + return applied != nil && applied.GreaterEq(*desired) +} + +// CanAcceptNewConnections implements frontend.SQLConnectionAdmissionController. +func (s *service) CanAcceptNewConnections() bool { + deadline := s.servingLeaseDeadline.Load() + return deadline != nil && time.Now().Before(*deadline) && s.globalSysVarCaughtUp() +} + +func (s *service) waitGlobalSysVarAdmission(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-s.hakeeperConnected: + } + for !s.globalSysVarCaughtUp() { + select { + case <-ctx.Done(): + return ctx.Err() + case <-s.globalSysVarAppliedC: + } + } + return nil +} + +func (s *service) globalSysVarWatermarkTask(ctx context.Context) { + retry := time.NewTimer(time.Hour) + if !retry.Stop() { + <-retry.C + } + defer retry.Stop() + var retryC <-chan time.Time + for { + desired := s.globalSysVarDesired.Load() + applied := s.globalSysVarApplied.Load() + if desired != nil && (applied == nil || applied.Less(*desired)) { + if err := s._txnClient.SyncLatestCommitTSWithContext(ctx, *desired); err != nil { + if ctx.Err() != nil { + return + } + s.logger.Error("failed to apply global sysvar commit timestamp", zap.Error(err)) + retry.Reset(100 * time.Millisecond) + retryC = retry.C + } else { + s.publishGlobalSysVarCommitTS(*desired) + retryC = nil + continue + } + } + select { + case <-ctx.Done(): + return + case <-s.globalSysVarWakeup: + if retryC != nil && !retry.Stop() { + select { + case <-retry.C: + default: + } + } + retryC = nil + case <-retryC: + retryC = nil + } + } } func (s *service) commandTask(ctx context.Context) { @@ -202,6 +344,11 @@ func (s *service) heartbeat(ctx context.Context) { CommitID: version.CommitID, AckedCommandBatchID: s.ackedCommandBatchID.Load(), CommandDeliveryAckSupported: true, + GlobalSysVarGeneration: s.globalSysVarGeneration, + ProtocolVersion: defines.MORPCLatestVersion, + } + if applied := s.globalSysVarApplied.Load(); applied != nil { + hb.GlobalSysVarCommitTS = *applied } if s.gossipNode != nil { hb.GossipAddress = s.gossipServiceAddr() @@ -213,6 +360,7 @@ func (s *service) heartbeat(ctx context.Context) { cb, err := s._hakeeperClient.SendCNHeartbeat(ctx2, hb) s.heartbeatInFlight.Store(false) if err != nil { + s.revokeServingLease() s.commandPollNeeded.Store(true) s.notifyCommandPoll() err = moerr.AttachCause(ctx2, err) @@ -221,6 +369,7 @@ func (s *service) heartbeat(ctx context.Context) { return } if ctx2.Err() != nil { + s.revokeServingLease() s.commandPollNeeded.Store(true) s.notifyCommandPoll() return @@ -228,17 +377,20 @@ func (s *service) heartbeat(ctx context.Context) { s.commandPollNeeded.Store(false) s.notifyCommandPoll() + s.config.DecrCount() + s.handleHeartbeatResponse(hb.AckedCommandBatchID, cb) + s.renewServingLease() + select { case <-s.hakeeperConnected: default: s.initTaskServiceHolder() close(s.hakeeperConnected) } - s.config.DecrCount() - s.handleHeartbeatResponse(hb.AckedCommandBatchID, cb) } func (s *service) handleCommandBatch(batch logservicepb.CommandBatch) { + s.observeGlobalSysVarCommitTS(batch.GlobalSysVarCommitTS) s.commandMu.Lock() defer s.commandMu.Unlock() s.handleCommandBatchLocked(batch) @@ -286,6 +438,7 @@ func (s *service) handleHeartbeatResponse( sentAck uint64, batch logservicepb.CommandBatch, ) { + s.observeGlobalSysVarCommitTS(batch.GlobalSysVarCommitTS) s.commandMu.Lock() defer s.commandMu.Unlock() if sentAck != 0 && sentAck == s.lastCommandBatchID && diff --git a/pkg/cnservice/server_heartbeat_test.go b/pkg/cnservice/server_heartbeat_test.go index f7a371b0ad98f..7127591d62f7b 100644 --- a/pkg/cnservice/server_heartbeat_test.go +++ b/pkg/cnservice/server_heartbeat_test.go @@ -21,14 +21,143 @@ import ( "testing" "time" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "github.com/matrixorigin/matrixone/pkg/defines" + mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" "github.com/matrixorigin/matrixone/pkg/logutil" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/taskservice" "github.com/matrixorigin/matrixone/pkg/util" ) +func TestGlobalSysVarWatermarkTaskAppliesLatestObservedCommit(t *testing.T) { + ctrl := gomock.NewController(t) + txnClient := mock_frontend.NewMockTxnClient(ctrl) + first := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 1} + latest := timestamp.Timestamp{PhysicalTime: 200, LogicalTime: 2} + firstEntered := make(chan struct{}) + firstRelease := make(chan struct{}) + txnClient.EXPECT().SyncLatestCommitTSWithContext(gomock.Any(), first). + DoAndReturn(func(context.Context, timestamp.Timestamp) error { + close(firstEntered) + <-firstRelease + return nil + }) + txnClient.EXPECT().SyncLatestCommitTSWithContext(gomock.Any(), latest).Return(nil) + + s := &service{ + _txnClient: txnClient, + globalSysVarWakeup: make(chan struct{}, 1), + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + s.globalSysVarWatermarkTask(ctx) + }() + + s.observeGlobalSysVarCommitTS(first) + select { + case <-firstEntered: + case <-time.After(time.Second): + t.Fatal("watermark worker did not start the first visibility wait") + } + s.observeGlobalSysVarCommitTS(latest) + close(firstRelease) + require.Eventually(t, func() bool { + applied := s.globalSysVarApplied.Load() + return applied != nil && applied.Equal(latest) + }, time.Second, time.Millisecond) + + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("watermark worker did not stop after cancellation") + } +} + +func TestGlobalSysVarWatermarkTaskCancelsVisibilityWait(t *testing.T) { + ctrl := gomock.NewController(t) + txnClient := mock_frontend.NewMockTxnClient(ctrl) + ts := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 1} + entered := make(chan struct{}) + txnClient.EXPECT().SyncLatestCommitTSWithContext(gomock.Any(), ts). + DoAndReturn(func(ctx context.Context, _ timestamp.Timestamp) error { + close(entered) + <-ctx.Done() + return ctx.Err() + }) + + svc := &service{ + _txnClient: txnClient, + globalSysVarWakeup: make(chan struct{}, 1), + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + svc.globalSysVarWatermarkTask(ctx) + }() + svc.observeGlobalSysVarCommitTS(ts) + + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("watermark worker did not enter the visibility wait") + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("watermark worker outlived its service context") + } + require.Nil(t, svc.globalSysVarApplied.Load()) +} + +func TestCNGlobalSysVarAdmissionWaitsForAppliedWatermark(t *testing.T) { + desired := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 1} + connected := make(chan struct{}) + close(connected) + s := &service{ + hakeeperConnected: connected, + globalSysVarAppliedC: make(chan struct{}, 1), + } + s.globalSysVarDesired.Store(&desired) + deadline := time.Now().Add(time.Minute) + s.servingLeaseDeadline.Store(&deadline) + require.False(t, s.CanAcceptNewConnections(), + "a CN must not admit direct SQL while the durable watermark is behind") + + done := make(chan error, 1) + go func() { + done <- s.waitGlobalSysVarAdmission(context.Background()) + }() + select { + case err := <-done: + t.Fatalf("startup admission returned before watermark apply: %v", err) + default: + } + + s.publishGlobalSysVarCommitTS(desired) + require.NoError(t, <-done) + require.True(t, s.CanAcceptNewConnections()) + deadline = time.Now().Add(-time.Nanosecond) + s.servingLeaseDeadline.Store(&deadline) + require.False(t, s.CanAcceptNewConnections(), + "an expired control-plane lease must fail-close direct SQL admission") +} + +func TestCNGlobalSysVarAdmissionHonorsContext(t *testing.T) { + s := &service{hakeeperConnected: make(chan struct{})} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, s.waitGlobalSysVarAdmission(ctx), context.Canceled) +} + type blockingCNHeartbeatCommandClient struct { *testHAKClient heartbeatEntered chan struct{} @@ -43,6 +172,7 @@ type canceledCNResponseClient struct { *testHAKClient heartbeatEntered chan struct{} pollEntered chan struct{} + heartbeat pb.CNStoreHeartbeat } type lateCNCommandClient struct { @@ -94,8 +224,9 @@ func (h *observingTaskHolder) Create(pb.CreateTaskService) error { func (c *canceledCNResponseClient) SendCNHeartbeat( ctx context.Context, - _ pb.CNStoreHeartbeat, + hb pb.CNStoreHeartbeat, ) (pb.CommandBatch, error) { + c.heartbeat = hb select { case <-c.heartbeatEntered: default: @@ -480,6 +611,8 @@ func TestCNHeartbeatDropsResponseAfterRequestDeadline(t *testing.T) { // A nil hakeeperConnected channel would panic if the successful-looking // late response escaped the per-request deadline guard. service.heartbeat(context.Background()) + require.Equal(t, defines.MORPCLatestVersion, + service._hakeeperClient.(*canceledCNResponseClient).heartbeat.ProtocolVersion) } func TestCNCommandGenerationRolloverDoesNotReplayInheritedCommands(t *testing.T) { diff --git a/pkg/cnservice/server_query.go b/pkg/cnservice/server_query.go index 2c99e5d0b759d..64175b1902d50 100644 --- a/pkg/cnservice/server_query.go +++ b/pkg/cnservice/server_query.go @@ -442,8 +442,7 @@ func (s *service) handleGetTxnInfo(ctx context.Context, req *query.Request, resp } func (s *service) handleSyncCommit(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { - s._txnClient.SyncLatestCommitTS(req.SycnCommit.LatestCommitTS) - return nil + return s._txnClient.SyncLatestCommitTSWithContext(ctx, req.SycnCommit.LatestCommitTS) } func (s *service) handleGetMinTimestamp(ctx context.Context, req *query.Request, resp *query.Response, _ *morpc.Buffer) error { diff --git a/pkg/cnservice/server_query_test.go b/pkg/cnservice/server_query_test.go index bb881375b135e..b3c950e1f0ca6 100644 --- a/pkg/cnservice/server_query_test.go +++ b/pkg/cnservice/server_query_test.go @@ -53,6 +53,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/query" "github.com/matrixorigin/matrixone/pkg/pb/statsinfo" "github.com/matrixorigin/matrixone/pkg/pb/task" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/queryservice" "github.com/matrixorigin/matrixone/pkg/shardservice" sqlmongodb "github.com/matrixorigin/matrixone/pkg/sql/mongodb" @@ -67,6 +68,36 @@ import ( var dummyBadRequestErr = moerr.NewInternalError(context.TODO(), "bad request") var dummyErr = moerr.NewInternalError(context.TODO(), "dummy error") +func TestHandleSyncCommitPropagatesCancellation(t *testing.T) { + ctrl := gomock.NewController(t) + txnClient := mock_frontend.NewMockTxnClient(ctrl) + ts := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + entered := make(chan struct{}) + txnClient.EXPECT().SyncLatestCommitTSWithContext(gomock.Any(), ts). + DoAndReturn(func(ctx context.Context, _ timestamp.Timestamp) error { + close(entered) + <-ctx.Done() + return ctx.Err() + }) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + s := &service{_txnClient: txnClient} + go func() { + done <- s.handleSyncCommit(ctx, &query.Request{ + SycnCommit: &query.SyncCommitRequest{LatestCommitTS: ts}, + }, &query.Response{}, nil) + }() + + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("handler did not enter the transaction client wait") + } + cancel() + require.ErrorIs(t, <-done, context.Canceled) +} + func Test_service_handleISCPDrainConsumerRenewFenceOnly(t *testing.T) { exec := &iscp.ISCPTaskExecutor{} iscp.RegisterExecutorRuntime("runner-cn", exec) diff --git a/pkg/cnservice/types.go b/pkg/cnservice/types.go index 2d01d80896194..a6c3f442df6d7 100644 --- a/pkg/cnservice/types.go +++ b/pkg/cnservice/types.go @@ -42,6 +42,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/partitionservice" logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/pb/txn" "github.com/matrixorigin/matrixone/pkg/queryservice" qclient "github.com/matrixorigin/matrixone/pkg/queryservice/client" @@ -342,6 +343,13 @@ func (c *Config) Validate() error { if c.HAKeeper.HeatbeatTimeout.Duration < 0 { return moerr.NewBadConfigNoCtx("hakeeper heartbeat timeout must be positive") } + if c.HAKeeper.HeatbeatInterval.Duration+c.HAKeeper.HeatbeatTimeout.Duration > + logservice.GlobalSysVarHeartbeatProgressBudget { + return moerr.NewBadConfigNoCtxf( + "hakeeper heartbeat cycle %s exceeds global-system-variable progress budget %s", + c.HAKeeper.HeatbeatInterval.Duration+c.HAKeeper.HeatbeatTimeout.Duration, + logservice.GlobalSysVarHeartbeatProgressBudget) + } if c.TaskRunner.Parallelism == 0 { c.TaskRunner.Parallelism = runtime.NumCPU() / 16 if c.TaskRunner.Parallelism <= ReservedTasks { @@ -698,21 +706,28 @@ type service struct { incrservice incrservice.AutoIncrementService txnTraceService trace.Service - stopper *stopper.Stopper - heartbeatInFlight atomic.Bool - commandPollNeeded atomic.Bool - commandPollWakeup chan struct{} - commandMu sync.Mutex - lastCommandBatchID uint64 - ackedCommandBatchID atomic.Uint64 - appliedCommandIDs map[logservice.ScheduleCommandIdentity]struct{} - lastCommandHash [32]byte - legacyDedupeArmed bool - aicm *defines.AutoIncrCacheManager - lifecycleMu sync.Mutex - lifecycle serviceLifecycleState - closeOnce sync.Once - closeErr error + stopper *stopper.Stopper + controlChannelsOnce sync.Once + heartbeatInFlight atomic.Bool + commandPollNeeded atomic.Bool + commandPollWakeup chan struct{} + commandMu sync.Mutex + lastCommandBatchID uint64 + ackedCommandBatchID atomic.Uint64 + globalSysVarDesired atomic.Pointer[timestamp.Timestamp] + globalSysVarApplied atomic.Pointer[timestamp.Timestamp] + globalSysVarWakeup chan struct{} + globalSysVarAppliedC chan struct{} + globalSysVarGeneration string + servingLeaseDeadline atomic.Pointer[time.Time] + appliedCommandIDs map[logservice.ScheduleCommandIdentity]struct{} + lastCommandHash [32]byte + legacyDedupeArmed bool + aicm *defines.AutoIncrCacheManager + lifecycleMu sync.Mutex + lifecycle serviceLifecycleState + closeOnce sync.Once + closeErr error task struct { sync.RWMutex diff --git a/pkg/cnservice/types_test.go b/pkg/cnservice/types_test.go index f4c118a732e27..bf562e211fef6 100644 --- a/pkg/cnservice/types_test.go +++ b/pkg/cnservice/types_test.go @@ -46,4 +46,10 @@ func TestValidateHeartbeatDurations(t *testing.T) { cfg := Config{UUID: "cn1"} cfg.HAKeeper.HeatbeatTimeout.Duration = -time.Nanosecond require.ErrorContains(t, cfg.Validate(), "hakeeper heartbeat timeout") + + cfg = Config{UUID: "cn1"} + cfg.HAKeeper.HeatbeatInterval.Duration = time.Second + cfg.HAKeeper.HeatbeatTimeout.Duration = + logservice.GlobalSysVarHeartbeatProgressBudget + require.ErrorContains(t, cfg.Validate(), "global-system-variable progress budget") } diff --git a/pkg/defines/const.go b/pkg/defines/const.go index 8edeefc1416cc..3e5a68fa584d9 100644 --- a/pkg/defines/const.go +++ b/pkg/defines/const.go @@ -47,9 +47,10 @@ const ( MORPCVersion9 int64 = 9 // AUTO_INCREMENT epoch-fenced commit MORPCVersion10 int64 = 10 // persisted appendable-object abort metadata MORPCVersion11 int64 = 11 // bounded Sorted64 membership-filter wire format - MORPCVersion12 int64 = 12 // prepared-parameter provenance in remote process metadata and aggregate trailers + MORPCVersion12 int64 = 12 // prepared provenance MORPCVersion13 int64 = 13 // lossless v2 prefix-index metadata - MORPCLatestVersion = MORPCVersion13 + MORPCVersion14 int64 = 14 // HAKeeper-fenced global-system-variable visibility + MORPCLatestVersion = MORPCVersion14 ) // DefaultLockWaitTimeoutSeconds is shared by the frontend default and by diff --git a/pkg/frontend/authenticate.go b/pkg/frontend/authenticate.go index d147d6286ed7e..22cefdd199898 100644 --- a/pkg/frontend/authenticate.go +++ b/pkg/frontend/authenticate.go @@ -1601,7 +1601,9 @@ const ( insertSystemVariableWithAccountFormat = `insert into mo_catalog.mo_mysql_compatibility_mode(account_id, account_name, variable_name, variable_value, system_variables) values (%d, "%s", "%s", "%s", %v);` - updateSystemVariableValueFormat = `update mo_catalog.mo_mysql_compatibility_mode set variable_value = '%s' where account_id = %d and variable_name = '%s' and system_variables = true;` + updateSystemVariableValueFormat = `update mo_catalog.mo_mysql_compatibility_mode set variable_value = '%s' where account_id = %d and variable_name = '%s' and system_variables = true;` + lockGlobalSystemVariableAccountFormat = `select account_id from mo_catalog.mo_account where account_id = %d for update;` + getGlobalSystemVariableEpochFormat = `select variable_value from mo_catalog.mo_mysql_compatibility_mode where account_id = %d and system_variables = true and variable_name = '%s' for update;` updateConfigurationByDbNameAndAccountNameFormat = `update mo_catalog.mo_mysql_compatibility_mode set variable_value = '%s' where account_name = '%s' and dat_name = '%s' and variable_name = '%s';` @@ -2255,6 +2257,16 @@ func getSqlForUpdateSysVarValue(varValue string, accountId uint64, varName strin return fmt.Sprintf(updateSystemVariableValueFormat, varValue, accountId, varName) } +const globalSystemVariableEpochName = "__mo_global_system_variable_epoch" + +func getSqlForLockGlobalSystemVariableAccount(accountID uint64) string { + return fmt.Sprintf(lockGlobalSystemVariableAccountFormat, accountID) +} + +func getSqlForGlobalSystemVariableEpoch(accountID uint64) string { + return fmt.Sprintf(getGlobalSystemVariableEpochFormat, accountID, globalSystemVariableEpochName) +} + func getSqlForupdateConfigurationByDbNameAndAccountName(ctx context.Context, varValue, accountName, dbName, varName string) (string, error) { err := inputNameIsInvalid(ctx, dbName) if err != nil { @@ -11846,7 +11858,12 @@ func doRevokePrivilegeImplicitly( return nil } -func doSetGlobalSystemVariable(ctx context.Context, ses *Session, varName string, varValue interface{}) (err error) { +func doSetGlobalSystemVariable( + ctx context.Context, + ses *Session, + varName string, + varValue interface{}, +) (epoch uint64, err error) { accountId := uint64(ses.GetTenantInfo().TenantID) accountName := ses.GetTenantName() varName = strings.ToLower(varName) @@ -11860,6 +11877,47 @@ func doSetGlobalSystemVariable(ctx context.Context, ses *Session, varName string err = finishTxn(ctx, bh, err) }() + // Serialize the durable account epoch across CNs. The epoch row is updated + // in the same catalog transaction as the sysvar value, so it is both a + // monotonic cache version and a replayable fence intent after a process + // crash between catalog COMMIT and the HAKeeper publication. + if err = bh.Exec(ctx, getSqlForLockGlobalSystemVariableAccount(accountId)); err != nil { + return + } + bh.ClearExecResultSet() + if err = bh.Exec(ctx, getSqlForGlobalSystemVariableEpoch(accountId)); err != nil { + return + } + epochExists := false + if erArray, resultErr := getResultSet(ctx, bh); resultErr != nil { + err = resultErr + return + } else if execResultArrayHasData(erArray) { + epochExists = true + var value string + if value, err = erArray[0].GetString(ctx, 0, 0); err != nil { + return + } + if epoch, err = strconv.ParseUint(value, 10, 64); err != nil { + return 0, moerr.NewInternalErrorf(ctx, + "invalid global system variable epoch %q", value) + } + } + if epoch == math.MaxUint64 { + return 0, moerr.NewInternalError(ctx, "global system variable epoch exhausted") + } + epoch++ + if epochExists { + err = bh.Exec(ctx, getSqlForUpdateSysVarValue( + strconv.FormatUint(epoch, 10), accountId, globalSystemVariableEpochName)) + } else { + err = bh.Exec(ctx, getSqlForInsertSysVarWithAccount( + accountId, accountName, globalSystemVariableEpochName, strconv.FormatUint(epoch, 10))) + } + if err != nil { + return + } + // check if var exists sql := getSqlForGetSysVarWithAccount(accountId, varName) bh.ClearExecResultSet() diff --git a/pkg/frontend/authenticate_test.go b/pkg/frontend/authenticate_test.go index 85e427db4f0a2..a7aed38481db4 100644 --- a/pkg/frontend/authenticate_test.go +++ b/pkg/frontend/authenticate_test.go @@ -530,7 +530,6 @@ func Test_initUser(t *testing.T) { bh.sql2result["begin;"] = nil bh.sql2result["commit;"] = nil bh.sql2result["rollback;"] = nil - ses := newSes(nil, ctrl) pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) @@ -10015,6 +10014,11 @@ func TestSetGlobalSysVar(t *testing.T) { bh.sql2result["begin;"] = nil bh.sql2result["commit;"] = nil bh.sql2result["rollback;"] = nil + bh.sql2result[getSqlForLockGlobalSystemVariableAccount(sysAccountID)] = nil + bh.sql2result[getSqlForGlobalSystemVariableEpoch(sysAccountID)] = + newMrsForSystemVariableNameOfAccount([][]interface{}{}) + bh.sql2result[getSqlForInsertSysVarWithAccount( + sysAccountID, sysAccountName, globalSystemVariableEpochName, "1")] = nil sql := getSqlForGetSysVarWithAccount(sysAccountID, "autocommit") mrs := newMrsForSystemVariableNameOfAccount([][]interface{}{}) bh.sql2result[sql] = mrs diff --git a/pkg/frontend/global_sysvars_generation_test.go b/pkg/frontend/global_sysvars_generation_test.go index 970676d49918e..01e4b07f69470 100644 --- a/pkg/frontend/global_sysvars_generation_test.go +++ b/pkg/frontend/global_sysvars_generation_test.go @@ -126,3 +126,34 @@ func TestGlobalSysVarsRefreshDoesNotAdvanceMutationGeneration(t *testing.T) { value := globalVars.Get(PasswordHistory) require.Equal(t, int64(0), value) } + +func TestGlobalSysVarsCatalogEpochRejectsReversePublication(t *testing.T) { + globalVars := SystemVariables{ + mp: map[string]interface{}{PasswordHistory: int64(0)}, + } + generation := globalVars.getMutationGeneration() + + globalVars.replaceIfCatalogEpoch(2, generation, map[string]interface{}{ + PasswordHistory: int64(5), + }) + globalVars.replaceIfCatalogEpoch(1, generation, map[string]interface{}{ + PasswordHistory: int64(0), + }) + + require.Equal(t, uint64(2), globalVars.catalogEpoch) + require.Equal(t, int64(5), globalVars.Get(PasswordHistory), + "a late catalog read from an older commit must not roll the cache back") +} + +func TestGlobalSysVarsCatalogEpochOrdersConcurrentSetPublication(t *testing.T) { + globalVars := SystemVariables{ + mp: map[string]interface{}{PasswordHistory: int64(0)}, + } + + globalVars.setAtCatalogEpoch(PasswordHistory, int64(6), 2) + globalVars.setAtCatalogEpoch(PasswordHistory, int64(5), 1) + + require.Equal(t, uint64(2), globalVars.catalogEpoch) + require.Equal(t, int64(6), globalVars.Get(PasswordHistory), + "local completion order must not override catalog commit order") +} diff --git a/pkg/frontend/global_sysvars_sync.go b/pkg/frontend/global_sysvars_sync.go new file mode 100644 index 0000000000000..6ff29bf5b7aba --- /dev/null +++ b/pkg/frontend/global_sysvars_sync.go @@ -0,0 +1,183 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package frontend + +import ( + "context" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/logservice" + logpb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" +) + +const ( + globalSysVarCommitSyncTimeout = logservice.GlobalSysVarFenceTimeout + globalSysVarFencePollInterval = 20 * time.Millisecond +) + +// validateGlobalSysVarSyncProtocol fails before the catalog mutation when a +// rolling deployment has not activated the HAKeeper routing-fence protocol. +func validateGlobalSysVarSyncProtocol(ctx context.Context, ses *Session) error { + pu := getPuIfPresent(ses.GetService()) + if pu == nil || pu.HAKeeperClient == nil { + return nil + } + rt := moruntime.ServiceRuntime(ses.GetService()) + if rt == nil { + return moerr.NewInternalError(ctx, "service runtime is not initialized") + } + value, ok := rt.GetGlobalVariables(moruntime.MOProtocolVersion) + version, valid := value.(int64) + if !ok || !valid || version < defines.MORPCVersion14 { + return moerr.NewInternalErrorf(ctx, + "SET GLOBAL requires MORPC protocol version %d", defines.MORPCVersion14) + } + if _, ok := pu.HAKeeperClient.(logservice.GlobalSysVarHAKeeperClient); !ok { + return moerr.NewInternalError(ctx, + "HAKeeper client does not support global system variable fencing") + } + details, err := pu.HAKeeperClient.GetClusterDetails(ctx) + if err != nil { + return err + } + hasServingCN := false + for _, cn := range details.CNStores { + if cn.SQLAddress == "" { + continue + } + hasServingCN = true + if cn.ProtocolVersion < defines.MORPCVersion14 { + return moerr.NewInternalErrorf(ctx, + "CN %s protocol version %d does not support global system variable fencing", + cn.UUID, cn.ProtocolVersion) + } + } + if !hasServingCN { + return moerr.NewInternalError(ctx, + "HAKeeper has no protocol-capable SQL CN") + } + if len(details.LogStores) == 0 { + return moerr.NewInternalError(ctx, + "HAKeeper has no protocol-capable LogStore") + } + for _, store := range details.LogStores { + if store.ProtocolVersion < defines.MORPCVersion14 { + return moerr.NewInternalErrorf(ctx, + "LogStore %s protocol version %d does not support global system variable fencing", + store.UUID, store.ProtocolVersion) + } + } + for _, proxy := range details.ProxyStores { + if proxy.ProtocolVersion < defines.MORPCVersion14 { + return moerr.NewInternalErrorf(ctx, + "Proxy %s protocol version %d does not support global system variable fencing", + proxy.UUID, proxy.ProtocolVersion) + } + } + return nil +} + +// syncGlobalSysVarCommit publishes the committed timestamp as a durable +// HAKeeper admission fence and waits until every CN routable at that +// linearization point has applied it. +func syncGlobalSysVarCommit(ctx context.Context, ses *Session) error { + pu := getPuIfPresent(ses.GetService()) + if pu == nil || pu.HAKeeperClient == nil { + return nil + } + if pu.TxnClient == nil { + return moerr.NewInternalError(ctx, "transaction client is not initialized") + } + commitTS := pu.TxnClient.GetLatestCommitTS() + if commitTS.IsEmpty() { + return moerr.NewInternalError(ctx, "global system variable commit timestamp is empty") + } + fenceClient, ok := pu.HAKeeperClient.(logservice.GlobalSysVarHAKeeperClient) + if !ok { + return moerr.NewInternalError(ctx, + "HAKeeper client does not support global system variable fencing") + } + + syncCtx, cancel := context.WithTimeoutCause( + ctx, globalSysVarCommitSyncTimeout, moerr.CauseSyncLatestCommitT) + defer cancel() + if err := fenceClient.UpdateGlobalSysVarCommitTS(syncCtx, commitTS); err != nil { + return moerr.AttachCause(syncCtx, err) + } + if err := waitGlobalSysVarCommitFence( + syncCtx, pu.HAKeeperClient.GetClusterDetails, commitTS); err != nil { + return moerr.AttachCause(syncCtx, err) + } + return nil +} + +func waitGlobalSysVarCommitFence( + ctx context.Context, + getDetails func(context.Context) (logpb.ClusterDetails, error), + commitTS timestamp.Timestamp, +) error { + if commitTS.IsEmpty() || getDetails == nil { + return nil + } + ticker := time.NewTicker(globalSysVarFencePollInterval) + defer ticker.Stop() + for { + details, err := getDetails(ctx) + if err != nil { + return err + } + if details.GlobalSysVarCommitTS.GreaterEq(commitTS) && + allRoutableCNsApplied(details.CNStores, commitTS) && + allProxiesApplied(details.ProxyStores, commitTS) { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +func allProxiesApplied(proxies []logpb.ProxyStore, commitTS timestamp.Timestamp) bool { + for _, proxy := range proxies { + if proxy.State != logpb.NormalState { + continue + } + if proxy.GlobalSysVarCommitTS.Less(commitTS) { + return false + } + } + return true +} + +func allRoutableCNsApplied(cns []logpb.CNStore, commitTS timestamp.Timestamp) bool { + for _, cn := range cns { + if cn.State != logpb.NormalState || cn.SQLAddress == "" || + (cn.WorkState != metadata.WorkState_Working && + cn.WorkState != metadata.WorkState_Unknown) { + continue + } + if cn.GlobalSysVarCommitTS.Less(commitTS) { + return false + } + } + return true +} diff --git a/pkg/frontend/global_sysvars_sync_test.go b/pkg/frontend/global_sysvars_sync_test.go new file mode 100644 index 0000000000000..e49c599d0a30f --- /dev/null +++ b/pkg/frontend/global_sysvars_sync_test.go @@ -0,0 +1,409 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package frontend + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/golang/mock/gomock" + "github.com/prashantv/gostub" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/config" + "github.com/matrixorigin/matrixone/pkg/defines" + mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" + "github.com/matrixorigin/matrixone/pkg/logservice" + logpb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" +) + +type globalSysVarFenceHAKeeper struct { + logservice.CNHAKeeperClient + mu sync.Mutex + updates []timestamp.Timestamp + details []logpb.ClusterDetails + updateErr error + detailErr error + gets int +} + +func (m *globalSysVarFenceHAKeeper) UpdateGlobalSysVarCommitTS( + _ context.Context, + ts timestamp.Timestamp, +) error { + m.mu.Lock() + defer m.mu.Unlock() + m.updates = append(m.updates, ts) + return m.updateErr +} + +func (m *globalSysVarFenceHAKeeper) GetClusterDetails( + _ context.Context, +) (logpb.ClusterDetails, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.detailErr != nil { + return logpb.ClusterDetails{}, m.detailErr + } + if len(m.details) == 0 { + m.gets++ + return logpb.ClusterDetails{}, nil + } + i := m.gets + if i >= len(m.details) { + i = len(m.details) - 1 + } + m.gets++ + return m.details[i], nil +} + +func (m *globalSysVarFenceHAKeeper) snapshot() ([]timestamp.Timestamp, int) { + m.mu.Lock() + defer m.mu.Unlock() + return append([]timestamp.Timestamp(nil), m.updates...), m.gets +} + +func setupGlobalSysVarFenceSession( + t *testing.T, + version int64, + hakeeper logservice.CNHAKeeperClient, + txnClient *mock_frontend.MockTxnClient, +) *Session { + t.Helper() + serviceID := t.Name() + pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, txnClient, nil) + pu.HAKeeperClient = hakeeper + InitServerLevelVars(serviceID) + setPu(serviceID, pu) + rt := runtime.NewRuntime(metadata.ServiceType_CN, serviceID, zap.NewNop()) + rt.SetGlobalVariables(runtime.MOProtocolVersion, version) + runtime.SetupServiceBasedRuntime(serviceID, rt) + return &Session{feSessionImpl: feSessionImpl{service: serviceID}} +} + +func TestValidateGlobalSysVarSyncProtocolRollingUpgrade(t *testing.T) { + capableDetails := logpb.ClusterDetails{CNStores: []logpb.CNStore{{ + UUID: "cn-capable", + SQLAddress: "127.0.0.1:6001", + ProtocolVersion: defines.MORPCVersion14, + }}, LogStores: []logpb.LogStore{{ + UUID: "log-capable", ProtocolVersion: defines.MORPCVersion14, + }}} + t.Run("previous latest version fails closed", func(t *testing.T) { + hakeeper := &globalSysVarFenceHAKeeper{} + ses := setupGlobalSysVarFenceSession( + t, defines.MORPCVersion13, hakeeper, nil) + err := validateGlobalSysVarSyncProtocol(context.Background(), ses) + require.ErrorContains(t, err, "protocol version 14") + updates, gets := hakeeper.snapshot() + require.Empty(t, updates) + require.Zero(t, gets) + }) + + t.Run("version 14 enables fence", func(t *testing.T) { + ses := setupGlobalSysVarFenceSession( + t, defines.MORPCVersion14, &globalSysVarFenceHAKeeper{ + details: []logpb.ClusterDetails{capableDetails}, + }, nil) + require.NoError(t, validateGlobalSysVarSyncProtocol(context.Background(), ses)) + }) + + t.Run("old CN capability fails closed", func(t *testing.T) { + details := logpb.ClusterDetails{ + CNStores: append([]logpb.CNStore(nil), capableDetails.CNStores...), + LogStores: append([]logpb.LogStore(nil), capableDetails.LogStores...), + } + details.CNStores[0].ProtocolVersion = defines.MORPCVersion13 + ses := setupGlobalSysVarFenceSession(t, defines.MORPCVersion14, + &globalSysVarFenceHAKeeper{details: []logpb.ClusterDetails{details}}, nil) + require.ErrorContains(t, + validateGlobalSysVarSyncProtocol(context.Background(), ses), + "CN cn-capable protocol version 13") + }) + + t.Run("old Proxy capability fails closed", func(t *testing.T) { + details := logpb.ClusterDetails{ + CNStores: append([]logpb.CNStore(nil), capableDetails.CNStores...), + LogStores: append([]logpb.LogStore(nil), capableDetails.LogStores...), + } + details.ProxyStores = []logpb.ProxyStore{{ + UUID: "proxy-old", ProtocolVersion: defines.MORPCVersion13, + }} + ses := setupGlobalSysVarFenceSession(t, defines.MORPCVersion14, + &globalSysVarFenceHAKeeper{details: []logpb.ClusterDetails{details}}, nil) + require.ErrorContains(t, + validateGlobalSysVarSyncProtocol(context.Background(), ses), + "Proxy proxy-old protocol version 13") + }) + + t.Run("old LogStore capability fails closed", func(t *testing.T) { + details := logpb.ClusterDetails{ + CNStores: append([]logpb.CNStore(nil), capableDetails.CNStores...), + LogStores: []logpb.LogStore{{ + UUID: "log-old", ProtocolVersion: defines.MORPCVersion13, + }}, + } + ses := setupGlobalSysVarFenceSession(t, defines.MORPCVersion14, + &globalSysVarFenceHAKeeper{details: []logpb.ClusterDetails{details}}, nil) + require.ErrorContains(t, + validateGlobalSysVarSyncProtocol(context.Background(), ses), + "LogStore log-old protocol version 13") + }) + + t.Run("missing capability fails closed", func(t *testing.T) { + hakeeper := struct{ logservice.CNHAKeeperClient }{} + ses := setupGlobalSysVarFenceSession(t, defines.MORPCVersion14, hakeeper, nil) + require.ErrorContains(t, + validateGlobalSysVarSyncProtocol(context.Background(), ses), + "does not support global system variable fencing") + }) + + t.Run("standalone remains compatible", func(t *testing.T) { + ses := setupGlobalSysVarFenceSession(t, defines.MORPCVersion13, nil, nil) + require.NoError(t, validateGlobalSysVarSyncProtocol(context.Background(), ses)) + }) +} + +func TestSetGlobalSysVarRollingUpgradeRejectsBeforeCatalogWrite(t *testing.T) { + ctrl := gomock.NewController(t) + ses := newSes(nil, ctrl) + previousRuntime := runtime.ServiceRuntime(ses.GetService()) + t.Cleanup(func() { + if previousRuntime != nil { + runtime.SetupServiceBasedRuntime(ses.GetService(), previousRuntime) + } + }) + hakeeper := &globalSysVarFenceHAKeeper{} + getPuIfPresent(ses.GetService()).HAKeeperClient = hakeeper + rt := runtime.NewRuntime(metadata.ServiceType_CN, ses.GetService(), zap.NewNop()) + rt.SetGlobalVariables(runtime.MOProtocolVersion, defines.MORPCVersion13) + runtime.SetupServiceBasedRuntime(ses.GetService(), rt) + + background := &backgroundExecTest{} + background.init() + stub := gostub.StubFunc(&NewBackgroundExec, background) + t.Cleanup(stub.Reset) + + err := ses.SetGlobalSysVar(context.Background(), "autocommit", int64(0)) + require.ErrorContains(t, err, "protocol version 14") + require.Empty(t, background.executedSQLs, + "rolling-upgrade rejection must happen before opening the catalog transaction") + updates, gets := hakeeper.snapshot() + require.Empty(t, updates) + require.Zero(t, gets) +} + +func TestSetGlobalSysVarFenceFailureLeavesDurableReconciliationEpoch(t *testing.T) { + ctrl := gomock.NewController(t) + ses := newSes(nil, ctrl) + previousRuntime := runtime.ServiceRuntime(ses.GetService()) + t.Cleanup(func() { + if previousRuntime != nil { + runtime.SetupServiceBasedRuntime(ses.GetService(), previousRuntime) + } + }) + commitTS := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + txnClient := mock_frontend.NewMockTxnClient(ctrl) + txnClient.EXPECT().GetLatestCommitTS().Return(commitTS) + hakeeper := &globalSysVarFenceHAKeeper{ + updateErr: errors.New("raft unavailable"), + details: []logpb.ClusterDetails{{CNStores: []logpb.CNStore{{ + UUID: "cn-capable", + SQLAddress: "127.0.0.1:6001", + ProtocolVersion: defines.MORPCVersion14, + }}, LogStores: []logpb.LogStore{{ + UUID: "log-capable", ProtocolVersion: defines.MORPCVersion14, + }}}}, + } + pu := getPuIfPresent(ses.GetService()) + pu.HAKeeperClient = hakeeper + pu.TxnClient = txnClient + rt := runtime.NewRuntime(metadata.ServiceType_CN, ses.GetService(), zap.NewNop()) + rt.SetGlobalVariables(runtime.MOProtocolVersion, defines.MORPCVersion14) + runtime.SetupServiceBasedRuntime(ses.GetService(), rt) + background := stubGlobalSysVarPersistence( + t, sysVarSet{PasswordHistory, int64(5)}) + + err := ses.SetGlobalSysVar(context.Background(), PasswordHistory, int64(5)) + require.ErrorContains(t, err, "raft unavailable") + require.Contains(t, background.executedSQLs, getSqlForInsertSysVarWithAccount( + sysAccountID, sysAccountName, globalSystemVariableEpochName, "1")) + require.Contains(t, background.executedSQLs, "commit;", + "the epoch intent must commit atomically with the catalog value") +} + +func TestSyncGlobalSysVarCommitPublishesAndWaitsForFence(t *testing.T) { + ctrl := gomock.NewController(t) + commitTS := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + txnClient := mock_frontend.NewMockTxnClient(ctrl) + txnClient.EXPECT().GetLatestCommitTS().Return(commitTS) + hakeeper := &globalSysVarFenceHAKeeper{details: []logpb.ClusterDetails{{ + GlobalSysVarCommitTS: commitTS, + ProxyStores: []logpb.ProxyStore{{ + UUID: "proxy-1", GlobalSysVarCommitTS: commitTS, + }}, + CNStores: []logpb.CNStore{ + {UUID: "cn-1", SQLAddress: "sql-1", State: logpb.NormalState, + WorkState: metadata.WorkState_Working, GlobalSysVarCommitTS: commitTS}, + {UUID: "cn-draining", SQLAddress: "sql-2", State: logpb.NormalState, + WorkState: metadata.WorkState_Draining}, + {UUID: "cn-expired", SQLAddress: "sql-3", State: logpb.TimeoutState, + WorkState: metadata.WorkState_Working}, + }, + }}} + ses := setupGlobalSysVarFenceSession( + t, defines.MORPCVersion14, hakeeper, txnClient) + + require.NoError(t, syncGlobalSysVarCommit(context.Background(), ses)) + updates, gets := hakeeper.snapshot() + require.Equal(t, []timestamp.Timestamp{commitTS}, updates) + require.Equal(t, 1, gets) +} + +func TestWaitGlobalSysVarCommitFenceWaitsForProxyRouteBarrier(t *testing.T) { + commitTS := timestamp.Timestamp{PhysicalTime: 100} + hakeeper := &globalSysVarFenceHAKeeper{details: []logpb.ClusterDetails{ + { + GlobalSysVarCommitTS: commitTS, + ProxyStores: []logpb.ProxyStore{{UUID: "proxy-1"}}, + }, + { + GlobalSysVarCommitTS: commitTS, + ProxyStores: []logpb.ProxyStore{{ + UUID: "proxy-1", GlobalSysVarCommitTS: commitTS, + }}, + }, + }} + require.NoError(t, waitGlobalSysVarCommitFence( + context.Background(), hakeeper.GetClusterDetails, commitTS)) + _, gets := hakeeper.snapshot() + require.Equal(t, 2, gets) +} + +func TestWaitGlobalSysVarCommitFenceIgnoresExpiredProxy(t *testing.T) { + commitTS := timestamp.Timestamp{PhysicalTime: 100} + details := logpb.ClusterDetails{ + GlobalSysVarCommitTS: commitTS, + ProxyStores: []logpb.ProxyStore{ + {UUID: "proxy-live", State: logpb.NormalState, GlobalSysVarCommitTS: commitTS}, + {UUID: "proxy-expired", State: logpb.TimeoutState}, + }, + } + require.NoError(t, waitGlobalSysVarCommitFence( + context.Background(), func(context.Context) (logpb.ClusterDetails, error) { + return details, nil + }, commitTS)) +} + +func TestWaitGlobalSysVarCommitFenceIncludesLateJoin(t *testing.T) { + commitTS := timestamp.Timestamp{PhysicalTime: 100} + hakeeper := &globalSysVarFenceHAKeeper{details: []logpb.ClusterDetails{ + { + GlobalSysVarCommitTS: commitTS, + CNStores: []logpb.CNStore{{ + UUID: "cn-a", SQLAddress: "sql-a", State: logpb.NormalState, + WorkState: metadata.WorkState_Working, GlobalSysVarCommitTS: commitTS, + }}, + }, + { + GlobalSysVarCommitTS: commitTS, + CNStores: []logpb.CNStore{ + {UUID: "cn-a", SQLAddress: "sql-a", State: logpb.NormalState, + WorkState: metadata.WorkState_Working, GlobalSysVarCommitTS: commitTS}, + {UUID: "cn-b", SQLAddress: "sql-b", State: logpb.NormalState, + WorkState: metadata.WorkState_Working}, + }, + }, + { + GlobalSysVarCommitTS: commitTS, + CNStores: []logpb.CNStore{ + {UUID: "cn-a", SQLAddress: "sql-a", State: logpb.NormalState, + WorkState: metadata.WorkState_Working, GlobalSysVarCommitTS: commitTS}, + {UUID: "cn-b", SQLAddress: "sql-b", State: logpb.NormalState, + WorkState: metadata.WorkState_Working, GlobalSysVarCommitTS: commitTS}, + }, + }, + }} + + // The first snapshot alone is intentionally not used as a success oracle: + // CN-B appears after it and must also acknowledge before the barrier opens. + _, err := hakeeper.GetClusterDetails(context.Background()) + require.NoError(t, err) + require.NoError(t, waitGlobalSysVarCommitFence( + context.Background(), hakeeper.GetClusterDetails, commitTS)) + _, gets := hakeeper.snapshot() + require.Equal(t, 3, gets) +} + +func TestWaitGlobalSysVarCommitFenceErrorsAndCancellation(t *testing.T) { + commitTS := timestamp.Timestamp{PhysicalTime: 100} + t.Run("hakeeper error", func(t *testing.T) { + want := errors.New("hakeeper unavailable") + hakeeper := &globalSysVarFenceHAKeeper{detailErr: want} + require.ErrorIs(t, waitGlobalSysVarCommitFence( + context.Background(), hakeeper.GetClusterDetails, commitTS), want) + }) + + t.Run("caller cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + hakeeper := &globalSysVarFenceHAKeeper{details: []logpb.ClusterDetails{{ + GlobalSysVarCommitTS: commitTS, + CNStores: []logpb.CNStore{{ + SQLAddress: "sql", State: logpb.NormalState, + WorkState: metadata.WorkState_Working, + }}, + }}} + require.ErrorIs(t, waitGlobalSysVarCommitFence( + ctx, hakeeper.GetClusterDetails, commitTS), context.Canceled) + }) +} + +func TestSyncGlobalSysVarCommitRejectsInvalidState(t *testing.T) { + t.Run("empty commit timestamp", func(t *testing.T) { + ctrl := gomock.NewController(t) + txnClient := mock_frontend.NewMockTxnClient(ctrl) + txnClient.EXPECT().GetLatestCommitTS().Return(timestamp.Timestamp{}) + hakeeper := &globalSysVarFenceHAKeeper{} + ses := setupGlobalSysVarFenceSession( + t, defines.MORPCVersion14, hakeeper, txnClient) + require.ErrorContains(t, + syncGlobalSysVarCommit(context.Background(), ses), + "commit timestamp is empty") + updates, gets := hakeeper.snapshot() + require.Empty(t, updates) + require.Zero(t, gets) + }) + + t.Run("watermark update error", func(t *testing.T) { + ctrl := gomock.NewController(t) + commitTS := timestamp.Timestamp{PhysicalTime: 100} + txnClient := mock_frontend.NewMockTxnClient(ctrl) + txnClient.EXPECT().GetLatestCommitTS().Return(commitTS) + want := errors.New("raft unavailable") + hakeeper := &globalSysVarFenceHAKeeper{updateErr: want} + ses := setupGlobalSysVarFenceSession( + t, defines.MORPCVersion14, hakeeper, txnClient) + require.ErrorIs(t, syncGlobalSysVarCommit(context.Background(), ses), want) + _, gets := hakeeper.snapshot() + require.Zero(t, gets) + }) +} diff --git a/pkg/frontend/mysql_cmd_executor_test.go b/pkg/frontend/mysql_cmd_executor_test.go index 230cf1e62db8d..668380290f868 100644 --- a/pkg/frontend/mysql_cmd_executor_test.go +++ b/pkg/frontend/mysql_cmd_executor_test.go @@ -24,6 +24,7 @@ import ( "math" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "testing" @@ -1213,8 +1214,12 @@ func TestShowGlobalVariablesRefreshesGlobalSysVarCache(t *testing.T) { bh := &backgroundExecTest{} bh.init() sql := getSqlForGetSystemVariablesWithAccount(sysAccountID) + ses.gSysVars.mu.Lock() + catalogEpoch := ses.gSysVars.catalogEpoch + ses.gSysVars.mu.Unlock() bh.sql2result[sql] = newMrsForGlobalSystemVariables([][]interface{}{ {"long_query_time", "1.1"}, + {globalSystemVariableEpochName, strconv.FormatUint(catalogEpoch, 10)}, }) bhStub := gostub.StubFunc(&NewBackgroundExec, bh) diff --git a/pkg/frontend/routine_manager_test.go b/pkg/frontend/routine_manager_test.go index b122ca3931f39..88f215931dd36 100644 --- a/pkg/frontend/routine_manager_test.go +++ b/pkg/frontend/routine_manager_test.go @@ -919,6 +919,7 @@ func receiveLegacyMigrationActionResult(t *testing.T, result <-chan error) error } func TestRoutineManagerResetSessionRejectsRequestAfterResponseWrite(t *testing.T) { + stubCachedSessionSystemVariables(t) const connID = uint32(1009) ctrl := gomock.NewController(t) oldSession := newTestSession(t, ctrl) diff --git a/pkg/frontend/routine_test.go b/pkg/frontend/routine_test.go index 79a5d20f1a1df..d743f99d75b9a 100644 --- a/pkg/frontend/routine_test.go +++ b/pkg/frontend/routine_test.go @@ -50,6 +50,13 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm/process" ) +func stubCachedSessionSystemVariables(t *testing.T) { + t.Helper() + stub := gostub.Stub(&initializeCachedSessionSystemVariables, + func(context.Context, *Session) error { return nil }) + t.Cleanup(stub.Reset) +} + type routineTraceIDGenerator struct{} func (routineTraceIDGenerator) NewIDs() (trace.TraceID, trace.SpanID) { @@ -343,6 +350,7 @@ func TestCanceledResetAdmissionDoesNotTouchSession(t *testing.T) { } func TestRoutineCloseCancelsResetRollback(t *testing.T) { + stubCachedSessionSystemVariables(t) ctrl := gomock.NewController(t) oldSession := newTestSession(t, ctrl) oldSession.GetTxnHandler().Close() @@ -463,6 +471,17 @@ func TestMigrateConnectionFromPreservesLastAffectedRows(t *testing.T) { } func TestRoutineResetSessionKeepsReplacementRegistered(t *testing.T) { + initialized := 0 + stub := gostub.Stub(&initializeCachedSessionSystemVariables, + func(_ context.Context, ses *Session) error { + initialized++ + ses.gSysVars = &SystemVariables{mp: map[string]interface{}{ + PasswordHistory: int64(5), + }} + ses.sesSysVars = ses.gSysVars.Clone() + return nil + }) + t.Cleanup(stub.Reset) ctrl := gomock.NewController(t) oldSession := newTestSession(t, ctrl) timeZone := time.FixedZone("reset-session-test", 8*60*60) @@ -491,6 +510,11 @@ func TestRoutineResetSessionKeepsReplacementRegistered(t *testing.T) { require.NotSame(t, oldSession, newSession) require.Equal(t, oldSession.GetUUIDString(), newSession.GetUUIDString()) require.Same(t, timeZone, newSession.GetTimeZone()) + require.Equal(t, 1, initialized) + value, err := newSession.GetSessionSysVar(PasswordHistory) + require.NoError(t, err) + require.Equal(t, int64(5), value, + "a cached backend must initialize the new login from the current account snapshot") registered := rm.sessionManager.GetAllSessions() require.Len(t, registered, 1, "successful reset must keep the replacement session registered") @@ -498,6 +522,7 @@ func TestRoutineResetSessionKeepsReplacementRegistered(t *testing.T) { } func TestRoutineResetSessionFailureRestoresProtocolState(t *testing.T) { + stubCachedSessionSystemVariables(t) ctrl := gomock.NewController(t) oldSession := newTestSession(t, ctrl) rm, err := NewRoutineManager(context.Background(), "") diff --git a/pkg/frontend/server.go b/pkg/frontend/server.go index 6309da5191f8d..77cecc04c8c5a 100644 --- a/pkg/frontend/server.go +++ b/pkg/frontend/server.go @@ -81,6 +81,9 @@ type MOServer struct { pu *config.ParameterUnit listeners []net.Listener service string + // canAcceptNewConnections is an optional fail-closed admission check owned + // by the CN control plane. Existing sessions are unaffected. + canAcceptNewConnections func() bool } // Server interface is for mock MOServer @@ -107,6 +110,14 @@ type BaseService interface { UpgradeTenant(ctx context.Context, tenantName string, retryCount uint32, isALLAccount bool) error } +// SQLConnectionAdmissionController lets a CN fail-close new direct SQL +// connections while its control-plane lease or durable visibility fence is not +// valid, without expanding the BaseService contract implemented by tests and +// embedders. +type SQLConnectionAdmissionController interface { + CanAcceptNewConnections() bool +} + func (mo *MOServer) GetRoutineManager() *RoutineManager { return mo.rm } @@ -226,6 +237,10 @@ func (mo *MOServer) startAccept(ctx context.Context, listener net.Listener) { return } tempDelay = 0 + if mo.canAcceptNewConnections != nil && !mo.canAcceptNewConnections() { + _ = conn.Close() + continue + } go mo.handleConn(ctx, conn) } @@ -689,6 +704,9 @@ func NewMOServer( handler: rm.Handler, service: service, } + if admission, ok := baseService.(SQLConnectionAdmissionController); ok { + mo.canAcceptNewConnections = admission.CanAcceptNewConnections + } listenerTcp, err := net.Listen("tcp", addr) if err != nil { logutil.Panicf("start server failed with %+v", err) diff --git a/pkg/frontend/server_test.go b/pkg/frontend/server_test.go index b3491f52c4c9b..2906258edd1bc 100644 --- a/pkg/frontend/server_test.go +++ b/pkg/frontend/server_test.go @@ -34,6 +34,18 @@ type closeErrorListener struct { err error } +type scriptedSQLAdmissionListener struct { + accept func() (net.Conn, error) +} + +func (l *scriptedSQLAdmissionListener) Accept() (net.Conn, error) { + return l.accept() +} + +func (l *scriptedSQLAdmissionListener) Close() error { return nil } + +func (l *scriptedSQLAdmissionListener) Addr() net.Addr { return nil } + type testMOServerBaseService struct { MockBaseService id string @@ -109,6 +121,27 @@ func TestMOServerStopBeforeStartReleasesListener(t *testing.T) { require.NoError(t, rebound.Close()) } +func TestMOServerRejectsDirectSQLWhenCNAdmissionIsClosed(t *testing.T) { + serverSide, clientSide := net.Pipe() + defer clientSide.Close() + sentinel := errors.New("listener stopped") + step := 0 + listener := &scriptedSQLAdmissionListener{accept: func() (net.Conn, error) { + step++ + if step == 1 { + return serverSide, nil + } + return nil, sentinel + }} + mo := &MOServer{canAcceptNewConnections: func() bool { return false }} + mo.wg.Add(1) + mo.startAccept(context.Background(), listener) + + buf := make([]byte, 1) + _, err := clientSide.Read(buf) + require.Error(t, err, "CN admission must close the direct SQL socket before session creation") +} + func Test_handshake(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/pkg/frontend/session.go b/pkg/frontend/session.go index 49f844c46afbb..e4db57532c27d 100644 --- a/pkg/frontend/session.go +++ b/pkg/frontend/session.go @@ -1917,7 +1917,10 @@ func (ses *Session) UpgradeTenant(ctx context.Context, tenantName string, retryC return ses.rm.baseService.UpgradeTenant(ctx, tenantName, retryCount, isALLAccount) } -func (ses *Session) getGlobalSysVars(ctx context.Context, bh BackgroundExec) (gSysVars map[string]interface{}, err error) { +func (ses *Session) getGlobalSysVars( + ctx context.Context, + bh BackgroundExec, +) (gSysVars map[string]interface{}, catalogEpoch uint64, err error) { var execResults []ExecResult tenantInfo := ses.GetTenantInfo() @@ -1944,6 +1947,14 @@ func (ses *Session) getGlobalSysVars(ctx context.Context, bh BackgroundExec) (gS if varValue, err = execResult.GetString(tenantCtx, i, 1); err != nil { return } + if varName == globalSystemVariableEpochName { + catalogEpoch, err = strconv.ParseUint(varValue, 10, 64) + if err != nil { + err = moerr.NewInternalErrorf( + tenantCtx, "invalid global system variable epoch %q", varValue) + } + continue + } // overwrite with the values from table `mo_mysql_compatibility` if sv, ok := gSysVarsDefs[varName]; ok { @@ -2130,6 +2141,12 @@ func (ses *Session) getCleanupContext() context.Context { return context.Background() } +var initializeCachedSessionSystemVariables = func(ctx context.Context, ses *Session) error { + bh := ses.GetBackgroundExec(ctx) + defer bh.Close() + return ses.InitSystemVariables(ctx, bh) +} + // reset resets the ses instance and copy some fields of prev, then // close the prev. func (ses *Session) reset(ctx context.Context, prev *Session) error { @@ -2197,6 +2214,17 @@ func (ses *Session) reset(ctx context.Context, prev *Session) error { return cause } } + // A cached backend connection represents a brand-new external login after + // ResetSession. It does not run the normal CN authentication path again, so + // initialize its account snapshot before Proxy can hand the connection to + // the new client. Roll back the old transaction first: it may itself hold a + // catalog lock needed by this read, and waiting before rollback would form a + // self-deadlock. On failure the speculative session is discarded. + if prev.gSysVars != nil { + if err := initializeCachedSessionSystemVariables(ctx, ses); err != nil { + return err + } + } // close the previous session. prev.ReserveConnAndClose() return nil diff --git a/pkg/frontend/test/txn_mock.go b/pkg/frontend/test/txn_mock.go index b72ec2c689984..4c991d48bb8c7 100644 --- a/pkg/frontend/test/txn_mock.go +++ b/pkg/frontend/test/txn_mock.go @@ -94,6 +94,20 @@ func (mr *MockTxnTimestampAwareMockRecorder) SyncLatestCommitTS(arg0 interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncLatestCommitTS", reflect.TypeOf((*MockTxnTimestampAware)(nil).SyncLatestCommitTS), arg0) } +// SyncLatestCommitTSWithContext mocks base method. +func (m *MockTxnTimestampAware) SyncLatestCommitTSWithContext(arg0 context.Context, arg1 timestamp.Timestamp) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SyncLatestCommitTSWithContext", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SyncLatestCommitTSWithContext indicates an expected call of SyncLatestCommitTSWithContext. +func (mr *MockTxnTimestampAwareMockRecorder) SyncLatestCommitTSWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncLatestCommitTSWithContext", reflect.TypeOf((*MockTxnTimestampAware)(nil).SyncLatestCommitTSWithContext), arg0, arg1) +} + // WaitLogTailAppliedAt mocks base method. func (m *MockTxnTimestampAware) WaitLogTailAppliedAt(ctx context.Context, ts timestamp.Timestamp) (timestamp.Timestamp, error) { m.ctrl.T.Helper() @@ -372,6 +386,20 @@ func (mr *MockTxnClientMockRecorder) SyncLatestCommitTS(arg0 interface{}) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncLatestCommitTS", reflect.TypeOf((*MockTxnClient)(nil).SyncLatestCommitTS), arg0) } +// SyncLatestCommitTSWithContext mocks base method. +func (m *MockTxnClient) SyncLatestCommitTSWithContext(arg0 context.Context, arg1 timestamp.Timestamp) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SyncLatestCommitTSWithContext", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SyncLatestCommitTSWithContext indicates an expected call of SyncLatestCommitTSWithContext. +func (mr *MockTxnClientMockRecorder) SyncLatestCommitTSWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncLatestCommitTSWithContext", reflect.TypeOf((*MockTxnClient)(nil).SyncLatestCommitTSWithContext), arg0, arg1) +} + // WaitLogTailAppliedAt mocks base method. func (m *MockTxnClient) WaitLogTailAppliedAt(ctx context.Context, ts timestamp.Timestamp) (timestamp.Timestamp, error) { m.ctrl.T.Helper() diff --git a/pkg/frontend/timeout_behavior_test.go b/pkg/frontend/timeout_behavior_test.go index 39f0df2b3479e..fc592713038dd 100644 --- a/pkg/frontend/timeout_behavior_test.go +++ b/pkg/frontend/timeout_behavior_test.go @@ -47,6 +47,11 @@ func stubGlobalSysVarPersistence(t *testing.T, vars ...sysVarSet) *backgroundExe bh.sql2result["begin;"] = nil bh.sql2result["commit;"] = nil bh.sql2result["rollback;"] = nil + bh.sql2result[getSqlForLockGlobalSystemVariableAccount(sysAccountID)] = nil + bh.sql2result[getSqlForGlobalSystemVariableEpoch(sysAccountID)] = + newMrsForSystemVariableNameOfAccount([][]interface{}{}) + bh.sql2result[getSqlForInsertSysVarWithAccount( + sysAccountID, sysAccountName, globalSystemVariableEpochName, "1")] = nil for _, v := range vars { bh.sql2result[getSqlForGetSysVarWithAccount(sysAccountID, v.name)] = newMrsForSystemVariableNameOfAccount([][]interface{}{}) diff --git a/pkg/frontend/types.go b/pkg/frontend/types.go index 6ed35db638c47..f43494fba4579 100644 --- a/pkg/frontend/types.go +++ b/pkg/frontend/types.go @@ -1451,12 +1451,30 @@ func (ses *Session) SetGlobalSysVar(ctx context.Context, name string, val interf } } + // A mixed-version deployment must reject before the catalog mutation. Old + // CNs implement SyncCommit with an uninterruptible five-minute/Fatal path + // and do not participate in the HAKeeper routing fence. + if err = validateGlobalSysVarSyncProtocol(ctx, ses); err != nil { + return err + } + accountID := ses.GetTenantInfo().TenantID + unlock := GSysVarsMgr.lockAccount(accountID) + defer unlock() + // save to table first - if err = doSetGlobalSystemVariable(ctx, ses, name, val); err != nil { + var catalogEpoch uint64 + if catalogEpoch, err = doSetGlobalSystemVariable(ctx, ses, name, val); err != nil { return } - ses.gSysVars.Set(name, val) - return + ses.gSysVars.setAtCatalogEpoch(name, val, catalogEpoch) + if err = syncGlobalSysVarCommit(ctx, ses); err != nil { + // The catalog epoch row remains as a durable reconciliation intent. + // A subsequent session on any CN will replay the fence before publishing + // that epoch into its local cache. + return err + } + GSysVarsMgr.markCatalogEpochReconciled(accountID, catalogEpoch) + return nil } func (ses *feSessionImpl) GetSessionSysVars() *SystemVariables { diff --git a/pkg/frontend/variables.go b/pkg/frontend/variables.go index 04ec4c8fda6a0..0d9b3e7077c42 100644 --- a/pkg/frontend/variables.go +++ b/pkg/frontend/variables.go @@ -966,6 +966,56 @@ func (sv SystemVariable) GetDefault() interface{} { type GlobalSysVarsMgr struct { sync.Mutex accountsGlobalSysVarsMap map[uint32]*SystemVariables + accountLocks map[uint32]*sync.Mutex + reconciledEpochs map[uint32]uint64 +} + +func (m *GlobalSysVarsMgr) lockAccount(accountID uint32) func() { + m.Lock() + if m.accountLocks == nil { + m.accountLocks = make(map[uint32]*sync.Mutex) + } + lock := m.accountLocks[accountID] + if lock == nil { + lock = &sync.Mutex{} + m.accountLocks[accountID] = lock + } + m.Unlock() + lock.Lock() + return lock.Unlock +} + +func (m *GlobalSysVarsMgr) reconcileCatalogEpoch( + accountID uint32, + epoch uint64, + ctx context.Context, + ses *Session, +) error { + if epoch == 0 { + return nil + } + m.Lock() + alreadyReconciled := m.reconciledEpochs != nil && m.reconciledEpochs[accountID] >= epoch + m.Unlock() + if alreadyReconciled { + return nil + } + if err := syncGlobalSysVarCommit(ctx, ses); err != nil { + return err + } + m.markCatalogEpochReconciled(accountID, epoch) + return nil +} + +func (m *GlobalSysVarsMgr) markCatalogEpochReconciled(accountID uint32, epoch uint64) { + m.Lock() + defer m.Unlock() + if m.reconciledEpochs == nil { + m.reconciledEpochs = make(map[uint32]uint64) + } + if m.reconciledEpochs[accountID] < epoch { + m.reconciledEpochs[accountID] = epoch + } } func useTomlConfigOverOtherConfigs(CNServiceConfig *config.FrontendParameters, sysVarsMp map[string]interface{}) { @@ -990,6 +1040,9 @@ func resolveServerID(ses *Session) string { // Get return sys vars of accountId func (m *GlobalSysVarsMgr) Get(accountId uint32, ses *Session, ctx context.Context, bh BackgroundExec) (*SystemVariables, error) { + unlock := m.lockAccount(accountId) + defer unlock() + m.Lock() sysVars, ok := m.accountsGlobalSysVarsMap[accountId] var mutationGeneration uint64 @@ -998,10 +1051,13 @@ func (m *GlobalSysVarsMgr) Get(accountId uint32, ses *Session, ctx context.Conte } m.Unlock() - sysVarsMp, err := ses.getGlobalSysVars(ctx, bh) + sysVarsMp, catalogEpoch, err := ses.getGlobalSysVars(ctx, bh) if err != nil { return nil, err } + if err = m.reconcileCatalogEpoch(accountId, catalogEpoch, ctx, ses); err != nil { + return nil, err + } CNServiceConfig := getPu(ses.service).SV useTomlConfigOverOtherConfigs(CNServiceConfig, sysVarsMp) @@ -1011,7 +1067,7 @@ func (m *GlobalSysVarsMgr) Get(accountId uint32, ses *Session, ctx context.Conte defer m.Unlock() current, exists := m.accountsGlobalSysVarsMap[accountId] if !exists { - current = &SystemVariables{mp: sysVarsMp} + current = &SystemVariables{mp: sysVarsMp, catalogEpoch: catalogEpoch} m.accountsGlobalSysVarsMap[accountId] = current return current, nil } @@ -1021,7 +1077,7 @@ func (m *GlobalSysVarsMgr) Get(accountId uint32, ses *Session, ctx context.Conte if !ok || current != sysVars { return current, nil } - current.replaceIfMutationGeneration(mutationGeneration, sysVarsMp) + current.replaceIfCatalogEpoch(catalogEpoch, mutationGeneration, sysVarsMp) return current, nil } @@ -1033,6 +1089,8 @@ func (m *GlobalSysVarsMgr) Put(accountId uint32, vars *SystemVariables) { var GSysVarsMgr = &GlobalSysVarsMgr{ accountsGlobalSysVarsMap: make(map[uint32]*SystemVariables), + accountLocks: make(map[uint32]*sync.Mutex), + reconciledEpochs: make(map[uint32]uint64), } // SystemVariables is account level @@ -1044,6 +1102,10 @@ type SystemVariables struct { // refresh is derived from the catalog and must not invalidate another // refresh that observed the same local generation. mutationGeneration uint64 + // catalogEpoch is advanced atomically with a SET GLOBAL catalog mutation. + // Cache publication is monotonic in this value, so older refreshes and + // reversed local completions cannot roll back a newer account snapshot. + catalogEpoch uint64 } func (sv *SystemVariables) getMutationGeneration() uint64 { @@ -1055,12 +1117,22 @@ func (sv *SystemVariables) getMutationGeneration() uint64 { // replaceIfMutationGeneration publishes a refreshed snapshot only when no // local mutation has been applied since the refresh started. func (sv *SystemVariables) replaceIfMutationGeneration(generation uint64, mp map[string]interface{}) { + sv.replaceIfCatalogEpoch(sv.catalogEpoch, generation, mp) +} + +func (sv *SystemVariables) replaceIfCatalogEpoch( + epoch uint64, + generation uint64, + mp map[string]interface{}, +) { sv.mu.Lock() defer sv.mu.Unlock() - if sv.mutationGeneration != generation { + if epoch < sv.catalogEpoch || + (epoch == sv.catalogEpoch && sv.mutationGeneration != generation) { return } sv.mp = mp + sv.catalogEpoch = epoch } // Clone returns a copy of sv @@ -1071,7 +1143,7 @@ func (sv *SystemVariables) Clone() *SystemVariables { for name, value := range sv.mp { mp[name] = value } - return &SystemVariables{mp: mp} + return &SystemVariables{mp: mp, catalogEpoch: sv.catalogEpoch} } func (sv *SystemVariables) Get(name string) interface{} { @@ -1089,6 +1161,23 @@ func (sv *SystemVariables) Set(name string, value interface{}) { sv.mutationGeneration++ } +func (sv *SystemVariables) setAtCatalogEpoch( + name string, + value interface{}, + epoch uint64, +) bool { + sv.mu.Lock() + defer sv.mu.Unlock() + if epoch < sv.catalogEpoch { + return false + } + name = strings.ToLower(name) + sv.mp[name] = value + sv.catalogEpoch = epoch + sv.mutationGeneration++ + return true +} + // definitions of system variables const ( enableExplainScheduling = "enable_explain_scheduling" diff --git a/pkg/hakeeper/rsm.go b/pkg/hakeeper/rsm.go index 584bc3a03c008..3ca858b698423 100644 --- a/pkg/hakeeper/rsm.go +++ b/pkg/hakeeper/rsm.go @@ -34,6 +34,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) var ( @@ -426,6 +427,16 @@ func GetUpdateNonVotingLocality(locality pb.Locality) []byte { return cmd } +// GetUpdateGlobalSysVarCommitTSCmd advances the durable CN admission fence. +func GetUpdateGlobalSysVarCommitTSCmd(ts timestamp.Timestamp) []byte { + cmd := make([]byte, headerSize+ts.ProtoSize()) + binaryEnc.PutUint32(cmd, uint32(pb.UpdateGlobalSysVarCommitTS)) + if _, err := ts.MarshalTo(cmd[headerSize:]); err != nil { + panic(err) + } + return cmd +} + func getHeartbeatCmd(data []byte, tag pb.HAKeeperUpdateType) []byte { cmd := make([]byte, headerSize+len(data)) binaryEnc.PutUint32(cmd, uint32(tag)) @@ -792,6 +803,19 @@ func (s *stateMachine) getCommandBatch(uuid string) sm.Result { return s.getCommandBatchFiltered(uuid, false) } +func (s *stateMachine) commandBatchResult(batch pb.CommandBatch) sm.Result { + batch.GlobalSysVarCommitTS = s.state.CNState.GlobalSysVarCommitTS + if len(batch.Commands) == 0 && len(batch.CommandIDs) == 0 && + batch.BatchID == 0 && batch.GlobalSysVarCommitTS.IsEmpty() { + return sm.Result{} + } + data, err := batch.Marshal() + if err != nil { + panic(err) + } + return sm.Result{Data: data} +} + func (s *stateMachine) getCommandBatchFiltered( uuid string, filterHAKeeperAdmissions bool, @@ -839,13 +863,9 @@ func (s *stateMachine) getCommandBatchFiltered( batch.Commands = deliver batch.CommandIDs = deliverIDs - data, err := batch.Marshal() - if err != nil { - panic(err) - } - return sm.Result{Data: data} + return s.commandBatchResult(batch) } - return sm.Result{} + return s.commandBatchResult(pb.CommandBatch{}) } @@ -898,7 +918,7 @@ func (s *stateMachine) logScheduleCommandDeliverable(cmd pb.ScheduleCommand) boo func (s *stateMachine) getCommandBatchWithAck(uuid string, ack uint64) sm.Result { batch, ok := s.state.ScheduleCommands[uuid] if !ok { - return sm.Result{} + return s.commandBatchResult(pb.CommandBatch{}) } if ensureScheduleCommandIDs(&batch, s.state.Index) { // A snapshot produced before delivery IDs were introduced can still @@ -919,17 +939,13 @@ func (s *stateMachine) getCommandBatchWithAck(uuid string, ack uint64) sm.Result } if len(pending) == 0 { delete(s.state.ScheduleCommands, uuid) - return sm.Result{} + return s.commandBatchResult(pb.CommandBatch{}) } batch.Commands = pending batch.CommandIDs = pendingIDs s.state.ScheduleCommands[uuid] = batch } - data, err := batch.Marshal() - if err != nil { - panic(err) - } - return sm.Result{Data: data} + return s.commandBatchResult(batch) } // bootstrapReplicaCommandStatus returns whether a bootstrap command must be @@ -1601,6 +1617,13 @@ func (s *stateMachine) Update(e sm.Entry) (sm.Result, error) { return s.handleCompleteLogServiceRecoveryCmd(), nil case pb.EnableCommandDeliveryUpdate: return s.handleEnableCommandDelivery(cmd), nil + case pb.UpdateGlobalSysVarCommitTS: + var ts timestamp.Timestamp + if err := ts.Unmarshal(cmd[headerSize:]); err != nil { + panic(err) + } + s.state.CNState.UpdateGlobalSysVarCommitTS(ts) + return sm.Result{}, nil case pb.SetTaskTableUserUpdate: s.assertState() return s.handleTaskTableUserCmd(cmd), nil @@ -1660,18 +1683,20 @@ func (s *stateMachine) handleScheduleCommandQuery(uuid string) *pb.CommandBatch if !ok { panic("deep copy failed") } + result.GlobalSysVarCommitTS = s.state.CNState.GlobalSysVarCommitTS return result } - return &pb.CommandBatch{} + return &pb.CommandBatch{GlobalSysVarCommitTS: s.state.CNState.GlobalSysVarCommitTS} } func (s *stateMachine) handleClusterDetailsQuery(cfg Config) *pb.ClusterDetails { cfg.Fill() cd := &pb.ClusterDetails{ - CNStores: make([]pb.CNStore, 0, len(s.state.CNState.Stores)), - TNStores: make([]pb.TNStore, 0, len(s.state.TNState.Stores)), - LogStores: make([]pb.LogStore, 0, len(s.state.LogState.Stores)), - ProxyStores: make([]pb.ProxyStore, 0, len(s.state.ProxyState.Stores)), + CNStores: make([]pb.CNStore, 0, len(s.state.CNState.Stores)), + TNStores: make([]pb.TNStore, 0, len(s.state.TNState.Stores)), + LogStores: make([]pb.LogStore, 0, len(s.state.LogState.Stores)), + ProxyStores: make([]pb.ProxyStore, 0, len(s.state.ProxyState.Stores)), + GlobalSysVarCommitTS: s.state.CNState.GlobalSysVarCommitTS, } for uuid, info := range s.state.CNState.Stores { state := pb.NormalState @@ -1679,20 +1704,23 @@ func (s *stateMachine) handleClusterDetailsQuery(cfg Config) *pb.ClusterDetails state = pb.TimeoutState } n := pb.CNStore{ - UUID: uuid, - Tick: info.Tick, - ServiceAddress: info.ServiceAddress, - SQLAddress: info.SQLAddress, - LockServiceAddress: info.LockServiceAddress, - ShardServiceAddress: info.ShardServiceAddress, - State: state, - WorkState: info.WorkState, - Labels: info.Labels, - QueryAddress: info.QueryAddress, - ConfigData: info.ConfigData, - Resource: info.Resource, - UpTime: info.UpTime, - CommitID: info.CommitID, + UUID: uuid, + Tick: info.Tick, + ServiceAddress: info.ServiceAddress, + SQLAddress: info.SQLAddress, + LockServiceAddress: info.LockServiceAddress, + ShardServiceAddress: info.ShardServiceAddress, + State: state, + WorkState: info.WorkState, + Labels: info.Labels, + QueryAddress: info.QueryAddress, + ConfigData: info.ConfigData, + Resource: info.Resource, + UpTime: info.UpTime, + CommitID: info.CommitID, + GlobalSysVarCommitTS: info.GlobalSysVarCommitTS, + GlobalSysVarGeneration: info.GlobalSysVarGeneration, + ProtocolVersion: info.ProtocolVersion, } cd.CNStores = append(cd.CNStores, n) } @@ -1722,22 +1750,31 @@ func (s *stateMachine) handleClusterDetailsQuery(cfg Config) *pb.ClusterDetails state = pb.TimeoutState } n := pb.LogStore{ - UUID: uuid, - Tick: info.Tick, - State: state, - ServiceAddress: info.ServiceAddress, - Replicas: info.Replicas, - ConfigData: info.ConfigData, - Locality: info.Locality, + UUID: uuid, + Tick: info.Tick, + State: state, + ServiceAddress: info.ServiceAddress, + Replicas: info.Replicas, + ConfigData: info.ConfigData, + Locality: info.Locality, + ProtocolVersion: info.ProtocolVersion, } cd.LogStores = append(cd.LogStores, n) } for uuid, info := range s.state.ProxyState.Stores { + state := pb.NormalState + if cfg.ProxyStoreExpired(info.Tick, s.state.Tick) { + state = pb.TimeoutState + } cd.ProxyStores = append(cd.ProxyStores, pb.ProxyStore{ - UUID: uuid, - Tick: info.Tick, - ListenAddress: info.ListenAddress, - ConfigData: info.ConfigData, + UUID: uuid, + Tick: info.Tick, + ListenAddress: info.ListenAddress, + ConfigData: info.ConfigData, + GlobalSysVarCommitTS: info.GlobalSysVarCommitTS, + GlobalSysVarGeneration: info.GlobalSysVarGeneration, + State: state, + ProtocolVersion: info.ProtocolVersion, }) } for _, store := range s.state.DeletedStores { diff --git a/pkg/hakeeper/rsm_test.go b/pkg/hakeeper/rsm_test.go index 472cb2c932764..4c86b8427cf84 100644 --- a/pkg/hakeeper/rsm_test.go +++ b/pkg/hakeeper/rsm_test.go @@ -26,6 +26,7 @@ import ( pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) func TestAssignID(t *testing.T) { @@ -210,6 +211,93 @@ func TestHandleCNHeartbeat(t *testing.T) { assert.Equal(t, hb.CommitID, cninfo.CommitID) } +func TestGlobalSysVarCommitWatermarkIsDurableAndMonotonic(t *testing.T) { + rsm := NewStateMachine(0, 1).(*stateMachine) + latest := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + older := timestamp.Timestamp{PhysicalTime: 90} + + _, err := rsm.Update(sm.Entry{Index: 1, Cmd: GetUpdateGlobalSysVarCommitTSCmd(latest)}) + require.NoError(t, err) + _, err = rsm.Update(sm.Entry{Index: 2, Cmd: GetUpdateGlobalSysVarCommitTSCmd(older)}) + require.NoError(t, err) + require.Equal(t, latest, rsm.state.CNState.GlobalSysVarCommitTS) + + hb := pb.CNStoreHeartbeat{ + UUID: "cn-1", + SQLAddress: "sql-1", + CommandDeliveryAckSupported: true, + GlobalSysVarCommitTS: older, + ProtocolVersion: 14, + } + data, err := hb.Marshal() + require.NoError(t, err) + result, err := rsm.Update(sm.Entry{Index: 3, Cmd: GetCNStoreHeartbeatCmd(data)}) + require.NoError(t, err) + var batch pb.CommandBatch + require.NoError(t, batch.Unmarshal(result.Data)) + require.Equal(t, latest, batch.GlobalSysVarCommitTS) + require.Equal(t, older, rsm.state.CNState.Stores[hb.UUID].GlobalSysVarCommitTS) + + hb.GlobalSysVarCommitTS = latest + data, err = hb.Marshal() + require.NoError(t, err) + _, err = rsm.Update(sm.Entry{Index: 4, Cmd: GetCNStoreHeartbeatCmd(data)}) + require.NoError(t, err) + details := rsm.handleClusterDetailsQuery(Config{}) + require.Equal(t, latest, details.GlobalSysVarCommitTS) + require.Len(t, details.CNStores, 1) + require.Equal(t, latest, details.CNStores[0].GlobalSysVarCommitTS) + require.Equal(t, int64(14), details.CNStores[0].ProtocolVersion) + + hb.GlobalSysVarGeneration = "generation-a" + data, err = hb.Marshal() + require.NoError(t, err) + _, err = rsm.Update(sm.Entry{Index: 5, Cmd: GetCNStoreHeartbeatCmd(data)}) + require.NoError(t, err) + hb.GlobalSysVarGeneration = "generation-b" + hb.GlobalSysVarCommitTS = timestamp.Timestamp{} + data, err = hb.Marshal() + require.NoError(t, err) + _, err = rsm.Update(sm.Entry{Index: 6, Cmd: GetCNStoreHeartbeatCmd(data)}) + require.NoError(t, err) + require.Empty(t, rsm.state.CNState.Stores[hb.UUID].GlobalSysVarCommitTS, + "a restarted CN must not inherit the previous process's visibility ack") +} + +func TestProxyGlobalSysVarCommitWatermarkTracksIncarnation(t *testing.T) { + state := pb.NewProxyState() + latest := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + state.Update(pb.ProxyHeartbeat{ + UUID: "proxy-1", GlobalSysVarGeneration: "generation-a", + GlobalSysVarCommitTS: latest, + }, 1) + require.Equal(t, latest, state.Stores["proxy-1"].GlobalSysVarCommitTS) + + state.Update(pb.ProxyHeartbeat{ + UUID: "proxy-1", GlobalSysVarGeneration: "generation-b", + }, 2) + require.Empty(t, state.Stores["proxy-1"].GlobalSysVarCommitTS, + "a restarted Proxy must not inherit the previous process's route-barrier ack") +} + +func TestClusterDetailsReportsExpiredProxy(t *testing.T) { + rsm := NewStateMachine(0, 1).(*stateMachine) + rsm.state.Tick = 10 + rsm.state.ProxyState.Stores["proxy-live"] = pb.ProxyStore{Tick: 10} + rsm.state.ProxyState.Stores["proxy-expired"] = pb.ProxyStore{Tick: 1} + + details := rsm.handleClusterDetailsQuery(Config{ + TickPerSecond: 1, + ProxyStoreTimeout: time.Second, + }) + states := make(map[string]pb.NodeState, len(details.ProxyStores)) + for _, proxy := range details.ProxyStores { + states[proxy.UUID] = proxy.State + } + require.Equal(t, pb.NormalState, states["proxy-live"]) + require.Equal(t, pb.TimeoutState, states["proxy-expired"]) +} + func TestGetIDCmd(t *testing.T) { tsm1 := NewStateMachine(0, 1).(*stateMachine) tsm1.state.State = pb.HAKeeperRunning @@ -456,8 +544,9 @@ func TestClusterDetailsQuery(t *testing.T) { } tsm.state.LogState.Stores["store-1"] = pb.LogStoreInfo{ - Tick: 100, - ServiceAddress: "addr-log-1", + Tick: 100, + ServiceAddress: "addr-log-1", + ProtocolVersion: 14, Replicas: []pb.LogReplicaInfo{{ LogShardInfo: pb.LogShardInfo{ ShardID: 1, @@ -528,10 +617,11 @@ func TestClusterDetailsQuery(t *testing.T) { }, LogStores: []pb.LogStore{ { - UUID: "store-1", - ServiceAddress: "addr-log-1", - Tick: 100, - State: 0, + UUID: "store-1", + ServiceAddress: "addr-log-1", + Tick: 100, + State: 0, + ProtocolVersion: 14, Replicas: []pb.LogReplicaInfo{{ LogShardInfo: pb.LogShardInfo{ ShardID: 1, diff --git a/pkg/logservice/hakeeper_client.go b/pkg/logservice/hakeeper_client.go index 04cb3982ab276..1756369eaf31f 100644 --- a/pkg/logservice/hakeeper_client.go +++ b/pkg/logservice/hakeeper_client.go @@ -32,6 +32,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/hakeeper" "github.com/matrixorigin/matrixone/pkg/logutil" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/util/trace" ) @@ -41,7 +42,12 @@ const ( // ScheduleCommandPollInterval bounds the start-to-start delay between // degraded-path command reads. ScheduleCommandPollTimeout bounds each read. // Neither value inherits the heartbeat RPC timeout. - ScheduleCommandPollInterval = time.Second + ScheduleCommandPollInterval = time.Second + // GlobalSysVarHeartbeatProgressBudget is the maximum supported interval + // from one successful service heartbeat attempt to the next. The SQL fence + // timeout covers two such cycles plus control-plane RPC slack. + GlobalSysVarHeartbeatProgressBudget = 10 * time.Second + GlobalSysVarFenceTimeout = 3 * GlobalSysVarHeartbeatProgressBudget ScheduleCommandPollTimeout = 3 * time.Second scheduleCommandInitialPollJitterRange = 250 * time.Millisecond ) @@ -209,6 +215,12 @@ type ScheduleCommandHAKeeperClient interface { GetScheduleCommands(ctx context.Context, serviceType pb.ServiceType) (pb.CommandBatch, error) } +// GlobalSysVarHAKeeperClient is the version-gated HAKeeper capability used to +// publish the durable global-system-variable routing watermark. +type GlobalSysVarHAKeeperClient interface { + UpdateGlobalSysVarCommitTS(context.Context, timestamp.Timestamp) error +} + // LogHAKeeperClient is the HAKeeper client used by a Log store. type LogHAKeeperClient interface { basicHAKeeperClient @@ -953,6 +965,39 @@ func (c *managedHAKeeperClient) UpdateCNWorkState( } } +// UpdateGlobalSysVarCommitTS advances the durable CN admission watermark. +func (c *managedHAKeeperClient) UpdateGlobalSysVarCommitTS( + ctx context.Context, + ts timestamp.Timestamp, +) error { + if err := validateHAKeeperClientContext(ctx); err != nil { + return err + } + for { + client, err := c.getPreparedClient(ctx) + if err != nil { + if c.isRetryableError(err) { + if err := c.waitRetry(ctx); err != nil { + return err + } + continue + } + return err + } + err = client.updateGlobalSysVarCommitTS(ctx, ts) + if shouldResetHAKeeperClient(err) { + c.resetClientIfCurrent(client) + } + if c.isRetryableError(err) { + if err := c.waitRetry(ctx); err != nil { + return err + } + continue + } + return err + } +} + // PatchCNStore implements the ProxyHAKeeperClient interface. func (c *managedHAKeeperClient) PatchCNStore( ctx context.Context, stateLabel pb.CNStateLabel, @@ -1618,6 +1663,18 @@ func (c *hakeeperClient) updateCNWorkState(ctx context.Context, state pb.CNWorkS return nil } +func (c *hakeeperClient) updateGlobalSysVarCommitTS( + ctx context.Context, + ts timestamp.Timestamp, +) error { + req := pb.Request{ + Method: pb.UPDATE_GLOBAL_SYS_VAR_COMMIT_TS, + GlobalSysVarCommitTS: ts, + } + _, err := c.request(ctx, req) + return err +} + func (c *hakeeperClient) patchCNStore(ctx context.Context, stateLabel pb.CNStateLabel) error { req := pb.Request{ Method: pb.PATCH_CN_STORE, diff --git a/pkg/logservice/hakeeper_client_test.go b/pkg/logservice/hakeeper_client_test.go index 3cb1dc186ebe7..20aa04dea6392 100644 --- a/pkg/logservice/hakeeper_client_test.go +++ b/pkg/logservice/hakeeper_client_test.go @@ -38,6 +38,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/logutil" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) type countingErrorRPCClient struct { @@ -2030,6 +2031,57 @@ func TestHAKeeperClientUpdateCNWorkState(t *testing.T) { runServiceTest(t, true, true, fn) } +func TestHAKeeperClientGlobalSysVarCommitWatermark(t *testing.T) { + fn := func(t *testing.T, s *Service) { + cfg := HAKeeperClientConfig{ + ServiceAddresses: []string{s.cfg.LogServiceServiceAddr()}, + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + client, err := NewCNHAKeeperClient(ctx, "", cfg) + require.NoError(t, err) + defer func() { require.NoError(t, client.Close()) }() + fenceClient, ok := client.(GlobalSysVarHAKeeperClient) + require.True(t, ok) + + commitTS := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + require.NoError(t, fenceClient.UpdateGlobalSysVarCommitTS(ctx, commitTS)) + batch, err := client.SendCNHeartbeat(ctx, pb.CNStoreHeartbeat{ + UUID: s.ID(), + SQLAddress: "sql-address", + GlobalSysVarCommitTS: commitTS, + ServiceAddress: "service-address", + QueryAddress: "query-address", + LockServiceAddress: "lock-address", + ShardServiceAddress: "shard-address", + TaskServiceCreated: true, + GossipAddress: "gossip-address", + GossipJoined: true, + }) + require.NoError(t, err) + require.Equal(t, commitTS, batch.GlobalSysVarCommitTS) + proxyClient, err := NewProxyHAKeeperClient(ctx, "", cfg) + require.NoError(t, err) + defer func() { require.NoError(t, proxyClient.Close()) }() + _, err = proxyClient.SendProxyHeartbeat(ctx, pb.ProxyHeartbeat{ + UUID: "proxy-1", + ListenAddress: "proxy-address", + GlobalSysVarCommitTS: commitTS, + GlobalSysVarGeneration: "proxy-generation", + }) + require.NoError(t, err) + + details, err := client.GetClusterDetails(ctx) + require.NoError(t, err) + require.Equal(t, commitTS, details.GlobalSysVarCommitTS) + require.Len(t, details.CNStores, 1) + require.Equal(t, commitTS, details.CNStores[0].GlobalSysVarCommitTS) + require.Len(t, details.ProxyStores, 1) + require.Equal(t, commitTS, details.ProxyStores[0].GlobalSysVarCommitTS) + } + runServiceTest(t, true, true, fn) +} + func TestHAKeeperClientPatchCNStore(t *testing.T) { fn := func(t *testing.T, s *Service) { cfg := HAKeeperClientConfig{ diff --git a/pkg/logservice/service.go b/pkg/logservice/service.go index 1cbe03d904679..e812bf7a4a5ac 100644 --- a/pkg/logservice/service.go +++ b/pkg/logservice/service.go @@ -353,6 +353,8 @@ func (s *Service) handle(ctx context.Context, req pb.Request, return s.handleUpdateCNLabel(ctx, req), pb.LogRecordResponse{} case pb.UPDATE_CN_WORK_STATE: return s.handleUpdateCNWorkState(ctx, req), pb.LogRecordResponse{} + case pb.UPDATE_GLOBAL_SYS_VAR_COMMIT_TS: + return s.handleUpdateGlobalSysVarCommitTS(ctx, req), pb.LogRecordResponse{} case pb.PATCH_CN_STORE: return s.handlePatchCNStore(ctx, req), pb.LogRecordResponse{} case pb.DELETE_CN_STORE: @@ -663,6 +665,14 @@ func (s *Service) handleUpdateCNWorkState(ctx context.Context, req pb.Request) p return resp } +func (s *Service) handleUpdateGlobalSysVarCommitTS(ctx context.Context, req pb.Request) pb.Response { + resp := getResponse(req) + if err := s.store.updateGlobalSysVarCommitTS(ctx, req.GlobalSysVarCommitTS); err != nil { + resp.ErrorCode, resp.ErrorMessage = toErrorCode(err) + } + return resp +} + func (s *Service) handlePatchCNStore(ctx context.Context, req pb.Request) pb.Response { stateLabel := req.CNStateLabel resp := getResponse(req) diff --git a/pkg/logservice/store.go b/pkg/logservice/store.go index 9caeaefdd0043..4f9539bd80a0b 100644 --- a/pkg/logservice/store.go +++ b/pkg/logservice/store.go @@ -36,6 +36,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/common/stopper" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/hakeeper" "github.com/matrixorigin/matrixone/pkg/hakeeper/bootstrap" @@ -44,6 +45,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/logutil" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/taskservice" ) @@ -1060,6 +1062,20 @@ func (l *store) updateCNWorkState(ctx context.Context, workState pb.CNWorkState) } } +func (l *store) updateGlobalSysVarCommitTS( + ctx context.Context, + ts timestamp.Timestamp, +) error { + cmd := hakeeper.GetUpdateGlobalSysVarCommitTSCmd(ts) + session := l.nh.GetNoOPSession(hakeeper.DefaultHAKeeperShardID) + if _, err := l.propose(ctx, session, cmd); err != nil { + l.runtime.Logger().Error("failed to propose global sysvar commit timestamp", + zap.Error(err)) + return handleNotHAKeeperError(ctx, err) + } + return nil +} + func (l *store) patchCNStore(ctx context.Context, stateLabel pb.CNStateLabel) error { state, err := l.getCheckerState() if err != nil { @@ -1431,6 +1447,7 @@ func (l *store) getHeartbeatMessage() pb.LogStoreHeartbeat { Replicas: make([]pb.LogReplicaInfo, 0), Locality: l.cfg.getLocality(), CommandDeliverySupported: true, + ProtocolVersion: defines.MORPCLatestVersion, } opts := dragonboat.NodeHostInfoOption{ SkipLogInfo: true, diff --git a/pkg/logservice/store_metadata_test.go b/pkg/logservice/store_metadata_test.go index dffe750ae9e07..4af14f845a94f 100644 --- a/pkg/logservice/store_metadata_test.go +++ b/pkg/logservice/store_metadata_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/pb/metadata" ) @@ -115,6 +117,7 @@ func TestStartReplicas(t *testing.T) { done := false for i := 0; i < 1000; i++ { hb := store.getHeartbeatMessage() + require.Equal(t, defines.MORPCLatestVersion, hb.ProtocolVersion) if len(hb.Replicas) != 2 { time.Sleep(10 * time.Millisecond) continue diff --git a/pkg/pb/logservice/logservice.go b/pkg/pb/logservice/logservice.go index 47b429d8fff28..531758143f36d 100644 --- a/pkg/pb/logservice/logservice.go +++ b/pkg/pb/logservice/logservice.go @@ -22,6 +22,7 @@ import ( "time" "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" ) const ( @@ -156,9 +157,23 @@ func (s *CNState) Update(hb CNStoreHeartbeat, tick uint64) { storeInfo.Resource = hb.Resource storeInfo.CommitID = hb.CommitID storeInfo.CommandDeliveryAckSupported = hb.CommandDeliveryAckSupported + storeInfo.ProtocolVersion = hb.ProtocolVersion + if storeInfo.GlobalSysVarGeneration != hb.GlobalSysVarGeneration { + storeInfo.GlobalSysVarGeneration = hb.GlobalSysVarGeneration + storeInfo.GlobalSysVarCommitTS = hb.GlobalSysVarCommitTS + } else if storeInfo.GlobalSysVarCommitTS.Less(hb.GlobalSysVarCommitTS) { + storeInfo.GlobalSysVarCommitTS = hb.GlobalSysVarCommitTS + } s.Stores[hb.UUID] = storeInfo } +// UpdateGlobalSysVarCommitTS advances the durable routing-admission watermark. +func (s *CNState) UpdateGlobalSysVarCommitTS(ts timestamp.Timestamp) { + if s.GlobalSysVarCommitTS.Less(ts) { + s.GlobalSysVarCommitTS = ts + } +} + // UpdateLabel updates labels of CN store. func (s *CNState) UpdateLabel(label CNStoreLabel) { storeInfo, ok := s.Stores[label.UUID] @@ -266,6 +281,7 @@ func (s *LogState) updateStores(hb LogStoreHeartbeat, tick uint64) { } storeInfo.Locality = hb.Locality storeInfo.CommandDeliverySupported = hb.CommandDeliverySupported + storeInfo.ProtocolVersion = hb.ProtocolVersion s.Stores[hb.UUID] = storeInfo } @@ -383,6 +399,13 @@ func (s *ProxyState) Update(hb ProxyHeartbeat, tick uint64) { if hb.ConfigData != nil { storeInfo.ConfigData = hb.ConfigData } + storeInfo.ProtocolVersion = hb.ProtocolVersion + if storeInfo.GlobalSysVarGeneration != hb.GlobalSysVarGeneration { + storeInfo.GlobalSysVarGeneration = hb.GlobalSysVarGeneration + storeInfo.GlobalSysVarCommitTS = hb.GlobalSysVarCommitTS + } else if storeInfo.GlobalSysVarCommitTS.Less(hb.GlobalSysVarCommitTS) { + storeInfo.GlobalSysVarCommitTS = hb.GlobalSysVarCommitTS + } s.Stores[hb.UUID] = storeInfo } diff --git a/pkg/pb/logservice/logservice.pb.go b/pkg/pb/logservice/logservice.pb.go index b8ba584555132..27742b0b85d9e 100644 --- a/pkg/pb/logservice/logservice.pb.go +++ b/pkg/pb/logservice/logservice.pb.go @@ -15,6 +15,7 @@ import ( proto "github.com/gogo/protobuf/proto" github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" metadata "github.com/matrixorigin/matrixone/pkg/pb/metadata" + timestamp "github.com/matrixorigin/matrixone/pkg/pb/timestamp" _ "google.golang.org/protobuf/types/known/timestamppb" ) @@ -92,36 +93,37 @@ func (NodeState) EnumDescriptor() ([]byte, []int) { type MethodType int32 const ( - TSO_UPDATE MethodType = 0 - APPEND MethodType = 1 - READ MethodType = 2 - TRUNCATE MethodType = 3 - GET_TRUNCATE MethodType = 4 - CONNECT MethodType = 5 - CONNECT_RO MethodType = 6 - LOG_HEARTBEAT MethodType = 7 - CN_HEARTBEAT MethodType = 8 - TN_HEARTBEAT MethodType = 9 - CHECK_HAKEEPER MethodType = 10 - GET_CLUSTER_DETAILS MethodType = 11 - GET_SHARD_INFO MethodType = 12 - CN_ALLOCATE_ID MethodType = 13 - GET_CLUSTER_STATE MethodType = 14 - UPDATE_CN_LABEL MethodType = 15 - UPDATE_CN_WORK_STATE MethodType = 16 - PATCH_CN_STORE MethodType = 17 - DELETE_CN_STORE MethodType = 18 - PROXY_HEARTBEAT MethodType = 19 - UPDATE_NON_VOTING_REPLICA_NUM MethodType = 20 - UPDATE_NON_VOTING_LOCALITY MethodType = 21 - SET_REQUIRED_LSN MethodType = 22 - GET_REQUIRED_LSN MethodType = 23 - GET_LATEST_LSN MethodType = 24 - GET_LEADER_ID MethodType = 25 - CHECK_HEALTH MethodType = 26 - READ_LSN MethodType = 27 - UPDATE_LEASEHOLDER_ID MethodType = 28 - GET_SCHEDULE_COMMANDS MethodType = 29 + TSO_UPDATE MethodType = 0 + APPEND MethodType = 1 + READ MethodType = 2 + TRUNCATE MethodType = 3 + GET_TRUNCATE MethodType = 4 + CONNECT MethodType = 5 + CONNECT_RO MethodType = 6 + LOG_HEARTBEAT MethodType = 7 + CN_HEARTBEAT MethodType = 8 + TN_HEARTBEAT MethodType = 9 + CHECK_HAKEEPER MethodType = 10 + GET_CLUSTER_DETAILS MethodType = 11 + GET_SHARD_INFO MethodType = 12 + CN_ALLOCATE_ID MethodType = 13 + GET_CLUSTER_STATE MethodType = 14 + UPDATE_CN_LABEL MethodType = 15 + UPDATE_CN_WORK_STATE MethodType = 16 + PATCH_CN_STORE MethodType = 17 + DELETE_CN_STORE MethodType = 18 + PROXY_HEARTBEAT MethodType = 19 + UPDATE_NON_VOTING_REPLICA_NUM MethodType = 20 + UPDATE_NON_VOTING_LOCALITY MethodType = 21 + SET_REQUIRED_LSN MethodType = 22 + GET_REQUIRED_LSN MethodType = 23 + GET_LATEST_LSN MethodType = 24 + GET_LEADER_ID MethodType = 25 + CHECK_HEALTH MethodType = 26 + READ_LSN MethodType = 27 + UPDATE_LEASEHOLDER_ID MethodType = 28 + GET_SCHEDULE_COMMANDS MethodType = 29 + UPDATE_GLOBAL_SYS_VAR_COMMIT_TS MethodType = 30 ) var MethodType_name = map[int32]string{ @@ -155,39 +157,41 @@ var MethodType_name = map[int32]string{ 27: "READ_LSN", 28: "UPDATE_LEASEHOLDER_ID", 29: "GET_SCHEDULE_COMMANDS", + 30: "UPDATE_GLOBAL_SYS_VAR_COMMIT_TS", } var MethodType_value = map[string]int32{ - "TSO_UPDATE": 0, - "APPEND": 1, - "READ": 2, - "TRUNCATE": 3, - "GET_TRUNCATE": 4, - "CONNECT": 5, - "CONNECT_RO": 6, - "LOG_HEARTBEAT": 7, - "CN_HEARTBEAT": 8, - "TN_HEARTBEAT": 9, - "CHECK_HAKEEPER": 10, - "GET_CLUSTER_DETAILS": 11, - "GET_SHARD_INFO": 12, - "CN_ALLOCATE_ID": 13, - "GET_CLUSTER_STATE": 14, - "UPDATE_CN_LABEL": 15, - "UPDATE_CN_WORK_STATE": 16, - "PATCH_CN_STORE": 17, - "DELETE_CN_STORE": 18, - "PROXY_HEARTBEAT": 19, - "UPDATE_NON_VOTING_REPLICA_NUM": 20, - "UPDATE_NON_VOTING_LOCALITY": 21, - "SET_REQUIRED_LSN": 22, - "GET_REQUIRED_LSN": 23, - "GET_LATEST_LSN": 24, - "GET_LEADER_ID": 25, - "CHECK_HEALTH": 26, - "READ_LSN": 27, - "UPDATE_LEASEHOLDER_ID": 28, - "GET_SCHEDULE_COMMANDS": 29, + "TSO_UPDATE": 0, + "APPEND": 1, + "READ": 2, + "TRUNCATE": 3, + "GET_TRUNCATE": 4, + "CONNECT": 5, + "CONNECT_RO": 6, + "LOG_HEARTBEAT": 7, + "CN_HEARTBEAT": 8, + "TN_HEARTBEAT": 9, + "CHECK_HAKEEPER": 10, + "GET_CLUSTER_DETAILS": 11, + "GET_SHARD_INFO": 12, + "CN_ALLOCATE_ID": 13, + "GET_CLUSTER_STATE": 14, + "UPDATE_CN_LABEL": 15, + "UPDATE_CN_WORK_STATE": 16, + "PATCH_CN_STORE": 17, + "DELETE_CN_STORE": 18, + "PROXY_HEARTBEAT": 19, + "UPDATE_NON_VOTING_REPLICA_NUM": 20, + "UPDATE_NON_VOTING_LOCALITY": 21, + "SET_REQUIRED_LSN": 22, + "GET_REQUIRED_LSN": 23, + "GET_LATEST_LSN": 24, + "GET_LEADER_ID": 25, + "CHECK_HEALTH": 26, + "READ_LSN": 27, + "UPDATE_LEASEHOLDER_ID": 28, + "GET_SCHEDULE_COMMANDS": 29, + "UPDATE_GLOBAL_SYS_VAR_COMMIT_TS": 30, } func (x MethodType) String() string { @@ -278,6 +282,7 @@ const ( RestoreIDWatermarkUpdate HAKeeperUpdateType = 18 CompleteLogServiceRecoveryUpdate HAKeeperUpdateType = 19 EnableCommandDeliveryUpdate HAKeeperUpdateType = 20 + UpdateGlobalSysVarCommitTS HAKeeperUpdateType = 21 ) var HAKeeperUpdateType_name = map[int32]string{ @@ -302,6 +307,7 @@ var HAKeeperUpdateType_name = map[int32]string{ 18: "RestoreIDWatermarkUpdate", 19: "CompleteLogServiceRecoveryUpdate", 20: "EnableCommandDeliveryUpdate", + 21: "UpdateGlobalSysVarCommitTS", } var HAKeeperUpdateType_value = map[string]int32{ @@ -326,6 +332,7 @@ var HAKeeperUpdateType_value = map[string]int32{ "RestoreIDWatermarkUpdate": 18, "CompleteLogServiceRecoveryUpdate": 19, "EnableCommandDeliveryUpdate": 20, + "UpdateGlobalSysVarCommitTS": 21, } func (x HAKeeperUpdateType) String() string { @@ -508,24 +515,27 @@ func (ServiceType) EnumDescriptor() ([]byte, []int) { } type CNStore struct { - UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` - ServiceAddress string `protobuf:"bytes,2,opt,name=ServiceAddress,proto3" json:"ServiceAddress,omitempty"` - SQLAddress string `protobuf:"bytes,3,opt,name=SQLAddress,proto3" json:"SQLAddress,omitempty"` - LockServiceAddress string `protobuf:"bytes,4,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` - Role metadata.CNRole `protobuf:"varint,6,opt,name=Role,proto3,enum=metadata.CNRole" json:"Role,omitempty"` - Tick uint64 `protobuf:"varint,7,opt,name=Tick,proto3" json:"Tick,omitempty"` - State NodeState `protobuf:"varint,8,opt,name=State,proto3,enum=logservice.NodeState" json:"State,omitempty"` - Labels map[string]metadata.LabelList `protobuf:"bytes,9,rep,name=Labels,proto3" json:"Labels" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - WorkState metadata.WorkState `protobuf:"varint,10,opt,name=WorkState,proto3,enum=metadata.WorkState" json:"WorkState,omitempty"` - QueryAddress string `protobuf:"bytes,11,opt,name=QueryAddress,proto3" json:"QueryAddress,omitempty"` - ConfigData *ConfigData `protobuf:"bytes,12,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` - Resource Resource `protobuf:"bytes,13,opt,name=Resource,proto3" json:"Resource"` - UpTime int64 `protobuf:"varint,14,opt,name=UpTime,proto3" json:"UpTime,omitempty"` - ShardServiceAddress string `protobuf:"bytes,15,opt,name=ShardServiceAddress,proto3" json:"ShardServiceAddress,omitempty"` - CommitID string `protobuf:"bytes,16,opt,name=CommitID,proto3" json:"CommitID,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` + ServiceAddress string `protobuf:"bytes,2,opt,name=ServiceAddress,proto3" json:"ServiceAddress,omitempty"` + SQLAddress string `protobuf:"bytes,3,opt,name=SQLAddress,proto3" json:"SQLAddress,omitempty"` + LockServiceAddress string `protobuf:"bytes,4,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` + Role metadata.CNRole `protobuf:"varint,6,opt,name=Role,proto3,enum=metadata.CNRole" json:"Role,omitempty"` + Tick uint64 `protobuf:"varint,7,opt,name=Tick,proto3" json:"Tick,omitempty"` + State NodeState `protobuf:"varint,8,opt,name=State,proto3,enum=logservice.NodeState" json:"State,omitempty"` + Labels map[string]metadata.LabelList `protobuf:"bytes,9,rep,name=Labels,proto3" json:"Labels" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + WorkState metadata.WorkState `protobuf:"varint,10,opt,name=WorkState,proto3,enum=metadata.WorkState" json:"WorkState,omitempty"` + QueryAddress string `protobuf:"bytes,11,opt,name=QueryAddress,proto3" json:"QueryAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,12,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` + Resource Resource `protobuf:"bytes,13,opt,name=Resource,proto3" json:"Resource"` + UpTime int64 `protobuf:"varint,14,opt,name=UpTime,proto3" json:"UpTime,omitempty"` + ShardServiceAddress string `protobuf:"bytes,15,opt,name=ShardServiceAddress,proto3" json:"ShardServiceAddress,omitempty"` + CommitID string `protobuf:"bytes,16,opt,name=CommitID,proto3" json:"CommitID,omitempty"` + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,17,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` + GlobalSysVarGeneration string `protobuf:"bytes,18,opt,name=GlobalSysVarGeneration,proto3" json:"GlobalSysVarGeneration,omitempty"` + ProtocolVersion int64 `protobuf:"varint,19,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CNStore) Reset() { *m = CNStore{} } @@ -666,6 +676,27 @@ func (m *CNStore) GetCommitID() string { return "" } +func (m *CNStore) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + +func (m *CNStore) GetGlobalSysVarGeneration() string { + if m != nil { + return m.GlobalSysVarGeneration + } + return "" +} + +func (m *CNStore) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + type TNStore struct { UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` ServiceAddress string `protobuf:"bytes,2,opt,name=ServiceAddress,proto3" json:"ServiceAddress,omitempty"` @@ -804,6 +835,7 @@ type LogStore struct { Replicas []LogReplicaInfo `protobuf:"bytes,5,rep,name=Replicas,proto3" json:"Replicas"` ConfigData *ConfigData `protobuf:"bytes,6,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` Locality Locality `protobuf:"bytes,7,opt,name=Locality,proto3" json:"Locality"` + ProtocolVersion int64 `protobuf:"varint,8,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -891,6 +923,13 @@ func (m *LogStore) GetLocality() Locality { return Locality{} } +func (m *LogStore) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + // LogShardInfo contains information a log shard. type LogShardInfo struct { // ShardID is the ID of a Log shard. @@ -1137,11 +1176,16 @@ type CNStoreHeartbeat struct { CommitID string `protobuf:"bytes,15,opt,name=CommitID,proto3" json:"CommitID,omitempty"` // AckedCommandBatchID is the last schedule-command batch accepted by this // service. It is meaningful only when CommandDeliveryAckSupported is true. - AckedCommandBatchID uint64 `protobuf:"varint,16,opt,name=AckedCommandBatchID,proto3" json:"AckedCommandBatchID,omitempty"` - CommandDeliveryAckSupported bool `protobuf:"varint,17,opt,name=CommandDeliveryAckSupported,proto3" json:"CommandDeliveryAckSupported,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + AckedCommandBatchID uint64 `protobuf:"varint,16,opt,name=AckedCommandBatchID,proto3" json:"AckedCommandBatchID,omitempty"` + CommandDeliveryAckSupported bool `protobuf:"varint,17,opt,name=CommandDeliveryAckSupported,proto3" json:"CommandDeliveryAckSupported,omitempty"` + // GlobalSysVarCommitTS is the latest global-system-variable watermark whose + // logtail has been applied locally. + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,18,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` + GlobalSysVarGeneration string `protobuf:"bytes,19,opt,name=GlobalSysVarGeneration,proto3" json:"GlobalSysVarGeneration,omitempty"` + ProtocolVersion int64 `protobuf:"varint,20,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CNStoreHeartbeat) Reset() { *m = CNStoreHeartbeat{} } @@ -1289,6 +1333,27 @@ func (m *CNStoreHeartbeat) GetCommandDeliveryAckSupported() bool { return false } +func (m *CNStoreHeartbeat) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + +func (m *CNStoreHeartbeat) GetGlobalSysVarGeneration() string { + if m != nil { + return m.GlobalSysVarGeneration + } + return "" +} + +func (m *CNStoreHeartbeat) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + // CNAllocateID is the periodic message sent tp the HAKeeper by CN stores. type CNAllocateID struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -1375,6 +1440,7 @@ type LogStoreHeartbeat struct { // acknowledged delivery protocol only after every HAKeeper replica has // upgraded. CommandDeliverySupported bool `protobuf:"varint,9,opt,name=CommandDeliverySupported,proto3" json:"CommandDeliverySupported,omitempty"` + ProtocolVersion int64 `protobuf:"varint,10,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1476,6 +1542,13 @@ func (m *LogStoreHeartbeat) GetCommandDeliverySupported() bool { return false } +func (m *LogStoreHeartbeat) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + // TNShardInfo contains information of a launched TN shard. type TNShardInfo struct { // ShardID uniquely identifies a TN shard. Each TN shard manages a Primary @@ -2264,6 +2337,7 @@ type Request struct { NonVotingLocality *Locality `protobuf:"bytes,15,opt,name=NonVotingLocality,proto3" json:"NonVotingLocality,omitempty"` CheckHealth *CheckHealth `protobuf:"bytes,16,opt,name=CheckHealth,proto3" json:"CheckHealth,omitempty"` ScheduleCommandQuery *ScheduleCommandQuery `protobuf:"bytes,17,opt,name=ScheduleCommandQuery,proto3" json:"ScheduleCommandQuery,omitempty"` + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,18,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2421,6 +2495,13 @@ func (m *Request) GetScheduleCommandQuery() *ScheduleCommandQuery { return nil } +func (m *Request) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + type LogResponse struct { ShardID uint64 `protobuf:"varint,1,opt,name=ShardID,proto3" json:"ShardID,omitempty"` Lsn uint64 `protobuf:"varint,2,opt,name=Lsn,proto3" json:"Lsn,omitempty"` @@ -3637,7 +3718,9 @@ type CommandBatch struct { // later, intentionally identical command without depending on payload // fingerprints. A non-empty acknowledged batch must contain one valid ID per // command before it is exposed to CN/TN services. - CommandIDs []ScheduleCommandID `protobuf:"bytes,4,rep,name=CommandIDs,proto3" json:"CommandIDs"` + CommandIDs []ScheduleCommandID `protobuf:"bytes,4,rep,name=CommandIDs,proto3" json:"CommandIDs"` + // GlobalSysVarCommitTS is the desired cluster-wide visibility watermark. + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,5,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3704,6 +3787,13 @@ func (m *CommandBatch) GetCommandIDs() []ScheduleCommandID { return nil } +func (m *CommandBatch) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + // CNStoreInfo contains information on a CN store. type CNStoreInfo struct { Tick uint64 `protobuf:"varint,1,opt,name=Tick,proto3" json:"Tick,omitempty"` @@ -3725,10 +3815,13 @@ type CNStoreInfo struct { // CommandDeliveryAckSupported reports whether this CN can use the // non-destructive acknowledged schedule-command protocol. It is copied // from CNStoreHeartbeat so HAKeeper can gate activation and admission. - CommandDeliveryAckSupported bool `protobuf:"varint,18,opt,name=CommandDeliveryAckSupported,proto3" json:"CommandDeliveryAckSupported,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + CommandDeliveryAckSupported bool `protobuf:"varint,18,opt,name=CommandDeliveryAckSupported,proto3" json:"CommandDeliveryAckSupported,omitempty"` + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,19,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` + GlobalSysVarGeneration string `protobuf:"bytes,20,opt,name=GlobalSysVarGeneration,proto3" json:"GlobalSysVarGeneration,omitempty"` + ProtocolVersion int64 `protobuf:"varint,21,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CNStoreInfo) Reset() { *m = CNStoreInfo{} } @@ -3883,13 +3976,36 @@ func (m *CNStoreInfo) GetCommandDeliveryAckSupported() bool { return false } +func (m *CNStoreInfo) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + +func (m *CNStoreInfo) GetGlobalSysVarGeneration() string { + if m != nil { + return m.GlobalSysVarGeneration + } + return "" +} + +func (m *CNStoreInfo) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + // CNState contains all CN details known to the HAKeeper. type CNState struct { // Stores is keyed by CN store UUID. - Stores map[string]CNStoreInfo `protobuf:"bytes,1,rep,name=Stores,proto3" json:"Stores" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Stores map[string]CNStoreInfo `protobuf:"bytes,1,rep,name=Stores,proto3" json:"Stores" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // GlobalSysVarCommitTS is the durable admission watermark for routable CNs. + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,2,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CNState) Reset() { *m = CNState{} } @@ -3932,6 +4048,13 @@ func (m *CNState) GetStores() map[string]CNStoreInfo { return nil } +func (m *CNState) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + // TNStoreInfo contains information on a TN store. type TNStoreInfo struct { Tick uint64 `protobuf:"varint,1,opt,name=Tick,proto3" json:"Tick,omitempty"` @@ -4124,13 +4247,17 @@ func (m *TNState) GetStores() map[string]TNStoreInfo { } type ProxyStore struct { - UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` - Tick uint64 `protobuf:"varint,2,opt,name=Tick,proto3" json:"Tick,omitempty"` - ListenAddress string `protobuf:"bytes,3,opt,name=ListenAddress,proto3" json:"ListenAddress,omitempty"` - ConfigData *ConfigData `protobuf:"bytes,4,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` + Tick uint64 `protobuf:"varint,2,opt,name=Tick,proto3" json:"Tick,omitempty"` + ListenAddress string `protobuf:"bytes,3,opt,name=ListenAddress,proto3" json:"ListenAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,4,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,5,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` + GlobalSysVarGeneration string `protobuf:"bytes,6,opt,name=GlobalSysVarGeneration,proto3" json:"GlobalSysVarGeneration,omitempty"` + State NodeState `protobuf:"varint,7,opt,name=State,proto3,enum=logservice.NodeState" json:"State,omitempty"` + ProtocolVersion int64 `protobuf:"varint,8,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ProxyStore) Reset() { *m = ProxyStore{} } @@ -4194,6 +4321,34 @@ func (m *ProxyStore) GetConfigData() *ConfigData { return nil } +func (m *ProxyStore) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + +func (m *ProxyStore) GetGlobalSysVarGeneration() string { + if m != nil { + return m.GlobalSysVarGeneration + } + return "" +} + +func (m *ProxyStore) GetState() NodeState { + if m != nil { + return m.State + } + return NormalState +} + +func (m *ProxyStore) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + type ProxyState struct { Stores map[string]ProxyStore `protobuf:"bytes,1,rep,name=Stores,proto3" json:"Stores" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -4242,12 +4397,15 @@ func (m *ProxyState) GetStores() map[string]ProxyStore { } type ProxyHeartbeat struct { - UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` - ListenAddress string `protobuf:"bytes,2,opt,name=ListenAddress,proto3" json:"ListenAddress,omitempty"` - ConfigData *ConfigData `protobuf:"bytes,3,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` + ListenAddress string `protobuf:"bytes,2,opt,name=ListenAddress,proto3" json:"ListenAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,3,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,4,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` + GlobalSysVarGeneration string `protobuf:"bytes,5,opt,name=GlobalSysVarGeneration,proto3" json:"GlobalSysVarGeneration,omitempty"` + ProtocolVersion int64 `protobuf:"varint,6,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ProxyHeartbeat) Reset() { *m = ProxyHeartbeat{} } @@ -4304,15 +4462,37 @@ func (m *ProxyHeartbeat) GetConfigData() *ConfigData { return nil } +func (m *ProxyHeartbeat) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + +func (m *ProxyHeartbeat) GetGlobalSysVarGeneration() string { + if m != nil { + return m.GlobalSysVarGeneration + } + return "" +} + +func (m *ProxyHeartbeat) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + type ClusterDetails struct { - TNStores []TNStore `protobuf:"bytes,1,rep,name=TNStores,proto3" json:"TNStores"` - CNStores []CNStore `protobuf:"bytes,2,rep,name=CNStores,proto3" json:"CNStores"` - LogStores []LogStore `protobuf:"bytes,3,rep,name=LogStores,proto3" json:"LogStores"` - ProxyStores []ProxyStore `protobuf:"bytes,4,rep,name=ProxyStores,proto3" json:"ProxyStores"` - DeletedStores []DeletedStore `protobuf:"bytes,5,rep,name=DeletedStores,proto3" json:"DeletedStores"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + TNStores []TNStore `protobuf:"bytes,1,rep,name=TNStores,proto3" json:"TNStores"` + CNStores []CNStore `protobuf:"bytes,2,rep,name=CNStores,proto3" json:"CNStores"` + LogStores []LogStore `protobuf:"bytes,3,rep,name=LogStores,proto3" json:"LogStores"` + ProxyStores []ProxyStore `protobuf:"bytes,4,rep,name=ProxyStores,proto3" json:"ProxyStores"` + DeletedStores []DeletedStore `protobuf:"bytes,5,rep,name=DeletedStores,proto3" json:"DeletedStores"` + GlobalSysVarCommitTS timestamp.Timestamp `protobuf:"bytes,6,opt,name=GlobalSysVarCommitTS,proto3" json:"GlobalSysVarCommitTS"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ClusterDetails) Reset() { *m = ClusterDetails{} } @@ -4383,6 +4563,13 @@ func (m *ClusterDetails) GetDeletedStores() []DeletedStore { return nil } +func (m *ClusterDetails) GetGlobalSysVarCommitTS() timestamp.Timestamp { + if m != nil { + return m.GlobalSysVarCommitTS + } + return timestamp.Timestamp{} +} + // ClusterInfo provides a global view of all shards in the cluster. It // describes the logical sharding of the system, rather than physical // distribution of all replicas that belong to those shards. @@ -4610,6 +4797,7 @@ type LogStoreInfo struct { ConfigData *ConfigData `protobuf:"bytes,7,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` Locality Locality `protobuf:"bytes,8,opt,name=Locality,proto3" json:"Locality"` CommandDeliverySupported bool `protobuf:"varint,9,opt,name=CommandDeliverySupported,proto3" json:"CommandDeliverySupported,omitempty"` + ProtocolVersion int64 `protobuf:"varint,10,opt,name=ProtocolVersion,proto3" json:"ProtocolVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -4711,6 +4899,13 @@ func (m *LogStoreInfo) GetCommandDeliverySupported() bool { return false } +func (m *LogStoreInfo) GetProtocolVersion() int64 { + if m != nil { + return m.ProtocolVersion + } + return 0 +} + type LogState struct { // Shards is keyed by ShardID, it contains details aggregated from all Log // stores. Each pb.LogShardInfo here contains data aggregated from @@ -5914,334 +6109,347 @@ func init() { func init() { proto.RegisterFile("logservice.proto", fileDescriptor_fd1040c5381ab5a7) } var fileDescriptor_fd1040c5381ab5a7 = []byte{ - // 5232 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7c, 0xdd, 0x6f, 0x1b, 0xd9, - 0x75, 0xb8, 0x87, 0xa4, 0x24, 0xf2, 0x50, 0x92, 0x47, 0x57, 0x92, 0x4d, 0xcb, 0xb6, 0xac, 0x30, - 0xfe, 0xed, 0x7a, 0x95, 0x5d, 0xfa, 0x57, 0x7b, 0x77, 0xb1, 0x49, 0xbd, 0xde, 0x50, 0x24, 0x6d, - 0xd1, 0xa6, 0x28, 0xed, 0xe5, 0x68, 0x37, 0x09, 0xb0, 0x10, 0x46, 0xe4, 0x35, 0xcd, 0x88, 0xe4, - 0x30, 0x33, 0x43, 0xaf, 0xdd, 0xbe, 0x15, 0x6d, 0x1f, 0x5a, 0xa0, 0x68, 0x8b, 0xa2, 0x08, 0x8a, - 0xb6, 0x69, 0x1f, 0x0b, 0x14, 0x08, 0x02, 0xb4, 0xaf, 0x41, 0xfa, 0x96, 0xa7, 0x62, 0xd1, 0x3f, - 0x20, 0x4d, 0xb6, 0x2f, 0x05, 0xf2, 0xde, 0x3c, 0x15, 0x28, 0xee, 0xd7, 0xcc, 0xbd, 0x33, 0x43, - 0x52, 0x92, 0xbd, 0x69, 0x52, 0xe4, 0x49, 0xbc, 0xe7, 0xe3, 0x7e, 0x9c, 0x7b, 0xbe, 0xee, 0xb9, - 0x77, 0x04, 0x66, 0xdf, 0xe9, 0x7a, 0xc4, 0x7d, 0xd6, 0x6b, 0x93, 0xd2, 0xc8, 0x75, 0x7c, 0x07, - 0x41, 0x08, 0xd9, 0x78, 0xab, 0xdb, 0xf3, 0x9f, 0x8e, 0x8f, 0x4b, 0x6d, 0x67, 0x70, 0xbb, 0xeb, - 0x74, 0x9d, 0xdb, 0x8c, 0xe4, 0x78, 0xfc, 0x84, 0xb5, 0x58, 0x83, 0xfd, 0xe2, 0xac, 0x1b, 0x37, - 0xba, 0x8e, 0xd3, 0xed, 0x93, 0x90, 0xca, 0xef, 0x0d, 0x88, 0xe7, 0xdb, 0x83, 0x91, 0x20, 0x58, - 0x1e, 0x10, 0xdf, 0xee, 0xd8, 0xbe, 0xcd, 0xdb, 0xc5, 0x1f, 0xce, 0xc1, 0x42, 0xa5, 0xd9, 0xf2, - 0x1d, 0x97, 0x20, 0x04, 0x99, 0xc3, 0xc3, 0x7a, 0xb5, 0x60, 0x6c, 0x19, 0xb7, 0x72, 0x98, 0xfd, - 0x46, 0xaf, 0xc1, 0x72, 0x8b, 0x4f, 0xa5, 0xdc, 0xe9, 0xb8, 0xc4, 0xf3, 0x0a, 0x29, 0x86, 0x8d, - 0x40, 0xd1, 0x26, 0x40, 0xeb, 0xc3, 0x86, 0xa4, 0x49, 0x33, 0x1a, 0x05, 0x82, 0x4a, 0x80, 0x1a, - 0x4e, 0xfb, 0x24, 0xd2, 0x57, 0x86, 0xd1, 0x25, 0x60, 0xd0, 0x4d, 0xc8, 0x60, 0xa7, 0x4f, 0x0a, - 0xf3, 0x5b, 0xc6, 0xad, 0xe5, 0x3b, 0x66, 0x29, 0x98, 0x76, 0xa5, 0x49, 0xe1, 0x98, 0x61, 0xe9, - 0x8c, 0xad, 0x5e, 0xfb, 0xa4, 0xb0, 0xb0, 0x65, 0xdc, 0xca, 0x60, 0xf6, 0x1b, 0x7d, 0x05, 0xe6, - 0x5a, 0xbe, 0xed, 0x93, 0x42, 0x96, 0xb1, 0xae, 0x97, 0x14, 0xf9, 0x36, 0x9d, 0x0e, 0x61, 0x48, - 0xcc, 0x69, 0xd0, 0xfb, 0x30, 0xdf, 0xb0, 0x8f, 0x49, 0xdf, 0x2b, 0xe4, 0xb6, 0xd2, 0xb7, 0xf2, - 0x77, 0x6e, 0xa8, 0xd4, 0x42, 0x2e, 0x25, 0x4e, 0x51, 0x1b, 0xfa, 0xee, 0x8b, 0x9d, 0xcc, 0x8f, - 0x7f, 0x72, 0xe3, 0x02, 0x16, 0x4c, 0xe8, 0xb7, 0x20, 0xf7, 0xb1, 0xe3, 0x9e, 0xf0, 0xf1, 0x80, - 0x8d, 0xb7, 0x1a, 0x4e, 0x35, 0x40, 0xe1, 0x90, 0x0a, 0x15, 0x61, 0xf1, 0xc3, 0x31, 0x71, 0x5f, - 0x48, 0x11, 0xe4, 0x99, 0x08, 0x34, 0x18, 0x7a, 0x17, 0xa0, 0xe2, 0x0c, 0x9f, 0xf4, 0xba, 0x55, - 0xdb, 0xb7, 0x0b, 0x8b, 0x5b, 0xc6, 0xad, 0xfc, 0x9d, 0x4b, 0xda, 0xcc, 0x02, 0x2c, 0x56, 0x28, - 0xd1, 0xbb, 0x90, 0xc5, 0xc4, 0x73, 0xc6, 0x6e, 0x9b, 0x14, 0x96, 0x18, 0xd7, 0x9a, 0xca, 0x25, - 0x71, 0x62, 0x11, 0x01, 0x2d, 0xba, 0x04, 0xf3, 0x87, 0x23, 0xab, 0x37, 0x20, 0x85, 0xe5, 0x2d, - 0xe3, 0x56, 0x1a, 0x8b, 0x16, 0xfa, 0xff, 0xb0, 0xda, 0x7a, 0x6a, 0xbb, 0x9d, 0xc8, 0xae, 0x5d, - 0x64, 0x53, 0x4e, 0x42, 0xa1, 0x0d, 0xc8, 0x56, 0x9c, 0xc1, 0xa0, 0xe7, 0xd7, 0xab, 0x05, 0x93, - 0x91, 0x05, 0xed, 0x8d, 0x26, 0xe4, 0x15, 0x49, 0x22, 0x13, 0xd2, 0x27, 0xe4, 0x85, 0x50, 0x36, - 0xfa, 0x13, 0xbd, 0x01, 0x73, 0xcf, 0xec, 0xfe, 0x98, 0x30, 0x15, 0xcb, 0xab, 0x92, 0x64, 0x7c, - 0x8d, 0x9e, 0xe7, 0x63, 0x4e, 0xf1, 0xb5, 0xd4, 0x7b, 0xc6, 0xa3, 0x4c, 0x76, 0xce, 0x9c, 0x2f, - 0xfe, 0x22, 0x0d, 0x0b, 0xd6, 0x2b, 0x50, 0x60, 0xa9, 0x4a, 0xe9, 0x24, 0x55, 0xca, 0x9c, 0x42, - 0x95, 0xde, 0x81, 0x79, 0x26, 0x11, 0xaf, 0x30, 0xc7, 0x54, 0xe9, 0xb2, 0x4a, 0x6d, 0x35, 0x19, - 0xae, 0x3e, 0x7c, 0xe2, 0x48, 0x15, 0xe2, 0xc4, 0xe8, 0x0e, 0xac, 0x35, 0x9c, 0xae, 0x6f, 0xf7, - 0xfa, 0x74, 0x42, 0xc4, 0x95, 0xb3, 0x9c, 0x67, 0xb3, 0x4c, 0xc4, 0x4d, 0x30, 0xa6, 0x85, 0x89, - 0xc6, 0xa4, 0xeb, 0x53, 0xee, 0xd4, 0xfa, 0x14, 0xd5, 0x55, 0x48, 0xd0, 0xd5, 0x09, 0x3a, 0x92, - 0x9f, 0xac, 0x23, 0x5f, 0x87, 0xab, 0xe5, 0xb1, 0xef, 0xd4, 0x87, 0x6d, 0xb7, 0x36, 0x72, 0xda, - 0x4f, 0x1f, 0x90, 0x61, 0x9b, 0xb4, 0xc6, 0xa3, 0x91, 0xe3, 0xfa, 0xa4, 0xc3, 0xd4, 0x3d, 0x8b, - 0xa7, 0x91, 0x3c, 0xca, 0x64, 0xb3, 0x66, 0xae, 0xf8, 0xcf, 0x29, 0xc8, 0x36, 0x9c, 0xee, 0xaf, - 0xc0, 0xd6, 0xdf, 0xa3, 0x76, 0x37, 0xea, 0xf7, 0xda, 0xb6, 0xdc, 0xfc, 0x0d, 0x95, 0xbe, 0xe1, - 0x74, 0x05, 0x5a, 0xd9, 0xff, 0x80, 0x23, 0xb2, 0x3b, 0xf3, 0x67, 0xb1, 0xf6, 0x86, 0xd3, 0xb6, - 0xfb, 0x3d, 0xff, 0x05, 0xdb, 0xfb, 0x88, 0xb5, 0x4b, 0x9c, 0x1c, 0x4f, 0xb6, 0x8b, 0x7f, 0x91, - 0x86, 0x45, 0x2a, 0x37, 0xa9, 0x90, 0xa8, 0x00, 0x0b, 0xbc, 0xc1, 0xc5, 0x97, 0xc1, 0xb2, 0x89, - 0x76, 0x94, 0x85, 0xa5, 0xd8, 0xc2, 0x5e, 0x8b, 0x2c, 0x2c, 0xe8, 0xa5, 0x24, 0x09, 0x99, 0x75, - 0x2b, 0xcb, 0x5b, 0x83, 0x39, 0xb6, 0x87, 0x42, 0xbc, 0xbc, 0x41, 0x1d, 0x45, 0x83, 0xd8, 0x1d, - 0xe2, 0xd6, 0xab, 0x4c, 0xc4, 0x19, 0x1c, 0xb4, 0xd9, 0x7e, 0x10, 0x77, 0x50, 0x98, 0x13, 0xfb, - 0x41, 0xdc, 0x01, 0xfa, 0x04, 0x56, 0x9a, 0xce, 0xf0, 0x23, 0xc7, 0xef, 0x0d, 0xbb, 0xc1, 0x94, - 0xe6, 0xd9, 0x94, 0x6e, 0x4f, 0x9c, 0x52, 0x8c, 0x83, 0xcf, 0x2d, 0xde, 0xd3, 0xc6, 0x6f, 0xc3, - 0x92, 0x46, 0xa3, 0x7a, 0xa7, 0x0c, 0xf7, 0x4e, 0x6b, 0xaa, 0x77, 0xca, 0x29, 0x8e, 0x68, 0xa3, - 0x0a, 0x97, 0x92, 0x47, 0x3a, 0x4b, 0x2f, 0xc5, 0xef, 0x1a, 0xb0, 0xac, 0x6b, 0x0a, 0x7a, 0xa0, - 0x6f, 0x14, 0xeb, 0x27, 0x7f, 0xa7, 0x30, 0x69, 0xbd, 0x3b, 0x59, 0xba, 0xd3, 0x9f, 0xfd, 0xe4, - 0x86, 0x81, 0xf5, 0x0d, 0xbe, 0x06, 0x39, 0xd9, 0x6d, 0x95, 0x0d, 0x9c, 0xc1, 0x21, 0x00, 0x6d, - 0x41, 0xbe, 0xee, 0x05, 0x0b, 0x60, 0xdb, 0x94, 0xc5, 0x2a, 0xa8, 0xf8, 0x47, 0x46, 0x18, 0x58, - 0x98, 0x8b, 0x3f, 0x38, 0xb4, 0x1c, 0xdf, 0xee, 0x8b, 0x85, 0x05, 0x6d, 0xea, 0x30, 0x2a, 0x07, - 0x87, 0xe5, 0x67, 0x76, 0xaf, 0x6f, 0x1f, 0xf7, 0xf9, 0x22, 0x0d, 0xac, 0xc1, 0x28, 0xff, 0x1e, - 0x19, 0x70, 0x7e, 0xae, 0x12, 0x41, 0x9b, 0xf2, 0xef, 0x91, 0x41, 0xc8, 0xcf, 0x35, 0x43, 0x83, - 0x15, 0x7f, 0x34, 0x07, 0xa6, 0x88, 0xcc, 0xbb, 0xc4, 0x76, 0xfd, 0x63, 0x62, 0xfb, 0xbf, 0x86, - 0xa9, 0x4b, 0x09, 0x90, 0x65, 0x7b, 0x92, 0xb7, 0xe2, 0x12, 0x9b, 0x3a, 0xbf, 0x05, 0x26, 0xfc, - 0x04, 0x4c, 0xcc, 0x17, 0x67, 0x13, 0x7c, 0xf1, 0x4d, 0x58, 0xaa, 0x0f, 0x7b, 0x7e, 0x98, 0x92, - 0xe4, 0x18, 0x91, 0x0e, 0xa4, 0x54, 0x0f, 0x1d, 0xcf, 0xeb, 0x8d, 0x74, 0xb7, 0xae, 0x03, 0xe9, - 0x78, 0x1c, 0xf0, 0xc8, 0xe9, 0x0d, 0x49, 0x87, 0x39, 0xf4, 0x2c, 0xd6, 0x60, 0xbf, 0xf4, 0x3c, - 0x65, 0x42, 0xac, 0x59, 0x3e, 0x5d, 0x3e, 0x72, 0x51, 0xcf, 0x47, 0x68, 0x6f, 0xe5, 0xf6, 0x09, - 0xe9, 0x50, 0x80, 0x3d, 0xec, 0xec, 0xd8, 0x7e, 0xfb, 0xa9, 0x48, 0x5b, 0x32, 0x38, 0x09, 0x45, - 0x23, 0x97, 0x80, 0x54, 0x49, 0xbf, 0xf7, 0x8c, 0x4a, 0xbe, 0x7d, 0x12, 0x46, 0xae, 0x15, 0x1e, - 0xb9, 0xa6, 0x90, 0x88, 0x9c, 0xc5, 0x82, 0xc5, 0x4a, 0xb3, 0xdc, 0xef, 0x3b, 0x6d, 0xdb, 0x27, - 0xf5, 0x6a, 0x42, 0x2a, 0xb4, 0x06, 0x73, 0x6c, 0x50, 0x61, 0xad, 0xbc, 0xc1, 0xed, 0xf8, 0x3b, - 0x63, 0xe2, 0xd1, 0xe5, 0x70, 0x45, 0x0d, 0x01, 0xc5, 0x1f, 0xa4, 0x61, 0x45, 0xc6, 0xc3, 0xe9, - 0x96, 0xb1, 0x05, 0x79, 0x6c, 0x3f, 0xf1, 0x75, 0xb3, 0x50, 0x41, 0x09, 0xb6, 0x93, 0x4e, 0xb4, - 0x9d, 0x98, 0x2e, 0x65, 0x92, 0x74, 0xe9, 0xe5, 0xe2, 0x63, 0xb2, 0xa5, 0xcc, 0x4f, 0xb4, 0x14, - 0x5d, 0x2b, 0x17, 0xce, 0x15, 0x4f, 0xb3, 0xa7, 0x8f, 0xa7, 0xe8, 0x6b, 0x50, 0x88, 0x6c, 0x79, - 0xa8, 0x12, 0x39, 0x36, 0xcb, 0x89, 0xf8, 0x62, 0x0d, 0xf2, 0x4a, 0x6a, 0x38, 0x25, 0x12, 0x4f, - 0x75, 0xe1, 0xc5, 0x3f, 0x98, 0x03, 0xd3, 0x7a, 0x95, 0x3e, 0x31, 0x4c, 0x66, 0xd3, 0x67, 0x49, - 0x66, 0x93, 0xb7, 0x2a, 0x33, 0x71, 0xab, 0x26, 0x25, 0xbf, 0x73, 0x67, 0x4e, 0x7e, 0xe7, 0x4f, - 0x99, 0xfc, 0x66, 0xcf, 0x9d, 0xfc, 0xe6, 0x4e, 0x9f, 0xfc, 0xc2, 0x64, 0x87, 0x44, 0x4d, 0x8f, - 0x8c, 0xfa, 0xf6, 0x0b, 0xd2, 0x69, 0x78, 0x43, 0xe6, 0x55, 0x33, 0x58, 0x05, 0xbd, 0x7c, 0x7a, - 0x3c, 0xc9, 0xb1, 0x2d, 0x9d, 0xdb, 0xb1, 0x2d, 0x9f, 0xc6, 0xb1, 0x2d, 0x98, 0xd9, 0xe2, 0xf7, - 0xe7, 0x20, 0x8b, 0x5b, 0x7b, 0x3c, 0xce, 0x98, 0x90, 0xb6, 0x3c, 0x47, 0x26, 0x3f, 0x96, 0xe7, - 0x50, 0xaf, 0x56, 0x1f, 0x76, 0xc8, 0x73, 0xe9, 0xd5, 0x58, 0x83, 0xfa, 0x90, 0x06, 0xb1, 0x3d, - 0xb2, 0xeb, 0xf4, 0x79, 0x3e, 0xc8, 0xb3, 0x02, 0x1d, 0x48, 0xb7, 0xc3, 0x72, 0xc7, 0x43, 0xea, - 0x31, 0x99, 0xe4, 0x44, 0x6a, 0xa0, 0xc2, 0xd0, 0x23, 0x58, 0xe4, 0x4c, 0x3d, 0xcf, 0x77, 0xdc, - 0x17, 0xc2, 0xd7, 0x68, 0x29, 0xab, 0x9c, 0x5d, 0x49, 0x25, 0xe4, 0x69, 0xa1, 0xc6, 0xcb, 0x37, - 0xea, 0x3b, 0xe3, 0x9e, 0xcb, 0x87, 0x9b, 0x97, 0x1b, 0x15, 0x80, 0xd0, 0x9b, 0xb0, 0xf2, 0x71, - 0xb9, 0x81, 0x49, 0xdb, 0xa1, 0xd2, 0xa8, 0xf6, 0xba, 0xc4, 0xf3, 0xc5, 0x21, 0x2c, 0x8e, 0x40, - 0x6f, 0xc3, 0xba, 0x02, 0x64, 0x23, 0x56, 0x9c, 0xf1, 0xd0, 0x67, 0x1a, 0x99, 0xc1, 0xc9, 0x48, - 0xba, 0x31, 0x0a, 0xa2, 0xe2, 0x0c, 0x46, 0x7d, 0xe2, 0x93, 0x0e, 0xa5, 0xe8, 0x11, 0xae, 0x93, - 0x19, 0x3c, 0x8d, 0x84, 0x9a, 0x8b, 0x82, 0xde, 0xb1, 0x3d, 0x42, 0x97, 0x03, 0x8c, 0x31, 0x01, - 0x13, 0xa1, 0x6f, 0xd8, 0x9e, 0x1f, 0xea, 0x69, 0x02, 0x06, 0x3d, 0x82, 0x2d, 0x05, 0xba, 0xef, - 0xf6, 0xba, 0xbd, 0xa1, 0xdd, 0xd7, 0x37, 0x74, 0x91, 0x71, 0xcf, 0xa4, 0xa3, 0x8a, 0x9b, 0xb0, - 0x14, 0xa6, 0xb8, 0x59, 0x9c, 0x84, 0xda, 0xf8, 0x00, 0x56, 0x62, 0x1b, 0x39, 0x2b, 0xeb, 0xce, - 0xa8, 0x59, 0xf7, 0x27, 0x90, 0x63, 0xe1, 0xa7, 0xed, 0xb8, 0x1d, 0xca, 0x48, 0x17, 0x2b, 0x18, - 0xe9, 0xea, 0xb6, 0x21, 0x63, 0xbd, 0x18, 0x71, 0xbe, 0x65, 0xdd, 0x6d, 0x70, 0x1e, 0x8a, 0xc5, - 0x8c, 0x86, 0xfa, 0x5b, 0xe6, 0x62, 0xa8, 0xfa, 0x2e, 0x62, 0xf6, 0xbb, 0xf8, 0xd3, 0x14, 0x00, - 0xeb, 0x9f, 0x05, 0x69, 0x4a, 0xd2, 0xb4, 0x07, 0x44, 0xba, 0x64, 0xfa, 0x5b, 0xf5, 0xf9, 0x29, - 0xdd, 0xe7, 0x8b, 0xe9, 0xa4, 0xc3, 0xe9, 0x14, 0x60, 0x61, 0xcf, 0x7e, 0xde, 0xea, 0xfd, 0x8e, - 0x4c, 0x8d, 0x65, 0x93, 0xc6, 0x07, 0xe9, 0x96, 0xab, 0xe2, 0xe0, 0x14, 0x02, 0xd8, 0x89, 0xaa, - 0x59, 0xaf, 0x0a, 0x2d, 0x66, 0xbf, 0xd1, 0xdb, 0x90, 0xb2, 0x5a, 0x22, 0x3c, 0x6e, 0x94, 0x78, - 0xdd, 0xb0, 0x24, 0xeb, 0x86, 0x25, 0x4b, 0xd6, 0x0d, 0xf9, 0xa1, 0xe2, 0x4f, 0xff, 0xfd, 0x86, - 0x81, 0x53, 0x56, 0x0b, 0x3d, 0x80, 0xcd, 0xfa, 0xb0, 0xdd, 0x1f, 0x77, 0x48, 0xed, 0xf9, 0x88, - 0x5a, 0x82, 0x08, 0x42, 0xc2, 0xbf, 0x11, 0x9e, 0x98, 0x66, 0xf1, 0x0c, 0x2a, 0xb4, 0x0b, 0x37, - 0x6a, 0xcf, 0x19, 0xc5, 0xae, 0xed, 0x76, 0xaa, 0xce, 0xa7, 0xc3, 0x58, 0x47, 0x3c, 0x76, 0xce, - 0x22, 0x2b, 0x16, 0x01, 0x2c, 0xcf, 0x91, 0x12, 0x5e, 0x83, 0x39, 0x6e, 0x56, 0x7c, 0x13, 0x79, - 0xa3, 0xf8, 0x73, 0x83, 0x66, 0x5c, 0x2c, 0x3e, 0xb2, 0x52, 0x52, 0x62, 0x6c, 0xbc, 0x0b, 0xb9, - 0xfd, 0x11, 0x71, 0x6d, 0xbf, 0xe7, 0x0c, 0xc5, 0x86, 0xaf, 0xeb, 0xe5, 0x40, 0xc6, 0xbb, 0x3f, - 0xc2, 0x21, 0x1d, 0xda, 0x09, 0x0a, 0x88, 0x3c, 0x50, 0xde, 0x4c, 0x28, 0x20, 0x32, 0x82, 0xc9, - 0x55, 0xc4, 0x57, 0x5d, 0x18, 0x2b, 0x36, 0x20, 0x5f, 0x69, 0x86, 0xf9, 0x7e, 0xd2, 0x5a, 0xdf, - 0x90, 0xe5, 0x8d, 0xd4, 0xe4, 0xa2, 0x25, 0xa7, 0x28, 0xfe, 0x4c, 0xc8, 0xce, 0xf6, 0xa7, 0xc8, - 0xee, 0xf4, 0xfd, 0xcd, 0x96, 0x98, 0x1c, 0xe8, 0x97, 0x28, 0xb1, 0x3f, 0xcb, 0xc2, 0x82, 0xd4, - 0x20, 0x2d, 0xc9, 0x36, 0x64, 0xa6, 0x25, 0x00, 0xa8, 0x04, 0xf3, 0x7b, 0xc4, 0x7f, 0xea, 0x74, - 0x92, 0x5c, 0x02, 0xc7, 0x30, 0x97, 0x20, 0xa8, 0xd0, 0x3d, 0xd5, 0xfe, 0x99, 0x29, 0x47, 0xb2, - 0x8f, 0x10, 0x2b, 0xd6, 0xa8, 0xfa, 0x8b, 0x32, 0x2b, 0x00, 0x04, 0x29, 0x1d, 0x33, 0xfa, 0xfc, - 0x9d, 0xeb, 0xd1, 0x02, 0x80, 0x96, 0xf7, 0x61, 0x8d, 0x05, 0xdd, 0xa7, 0xca, 0x10, 0xf6, 0x30, - 0xc7, 0x7a, 0xb8, 0x96, 0xa0, 0xa5, 0x61, 0x07, 0x2a, 0x03, 0xe5, 0xb7, 0x14, 0xfe, 0xf9, 0x38, - 0xbf, 0x15, 0xe3, 0x57, 0x18, 0x68, 0xfa, 0x15, 0x9a, 0x67, 0x52, 0x36, 0x1e, 0x62, 0xb1, 0x6a, - 0xc8, 0xf7, 0xf4, 0x33, 0x92, 0x48, 0xdc, 0x0a, 0xfa, 0xc4, 0x43, 0x3c, 0xd6, 0x4f, 0x54, 0xf7, - 0x74, 0x7b, 0x17, 0x35, 0xcf, 0xc2, 0x24, 0xe3, 0xc4, 0xba, 0x77, 0xf8, 0xaa, 0x66, 0x40, 0x2c, - 0x58, 0x46, 0x52, 0x60, 0x05, 0x8d, 0x35, 0x63, 0xbb, 0xa7, 0x1b, 0x0b, 0x0b, 0x9c, 0x09, 0x03, - 0x4b, 0x3c, 0xd6, 0x4d, 0xeb, 0x03, 0x58, 0xaa, 0x12, 0x1a, 0xd8, 0xc4, 0x74, 0xc4, 0x99, 0xfa, - 0x8a, 0xca, 0xae, 0x11, 0x60, 0x9d, 0x1e, 0xed, 0xc0, 0xf2, 0x81, 0xeb, 0x3c, 0x7f, 0x11, 0x6e, - 0xd8, 0x92, 0x70, 0xf0, 0x4a, 0x0f, 0x3a, 0x05, 0x8e, 0x70, 0xd0, 0x28, 0x1c, 0x2d, 0x67, 0x35, - 0xc7, 0x03, 0x96, 0x04, 0x66, 0x70, 0x12, 0x0a, 0xed, 0x28, 0xc5, 0xb9, 0xe0, 0x08, 0x75, 0x71, - 0xf2, 0x11, 0x0a, 0xc7, 0xc9, 0x99, 0xcc, 0x9f, 0x92, 0xf6, 0xc9, 0x2e, 0xb1, 0xfb, 0xfe, 0x53, - 0x76, 0x0a, 0x8f, 0xca, 0x3c, 0x44, 0x63, 0x95, 0x16, 0x59, 0xb0, 0xd6, 0x6a, 0x3f, 0x25, 0x9d, - 0x71, 0x9f, 0x88, 0x14, 0x95, 0x25, 0xe9, 0xec, 0x3c, 0x9e, 0xbf, 0xb3, 0xa5, 0xf6, 0x91, 0x44, - 0x87, 0x13, 0xb9, 0x8b, 0x0e, 0xe4, 0x99, 0x25, 0x7a, 0x23, 0x67, 0xe8, 0x91, 0x29, 0x47, 0x33, - 0x11, 0xa6, 0x53, 0x5a, 0x98, 0x96, 0x89, 0x13, 0x0f, 0xde, 0xb2, 0x39, 0xad, 0xec, 0x59, 0x2c, - 0x01, 0x52, 0xf4, 0x59, 0x19, 0xf7, 0x41, 0xcf, 0x55, 0x9c, 0x91, 0x6c, 0x16, 0xff, 0x2b, 0xc3, - 0xca, 0x28, 0x9c, 0xec, 0xd5, 0x7a, 0xad, 0x6b, 0x90, 0xab, 0xb9, 0xae, 0xe3, 0x56, 0x9c, 0x0e, - 0x61, 0x4b, 0x58, 0xc2, 0x21, 0x80, 0xa6, 0xe2, 0xac, 0xb1, 0x47, 0x3c, 0xcf, 0xee, 0x12, 0x71, - 0xe6, 0xd7, 0x60, 0x68, 0x13, 0xa0, 0xee, 0xed, 0x96, 0x1f, 0x13, 0x32, 0x22, 0x2e, 0xf3, 0x3a, - 0x59, 0xac, 0x40, 0xd0, 0x07, 0x9a, 0x74, 0x85, 0x5b, 0xb9, 0x1c, 0x73, 0x8c, 0x1c, 0x2d, 0x3c, - 0xa3, 0xb6, 0x1f, 0xd4, 0xd0, 0x94, 0x43, 0x8c, 0xf0, 0x2c, 0xba, 0xa1, 0x29, 0x78, 0xac, 0x51, - 0x53, 0x6d, 0x63, 0xbe, 0x46, 0x0c, 0x9f, 0x8d, 0x0f, 0xaf, 0xa0, 0xb1, 0x4a, 0x4b, 0x4d, 0xac, - 0xd2, 0x1f, 0x7b, 0x3e, 0x71, 0xab, 0x84, 0x9e, 0x4e, 0x3d, 0xe1, 0x5c, 0x34, 0x13, 0xd3, 0x29, - 0x70, 0x84, 0x03, 0xdd, 0x87, 0x5c, 0x58, 0xd5, 0x85, 0x04, 0x35, 0x95, 0x48, 0xae, 0xa0, 0xc4, - 0x1b, 0xf7, 0x7d, 0x1c, 0xb2, 0xa0, 0xfb, 0x00, 0x8a, 0x6b, 0xe4, 0x3e, 0x66, 0x53, 0xed, 0x20, - 0xae, 0x48, 0x18, 0x22, 0xee, 0x91, 0x1a, 0x10, 0x71, 0xb9, 0x87, 0x5b, 0x4c, 0x10, 0x9e, 0x82, - 0xc7, 0x1a, 0x75, 0xf1, 0x11, 0xab, 0x33, 0xf1, 0xfc, 0x37, 0x10, 0xcb, 0x3b, 0x34, 0x82, 0x52, - 0x88, 0x57, 0x30, 0x58, 0x5c, 0x5f, 0x8f, 0x6d, 0x26, 0xc5, 0x8a, 0xad, 0x94, 0xb4, 0xc5, 0x2f, - 0x6b, 0x1b, 0x41, 0xd3, 0xb7, 0x8f, 0x58, 0xdc, 0x16, 0xe9, 0x1b, 0x6b, 0x14, 0x1f, 0xc2, 0x92, - 0x65, 0x7b, 0x27, 0x96, 0x7d, 0xdc, 0x27, 0x87, 0x1e, 0x71, 0xa9, 0x19, 0xd1, 0xbf, 0xc3, 0x30, - 0x97, 0x0e, 0xda, 0x14, 0x77, 0x60, 0x7b, 0xde, 0xa7, 0x8e, 0xdb, 0x11, 0xc5, 0x8d, 0xa0, 0x5d, - 0xfc, 0x63, 0x83, 0xce, 0x92, 0xf9, 0xad, 0xc4, 0x34, 0x66, 0x72, 0x2e, 0xae, 0xd5, 0x5f, 0xd2, - 0xd1, 0x12, 0x7a, 0x70, 0xc7, 0x91, 0x51, 0xef, 0x38, 0x36, 0x59, 0xec, 0xd7, 0x93, 0x72, 0x05, - 0x52, 0xfc, 0xab, 0x14, 0xd5, 0xe1, 0xe1, 0x93, 0x5e, 0xb7, 0xf2, 0xd4, 0x1e, 0x76, 0x09, 0xba, - 0x1b, 0xcc, 0x4e, 0x94, 0xfa, 0x57, 0xf5, 0x03, 0x07, 0x43, 0x85, 0x12, 0xe4, 0xeb, 0xb8, 0x07, - 0xc0, 0xd9, 0x95, 0x83, 0xca, 0xb5, 0x78, 0x7d, 0x23, 0xa4, 0xc1, 0x0a, 0x3d, 0xb2, 0x60, 0xb9, - 0x3e, 0xec, 0xf9, 0x3d, 0xbb, 0xbf, 0x47, 0x06, 0xc7, 0xc4, 0x95, 0x59, 0xd9, 0x9b, 0x93, 0x7a, - 0x28, 0xe9, 0xe4, 0xfc, 0xe8, 0x1c, 0xe9, 0x63, 0xa3, 0x0c, 0xab, 0x09, 0x64, 0x67, 0xba, 0x0e, - 0x79, 0x03, 0x96, 0x5a, 0x4f, 0xc7, 0x7e, 0xc7, 0xf9, 0x74, 0xc8, 0x43, 0x1b, 0xdd, 0x1b, 0xfa, - 0x23, 0xd8, 0x32, 0xd9, 0x2c, 0xfe, 0xc3, 0x1c, 0x5c, 0x8c, 0xb8, 0xf0, 0xc4, 0xdd, 0xbd, 0x09, - 0x4b, 0x3b, 0x8e, 0xe3, 0x7b, 0xbe, 0x6b, 0x8f, 0x46, 0xbd, 0x61, 0x97, 0x0d, 0x9a, 0xc5, 0x3a, - 0x90, 0xba, 0x06, 0x51, 0xb3, 0x61, 0x02, 0x4d, 0x33, 0x81, 0x6a, 0xae, 0x41, 0x41, 0x63, 0x95, - 0x96, 0xfb, 0xa4, 0x50, 0x54, 0x22, 0x5d, 0x2b, 0x4c, 0x12, 0x25, 0xd6, 0x77, 0xff, 0x83, 0xc8, - 0x8a, 0x45, 0xae, 0x76, 0x45, 0x77, 0x0c, 0x0a, 0x01, 0x8e, 0x48, 0xe8, 0x31, 0xac, 0xf0, 0xc2, - 0x9a, 0x52, 0x69, 0x13, 0x9e, 0x55, 0x4b, 0x19, 0x63, 0x44, 0x38, 0xce, 0x17, 0x4f, 0x45, 0x16, - 0xce, 0x98, 0x8a, 0x3c, 0x86, 0x95, 0x47, 0x4e, 0x6f, 0xc8, 0x2b, 0xc1, 0xc2, 0xff, 0x09, 0x47, - 0xab, 0xcd, 0x26, 0x46, 0x84, 0xe3, 0x7c, 0x68, 0x17, 0x4c, 0xde, 0x3b, 0xcb, 0x55, 0xf8, 0x84, - 0x72, 0xf1, 0x54, 0x34, 0x4a, 0x83, 0x63, 0x5c, 0x74, 0x7b, 0xcb, 0x9d, 0x8e, 0xb4, 0xc2, 0xa4, - 0xdc, 0x4e, 0x41, 0x63, 0x95, 0x96, 0x7a, 0xfe, 0x40, 0x55, 0x38, 0x77, 0x3e, 0xee, 0xf9, 0x75, - 0x0a, 0x1c, 0xe1, 0x28, 0x92, 0xe4, 0x5c, 0x25, 0x51, 0x5f, 0x23, 0x9a, 0x98, 0x3a, 0xbd, 0x26, - 0x16, 0xff, 0x32, 0x05, 0x97, 0x22, 0xe5, 0x3a, 0xcb, 0x76, 0xbb, 0xc4, 0x67, 0x17, 0x3b, 0x62, - 0x8b, 0xe8, 0x20, 0xdc, 0x5b, 0xe7, 0xb0, 0x06, 0x63, 0xc5, 0x36, 0x95, 0x26, 0xc5, 0x69, 0x54, - 0x18, 0xf5, 0xb3, 0xb5, 0xe7, 0xd4, 0x05, 0xf5, 0x7c, 0x71, 0x67, 0x18, 0xb4, 0x69, 0x0a, 0x29, - 0xfa, 0xb3, 0x7a, 0x03, 0xe2, 0x8c, 0x7d, 0xab, 0xd7, 0x3e, 0xf1, 0x84, 0x77, 0x4c, 0x42, 0x51, - 0x0e, 0x2b, 0x81, 0x83, 0x3b, 0xcd, 0x24, 0x14, 0x7a, 0x1b, 0xd6, 0x6b, 0xd4, 0x5d, 0xd8, 0x3e, - 0xa9, 0x8c, 0x5d, 0x97, 0x0c, 0x7d, 0x46, 0xe3, 0x89, 0x9b, 0x81, 0x64, 0x64, 0xf1, 0x76, 0x82, - 0x56, 0xf2, 0xa5, 0xf4, 0x3c, 0x76, 0xfd, 0xc9, 0xc5, 0x11, 0xb4, 0x8b, 0xfd, 0x04, 0xa3, 0x42, - 0x77, 0x21, 0x43, 0xe3, 0x8d, 0xf0, 0xd2, 0x9a, 0x4d, 0x68, 0x81, 0x4a, 0xf8, 0x6a, 0x46, 0xcc, - 0x84, 0x6a, 0x7b, 0x27, 0x55, 0xdb, 0xb7, 0x8f, 0x6d, 0x4f, 0xba, 0x3c, 0x0d, 0x46, 0xbd, 0x9e, - 0x6e, 0x45, 0x93, 0xbd, 0xde, 0x9b, 0x71, 0x93, 0x98, 0x42, 0xfd, 0xba, 0xa6, 0xf6, 0x93, 0xb3, - 0xd9, 0xe2, 0x2f, 0x8c, 0xa8, 0x96, 0x9f, 0xf7, 0x56, 0x02, 0x7d, 0x34, 0x21, 0xb6, 0x94, 0x26, - 0xdb, 0xcb, 0x69, 0xa2, 0x0b, 0xb5, 0x15, 0xba, 0x87, 0xe2, 0x5e, 0x81, 0xfd, 0x7e, 0x15, 0x11, - 0xe7, 0x47, 0x86, 0x9e, 0x52, 0x06, 0xef, 0x10, 0x0c, 0xe5, 0x1d, 0xc2, 0xfb, 0xfc, 0x42, 0xd1, - 0x1e, 0x76, 0xe4, 0x8b, 0x88, 0xab, 0x53, 0xce, 0x17, 0xf2, 0xae, 0x48, 0xb2, 0x50, 0x51, 0xca, - 0x72, 0xbc, 0x38, 0x19, 0xc8, 0x12, 0x7c, 0x05, 0x40, 0x50, 0x51, 0x83, 0xcb, 0xb0, 0xae, 0xaf, - 0x4f, 0xe9, 0xba, 0x5e, 0x95, 0xf5, 0x82, 0x90, 0xad, 0xf8, 0x2f, 0xf3, 0xf4, 0xe4, 0xca, 0xf7, - 0x9c, 0xe6, 0x89, 0xf2, 0x65, 0x8b, 0xa1, 0xbc, 0x6c, 0xf9, 0xbf, 0x75, 0x2d, 0x5e, 0x0e, 0xaa, - 0x49, 0x59, 0x26, 0xb2, 0x2f, 0x27, 0x1c, 0xf1, 0xd9, 0x5b, 0x90, 0x53, 0x3e, 0xe2, 0xcb, 0x9d, - 0xeb, 0x11, 0x1f, 0x24, 0x5f, 0xc6, 0xeb, 0x57, 0xa3, 0xf9, 0xd3, 0x5c, 0xb3, 0x2f, 0xce, 0xbc, - 0x66, 0x5f, 0x3a, 0xd7, 0x35, 0xfb, 0xf2, 0xb9, 0x9e, 0x03, 0x5e, 0x3c, 0xcd, 0x73, 0x40, 0xf3, - 0x74, 0xd7, 0xef, 0x2b, 0x91, 0xeb, 0xf7, 0x19, 0x77, 0x4e, 0x68, 0xe6, 0x9d, 0xd3, 0x17, 0xf4, - 0xa0, 0xf0, 0xaf, 0x0d, 0xfe, 0x22, 0x56, 0x3c, 0x0f, 0x15, 0x41, 0xc5, 0x48, 0x7e, 0x1e, 0x6a, - 0xfb, 0xa4, 0xc4, 0x29, 0x34, 0xcd, 0xe2, 0xa0, 0x0d, 0x0c, 0x79, 0x05, 0x99, 0x30, 0xc1, 0xb7, - 0xf4, 0x09, 0x5e, 0x9e, 0xa0, 0xbc, 0xaa, 0x97, 0xfa, 0xd7, 0x0c, 0xbb, 0x32, 0x7e, 0x25, 0x26, - 0xfe, 0x9b, 0x5b, 0xde, 0x5f, 0xe3, 0x5b, 0xde, 0x19, 0xf6, 0xb3, 0x74, 0xda, 0x3b, 0x5b, 0xaa, - 0xef, 0xd6, 0x69, 0xf4, 0xdd, 0xfa, 0x62, 0xf5, 0xdd, 0x4a, 0xd6, 0xf7, 0x3f, 0x37, 0x00, 0x94, - 0x0c, 0x27, 0x29, 0x4f, 0x96, 0x26, 0x90, 0x52, 0x4c, 0xe0, 0x26, 0x2c, 0x51, 0xf3, 0x26, 0x43, - 0x3d, 0x80, 0xe9, 0xc0, 0x88, 0xd6, 0x64, 0x4e, 0xab, 0x35, 0xc5, 0xbf, 0x0f, 0x27, 0x45, 0xc5, - 0xf6, 0xf5, 0x88, 0xd8, 0x8a, 0xb1, 0x6a, 0xeb, 0x2c, 0xc9, 0x7d, 0x38, 0x4b, 0x72, 0x6f, 0xea, - 0x92, 0xbb, 0x94, 0x30, 0x02, 0x3d, 0xef, 0x28, 0x82, 0xfb, 0x3d, 0x23, 0x5a, 0x0b, 0x9e, 0x74, - 0x28, 0xd6, 0x05, 0x95, 0x9a, 0x2d, 0xa8, 0xf4, 0xa9, 0x05, 0xf5, 0xc3, 0x54, 0xb4, 0x5a, 0x86, - 0xde, 0x81, 0xac, 0xd8, 0x6a, 0x29, 0xae, 0xd5, 0x04, 0x35, 0x90, 0x41, 0x49, 0x92, 0x52, 0xb6, - 0x8a, 0x64, 0x4b, 0xc5, 0xd9, 0x2a, 0x3a, 0x9b, 0x24, 0x45, 0xef, 0xb1, 0xfb, 0x5d, 0xc1, 0xc7, - 0xbd, 0xdc, 0x5a, 0xd2, 0xf5, 0x89, 0x60, 0x0c, 0x89, 0xd1, 0x7d, 0xc8, 0x87, 0x82, 0x95, 0x19, - 0xd9, 0x04, 0xb9, 0xcb, 0x02, 0xa5, 0xc2, 0x80, 0xaa, 0x32, 0x95, 0xef, 0x88, 0x1e, 0xf8, 0x6b, - 0x84, 0x42, 0xfc, 0xbc, 0xda, 0x51, 0xfb, 0xd0, 0x99, 0x8a, 0x7f, 0x68, 0x40, 0x5e, 0x08, 0x90, - 0xb9, 0xfb, 0xaf, 0x32, 0xe9, 0x71, 0xa7, 0x6d, 0x08, 0xa7, 0x1d, 0x44, 0x35, 0x81, 0xd1, 0x2a, - 0x6d, 0x01, 0x39, 0xba, 0xc7, 0x45, 0xc1, 0x79, 0x53, 0x62, 0x32, 0x61, 0x44, 0x94, 0x47, 0x5e, - 0x95, 0x39, 0x64, 0x28, 0xfe, 0x4d, 0x06, 0xd6, 0x45, 0x86, 0x2d, 0xcf, 0xe9, 0xe2, 0xa6, 0xe6, - 0x35, 0x58, 0x6e, 0x8e, 0x07, 0xfb, 0x4f, 0xc2, 0xce, 0x79, 0x2c, 0x8a, 0x40, 0xa9, 0xa6, 0x31, - 0x48, 0x30, 0x7f, 0x6e, 0xaf, 0x3a, 0x10, 0x6d, 0x83, 0x29, 0xf9, 0x82, 0x37, 0x63, 0x3c, 0x55, - 0x8e, 0xc1, 0x69, 0xa2, 0xd2, 0x24, 0xcf, 0xfd, 0xa0, 0x96, 0x2e, 0x5a, 0xc8, 0x82, 0x3c, 0xff, - 0xb5, 0xf3, 0xe2, 0x31, 0x91, 0xcf, 0x40, 0xee, 0xa8, 0x82, 0x4f, 0x5c, 0x49, 0x49, 0x61, 0xe2, - 0x27, 0x0f, 0xb5, 0x1b, 0xf4, 0x24, 0xe9, 0x96, 0x83, 0x3f, 0x41, 0x7e, 0xef, 0x14, 0x7d, 0x47, - 0x59, 0xa3, 0x6f, 0x91, 0x83, 0x9b, 0x10, 0x16, 0xfa, 0xba, 0xb2, 0x34, 0x23, 0x5e, 0x3c, 0xc8, - 0x14, 0x38, 0x8e, 0xd9, 0xb8, 0x0f, 0x66, 0x74, 0xe2, 0xc9, 0x2f, 0x0a, 0x93, 0x9f, 0x40, 0x68, - 0xcf, 0x97, 0xb5, 0xc9, 0xcd, 0xea, 0x45, 0x3b, 0x3d, 0xfd, 0xb7, 0x01, 0x57, 0x30, 0xf1, 0xf8, - 0x71, 0xf3, 0x63, 0xdb, 0x27, 0xee, 0xc0, 0x76, 0x4f, 0xa4, 0x8e, 0x84, 0x3b, 0x65, 0x68, 0x3b, - 0xf5, 0x0d, 0x7d, 0xa7, 0xb8, 0x56, 0xbe, 0x1b, 0xc9, 0x52, 0x93, 0xfb, 0x9c, 0xb1, 0x5b, 0xc9, - 0x52, 0x4c, 0x7f, 0x51, 0x52, 0x2c, 0xfe, 0xa3, 0x78, 0x55, 0x3f, 0x35, 0x31, 0xfb, 0xcd, 0xc3, - 0xcb, 0x5f, 0x95, 0x87, 0x97, 0xff, 0x24, 0x3f, 0x1e, 0xa1, 0xf1, 0xfb, 0x7e, 0x90, 0x07, 0x73, - 0x97, 0xba, 0x15, 0x8b, 0x10, 0x2c, 0x7a, 0x33, 0x12, 0x3d, 0x7a, 0x73, 0x9f, 0x75, 0x3f, 0x88, - 0xff, 0xa9, 0x69, 0xfc, 0x13, 0xa3, 0x7f, 0x0b, 0xf2, 0x4a, 0xe7, 0x09, 0x45, 0x8b, 0x92, 0x1e, - 0xfd, 0x27, 0x7e, 0x01, 0xa0, 0x9a, 0x75, 0x6b, 0x56, 0x4a, 0x31, 0xab, 0xd3, 0xa4, 0x6c, 0xec, - 0x3f, 0xb3, 0xfa, 0xcd, 0x51, 0xa2, 0x96, 0x7f, 0xa0, 0x85, 0xac, 0xc4, 0xb3, 0x4d, 0x88, 0x96, - 0xa1, 0x53, 0x0d, 0x72, 0x77, 0x83, 0x8c, 0x54, 0xa4, 0x1a, 0xab, 0x09, 0x79, 0xa8, 0xbc, 0x07, - 0x91, 0xb9, 0xeb, 0xbb, 0xe1, 0x86, 0x8a, 0x4c, 0x6e, 0x2d, 0x69, 0x1b, 0x42, 0x2d, 0x12, 0x9b, - 0x7f, 0x37, 0x38, 0xee, 0x89, 0x82, 0xfb, 0x6a, 0xc2, 0x21, 0x4f, 0x0e, 0x26, 0x0f, 0x86, 0xb7, - 0xe5, 0x7b, 0x17, 0x5e, 0xcd, 0xd0, 0x2a, 0x80, 0xf2, 0x8e, 0x53, 0x7b, 0xf5, 0xd2, 0x14, 0xb6, - 0x24, 0x8a, 0x38, 0xe2, 0xde, 0x6d, 0x81, 0x71, 0x6f, 0x46, 0xeb, 0x87, 0x3a, 0x15, 0x4e, 0xe0, - 0x44, 0xb5, 0xc8, 0x95, 0x98, 0x30, 0x9c, 0x99, 0xa5, 0xc8, 0xc8, 0x45, 0x9a, 0xf4, 0xcb, 0x1d, - 0xf1, 0x94, 0x50, 0xb4, 0xd0, 0x63, 0xdd, 0x2f, 0x03, 0x53, 0xeb, 0x37, 0x26, 0xdd, 0x0f, 0xce, - 0x70, 0xc5, 0xf7, 0xd4, 0x64, 0x59, 0xd4, 0xcc, 0x2f, 0x25, 0xa7, 0xc8, 0xb2, 0xa6, 0xa5, 0x24, - 0xd7, 0x6a, 0x15, 0x63, 0xf1, 0x6c, 0x1f, 0x0b, 0x24, 0x3d, 0x63, 0x58, 0x9a, 0xfc, 0x8c, 0x61, - 0x37, 0x29, 0xc0, 0x2f, 0xcf, 0x74, 0x48, 0x09, 0x21, 0xfc, 0x1e, 0x5c, 0x89, 0x87, 0x98, 0x03, - 0x32, 0xec, 0xf4, 0x86, 0x5d, 0x56, 0x54, 0xc9, 0xe2, 0xc9, 0x04, 0x68, 0x07, 0xae, 0x69, 0xe1, - 0x8e, 0x05, 0xc0, 0x87, 0x64, 0x28, 0xdf, 0xa6, 0xf1, 0x2f, 0x14, 0xa6, 0xd2, 0xa0, 0xfb, 0xb0, - 0x91, 0x30, 0x80, 0x4b, 0x46, 0xb6, 0x1b, 0x7c, 0xa9, 0x30, 0x85, 0x82, 0x9e, 0x2e, 0xe3, 0xd8, - 0xe0, 0x71, 0xa9, 0xac, 0xce, 0x4c, 0x21, 0x79, 0xe9, 0x80, 0xfa, 0x27, 0x06, 0x2c, 0xaa, 0xb9, - 0x70, 0xe2, 0xe9, 0xe5, 0x1a, 0xe4, 0xf8, 0xcd, 0x80, 0xbc, 0x20, 0xc9, 0xe1, 0x10, 0x80, 0x0a, - 0xb0, 0xa0, 0x47, 0x51, 0xd9, 0x54, 0x4a, 0x5c, 0x19, 0xad, 0xc4, 0xb5, 0x01, 0xd9, 0xaa, 0xf3, - 0xe9, 0x90, 0x61, 0xe6, 0x18, 0x26, 0x68, 0x17, 0x7f, 0x80, 0xc0, 0x94, 0xb6, 0x1d, 0x3c, 0x72, - 0x0e, 0x9e, 0x34, 0x1b, 0xea, 0x93, 0xe6, 0xa4, 0x13, 0x69, 0x98, 0x02, 0xa5, 0xb5, 0x14, 0x68, - 0x5f, 0x37, 0x35, 0x7e, 0xce, 0x78, 0x2b, 0xc9, 0xa1, 0x04, 0x6f, 0x97, 0xa7, 0x9b, 0x5b, 0xd2, - 0xe7, 0x73, 0xff, 0xeb, 0xfe, 0xaa, 0x03, 0x66, 0xa4, 0x80, 0x2d, 0x2b, 0xb6, 0x77, 0xa6, 0x2e, - 0x35, 0xca, 0xa4, 0x86, 0xcf, 0x58, 0x8f, 0xa8, 0xae, 0x1e, 0x71, 0xf8, 0x17, 0xdd, 0x5f, 0x99, - 0xda, 0x7d, 0x40, 0xcd, 0xe5, 0x18, 0x72, 0xab, 0x61, 0x01, 0x4e, 0x1d, 0x16, 0x94, 0xc0, 0x95, - 0x3f, 0x57, 0xe0, 0x5a, 0x3c, 0x43, 0xe0, 0x8a, 0x84, 0xd9, 0xa5, 0x33, 0x87, 0xd9, 0x58, 0x0c, - 0x59, 0x3e, 0x57, 0x0c, 0xd1, 0xdd, 0xfb, 0xc5, 0x33, 0xba, 0xf7, 0xd8, 0x31, 0xd9, 0x3c, 0xc7, - 0x31, 0x79, 0x92, 0xb3, 0x5f, 0x39, 0xa3, 0xb3, 0x47, 0xaf, 0xdc, 0xd9, 0xaf, 0xbe, 0xac, 0xb3, - 0x5f, 0x7b, 0x69, 0x67, 0xbf, 0xfe, 0xb2, 0xce, 0xfe, 0xd2, 0x4c, 0x67, 0x8f, 0xde, 0x8d, 0x5d, - 0x37, 0xd7, 0x86, 0x54, 0x41, 0x3a, 0x85, 0xcb, 0x8c, 0x79, 0x02, 0x36, 0x21, 0x85, 0xe7, 0x93, - 0xa2, 0xa2, 0x2b, 0x24, 0xa6, 0xf0, 0x01, 0x1e, 0x7d, 0x1b, 0xd6, 0x22, 0x38, 0x4c, 0xec, 0xce, - 0x8b, 0xc2, 0x95, 0xf8, 0x21, 0x32, 0x66, 0xf7, 0x49, 0x8c, 0xdc, 0x05, 0x24, 0xf6, 0x89, 0x46, - 0xb1, 0xf5, 0x55, 0x9a, 0x7c, 0xb4, 0x8d, 0x78, 0x01, 0x60, 0xd6, 0x68, 0x82, 0x95, 0x8f, 0x37, - 0xa1, 0xdf, 0x84, 0x11, 0x2d, 0x31, 0xe2, 0xd5, 0xb3, 0x8f, 0x68, 0x4d, 0x1b, 0x51, 0x20, 0xd1, - 0x2e, 0xdc, 0x88, 0x60, 0xc4, 0xdd, 0xa4, 0x57, 0xf6, 0xbc, 0x5e, 0x77, 0x48, 0x3a, 0x85, 0x6b, - 0xfc, 0x49, 0xfe, 0x0c, 0x32, 0xd4, 0x80, 0x2f, 0x45, 0x57, 0x15, 0xdc, 0x51, 0x06, 0x7d, 0x5d, - 0x67, 0x7d, 0xcd, 0x26, 0x7c, 0xe9, 0xfa, 0xc6, 0x27, 0xb0, 0x9e, 0x18, 0x45, 0xce, 0x78, 0x24, - 0xd2, 0x5e, 0x13, 0x2a, 0xdd, 0xdf, 0x63, 0x9f, 0x6d, 0x4f, 0x38, 0xbf, 0xcd, 0x9c, 0xdc, 0x43, - 0xb8, 0x32, 0x51, 0x17, 0x67, 0x75, 0x94, 0x55, 0x3b, 0xaa, 0xc7, 0xae, 0x03, 0x54, 0x35, 0x7b, - 0xc9, 0xae, 0xac, 0x73, 0x76, 0x55, 0xac, 0xf3, 0x8b, 0x10, 0xf9, 0x41, 0xfb, 0x4b, 0x7c, 0x92, - 0x58, 0xfc, 0xdb, 0x14, 0xac, 0x25, 0xbd, 0x8b, 0x9c, 0xf2, 0x3c, 0xe1, 0x20, 0xf6, 0xef, 0x0b, - 0x4a, 0xb3, 0x5e, 0x59, 0xea, 0xff, 0xc6, 0x20, 0x56, 0x12, 0x79, 0x25, 0xff, 0xcc, 0x60, 0xc3, - 0x9a, 0xfd, 0xdf, 0x06, 0xa6, 0xdd, 0x94, 0x28, 0x12, 0x55, 0x65, 0xfd, 0x7d, 0x03, 0x60, 0xc7, - 0x6e, 0x9f, 0x8c, 0x47, 0xac, 0xaa, 0x32, 0xa9, 0xe4, 0x56, 0x4f, 0x2a, 0xb9, 0xbd, 0xae, 0x3d, - 0xc9, 0x08, 0x3a, 0x99, 0x9e, 0x69, 0xbe, 0x74, 0x8a, 0xff, 0xfb, 0x86, 0xac, 0x18, 0xd5, 0x7d, - 0x32, 0x48, 0xfc, 0x3a, 0xaa, 0x08, 0x8b, 0xe2, 0x01, 0xcf, 0x47, 0x4a, 0xdd, 0x51, 0x83, 0x51, - 0x9a, 0x2a, 0x79, 0x62, 0x8f, 0xfb, 0x82, 0x86, 0xe7, 0xfa, 0x1a, 0x8c, 0x6e, 0x51, 0x7d, 0xe8, - 0x13, 0x77, 0x68, 0xf7, 0x45, 0xa9, 0x2c, 0x68, 0x17, 0xbf, 0x67, 0xa8, 0x85, 0x2b, 0xf4, 0x3e, - 0x2c, 0x54, 0x9c, 0xa1, 0x4f, 0xd8, 0x47, 0x44, 0xf1, 0x37, 0x05, 0x01, 0x61, 0x49, 0x50, 0x71, - 0xc1, 0x48, 0x9e, 0x0d, 0xcc, 0x1e, 0x01, 0x06, 0x88, 0x33, 0xde, 0xe5, 0x84, 0xe2, 0x50, 0x05, - 0xf5, 0xbb, 0x61, 0x85, 0x0c, 0xbd, 0x13, 0x3e, 0x91, 0x8d, 0x5d, 0xd1, 0x49, 0xa2, 0x12, 0xa3, - 0xe0, 0x13, 0xe3, 0xd4, 0x1b, 0xef, 0x01, 0x84, 0xc0, 0x33, 0x55, 0x76, 0x5f, 0xd7, 0x5e, 0xe6, - 0x4f, 0x79, 0x3a, 0xf4, 0x09, 0xac, 0xc4, 0x1e, 0xa9, 0xa0, 0x9b, 0xb0, 0xc4, 0x3f, 0xf6, 0x93, - 0xef, 0x5e, 0x38, 0x93, 0x0e, 0x64, 0xdb, 0x2c, 0x58, 0x94, 0x0f, 0x44, 0x35, 0xd8, 0xf6, 0xa7, - 0x00, 0x87, 0xa3, 0x8e, 0xed, 0xf3, 0xb3, 0xdd, 0x65, 0x58, 0xd5, 0x3e, 0x1e, 0xe4, 0x28, 0xf3, - 0x02, 0x5a, 0x87, 0x15, 0xf9, 0x51, 0x68, 0xa3, 0xd5, 0x14, 0x60, 0x03, 0xad, 0xc2, 0x45, 0x9a, - 0xad, 0xb2, 0xe5, 0x0b, 0x60, 0x0a, 0x2d, 0x41, 0xce, 0x6a, 0xed, 0x8b, 0x66, 0x9a, 0xb2, 0x06, - 0x1f, 0x78, 0x06, 0xac, 0x99, 0xed, 0x12, 0xe4, 0x82, 0x7f, 0xf9, 0x82, 0x2e, 0x42, 0xbe, 0xe9, - 0xb8, 0x03, 0xbb, 0xcf, 0x9a, 0xe6, 0x05, 0x64, 0xc2, 0xa2, 0x78, 0x97, 0xc6, 0x21, 0xc6, 0xf6, - 0xcf, 0x33, 0x00, 0xe1, 0xa3, 0x7a, 0xb4, 0x0c, 0x60, 0xb5, 0xf6, 0x8f, 0x0e, 0x0f, 0xaa, 0x65, - 0xab, 0x66, 0x5e, 0x40, 0x00, 0xf3, 0xe5, 0x83, 0x83, 0x5a, 0xb3, 0x6a, 0x1a, 0x28, 0x0b, 0x19, - 0x5c, 0x2b, 0x57, 0xcd, 0x14, 0x5a, 0x84, 0xac, 0x85, 0x0f, 0x9b, 0x15, 0x4a, 0x93, 0xa6, 0x9d, - 0x3e, 0xac, 0x59, 0x47, 0x01, 0x24, 0x83, 0xf2, 0xb0, 0x50, 0xd9, 0x6f, 0x36, 0x6b, 0x15, 0xcb, - 0x9c, 0xa3, 0x5d, 0x8a, 0xc6, 0x11, 0xde, 0x37, 0xe7, 0xd1, 0x0a, 0x2c, 0x35, 0xf6, 0x1f, 0x1e, - 0xed, 0xd6, 0xca, 0xd8, 0xda, 0xa9, 0x95, 0x2d, 0x73, 0x81, 0xf6, 0x50, 0x69, 0x2a, 0x90, 0x2c, - 0x9b, 0xa8, 0x0a, 0xc9, 0x21, 0x04, 0xcb, 0x95, 0xdd, 0x5a, 0xe5, 0xf1, 0xd1, 0x6e, 0xf9, 0x71, - 0xad, 0x76, 0x50, 0xc3, 0x26, 0x50, 0xb9, 0xd2, 0x91, 0x2b, 0x8d, 0xc3, 0x96, 0x55, 0xc3, 0x47, - 0xd5, 0x9a, 0x55, 0xae, 0x37, 0x5a, 0x66, 0x9e, 0x12, 0x53, 0x44, 0x6b, 0xb7, 0x8c, 0xab, 0x47, - 0xf5, 0xe6, 0x83, 0x7d, 0x73, 0x91, 0x75, 0xd0, 0x3c, 0x2a, 0x37, 0x1a, 0xfb, 0x74, 0x96, 0x47, - 0xf5, 0xaa, 0xb9, 0x44, 0x85, 0xa8, 0x76, 0xd0, 0xb2, 0xe8, 0xfc, 0x97, 0x99, 0xfc, 0x99, 0x04, - 0x8e, 0x2a, 0xcd, 0xa3, 0x46, 0x79, 0xa7, 0xd6, 0x30, 0x2f, 0xa2, 0x02, 0xac, 0x85, 0xc0, 0x8f, - 0xf7, 0xf1, 0x63, 0x41, 0x6e, 0xd2, 0x9e, 0x0f, 0xca, 0x56, 0x65, 0x97, 0x22, 0x5a, 0xd6, 0x3e, - 0xae, 0x99, 0x2b, 0xb4, 0x8b, 0x6a, 0xad, 0x51, 0xe3, 0xd4, 0x1c, 0x88, 0x28, 0xf0, 0x00, 0xef, - 0x7f, 0xe3, 0x9b, 0xca, 0xc2, 0x56, 0xd1, 0x97, 0xe0, 0xba, 0xe8, 0xb7, 0xb9, 0xdf, 0x3c, 0xfa, - 0x68, 0xdf, 0xaa, 0x37, 0x1f, 0x1e, 0xe1, 0xda, 0x41, 0xa3, 0x5e, 0x29, 0x1f, 0x35, 0x0f, 0xf7, - 0xcc, 0x35, 0xb4, 0x09, 0x1b, 0x71, 0x12, 0xba, 0x8e, 0x46, 0xdd, 0xfa, 0xa6, 0xb9, 0x8e, 0xd6, - 0xc0, 0x6c, 0xd5, 0xac, 0x23, 0x5c, 0xfb, 0xf0, 0xb0, 0x8e, 0x6b, 0xd5, 0xa3, 0x46, 0xab, 0x69, - 0x5e, 0xa2, 0xd0, 0x87, 0x51, 0xe8, 0x65, 0x29, 0x9a, 0x46, 0xd9, 0xaa, 0xb5, 0x2c, 0x06, 0x2b, - 0xd0, 0x2d, 0x61, 0xb0, 0x5a, 0xb9, 0x5a, 0xc3, 0x54, 0x32, 0x57, 0xd8, 0x96, 0x70, 0x71, 0xd7, - 0xca, 0x0d, 0x6b, 0xd7, 0xdc, 0xa0, 0x9b, 0x4e, 0xb7, 0x9f, 0xb1, 0x5c, 0x45, 0x57, 0x60, 0x5d, - 0x4c, 0xa9, 0x51, 0x2b, 0xb7, 0x6a, 0xbb, 0xfb, 0x0d, 0xc1, 0x7a, 0x8d, 0xa2, 0x98, 0xf0, 0x2b, - 0xbb, 0xb5, 0xea, 0x61, 0xa3, 0x76, 0x54, 0xd9, 0xdf, 0xdb, 0x2b, 0x37, 0xab, 0x2d, 0xf3, 0xfa, - 0x76, 0x13, 0x20, 0xfc, 0x14, 0x95, 0x6a, 0x06, 0x55, 0x73, 0x0e, 0x31, 0x2f, 0xd0, 0x11, 0xa4, - 0x9f, 0x33, 0x0d, 0xaa, 0xbc, 0xcc, 0x68, 0x02, 0x03, 0x58, 0x11, 0xdf, 0x5e, 0x63, 0xf2, 0x6d, - 0xd2, 0xf6, 0x49, 0xc7, 0x4c, 0x6f, 0x6f, 0x43, 0x2e, 0xf8, 0xd2, 0x91, 0xb2, 0xb7, 0x88, 0xcf, - 0x5a, 0xe6, 0x05, 0xca, 0xce, 0x8f, 0x5d, 0x1c, 0x60, 0x6c, 0x7f, 0x2f, 0x03, 0x48, 0xe6, 0x9e, - 0x8a, 0x6d, 0x52, 0x8d, 0xef, 0xb5, 0x4f, 0x54, 0x93, 0x54, 0x3e, 0x29, 0x0b, 0x4c, 0x92, 0x5a, - 0x6a, 0x0c, 0x9c, 0x42, 0x97, 0xd8, 0xcd, 0x4d, 0x14, 0x9e, 0xa6, 0xa3, 0x3f, 0x24, 0x7e, 0x60, - 0xe9, 0x19, 0x2a, 0x94, 0x88, 0xbf, 0x11, 0xa8, 0x39, 0xba, 0x23, 0x2d, 0xc2, 0x0d, 0x52, 0xc0, - 0xe6, 0xa9, 0xb2, 0xe9, 0x57, 0x73, 0x02, 0xb3, 0x80, 0x6e, 0xc0, 0xd5, 0x16, 0xf1, 0xe3, 0x55, - 0x0b, 0x41, 0x90, 0x45, 0x1b, 0x70, 0x49, 0x10, 0x04, 0xc7, 0x5e, 0x81, 0xcb, 0x51, 0x11, 0xf2, - 0xdf, 0x42, 0x6a, 0x26, 0xd0, 0x85, 0x49, 0x50, 0xf0, 0x66, 0xcc, 0xcc, 0xd3, 0xfd, 0x3f, 0xa0, - 0xfe, 0x4e, 0x5c, 0x4e, 0x9b, 0x8b, 0x94, 0x17, 0x93, 0x81, 0xf3, 0x4c, 0x3e, 0xf7, 0x34, 0x97, - 0xe8, 0x2c, 0xf5, 0x5b, 0x7b, 0x31, 0xd0, 0x32, 0xba, 0x0e, 0x57, 0xf8, 0xef, 0x84, 0xe3, 0xac, - 0x79, 0x11, 0x5d, 0x85, 0xcb, 0x11, 0xb4, 0x8c, 0x06, 0xdc, 0x9c, 0x64, 0x92, 0x2a, 0xfa, 0x5b, - 0x41, 0xd7, 0xa0, 0x10, 0xbf, 0x5c, 0x13, 0x58, 0x84, 0x6e, 0xc2, 0x96, 0x3c, 0xde, 0xc5, 0x0f, - 0x7e, 0x82, 0x6a, 0x95, 0x4a, 0x8e, 0x1f, 0xe5, 0x22, 0x09, 0xa3, 0x20, 0x58, 0xdb, 0xfe, 0xae, - 0x01, 0x4b, 0x5a, 0x81, 0x89, 0x1a, 0xac, 0x04, 0x88, 0x8b, 0x23, 0xf3, 0x02, 0xdd, 0x4a, 0x09, - 0xd4, 0x5e, 0xe3, 0x9b, 0x06, 0xfa, 0x7f, 0xf0, 0xa5, 0x18, 0x4a, 0xe6, 0xf1, 0x98, 0xb4, 0x49, - 0xef, 0x19, 0xe9, 0x98, 0x29, 0xba, 0xfc, 0x18, 0xd9, 0x03, 0xbb, 0xd7, 0xa7, 0x3a, 0xad, 0x8e, - 0x89, 0xc7, 0xc3, 0x21, 0xed, 0x38, 0xb3, 0x7d, 0x9c, 0x54, 0xe2, 0xa2, 0xf2, 0xd7, 0xa0, 0xe1, - 0x1c, 0xa3, 0x18, 0xd9, 0x93, 0x11, 0xc3, 0xb4, 0x7c, 0x67, 0x34, 0xa2, 0xb3, 0xda, 0xfe, 0x37, - 0x03, 0xcc, 0xe8, 0xf7, 0x17, 0xd4, 0x3c, 0xca, 0x1d, 0xf9, 0x49, 0xb4, 0x79, 0x21, 0xd4, 0x02, - 0x09, 0x32, 0xa8, 0xaa, 0xb4, 0x7c, 0xdb, 0xf5, 0x25, 0x24, 0x45, 0xb5, 0x9f, 0x76, 0x2b, 0x01, - 0x69, 0xda, 0xcb, 0xe3, 0x5e, 0xbf, 0xff, 0x2d, 0x67, 0x70, 0xdc, 0xa3, 0xd6, 0x70, 0x19, 0x56, - 0xcb, 0x9d, 0x4e, 0x54, 0x37, 0xcc, 0x39, 0xaa, 0xbc, 0xbc, 0xfb, 0x18, 0x6e, 0x9e, 0x99, 0x10, - 0x1d, 0x27, 0x86, 0x5a, 0xa0, 0x8b, 0xa2, 0x03, 0xc6, 0x30, 0xd9, 0xed, 0x3d, 0xed, 0x5d, 0x3a, - 0x9d, 0x48, 0xa8, 0x21, 0xe6, 0x05, 0x16, 0x54, 0x9b, 0xb2, 0x69, 0xd0, 0x66, 0x25, 0x68, 0xa6, - 0x98, 0x11, 0xb0, 0xea, 0x8f, 0x80, 0xa4, 0x77, 0xea, 0x9f, 0xfd, 0x6c, 0xf3, 0xc2, 0x8f, 0x3f, - 0xdf, 0x34, 0x3e, 0xfb, 0x7c, 0xd3, 0xf8, 0xe9, 0xe7, 0x9b, 0x17, 0xfe, 0xee, 0x3f, 0x36, 0x8d, - 0x6f, 0xdd, 0x55, 0xfe, 0xf1, 0xe5, 0xc0, 0xf6, 0xdd, 0xde, 0x73, 0x87, 0x65, 0x0c, 0xb2, 0x31, - 0x24, 0xb7, 0x47, 0x27, 0xdd, 0xdb, 0xa3, 0xe3, 0xdb, 0x61, 0xfe, 0x73, 0x3c, 0xcf, 0xbe, 0x5f, - 0xbf, 0xfb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xff, 0x4e, 0x41, 0xa4, 0x54, 0x53, 0x00, 0x00, + // 5431 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7c, 0x4b, 0x6c, 0x1b, 0x49, + 0x7a, 0xb0, 0x9b, 0xa4, 0x28, 0xf2, 0xa3, 0x1e, 0xad, 0x92, 0x64, 0xd3, 0xb2, 0x2d, 0x6b, 0xb9, + 0xfe, 0x67, 0x3c, 0xda, 0x19, 0x79, 0x7f, 0x7b, 0xc6, 0x98, 0xdd, 0x78, 0x3c, 0x43, 0x91, 0xb4, + 0x45, 0x9b, 0xa6, 0x34, 0xcd, 0x96, 0x67, 0x67, 0x81, 0x81, 0xd0, 0x22, 0xcb, 0x32, 0x57, 0x14, + 0x9b, 0xdb, 0xdd, 0xf4, 0xd8, 0xc9, 0x35, 0xc9, 0x21, 0x01, 0x16, 0x73, 0x08, 0x82, 0x45, 0x90, + 0xe7, 0x31, 0xa7, 0x45, 0x80, 0xbd, 0xe7, 0xba, 0xa7, 0x60, 0x90, 0x43, 0x2e, 0x01, 0x76, 0x77, + 0x26, 0x97, 0x00, 0xc9, 0x21, 0x0f, 0x20, 0x7b, 0x0a, 0x10, 0xd4, 0xab, 0xbb, 0xaa, 0xbb, 0x48, + 0xea, 0x95, 0x49, 0x26, 0xd8, 0x93, 0x58, 0xdf, 0xa3, 0xaa, 0xba, 0xea, 0x7b, 0x57, 0x95, 0xc0, + 0xec, 0xb9, 0x07, 0x3e, 0xf6, 0x5e, 0x74, 0xdb, 0x78, 0x63, 0xe0, 0xb9, 0x81, 0x8b, 0x20, 0x82, + 0xac, 0xbc, 0x75, 0xd0, 0x0d, 0x9e, 0x0f, 0xf7, 0x37, 0xda, 0xee, 0xd1, 0xad, 0x03, 0xf7, 0xc0, + 0xbd, 0x45, 0x49, 0xf6, 0x87, 0xcf, 0x68, 0x8b, 0x36, 0xe8, 0x2f, 0xc6, 0xba, 0x72, 0xfd, 0xc0, + 0x75, 0x0f, 0x7a, 0x38, 0xa2, 0x0a, 0xba, 0x47, 0xd8, 0x0f, 0x9c, 0xa3, 0x01, 0x27, 0x98, 0x3b, + 0xc2, 0x81, 0xd3, 0x71, 0x02, 0x87, 0xb7, 0xe7, 0x63, 0x04, 0xa5, 0x7f, 0xc9, 0xc2, 0x74, 0xa5, + 0xd9, 0x0a, 0x5c, 0x0f, 0x23, 0x04, 0x99, 0xdd, 0xdd, 0x7a, 0xb5, 0x68, 0xac, 0x19, 0x37, 0xf3, + 0x16, 0xfd, 0x8d, 0x5e, 0x83, 0xb9, 0x16, 0x9b, 0x5b, 0xb9, 0xd3, 0xf1, 0xb0, 0xef, 0x17, 0x53, + 0x14, 0x1b, 0x83, 0xa2, 0x55, 0x80, 0xd6, 0x87, 0x0d, 0x41, 0x93, 0xa6, 0x34, 0x12, 0x04, 0x6d, + 0x00, 0x6a, 0xb8, 0xed, 0xc3, 0x58, 0x5f, 0x19, 0x4a, 0xa7, 0xc1, 0xa0, 0x1b, 0x90, 0xb1, 0xdc, + 0x1e, 0x2e, 0x66, 0xd7, 0x8c, 0x9b, 0x73, 0xb7, 0xcd, 0x8d, 0xf0, 0x3b, 0x2a, 0x4d, 0x02, 0xb7, + 0x28, 0x96, 0xcc, 0xd8, 0xee, 0xb6, 0x0f, 0x8b, 0xd3, 0x6b, 0xc6, 0xcd, 0x8c, 0x45, 0x7f, 0xa3, + 0x6f, 0xc1, 0x54, 0x2b, 0x70, 0x02, 0x5c, 0xcc, 0x51, 0xd6, 0xe5, 0x0d, 0x69, 0xc1, 0x9b, 0x6e, + 0x07, 0x53, 0xa4, 0xc5, 0x68, 0xd0, 0x7b, 0x90, 0x6d, 0x38, 0xfb, 0xb8, 0xe7, 0x17, 0xf3, 0x6b, + 0xe9, 0x9b, 0x85, 0xdb, 0xd7, 0x65, 0x6a, 0xbe, 0x2e, 0x1b, 0x8c, 0xa2, 0xd6, 0x0f, 0xbc, 0x57, + 0x9b, 0x99, 0x9f, 0xfd, 0xfc, 0xfa, 0x05, 0x8b, 0x33, 0xa1, 0xff, 0x0f, 0xf9, 0x8f, 0x5c, 0xef, + 0x90, 0x8d, 0x07, 0x74, 0xbc, 0xc5, 0x68, 0xaa, 0x21, 0xca, 0x8a, 0xa8, 0x50, 0x09, 0x66, 0x3e, + 0x1c, 0x62, 0xef, 0x95, 0x58, 0x82, 0x02, 0x5d, 0x02, 0x05, 0x86, 0xee, 0x02, 0x54, 0xdc, 0xfe, + 0xb3, 0xee, 0x41, 0xd5, 0x09, 0x9c, 0xe2, 0xcc, 0x9a, 0x71, 0xb3, 0x70, 0xfb, 0xa2, 0x32, 0xb3, + 0x10, 0x6b, 0x49, 0x94, 0xe8, 0x2e, 0xe4, 0x2c, 0xec, 0xbb, 0x43, 0xaf, 0x8d, 0x8b, 0xb3, 0x94, + 0x6b, 0x49, 0xe6, 0x12, 0x38, 0xfe, 0x11, 0x21, 0x2d, 0xba, 0x08, 0xd9, 0xdd, 0x81, 0xdd, 0x3d, + 0xc2, 0xc5, 0xb9, 0x35, 0xe3, 0x66, 0xda, 0xe2, 0x2d, 0xf4, 0x6d, 0x58, 0x6c, 0x3d, 0x77, 0xbc, + 0x4e, 0x6c, 0xd7, 0xe6, 0xe9, 0x94, 0x75, 0x28, 0xb4, 0x02, 0xb9, 0x8a, 0x7b, 0x74, 0xd4, 0x0d, + 0xea, 0xd5, 0xa2, 0x49, 0xc9, 0xc2, 0x36, 0x6a, 0xc2, 0xd2, 0xc3, 0x9e, 0xbb, 0xef, 0xf4, 0x5a, + 0xaf, 0xfc, 0xa7, 0x8e, 0xc7, 0xe0, 0x76, 0xab, 0xb8, 0xc0, 0x67, 0x1a, 0x89, 0xa6, 0x2d, 0x7e, + 0xf1, 0x99, 0x6a, 0xf9, 0xd0, 0x5d, 0xb8, 0x28, 0xc3, 0x1f, 0xe2, 0x3e, 0xf6, 0x9c, 0xa0, 0xeb, + 0xf6, 0x8b, 0x88, 0x8e, 0x3c, 0x02, 0x8b, 0x6e, 0xc2, 0xfc, 0x0e, 0x91, 0xfd, 0xb6, 0xdb, 0x7b, + 0x8a, 0x3d, 0x9f, 0x30, 0x2c, 0xd2, 0xcf, 0x8e, 0x83, 0x57, 0x9a, 0x50, 0x90, 0xf6, 0x1e, 0x99, + 0x90, 0x3e, 0xc4, 0xaf, 0xb8, 0x7a, 0x90, 0x9f, 0xe8, 0x0d, 0x98, 0x7a, 0xe1, 0xf4, 0x86, 0x98, + 0x2a, 0x45, 0x41, 0xde, 0x7b, 0xca, 0xd7, 0xe8, 0xfa, 0x81, 0xc5, 0x28, 0xbe, 0x9b, 0x7a, 0xd7, + 0x78, 0x94, 0xc9, 0x4d, 0x99, 0xd9, 0xd2, 0xaf, 0xd2, 0x30, 0x6d, 0x9f, 0x83, 0xca, 0x09, 0xe1, + 0x4f, 0xeb, 0x84, 0x3f, 0x73, 0x0c, 0xe1, 0x7f, 0x07, 0xb2, 0x74, 0x0f, 0xfd, 0xe2, 0x14, 0x15, + 0xfe, 0x4b, 0x32, 0xb5, 0xdd, 0xa4, 0xb8, 0x7a, 0xff, 0x99, 0x2b, 0x84, 0x9e, 0x11, 0xa3, 0xdb, + 0xb0, 0xd4, 0x70, 0x0f, 0x02, 0xa7, 0xdb, 0x23, 0x13, 0xc2, 0x9e, 0x98, 0x65, 0x96, 0xce, 0x52, + 0x8b, 0x1b, 0xa1, 0xfe, 0xd3, 0x23, 0xd5, 0x5f, 0xd5, 0x80, 0xfc, 0xb1, 0x35, 0x20, 0xae, 0x5d, + 0xa0, 0xd1, 0xae, 0x11, 0x52, 0x5d, 0x18, 0x2d, 0xd5, 0x1f, 0xc0, 0x95, 0xf2, 0x30, 0x70, 0xeb, + 0xfd, 0xb6, 0x57, 0x1b, 0xb8, 0xed, 0xe7, 0x0f, 0x70, 0xbf, 0x8d, 0x5b, 0xc3, 0xc1, 0xc0, 0xf5, + 0x02, 0xdc, 0xa1, 0x0a, 0x9a, 0xb3, 0xc6, 0x91, 0x3c, 0xca, 0xe4, 0x72, 0x66, 0xbe, 0xf4, 0x8b, + 0x14, 0xe4, 0x1a, 0xee, 0xc1, 0xff, 0x82, 0xad, 0xbf, 0x47, 0x2c, 0xc5, 0xa0, 0xd7, 0x6d, 0x3b, + 0x62, 0xf3, 0x57, 0x64, 0xfa, 0x86, 0x7b, 0xc0, 0xd1, 0xd2, 0xfe, 0x87, 0x1c, 0xb1, 0xdd, 0xc9, + 0x9e, 0xc4, 0x3e, 0x35, 0xdc, 0xb6, 0xd3, 0xeb, 0x06, 0xaf, 0xe8, 0xde, 0xc7, 0xec, 0x93, 0xc0, + 0x89, 0xf1, 0x44, 0x5b, 0xa7, 0xb1, 0x39, 0xad, 0xc6, 0x96, 0xfe, 0x20, 0x0d, 0x33, 0x64, 0x85, + 0x85, 0xe8, 0xa2, 0x22, 0x4c, 0xb3, 0x06, 0x5b, 0xe8, 0x8c, 0x25, 0x9a, 0x68, 0x53, 0x5a, 0x82, + 0x14, 0x5d, 0x82, 0xd7, 0x62, 0x4b, 0x10, 0xf6, 0xb2, 0x21, 0x08, 0xa9, 0x1d, 0x90, 0x16, 0x62, + 0x09, 0xa6, 0xe8, 0x6e, 0xf3, 0x8d, 0x60, 0x0d, 0x62, 0x04, 0x1b, 0xd8, 0xe9, 0x60, 0xaf, 0x5e, + 0xa5, 0x9b, 0x91, 0xb1, 0xc2, 0x36, 0xdd, 0x39, 0xec, 0x1d, 0x15, 0xa7, 0xf8, 0xce, 0x61, 0xef, + 0x08, 0x7d, 0x02, 0x0b, 0x4d, 0xb7, 0xff, 0xd4, 0x0d, 0xba, 0xfd, 0x83, 0x70, 0x4a, 0x59, 0x3a, + 0xa5, 0x5b, 0x23, 0xa7, 0x94, 0xe0, 0x60, 0x73, 0x4b, 0xf6, 0xb4, 0xf2, 0x1b, 0x30, 0xab, 0xd0, + 0xc8, 0x76, 0x2c, 0xc3, 0xec, 0xd8, 0x92, 0x6c, 0xc7, 0xf2, 0x92, 0xc9, 0x5a, 0xa9, 0xc2, 0x45, + 0xfd, 0x48, 0x27, 0xe9, 0xa5, 0xf4, 0x63, 0x03, 0xe6, 0x54, 0x99, 0x42, 0x0f, 0xd4, 0x8d, 0xa2, + 0xfd, 0x14, 0x6e, 0x17, 0x47, 0x7d, 0xef, 0x66, 0x8e, 0xc8, 0xc4, 0xe7, 0x3f, 0xbf, 0x6e, 0x58, + 0xea, 0x06, 0x5f, 0x85, 0xbc, 0xe8, 0xb6, 0x4a, 0x07, 0xce, 0x58, 0x11, 0x00, 0xad, 0x41, 0xa1, + 0xee, 0x87, 0x1f, 0x40, 0xb7, 0x29, 0x67, 0xc9, 0xa0, 0xd2, 0xef, 0x19, 0x91, 0xd3, 0xa4, 0xee, + 0x6b, 0x67, 0xd7, 0x76, 0x03, 0xa7, 0xc7, 0x3f, 0x2c, 0x6c, 0x13, 0xd3, 0x52, 0xd9, 0xd9, 0x2d, + 0xbf, 0x70, 0xba, 0x3d, 0x67, 0xbf, 0xc7, 0x3e, 0xd2, 0xb0, 0x14, 0x18, 0xe1, 0x7f, 0x82, 0x8f, + 0x18, 0x3f, 0x13, 0x89, 0xb0, 0x4d, 0xf8, 0x9f, 0xe0, 0xa3, 0x88, 0x9f, 0x49, 0x86, 0x02, 0x2b, + 0xfd, 0x5b, 0x16, 0x4c, 0x1e, 0x75, 0x6c, 0x61, 0xc7, 0x0b, 0xf6, 0xb1, 0x13, 0x7c, 0x0d, 0xc3, + 0xb2, 0x0d, 0x40, 0xb6, 0xe3, 0x0b, 0xde, 0x8a, 0x87, 0x1d, 0x62, 0x26, 0xa7, 0xe9, 0xe2, 0x6b, + 0x30, 0x09, 0xab, 0x9d, 0xd3, 0x58, 0xed, 0x1b, 0x30, 0x5b, 0xef, 0x77, 0x83, 0x28, 0xdc, 0xca, + 0x53, 0x22, 0x15, 0x48, 0xa8, 0x1e, 0xba, 0xbe, 0xdf, 0x1d, 0xa8, 0x0e, 0x40, 0x05, 0x92, 0xf1, + 0x18, 0xe0, 0x91, 0xdb, 0xed, 0xe3, 0x0e, 0x35, 0xfd, 0x39, 0x4b, 0x81, 0x7d, 0xe5, 0x31, 0xd8, + 0x08, 0xaf, 0x34, 0x77, 0xbc, 0x58, 0x6b, 0x3e, 0x16, 0x6b, 0x7d, 0x1b, 0x16, 0xcb, 0xed, 0x43, + 0xdc, 0x21, 0x00, 0xa7, 0xdf, 0xd9, 0x74, 0x82, 0xf6, 0x73, 0x1e, 0x92, 0x65, 0x2c, 0x1d, 0x8a, + 0xf8, 0x38, 0x0e, 0xa9, 0xe2, 0x5e, 0xf7, 0x05, 0x59, 0xf9, 0xf6, 0x61, 0xe4, 0xe3, 0x16, 0x98, + 0x8f, 0x1b, 0x43, 0x32, 0x32, 0xbe, 0x43, 0xe7, 0x1e, 0xdf, 0x2d, 0x9e, 0x34, 0xbe, 0x5b, 0xd2, + 0x7a, 0x0b, 0x1e, 0x8f, 0xd9, 0x30, 0x53, 0x69, 0x96, 0x7b, 0x3d, 0xb7, 0xed, 0x04, 0xb8, 0x5e, + 0xd5, 0x84, 0x79, 0x4b, 0x30, 0x45, 0x97, 0x89, 0xdb, 0x17, 0xd6, 0x60, 0x96, 0xe7, 0x87, 0x43, + 0xec, 0x93, 0x0d, 0x60, 0xaa, 0x15, 0x01, 0x4a, 0x7f, 0x9f, 0x86, 0x05, 0xe1, 0xeb, 0xc7, 0xeb, + 0xf2, 0x1a, 0x14, 0x2c, 0xe7, 0x59, 0xa0, 0x2a, 0xb2, 0x0c, 0xd2, 0x68, 0x7b, 0x5a, 0xab, 0xed, + 0x09, 0xe9, 0xcf, 0xe8, 0xa4, 0xff, 0x6c, 0xbe, 0x5f, 0xaf, 0xdb, 0xd9, 0x91, 0xba, 0xad, 0xea, + 0xd1, 0xf4, 0xa9, 0x62, 0x85, 0xdc, 0x09, 0x62, 0x85, 0xef, 0x42, 0x31, 0x26, 0xa4, 0x91, 0x10, + 0xe7, 0xe9, 0x2c, 0x47, 0xe2, 0x75, 0x92, 0x03, 0xfa, 0x38, 0xa3, 0x06, 0x05, 0x29, 0x40, 0x1e, + 0x13, 0x65, 0x8c, 0x75, 0x4f, 0xa5, 0xdf, 0x99, 0x02, 0xd3, 0x3e, 0x4f, 0x7b, 0x1f, 0x85, 0xf4, + 0xe9, 0x93, 0x84, 0xf4, 0xfa, 0x4d, 0xcd, 0x8c, 0xdc, 0xd4, 0x51, 0x29, 0xc0, 0xd4, 0x89, 0x53, + 0x80, 0xec, 0x31, 0x53, 0x80, 0xdc, 0xa9, 0x53, 0x80, 0xfc, 0xf1, 0x53, 0x00, 0x18, 0x6d, 0x6c, + 0x89, 0x92, 0xe2, 0x41, 0xcf, 0x79, 0x85, 0x3b, 0x0d, 0xbf, 0x4f, 0x3d, 0x46, 0xc6, 0x92, 0x41, + 0x67, 0x4f, 0x12, 0x46, 0x19, 0xed, 0xd9, 0x53, 0x1b, 0xed, 0xb9, 0x89, 0x46, 0xfb, 0x51, 0x26, + 0x37, 0x6d, 0xe6, 0x4a, 0x3f, 0x99, 0x82, 0x9c, 0xd5, 0x7a, 0xc2, 0x7c, 0xa8, 0x09, 0x69, 0xdb, + 0x77, 0x45, 0x60, 0x67, 0xfb, 0x2e, 0xb1, 0x7f, 0xf5, 0x7e, 0x07, 0xbf, 0x14, 0xf6, 0x8f, 0x36, + 0x88, 0xb5, 0x69, 0x60, 0xc7, 0xc7, 0x5b, 0x6e, 0x8f, 0xc5, 0xba, 0x2c, 0xe2, 0x51, 0x81, 0x64, + 0x3b, 0x6c, 0x6f, 0xd8, 0x27, 0xb6, 0x95, 0xae, 0x1c, 0x0f, 0x7b, 0x64, 0x18, 0x7a, 0x04, 0x33, + 0x8c, 0xa9, 0xeb, 0x07, 0xae, 0xf7, 0x8a, 0x5b, 0x25, 0x25, 0x1c, 0x17, 0xb3, 0xdb, 0x90, 0x09, + 0x59, 0xc8, 0xab, 0xf0, 0xb2, 0x8d, 0xfa, 0xe1, 0xb0, 0xeb, 0xb1, 0xe1, 0xb2, 0x62, 0xa3, 0x42, + 0x10, 0x7a, 0x13, 0x16, 0x3e, 0x2a, 0x37, 0x2c, 0xdc, 0x76, 0xc9, 0x6a, 0x54, 0xbb, 0x07, 0xd8, + 0x0f, 0x78, 0x2a, 0x9a, 0x44, 0xa0, 0xb7, 0x61, 0x59, 0x02, 0xd2, 0x11, 0x2b, 0xee, 0xb0, 0x1f, + 0x50, 0x89, 0xcc, 0x58, 0x7a, 0x24, 0xd9, 0x18, 0x09, 0x51, 0x71, 0x8f, 0x06, 0x3d, 0x1c, 0xe0, + 0x0e, 0xa1, 0xe8, 0x62, 0x26, 0x93, 0x19, 0x6b, 0x1c, 0x09, 0x51, 0x17, 0x09, 0xbd, 0xe9, 0xf8, + 0x98, 0x7c, 0x0e, 0x50, 0x46, 0x0d, 0x26, 0x46, 0xdf, 0x70, 0xfc, 0x20, 0x92, 0x53, 0x0d, 0x06, + 0x3d, 0x82, 0x35, 0x09, 0xba, 0xed, 0x75, 0x0f, 0xba, 0x7d, 0xa7, 0xa7, 0x6e, 0xe8, 0x0c, 0xe5, + 0x9e, 0x48, 0x47, 0x04, 0x57, 0xf3, 0x29, 0x54, 0x70, 0x73, 0x96, 0x0e, 0xb5, 0xf2, 0x3e, 0x2c, + 0x24, 0x36, 0x72, 0x52, 0x46, 0x91, 0x91, 0x33, 0x8a, 0x4f, 0x20, 0x4f, 0x1d, 0x55, 0xdb, 0xf5, + 0x3a, 0x84, 0x91, 0x7c, 0x2c, 0x67, 0x24, 0x5f, 0xb7, 0x0e, 0x19, 0xfb, 0xd5, 0x80, 0xf1, 0xcd, + 0xa9, 0x66, 0x83, 0xf1, 0x10, 0xac, 0x45, 0x69, 0x88, 0xbd, 0xa5, 0x26, 0x86, 0x88, 0xef, 0x8c, + 0x45, 0x7f, 0x97, 0x7e, 0x99, 0x02, 0xa0, 0xfd, 0x53, 0x77, 0x4e, 0x48, 0x9a, 0xce, 0x11, 0x16, + 0x26, 0x99, 0xfc, 0x96, 0x6d, 0x7e, 0x4a, 0xb5, 0xf9, 0x7c, 0x3a, 0xe9, 0x68, 0x3a, 0x45, 0x98, + 0x7e, 0xe2, 0xbc, 0x6c, 0x75, 0x7f, 0x53, 0x84, 0xfd, 0xa2, 0x49, 0xfc, 0x83, 0x30, 0xcb, 0x55, + 0x9e, 0x14, 0x46, 0x00, 0x9a, 0x2d, 0x36, 0xeb, 0x55, 0x2e, 0xc5, 0xf4, 0x37, 0x7a, 0x1b, 0x52, + 0x76, 0x8b, 0x3b, 0xd2, 0x95, 0x0d, 0x56, 0x00, 0xde, 0x10, 0x05, 0x60, 0x29, 0xb4, 0xa2, 0x09, + 0xd3, 0x67, 0xbf, 0xb8, 0x6e, 0x58, 0x29, 0xbb, 0x85, 0x1e, 0xc0, 0x6a, 0xbd, 0xdf, 0xee, 0x0d, + 0x3b, 0xb8, 0xf6, 0x72, 0x40, 0x34, 0x81, 0x3b, 0x21, 0x6e, 0xdf, 0x30, 0x0b, 0xba, 0x73, 0xd6, + 0x04, 0x2a, 0xb4, 0x05, 0xd7, 0x6b, 0x2f, 0x29, 0xc5, 0x96, 0xe3, 0x75, 0xaa, 0xee, 0xa7, 0xfd, + 0x44, 0x47, 0xcc, 0xcb, 0x4e, 0x22, 0x2b, 0x95, 0x00, 0x6c, 0xdf, 0x15, 0x2b, 0xbc, 0x04, 0x53, + 0x4c, 0xad, 0xd8, 0x26, 0xb2, 0x46, 0xe9, 0x9f, 0x0c, 0x12, 0x9b, 0x51, 0xff, 0x48, 0x0b, 0x6a, + 0x5a, 0xdf, 0x78, 0x07, 0xf2, 0xdb, 0x03, 0x11, 0x1a, 0xa6, 0x92, 0xc5, 0x8f, 0x4a, 0x93, 0xf2, + 0x6e, 0x0f, 0xac, 0x88, 0x0e, 0x6d, 0x86, 0x85, 0x5f, 0xe6, 0x28, 0x6f, 0x68, 0x0a, 0xbf, 0x94, + 0x60, 0x74, 0xf5, 0xf7, 0xbc, 0xcb, 0x83, 0xa5, 0x06, 0x14, 0x2a, 0xcd, 0x28, 0x97, 0xd1, 0x7d, + 0xeb, 0x1b, 0xa2, 0xc8, 0x93, 0x1a, 0x5d, 0x6c, 0x66, 0x14, 0xa5, 0x2f, 0xf8, 0xda, 0x39, 0xc1, + 0x98, 0xb5, 0x3b, 0x7e, 0x7f, 0x93, 0x57, 0x4c, 0x0c, 0xf4, 0x15, 0xae, 0xd8, 0x17, 0x39, 0x98, + 0x16, 0x12, 0xa4, 0x84, 0xe3, 0x86, 0x88, 0xb4, 0x38, 0x00, 0x6d, 0x40, 0xf6, 0x09, 0x0e, 0x9e, + 0xbb, 0x1d, 0x9d, 0x49, 0x60, 0x18, 0x6a, 0x12, 0x38, 0x15, 0xba, 0x27, 0xeb, 0x3f, 0x55, 0xe5, + 0x58, 0xf4, 0x11, 0x61, 0xf9, 0x37, 0xca, 0xf6, 0xa2, 0x4c, 0x8b, 0x1b, 0x61, 0x48, 0x47, 0x95, + 0xbe, 0x70, 0xfb, 0x5a, 0xbc, 0xb8, 0xa1, 0xc4, 0x7d, 0x96, 0xc2, 0x82, 0xee, 0x13, 0x61, 0x88, + 0x7a, 0x98, 0xa2, 0x3d, 0x5c, 0xd5, 0x48, 0x69, 0xd4, 0x81, 0xcc, 0x40, 0xf8, 0x6d, 0x89, 0x3f, + 0x9b, 0xe4, 0xb7, 0x13, 0xfc, 0x12, 0x03, 0x09, 0xbf, 0x22, 0xf5, 0xd4, 0xc5, 0xed, 0x11, 0xd6, + 0x92, 0x15, 0xf9, 0x9e, 0x9a, 0x4d, 0xf1, 0xc0, 0xad, 0xa8, 0x4e, 0x3c, 0xc2, 0x5b, 0x6a, 0xee, + 0x75, 0x4f, 0xd5, 0x77, 0x5e, 0xf9, 0x2d, 0x8e, 0x52, 0x4e, 0x4b, 0xb5, 0x0e, 0xdf, 0x51, 0x14, + 0x88, 0x3a, 0xcb, 0x58, 0x08, 0x2c, 0xa1, 0x2d, 0x45, 0xd9, 0xee, 0xa9, 0xca, 0x42, 0x1d, 0xa7, + 0x66, 0x60, 0x81, 0xb7, 0x54, 0xd5, 0x7a, 0x1f, 0x66, 0xab, 0x98, 0x38, 0x36, 0x3e, 0x1d, 0x5e, + 0x2f, 0xb8, 0x2c, 0xb3, 0x2b, 0x04, 0x96, 0x4a, 0x8f, 0x36, 0x61, 0x6e, 0xc7, 0x73, 0x5f, 0xbe, + 0x8a, 0x36, 0x6c, 0x96, 0x1b, 0x78, 0xa9, 0x07, 0x95, 0xc2, 0x8a, 0x71, 0x10, 0x2f, 0x1c, 0x2f, + 0xd5, 0x35, 0x87, 0x47, 0x34, 0x08, 0xcc, 0x58, 0x3a, 0x14, 0xda, 0x94, 0x0a, 0x8f, 0x61, 0xb2, + 0x35, 0x3f, 0x3a, 0xd9, 0xb2, 0x92, 0xe4, 0x74, 0xcd, 0x9f, 0xe3, 0xf6, 0xe1, 0x16, 0x76, 0x7a, + 0xc1, 0x73, 0x5a, 0x61, 0x88, 0xaf, 0x79, 0x84, 0xb6, 0x64, 0x5a, 0x64, 0xc3, 0x52, 0xab, 0xfd, + 0x1c, 0x77, 0x86, 0x3d, 0xcc, 0x43, 0x54, 0x1a, 0xa4, 0xf3, 0x03, 0xa1, 0x35, 0xb9, 0x0f, 0x1d, + 0x9d, 0xa5, 0xe5, 0x3e, 0xef, 0x32, 0x44, 0xc9, 0x85, 0x02, 0xd5, 0x6c, 0x7f, 0xe0, 0xf6, 0x7d, + 0x3c, 0x26, 0xd5, 0xe3, 0x6e, 0x3f, 0xa5, 0xb8, 0x7d, 0x11, 0x88, 0xb1, 0x60, 0x40, 0x34, 0xc7, + 0x95, 0x88, 0x4b, 0x1b, 0x80, 0x24, 0xfd, 0x90, 0xc6, 0x7d, 0xd0, 0xf5, 0x24, 0xe3, 0x26, 0x9a, + 0xa5, 0xff, 0xc8, 0xd0, 0x92, 0x13, 0x23, 0x3b, 0x5f, 0x2b, 0x78, 0x15, 0xf2, 0x35, 0xcf, 0x73, + 0xbd, 0x8a, 0xdb, 0xc1, 0xf4, 0x13, 0x66, 0xad, 0x08, 0x40, 0x42, 0x7b, 0xda, 0x78, 0x82, 0x7d, + 0xdf, 0x39, 0xc0, 0xbc, 0xda, 0xa0, 0xc0, 0xd0, 0x2a, 0x40, 0xdd, 0xdf, 0x2a, 0x3f, 0xc6, 0x78, + 0x80, 0x3d, 0x6a, 0xc5, 0x72, 0x96, 0x04, 0x41, 0xef, 0x2b, 0xab, 0xcb, 0xcd, 0xd4, 0xa5, 0x84, + 0xa1, 0x65, 0x68, 0xbe, 0x4f, 0xca, 0x7e, 0x10, 0xc5, 0x95, 0x92, 0x22, 0x6e, 0xa9, 0x54, 0xc5, + 0x95, 0xf0, 0x96, 0x42, 0x4d, 0xa4, 0x97, 0xda, 0x2e, 0x3e, 0x7c, 0x2e, 0x39, 0xbc, 0x84, 0xb6, + 0x64, 0x5a, 0xa2, 0xb2, 0x95, 0xde, 0xd0, 0x0f, 0xb0, 0x57, 0xc5, 0x24, 0xdb, 0xf5, 0xb9, 0xb1, + 0x52, 0x54, 0x56, 0xa5, 0xb0, 0x62, 0x1c, 0xe8, 0x3e, 0xe4, 0xa3, 0x0a, 0x38, 0x68, 0xc4, 0x5e, + 0x20, 0x99, 0xc0, 0x63, 0x7f, 0xd8, 0x0b, 0xac, 0x88, 0x05, 0xdd, 0x07, 0x90, 0x4c, 0x2d, 0xb3, + 0x59, 0xab, 0x72, 0x07, 0x49, 0x41, 0xb2, 0x20, 0x66, 0x6e, 0x89, 0x42, 0x62, 0x8f, 0x59, 0xcc, + 0x19, 0xcd, 0xe2, 0x49, 0x78, 0x4b, 0xa1, 0x2e, 0x3d, 0xa2, 0x15, 0x2e, 0x16, 0x4f, 0x87, 0xcb, + 0xf2, 0x0e, 0xf1, 0xc8, 0x04, 0xe2, 0x17, 0x0d, 0x1a, 0x27, 0x2c, 0x27, 0x36, 0x93, 0x60, 0xf9, + 0x56, 0x0a, 0xda, 0xd2, 0x37, 0x95, 0x8d, 0x20, 0xe1, 0xe0, 0x53, 0x1a, 0x07, 0xf0, 0x70, 0x90, + 0x36, 0x4a, 0x0f, 0x61, 0xd6, 0x76, 0xfc, 0x43, 0xdb, 0xd9, 0xef, 0xe1, 0x5d, 0x1f, 0x7b, 0x44, + 0x8d, 0xc8, 0xdf, 0x7e, 0x14, 0x9b, 0x87, 0x6d, 0x82, 0xdb, 0x71, 0x7c, 0xff, 0x53, 0xd7, 0xeb, + 0xf0, 0x62, 0x49, 0xd8, 0x2e, 0xfd, 0xbe, 0x41, 0x66, 0x49, 0xed, 0xa0, 0x36, 0x2c, 0x1a, 0x1d, + 0xdb, 0x2b, 0xf5, 0x9c, 0x74, 0xfc, 0xb8, 0x21, 0x3c, 0x0f, 0xca, 0xc8, 0xe7, 0x41, 0xab, 0x34, + 0x96, 0x50, 0x83, 0x7c, 0x09, 0x52, 0xfa, 0xa3, 0x14, 0x91, 0xe1, 0xfe, 0xb3, 0xee, 0x41, 0xe5, + 0xb9, 0xd3, 0x3f, 0xc0, 0xe8, 0x4e, 0x38, 0x3b, 0x7e, 0x2c, 0xb2, 0xa8, 0x26, 0x30, 0x14, 0x15, + 0xad, 0x20, 0xfb, 0x8e, 0x7b, 0x00, 0x8c, 0x5d, 0x4a, 0x7c, 0xae, 0x26, 0xeb, 0x25, 0x11, 0x8d, + 0x25, 0xd1, 0x23, 0x1b, 0xe6, 0xea, 0xfd, 0x6e, 0xd0, 0x75, 0x7a, 0x4f, 0xf0, 0xd1, 0x3e, 0xf6, + 0x44, 0x94, 0xf7, 0xe6, 0xa8, 0x1e, 0x36, 0x54, 0x72, 0x96, 0x8a, 0xc7, 0xfa, 0x58, 0x29, 0xc3, + 0xa2, 0x86, 0xec, 0x44, 0x47, 0x47, 0x6f, 0xc0, 0x6c, 0xeb, 0xf9, 0x30, 0xe8, 0xb8, 0x9f, 0xf6, + 0x99, 0xab, 0x24, 0x7b, 0x43, 0x7e, 0x84, 0x5b, 0x26, 0x9a, 0xa5, 0xbf, 0x9c, 0x82, 0xf9, 0x98, + 0x4b, 0xd0, 0xee, 0xee, 0x0d, 0x98, 0xdd, 0x74, 0xdd, 0xc0, 0x0f, 0x3c, 0x67, 0x30, 0xe8, 0xf6, + 0x0f, 0xe8, 0xa0, 0x39, 0x4b, 0x05, 0x12, 0xd3, 0xc0, 0x6b, 0x40, 0x74, 0x41, 0xd3, 0x74, 0x41, + 0x15, 0xd3, 0x20, 0xa1, 0x2d, 0x99, 0x96, 0xd9, 0xa4, 0x68, 0xa9, 0x78, 0xf8, 0x57, 0x1c, 0xb5, + 0x94, 0x96, 0xba, 0xfb, 0xef, 0xc7, 0xbe, 0x98, 0xc7, 0x7e, 0x97, 0x55, 0xc3, 0x20, 0x11, 0x58, + 0xb1, 0x15, 0x7a, 0x0c, 0x0b, 0xac, 0x50, 0x27, 0x55, 0xee, 0xb8, 0x65, 0x55, 0x42, 0xd0, 0x04, + 0x91, 0x95, 0xe4, 0x4b, 0x86, 0x36, 0xd3, 0x27, 0x0c, 0x6d, 0x1e, 0xc3, 0xc2, 0x23, 0xb7, 0xdb, + 0x67, 0x35, 0x68, 0x6e, 0xff, 0xb8, 0xa1, 0x55, 0x66, 0x93, 0x20, 0xb2, 0x92, 0x7c, 0x68, 0x0b, + 0x4c, 0xd6, 0x3b, 0x8d, 0x7d, 0xd8, 0x84, 0xf2, 0xc9, 0xd0, 0x36, 0x4e, 0x63, 0x25, 0xb8, 0xc8, + 0xf6, 0x96, 0x3b, 0x1d, 0xa1, 0x85, 0xba, 0x58, 0x51, 0x42, 0x5b, 0x32, 0x2d, 0xb1, 0xfc, 0xa1, + 0xa8, 0x30, 0xee, 0x42, 0xd2, 0xf2, 0xab, 0x14, 0x56, 0x8c, 0xa3, 0x84, 0xf5, 0xb1, 0x8f, 0x56, + 0x5e, 0x63, 0x92, 0x98, 0x3a, 0xbe, 0x24, 0x96, 0xfe, 0x30, 0x05, 0x17, 0x63, 0xe5, 0x3f, 0xdb, + 0xf1, 0x0e, 0x70, 0x40, 0x0f, 0xc1, 0xf8, 0x16, 0x91, 0x41, 0x98, 0xb5, 0xce, 0x5b, 0x0a, 0x8c, + 0x16, 0xef, 0x64, 0x9a, 0x14, 0xa3, 0x91, 0x61, 0xc4, 0xce, 0xd6, 0x5e, 0x12, 0x13, 0xd4, 0x0d, + 0xf8, 0xf9, 0x6a, 0xd8, 0x26, 0x21, 0x29, 0xef, 0x8f, 0xc4, 0x5a, 0xee, 0x30, 0xb0, 0xbb, 0xed, + 0x43, 0x9f, 0x5b, 0x47, 0x1d, 0x8a, 0x70, 0xd8, 0x1a, 0x0e, 0x66, 0x34, 0x75, 0x28, 0xf4, 0x36, + 0x2c, 0xd7, 0x88, 0xb9, 0x70, 0x02, 0x5c, 0x19, 0x7a, 0x1e, 0xee, 0x07, 0x94, 0xc6, 0xe7, 0x67, + 0x12, 0x7a, 0x64, 0xe9, 0x96, 0x46, 0x2a, 0xd9, 0xa7, 0x74, 0x7d, 0x7a, 0x54, 0xcc, 0x96, 0x23, + 0x6c, 0x97, 0x7a, 0x1a, 0xa5, 0x42, 0x77, 0x20, 0x43, 0xfc, 0x0d, 0xb7, 0xd2, 0x8a, 0x4e, 0x28, + 0x8e, 0x8a, 0xdb, 0x6a, 0x4a, 0x4c, 0x17, 0xd5, 0xf1, 0x0f, 0xab, 0x4e, 0xe0, 0xec, 0x3b, 0xbe, + 0x30, 0x79, 0x0a, 0x8c, 0x58, 0x3d, 0x55, 0x8b, 0x46, 0x5b, 0xbd, 0x37, 0x93, 0x2a, 0x31, 0x86, + 0xfa, 0x75, 0x45, 0xec, 0x47, 0x47, 0xb3, 0xa5, 0x5f, 0x19, 0x71, 0x29, 0x3f, 0xed, 0x29, 0x07, + 0x7a, 0x3a, 0xc2, 0xb7, 0x6c, 0x8c, 0xd6, 0x97, 0xe3, 0x78, 0x17, 0xa2, 0x2b, 0x64, 0x0f, 0xf9, + 0x39, 0x05, 0xfd, 0x7d, 0x1e, 0x1e, 0xe7, 0xb3, 0x94, 0x1a, 0x52, 0x86, 0x77, 0x36, 0x0c, 0xe9, + 0xce, 0xc6, 0x7b, 0xec, 0xf0, 0xd5, 0xe9, 0x77, 0xc4, 0xed, 0x91, 0x2b, 0x63, 0xf2, 0x15, 0x71, + 0x4a, 0x25, 0x58, 0xc8, 0x52, 0x8a, 0xf2, 0x3e, 0xcf, 0x0c, 0x44, 0x49, 0xbf, 0x02, 0xc0, 0xa9, + 0x88, 0xc2, 0x65, 0x68, 0xd7, 0xd7, 0xc6, 0x74, 0x5d, 0xaf, 0x8a, 0xfa, 0x43, 0xc4, 0x36, 0x32, + 0x07, 0x9a, 0x3a, 0x65, 0x0e, 0xf4, 0xef, 0xd3, 0x24, 0xb3, 0x66, 0x32, 0x44, 0xe2, 0x4e, 0x71, + 0xff, 0xc8, 0x90, 0xee, 0x1f, 0xfd, 0xdf, 0xba, 0x92, 0x50, 0x0e, 0xab, 0x5d, 0x39, 0xba, 0x05, + 0xdf, 0xd4, 0x94, 0x20, 0xe8, 0x3d, 0x9c, 0x63, 0x5e, 0x0e, 0xcd, 0x9f, 0xea, 0x72, 0x28, 0xe8, + 0x2f, 0x42, 0xa8, 0x87, 0xbc, 0x85, 0xe3, 0x5c, 0x71, 0x98, 0x99, 0x78, 0xc5, 0x61, 0xf6, 0x54, + 0x57, 0x1c, 0xe6, 0x4e, 0x75, 0xcd, 0x74, 0xfe, 0x38, 0xd7, 0x4c, 0xcd, 0xe3, 0x5d, 0x7d, 0x58, + 0x88, 0x5d, 0x7d, 0x98, 0x70, 0x26, 0x86, 0x4e, 0x7f, 0x91, 0x61, 0xf1, 0xdc, 0x2f, 0x32, 0x2c, + 0x9d, 0xf4, 0x22, 0xc3, 0xf2, 0x57, 0x79, 0x51, 0xf5, 0x9f, 0x0d, 0x76, 0x37, 0x9c, 0x5f, 0x94, + 0xe6, 0x6e, 0xd5, 0xd0, 0x5f, 0x94, 0x76, 0x02, 0xbc, 0xc1, 0x28, 0x14, 0x5d, 0x60, 0xa0, 0x91, + 0x4b, 0x9a, 0x3a, 0xdd, 0x92, 0xae, 0x58, 0x50, 0x90, 0x06, 0xd3, 0x7c, 0xf0, 0x5b, 0xea, 0x07, + 0x5f, 0x1a, 0xa1, 0xbe, 0xb2, 0xdd, 0xff, 0x9b, 0x0c, 0x3d, 0xd4, 0x3f, 0x17, 0x23, 0xf7, 0xeb, + 0x73, 0xf8, 0xaf, 0xf1, 0x39, 0xfc, 0x04, 0x0b, 0x32, 0x7b, 0xdc, 0x53, 0xf5, 0x3f, 0x36, 0xd8, + 0x45, 0xef, 0x89, 0xfa, 0x63, 0x4f, 0xd2, 0x9f, 0xb3, 0xc9, 0xbb, 0xad, 0x97, 0xf7, 0x7f, 0x4d, + 0x01, 0x48, 0x31, 0xa3, 0x2e, 0xf3, 0x10, 0x2a, 0x90, 0x92, 0x54, 0xe0, 0x06, 0xcc, 0x12, 0x73, + 0x81, 0xfb, 0xaa, 0x0b, 0x57, 0x81, 0x31, 0xa9, 0xc9, 0x1c, 0x5b, 0x6a, 0xce, 0x39, 0x72, 0x19, + 0x63, 0x7b, 0xb3, 0x63, 0x6d, 0x6f, 0x78, 0x9b, 0x7a, 0xfa, 0x18, 0xb7, 0xa9, 0x8f, 0x7f, 0x3f, + 0xf9, 0x2f, 0x8c, 0x70, 0xcd, 0x09, 0xe3, 0x07, 0x31, 0xa9, 0x28, 0x25, 0xca, 0xfd, 0x93, 0x04, + 0xe3, 0xc3, 0x49, 0x82, 0xf1, 0xa6, 0x2a, 0x18, 0x17, 0x35, 0x23, 0x90, 0x04, 0x59, 0x92, 0x8b, + 0x9f, 0xa6, 0xe2, 0x87, 0x11, 0xa3, 0xaa, 0x28, 0xaa, 0x1c, 0xa4, 0x26, 0xcb, 0x41, 0xfa, 0xcc, + 0x72, 0x90, 0x39, 0x77, 0x39, 0x98, 0x3a, 0xa9, 0x0f, 0xce, 0xea, 0xb7, 0xf6, 0x47, 0xe9, 0x78, + 0x41, 0x18, 0xbd, 0x03, 0x39, 0xae, 0x7b, 0x62, 0x83, 0x17, 0x35, 0x7a, 0x29, 0xe2, 0x24, 0x41, + 0x4a, 0xd8, 0x2a, 0x82, 0x2d, 0x95, 0x64, 0xab, 0xa8, 0x6c, 0x82, 0x14, 0xbd, 0x4b, 0xaf, 0x44, + 0x70, 0x3e, 0xe6, 0x76, 0x96, 0x74, 0x27, 0x8e, 0x9c, 0x31, 0x22, 0x46, 0xf7, 0xa1, 0x10, 0x89, + 0x82, 0x48, 0x3a, 0x46, 0x48, 0x8a, 0xa8, 0xc1, 0x4b, 0x0c, 0xa8, 0x2a, 0xb2, 0xd5, 0x0e, 0xef, + 0x81, 0x5d, 0xe0, 0x29, 0x26, 0x4b, 0x32, 0x1d, 0xb9, 0x0f, 0x95, 0x69, 0xe4, 0x96, 0x67, 0x4f, + 0x99, 0xb4, 0xfc, 0xae, 0x01, 0x05, 0xbe, 0x21, 0xd4, 0x9f, 0x7f, 0x87, 0xee, 0x06, 0xf3, 0xca, + 0x06, 0xf7, 0xca, 0x61, 0x18, 0xc4, 0x31, 0x4a, 0x71, 0x3a, 0x24, 0x47, 0xf7, 0xd8, 0xd2, 0x32, + 0xde, 0x14, 0xff, 0xb8, 0x28, 0x84, 0x12, 0x55, 0x22, 0x99, 0x39, 0x62, 0x28, 0xfd, 0x49, 0x06, + 0x96, 0x79, 0x52, 0x2a, 0x4a, 0x5b, 0xfc, 0xb0, 0xf4, 0x35, 0x98, 0x6b, 0x0e, 0x8f, 0xb6, 0x9f, + 0x45, 0x9d, 0xb3, 0x60, 0x23, 0x06, 0x25, 0xba, 0x46, 0x21, 0xe1, 0xfc, 0x99, 0x41, 0x56, 0x81, + 0x68, 0x1d, 0x4c, 0xc1, 0x17, 0x5e, 0xf0, 0x64, 0xd9, 0x65, 0x02, 0x4e, 0x62, 0xf1, 0x26, 0x7e, + 0x19, 0x84, 0xc7, 0x4f, 0xbc, 0x85, 0x6c, 0x28, 0xb0, 0x5f, 0x9b, 0xaf, 0x1e, 0x63, 0x71, 0x13, + 0xeb, 0xb6, 0xbc, 0x91, 0xda, 0x2f, 0xd9, 0x90, 0x98, 0x58, 0xb2, 0x2e, 0x77, 0x83, 0x9e, 0xe9, + 0x0e, 0x1a, 0xd9, 0x0b, 0x87, 0x77, 0x8f, 0xd1, 0x77, 0x9c, 0x35, 0xfe, 0xd4, 0x21, 0x3c, 0x8c, + 0xa4, 0xb1, 0xcd, 0x81, 0xa8, 0x66, 0xf2, 0x4b, 0x47, 0x22, 0xcb, 0x4b, 0x62, 0x56, 0xee, 0x83, + 0x19, 0x9f, 0xb8, 0xfe, 0xfa, 0xaf, 0xfe, 0x16, 0x92, 0xf2, 0x3a, 0x42, 0x99, 0xdc, 0xa4, 0x5e, + 0x94, 0x82, 0xc3, 0x7f, 0x1a, 0x70, 0xd9, 0xc2, 0x3e, 0xab, 0xd0, 0x7c, 0xe4, 0x04, 0xd8, 0x3b, + 0x72, 0xbc, 0x43, 0x21, 0x23, 0xd1, 0x4e, 0x19, 0xca, 0x4e, 0x7d, 0x4f, 0xdd, 0x29, 0x26, 0x95, + 0x77, 0x63, 0x89, 0x98, 0xbe, 0xcf, 0x09, 0xbb, 0xa5, 0x5f, 0xc5, 0xf4, 0x7f, 0xd7, 0x2a, 0x96, + 0xfe, 0x8e, 0x3f, 0xda, 0x19, 0x1b, 0x79, 0xff, 0xfa, 0x96, 0xf4, 0xd7, 0xef, 0x96, 0xf4, 0x4f, + 0xc5, 0x7b, 0x37, 0x12, 0xeb, 0xdc, 0x0f, 0x53, 0x22, 0x66, 0x7c, 0xd7, 0x12, 0xbe, 0x89, 0x46, + 0x3a, 0x94, 0x44, 0x8d, 0x74, 0x98, 0x75, 0xbb, 0x1f, 0xc6, 0x4a, 0xa9, 0x71, 0xfc, 0x23, 0x23, + 0xa5, 0x16, 0x14, 0xa4, 0xce, 0x35, 0x15, 0xc1, 0x0d, 0x35, 0x52, 0x1a, 0xf9, 0x14, 0x49, 0x36, + 0x00, 0xad, 0x49, 0xe1, 0xd7, 0xa4, 0x4e, 0x75, 0x81, 0xf9, 0x3f, 0xe6, 0xd4, 0x63, 0x59, 0xad, + 0x3e, 0xbc, 0xaf, 0x38, 0x37, 0x6d, 0x9a, 0x1b, 0xa1, 0x85, 0xd3, 0x96, 0xdd, 0xe1, 0x9d, 0x30, + 0x39, 0xe1, 0x61, 0xd9, 0xa2, 0x26, 0x25, 0x11, 0x87, 0x8c, 0x22, 0x8d, 0xb9, 0x1b, 0x6d, 0x68, + 0x18, 0x8a, 0x69, 0xb6, 0x21, 0x92, 0x37, 0xbe, 0xf9, 0x77, 0xc2, 0x4a, 0x02, 0x8f, 0xe4, 0x17, + 0x35, 0xf5, 0x03, 0x31, 0x98, 0xa8, 0x39, 0xdc, 0x12, 0x31, 0x38, 0x2b, 0xed, 0x29, 0xe5, 0x75, + 0x71, 0x81, 0x40, 0x89, 0xc3, 0x9b, 0x5c, 0xeb, 0x78, 0x85, 0xd4, 0x93, 0x23, 0xf8, 0xd5, 0x78, + 0x71, 0x5e, 0xa5, 0xb2, 0x34, 0x9c, 0xa8, 0x16, 0x3b, 0x6f, 0xe6, 0x2a, 0x36, 0xb1, 0xce, 0x1f, + 0x3b, 0xa5, 0x16, 0x16, 0xbc, 0xc3, 0xef, 0xfd, 0xf2, 0x16, 0x7a, 0xac, 0x5a, 0x70, 0xa0, 0x62, + 0xfd, 0xc6, 0xa8, 0xc3, 0xf7, 0x09, 0x46, 0xfb, 0x9e, 0x9c, 0x58, 0xf0, 0x03, 0xa9, 0x8b, 0xfa, + 0x74, 0x42, 0x14, 0x8c, 0xa5, 0x44, 0x44, 0x2e, 0xe9, 0xcd, 0x9c, 0xec, 0xd5, 0x92, 0xee, 0xce, + 0xd1, 0xec, 0xe8, 0x3b, 0x47, 0x5b, 0xba, 0x50, 0x60, 0x6e, 0xa2, 0xe9, 0xd2, 0x38, 0xfb, 0x7b, + 0x70, 0x39, 0xe9, 0x8c, 0x76, 0x70, 0xbf, 0xd3, 0xed, 0x1f, 0xd0, 0x0a, 0x63, 0xce, 0x1a, 0x4d, + 0x80, 0x36, 0xe1, 0xaa, 0xe2, 0x18, 0xa9, 0xab, 0x94, 0xd2, 0x02, 0xf6, 0x54, 0x6a, 0x2c, 0x0d, + 0xba, 0x0f, 0x2b, 0x9a, 0x01, 0x3c, 0x3c, 0x70, 0xbc, 0xf0, 0xc9, 0xd4, 0x18, 0x0a, 0xf4, 0x01, + 0x5c, 0x49, 0x62, 0xc3, 0x9b, 0xe0, 0xa2, 0x54, 0x39, 0x86, 0xe4, 0xcc, 0xae, 0xf7, 0x47, 0x06, + 0xcc, 0xc8, 0x51, 0xb8, 0x36, 0xd3, 0xbb, 0x0a, 0x79, 0x76, 0xec, 0x26, 0x4e, 0x1f, 0xf3, 0x56, + 0x04, 0x40, 0x45, 0x98, 0x56, 0xfd, 0xad, 0x68, 0x4a, 0xf5, 0xde, 0x8c, 0x52, 0xef, 0x5d, 0x81, + 0x5c, 0xd5, 0xfd, 0xb4, 0x4f, 0x31, 0x53, 0x14, 0x13, 0xb6, 0x4b, 0x7f, 0x85, 0xc0, 0x14, 0xba, + 0x1d, 0xbe, 0x48, 0x08, 0xdf, 0x1f, 0x18, 0xf2, 0xfb, 0x03, 0x5d, 0x71, 0x22, 0x0a, 0x96, 0xd2, + 0x4a, 0xb0, 0xb4, 0xad, 0xaa, 0x1a, 0xcb, 0x70, 0xde, 0xd2, 0x19, 0x94, 0xf0, 0xa1, 0xc1, 0x78, + 0x75, 0xd3, 0xbd, 0xe3, 0xfd, 0x1f, 0xb7, 0x57, 0x1d, 0x30, 0x63, 0xa7, 0x43, 0xe2, 0xf8, 0xe2, + 0xf6, 0xd8, 0x4f, 0x8d, 0x33, 0xc9, 0xee, 0x33, 0xd1, 0x23, 0xaa, 0xcb, 0xc9, 0x10, 0xfb, 0xb7, + 0x19, 0xdf, 0x1a, 0xdb, 0x7d, 0x48, 0xcd, 0xd6, 0x31, 0xe2, 0x96, 0xdd, 0x02, 0x1c, 0xdb, 0x2d, + 0x48, 0x8e, 0xab, 0x70, 0x2a, 0xc7, 0x35, 0x73, 0x02, 0xc7, 0x15, 0x73, 0xb3, 0xb3, 0x27, 0x76, + 0xb3, 0x09, 0x1f, 0x32, 0x77, 0x2a, 0x1f, 0xa2, 0x9a, 0xf7, 0xf9, 0x13, 0x9a, 0xf7, 0x44, 0x82, + 0x6e, 0x9e, 0x26, 0x41, 0x1f, 0x61, 0xec, 0x17, 0x4e, 0x68, 0xec, 0xd1, 0xb9, 0x1b, 0xfb, 0xc5, + 0xb3, 0x1a, 0xfb, 0xa5, 0x33, 0x1b, 0xfb, 0xe5, 0xb3, 0x1a, 0xfb, 0x8b, 0x13, 0x8d, 0x3d, 0xba, + 0x9b, 0xb8, 0xcb, 0x51, 0xeb, 0x13, 0x01, 0xe9, 0x14, 0x2f, 0x51, 0xe6, 0x11, 0x58, 0x4d, 0xb0, + 0xcf, 0x26, 0x45, 0x96, 0xae, 0xa8, 0x0d, 0xf6, 0x43, 0x3c, 0xfa, 0x01, 0x2c, 0xc5, 0x70, 0x16, + 0x76, 0x3a, 0xaf, 0x8a, 0x97, 0x93, 0xe9, 0x66, 0x42, 0xef, 0x75, 0x8c, 0xcc, 0x04, 0x68, 0xfb, + 0x44, 0x83, 0xc4, 0xf7, 0x55, 0x9a, 0x6c, 0xb4, 0x95, 0x64, 0xa9, 0x60, 0xd2, 0x68, 0x9c, 0x95, + 0x8d, 0x37, 0xa2, 0x5f, 0xcd, 0x88, 0x36, 0x1f, 0xf1, 0xca, 0xc9, 0x47, 0xb4, 0xc7, 0x8d, 0xc8, + 0x91, 0x68, 0x0b, 0xae, 0xc7, 0x30, 0xfc, 0xe0, 0xdf, 0x2f, 0xfb, 0x7e, 0xf7, 0xa0, 0x8f, 0x3b, + 0xc5, 0xab, 0xec, 0xfd, 0xcc, 0x04, 0x32, 0xd4, 0x80, 0x6f, 0xc4, 0xbf, 0x2a, 0xbc, 0x00, 0x10, + 0xf6, 0x75, 0x8d, 0xf6, 0x35, 0x99, 0xf0, 0xcc, 0x95, 0x90, 0x4f, 0x60, 0x59, 0xeb, 0x45, 0x4e, + 0x98, 0x12, 0x29, 0x57, 0x75, 0xa5, 0xee, 0xef, 0xd1, 0xff, 0x1f, 0x31, 0x22, 0x7f, 0x9b, 0x38, + 0xb9, 0x87, 0x70, 0x79, 0xa4, 0x2c, 0x4e, 0xea, 0x28, 0x27, 0x77, 0x54, 0x4f, 0x9c, 0x0c, 0xc9, + 0x62, 0x76, 0xc6, 0xae, 0xec, 0x53, 0x76, 0x55, 0xaa, 0xb3, 0x33, 0x31, 0xf1, 0x9f, 0x35, 0xce, + 0xf0, 0x7e, 0xb8, 0xf4, 0xa7, 0x29, 0x58, 0xd2, 0x5d, 0x3a, 0x1e, 0x73, 0xf7, 0x67, 0x27, 0xf1, + 0x7f, 0x54, 0x36, 0x26, 0x5d, 0x61, 0x56, 0xff, 0x9f, 0x4a, 0xa2, 0x78, 0x72, 0x2e, 0xff, 0x55, + 0x65, 0xc5, 0x9e, 0xfc, 0x6f, 0x4f, 0xc6, 0x1d, 0x9a, 0x49, 0x2b, 0x2a, 0xaf, 0xf5, 0x4f, 0x0c, + 0x80, 0x4d, 0xa7, 0x7d, 0x38, 0x1c, 0xd0, 0xfa, 0xcb, 0xa8, 0xe2, 0x5c, 0x5d, 0x57, 0x9c, 0x7b, + 0x5d, 0xb9, 0xef, 0x14, 0x76, 0x32, 0x3e, 0xd2, 0x3c, 0x73, 0x88, 0xff, 0xdb, 0x86, 0xa8, 0x2d, + 0xd5, 0x03, 0x7c, 0xa4, 0x7d, 0xca, 0x58, 0x82, 0x19, 0x7e, 0x3b, 0xee, 0xa9, 0x54, 0xa1, 0x54, + 0x60, 0x84, 0xa6, 0x8a, 0x9f, 0x39, 0xc3, 0x1e, 0xa7, 0x61, 0xb1, 0xbe, 0x02, 0x23, 0x5b, 0x54, + 0xef, 0x07, 0xd8, 0xeb, 0x3b, 0x3d, 0x5e, 0x54, 0x0b, 0xdb, 0xa5, 0x3f, 0x33, 0xe4, 0x12, 0x17, + 0x7a, 0x0f, 0xa6, 0x2b, 0x6e, 0x3f, 0xc0, 0xf4, 0xc5, 0x5f, 0xf2, 0x82, 0x4d, 0x48, 0xb8, 0xc1, + 0xa9, 0xd8, 0xc2, 0x08, 0x9e, 0x15, 0x8b, 0xde, 0xb0, 0x0d, 0x11, 0x27, 0x3c, 0xf7, 0x8a, 0x96, + 0x43, 0x5e, 0xa8, 0xdf, 0x8a, 0x6a, 0x69, 0xe8, 0x9d, 0xe8, 0xfe, 0x79, 0xe2, 0xb4, 0x56, 0x10, + 0x6d, 0x50, 0x0a, 0x36, 0x31, 0x46, 0xbd, 0xf2, 0x2e, 0x40, 0x04, 0x3c, 0x51, 0x0d, 0xf8, 0x75, + 0xe5, 0x19, 0xcd, 0x98, 0x7b, 0x79, 0x9f, 0xc0, 0x42, 0xe2, 0x06, 0x18, 0xba, 0x01, 0xb3, 0xec, + 0x65, 0xae, 0xb8, 0x54, 0xc6, 0x98, 0x54, 0x20, 0xdd, 0x66, 0xce, 0x22, 0xbd, 0xe6, 0x56, 0x60, + 0xeb, 0x9f, 0x02, 0xec, 0x0e, 0x3a, 0x4e, 0xc0, 0x72, 0xbb, 0x4b, 0xb0, 0xa8, 0xbc, 0xf4, 0x65, + 0x28, 0xf3, 0x02, 0x5a, 0x86, 0x05, 0xf1, 0x82, 0xbb, 0xd1, 0x6a, 0x72, 0xb0, 0x81, 0x16, 0x61, + 0x9e, 0x44, 0xab, 0xf4, 0xf3, 0x39, 0x30, 0x85, 0x66, 0x21, 0x6f, 0xb7, 0xb6, 0x79, 0x33, 0x4d, + 0x58, 0xc3, 0xd7, 0xd8, 0x21, 0x6b, 0x66, 0x7d, 0x03, 0xf2, 0xe1, 0xb9, 0x2a, 0x9a, 0x87, 0x42, + 0xd3, 0xf5, 0x8e, 0x9c, 0x1e, 0x6d, 0x9a, 0x17, 0x90, 0x09, 0x33, 0xfc, 0xd2, 0x27, 0x83, 0x18, + 0xeb, 0x9f, 0x4d, 0x01, 0x44, 0x2f, 0x56, 0xd0, 0x1c, 0x80, 0xdd, 0xda, 0xde, 0xdb, 0xdd, 0xa9, + 0x96, 0xed, 0x9a, 0x79, 0x01, 0x01, 0x64, 0xcb, 0x3b, 0x3b, 0xb5, 0x66, 0xd5, 0x34, 0x50, 0x0e, + 0x32, 0x56, 0xad, 0x5c, 0x35, 0x53, 0x68, 0x06, 0x72, 0xb6, 0xb5, 0xdb, 0xac, 0x10, 0x9a, 0x34, + 0xe9, 0xf4, 0x61, 0xcd, 0xde, 0x0b, 0x21, 0x19, 0x54, 0x80, 0xe9, 0xca, 0x76, 0xb3, 0x59, 0xab, + 0xd8, 0xe6, 0x14, 0xe9, 0x92, 0x37, 0xf6, 0xac, 0x6d, 0x33, 0x8b, 0x16, 0x60, 0xb6, 0xb1, 0xfd, + 0x70, 0x6f, 0xab, 0x56, 0xb6, 0xec, 0xcd, 0x5a, 0xd9, 0x36, 0xa7, 0x49, 0x0f, 0x95, 0xa6, 0x04, + 0xc9, 0xd1, 0x89, 0xca, 0x90, 0x3c, 0x42, 0x30, 0x57, 0xd9, 0xaa, 0x55, 0x1e, 0xef, 0x6d, 0x95, + 0x1f, 0xd7, 0x6a, 0x3b, 0x35, 0xcb, 0x04, 0xb2, 0xae, 0x64, 0xe4, 0x4a, 0x63, 0xb7, 0x65, 0xd7, + 0xac, 0xbd, 0x6a, 0xcd, 0x2e, 0xd7, 0x1b, 0x2d, 0xb3, 0x40, 0x88, 0x09, 0xa2, 0xb5, 0x55, 0xb6, + 0xaa, 0x7b, 0xf5, 0xe6, 0x83, 0x6d, 0x73, 0x86, 0x76, 0xd0, 0xdc, 0x2b, 0x37, 0x1a, 0xdb, 0x64, + 0x96, 0x7b, 0xf5, 0xaa, 0x39, 0x4b, 0x16, 0x51, 0xee, 0xa0, 0x65, 0x93, 0xf9, 0xcf, 0xd1, 0xf5, + 0xa7, 0x2b, 0xb0, 0x57, 0x69, 0xee, 0x35, 0xca, 0x9b, 0xb5, 0x86, 0x39, 0x8f, 0x8a, 0xb0, 0x14, + 0x01, 0x3f, 0xda, 0xb6, 0x1e, 0x73, 0x72, 0x93, 0xf4, 0xbc, 0x53, 0xb6, 0x2b, 0x5b, 0x04, 0xd1, + 0xb2, 0xb7, 0xad, 0x9a, 0xb9, 0x40, 0xba, 0xa8, 0xd6, 0x1a, 0x35, 0x46, 0xcd, 0x80, 0x88, 0x00, + 0x77, 0xac, 0xed, 0xef, 0x7d, 0x2c, 0x7d, 0xd8, 0x22, 0xfa, 0x06, 0x5c, 0xe3, 0xfd, 0x36, 0xb7, + 0x9b, 0x7b, 0x4f, 0xb7, 0xed, 0x7a, 0xf3, 0xe1, 0x9e, 0x55, 0xdb, 0x69, 0xd4, 0x2b, 0xe5, 0xbd, + 0xe6, 0xee, 0x13, 0x73, 0x09, 0xad, 0xc2, 0x4a, 0x92, 0x84, 0x7c, 0x47, 0xa3, 0x6e, 0x7f, 0x6c, + 0x2e, 0xa3, 0x25, 0x30, 0x5b, 0x35, 0x7b, 0xcf, 0xaa, 0x7d, 0xb8, 0x5b, 0xb7, 0x6a, 0xd5, 0xbd, + 0x46, 0xab, 0x69, 0x5e, 0x24, 0xd0, 0x87, 0x71, 0xe8, 0x25, 0xb1, 0x34, 0x8d, 0xb2, 0x5d, 0x6b, + 0xd9, 0x14, 0x56, 0x24, 0x5b, 0x42, 0x61, 0xb5, 0x72, 0xb5, 0x66, 0x91, 0x95, 0xb9, 0x4c, 0xb7, + 0x84, 0x2d, 0x77, 0xad, 0xdc, 0xb0, 0xb7, 0xcc, 0x15, 0xb2, 0xe9, 0x64, 0xfb, 0x29, 0xcb, 0x15, + 0x74, 0x19, 0x96, 0xf9, 0x94, 0x1a, 0xb5, 0x72, 0xab, 0xb6, 0xb5, 0xdd, 0xe0, 0xac, 0x57, 0x09, + 0x8a, 0x2e, 0x7e, 0x65, 0xab, 0x56, 0xdd, 0x6d, 0xd4, 0xf6, 0x2a, 0xdb, 0x4f, 0x9e, 0x94, 0x9b, + 0xd5, 0x96, 0x79, 0x0d, 0x7d, 0x13, 0xae, 0x73, 0xae, 0x87, 0x8d, 0xed, 0xcd, 0x72, 0x63, 0xaf, + 0xf5, 0x71, 0x6b, 0xef, 0x69, 0xd9, 0xa2, 0x34, 0x75, 0x7b, 0xcf, 0x6e, 0x99, 0xab, 0xeb, 0x4d, + 0x80, 0xe8, 0x71, 0x39, 0x11, 0x1f, 0xa2, 0x0b, 0x0c, 0x62, 0x5e, 0x20, 0xd3, 0x10, 0xc6, 0xd0, + 0x34, 0x88, 0x84, 0x53, 0xcd, 0x0a, 0xb5, 0x64, 0x81, 0xff, 0x37, 0x05, 0x0b, 0xff, 0x00, 0xb7, + 0x03, 0xdc, 0x31, 0xd3, 0xeb, 0xeb, 0x90, 0x0f, 0xdf, 0x2e, 0x13, 0xf6, 0x16, 0x0e, 0x68, 0xcb, + 0xbc, 0x40, 0xd8, 0x59, 0x6e, 0xc6, 0x00, 0xc6, 0xfa, 0x5f, 0x67, 0x00, 0x89, 0x00, 0x55, 0x52, + 0x60, 0xa2, 0x16, 0xdd, 0xf6, 0xa1, 0xac, 0xb7, 0xd2, 0x23, 0xd1, 0x50, 0x6f, 0x89, 0x3a, 0x27, + 0xc0, 0x29, 0x74, 0x91, 0x1e, 0x04, 0xc5, 0xe1, 0x69, 0x32, 0xfa, 0x43, 0x1c, 0x84, 0xe6, 0x20, + 0x43, 0x56, 0x2e, 0x66, 0x94, 0x38, 0x6a, 0x8a, 0x6c, 0x5b, 0x0b, 0x33, 0xad, 0xe5, 0xb0, 0x2c, + 0x91, 0x48, 0xf5, 0xa4, 0x8f, 0x63, 0xa6, 0xd1, 0x75, 0xb8, 0xd2, 0xc2, 0x41, 0xb2, 0xb4, 0xc1, + 0x09, 0x72, 0x68, 0x05, 0x2e, 0x72, 0x82, 0x30, 0x37, 0xe6, 0xb8, 0x3c, 0x59, 0x42, 0xf6, 0x9b, + 0xaf, 0x9a, 0x09, 0xe4, 0xc3, 0x04, 0x28, 0xbc, 0x65, 0x69, 0x16, 0x88, 0x90, 0xec, 0x10, 0xa3, + 0xc8, 0xcf, 0xce, 0xcd, 0x19, 0xc2, 0x6b, 0xe1, 0x23, 0xf7, 0x85, 0xb8, 0x70, 0x6d, 0xce, 0x92, + 0x59, 0xaa, 0xd7, 0x20, 0xf8, 0x40, 0x73, 0xe8, 0x1a, 0x5c, 0x66, 0xbf, 0x35, 0x39, 0xaf, 0x39, + 0x8f, 0xae, 0xc0, 0xa5, 0x18, 0x5a, 0xb8, 0x0c, 0xa6, 0x73, 0x22, 0x92, 0xe5, 0xfd, 0x2d, 0xa0, + 0xab, 0x50, 0x4c, 0x9e, 0xd5, 0x71, 0x2c, 0x42, 0x37, 0x60, 0x4d, 0xe4, 0x80, 0xc9, 0xec, 0x90, + 0x53, 0x2d, 0x92, 0x95, 0x63, 0xf9, 0x5e, 0x2c, 0xaa, 0xe4, 0x04, 0x4c, 0x17, 0xe9, 0x6f, 0xdd, + 0x61, 0xb9, 0xb9, 0xbc, 0xfe, 0x63, 0x03, 0x66, 0x95, 0x2a, 0x15, 0xd1, 0x7a, 0x01, 0xe0, 0xe7, + 0x54, 0xe6, 0x05, 0xb2, 0xd5, 0x02, 0xa8, 0xbc, 0x97, 0x31, 0x0d, 0xf4, 0xff, 0xe0, 0x1b, 0x09, + 0x94, 0x48, 0x06, 0x2c, 0xdc, 0xc6, 0xdd, 0x17, 0xb8, 0x63, 0xa6, 0xc8, 0xf2, 0x24, 0xc8, 0x1e, + 0x38, 0xdd, 0x1e, 0x91, 0x79, 0x79, 0x4c, 0x6b, 0xd8, 0xef, 0x93, 0x8e, 0x33, 0xeb, 0xfb, 0xba, + 0x3a, 0x19, 0xd9, 0x1f, 0x05, 0x1a, 0xcd, 0x31, 0x8e, 0x11, 0x3d, 0x19, 0x09, 0x4c, 0x2b, 0x70, + 0x07, 0x03, 0x32, 0xab, 0xf5, 0xbf, 0x35, 0xc0, 0x8c, 0xbf, 0x90, 0x22, 0xea, 0x53, 0xee, 0x88, + 0x7f, 0x82, 0x60, 0x5e, 0x88, 0xa4, 0x44, 0x80, 0x0c, 0x22, 0x4a, 0xad, 0xc0, 0xf1, 0x02, 0x01, + 0x49, 0x11, 0xed, 0x20, 0xdd, 0x0a, 0x40, 0x9a, 0xf4, 0xf2, 0xb8, 0xdb, 0xeb, 0x7d, 0xdf, 0x3d, + 0xda, 0xef, 0x12, 0x6d, 0xb9, 0x04, 0x8b, 0xe5, 0x4e, 0x27, 0x2e, 0x3b, 0xe6, 0x14, 0x11, 0x6e, + 0xd6, 0x7d, 0x02, 0x97, 0xa5, 0x2a, 0x46, 0xc6, 0x49, 0xa0, 0xa6, 0xc9, 0x47, 0x91, 0x01, 0x13, + 0x98, 0xdc, 0xfa, 0x13, 0xe5, 0xe5, 0x08, 0x99, 0x48, 0x24, 0x41, 0xe6, 0x05, 0xea, 0x99, 0x9b, + 0xa2, 0x69, 0x90, 0x66, 0x25, 0x6c, 0xa6, 0xa8, 0x92, 0xd0, 0x12, 0x12, 0x87, 0xa4, 0x37, 0xeb, + 0x9f, 0x7f, 0xb1, 0x7a, 0xe1, 0x67, 0x5f, 0xae, 0x1a, 0x9f, 0x7f, 0xb9, 0x6a, 0xfc, 0xf2, 0xcb, + 0xd5, 0x0b, 0x7f, 0xfe, 0x0f, 0xab, 0xc6, 0xf7, 0xef, 0x48, 0xff, 0xb3, 0xf8, 0xc8, 0x09, 0xbc, + 0xee, 0x4b, 0x97, 0x86, 0x1d, 0xa2, 0xd1, 0xc7, 0xb7, 0x06, 0x87, 0x07, 0xb7, 0x06, 0xfb, 0xb7, + 0xa2, 0x20, 0x6a, 0x3f, 0x4b, 0xff, 0x63, 0xc5, 0x9d, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0xb2, + 0x99, 0x52, 0x58, 0x0f, 0x59, 0x00, 0x00, } func (m *CNStore) Marshal() (dAtA []byte, err error) { @@ -6268,6 +6476,34 @@ func (m *CNStore) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x98 + } + if len(m.GlobalSysVarGeneration) > 0 { + i -= len(m.GlobalSysVarGeneration) + copy(dAtA[i:], m.GlobalSysVarGeneration) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.GlobalSysVarGeneration))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x8a if len(m.CommitID) > 0 { i -= len(m.CommitID) copy(dAtA[i:], m.CommitID) @@ -6532,6 +6768,11 @@ func (m *LogStore) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) + i-- + dAtA[i] = 0x40 + } { size, err := m.Locality.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -6800,6 +7041,34 @@ func (m *CNStoreHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa0 + } + if len(m.GlobalSysVarGeneration) > 0 { + i -= len(m.GlobalSysVarGeneration) + copy(dAtA[i:], m.GlobalSysVarGeneration) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.GlobalSysVarGeneration))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x9a + } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 if m.CommandDeliveryAckSupported { i-- if m.CommandDeliveryAckSupported { @@ -7002,6 +7271,11 @@ func (m *LogStoreHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) + i-- + dAtA[i] = 0x50 + } if m.CommandDeliverySupported { i-- if m.CommandDeliverySupported { @@ -7458,12 +7732,12 @@ func (m *LogRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x40 } - n13, err13 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.TS, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.TS):]) - if err13 != nil { - return 0, err13 + n15, err15 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.TS, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.TS):]) + if err15 != nil { + return 0, err15 } - i -= n13 - i = encodeVarintLogservice(dAtA, i, uint64(n13)) + i -= n15 + i = encodeVarintLogservice(dAtA, i, uint64(n15)) i-- dAtA[i] = 0x3a if m.TNID != 0 { @@ -7722,6 +7996,18 @@ func (m *Request) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 if m.ScheduleCommandQuery != nil { { size, err := m.ScheduleCommandQuery.MarshalToSizedBuffer(dAtA[:i]) @@ -8918,6 +9204,16 @@ func (m *CommandBatch) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a if len(m.CommandIDs) > 0 { for iNdEx := len(m.CommandIDs) - 1; iNdEx >= 0; iNdEx-- { { @@ -8983,6 +9279,34 @@ func (m *CNStoreInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa8 + } + if len(m.GlobalSysVarGeneration) > 0 { + i -= len(m.GlobalSysVarGeneration) + copy(dAtA[i:], m.GlobalSysVarGeneration) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.GlobalSysVarGeneration))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa2 + } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x9a if m.CommandDeliveryAckSupported { i-- if m.CommandDeliveryAckSupported { @@ -9161,6 +9485,16 @@ func (m *CNState) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 if len(m.Stores) > 0 { for k := range m.Stores { v := m.Stores[k] @@ -9391,6 +9725,33 @@ func (m *ProxyStore) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) + i-- + dAtA[i] = 0x40 + } + if m.State != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.State)) + i-- + dAtA[i] = 0x38 + } + if len(m.GlobalSysVarGeneration) > 0 { + i -= len(m.GlobalSysVarGeneration) + copy(dAtA[i:], m.GlobalSysVarGeneration) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.GlobalSysVarGeneration))) + i-- + dAtA[i] = 0x32 + } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a if m.ConfigData != nil { { size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) @@ -9500,17 +9861,39 @@ func (m *ProxyHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ConfigData != nil { - { - size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintLogservice(dAtA, i, uint64(size)) - } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) i-- - dAtA[i] = 0x1a + dAtA[i] = 0x30 + } + if len(m.GlobalSysVarGeneration) > 0 { + i -= len(m.GlobalSysVarGeneration) + copy(dAtA[i:], m.GlobalSysVarGeneration) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.GlobalSysVarGeneration))) + i-- + dAtA[i] = 0x2a + } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a } if len(m.ListenAddress) > 0 { i -= len(m.ListenAddress) @@ -9553,6 +9936,16 @@ func (m *ClusterDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + { + size, err := m.GlobalSysVarCommitTS.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 if len(m.DeletedStores) > 0 { for iNdEx := len(m.DeletedStores) - 1; iNdEx >= 0; iNdEx-- { { @@ -9857,6 +10250,11 @@ func (m *LogStoreInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ProtocolVersion != 0 { + i = encodeVarintLogservice(dAtA, i, uint64(m.ProtocolVersion)) + i-- + dAtA[i] = 0x50 + } if m.CommandDeliverySupported { i-- if m.CommandDeliverySupported { @@ -11093,6 +11491,15 @@ func (m *CNStore) ProtoSize() (n int) { if l > 0 { n += 2 + l + sovLogservice(uint64(l)) } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 2 + l + sovLogservice(uint64(l)) + l = len(m.GlobalSysVarGeneration) + if l > 0 { + n += 2 + l + sovLogservice(uint64(l)) + } + if m.ProtocolVersion != 0 { + n += 2 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -11186,6 +11593,9 @@ func (m *LogStore) ProtoSize() (n int) { } l = m.Locality.ProtoSize() n += 1 + l + sovLogservice(uint64(l)) + if m.ProtocolVersion != 0 { + n += 1 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -11339,6 +11749,15 @@ func (m *CNStoreHeartbeat) ProtoSize() (n int) { if m.CommandDeliveryAckSupported { n += 3 } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 2 + l + sovLogservice(uint64(l)) + l = len(m.GlobalSysVarGeneration) + if l > 0 { + n += 2 + l + sovLogservice(uint64(l)) + } + if m.ProtocolVersion != 0 { + n += 2 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -11408,6 +11827,9 @@ func (m *LogStoreHeartbeat) ProtoSize() (n int) { if m.CommandDeliverySupported { n += 2 } + if m.ProtocolVersion != 0 { + n += 1 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -11770,6 +12192,8 @@ func (m *Request) ProtoSize() (n int) { l = m.ScheduleCommandQuery.ProtoSize() n += 2 + l + sovLogservice(uint64(l)) } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 2 + l + sovLogservice(uint64(l)) if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12236,6 +12660,8 @@ func (m *CommandBatch) ProtoSize() (n int) { n += 1 + l + sovLogservice(uint64(l)) } } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 1 + l + sovLogservice(uint64(l)) if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12312,6 +12738,15 @@ func (m *CNStoreInfo) ProtoSize() (n int) { if m.CommandDeliveryAckSupported { n += 3 } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 2 + l + sovLogservice(uint64(l)) + l = len(m.GlobalSysVarGeneration) + if l > 0 { + n += 2 + l + sovLogservice(uint64(l)) + } + if m.ProtocolVersion != 0 { + n += 2 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12333,6 +12768,8 @@ func (m *CNState) ProtoSize() (n int) { n += mapEntrySize + 1 + sovLogservice(uint64(mapEntrySize)) } } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 1 + l + sovLogservice(uint64(l)) if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12438,6 +12875,18 @@ func (m *ProxyStore) ProtoSize() (n int) { l = m.ConfigData.ProtoSize() n += 1 + l + sovLogservice(uint64(l)) } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 1 + l + sovLogservice(uint64(l)) + l = len(m.GlobalSysVarGeneration) + if l > 0 { + n += 1 + l + sovLogservice(uint64(l)) + } + if m.State != 0 { + n += 1 + sovLogservice(uint64(m.State)) + } + if m.ProtocolVersion != 0 { + n += 1 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12483,6 +12932,15 @@ func (m *ProxyHeartbeat) ProtoSize() (n int) { l = m.ConfigData.ProtoSize() n += 1 + l + sovLogservice(uint64(l)) } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 1 + l + sovLogservice(uint64(l)) + l = len(m.GlobalSysVarGeneration) + if l > 0 { + n += 1 + l + sovLogservice(uint64(l)) + } + if m.ProtocolVersion != 0 { + n += 1 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12525,6 +12983,8 @@ func (m *ClusterDetails) ProtoSize() (n int) { n += 1 + l + sovLogservice(uint64(l)) } } + l = m.GlobalSysVarCommitTS.ProtoSize() + n += 1 + l + sovLogservice(uint64(l)) if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12663,6 +13123,9 @@ func (m *LogStoreInfo) ProtoSize() (n int) { if m.CommandDeliverySupported { n += 2 } + if m.ProtocolVersion != 0 { + n += 1 + sovLogservice(uint64(m.ProtocolVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -13652,6 +14115,90 @@ func (m *CNStore) Unmarshal(dAtA []byte) error { } m.CommitID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 17: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarGeneration", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GlobalSysVarGeneration = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 19: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) + } + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -14279,6 +14826,25 @@ func (m *LogStore) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) + } + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -15380,6 +15946,90 @@ func (m *CNStoreHeartbeat) Unmarshal(dAtA []byte) error { } } m.CommandDeliveryAckSupported = bool(v != 0) + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 19: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarGeneration", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GlobalSysVarGeneration = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 20: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) + } + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -15836,19 +16486,38 @@ func (m *LogStoreHeartbeat) Unmarshal(dAtA []byte) error { } } m.CommandDeliverySupported = bool(v != 0) - default: - iNdEx = preIndex - skippy, err := skipLogservice(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthLogservice - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipLogservice(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthLogservice + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -18345,6 +19014,39 @@ func (m *Request) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -21198,6 +21900,39 @@ func (m *CommandBatch) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -21807,6 +22542,90 @@ func (m *CNStoreInfo) Unmarshal(dAtA []byte) error { } } m.CommandDeliveryAckSupported = bool(v != 0) + case 19: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 20: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarGeneration", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GlobalSysVarGeneration = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 21: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) + } + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -21987,6 +22806,39 @@ func (m *CNState) Unmarshal(dAtA []byte) error { } m.Stores[mapkey] = *mapvalue iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -22716,6 +23568,109 @@ func (m *ProxyStore) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarGeneration", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GlobalSysVarGeneration = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + m.State = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.State |= NodeState(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) + } + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -23047,6 +24002,90 @@ func (m *ProxyHeartbeat) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarGeneration", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GlobalSysVarGeneration = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) + } + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -23268,6 +24307,39 @@ func (m *ClusterDetails) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GlobalSysVarCommitTS", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.GlobalSysVarCommitTS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -24286,6 +25358,25 @@ func (m *LogStoreInfo) Unmarshal(dAtA []byte) error { } } m.CommandDeliverySupported = bool(v != 0) + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProtocolVersion", wireType) + } + m.ProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProtocolVersion |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) diff --git a/pkg/pb/logservice/logservice_test.go b/pkg/pb/logservice/logservice_test.go index ba264a35435b7..c1c9781ff4d3d 100644 --- a/pkg/pb/logservice/logservice_test.go +++ b/pkg/pb/logservice/logservice_test.go @@ -39,6 +39,7 @@ func TestCNStateUpdate(t *testing.T) { ServiceAddress: "addr-a", Role: metadata.CNRole_AP, CommandDeliveryAckSupported: true, + ProtocolVersion: 14, } tick1 := uint64(100) @@ -51,6 +52,7 @@ func TestCNStateUpdate(t *testing.T) { Labels: map[string]metadata.LabelList{}, UpTime: state.Stores[hb1.UUID].UpTime, CommandDeliveryAckSupported: true, + ProtocolVersion: 14, }) hb2 := CNStoreHeartbeat{UUID: "cn-b", ServiceAddress: "addr-b", Role: metadata.CNRole_TP} @@ -159,6 +161,7 @@ func TestLogStateUpdateStores(t *testing.T) { ServiceAddress: "addr-a", GossipAddress: "gossip-a", CommandDeliverySupported: true, + ProtocolVersion: 14, Replicas: []LogReplicaInfo{{ LogShardInfo: LogShardInfo{ ShardID: 1, @@ -179,6 +182,7 @@ func TestLogStateUpdateStores(t *testing.T) { GossipAddress: hb1.GossipAddress, Replicas: hb1.Replicas, CommandDeliverySupported: true, + ProtocolVersion: 14, }) hb2 := LogStoreHeartbeat{ @@ -628,16 +632,18 @@ func TestProxyStateUpdate(t *testing.T) { state := ProxyState{Stores: map[string]ProxyStore{}} hb1 := ProxyHeartbeat{ - UUID: "proxy-1", - ListenAddress: "addr-a", + UUID: "proxy-1", + ListenAddress: "addr-a", + ProtocolVersion: 14, } tick1 := uint64(100) state.Update(hb1, tick1) assert.Equal(t, state.Stores[hb1.UUID], ProxyStore{ - UUID: hb1.UUID, - Tick: tick1, - ListenAddress: hb1.ListenAddress, + UUID: hb1.UUID, + Tick: tick1, + ListenAddress: hb1.ListenAddress, + ProtocolVersion: 14, }) hb2 := ProxyHeartbeat{ diff --git a/pkg/proxy/config.go b/pkg/proxy/config.go index a237c1f22019a..7c8650dad6015 100644 --- a/pkg/proxy/config.go +++ b/pkg/proxy/config.go @@ -328,6 +328,13 @@ func (c *Config) FillDefault() { // Validate validates the configuration of proxy server. func (c *Config) Validate() error { noReport := errutil.ContextWithNoReport(context.Background(), true) + if c.HAKeeper.HeartbeatInterval.Duration+c.HAKeeper.HeartbeatTimeout.Duration > + logservice.GlobalSysVarHeartbeatProgressBudget { + return moerr.NewInternalErrorf(noReport, + "proxy hakeeper heartbeat cycle %s exceeds global-system-variable progress budget %s", + c.HAKeeper.HeartbeatInterval.Duration+c.HAKeeper.HeartbeatTimeout.Duration, + logservice.GlobalSysVarHeartbeatProgressBudget) + } if c.MaxConnections < 0 { return moerr.NewInternalError(noReport, "proxy max-connections must be positive") } diff --git a/pkg/proxy/config_test.go b/pkg/proxy/config_test.go index 1db90de91ea8d..f582bbcf5b23f 100644 --- a/pkg/proxy/config_test.go +++ b/pkg/proxy/config_test.go @@ -19,6 +19,7 @@ import ( "time" "github.com/matrixorigin/matrixone/pkg/frontend" + "github.com/matrixorigin/matrixone/pkg/logservice" "github.com/matrixorigin/matrixone/pkg/util/toml" "github.com/stretchr/testify/require" ) @@ -64,6 +65,16 @@ func TestValidate(t *testing.T) { }{{ name: "empty", cfg: Config{}, + }, { + name: "heartbeat cycle exceeds global sysvar progress budget", + cfg: func() Config { + var cfg Config + cfg.HAKeeper.HeartbeatInterval.Duration = time.Second + cfg.HAKeeper.HeartbeatTimeout.Duration = + logservice.GlobalSysVarHeartbeatProgressBudget + return cfg + }(), + wantErr: true, }, { name: "negative client handshake timeout", cfg: Config{ diff --git a/pkg/proxy/connection_limit.go b/pkg/proxy/connection_limit.go index 71ae5947d65e7..716a7990ce7b0 100644 --- a/pkg/proxy/connection_limit.go +++ b/pkg/proxy/connection_limit.go @@ -53,17 +53,24 @@ type connectionAdmissionListener struct { net.Listener limiter *connectionLimiter reject func(net.Conn) + admit func() bool } func newConnectionAdmissionListener( listener net.Listener, limiter *connectionLimiter, reject func(net.Conn), + admit ...func() bool, ) net.Listener { + var admission func() bool + if len(admit) > 0 { + admission = admit[0] + } return &connectionAdmissionListener{ Listener: listener, limiter: limiter, reject: reject, + admit: admission, } } @@ -73,6 +80,10 @@ func (l *connectionAdmissionListener) Accept() (net.Conn, error) { if err != nil { return nil, err } + if l.admit != nil && !l.admit() { + _ = conn.Close() + continue + } lease, ok := l.limiter.acquire() if ok { return &connectionAdmissionConn{Conn: conn, lease: lease}, nil diff --git a/pkg/proxy/connection_limit_test.go b/pkg/proxy/connection_limit_test.go index 0b40f6bbc0335..cbfdf30acf5e7 100644 --- a/pkg/proxy/connection_limit_test.go +++ b/pkg/proxy/connection_limit_test.go @@ -331,6 +331,31 @@ func TestConnectionAdmissionOwnershipTransfer(t *testing.T) { }) } +func TestConnectionAdmissionListenerRejectsInvalidServingLeaseBeforeLimiter(t *testing.T) { + limiter := newConnectionLimiter(1, 1) + proxySide, peerSide := net.Pipe() + defer peerSide.Close() + sentinel := errors.New("listener stopped") + step := 0 + raw := &scriptedAdmissionListener{accept: func() (net.Conn, error) { + step++ + if step == 1 { + return proxySide, nil + } + return nil, sentinel + }} + listener := newConnectionAdmissionListener(raw, limiter, nil, func() bool { return false }) + + conn, err := listener.Accept() + require.Nil(t, conn) + require.ErrorIs(t, err, sentinel) + require.Zero(t, limiter.total, + "a control-plane-rejected socket must not consume a connection slot") + buf := make([]byte, 1) + _, err = peerSide.Read(buf) + require.Error(t, err) +} + func TestRewriteProxyError(t *testing.T) { t.Run("connection limit", func(t *testing.T) { err := fmt.Errorf("wrapped: %w", errProxyConnectionLimit) diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index e9c4e054b65a3..e4df2b7e29e55 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -97,7 +97,12 @@ func newProxyHandler( ) // Create the MO cluster. - mc := clusterservice.NewMOCluster(cfg.UUID, haKeeperClient, cfg.Cluster.RefreshInterval.Duration) + mc := clusterservice.NewMOCluster( + cfg.UUID, + haKeeperClient, + cfg.Cluster.RefreshInterval.Duration, + clusterservice.WithGlobalSysVarRoutingFilter(), + ) rt.SetGlobalVariables(runtime.ClusterService, mc) // Create the rebalancer. diff --git a/pkg/proxy/heartbeat.go b/pkg/proxy/heartbeat.go index 23ed19d31c10e..6cbcbd22c3e2c 100644 --- a/pkg/proxy/heartbeat.go +++ b/pkg/proxy/heartbeat.go @@ -20,7 +20,9 @@ import ( "go.uber.org/zap" + "github.com/matrixorigin/matrixone/pkg/clusterservice" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/defines" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" ) @@ -48,14 +50,61 @@ func (s *Server) heartbeat(ctx context.Context) { func (s *Server) doHeartbeat(ctx context.Context) { ctx, cancel := context.WithTimeoutCause(ctx, s.config.HAKeeper.HeartbeatTimeout.Duration, moerr.CauseDoHeartbeat) defer cancel() - _, err := s.haKeeperClient.SendProxyHeartbeat(ctx, pb.ProxyHeartbeat{ - UUID: s.config.UUID, - ListenAddress: s.config.ListenAddress, - ConfigData: s.configData.GetData(), - }) - if err != nil { + if err := s.sendHeartbeat(ctx); err != nil { err = moerr.AttachCause(ctx, err) s.runtime.Logger().Error("failed to send heartbeat", zap.Error(err)) } +} + +func (s *Server) renewServingLease() { + duration := s.config.HAKeeper.HeartbeatInterval.Duration + + s.config.HAKeeper.HeartbeatTimeout.Duration + deadline := time.Now().Add(duration) + s.servingLeaseDeadline.Store(&deadline) +} + +func (s *Server) revokeServingLease() { + s.servingLeaseDeadline.Store(nil) +} + +func (s *Server) canAcceptNewConnections() bool { + deadline := s.servingLeaseDeadline.Load() + return deadline != nil && time.Now().Before(*deadline) +} + +func (s *Server) sendHeartbeat(ctx context.Context) error { + hb := pb.ProxyHeartbeat{ + UUID: s.config.UUID, + ListenAddress: s.config.ListenAddress, + ConfigData: s.configData.GetData(), + GlobalSysVarGeneration: s.globalSysVarGeneration, + ProtocolVersion: defines.MORPCLatestVersion, + } + if s.handler != nil { + hb.GlobalSysVarCommitTS = clusterservice.GlobalSysVarCommitTS(s.handler.moCluster) + } + _, err := s.haKeeperClient.SendProxyHeartbeat(ctx, hb) s.configData.DecrCount() + if err != nil { + s.revokeServingLease() + return err + } + if err := ctx.Err(); err != nil { + s.revokeServingLease() + return err + } + s.renewServingLease() + return nil +} + +func (s *Server) initializeGlobalSysVarRouteBarrier(ctx context.Context) error { + refresher, ok := s.handler.moCluster.(clusterservice.AuthoritativeRefresher) + if !ok { + return moerr.NewInternalError(ctx, + "proxy cluster service does not support authoritative refresh") + } + if err := refresher.Refresh(ctx); err != nil { + return err + } + return s.sendHeartbeat(ctx) } diff --git a/pkg/proxy/heartbeat_test.go b/pkg/proxy/heartbeat_test.go index 6627d266af56b..c327c0b22094a 100644 --- a/pkg/proxy/heartbeat_test.go +++ b/pkg/proxy/heartbeat_test.go @@ -16,15 +16,20 @@ package proxy import ( "context" + "sync" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/matrixorigin/matrixone/pkg/clusterservice" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/logservice" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/util" ) @@ -33,6 +38,13 @@ var _ logservice.ProxyHAKeeperClient = new(testHAClient) type testHAClient struct { } +type watermarkHAClient struct { + *testHAClient + sync.Mutex + details pb.ClusterDetails + heartbeat pb.ProxyHeartbeat +} + func (tclient *testHAClient) Close() error { //TODO implement me panic("implement me") @@ -97,6 +109,22 @@ func (tclient *testHAClient) SendProxyHeartbeat(ctx context.Context, hb pb.Proxy return pb.CommandBatch{}, moerr.NewInternalErrorNoCtx("return err") } +func (client *watermarkHAClient) GetClusterDetails(context.Context) (pb.ClusterDetails, error) { + client.Lock() + defer client.Unlock() + return client.details, nil +} + +func (client *watermarkHAClient) SendProxyHeartbeat( + _ context.Context, + hb pb.ProxyHeartbeat, +) (pb.CommandBatch, error) { + client.Lock() + defer client.Unlock() + client.heartbeat = hb + return pb.CommandBatch{}, nil +} + func TestServer_doHeartbeat(t *testing.T) { rt := runtime.DefaultRuntime() runtime.SetupServiceBasedRuntime("", rt) @@ -110,6 +138,71 @@ func TestServer_doHeartbeat(t *testing.T) { ser.doHeartbeat(ctx) } +func TestServerInitialRouteBarrierAcknowledgesPublishedWatermark(t *testing.T) { + rt := runtime.DefaultRuntime() + runtime.SetupServiceBasedRuntime("", rt) + commitTS := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + client := &watermarkHAClient{ + testHAClient: &testHAClient{}, + details: pb.ClusterDetails{GlobalSysVarCommitTS: commitTS}, + } + cluster := clusterservice.NewMOCluster("", client, time.Hour) + defer cluster.Close() + server := &Server{ + haKeeperClient: client, + configData: util.NewConfigData(nil), + runtime: runtime.ServiceRuntime(""), + globalSysVarGeneration: "proxy-generation", + handler: &handler{moCluster: cluster}, + } + server.config.UUID = "proxy-1" + server.config.HAKeeper.HeartbeatInterval.Duration = time.Second + server.config.HAKeeper.HeartbeatTimeout.Duration = time.Second + require.NoError(t, server.initializeGlobalSysVarRouteBarrier(context.Background())) + require.True(t, server.canAcceptNewConnections()) + + client.Lock() + hb := client.heartbeat + client.Unlock() + require.Equal(t, commitTS, hb.GlobalSysVarCommitTS) + require.Equal(t, "proxy-generation", hb.GlobalSysVarGeneration) + require.Equal(t, defines.MORPCLatestVersion, hb.ProtocolVersion) +} + +func TestProxyServingLeaseExpiresAndHeartbeatFailureRevokesIt(t *testing.T) { + server := &Server{ + haKeeperClient: &testHAClient{}, + configData: util.NewConfigData(nil), + runtime: runtime.ServiceRuntime(""), + } + server.config.HAKeeper.HeartbeatInterval.Duration = time.Second + server.config.HAKeeper.HeartbeatTimeout.Duration = time.Second + deadline := time.Now().Add(time.Minute) + server.servingLeaseDeadline.Store(&deadline) + require.True(t, server.canAcceptNewConnections()) + server.doHeartbeat(context.Background()) + require.False(t, server.canAcceptNewConnections(), + "a failed HAKeeper heartbeat must immediately fail-close Proxy admission") + + deadline = time.Now().Add(-time.Nanosecond) + server.servingLeaseDeadline.Store(&deadline) + require.False(t, server.canAcceptNewConnections()) +} + +func TestProxyHeartbeatDoesNotRenewLeaseAfterCallerCancellation(t *testing.T) { + client := &watermarkHAClient{testHAClient: &testHAClient{}} + server := &Server{ + haKeeperClient: client, + configData: util.NewConfigData(nil), + } + server.config.HAKeeper.HeartbeatInterval.Duration = time.Second + server.config.HAKeeper.HeartbeatTimeout.Duration = time.Second + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, server.sendHeartbeat(ctx), context.Canceled) + require.False(t, server.canAcceptNewConnections()) +} + func TestServer_NewServer(t *testing.T) { rt := runtime.DefaultRuntime() runtime.SetupServiceBasedRuntime("", rt) diff --git a/pkg/proxy/plugin.go b/pkg/proxy/plugin.go index d7475b3b39693..9c03984329ebf 100644 --- a/pkg/proxy/plugin.go +++ b/pkg/proxy/plugin.go @@ -96,20 +96,23 @@ func (r *pluginRouter) RouteForTransfer( if re.CN == nil { return nil, moerr.NewInternalErrorNoCtx("no CN server selected") } - if filter != nil && filter(re.CN.SQLAddress) { + resolver, ok := r.Router.(authoritativeRouteCandidateResolver) + if !ok { + if rr, ok := r.Router.(transferRouter); ok { + return rr.RouteForTransfer(ctx, sid, ci, filter) + } + return r.Router.Route(ctx, sid, ci, filter) + } + cn, eligible := resolver.resolveRouteCandidate( + sid, ci, filter, re.CN.ServiceID, re.CN.SQLAddress) + if !eligible { if rr, ok := r.Router.(transferRouter); ok { return rr.RouteForTransfer(ctx, sid, ci, filter) } return r.Router.Route(ctx, sid, ci, filter) } v2.ProxyConnectSelectCounter.Inc() - return &CNServer{ - reqLabel: ci.labelInfo, - cnLabel: re.CN.Labels, - uuid: re.CN.ServiceID, - addr: re.CN.SQLAddress, - hash: ci.hash, - }, nil + return cn, nil case plugin.Reject: v2.ProxyConnectRejectCounter.Inc() return nil, withCode(moerr.NewInfoNoCtx(re.Message), codeAuthFailed) @@ -149,18 +152,16 @@ func (r *pluginRouter) Route( if re.CN == nil { return nil, moerr.NewInternalErrorNoCtx("no CN server selected") } - // selected CN should be filtered out, fall back to the delegated router - if filter != nil && filter(re.CN.SQLAddress) { + resolver, ok := r.Router.(authoritativeRouteCandidateResolver) + if !ok { return r.Router.Route(ctx, sid, ci, filter) } - v2.ProxyConnectSelectCounter.Inc() - cn := &CNServer{ - reqLabel: ci.labelInfo, - cnLabel: re.CN.Labels, - uuid: re.CN.ServiceID, - addr: re.CN.SQLAddress, - hash: ci.hash, + cn, eligible := resolver.resolveRouteCandidate( + sid, ci, filter, re.CN.ServiceID, re.CN.SQLAddress) + if !eligible { + return r.Router.Route(ctx, sid, ci, filter) } + v2.ProxyConnectSelectCounter.Inc() // In plugin mode, a plugin-selected CN must still honor the same // breaker/probe gate as normal routing: a CN in active cooldown should // be skipped, an expired breaker should get at most one half-open diff --git a/pkg/proxy/plugin_test.go b/pkg/proxy/plugin_test.go index e260069cc7bc8..e180250e155ec 100644 --- a/pkg/proxy/plugin_test.go +++ b/pkg/proxy/plugin_test.go @@ -72,6 +72,19 @@ func (r *mockRouter) Route(ctx context.Context, sid string, ci clientInfo, f fun return nil, nil } +func (r *mockRouter) resolveRouteCandidate( + _ string, + _ clientInfo, + filter func(string) bool, + serviceID string, + address string, +) (*CNServer, bool) { + if filter != nil && filter(address) { + return nil, false + } + return &CNServer{uuid: serviceID, addr: address}, true +} + func (r *mockRouter) SelectByConnID(connID uint32) (*CNServer, error) { return nil, nil } @@ -396,6 +409,41 @@ func TestPluginRouter_SelectCooldownFallsBackToDelegatedRoute(t *testing.T) { require.Equal(t, "cn1", cn.uuid) } +func TestPluginRouterRejectsCandidateOutsideAuthoritativeSnapshot(t *testing.T) { + defer leaktest.AfterTest(t)() + + rt := runtime.DefaultRuntime() + runtime.SetupServiceBasedRuntime("", rt) + st := stopper.NewStopper("test-proxy", stopper.WithLogger(rt.Logger().RawLogger())) + defer st.Stop() + hc := &mockHAKeeperClient{} + hc.updateCN("fresh-cn", "8.8.8.8:6002", map[string]metadata.LabelList{ + tenantLabelKey: {Labels: []string{"t1"}}, + }) + mc := clusterservice.NewMOCluster("", hc, time.Hour) + defer mc.Close() + rt.SetGlobalVariables(runtime.ClusterService, mc) + mc.ForceRefresh(true) + + base := newRouter( + mc, testRebalancer(t, st, rt.Logger(), mc), newMockSQLWorker(), true, + ).(*router) + p := &mockPlugin{mockRecommendCNFn: func(context.Context, clientInfo) (*plugin.Recommendation, error) { + return &plugin.Recommendation{ + Action: plugin.Select, + CN: &metadata.CNService{ + ServiceID: "stale-cn", + SQLAddress: "8.8.8.8:6001", + }, + }, nil + }} + + cn, err := newPluginRouter("", base, p).Route( + context.Background(), "", clientInfo{labelInfo: labelInfo{Tenant: "t1"}}, nil) + require.NoError(t, err) + require.Equal(t, "fresh-cn", cn.uuid) +} + func TestRPCPlugin(t *testing.T) { defer leaktest.AfterTest(t)() diff --git a/pkg/proxy/router.go b/pkg/proxy/router.go index 38224afa83b2e..52bbd78009b89 100644 --- a/pkg/proxy/router.go +++ b/pkg/proxy/router.go @@ -137,6 +137,20 @@ type cacheReuseChecker interface { CanReuseCachedCN(cn *CNServer, client clientInfo) bool } +// authoritativeRouteCandidateResolver resolves an external recommendation +// against the router's current SQL-admission snapshot. A recommendation is a +// hint, not membership authority: labels, work state and the global-sysvar +// watermark must all come from MOCluster. +type authoritativeRouteCandidateResolver interface { + resolveRouteCandidate( + sid string, + client clientInfo, + filter func(string) bool, + serviceID string, + sqlAddress string, + ) (*CNServer, bool) +} + // RefreshableRouter is a router that can be refreshed to get latest route strategy type RefreshableRouter interface { Router @@ -333,6 +347,22 @@ func (r *router) routeCandidates(sid string, c clientInfo, filter func(string) b return cns } +func (r *router) resolveRouteCandidate( + sid string, + c clientInfo, + filter func(string) bool, + serviceID string, + sqlAddress string, +) (*CNServer, bool) { + for _, cn := range r.routeCandidates(sid, c, filter) { + if cn.uuid == serviceID && cn.addr == sqlAddress { + cn.hash = c.hash + return cn, true + } + } + return nil, false +} + // Route implements the Router interface. func (r *router) Route(ctx context.Context, sid string, c clientInfo, filter func(string) bool) (*CNServer, error) { cns := r.routeCandidates(sid, c, filter) diff --git a/pkg/proxy/server.go b/pkg/proxy/server.go index 334d1107d78b3..8af0ec72dc086 100644 --- a/pkg/proxy/server.go +++ b/pkg/proxy/server.go @@ -17,10 +17,12 @@ package proxy import ( "context" "net" + "sync/atomic" "time" "github.com/fagongzi/goetty/v2" "github.com/fagongzi/goetty/v2/codec" + "github.com/google/uuid" "go.uber.org/zap" "github.com/matrixorigin/matrixone/pkg/common/moerr" @@ -62,8 +64,10 @@ type Server struct { counterSet *counterSet haKeeperClient logservice.ProxyHAKeeperClient // configData will be sent to HAKeeper. - configData *util.ConfigData - test bool + configData *util.ConfigData + test bool + globalSysVarGeneration string + servingLeaseDeadline atomic.Pointer[time.Time] } // NewServer creates the proxy server. @@ -81,8 +85,9 @@ func NewServer(ctx context.Context, config Config, opts ...Option) (*Server, err opts = append(opts, WithConfigData(configKVMap)) s := &Server{ - config: config, - counterSet: newCounterSet(), + config: config, + counterSet: newCounterSet(), + globalSysVarGeneration: uuid.NewString(), } for _, opt := range opts { opt(s) @@ -126,11 +131,19 @@ func NewServer(ctx context.Context, config Config, opts ...Option) (*Server, err return nil, err } + s.handler = h + barrierCtx, barrierCancel := context.WithTimeoutCause( + ctx, s.config.HAKeeper.HeartbeatTimeout.Duration, moerr.CauseNewServer) + defer barrierCancel() + if err := s.initializeGlobalSysVarRouteBarrier(barrierCtx); err != nil { + _ = h.Close() + s.stopper.Stop() + stats.Unregister(statsFamilyName) + return nil, moerr.AttachCause(barrierCtx, err) + } if err := s.stopper.RunNamedTask("proxy heartbeat", s.heartbeat); err != nil { return nil, err } - - s.handler = h listener, err := newProxyListener(config.ListenAddress) if err != nil { return nil, err @@ -139,6 +152,7 @@ func NewServer(ctx context.Context, config Config, opts ...Option) (*Server, err listener, s.handler.connectionLimiter, s.handler.rejectBeforeSession, + s.canAcceptNewConnections, ) app, err := goetty.NewApplicationWithListeners([]net.Listener{listener}, nil, goetty.WithAppLogger(s.runtime.Logger().RawLogger()), diff --git a/pkg/txn/client/client.go b/pkg/txn/client/client.go index 46d9d0b50eaa9..9af3fbdc52c84 100644 --- a/pkg/txn/client/client.go +++ b/pkg/txn/client/client.go @@ -947,20 +947,28 @@ func (client *txnClient) GetLatestCommitTS() timestamp.Timestamp { } func (client *txnClient) SyncLatestCommitTS(ts timestamp.Timestamp) { - client.updateLastCommitTS(context.TODO(), nil, TxnEvent{Txn: txn.TxnMeta{CommitTS: ts}}, nil) + ctx, cancel := context.WithTimeoutCause(context.Background(), time.Minute*5, moerr.CauseSyncLatestCommitT) + defer cancel() + if err := client.SyncLatestCommitTSWithContext(ctx, ts); err != nil { + client.logger.Fatal("wait latest commit ts failed", zap.Error(err)) + } +} + +func (client *txnClient) SyncLatestCommitTSWithContext(ctx context.Context, ts timestamp.Timestamp) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + client.updateLastCommitTS(ctx, nil, TxnEvent{Txn: txn.TxnMeta{CommitTS: ts}}, nil) if client.timestampWaiter != nil { - ctx, cancel := context.WithTimeoutCause(context.Background(), time.Minute*5, moerr.CauseSyncLatestCommitT) - defer cancel() - for { - _, err := client.timestampWaiter.GetTimestamp(ctx, ts) - if err == nil { - break - } - err = moerr.AttachCause(ctx, err) - client.logger.Fatal("wait latest commit ts failed", zap.Error(err)) + if _, err := client.timestampWaiter.GetTimestamp(ctx, ts); err != nil { + return moerr.AttachCause(ctx, err) } } client.atomic.forceSyncCommitTimes.Add(1) + return nil } func (client *txnClient) GetSyncLatestCommitTSTimes() uint64 { diff --git a/pkg/txn/client/client_test.go b/pkg/txn/client/client_test.go index fe66a1f0c3a23..2b65589d781b0 100644 --- a/pkg/txn/client/client_test.go +++ b/pkg/txn/client/client_test.go @@ -1912,6 +1912,50 @@ func TestCloseCancelsAdmittedSnapshotWait(t *testing.T) { ) } +func TestSyncLatestCommitTSWithContextCancelsTimestampWait(t *testing.T) { + waiter := &blockingTimestampWaiter{entered: make(chan struct{}, 1)} + RunTxnTests( + func(tc TxnClient, _ rpc.TxnSender) { + ctx, cancel := context.WithCancel(context.Background()) + errC := make(chan error, 1) + ts := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + go func() { + errC <- tc.SyncLatestCommitTSWithContext(ctx, ts) + }() + + select { + case <-waiter.entered: + case <-time.After(time.Second): + t.Fatal("commit timestamp sync did not enter the logtail wait") + } + cancel() + require.ErrorIs(t, <-errC, context.Canceled) + require.Equal(t, ts, tc.GetLatestCommitTS()) + require.Zero(t, tc.GetSyncLatestCommitTSTimes()) + }, + WithTimestampWaiter(waiter), + ) +} + +func TestSyncLatestCommitTSWithContextWaitsForVisibility(t *testing.T) { + RunTxnTests( + func(tc TxnClient, _ rpc.TxnSender) { + ts := timestamp.Timestamp{PhysicalTime: 100, LogicalTime: 7} + require.NoError(t, tc.SyncLatestCommitTSWithContext(nil, ts)) + require.Equal(t, ts, tc.GetLatestCommitTS()) + require.Equal(t, uint64(1), tc.GetSyncLatestCommitTSTimes()) + + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, tc.SyncLatestCommitTSWithContext( + canceledCtx, timestamp.Timestamp{PhysicalTime: 200}), context.Canceled) + require.Equal(t, ts, tc.GetLatestCommitTS()) + require.Equal(t, uint64(1), tc.GetSyncLatestCommitTSTimes()) + }, + WithTimestampWaiter(immediateTimestampWaiter{}), + ) +} + func TestCloseCancelsRealTimestampWait(t *testing.T) { tw := NewTimestampWaiter(runtime.DefaultRuntime().Logger()).(*timestampWaiter) defer tw.Close() diff --git a/pkg/txn/client/types.go b/pkg/txn/client/types.go index 635c2748f1fa7..c54eb6295f66d 100644 --- a/pkg/txn/client/types.go +++ b/pkg/txn/client/types.go @@ -47,6 +47,9 @@ type TxnTimestampAware interface { GetLatestCommitTS() timestamp.Timestamp // SyncLatestCommitTS sync latest commit timestamp SyncLatestCommitTS(timestamp.Timestamp) + // SyncLatestCommitTSWithContext syncs the latest commit timestamp and waits + // for local logtail visibility until ctx is canceled. + SyncLatestCommitTSWithContext(context.Context, timestamp.Timestamp) error // GetSyncLatestCommitTSTimes returns times of sync latest commit ts GetSyncLatestCommitTSTimes() uint64 } diff --git a/proto/logservice.proto b/proto/logservice.proto index c8839563589f9..c7ae3fde458cf 100644 --- a/proto/logservice.proto +++ b/proto/logservice.proto @@ -22,6 +22,7 @@ option go_package = "github.com/matrixorigin/matrixone/pkg/pb/logservice"; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; import "google/protobuf/timestamp.proto"; import "metadata.proto"; +import "timestamp.proto"; option (gogoproto.goproto_enum_prefix_all) = false; option (gogoproto.sizer_all) = false; @@ -57,6 +58,9 @@ message CNStore { int64 UpTime = 14; string ShardServiceAddress = 15; string CommitID = 16; + timestamp.Timestamp GlobalSysVarCommitTS = 17 [(gogoproto.nullable) = false]; + string GlobalSysVarGeneration = 18; + int64 ProtocolVersion = 19; } message TNStore { @@ -88,6 +92,7 @@ message LogStore { repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false]; ConfigData ConfigData = 6; Locality Locality = 7 [(gogoproto.nullable) = false]; + int64 ProtocolVersion = 8; } // LogShardInfo contains information a log shard. @@ -149,6 +154,11 @@ message CNStoreHeartbeat { // service. It is meaningful only when CommandDeliveryAckSupported is true. uint64 AckedCommandBatchID = 16; bool CommandDeliveryAckSupported = 17; + // GlobalSysVarCommitTS is the latest global-system-variable watermark whose + // logtail has been applied locally. + timestamp.Timestamp GlobalSysVarCommitTS = 18 [(gogoproto.nullable) = false]; + string GlobalSysVarGeneration = 19; + int64 ProtocolVersion = 20; } // CNAllocateID is the periodic message sent tp the HAKeeper by CN stores. @@ -183,6 +193,7 @@ message LogStoreHeartbeat { // acknowledged delivery protocol only after every HAKeeper replica has // upgraded. bool CommandDeliverySupported = 9; + int64 ProtocolVersion = 10; }; // TNShardInfo contains information of a launched TN shard. @@ -273,6 +284,7 @@ enum MethodType { READ_LSN = 27; UPDATE_LEASEHOLDER_ID = 28; GET_SCHEDULE_COMMANDS = 29; + UPDATE_GLOBAL_SYS_VAR_COMMIT_TS = 30; }; enum RecordType { @@ -355,6 +367,7 @@ message Request { Locality NonVotingLocality = 15; CheckHealth CheckHealth = 16; ScheduleCommandQuery ScheduleCommandQuery = 17; + timestamp.Timestamp GlobalSysVarCommitTS = 18 [(gogoproto.nullable) = false]; }; message LogResponse { @@ -417,6 +430,7 @@ enum HAKeeperUpdateType { RestoreIDWatermarkUpdate = 18; CompleteLogServiceRecoveryUpdate = 19; EnableCommandDeliveryUpdate = 20; + UpdateGlobalSysVarCommitTS = 21; } // HAKeeperState state transition diagram @@ -607,6 +621,8 @@ message CommandBatch { // fingerprints. A non-empty acknowledged batch must contain one valid ID per // command before it is exposed to CN/TN services. repeated ScheduleCommandID CommandIDs = 4 [(gogoproto.nullable) = false]; + // GlobalSysVarCommitTS is the desired cluster-wide visibility watermark. + timestamp.Timestamp GlobalSysVarCommitTS = 5 [(gogoproto.nullable) = false]; } // CNStoreInfo contains information on a CN store. @@ -632,12 +648,17 @@ message CNStoreInfo { // non-destructive acknowledged schedule-command protocol. It is copied // from CNStoreHeartbeat so HAKeeper can gate activation and admission. bool CommandDeliveryAckSupported = 18; + timestamp.Timestamp GlobalSysVarCommitTS = 19 [(gogoproto.nullable) = false]; + string GlobalSysVarGeneration = 20; + int64 ProtocolVersion = 21; } // CNState contains all CN details known to the HAKeeper. message CNState { // Stores is keyed by CN store UUID. map Stores = 1 [(gogoproto.nullable) = false]; + // GlobalSysVarCommitTS is the durable admission watermark for routable CNs. + timestamp.Timestamp GlobalSysVarCommitTS = 2 [(gogoproto.nullable) = false]; } @@ -677,6 +698,10 @@ message ProxyStore { uint64 Tick = 2; string ListenAddress = 3; ConfigData ConfigData = 4; + timestamp.Timestamp GlobalSysVarCommitTS = 5 [(gogoproto.nullable) = false]; + string GlobalSysVarGeneration = 6; + NodeState State = 7; + int64 ProtocolVersion = 8; } message ProxyState { @@ -687,6 +712,9 @@ message ProxyHeartbeat { string UUID = 1; string ListenAddress = 2; ConfigData ConfigData = 3; + timestamp.Timestamp GlobalSysVarCommitTS = 4 [(gogoproto.nullable) = false]; + string GlobalSysVarGeneration = 5; + int64 ProtocolVersion = 6; }; message ClusterDetails { @@ -695,6 +723,7 @@ message ClusterDetails { repeated LogStore LogStores = 3 [(gogoproto.nullable) = false]; repeated ProxyStore ProxyStores = 4 [(gogoproto.nullable) = false]; repeated DeletedStore DeletedStores = 5 [(gogoproto.nullable) = false]; + timestamp.Timestamp GlobalSysVarCommitTS = 6 [(gogoproto.nullable) = false]; } // ClusterInfo provides a global view of all shards in the cluster. It @@ -733,6 +762,7 @@ message LogStoreInfo { ConfigData ConfigData = 7; Locality Locality = 8 [(gogoproto.nullable) = false]; bool CommandDeliverySupported = 9; + int64 ProtocolVersion = 10; } message LogState {