Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions api/v1alpha1/seinodetaskworkflow_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,12 @@ type SeiNodeTaskWorkflowKind string
const (
// SeiNodeTaskWorkflowKindStateSync re-bootstraps the target node through
// CometBFT state sync: mark-not-ready -> stop-seid -> reset-data ->
// config-patch -> configure-state-sync -> mark-ready ->
// await-condition(catchingUp). It is the paved road for STO-624.
// config-patch -> configure-state-sync -> mark-ready. The recipe ends at
// mark-ready, so Complete means every mutation was performed and the node
// was released to re-bootstrap, NOT that it caught up; catch-up is
// verified node-side (sdk WaitCaughtUp, RPC, alerts) by whoever triggered
// the workflow. The clean consequence: Failed always means the node is
// still held. It is the paved road for STO-624.
SeiNodeTaskWorkflowKindStateSync SeiNodeTaskWorkflowKind = "StateSync"
)

Expand Down
16 changes: 13 additions & 3 deletions internal/controller/node/envtest/workflow_lifecycle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,16 @@ func TestWorkflowLifecycle_AdoptAndComplete(t *testing.T) {
g.Expect(ready.Status).To(Equal(metav1.ConditionTrue))
g.Expect(w.Finalizers).NotTo(ContainElement(seiv1alpha1.SeiNodeTaskWorkflowFinalizer))

// Complete-at-release semantics pin: the recipe ends at mark-ready and
// carries no catch-up gate. Complete means the node was released, not
// that it caught up; catch-up verification is node-side (STO-624).
g.Expect(w.Status.Plan).NotTo(BeNil())
g.Expect(w.Status.Plan.Tasks).NotTo(BeEmpty())
g.Expect(w.Status.Plan.Tasks[len(w.Status.Plan.Tasks)-1].Type).To(Equal(sidecar.TaskTypeMarkReady))
for _, tk := range w.Status.Plan.Tasks {
g.Expect(tk.Type).NotTo(Equal(sidecar.TaskTypeAwaitCondition))
}

n := getNode(g, cli, "lc-complete")
g.Expect(n.Status.AdoptedWorkflow).To(BeNil())
wip := apimeta.FindStatusCondition(n.Status.Conditions, seiv1alpha1.ConditionWorkflowInProgress)
Expand Down Expand Up @@ -278,7 +288,7 @@ func TestWorkflowLifecycle_FailurePath(t *testing.T) {

func TestWorkflowLifecycle_ReadoptByUIDAfterRestart(t *testing.T) {
g := NewWithT(t)
fake := newFakeSidecar().stallAt(sidecar.TaskTypeAwaitCondition)
fake := newFakeSidecar().stallAt(sidecar.TaskTypeConfigureStateSync)
cli, cancel := startNodeManagerC(t, fake)

node := lifecycleNode("lc-restart")
Expand Down Expand Up @@ -343,7 +353,7 @@ func TestWorkflowLifecycle_QueueThenSequential(t *testing.T) {

func TestWorkflowLifecycle_FinalizerBlocksDeleteThenForce(t *testing.T) {
g := NewWithT(t)
fake := newFakeSidecar().stallAt(sidecar.TaskTypeAwaitCondition)
fake := newFakeSidecar().stallAt(sidecar.TaskTypeConfigureStateSync)
cli := startNodeManager(t, fake)

node := lifecycleNode("lc-final")
Expand Down Expand Up @@ -491,7 +501,7 @@ func forceDelete(cli client.Client, wf *seiv1alpha1.SeiNodeTaskWorkflow) {
// takes effect.
func TestWorkflowLifecycle_PauseFreezesForceDelete(t *testing.T) {
g := NewWithT(t)
fake := newFakeSidecar().stallAt(sidecar.TaskTypeAwaitCondition)
fake := newFakeSidecar().stallAt(sidecar.TaskTypeConfigureStateSync)
cli := startNodeManager(t, fake)

node := lifecycleNode("lc-pausedel")
Expand Down
4 changes: 2 additions & 2 deletions internal/controller/node/workflow_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func TestReconcileWorkflow_AdoptsOldestPending(t *testing.T) {
adopted := getWorkflow(t, c, "ss-0")
g.Expect(adopted.Finalizers).To(ContainElement(seiv1alpha1.SeiNodeTaskWorkflowFinalizer))
g.Expect(adopted.Status.Plan).NotTo(BeNil())
g.Expect(adopted.Status.Plan.Tasks).To(HaveLen(6)) // config-patch omitted (no Migration in fixture)
g.Expect(adopted.Status.Plan.Tasks).To(HaveLen(5)) // config-patch omitted (no Migration in fixture); mark-ready is terminal
g.Expect(adopted.Status.Phase).To(Equal(seiv1alpha1.SeiNodeTaskWorkflowPhaseRunning))
}

Expand Down Expand Up @@ -265,7 +265,7 @@ func TestReconcileWorkflow_ReadoptsByUIDAfterRestart(t *testing.T) {
// The interrupted plan is rebuilt and persisted deterministically.
resumed := getWorkflow(t, c, "ss-0")
g.Expect(resumed.Status.Plan).NotTo(BeNil())
g.Expect(resumed.Status.Plan.Tasks).To(HaveLen(6))
g.Expect(resumed.Status.Plan.Tasks).To(HaveLen(5))
// The deletion gate is (re-)ensured on the drive step, not just at adoption:
// an adoption interrupted before the finalizer add must not resume into
// destructive execution ungated (guards the 1a regression).
Expand Down
13 changes: 9 additions & 4 deletions internal/planner/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,13 @@ func (p *stateSyncWorkflowPlanner) Validate(node *seiv1alpha1.SeiNode, wf *seiv1
// BuildPlan compiles the StateSync progression:
//
// mark-not-ready -> stop-seid -> reset-data -> config-patch ->
// configure-state-sync -> mark-ready -> await-condition(catchingUp)
// configure-state-sync -> mark-ready
//
// The recipe ends at mark-ready: Complete means every mutation was performed
// and the node was released to re-bootstrap. Catch-up is verified node-side
// (sdk WaitCaughtUp, RPC, alerts) by whoever triggered the workflow. Because
// mark-ready is the terminal step, a Failed workflow always leaves the node
// held, and the adoption slot frees at release.
//
// TargetPhase and FailedPhase are left empty: a workflow never drives the
// node's phase (a failure parks the node held, it does not fail the node).
Expand Down Expand Up @@ -104,8 +110,8 @@ func (p *stateSyncWorkflowPlanner) BuildPlan(node *seiv1alpha1.SeiNode, wf *seiv
// gate and purges mark-ready records; stop-seid brings seid down; only then
// does reset-data run (it refuses if seid's RPC still answers). The release
// step's mark-ready always executes fresh because mark-not-ready purged the
// prior record. The final catchingUp await-condition carries no
// targetHeight (the live head is unknown at plan-build time).
// prior record, and it is the terminal step, so the workflow completes at
// release.
type step struct {
taskType string
params any
Expand All @@ -125,7 +131,6 @@ func (p *stateSyncWorkflowPlanner) BuildPlan(node *seiv1alpha1.SeiNode, wf *seiv
steps = append(steps,
step{TaskConfigureStateSync, cfgSS},
step{TaskMarkReady, sidecar.MarkReadyTask{}},
step{TaskAwaitCondition, sidecar.AwaitConditionTask{Condition: sidecar.ConditionCatchingUp}},
)

tasks := make([]seiv1alpha1.PlannedTask, 0, len(steps))
Expand Down
5 changes: 3 additions & 2 deletions internal/planner/workflow_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,10 @@ func TestStateSyncWorkflow_Progression(t *testing.T) {
TaskConfigPatch,
TaskConfigureStateSync,
TaskMarkReady,
TaskAwaitCondition,
}))
// mark-ready is the terminal step: Complete means the node was released,
// and catch-up is verified node-side, never by the recipe (STO-624).
g.Expect(gotTypes).NotTo(ContainElement(TaskAwaitCondition))
g.Expect(plan.Phase).To(Equal(seiv1alpha1.TaskPlanActive))

// A workflow never drives the node's phase.
Expand Down Expand Up @@ -119,7 +121,6 @@ func TestStateSyncWorkflow_OmitsConfigPatchWhenEmpty(t *testing.T) {
taskTypeResetData,
TaskConfigureStateSync,
TaskMarkReady,
TaskAwaitCondition,
}))
g.Expect(gotTypes).NotTo(ContainElement(TaskConfigPatch))
}
Expand Down
6 changes: 4 additions & 2 deletions sdk/sei/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,10 @@ type WorkflowSpec struct {

// StateSyncWorkflow is the payload for WorkflowStateSync — re-bootstrap the
// target through CometBFT state sync (quiesce -> wipe data/ -> re-configure ->
// resync -> await caught-up). The recipe is a paved road; its step ordering and
// guardrails live in the controller, not here.
// release). Complete means every mutation was performed and the node was
// released to re-bootstrap; the resync runs after Complete, so callers verify
// catch-up node-side (WaitCaughtUp). The recipe is a paved road; its step
// ordering and guardrails live in the controller, not here.
type StateSyncWorkflow struct {
// Migration, when set, runs a named seid config migration inside the
// re-bootstrap (the controller materializes it into the config-patch step).
Expand Down
13 changes: 8 additions & 5 deletions test/integration/workflow_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ import (
// memiavl baseline for the network in this suite, it reaches BOTH witnesses: it
// is applied to the network (so the genesis validator inherits it) and to the
// rpc follower via provision's storageConfig, so both witnesses serve chunks.
// Without it, the resync has nothing to sync FROM and the workflow's
// await-caught-up step never clears.
// Without it, the resync has nothing to sync FROM and the node-side
// WaitCaughtUp assertion after workflow Complete never clears (the workflow
// itself completes at mark-ready and does not gate on catch-up).
//
// Keys are the sei-config storage.* overrides that sei-config.SnapshotGenerationOverrides
// emits (storage.snapshot_interval / storage.snapshot_keep_recent, plus
Expand Down Expand Up @@ -316,9 +317,11 @@ func bringUpStateSyncFollower(ctx context.Context, t *testing.T, c *sei.Client,
}

// workflowWaitTimeout is the TIGHT child budget for one resync workflow's
// WaitTerminal — well under the 60m scenario ctx so a wedged recipe (a giga boot
// that refuses to start, or a step whose await carries no timeout) fails FAST and
// legibly with the recorded status instead of stalling to the scenario deadline.
// WaitTerminal — well under the 60m scenario ctx so a wedged recipe step fails
// FAST and legibly with the recorded status instead of stalling to the scenario
// deadline. The recipe ends at mark-ready (Complete == released, catch-up is
// asserted node-side below), so a healthy run completes in minutes and this
// budget bounds only the mutation steps, of which reset-data is the slowest.
const workflowWaitTimeout = 15 * time.Minute

// awaitWorkflowComplete blocks on wf.WaitTerminal under a tight child timeout and
Expand Down
Loading