diff --git a/test/integration/Dockerfile b/test/integration/Dockerfile index 98b848e..37b21f7 100644 --- a/test/integration/Dockerfile +++ b/test/integration/Dockerfile @@ -1,7 +1,7 @@ # The integration test suite compiled to an image: the build-tagged test binary, -# run by one in-cluster CronJob per target (args: -test.run TestX). The suites -# carry their fault/seiload templates via //go:embed, so the binary is -# self-contained (no extra files to COPY). +# run by one in-cluster CronJob as the whole nightly set (args: -test.run +# '^TestNightly'). The suites carry their fault/seiload templates via //go:embed, +# so the binary is self-contained (no extra files to COPY). FROM golang:1.26 AS builder ARG TARGETOS ARG TARGETARCH @@ -23,6 +23,6 @@ WORKDIR / COPY --from=builder /workspace/harness.test /harness.test USER 65532:65532 -# A CronJob selects a suite + budget via args, e.g. -# ["-test.run", "TestBenchmark", "-test.v", "-test.timeout", "0"] +# The CronJob passes the selection + budget via args, e.g. +# ["-test.run", "^TestNightly", "-test.v", "-test.timeout", "0"] ENTRYPOINT ["/harness.test"] diff --git a/test/integration/benchmark_test.go b/test/integration/benchmark_test.go index 80c6c5d..8cac232 100644 --- a/test/integration/benchmark_test.go +++ b/test/integration/benchmark_test.go @@ -10,14 +10,14 @@ import ( "time" ) -// TestBenchmark provisions a validator chain + RPC fleet, drives seiload against +// TestNightlyBenchmark provisions a validator chain + RPC fleet, drives seiload against // the fleet for the configured duration, and asserts the chain stayed live under // load. The load suite. // // Inputs (env, mirroring k8s_nightly.yml): // // SEI_CHAIN_ID base chain id (a per-run token is appended) [required] -// SEID_IMAGE seid image under test [required] +// SEID_IMAGE_MOCK seid image under test, mock_balances flavor [required] // SEILOAD_IMAGE sei-load benchmark image [required] // SEI_RUN_ID unique run id (sei.io/harness-run) [default: SEI_CHAIN_ID] // SEI_NAMESPACE shared nightly namespace [default: SDK default] @@ -30,7 +30,7 @@ import ( // so the scenario ctx below — not the test-runner alarm — must own the deadline, // nested inside the CronJob activeDeadlineSeconds (the SIGKILL backstop the // label-GC sweep covers). -func TestBenchmark(t *testing.T) { +func TestNightlyBenchmark(t *testing.T) { requireCluster(t) chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID")) @@ -38,7 +38,7 @@ func TestBenchmark(t *testing.T) { chainID: chainID, runID: envOr("SEI_RUN_ID", chainID), namespace: envOr("SEI_NAMESPACE", ""), - seidImage: mustEnv(t, "SEID_IMAGE"), + seidImage: mustEnv(t, "SEID_IMAGE_MOCK"), validators: 4, rpcNodes: 2, // seiload fans across both via the EVM endpoint list timeout: 90 * time.Minute, diff --git a/test/integration/chaos_test.go b/test/integration/chaos_test.go index 662d2eb..62b3cff 100644 --- a/test/integration/chaos_test.go +++ b/test/integration/chaos_test.go @@ -28,7 +28,7 @@ import ( // daemon on a freshly scaled node can legitimately take a while, so an expiry // here isn't proof of a stuck daemon, just a signal to stop waiting. Sized // well above steady-state inject latency (seconds) while staying below -// CHAOS_DURATION's default (3m) — see TestChaosSuite's injectWindow-vs-faultDur +// CHAOS_DURATION's default (3m) — see TestNightlyChaosSuite's injectWindow-vs-faultDur // guard, which exists so a late-but-successful injection still leaves real // time for the under-fault liveness check. const injectWindow = 90 * time.Second diff --git a/test/integration/chaossuite_test.go b/test/integration/chaossuite_test.go index 3337ee2..6fc5115 100644 --- a/test/integration/chaossuite_test.go +++ b/test/integration/chaossuite_test.go @@ -58,18 +58,18 @@ var chaosScenarios = []chaosScenario{ // in chaos_deferred_test.go for why and the re-add condition. } -// TestChaosSuite runs each fault against its own fresh chain: provision → inject +// TestNightlyChaosSuite runs each fault against its own fresh chain: provision → inject // the Chaos-Mesh fault → gate it injected → assert the chain stays live under it // (faults are bounded to f=1, so 2/3 quorum holds) → gate recovery → assert the // chain reconverged. Each fault is a subtest so one failure doesn't abort the // rest (matching the platform suite's continue-on-failure). // -// Inputs (env): SEI_CHAIN_ID (base), SEID_IMAGE [required]; SEI_NAMESPACE, -// CHAOS_DURATION [optional]. Run with -test.timeout 0 (see TestBenchmark). -func TestChaosSuite(t *testing.T) { +// Inputs (env): SEI_CHAIN_ID (base), SEID_IMAGE_CHAOS [required]; SEI_NAMESPACE, +// CHAOS_DURATION [optional]. Run with -test.timeout 0 (see TestNightlyBenchmark). +func TestNightlyChaosSuite(t *testing.T) { requireCluster(t) base := runChainID(mustEnv(t, "SEI_CHAIN_ID")) - seid := mustEnv(t, "SEID_IMAGE") + seid := mustEnv(t, "SEID_IMAGE_CHAOS") ns := envOr("SEI_NAMESPACE", "") duration := envOr("CHAOS_DURATION", "3m") faultDur, err := time.ParseDuration(duration) diff --git a/test/integration/genesis_test.go b/test/integration/genesis_test.go index db26b55..b58e6cb 100644 --- a/test/integration/genesis_test.go +++ b/test/integration/genesis_test.go @@ -60,7 +60,7 @@ import ( // appended) [default: genesis-ceremony]; SEI_NAMESPACE [default: SDK default]; // SEI_VALIDATORS (>= 2) [default: 2]; SEI_SIDECAR_IMAGE (pin the seictl sidecar // build under test) [default: platform default]. Run as the nightly CronJob with -// -test.timeout 0 (see TestBenchmark for why the scenario ctx, not the test-runner +// -test.timeout 0 (see TestNightlyBenchmark for why the scenario ctx, not the test-runner // alarm, owns the deadline): // // ["-test.run", "TestGenesisCeremonyProducesBlocks", "-test.v", "-test.timeout", "0"] diff --git a/test/integration/giga_migration_test.go b/test/integration/giga_migration_test.go index c213ad2..023790a 100644 --- a/test/integration/giga_migration_test.go +++ b/test/integration/giga_migration_test.go @@ -23,8 +23,8 @@ import ( // for the dependency boundary). const taskTypeConfigPatch = "config-patch" -// TestGigaStoreMigration drives the giga SS-store migration through the SDK end -// to end, as a sibling of TestWorkflowStateSync (it reuses that round trip's +// TestNightlyGigaStoreMigration drives the giga SS-store migration through the SDK end +// to end, as a sibling of TestNightlyWorkflowStateSync (it reuses that round trip's // provision + witness + follower bring-up verbatim via bringUpStateSyncFollower) // but with a ConfigMigration carried inside the StateSync recipe. Four signals, // in order: @@ -55,10 +55,10 @@ const taskTypeConfigPatch = "config-patch" // so a served EVM endpoint is strong (not absolute) evidence. // // Inputs (env): SEI_CHAIN_ID, SEID_IMAGE [required]; SEI_NAMESPACE, -// SEI_VALIDATORS [optional]. Run as the nightly CronJob: -// -// ["-test.run", "TestGigaStoreMigration", "-test.v", "-test.timeout", "0"] -func TestGigaStoreMigration(t *testing.T) { +// SEI_VALIDATORS [optional]. Selected into the nightly's combined run by its +// TestNightly prefix (harness_test.go); standalone: -test.run +// TestNightlyGigaStoreMigration. +func TestNightlyGigaStoreMigration(t *testing.T) { requireCluster(t) chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID")) seid := mustEnv(t, "SEID_IMAGE") @@ -70,7 +70,7 @@ func TestGigaStoreMigration(t *testing.T) { defer stop() c := openClient(ctx, t) - // Four validators, the harness convention — see TestWorkflowStateSync's + // Four validators, the harness convention — see TestNightlyWorkflowStateSync's // rationale (consensus-paced production keeps the witness follower at head). validators := envInt(t, "SEI_VALIDATORS", 4) @@ -143,7 +143,7 @@ func TestGigaStoreMigration(t *testing.T) { if err := sei.WaitEVMServing(ctx, f.hc, f.node.EVMRPC()); err != nil { t.Fatalf("follower %s EVM not serving under evm-ss-split=true after giga resync: %v (a node with an unpopulated EVM-SS layout refuses to boot, so a non-serving EVM here means giga did not go live)", f.node.Name(), err) } - t.Logf("follower %s: caught up + EVM serving under evm-ss-split=true, block-store base jumped %d -> %d (>= interval %d) — TestGigaStoreMigration OK", f.node.Name(), f.preEarliest, postEarliest, f.interval) + t.Logf("follower %s: caught up + EVM serving under evm-ss-split=true, block-store base jumped %d -> %d (>= interval %d) — TestNightlyGigaStoreMigration OK", f.node.Name(), f.preEarliest, postEarliest, f.interval) } // assertGigaConfigPatch is the anchor: it casts the workflow's raw CR, walks the diff --git a/test/integration/giga_mixed_release_test.go b/test/integration/giga_mixed_release_test.go index 7ae4d4a..c472d57 100644 --- a/test/integration/giga_mixed_release_test.go +++ b/test/integration/giga_mixed_release_test.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "net/http" - "net/url" "os/signal" "strconv" "strings" @@ -23,11 +22,11 @@ import ( "github.com/sei-protocol/sei-k8s-controller/sdk/sei" ) -// TestGigaMixedRelease proves giga_store_migration.md's central safety claim — +// TestNightlyGigaMixedRelease proves giga_store_migration.md's central safety claim — // "Giga SS Store is a per-node SS change that is invisible to the network" — // under real conformance load, not just a healthy-boot check. It provisions one // 4-validator chain with two plain RPC followers, migrates ONE follower to giga -// (evm-ss-split=true) via the same StateSync recipe TestGigaStoreMigration +// (evm-ss-split=true) via the same StateSync recipe TestNightlyGigaStoreMigration // drives, leaves the other at the shipped default (every node ships with // ss-enable=true already; evm-ss-split=false is the only thing distinguishing a // "giga" node from a plain one), then runs the external release-test conformance @@ -47,7 +46,7 @@ import ( // // Inputs (env): SEI_CHAIN_ID, SEID_IMAGE, RELEASE_TEST_IMAGE [required]; // SEI_NAMESPACE [optional]. Run with -test.timeout 0. -func TestGigaMixedRelease(t *testing.T) { +func TestNightlyGigaMixedRelease(t *testing.T) { requireCluster(t) chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID")) seid := mustEnv(t, "SEID_IMAGE") @@ -116,42 +115,18 @@ func TestGigaMixedRelease(t *testing.T) { t.Fatalf("network did not advance past the snapshot interval: %v", err) } - // Namespace for witness service hostnames, derived from the aggregate RPC - // host the same way TestWorkflowStateSync does (ns may be ""). - u, err := url.Parse(net.TendermintRPC()) - if err != nil { - t.Fatalf("parse network TM RPC %q: %v", net.TendermintRPC(), err) - } - hostLabels := strings.Split(u.Hostname(), ".") - if len(hostLabels) < 2 || hostLabels[1] == "" { - t.Fatalf("cannot derive namespace from aggregate RPC host %q", u.Host) - } - witnessNS := hostLabels[1] - // Migrate rpcNodes[1] to giga. Witnesses are explicit (validator-0 + the v2 // control node): a plain follower's ResolvedStateSyncers is populated only // for nodes created with an explicit StateSync spec, which neither follower // has here. A witness serves RPC trust points only; its own storage layout // is not consensus-relevant to serving them, so the v2 node is a valid // witness for the giga target's migration. - // - // Freshness gate: a witness must be at head to serve light blocks at the - // snapshot height the migration cross-checks; catching_up cannot certify that - // (a one-way latch), so assert it directly — same gate - // bringUpStateSyncFollower applies before its own bootstrap. - vHeight, vOK := sei.LatestHeight(ctx, hc, "http://"+nodeRPC(fmt.Sprintf("%s-0", chainID), witnessNS)) - wHeight, wOK := sei.LatestHeight(ctx, hc, "http://"+nodeRPC(v2Node.Name(), witnessNS)) - if !vOK || !wOK { - t.Fatalf("witness freshness read: validator ok=%v, v2 follower ok=%v", vOK, wOK) - } - if gap := vHeight - wHeight; gap > int64(interval) { - t.Fatalf("witness %s lags validator head by %d blocks (> interval %d): a diverging follower cannot serve state-sync light blocks", v2Node.Name(), gap, interval) - } - + witnessNS := witnessNamespace(t, net) witnesses := []string{ - nodeRPC(fmt.Sprintf("%s-0", chainID), witnessNS), - nodeRPC(v2Node.Name(), witnessNS), + nodeRPC(fmt.Sprintf("%s-0", chainID), witnessNS), // genesis validator-0 + nodeRPC(v2Node.Name(), witnessNS), // the v2 control node } + assertWitnessFresh(ctx, t, hc, witnesses[0], witnesses[1], interval) wf, err := c.CreateWorkflow(ctx, sei.WorkflowSpec{ Name: "giga-" + chainID, Namespace: ns, @@ -279,7 +254,7 @@ func TestGigaMixedRelease(t *testing.T) { if err := sei.WaitCaughtUp(ctx, hc, gigaNode.TendermintRPC()); err != nil { t.Errorf("post-release giga node %s not caught up: %v", gigaNode.Name(), err) } - t.Logf("release-test PASSED against both v2 and giga followers — parity confirmed, TestGigaMixedRelease OK") + t.Logf("release-test PASSED against both v2 and giga followers — parity confirmed, TestNightlyGigaMixedRelease OK") } // waitJobErr mirrors waitJob's polling logic but returns an error instead of diff --git a/test/integration/harness_test.go b/test/integration/harness_test.go index afdee03..e826be9 100644 --- a/test/integration/harness_test.go +++ b/test/integration/harness_test.go @@ -1,14 +1,22 @@ //go:build integration // Package integration holds the Sei nightly integration suites as plain `go test` -// targets (TestBenchmark, TestChaosSuite, TestChainUpgrade, TestRelease, -// TestWorkflowStateSync, TestGigaStoreMigration), selected with -run. Orchestration is statement order in -// one process; cross-step state is local Go values, not external config. +// targets. Every suite the nightly schedule runs is named TestNightly +// (TestNightlyBenchmark, TestNightlyChaosSuite, TestNightlyChainUpgrade, +// TestNightlyRelease, TestNightlyWorkflowStateSync, TestNightlyGigaStoreMigration, +// TestNightlyGigaMixedRelease); a new suite joins the nightly schedule by taking +// that name, with no other wiring required. TestGenesisCeremonyProducesBlocks is +// the one suite deliberately outside the prefix — an onboarding-validation check, +// run standalone, never nightly. Orchestration within one run is statement order +// in one process; cross-step state is local Go values, not external config. A +// suite whose required image can't be the shared SEID_IMAGE default (a different +// flavor, or a pinned version pair) reads its own env var name, since one process +// has one environment. // // Everything lives in *_test.go behind //go:build integration, so it never links // into a production binary and is excluded from the default `go test ./...`. The -// nightly compiles it once (go test -c -tags integration) and runs each target as -// an in-cluster CronJob (-test.run TestX). +// nightly compiles it once (go test -c -tags integration) and runs the whole set +// in one in-cluster CronJob (-test.run '^TestNightly'). // // Depends on sdk/sei (+ the k8s provider blank import), api/v1alpha1 (public API // types, for reading status.plan), and k8s.io/apimachinery/.../metav1 (for a @@ -21,8 +29,11 @@ import ( "fmt" "maps" "net/http" + "net/url" "os" "strconv" + "strings" + "sync/atomic" "testing" "time" @@ -116,6 +127,42 @@ func rpcNodeName(chainID string, ordinal int) string { return fmt.Sprintf("%s-rpc-%d", chainID, ordinal) } +// witnessNamespace derives the namespace for per-node witness service hostnames +// from a network's aggregate RPC host: -internal..svc[.cluster.local], +// and chainID carries no dots, so the second label is the namespace. The +// authoritative source is the served endpoint, not the spec namespace, which the +// SDK may resolve to a kubeconfig/SA default only the endpoint reflects. +func witnessNamespace(t *testing.T, network *sei.Network) string { + t.Helper() + u, err := url.Parse(network.TendermintRPC()) + if err != nil { + t.Fatalf("parse network TM RPC %q: %v", network.TendermintRPC(), err) + } + labels := strings.Split(u.Hostname(), ".") + if len(labels) < 2 || labels[1] == "" { + t.Fatalf("cannot derive namespace from aggregate RPC host %q", u.Host) + } + return labels[1] +} + +// assertWitnessFresh fails the suite unless the state-sync witness sits within one +// snapshot interval of the validator head. A witness must be at head to serve +// light blocks at the snapshot height a bootstrap cross-checks; catching_up cannot +// certify that (a one-way latch meaning "left blocksync once", never "at head +// now"), so freshness is asserted directly on the RPC heights. validatorRPC and +// witnessRPC are bare host:port (nodeRPC output). +func assertWitnessFresh(ctx context.Context, t *testing.T, hc *http.Client, validatorRPC, witnessRPC string, interval int) { + t.Helper() + vHeight, vOK := sei.LatestHeight(ctx, hc, "http://"+validatorRPC) + wHeight, wOK := sei.LatestHeight(ctx, hc, "http://"+witnessRPC) + if !vOK || !wOK { + t.Fatalf("witness freshness read: validator %s ok=%v, witness %s ok=%v", validatorRPC, vOK, witnessRPC, wOK) + } + if gap := vHeight - wHeight; gap > int64(interval) { + t.Fatalf("witness %s lags validator head by %d blocks (> interval %d): a diverging follower cannot serve state-sync light blocks", witnessRPC, gap, interval) + } +} + // provision stands up the genesis SeiNetwork + N standalone RPC SeiNodes via the // SDK in-process (not a seictl subprocess), waiting each follower // to Running + caught-up + EVM-serving before returning. The returned chain is @@ -256,14 +303,44 @@ func cleanupChain(t *testing.T, ch *chain) { }) } -// requireCluster skips a suite unless it is pointed at a real cluster. The SDK -// selects its k8s provider on SEI_NODE_CLUSTER presence; without it there is -// nothing to provision against. +// suitesStarted counts suites that began executing against a live cluster +// (incremented past requireCluster's skip). TestMain reads it to turn go test's +// silent exit-0-on-zero-match into a hard failure — see TestMain. +var suitesStarted atomic.Int64 + +// TestMain runs the suites, then guards the one failure mode go test hides: a +// -test.run pattern that matches nothing exits 0. Against a live cluster +// (SEI_NODE_CLUSTER set — the nightly's environment) a run that started zero +// suites means the pattern selected none of them, the classic false-healthy from +// a harness binary built before a suite rename, or a mistyped -test.run in the +// manifest. Fail loud instead of reporting green. Locally (SEI_NODE_CLUSTER +// unset) every suite t.Skips, so a zero count is expected and not enforced. This +// bounds only the zero-match case; positive proof that every expected suite ran +// is a completion-count SLI owned by the metrics layer, not this binary. +func TestMain(m *testing.M) { + code := m.Run() + if code == 0 && os.Getenv("SEI_NODE_CLUSTER") != "" && suitesStarted.Load() == 0 { + fmt.Fprintln(os.Stderr, "integration guard: -test.run selected zero suites against a live cluster — "+ + "refusing to report green (the harness binary and the -test.run pattern are out of sync; "+ + "an image built before a suite rename is the usual cause)") + code = 1 + } + os.Exit(code) +} + +// requireCluster skips a suite unless it is pointed at a real cluster, and +// counts started suites for TestMain's zero-match guard. The SDK selects its +// k8s provider on SEI_NODE_CLUSTER presence; without it there is nothing to +// provision against. func requireCluster(t *testing.T) { t.Helper() if os.Getenv("SEI_NODE_CLUSTER") == "" { t.Skip("integration suite: set SEI_NODE_CLUSTER to run against a cluster") } + // The one universal suite entry point: every suite gates on requireCluster + // first, so counting here needs no per-suite wiring and upholds the + // "a new suite joins by taking the TestNightly prefix, nothing else" rule. + suitesStarted.Add(1) } // openClient opens the SDK in k8s mode (config resolved from the ambient diff --git a/test/integration/release_test.go b/test/integration/release_test.go index 31af71b..21ec28c 100644 --- a/test/integration/release_test.go +++ b/test/integration/release_test.go @@ -45,7 +45,7 @@ var releaseRPCConfig = map[string]string{ "evm.enabled_legacy_sei_apis": releaseLegacyEVMAPIs, } -// TestRelease drives the release-validation flow: provision a 4-validator +// TestNightlyRelease drives the release-validation flow: provision a 4-validator // chain + one EVM-serving RPC follower, generate a funded admin account, and run // the external release-test image against the RPC node as a Job. The release-test // image owns the functional assertions (TEST_TARGET=chain-agnostic); the suite's @@ -58,8 +58,8 @@ var releaseRPCConfig = map[string]string{ // // Inputs (env): SEI_CHAIN_ID, SEID_IMAGE [required], RELEASE_TEST_IMAGE // (the external harness) [required]; SEI_NAMESPACE [optional]. Run with -// -test.timeout 0 (see TestBenchmark). -func TestRelease(t *testing.T) { +// -test.timeout 0 (see TestNightlyBenchmark). +func TestNightlyRelease(t *testing.T) { requireCluster(t) chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID")) seid := mustEnv(t, "SEID_IMAGE") @@ -158,7 +158,7 @@ func TestRelease(t *testing.T) { if err := sei.WaitCaughtUp(ctx, hc, node.TendermintRPC()); err != nil { t.Errorf("post-release %s not caught up: %v", rpcName, err) } - t.Logf("chain live post-release — TestRelease OK") + t.Logf("chain live post-release — TestNightlyRelease OK") } // createMnemonicSecret writes the admin mnemonic to a Secret the release-test pod diff --git a/test/integration/upgrade_test.go b/test/integration/upgrade_test.go index 923b99f..6cb1039 100644 --- a/test/integration/upgrade_test.go +++ b/test/integration/upgrade_test.go @@ -34,7 +34,7 @@ const ( // voting period and passes before the chain reaches it. At Sei's ~600ms-1s // blocks, 200 blocks comfortably outlasts the 60s voting period + tally; // env-overridable for faster/slower chains. The whole flow's safety rests on - // this margin, so it must stay generous (see TestChainUpgrade). + // this margin, so it must stay generous (see TestNightlyChainUpgrade). defaultUpgradeHeightDelta = 200 // minUpgradeHeightDelta guards against a misconfigured delta too small for the // pre-halt poll margin below. @@ -73,7 +73,7 @@ var upgradeConfig = map[string]string{ // restUnreachable is the last-seen note when a gov REST poll gets no 200. const restUnreachable = "REST unreachable / non-200" -// TestChainUpgrade drives a Sei major software upgrade end-to-end through the SDK +// TestNightlyChainUpgrade drives a Sei major software upgrade end-to-end through the SDK // task surface: provision a 4-validator chain on the pre-upgrade image -> submit a // GovSoftwareUpgrade proposal and read its minted ID off the task output -> // vote yes from every validator -> wait for it to pass -> let the chain halt at @@ -86,15 +86,16 @@ const restUnreachable = "REST unreachable / non-200" // (waitUpgradeApplied) is the safety net that fails loud if a too-fast chain // passed the height while still voting (no plan scheduled, no real upgrade). // -// Inputs (env): SEI_CHAIN_ID (base), SEID_IMAGE (pre-upgrade) [required], -// SEID_UPGRADE_IMAGE (post-upgrade) [required], SEI_UPGRADE_NAME (the upgrade -// handler name registered in the post image) [required]; SEI_NAMESPACE, -// UPGRADE_HEIGHT_DELTA [optional]. Run with -test.timeout 0 (see TestBenchmark). -func TestChainUpgrade(t *testing.T) { +// Inputs (env): SEI_CHAIN_ID (base), SEID_UPGRADE_FROM_IMAGE (pre-upgrade) +// [required], SEID_UPGRADE_TO_IMAGE (post-upgrade) [required], SEI_UPGRADE_NAME +// (the upgrade handler name registered in the post image) [required]; +// SEI_NAMESPACE, UPGRADE_HEIGHT_DELTA [optional]. Run with -test.timeout 0 (see +// TestNightlyBenchmark). +func TestNightlyChainUpgrade(t *testing.T) { requireCluster(t) chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID")) - preImage := mustEnv(t, "SEID_IMAGE") - postImage := mustEnv(t, "SEID_UPGRADE_IMAGE") + preImage := mustEnv(t, "SEID_UPGRADE_FROM_IMAGE") + postImage := mustEnv(t, "SEID_UPGRADE_TO_IMAGE") upgradeName := mustEnv(t, "SEI_UPGRADE_NAME") ns := envOr("SEI_NAMESPACE", "") delta := int64(envInt(t, "UPGRADE_HEIGHT_DELTA", defaultUpgradeHeightDelta)) @@ -234,7 +235,7 @@ func TestChainUpgrade(t *testing.T) { // and proves each validator individually — not merely "a pod is Ready". target := upgradeHeight + postUpgradeProgress awaitAllValidatorsAtHeight(ctx, t, c, chainID, ns, validators, target, runLabels) - t.Logf("all %d validators advanced past the upgrade to height %d — TestChainUpgrade OK", validators, target) + t.Logf("all %d validators advanced past the upgrade to height %d — TestNightlyChainUpgrade OK", validators, target) } // networkSpec builds the SeiNetwork spec for the upgrade chain. provision and the diff --git a/test/integration/workflow_test.go b/test/integration/workflow_test.go index 4f22e67..5119aa9 100644 --- a/test/integration/workflow_test.go +++ b/test/integration/workflow_test.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "net/http" - "net/url" "os/signal" "strconv" "strings" @@ -44,7 +43,7 @@ var snapshotProductionConfig = map[string]string{ "storage.snapshot_keep_recent": "3", } -// TestWorkflowStateSync drives the full state-sync procedure through the SDK end +// TestNightlyWorkflowStateSync drives the full state-sync procedure through the SDK end // to end: provision a snapshot-producing chain, wait past the snapshot interval, // then bring up a state-sync-bootstrapped follower and assert it started FROM a // snapshot (earliest retained height > 1, not a genesis replay's 1). It then runs @@ -56,7 +55,7 @@ var snapshotProductionConfig = map[string]string{ // criterion 5. // // The provision + witness + follower bring-up is shared verbatim with -// TestGigaStoreMigration via bringUpStateSyncFollower (that sibling reruns this +// TestNightlyGigaStoreMigration via bringUpStateSyncFollower (that sibling reruns this // same round trip WITH a giga config migration). This test is the plain-resync // baseline: it stays independent and carries no migration. // @@ -81,10 +80,10 @@ var snapshotProductionConfig = map[string]string{ // over p2p from the validators. // // Inputs (env): SEI_CHAIN_ID, SEID_IMAGE [required]; SEI_NAMESPACE, -// SEI_VALIDATORS [optional]. Run as the nightly CronJob: -// -// ["-test.run", "TestWorkflowStateSync", "-test.v", "-test.timeout", "0"] -func TestWorkflowStateSync(t *testing.T) { +// SEI_VALIDATORS [optional]. Selected into the nightly's combined run by its +// TestNightly prefix (harness_test.go); standalone: -test.run +// TestNightlyWorkflowStateSync. +func TestNightlyWorkflowStateSync(t *testing.T) { requireCluster(t) chainID := runChainID(mustEnv(t, "SEI_CHAIN_ID")) seid := mustEnv(t, "SEID_IMAGE") @@ -167,7 +166,7 @@ func TestWorkflowStateSync(t *testing.T) { t.Fatalf("follower %s earliest height %d -> %d after resync (jump %d), want a jump >= snapshot_interval %d (a genuine wipe + re-sync lands on a snapshot at least one interval higher; a smaller move is routine pruning, not a fresh restore)", f.node.Name(), f.preEarliest, postEarliest, postEarliest-f.preEarliest, f.interval) } - t.Logf("follower %s: caught up + EVM serving, block-store base jumped %d -> %d (>= interval %d) after state-sync re-bootstrap — TestWorkflowStateSync OK", f.node.Name(), f.preEarliest, postEarliest, f.interval) + t.Logf("follower %s: caught up + EVM serving, block-store base jumped %d -> %d (>= interval %d) after state-sync re-bootstrap — TestNightlyWorkflowStateSync OK", f.node.Name(), f.preEarliest, postEarliest, f.interval) // Interrupt-resume variant (deferred): kill the follower pod mid-recipe and // assert the workflow resumes to Complete. Deferred here — it needs a @@ -247,33 +246,12 @@ func bringUpStateSyncFollower(ctx context.Context, t *testing.T, c *sei.Client, t.Fatalf("network did not advance past the snapshot interval: %v", err) } - u, err := url.Parse(ch.network.TendermintRPC()) - if err != nil { - t.Fatalf("parse network TM RPC %q: %v", ch.network.TendermintRPC(), err) - } - hostLabels := strings.Split(u.Hostname(), ".") - if len(hostLabels) < 2 || hostLabels[1] == "" { - t.Fatalf("cannot derive namespace from aggregate RPC host %q", u.Host) - } - witnessNS := hostLabels[1] + witnessNS := witnessNamespace(t, ch.network) witnesses := []string{ nodeRPC(fmt.Sprintf("%s-0", chainID), witnessNS), // genesis validator-0 nodeRPC(rpcNodeName(chainID, 0), witnessNS), // rpc follower 0 } - - // A witness must be at head to serve light blocks at the snapshot height - // the bootstrap cross-checks. catching_up cannot certify that (it is a - // one-way latch meaning "left blocksync once", never "at head now"), so - // assert freshness directly: the follower witness sits within one - // snapshot interval of the validator head. - vHeight, vOK := sei.LatestHeight(ctx, hc, "http://"+witnesses[0]) - wHeight, wOK := sei.LatestHeight(ctx, hc, "http://"+witnesses[1]) - if !vOK || !wOK { - t.Fatalf("witness freshness read: validator ok=%v, follower ok=%v", vOK, wOK) - } - if gap := vHeight - wHeight; gap > int64(interval) { - t.Fatalf("witness %s lags validator head by %d blocks (> interval %d): a diverging follower cannot serve state-sync light blocks", witnesses[1], gap, interval) - } + assertWitnessFresh(ctx, t, hc, witnesses[0], witnesses[1], interval) // Bring up a state-sync-bootstrapped follower: it must fetch a snapshot from a // peer rather than replay from genesis. Appended to ch.rpcNodes so cleanupChain