fix: recover from phantom 'running' state after crash/restart

The run-cancel registry is in-memory; a container restart mid-run leaves accounts/tasks persisted as 'running' with no goroutine, wedging cancel (not-in-map -> 409) and blocking remove/re-run. - startup: ResetRunningOnStartup clears stale 'running' -> 'idle' on boot - cancel handler: when no live goroutine, ClearStuckAccount + ReconcileTaskStatus reset the stuck account (and its task) instead of returning 409 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MMHQTtnQtQqL8muAXHr9kd
2026-07-02 12:57:39 +07:00
parent 6a10697548
commit fa72f1b323
4 changed files with 136 additions and 1 deletions
@@ -0,0 +1,62 @@
+package store
+
+import (
+	"context"
+	"testing"
+)
+
+func TestResetRunningOnStartup(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	e1, _ := s.CreateEndpoint(ctx, Endpoint{RoleLabel: "s", Host: "a", Port: 993, TLSMode: "ssl"})
+	e2, _ := s.CreateEndpoint(ctx, Endpoint{RoleLabel: "d", Host: "b", Port: 993, TLSMode: "ssl"})
+	taskID, _ := s.CreateTask(ctx, Task{Name: "t", SrcEndpointID: e1, DstEndpointID: e2})
+	accID, _ := s.CreateAccount(ctx, Account{TaskID: taskID, SrcLogin: "u", SrcPassEnc: "x", DstLogin: "v", DstPassEnc: "y"})
+	// simulate a crash mid-run
+	_ = s.SetTaskStatus(ctx, taskID, "running")
+	_ = s.SetAccountStatus(ctx, accID, "running")
+
+	tn, an, err := s.ResetRunningOnStartup(ctx)
+	if err != nil {
+		t.Fatalf("reset: %v", err)
+	}
+	if tn != 1 || an != 1 {
+		t.Fatalf("reset counts tasks=%d accounts=%d, want 1/1", tn, an)
+	}
+	task, _ := s.GetTask(ctx, taskID)
+	if task.Status == "running" {
+		t.Fatal("task still running after reset")
+	}
+	accs, _ := s.ListAccountsByTask(ctx, taskID)
+	if accs[0].Status == "running" {
+		t.Fatal("account still running after reset")
+	}
+}
+
+func TestClearStuckAccountAndReconcile(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	e1, _ := s.CreateEndpoint(ctx, Endpoint{RoleLabel: "s", Host: "a", Port: 993, TLSMode: "ssl"})
+	e2, _ := s.CreateEndpoint(ctx, Endpoint{RoleLabel: "d", Host: "b", Port: 993, TLSMode: "ssl"})
+	taskID, _ := s.CreateTask(ctx, Task{Name: "t", SrcEndpointID: e1, DstEndpointID: e2})
+	accID, _ := s.CreateAccount(ctx, Account{TaskID: taskID, SrcLogin: "u", SrcPassEnc: "x", DstLogin: "v", DstPassEnc: "y"})
+	_ = s.SetTaskStatus(ctx, taskID, "running")
+	_ = s.SetAccountStatus(ctx, accID, "running")
+
+	cleared, err := s.ClearStuckAccount(ctx, accID)
+	if err != nil || !cleared {
+		t.Fatalf("clear stuck: cleared=%v err=%v", cleared, err)
+	}
+	// second call finds nothing to clear
+	again, _ := s.ClearStuckAccount(ctx, accID)
+	if again {
+		t.Fatal("second clear should be false")
+	}
+	if err := s.ReconcileTaskStatus(ctx, taskID); err != nil {
+		t.Fatalf("reconcile: %v", err)
+	}
+	task, _ := s.GetTask(ctx, taskID)
+	if task.Status == "running" {
+		t.Fatal("task should no longer be running after reconcile")
+	}
+}