dreampiler/kandev
1
0
Fork 0
Code Issues 43 Pull Requests Actions 6 Packages Projects Releases Wiki Activity

fix: scope PR auto-fix outcome reporting (#3602)

Browse Source
This commit is contained in:
Carlos Florêncio
2026-09-12 16:24:56 +01:00
committed by GitHub
parent 140ef39cc6
commit 2b9d5f1a5d
13 changed files with 757 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/internal/github/store_ci_outcome_scope_test.go
+213
View File
@@ -0,0 +1,213 @@
package github
import (
"context"
"errors"
"reflect"
"testing"
"time"
)
type ciOutcomeScopeSnapshot struct {
legacyOptionRows int
legacyOptions []TaskCIOptions
prOptions []*TaskPRAutomationOptions
states []*TaskCIPRAutomationState
}
func snapshotCIAutoFixScope(t *testing.T, store *Store, taskID string) ciOutcomeScopeSnapshot {
t.Helper()
var legacyOptionRows int
if err := store.db.GetContext(context.Background(), &legacyOptionRows,
"SELECT COUNT(*) FROM github_task_ci_options WHERE task_id = ?", taskID); err != nil {
t.Fatalf("count legacy options: %v", err)
}
var legacyOptions []TaskCIOptions
if err := store.ro.SelectContext(context.Background(), &legacyOptions,
store.ro.Rebind("SELECT * FROM github_task_ci_options WHERE task_id = ?"), taskID); err != nil {
t.Fatalf("list legacy options: %v", err)
}
prOptions, err := store.ListTaskPRAutomationOptions(context.Background(), taskID)
if err != nil {
t.Fatalf("list PR options: %v", err)
}
states, err := store.ListTaskCIPRStates(context.Background(), taskID)
if err != nil {
t.Fatalf("list PR states: %v", err)
}
return ciOutcomeScopeSnapshot{
legacyOptionRows: legacyOptionRows,
legacyOptions: legacyOptions,
prOptions: prOptions,
states: states,
}
}
func requireCIAutoFixScopeUnchanged(t *testing.T, before, after ciOutcomeScopeSnapshot) {
t.Helper()
if !reflect.DeepEqual(before, after) {
t.Fatalf("auto-fix state changed after rejected report:\nbefore: %+v\nafter: %+v", before, after)
}
}
// @covers AC-UI-CI-PR-AUTOMATION-001.10
// @covers AC-UI-CI-PR-AUTOMATION-001.17
func TestStorePRAutoFixOutcomeOrdinaryTurnHasNoSideEffects(t *testing.T) {
ctx := context.Background()
report := func(store *Store, taskID, sessionID, turnID string) error {
return store.ReportTaskCIAutoFixOutcome(ctx, TaskCIAutoFixOutcomeReport{
TaskID: taskID, SessionID: sessionID, TurnID: turnID,
Outcome: TaskCIAutoFixOutcomeBlocked, Summary: "ordinary PR work",
})
}
t.Run("absent settings", func(t *testing.T) {
store := newTestStore(t)
before := snapshotCIAutoFixScope(t, store, "task-absent")
if err := report(store, "task-absent", "session-current", "turn-current"); !errors.Is(err, ErrTaskCIAutoFixAttemptNotFound) {
t.Fatalf("report error = %v, want %v", err, ErrTaskCIAutoFixAttemptNotFound)
}
after := snapshotCIAutoFixScope(t, store, "task-absent")
requireCIAutoFixScopeUnchanged(t, before, after)
})
t.Run("explicitly disabled settings", func(t *testing.T) {
store := newTestStore(t)
disabled := false
if _, err := store.UpdateTaskPRAutomationOptions(ctx, "task-disabled", "repo-a", 101,
TaskPRAutomationOptionsPatch{AutoFixEnabled: &disabled}, false); err != nil {
t.Fatalf("seed disabled options: %v", err)
}
before := snapshotCIAutoFixScope(t, store, "task-disabled")
if err := report(store, "task-disabled", "session-current", "turn-current"); !errors.Is(err, ErrTaskCIAutoFixAttemptNotFound) {
t.Fatalf("report error = %v, want %v", err, ErrTaskCIAutoFixAttemptNotFound)
}
after := snapshotCIAutoFixScope(t, store, "task-disabled")
requireCIAutoFixScopeUnchanged(t, before, after)
})
t.Run("enabled without attempt", func(t *testing.T) {
store := newTestStore(t)
enabled := true
if _, err := store.UpdateTaskPRAutomationOptions(ctx, "task-enabled", "repo-a", 101,
TaskPRAutomationOptionsPatch{AutoFixEnabled: &enabled}, false); err != nil {
t.Fatalf("seed enabled options: %v", err)
}
before := snapshotCIAutoFixScope(t, store, "task-enabled")
if err := report(store, "task-enabled", "session-current", "turn-current"); !errors.Is(err, ErrTaskCIAutoFixAttemptNotFound) {
t.Fatalf("report error = %v, want %v", err, ErrTaskCIAutoFixAttemptNotFound)
}
after := snapshotCIAutoFixScope(t, store, "task-enabled")
requireCIAutoFixScopeUnchanged(t, before, after)
})
t.Run("foreign bound attempt across differently configured PRs", func(t *testing.T) {
store := newTestStore(t)
enabled := true
disabled := false
for _, target := range []struct {
repositoryID string
prNumber int
value *bool
}{
{repositoryID: "repo-a", prNumber: 101, value: &enabled},
{repositoryID: "repo-b", prNumber: 202, value: &disabled},
} {
if _, err := store.UpdateTaskPRAutomationOptions(ctx, "task-foreign", target.repositoryID, target.prNumber,
TaskPRAutomationOptionsPatch{AutoFixEnabled: target.value}, false); err != nil {
t.Fatalf("seed %s#%d options: %v", target.repositoryID, target.prNumber, err)
}
}
if err := store.RecordTaskCIFixAttempt(ctx, TaskCIFixAttempt{
TaskID: "task-foreign", RepositoryID: "repo-a", PRNumber: 101,
Signature: "foreign-feedback", CheckpointJSON: `{}`,
SessionID: "session-foreign", TurnID: "turn-foreign",
ProviderGeneration: "head-foreign", State: TaskCIAutoFixAttemptRunning,
IncrementRound: true, EnqueuedAt: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
}); err != nil {
t.Fatalf("seed foreign attempt: %v", err)
}
before := snapshotCIAutoFixScope(t, store, "task-foreign")
if err := report(store, "task-foreign", "session-current", "turn-current"); !errors.Is(err, ErrTaskCIAutoFixAttemptNotFound) {
t.Fatalf("report error = %v, want %v", err, ErrTaskCIAutoFixAttemptNotFound)
}
after := snapshotCIAutoFixScope(t, store, "task-foreign")
requireCIAutoFixScopeUnchanged(t, before, after)
})
t.Run("already reported attempt", func(t *testing.T) {
store := newTestStore(t)
attempt := TaskCIFixAttempt{
TaskID: "task-reported", RepositoryID: "repo-a", PRNumber: 101,
Signature: "reported-feedback", CheckpointJSON: `{}`,
SessionID: "session-current", TurnID: "turn-current",
ProviderGeneration: "head-reported", State: TaskCIAutoFixAttemptRunning,
IncrementRound: true,
}
if err := store.RecordTaskCIFixAttempt(ctx, attempt); err != nil {
t.Fatalf("seed reported attempt: %v", err)
}
if err := store.ReportTaskCIAutoFixOutcome(ctx, TaskCIAutoFixOutcomeReport{
TaskID: attempt.TaskID, SessionID: attempt.SessionID, TurnID: attempt.TurnID,
Outcome: TaskCIAutoFixOutcomeNonActionable, Summary: "first report",
}); err != nil {
t.Fatalf("record first outcome: %v", err)
}
before := snapshotCIAutoFixScope(t, store, attempt.TaskID)
if err := store.ReportTaskCIAutoFixOutcome(ctx, TaskCIAutoFixOutcomeReport{
TaskID: attempt.TaskID, SessionID: attempt.SessionID, TurnID: attempt.TurnID,
Outcome: TaskCIAutoFixOutcomeActionTaken, Summary: "duplicate report",
}); !errors.Is(err, ErrTaskCIAutoFixAttemptNotFound) {
t.Fatalf("duplicate report error = %v, want %v", err, ErrTaskCIAutoFixAttemptNotFound)
}
after := snapshotCIAutoFixScope(t, store, attempt.TaskID)
requireCIAutoFixScopeUnchanged(t, before, after)
})
}
// @covers AC-UI-CI-PR-AUTOMATION-001.10
func TestStorePRAutoFixOutcomeAcceptsAllValidOutcomes(t *testing.T) {
for _, test := range []struct {
name string
outcome TaskCIAutoFixOutcome
wantState TaskCIAutoFixAttemptState
wantLastError bool
}{
{name: "action taken", outcome: TaskCIAutoFixOutcomeActionTaken, wantState: TaskCIAutoFixAttemptAwaitingProviderProgress},
{name: "non actionable", outcome: TaskCIAutoFixOutcomeNonActionable, wantState: TaskCIAutoFixAttemptAcknowledged},
{name: "blocked", outcome: TaskCIAutoFixOutcomeBlocked, wantState: TaskCIAutoFixAttemptAcknowledged, wantLastError: true},
} {
t.Run(test.name, func(t *testing.T) {
store := newTestStore(t)
ctx := context.Background()
attempt := TaskCIFixAttempt{
TaskID: "task-valid-" + test.name, RepositoryID: "repo-valid", PRNumber: 303,
Signature: "valid-feedback", CheckpointJSON: `{}`,
SessionID: "session-valid", TurnID: "turn-valid",
ProviderGeneration: "head-valid", State: TaskCIAutoFixAttemptRunning,
}
if err := store.RecordTaskCIFixAttempt(ctx, attempt); err != nil {
t.Fatalf("seed attempt: %v", err)
}
if err := store.ReportTaskCIAutoFixOutcome(ctx, TaskCIAutoFixOutcomeReport{
TaskID: attempt.TaskID, SessionID: attempt.SessionID, TurnID: attempt.TurnID,
Outcome: test.outcome, Summary: "valid outcome",
}); err != nil {
t.Fatalf("record %s outcome: %v", test.outcome, err)
}
state, err := store.GetTaskCIPRState(ctx, attempt.TaskID, attempt.RepositoryID, attempt.PRNumber)
if err != nil {
t.Fatalf("get state: %v", err)
}
if state == nil {
t.Fatal("attempt state is missing")
}
if state.AutoFixAttemptState != test.wantState || state.AutoFixAttemptOutcome != test.outcome {
t.Fatalf("state = %+v, want state %q and outcome %q", state, test.wantState, test.outcome)
}
if (state.LastError != nil) != test.wantLastError {
t.Fatalf("last error presence = %t, want %t", state.LastError != nil, test.wantLastError)
}
})
}
}
apps/backend/internal/mcp/handlers/task_pr_automation.go
+5 -2
View File
@@ -29,6 +29,8 @@ type TaskPRAutoFixOutcomeService interface {
ReportTaskPRAutoFixOutcome(ctx context.Context, taskID, sessionID, outcome, summary string) error
}
const taskPRAutoFixOutcomeUnmatchedMessage = "No matching unresolved GitHub PR auto-fix attempt exists for this turn. Finish ordinary work without retrying this report or enabling auto-fix."
func (h *Handlers) SetTaskPRAutomationService(automation TaskPRAutomationService) {
h.taskPRAutomation = automation
}
@@ -153,8 +155,9 @@ func (h *Handlers) handleReportTaskPRAutoFixOutcome(ctx context.Context, msg *ws
ctx, req.TaskID, req.SessionID, req.Outcome, req.Summary,
); err != nil {
switch {
case errors.Is(err, github.ErrTaskCIAutoFixAttemptNotFound),
errors.Is(err, github.ErrTaskCIAutoFixOutcomeInvalid):
case errors.Is(err, github.ErrTaskCIAutoFixAttemptNotFound):
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, taskPRAutoFixOutcomeUnmatchedMessage, nil)
case errors.Is(err, github.ErrTaskCIAutoFixOutcomeInvalid):
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, err.Error(), nil)
default:
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to record PR auto-fix outcome: "+err.Error(), nil)
apps/backend/internal/mcp/handlers/task_pr_automation_outcome_scope_test.go
+33
View File
@@ -0,0 +1,33 @@
package handlers
import (
"strings"
"testing"
"github.com/kandev/kandev/internal/github"
ws "github.com/kandev/kandev/pkg/websocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReportPRAutoFixOutcomeExplainsUnmatchedTurn(t *testing.T) {
automation := &recordingTaskPRAutomationService{outcomeErr: github.ErrTaskCIAutoFixAttemptNotFound}
h := &Handlers{taskPRAutoFixOutcome: automation, logger: testLogger(t).WithFields()}
response, err := h.handleReportTaskPRAutoFixOutcome(
taskPRAutoFixOutcomeContext(),
makeWSMessage(t, ws.ActionMCPReportPRAutoFixOutcome, map[string]any{
"outcome": "blocked",
"summary": "provider is unavailable",
}),
)
require.NoError(t, err)
require.Equal(t, ws.MessageTypeError, response.Type)
var payload ws.ErrorPayload
require.NoError(t, response.ParsePayload(&payload))
assert.Equal(t, ws.ErrorCodeValidation, payload.Code)
assert.Contains(t, payload.Message, "No matching unresolved GitHub PR auto-fix attempt exists for this turn")
assert.Contains(t, payload.Message, "Finish ordinary work without retrying this report or enabling auto-fix")
assert.NotContains(t, strings.ToLower(payload.Message), "disabled")
}
apps/backend/internal/mcp/handlers/task_pr_automation_test.go
+1 -1
View File
@@ -203,7 +203,7 @@ func TestHandleReportTaskPRAutoFixOutcomeRejectsInvalidAndStaleReports(t *testin
}))
require.NoError(t, err)
assert.Equal(t, ws.MessageTypeError, response.Type)
assert.Contains(t, string(response.Payload), "not found")
assert.Contains(t, string(response.Payload), "Finish ordinary work without retrying this report or enabling auto-fix")
}
func TestHandleReportTaskPRAutoFixOutcomeRejectsSpoofedIdentity(t *testing.T) {
apps/backend/internal/mcp/server/pr_auto_fix_outcome_scope_test.go
+44
View File
@@ -0,0 +1,44 @@
package mcp
import (
"errors"
"testing"
"github.com/mark3labs/mcp-go/mcp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReportPRAutoFixOutcomeToolDescriptionScopesCurrentTurn(t *testing.T) {
s := newTaskModeServer(t, &testBackend{}, "task-current")
tool, ok := s.mcpServer.ListTools()["report_pr_auto_fix_outcome_kandev"]
require.True(t, ok, "report outcome tool must be registered")
description := tool.Tool.Description
assert.Contains(t, description, "current Kandev-dispatched auto-fix turn")
assert.Contains(t, description, "server-owned outcome protocol")
assert.Contains(t, description, "manual PR fixup")
assert.Contains(t, description, "sibling review")
assert.Contains(t, description, "earlier auto-fix turn")
assert.Contains(t, description, "Tool availability or enabled automation settings alone do not establish an obligation to report")
assert.Contains(t, description, "exactly once")
}
func TestReportPRAutoFixOutcomeServerSurfacesBackendError(t *testing.T) {
// Keep this transport fixture synchronized with handlers.taskPRAutoFixOutcomeUnmatchedMessage.
const explanation = "No matching unresolved GitHub PR auto-fix attempt exists for this turn. Finish ordinary work without retrying this report or enabling auto-fix."
backend := &testBackend{err: errors.New(explanation)}
s := newTaskModeServer(t, backend, "task-current")
result := callTool(t, s, "report_pr_auto_fix_outcome_kandev", map[string]interface{}{
"outcome": "blocked",
"summary": "provider is unavailable",
})
require.True(t, result.IsError)
require.Len(t, result.Content, 1)
content, ok := result.Content[0].(mcp.TextContent)
require.True(t, ok)
assert.Contains(t, content.Text, explanation)
}
apps/backend/internal/mcp/server/server.go
+5 -1
View File
@@ -1371,10 +1371,14 @@ func (s *Server) registerPRAutomationTools() {
s.mcpServer.AddTool(
mcp.NewTool("report_pr_auto_fix_outcome_kandev",
mcp.WithDescription(
"Report the one explicit outcome for the current GitHub PR auto-fix turn. "+
"Report the one explicit outcome for the current Kandev-dispatched auto-fix turn. "+
"Use this tool only when this turn received the server-owned outcome protocol. "+
"Do not use it for manual PR fixup, sibling review messages, or instructions carried over from an earlier auto-fix turn. "+
"Tool availability or enabled automation settings alone do not establish an obligation to report. "+
"Use action_taken when a concrete provider-visible change was made, "+
"non_actionable when the feedback identifies no change this task can make, "+
"or blocked when an external condition prevents the needed change. "+
"Report exactly once for the dispatched turn. "+
"The task, session, turn, and PR are bound by Kandev and are not tool arguments.",
),
mcp.WithString("outcome", mcp.Required(), mcp.Enum("action_taken", "non_actionable", "blocked"), mcp.Description("The disposition of this auto-fix turn.")),
apps/backend/internal/orchestrator/ci_automation_outcome_scope_test.go
+35
View File
@@ -0,0 +1,35 @@
package orchestrator
import (
"strings"
"testing"
"github.com/kandev/kandev/internal/sysprompt"
)
func TestCIAutomationOutcomeProtocolScopesCurrentTurn(t *testing.T) {
for _, passthrough := range []bool{false, true} {
t.Run(map[bool]string{false: "structured", true: "passthrough"}[passthrough], func(t *testing.T) {
prompt := ciAutomationAppendOutcomeProtocol("repair the PR", passthrough)
visible := prompt
if !passthrough {
visible = sysprompt.StripSystemContent(prompt)
}
assertions := []string{
"current Kandev-dispatched auto-fix turn",
"expires when this turn ends",
"manual PR fixup",
"sibling review",
"historical auto-fix instructions",
"Tool availability or enabled automation settings alone do not establish this obligation",
"report_pr_auto_fix_outcome_kandev exactly once",
}
for _, want := range assertions {
if !strings.Contains(prompt, want) {
t.Errorf("protocol does not contain %q: %s", want, visible)
}
}
})
}
}
apps/backend/internal/orchestrator/event_handlers_github_ci_automation.go
+2
View File
@@ -1203,10 +1203,12 @@ func ciAutomationRenderPromptTemplate(base, snapshot string) string {
}
const ciAutomationOutcomeProtocol = `Kandev PR auto-fix outcome protocol:
This instruction applies only to the current Kandev-dispatched auto-fix turn that received this protocol. It expires when this turn ends.
Before this turn ends, call report_pr_auto_fix_outcome_kandev exactly once with one of these outcomes:
- action_taken: you made or requested a concrete provider-visible change and want Kandev to wait for CI or PR progress.
- non_actionable: the current feedback does not identify a change this task can make.
- blocked: a concrete change is needed, but an external condition prevents it. Include a short reason.
Do not report an outcome for manual PR fixup, sibling review messages, or historical auto-fix instructions. Tool availability or enabled automation settings alone do not establish this obligation.
Do not claim action_taken from a plan or an attempted command alone. If the tool is unavailable, continue the repair work and explain that the outcome could not be recorded.`
func ciAutomationAppendOutcomeProtocol(prompt string, passthrough bool) string {