feat(exec): add Claude verifier for local model output quality gate

internal/exec/verifier.go

@@ -0,0 +1,99 @@
+package exec
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+)
+
+// Verdict is the output of a Claude verification call.
+type Verdict struct {
+	Accept   bool   `json:"accept"`
+	Feedback string `json:"feedback"` // empty when Accept is true
+}
+
+// Verifier runs a focused Claude call to judge local model output.
+type Verifier struct {
+	claudeBinary string
+	model        string
+	timeout      time.Duration
+}
+
+// NewVerifier creates a Verifier that calls claude with the given binary path and model.
+// Empty claudeBinary defaults to "claude". Zero timeout defaults to 30s.
+func NewVerifier(claudeBinary, model string, timeout time.Duration) *Verifier {
+	if claudeBinary == "" {
+		claudeBinary = "claude"
+	}
+	if timeout == 0 {
+		timeout = 30 * time.Second
+	}
+	return &Verifier{
+		claudeBinary: claudeBinary,
+		model:        model,
+		timeout:      timeout,
+	}
+}
+
+// Verify asks Claude whether output satisfies the skill discipline's iron laws.
+// Returns Verdict{Accept: true} to accept or Verdict{Accept: false, Feedback: "..."}
+// to escalate. Returns an error on subprocess failure or unparseable response.
+func (v *Verifier) Verify(ctx context.Context, skillPrompt, taskPrompt string, output Result) (Verdict, error) {
+	ctx, cancel := context.WithTimeout(ctx, v.timeout)
+	defer cancel()
+
+	outputJSON, err := json.Marshal(output)
+	if err != nil {
+		return Verdict{}, fmt.Errorf("verifier: marshal output: %w", err)
+	}
+
+	prompt := fmt.Sprintf(`You are a quality verifier for an AI supervisor system.
+
+Given the skill discipline, the original task, and the generated output, decide whether the output satisfies the discipline's iron laws and output contract.
+
+Reply with JSON only — no other text:
+{"accept": true, "feedback": ""}
+or
+{"accept": false, "feedback": "<one sentence reason>"}
+
+## Skill discipline
+%s
+
+## Original task
+%s
+
+## Generated output
+%s`, skillPrompt, taskPrompt, string(outputJSON))
+
+	args := []string{
+		"--print",
+		"--permission-mode", "bypassPermissions",
+	}
+	if v.model != "" {
+		args = append(args, "--model", v.model)
+	}
+	args = append(args, prompt)
+
+	cmd := exec.CommandContext(ctx, v.claudeBinary, args...)
+	cmd.Env = os.Environ()
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	if err := cmd.Run(); err != nil {
+		if ctx.Err() != nil {
+			return Verdict{}, fmt.Errorf("verifier: timeout after %s", v.timeout)
+		}
+		return Verdict{}, fmt.Errorf("verifier: claude exited with error: %w — stderr: %s", err, stderr.String())
+	}
+
+	var verdict Verdict
+	if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &verdict); err != nil {
+		return Verdict{}, fmt.Errorf("verifier: parse verdict JSON: %w — raw: %s", err, stdout.String())
+	}
+	return verdict, nil
+}

internal/exec/verifier_test.go

@@ -0,0 +1,74 @@
+package exec_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	iexec "github.com/mathiasbq/supervisor/internal/exec"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func fakeVerifierClaude(t *testing.T, verdict iexec.Verdict) string {
+	t.Helper()
+	data, err := json.Marshal(verdict)
+	require.NoError(t, err)
+	dir := t.TempDir()
+	script := filepath.Join(dir, "claude")
+	content := fmt.Sprintf("#!/bin/sh\necho '%s'\n", string(data))
+	require.NoError(t, os.WriteFile(script, []byte(content), 0755))
+	return script
+}
+
+func TestVerifierAccepts(t *testing.T) {
+	claude := fakeVerifierClaude(t, iexec.Verdict{Accept: true, Feedback: ""})
+	v := iexec.NewVerifier(claude, "claude-sonnet-4-6", 5*time.Second)
+
+	verdict, err := v.Verify(context.Background(), "skill rules", "do the task", iexec.Result{
+		Status: "pass", Phase: "review", Skill: "review", Message: "ok",
+	})
+	require.NoError(t, err)
+	assert.True(t, verdict.Accept)
+	assert.Empty(t, verdict.Feedback)
+}
+
+func TestVerifierEscalates(t *testing.T) {
+	claude := fakeVerifierClaude(t, iexec.Verdict{Accept: false, Feedback: "missing line references"})
+	v := iexec.NewVerifier(claude, "claude-sonnet-4-6", 5*time.Second)
+
+	verdict, err := v.Verify(context.Background(), "skill rules", "do the task", iexec.Result{
+		Status: "pass", Phase: "review", Skill: "review", Message: "incomplete",
+	})
+	require.NoError(t, err)
+	assert.False(t, verdict.Accept)
+	assert.Equal(t, "missing line references", verdict.Feedback)
+}
+
+func TestVerifierErrorOnUnparsableOutput(t *testing.T) {
+	dir := t.TempDir()
+	script := filepath.Join(dir, "claude")
+	require.NoError(t, os.WriteFile(script, []byte("#!/bin/sh\necho 'not json'\n"), 0755))
+
+	v := iexec.NewVerifier(script, "claude-sonnet-4-6", 5*time.Second)
+	_, err := v.Verify(context.Background(), "rules", "task", iexec.Result{
+		Status: "pass", Phase: "review", Skill: "review", Message: "ok",
+	})
+	assert.Error(t, err)
+}
+
+func TestVerifierErrorOnNonZeroExit(t *testing.T) {
+	dir := t.TempDir()
+	script := filepath.Join(dir, "claude")
+	require.NoError(t, os.WriteFile(script, []byte("#!/bin/sh\nexit 1\n"), 0755))
+
+	v := iexec.NewVerifier(script, "claude-sonnet-4-6", 5*time.Second)
+	_, err := v.Verify(context.Background(), "rules", "task", iexec.Result{
+		Status: "pass", Phase: "review", Skill: "review", Message: "ok",
+	})
+	assert.Error(t, err)
+}