diff --git a/.gitignore b/.gitignore index 0091b7092a..e7be9bbe06 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ bazel-src-cli .DS_Store samples .amp +bin/ diff --git a/cmd/src/batch_exec.go b/cmd/src/batch_exec.go index 11aadf39e3..a8cc9584c9 100644 --- a/cmd/src/batch_exec.go +++ b/cmd/src/batch_exec.go @@ -278,6 +278,7 @@ func convertWorkspace(w batcheslib.WorkspacesExecutionInput) *executor.Task { BatchChangeAttributes: &w.BatchChangeAttributes, CachedStepResultFound: w.CachedStepResultFound, CachedStepResult: w.CachedStepResult, + ModelProviderURL: w.ModelProviderURL, } return task diff --git a/internal/batches/executor/run_steps.go b/internal/batches/executor/run_steps.go index 47b5715887..076e269e8b 100644 --- a/internal/batches/executor/run_steps.go +++ b/internal/batches/executor/run_steps.go @@ -14,6 +14,8 @@ import ( "time" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" + "github.com/sourcegraph/sourcegraph/lib/batches/codingagent" + codingagenttypes "github.com/sourcegraph/sourcegraph/lib/batches/codingagent/types" "github.com/sourcegraph/sourcegraph/lib/batches/execution" "github.com/sourcegraph/sourcegraph/lib/batches/git" "github.com/sourcegraph/sourcegraph/lib/batches/template" @@ -272,12 +274,32 @@ func executeSingleStep( return bytes.Buffer{}, bytes.Buffer{}, err } - runScriptFile, runScript, cleanup, err := createRunScriptFile(ctx, opts.TempDir, step.Run, stepContext) + var ( + runScriptFile string + runScript string + runScriptCleanup func() + ) + if step.CodingAgent != nil { + if opts.Task.ModelProviderURL == "" { + err = errors.New("codingAgent step requires a model-provider URL") + opts.UI.StepPreparingFailed(stepIdx+1, err) + return bytes.Buffer{}, bytes.Buffer{}, err + } + runScript, err = codingagent.RenderRunCommand(step.CodingAgent, opts.Task.ModelProviderURL, stepContext) + if err != nil { + err = errors.Wrap(err, "rendering codingAgent step") + opts.UI.StepPreparingFailed(stepIdx+1, err) + return bytes.Buffer{}, bytes.Buffer{}, err + } + runScriptFile, runScriptCleanup, err = writeRunScriptFile(opts.TempDir, runScript) + } else { + runScriptFile, runScript, runScriptCleanup, err = createRunScriptFile(ctx, opts.TempDir, step.Run, stepContext) + } if err != nil { opts.UI.StepPreparingFailed(stepIdx+1, err) return bytes.Buffer{}, bytes.Buffer{}, err } - defer cleanup() + defer runScriptCleanup() // Parse and render the step.Files. filesToMount, cleanup, err := createFilesToMount(opts.TempDir, step, stepContext) @@ -303,12 +325,16 @@ func executeSingleStep( return bytes.Buffer{}, bytes.Buffer{}, err } + if step.CodingAgent != nil { + forwardCodingAgentEnv(opts.GlobalEnv, env) + } + opts.UI.StepPreparingSuccess(stepIdx + 1) // ---------- // EXECUTION // ---------- - opts.UI.StepStarted(stepIdx+1, runScript, env) + opts.UI.StepStarted(stepIdx+1, runScript, redactSensitiveEnv(env)) workspaceOpts, err := workspace.DockerRunOpts(ctx, workDir) if err != nil { @@ -394,7 +420,7 @@ func executeSingleStep( } opts.Logger.Logf("[Step %d] run: %q, container: %q", stepIdx+1, step.Run, step.Container) - opts.Logger.Logf("[Step %d] full command: %q", stepIdx+1, strings.Join(cmd.Args, " ")) + opts.Logger.Logf("[Step %d] full command: %q", stepIdx+1, strings.Join(redactSensitiveArgs(cmd.Args), " ")) // Start the command. t0 := time.Now() @@ -573,6 +599,86 @@ func createRunScriptFile(ctx context.Context, tempDir string, stepRun string, st return runScriptFile.Name(), runScript.String(), cleanup, nil } +// forwardCodingAgentEnv copies the model-provider auth env vars +// (SRC_BATCHES_MODEL_PROVIDER_TOKEN, SRC_BATCHES_JOB_ID) from globalEnv +// into stepEnv so they reach the user container. +func forwardCodingAgentEnv(globalEnv []string, stepEnv map[string]string) { + for _, key := range []string{codingagenttypes.ModelProviderTokenEnvVar, codingagenttypes.JobIDEnvVar} { + for _, e := range globalEnv { + if v, ok := strings.CutPrefix(e, key+"="); ok { + stepEnv[key] = v + break + } + } + } +} + +// sensitiveEnvKeys names env vars that get passed verbatim into the user +// container but must be scrubbed from UI sinks and log lines. +var sensitiveEnvKeys = map[string]struct{}{ + codingagenttypes.ModelProviderTokenEnvVar: {}, +} + +const redactedPlaceholder = "REDACTED" + +func redactSensitiveEnv(env map[string]string) map[string]string { + out := make(map[string]string, len(env)) + for k, v := range env { + if _, sensitive := sensitiveEnvKeys[k]; sensitive && v != "" { + out[k] = redactedPlaceholder + } else { + out[k] = v + } + } + return out +} + +// redactSensitiveArgs scrubs the value side of `-e KEY=VALUE` pairs whose +// KEY is sensitive, returning a copy of args suitable for logging. +func redactSensitiveArgs(args []string) []string { + out := make([]string, len(args)) + copy(out, args) + for i := 0; i+1 < len(out); i++ { + if out[i] != "-e" { + continue + } + key, _, ok := strings.Cut(out[i+1], "=") + if !ok { + continue + } + if _, sensitive := sensitiveEnvKeys[key]; sensitive { + out[i+1] = key + "=" + redactedPlaceholder + } + } + return out +} + +// writeRunScriptFile writes a pre-rendered run script verbatim to a temp +// file. Unlike createRunScriptFile it does NOT pass the content through +// template.RenderStepTemplate, so embedded `{{` sequences in a +// shell-quoted prompt are not re-parsed as templates. +func writeRunScriptFile(tempDir, script string) (string, func(), error) { + runScriptFile, err := os.CreateTemp(tempDir, "") + if err != nil { + return "", nil, errors.Wrap(err, "creating temporary file") + } + cleanup := func() { os.Remove(runScriptFile.Name()) } + + if _, err := runScriptFile.WriteString(script); err != nil { + cleanup() + return "", nil, errors.Wrap(err, "writing to temporary file") + } + if err := runScriptFile.Close(); err != nil { + cleanup() + return "", nil, errors.Wrap(err, "closing temporary file") + } + if err := os.Chmod(runScriptFile.Name(), 0644); err != nil { + cleanup() + return "", nil, errors.Wrap(err, "setting permissions on the temporary file") + } + return runScriptFile.Name(), cleanup, nil +} + // createCidFile creates a temporary file that will contain the container ID // when executing steps. // It returns the location of the file and a function that cleans up the diff --git a/internal/batches/executor/run_steps_test.go b/internal/batches/executor/run_steps_test.go new file mode 100644 index 0000000000..7022ba1b87 --- /dev/null +++ b/internal/batches/executor/run_steps_test.go @@ -0,0 +1,112 @@ +package executor + +import ( + "slices" + "testing" + + codingagenttypes "github.com/sourcegraph/sourcegraph/lib/batches/codingagent/types" +) + +func TestRedactSensitiveEnv(t *testing.T) { + in := map[string]string{ + codingagenttypes.ModelProviderTokenEnvVar: "tok-abc", + "PATH": "/bin", + } + out := redactSensitiveEnv(in) + if got := out[codingagenttypes.ModelProviderTokenEnvVar]; got != redactedPlaceholder { + t.Errorf("token: got %q want %q", got, redactedPlaceholder) + } + if got := out["PATH"]; got != "/bin" { + t.Errorf("PATH should not be redacted: got %q", got) + } + if in[codingagenttypes.ModelProviderTokenEnvVar] != "tok-abc" { + t.Errorf("input must not be mutated") + } +} + +func TestRedactSensitiveArgs(t *testing.T) { + in := []string{ + "docker", "run", + "-e", codingagenttypes.ModelProviderTokenEnvVar + "=tok-abc", + "-e", codingagenttypes.JobIDEnvVar + "=job-123", + "-e", "PATH=/bin", + "--", "image:tag", "/script", + } + out := redactSensitiveArgs(in) + if slices.Contains(out, codingagenttypes.ModelProviderTokenEnvVar+"=tok-abc") { + t.Errorf("token value still present in args: %v", out) + } + if !slices.Contains(out, codingagenttypes.ModelProviderTokenEnvVar+"="+redactedPlaceholder) { + t.Errorf("token not redacted in args: %v", out) + } + if !slices.Contains(out, codingagenttypes.JobIDEnvVar+"=job-123") { + t.Errorf("job id should pass through: %v", out) + } +} + +func TestForwardCodingAgentEnv(t *testing.T) { + cases := []struct { + name string + globalEnv []string + stepEnv map[string]string + want map[string]string + }{ + { + name: "forwards both vars", + globalEnv: []string{ + "PATH=/bin", + codingagenttypes.ModelProviderTokenEnvVar + "=tok-abc", + codingagenttypes.JobIDEnvVar + "=job-123", + }, + stepEnv: map[string]string{}, + want: map[string]string{ + codingagenttypes.ModelProviderTokenEnvVar: "tok-abc", + codingagenttypes.JobIDEnvVar: "job-123", + }, + }, + { + name: "forwards only what is set", + globalEnv: []string{ + codingagenttypes.JobIDEnvVar + "=job-456", + }, + stepEnv: map[string]string{}, + want: map[string]string{ + codingagenttypes.JobIDEnvVar: "job-456", + }, + }, + { + name: "preserves preexisting step env and overwrites on match", + globalEnv: []string{ + codingagenttypes.ModelProviderTokenEnvVar + "=from-global", + }, + stepEnv: map[string]string{ + "OTHER": "x", + codingagenttypes.ModelProviderTokenEnvVar: "from-step", + }, + want: map[string]string{ + "OTHER": "x", + codingagenttypes.ModelProviderTokenEnvVar: "from-global", + }, + }, + { + name: "no-op when env not present", + globalEnv: []string{"PATH=/bin"}, + stepEnv: map[string]string{}, + want: map[string]string{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + forwardCodingAgentEnv(tc.globalEnv, tc.stepEnv) + if len(tc.stepEnv) != len(tc.want) { + t.Fatalf("len mismatch: got %d want %d (got=%v want=%v)", len(tc.stepEnv), len(tc.want), tc.stepEnv, tc.want) + } + for k, v := range tc.want { + if got := tc.stepEnv[k]; got != v { + t.Errorf("env[%q]: got %q want %q", k, got, v) + } + } + }) + } +} diff --git a/internal/batches/executor/task.go b/internal/batches/executor/task.go index ee7f8ad55e..7c9718396c 100644 --- a/internal/batches/executor/task.go +++ b/internal/batches/executor/task.go @@ -29,6 +29,9 @@ type Task struct { // When this field is true, CachedStepResult is also populated. CachedStepResultFound bool CachedStepResult execution.AfterStepResult + // ModelProviderURL is the resolved proxy base URL for coding-agent + // steps; empty unless the spec contains at least one codingAgent step. + ModelProviderURL string } func (t *Task) ArchivePathToFetch() string { diff --git a/lib/batches/batch_spec.go b/lib/batches/batch_spec.go index 269093a82d..6fd6cde355 100644 --- a/lib/batches/batch_spec.go +++ b/lib/batches/batch_spec.go @@ -90,13 +90,26 @@ func (oqor *OnQueryOrRepository) GetBranches() ([]string, error) { } type Step struct { - Run string `json:"run,omitempty" yaml:"run"` - Container string `json:"container,omitempty" yaml:"container"` - Env env.Environment `json:"env" yaml:"env"` - Files map[string]string `json:"files,omitempty" yaml:"files,omitempty"` - Outputs Outputs `json:"outputs,omitempty" yaml:"outputs,omitempty"` - Mount []Mount `json:"mount,omitempty" yaml:"mount,omitempty"` - If any `json:"if,omitempty" yaml:"if,omitempty"` + Run string `json:"run,omitempty" yaml:"run"` + CodingAgent *CodingAgentStep `json:"codingAgent,omitempty" yaml:"codingAgent,omitempty"` + Container string `json:"container,omitempty" yaml:"container"` + Image string `json:"image,omitempty" yaml:"image"` + Env env.Environment `json:"env" yaml:"env"` + Files map[string]string `json:"files,omitempty" yaml:"files,omitempty"` + Outputs Outputs `json:"outputs,omitempty" yaml:"outputs,omitempty"` + Mount []Mount `json:"mount,omitempty" yaml:"mount,omitempty"` + If any `json:"if,omitempty" yaml:"if,omitempty"` +} + +type CodingAgentType string + +const ( + CodingAgentTypeCodex CodingAgentType = "codex" +) + +type CodingAgentStep struct { + Type CodingAgentType `json:"type,omitempty" yaml:"type"` + Prompt string `json:"prompt,omitempty" yaml:"prompt"` } func (s *Step) IfCondition() string { @@ -181,6 +194,12 @@ func parseBatchSpec(schema string, data []byte) (*BatchSpec, error) { errs = errors.Append(errs, NewValidationError(errors.Newf("step %d files target path contains invalid characters", i+1))) } } + if step.CodingAgent != nil && step.Run != "" { + errs = errors.Append(errs, NewValidationError(errors.Newf("step %d: codingAgent and run cannot be combined in the same step", i+1))) + } + if step.CodingAgent != nil && step.Container == "" && step.Image == "" { + errs = errors.Append(errs, NewValidationError(errors.Newf("step %d: codingAgent step requires an image", i+1))) + } } return &spec, errs diff --git a/lib/batches/batch_spec_test.go b/lib/batches/batch_spec_test.go new file mode 100644 index 0000000000..55faea2e3c --- /dev/null +++ b/lib/batches/batch_spec_test.go @@ -0,0 +1,60 @@ +package batches + +import ( + "fmt" + "strings" + "testing" +) + +// v3SpecWith wraps stepsYAML in the minimum scaffold needed for ParseBatchSpec +// to accept a v3 spec. +func v3SpecWith(stepsYAML string) []byte { + return []byte(fmt.Sprintf(` +version: 3 +name: test +description: test +on: + - repository: github.com/sourcegraph/sourcegraph +steps: +%s +changesetTemplate: + title: test + body: test + branch: test + commit: + message: test +`, stepsYAML)) +} + +func TestParseBatchSpec_v3_codingAgentRequiresImage(t *testing.T) { + _, err := ParseBatchSpec(v3SpecWith(" - codingAgent:\n type: codex\n prompt: do the thing")) + if err == nil { + t.Fatal("expected validation error, got nil") + } + if !strings.Contains(err.Error(), "requires an image") { + t.Errorf("error should mention missing image, got: %v", err) + } +} + +func TestParseBatchSpec_v3_codingAgentStep(t *testing.T) { + got, err := ParseBatchSpec(v3SpecWith(" - codingAgent:\n type: codex\n prompt: do the thing\n image: alpine:3")) + if err != nil { + t.Fatalf("ParseBatchSpec failed: %v", err) + } + if len(got.Steps) != 1 { + t.Fatalf("expected 1 step, got %d", len(got.Steps)) + } + step := got.Steps[0] + if step.CodingAgent == nil { + t.Fatal("expected step.CodingAgent to be set") + } + if step.CodingAgent.Type != CodingAgentTypeCodex { + t.Errorf("CodingAgent.Type: got %q want %q", step.CodingAgent.Type, CodingAgentTypeCodex) + } + if step.CodingAgent.Prompt != "do the thing" { + t.Errorf("CodingAgent.Prompt: got %q want %q", step.CodingAgent.Prompt, "do the thing") + } + if step.Image != "alpine:3" { + t.Errorf("Step.Image: got %q want %q", step.Image, "alpine:3") + } +} diff --git a/lib/batches/codex/codex.go b/lib/batches/codex/codex.go new file mode 100644 index 0000000000..56155dbd9c --- /dev/null +++ b/lib/batches/codex/codex.go @@ -0,0 +1,67 @@ +// Package codex implements the codex coding agent. +package codex + +import ( + "fmt" + + "github.com/kballard/go-shellquote" + + batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" + "github.com/sourcegraph/sourcegraph/lib/batches/codingagent/types" +) + +const ( + model = "gpt-5.4" + pinnedVersion = "0.134.0" +) + +type Agent struct{} + +func (Agent) Type() batcheslib.CodingAgentType { return batcheslib.CodingAgentTypeCodex } +func (Agent) ImageRequirements() []string { return []string{"curl", "tar"} } + +func (Agent) InstallScript() string { + return fmt.Sprintf(`_install_dir=%s +_version=%s + +case "$(uname -m)" in + x86_64) _triple=x86_64-unknown-linux-musl ;; + aarch64) _triple=aarch64-unknown-linux-musl ;; + *) echo "codingAgent codex: unsupported architecture $(uname -m)" >&2; exit 1 ;; +esac + +# Stage in a temp dir and mv into place so a failed retry can't leave a half-written binary behind. +_url="https://github.com/openai/codex/releases/download/rust-v${_version}/codex-${_triple}.tar.gz" +_tmp=$(mktemp -d "${TMPDIR:-/tmp}/sg-codex.XXXXXX") +mkdir -p "$_install_dir" +curl -fsSL "$_url" | tar -xz -C "$_tmp" || { echo "codingAgent codex: download/extract failed: $_url" >&2; exit 1; } +chmod +x "$_tmp/codex-${_triple}" +mv -f "$_tmp/codex-${_triple}" "$_install_dir/codex" +rm -rf "$_tmp" + +_actual=$("$_install_dir/codex" --version 2>&1) || { echo "codingAgent codex: cannot exec $_install_dir/codex: $_actual" >&2; exit 1; } +case "$_actual" in + *"$_version"*) ;; + *) echo "codingAgent codex: version mismatch: want $_version, got: $_actual" >&2; exit 1 ;; +esac +`, types.InstallDir, pinnedVersion) +} + +func (Agent) RunCommand(prompt, modelProviderURL string) string { + return shellquote.Join( + types.InstallDir+"/codex", + "exec", + "--json", + "--sandbox", "danger-full-access", + "--ephemeral", + "--model", model, + "-c", `approval_policy="never"`, + "-c", `model_reasoning_effort="medium"`, + "-c", `model_provider="sourcegraph"`, + "-c", `model_providers.sourcegraph.name="Sourcegraph"`, + "-c", fmt.Sprintf(`model_providers.sourcegraph.base_url=%q`, modelProviderURL), + "-c", fmt.Sprintf(`model_providers.sourcegraph.env_key=%q`, types.ModelProviderTokenEnvVar), + "-c", fmt.Sprintf(`model_providers.sourcegraph.env_http_headers={%q=%q}`, types.JobIDHeaderName, types.JobIDEnvVar), + prompt, + ) +} diff --git a/lib/batches/codingagent/codingagent.go b/lib/batches/codingagent/codingagent.go new file mode 100644 index 0000000000..4f3efd6aad --- /dev/null +++ b/lib/batches/codingagent/codingagent.go @@ -0,0 +1,64 @@ +// Package codingagent rewrites v3 batch spec coding-agent steps into shell +// commands. Register new agents in the agents list. +package codingagent + +import ( + "bytes" + "fmt" + "strings" + + "github.com/kballard/go-shellquote" + + batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" + "github.com/sourcegraph/sourcegraph/lib/batches/codex" + "github.com/sourcegraph/sourcegraph/lib/batches/codingagent/types" + "github.com/sourcegraph/sourcegraph/lib/batches/template" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +var ErrUnknownType = errors.New("unknown codingAgent type") + +// RenderRunCommand returns the shell command that runs agentStep. +func RenderRunCommand(agentStep *batcheslib.CodingAgentStep, modelProviderURL string, stepCtx *template.StepContext) (string, error) { + a, ok := agents[agentStep.Type] + if !ok { + return "", errors.Wrapf(ErrUnknownType, "codingAgent type %q", agentStep.Type) + } + var renderedPrompt bytes.Buffer + if err := template.RenderStepTemplate("codingagent-prompt", agentStep.Prompt, &renderedPrompt, stepCtx); err != nil { + return "", errors.Wrap(err, "rendering codingAgent.prompt") + } + prefixed := strings.TrimRight(modelProviderURL, "/") + "/" + string(a.Type()) + + var b strings.Builder + for _, binary := range a.ImageRequirements() { + b.WriteString(failIfMissing(a.Type(), binary)) + } + b.WriteString(a.InstallScript()) + b.WriteString(a.RunCommand(renderedPrompt.String(), prefixed)) + return b.String(), nil +} + +func failIfMissing(agentType batcheslib.CodingAgentType, binary string) string { + msg := fmt.Sprintf( + "codingAgent %q requires %q on PATH in the run container", + agentType, binary, + ) + return fmt.Sprintf("command -v %s >/dev/null 2>&1 || { echo %s >&2; exit 1; }\n", + binary, + shellquote.Join(msg), + ) +} + +var agents = func() map[batcheslib.CodingAgentType]types.Agent { + out := map[batcheslib.CodingAgentType]types.Agent{} + for _, a := range []types.Agent{ + codex.Agent{}, + } { + if _, exists := out[a.Type()]; exists { + panic("duplicate codingagent agent for " + a.Type()) + } + out[a.Type()] = a + } + return out +}() diff --git a/lib/batches/codingagent/codingagent_test.go b/lib/batches/codingagent/codingagent_test.go new file mode 100644 index 0000000000..5fc1e75723 --- /dev/null +++ b/lib/batches/codingagent/codingagent_test.go @@ -0,0 +1,56 @@ +package codingagent_test + +import ( + "errors" + "strings" + "testing" + + "github.com/kballard/go-shellquote" + + batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" + "github.com/sourcegraph/sourcegraph/lib/batches/codingagent" + "github.com/sourcegraph/sourcegraph/lib/batches/template" +) + +func TestRenderRunCommand_unknownType(t *testing.T) { + agentStep := &batcheslib.CodingAgentStep{Type: "nope", Prompt: "x"} + _, err := codingagent.RenderRunCommand(agentStep, "https://example/", &template.StepContext{}) + if err == nil { + t.Fatal("expected error, got nil") + } + if !errors.Is(err, codingagent.ErrUnknownType) { + t.Fatalf("expected ErrUnknownType, got %v", err) + } +} + +func TestRenderRunCommand_promptShellQuoting(t *testing.T) { + const repoName = `github.com/sourcegraph/sourcegraph` + prompt := "You're working in the ${{ repository.name }} repository.\n" + + "Add a README section describing the project; don't touch existing files." + agentStep := &batcheslib.CodingAgentStep{ + Type: batcheslib.CodingAgentTypeCodex, + Prompt: prompt, + } + stepCtx := &template.StepContext{Repository: template.Repository{Name: repoName}} + cmd, err := codingagent.RenderRunCommand(agentStep, "https://example/", stepCtx) + if err != nil { + t.Fatal(err) + } + + // The rendered command appends the shell-quoted prompt as its final + // argument. Round-tripping via shellquote.Split is unreliable here + // because the install script contains POSIX shell comments with + // apostrophes (e.g. "can't"), which shellquote does not understand + // and would treat as unterminated quoted strings. Assert on the + // suffix instead. + wantPrompt := "You're working in the " + repoName + " repository.\n" + + "Add a README section describing the project; don't touch existing files." + wantQuoted := shellquote.Join(wantPrompt) + if !strings.HasSuffix(cmd, wantQuoted) { + tail := cmd + if n := len(wantQuoted) + 50; len(cmd) > n { + tail = cmd[len(cmd)-n:] + } + t.Fatalf("rendered cmd does not end with shell-quoted prompt:\n want suffix: %q\n cmd tail: %q", wantQuoted, tail) + } +} diff --git a/lib/batches/codingagent/types/types.go b/lib/batches/codingagent/types/types.go new file mode 100644 index 0000000000..7be4be11ff --- /dev/null +++ b/lib/batches/codingagent/types/types.go @@ -0,0 +1,29 @@ +// Package types holds the codingagent contract shared with individual +// coding-agent implementations; split out to avoid an import cycle. +package types + +import ( + batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" +) + +// Mirrored in sourcegraph/sourcegraph lib/batches/codingagent/codingagent.go; keep in sync. +const ( + ModelProviderTokenEnvVar = "SRC_BATCHES_MODEL_PROVIDER_TOKEN" + JobIDEnvVar = "SRC_BATCHES_JOB_ID" + JobIDHeaderName = "X-Sourcegraph-Job-ID" +) + +// InstallDir is src-cli-local; not part of the cross-repo contract. +const InstallDir = "/tmp/sg-codingagent/bin" + +type Agent interface { + Type() batcheslib.CodingAgentType + // RunCommand returns the shell command for the agent. The rendered + // prompt MUST be shell-quoted. + RunCommand(renderedPrompt, modelProviderURL string) string + // ImageRequirements lists binaries that must be on PATH in the run + // container before InstallScript runs. + ImageRequirements() []string + // InstallScript installs the agent at a pinned version into InstallDir. + InstallScript() string +} diff --git a/lib/batches/schema/batch_spec_stringdata.go b/lib/batches/schema/batch_spec_stringdata.go index 1123970faf..5acf2223a5 100644 --- a/lib/batches/schema/batch_spec_stringdata.go +++ b/lib/batches/schema/batch_spec_stringdata.go @@ -11,6 +11,317 @@ const BatchSpecJSON = `{ "type": "object", "additionalProperties": false, "required": ["name"], + "allOf": [ + { + "if": { "properties": { "version": { "const": 3 } }, "required": ["version"] }, + "then": { + "properties": { + "steps": { + "items": { + "anyOf": [{ "required": ["run", "image"] }, { "required": ["codingAgent"] }] + } + } + } + }, + "else": { + "properties": { + "steps": { + "items": { + "required": ["run", "container"], + "not": { "required": ["codingAgent"] } + } + } + } + } + }, + { + "$comment": "The ` + "`" + `changesetHooks` + "`" + ` property is only allowed when ` + "`" + `version: 3` + "`" + ` is set.", + "if": { + "required": ["changesetHooks"] + }, + "then": { + "required": ["version"], + "properties": { + "version": { + "const": 3 + } + } + } + } + ], + "definitions": { + "OutputVariable": { + "title": "OutputVariable", + "type": "object", + "required": ["value"], + "properties": { + "value": { + "type": "string", + "description": "The value of the output, which can be a template string.", + "examples": ["hello world", "${{ step.stdout }}", "${{ repository.name }}"] + }, + "format": { + "type": "string", + "description": "The expected format of the output. If set, the output is being parsed in that format before being stored in the var. If not set, 'text' is assumed to the format.", + "enum": ["json", "yaml", "text"] + } + } + }, + "Mount": { + "title": "Mount", + "type": "object", + "additionalProperties": false, + "required": ["path", "mountpoint"], + "properties": { + "path": { + "type": "string", + "description": "The path on the local machine to mount. The path must be in the same directory or a subdirectory of the batch spec.", + "examples": ["local/path/to/file.text", "local/path/to/directory"] + }, + "mountpoint": { + "type": "string", + "description": "The path in the container to mount the path on the local machine to.", + "examples": ["path/to/file.txt", "path/to/directory"] + } + } + }, + "CodingAgent": { + "title": "CodingAgent", + "type": "object", + "description": "An out-of-the-box coding agent step that runs the given prompt inside a managed container. Mutually exclusive with run. Only supported in version 3 batch specs. Use the step's top-level container/image field to override the default agent image.", + "additionalProperties": false, + "required": ["type", "prompt"], + "properties": { + "type": { + "type": "string", + "description": "The coding agent to use.", + "enum": ["codex", "claude-code"] + }, + "prompt": { + "type": "string", + "description": "The prompt to send to the agent." + } + } + }, + "Step": { + "title": "Step", + "type": "object", + "description": "A command to run (as part of a sequence) in a repository branch to produce the required changes.", + "additionalProperties": false, + "properties": { + "run": { + "type": "string", + "description": "The shell command to run in the container. It can also be a multi-line shell script. The working directory is the root directory of the repository checkout." + }, + "container": { + "type": "string", + "description": "The Docker image used to launch the Docker container in which the shell command is run.", + "examples": ["alpine:3"] + }, + "image": { + "type": "string", + "description": "The Docker image used to launch the Docker container in which the shell command is run.", + "examples": ["alpine:3"] + }, + "codingAgent": { + "$ref": "#/definitions/CodingAgent" + }, + "outputs": { + "type": ["object", "null"], + "description": "Output variables of this step that can be referenced in the changesetTemplate or other steps via outputs.", + "additionalProperties": { + "$ref": "#/definitions/OutputVariable" + } + }, + "env": { + "description": "Environment variables to set in the step environment.", + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "description": "Environment variables to set in the step environment.", + "additionalProperties": { + "type": "string" + } + }, + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "description": "An environment variable to set in the step environment: the value will be passed through from the environment src is running within." + }, + { + "type": "object", + "description": "An environment variable to set in the step environment: the key is used as the environment variable name and the value as the value.", + "additionalProperties": { + "type": "string" + }, + "minProperties": 1, + "maxProperties": 1 + } + ] + } + } + ] + }, + "files": { + "type": ["object", "null"], + "description": "Files that should be mounted into or be created inside the Docker container.", + "additionalProperties": { + "type": "string" + } + }, + "if": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A condition to check before executing steps. Supports templating. The value 'true' is interpreted as true.", + "examples": [ + "true", + "${{ matches repository.name \"github.com/my-org/my-repo*\" }}", + "${{ outputs.goModFileExists }}", + "${{ eq previous_step.stdout \"success\" }}" + ] + }, + "mount": { + "description": "Files that are mounted to the Docker container.", + "type": ["array", "null"], + "items": { + "$ref": "#/definitions/Mount" + } + } + } + }, + "HookStep": { + "title": "HookStep", + "type": "object", + "description": "A command to run (as part of a sequence) inside a changeset hook action. Uses the version 3 step shape and supports additional hook-only fields such as maxAttempts.", + "additionalProperties": false, + "properties": { + "run": { + "type": "string", + "description": "The shell command to run in the container. It can also be a multi-line shell script. The working directory is the root directory of the repository checkout." + }, + "image": { + "type": "string", + "description": "The Docker image used to launch the Docker container in which the shell command is run.", + "examples": ["alpine:3"] + }, + "codingAgent": { + "$ref": "#/definitions/CodingAgent" + }, + "outputs": { + "type": ["object", "null"], + "description": "Output variables of this step that can be referenced in the changesetTemplate or other steps via outputs.", + "additionalProperties": { + "$ref": "#/definitions/OutputVariable" + } + }, + "env": { + "description": "Environment variables to set in the step environment.", + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "description": "Environment variables to set in the step environment.", + "additionalProperties": { + "type": "string" + } + }, + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "description": "An environment variable to set in the step environment: the value will be passed through from the environment src is running within." + }, + { + "type": "object", + "description": "An environment variable to set in the step environment: the key is used as the environment variable name and the value as the value.", + "additionalProperties": { + "type": "string" + }, + "minProperties": 1, + "maxProperties": 1 + } + ] + } + } + ] + }, + "files": { + "type": ["object", "null"], + "description": "Files that should be mounted into or be created inside the Docker container.", + "additionalProperties": { + "type": "string" + } + }, + "if": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A condition to check before executing steps. Supports templating. The value 'true' is interpreted as true.", + "examples": [ + "true", + "${{ matches repository.name \"github.com/my-org/my-repo*\" }}", + "${{ outputs.goModFileExists }}", + "${{ eq previous_step.stdout \"success\" }}" + ] + }, + "mount": { + "description": "Files that are mounted to the Docker container.", + "type": ["array", "null"], + "items": { + "$ref": "#/definitions/Mount" + } + }, + "maxAttempts": { + "type": "integer", + "description": "The maximum number of times this step will be attempted before the hook action is considered failed. Defaults to 1 (no retries).", + "minimum": 1 + } + } + }, + "ChangesetHookAction": { + "title": "ChangesetHookAction", + "type": "object", + "description": "A side-effect action to run at a changeset lifecycle event. Hook actions run in an executor.", + "additionalProperties": false, + "required": ["steps"], + "properties": { + "steps": { + "type": "array", + "description": "The sequence of steps to execute when the hook fires. Uses the version 3 step shape, with additional hook-only fields.", + "items": { + "$ref": "#/definitions/HookStep" + } + } + } + } + }, "properties": { "version": { "type": "number", @@ -28,20 +339,20 @@ const BatchSpecJSON = `{ }, "on": { "type": ["array", "null"], - "description": "The set of repositories (and branches) to run the batch change on, specified as a list of search queries (that match repositories) and/or specific repositories.", + "description": "The set of repositories to run the batch change on, specified as a list of search queries (that match repositories) and/or specific repositories. Repositories matched by repositoriesMatchingQuery use each repository's default branch unless overridden by an explicit repository entry with branch or branches.", "items": { "title": "OnQueryOrRepository", "oneOf": [ { "title": "OnQuery", "type": "object", - "description": "A Sourcegraph search query that matches a set of repositories (and branches). Each matched repository branch is added to the list of repositories that the batch change will be run on.", + "description": "A Sourcegraph search query that matches a set of repositories. Each matched repository is added to the list of repositories that the batch change will be run on using the repository's default branch.", "additionalProperties": false, "required": ["repositoriesMatchingQuery"], "properties": { "repositoriesMatchingQuery": { "type": "string", - "description": "A Sourcegraph search query that matches a set of repositories (and branches). If the query matches files, symbols, or some other object inside a repository, the object's repository is included.", + "description": "A Sourcegraph search query that matches a set of repositories. If the query matches files, symbols, or some other object inside a repository, the object's repository is included. Matched repositories are run on their default branch; use explicit repository entries with branch or branches to target non-default branches.", "examples": ["file:README.md"] } } @@ -124,126 +435,7 @@ const BatchSpecJSON = `{ "type": ["array", "null"], "description": "The sequence of commands to run (for each repository branch matched in the ` + "`" + `on` + "`" + ` property) to produce the workspace changes that will be included in the batch change.", "items": { - "title": "Step", - "type": "object", - "description": "A command to run (as part of a sequence) in a repository branch to produce the required changes.", - "additionalProperties": false, - "required": ["run", "container"], - "properties": { - "run": { - "type": "string", - "description": "The shell command to run in the container. It can also be a multi-line shell script. The working directory is the root directory of the repository checkout." - }, - "container": { - "type": "string", - "description": "The Docker image used to launch the Docker container in which the shell command is run.", - "examples": ["alpine:3"] - }, - "outputs": { - "type": ["object", "null"], - "description": "Output variables of this step that can be referenced in the changesetTemplate or other steps via outputs.", - "additionalProperties": { - "title": "OutputVariable", - "type": "object", - "required": ["value"], - "properties": { - "value": { - "type": "string", - "description": "The value of the output, which can be a template string.", - "examples": ["hello world", "${{ step.stdout }}", "${{ repository.name }}"] - }, - "format": { - "type": "string", - "description": "The expected format of the output. If set, the output is being parsed in that format before being stored in the var. If not set, 'text' is assumed to the format.", - "enum": ["json", "yaml", "text"] - } - } - } - }, - "env": { - "description": "Environment variables to set in the step environment.", - "oneOf": [ - { - "type": "null" - }, - { - "type": "object", - "description": "Environment variables to set in the step environment.", - "additionalProperties": { - "type": "string" - } - }, - { - "type": "array", - "items": { - "oneOf": [ - { - "type": "string", - "description": "An environment variable to set in the step environment: the value will be passed through from the environment src is running within." - }, - { - "type": "object", - "description": "An environment variable to set in the step environment: the key is used as the environment variable name and the value as the value.", - "additionalProperties": { - "type": "string" - }, - "minProperties": 1, - "maxProperties": 1 - } - ] - } - } - ] - }, - "files": { - "type": ["object", "null"], - "description": "Files that should be mounted into or be created inside the Docker container.", - "additionalProperties": { - "type": "string" - } - }, - "if": { - "oneOf": [ - { - "type": "boolean" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "A condition to check before executing steps. Supports templating. The value 'true' is interpreted as true.", - "examples": [ - "true", - "${{ matches repository.name \"github.com/my-org/my-repo*\" }}", - "${{ outputs.goModFileExists }}", - "${{ eq previous_step.stdout \"success\" }}" - ] - }, - "mount": { - "description": "Files that are mounted to the Docker container.", - "type": ["array", "null"], - "items": { - "type": "object", - "additionalProperties": false, - "required": ["path", "mountpoint"], - "properties": { - "path": { - "type": "string", - "description": "The path on the local machine to mount. The path must be in the same directory or a subdirectory of the batch spec.", - "examples": ["local/path/to/file.text", "local/path/to/directory"] - }, - "mountpoint": { - "type": "string", - "description": "The path in the container to mount the path on the local machine to.", - "examples": ["path/to/file.txt", "path/to/directory"] - } - } - } - } - } + "$ref": "#/definitions/Step" } }, "transformChanges": { @@ -406,6 +598,21 @@ const BatchSpecJSON = `{ ] } } + }, + "changesetHooks": { + "type": "object", + "description": "Side-effect actions to run at well-defined changeset lifecycle events. Only allowed when ` + "`" + `version: 3` + "`" + ` is set.", + "additionalProperties": false, + "properties": { + "onCIFailure": { + "description": "Action to run when the changeset's combined check state transitions into ` + "`" + `failed` + "`" + ` for a given head SHA.", + "$ref": "#/definitions/ChangesetHookAction" + }, + "onMergeConflict": { + "description": "Action to run when the changeset's external mergeability transitions into ` + "`" + `conflicting` + "`" + ` for a given (base, head) SHA pair.", + "$ref": "#/definitions/ChangesetHookAction" + } + } } } } diff --git a/lib/batches/workspaces_execution_input.go b/lib/batches/workspaces_execution_input.go index 76a5a8cb18..c63e0d881c 100644 --- a/lib/batches/workspaces_execution_input.go +++ b/lib/batches/workspaces_execution_input.go @@ -21,6 +21,9 @@ type WorkspacesExecutionInput struct { CachedStepResult execution.AfterStepResult `json:"cachedStepResult"` // SkippedSteps determines which steps are skipped in the execution. SkippedSteps map[int]struct{} `json:"skippedSteps"` + // ModelProviderURL is the resolved proxy base URL for coding-agent + // steps; only set when the spec contains at least one codingAgent step. + ModelProviderURL string `json:"modelProviderURL,omitempty"` } type WorkspaceRepo struct { diff --git a/lib/go.mod b/lib/go.mod index 44fdc19356..67fe8a2790 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -56,6 +56,7 @@ require ( github.com/gorilla/css v1.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect diff --git a/lib/go.sum b/lib/go.sum index 434c58adc8..dd1329c5ff 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -93,6 +93,8 @@ github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=