diff --git a/docs/CLI.md b/docs/CLI.md index e814a29d..509e2647 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -74,7 +74,7 @@ chunk │ │ --sidecar-id # Sidecar ID (defaults to active sidecar) │ │ --public-key # SSH public key string │ │ --public-key-file # Path to public key file -│ ├── ssh # SSH into sidecar +│ ├── ssh # SSH into sidecar (stdin forwarded when piped) │ │ --sidecar-id # Sidecar ID (defaults to active sidecar) │ │ --identity-file # SSH identity file │ │ -e / --env KEY=VALUE # Set env var in remote session (repeatable) @@ -83,6 +83,7 @@ chunk │ │ --sidecar-id # Sidecar ID (defaults to active sidecar) │ │ --identity-file # SSH identity file │ │ --workdir # Destination path on sidecar (auto-detected when omitted) +│ │ --bundle # Sync via git bundle (no GitHub access required) │ ├── env # Detect tech stack and print environment spec as JSON │ │ --dir # Directory to analyse (default: .) │ │ --no-save # Print only, do not save to .chunk/config.json @@ -122,6 +123,13 @@ chunk to disable. - `config set` accepts only `model` and `apiKey` as keys. - `chunk init` uses Claude to auto-detect the test command for the project. +- `sidecar sync --bundle` sends a full git bundle on first use, then incremental + bundles (`..HEAD`) on subsequent syncs. The branch does not need to be + pushed to GitHub. Set `bundleSync: true` in `.chunk/config.json` to use bundle + sync by default without passing `--bundle` on every invocation; `sidecar setup` + reads the same config key. +- `sidecar ssh -- ` forwards stdin when the process stdin is a pipe, enabling + patterns like `cat bundle | chunk sidecar ssh -- git fetch ...`. It generates `.claude/settings.json` with pre-commit hooks. It never touches CircleCI — tokens are prompted inline only when a command actually needs them. - Commands that require a CircleCI token (`task run`, `task config`, `sidecar *`, diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index f86fa57e..7e84b5fc 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -195,6 +195,26 @@ run the tests on the sidecar The skill handles the full loop: auth checks → find active sidecar → sync → validate → interpret failures → fix locally → repeat. +### Syncing without pushing to GitHub + +By default `chunk sidecar sync` requires the branch to be pushed so the sidecar can clone it. If you want to sync unpushed commits, use `--bundle`: + +```bash +chunk sidecar sync --bundle +``` + +The first sync sends a full git bundle of HEAD. Subsequent syncs send only the new commits since the last sync (`..HEAD`), so incremental syncs are fast. Uncommitted working-tree changes are applied on top as a patch after each bundle transfer. + +To avoid passing `--bundle` every time, set it in `.chunk/config.json`: + +```json +{ + "bundleSync": true +} +``` + +`chunk sidecar setup` reads the same setting, so the initial environment setup also uses bundle sync when it is enabled. + ### Environment setup Auto-detect your tech stack and build a sidecar image for it: diff --git a/internal/cmd/sidecar.go b/internal/cmd/sidecar.go index 11b447b0..c680e2cf 100644 --- a/internal/cmd/sidecar.go +++ b/internal/cmd/sidecar.go @@ -355,7 +355,11 @@ func newSidecarSSHCmd() *cobra.Command { return &userError{msg: fmt.Sprintf("resolve secrets: %s", err), err: err} } envVars = resolved - err = sidecar.SSH(cmd.Context(), client, sidecarID, identityFile, authSock, args, envVars, io) + var stdin *os.File + if fi, statErr := os.Stdin.Stat(); statErr == nil && fi.Mode()&os.ModeCharDevice == 0 { + stdin = os.Stdin + } + err = sidecar.SSH(cmd.Context(), client, sidecarID, identityFile, authSock, args, envVars, io, stdin) if err != nil { if err := sshSessionError(err); err != nil { return err @@ -380,6 +384,7 @@ func newSidecarSSHCmd() *cobra.Command { func newSidecarSyncCmd() *cobra.Command { var sidecarID, identityFile, workdir string + var bundle bool cmd := &cobra.Command{ Use: "sync", @@ -394,7 +399,24 @@ func newSidecarSyncCmd() *cobra.Command { if err != nil { return err } - err = sidecar.Sync(cmd.Context(), client, sidecarID, identityFile, authSock, workdir, newStatusFunc(io)) + cwd, cwdErr := os.Getwd() + if cwdErr != nil { + return fmt.Errorf("sync: %w", cwdErr) + } + useBundle := bundle + if !useBundle { + cfg, cfgErr := config.LoadProjectConfig(cwd) + if cfgErr != nil { + io.ErrPrintf("warning: could not load project config: %v\n", cfgErr) + } else { + useBundle = cfg.BundleSync + } + } + if useBundle { + err = sidecar.BundleSync(cmd.Context(), client, sidecarID, identityFile, authSock, workdir, cwd, newStatusFunc(io)) + } else { + err = sidecar.Sync(cmd.Context(), client, sidecarID, identityFile, authSock, workdir, newStatusFunc(io)) + } if err != nil { if _, ok := errors.AsType[*sidecar.RemoteBaseError](err); ok { return &userError{ @@ -421,6 +443,7 @@ func newSidecarSyncCmd() *cobra.Command { cmd.Flags().StringVar(&sidecarID, "sidecar-id", "", "Sidecar ID (defaults to active sidecar)") cmd.Flags().StringVar(&identityFile, "identity-file", "", "SSH identity file") cmd.Flags().StringVar(&workdir, "workdir", "", "Destination path on sidecar (auto-detected as /workspace/ when omitted)") + cmd.Flags().BoolVar(&bundle, "bundle", false, "Sync via git bundle instead of patch (no GitHub access required)") return cmd } @@ -810,7 +833,13 @@ Example: // Step 4: Sync files to sidecar. if !skipSync { - if err := sidecarSetupSync(cmd.Context(), client, sidecarID, identityFile, authSock, status); err != nil { + useBundle := false + if cfg, cfgErr := config.LoadProjectConfig(dir); cfgErr != nil { + status(iostream.LevelWarn, fmt.Sprintf("warning: could not load project config: %v", cfgErr)) + } else { + useBundle = cfg.BundleSync + } + if err := sidecarSetupSync(cmd.Context(), client, sidecarID, identityFile, authSock, useBundle, dir, status); err != nil { return err } } @@ -901,10 +930,17 @@ func sidecarSetupSync( ctx context.Context, client *circleci.Client, sidecarID, identityFile, authSock string, + useBundle bool, + cwd string, status iostream.StatusFunc, ) error { status(iostream.LevelStep, "Syncing files to sidecar...") - err := sidecar.Sync(ctx, client, sidecarID, identityFile, authSock, "", status) + var err error + if useBundle { + err = sidecar.BundleSync(ctx, client, sidecarID, identityFile, authSock, "", cwd, status) + } else { + err = sidecar.Sync(ctx, client, sidecarID, identityFile, authSock, "", status) + } if err == nil { return nil } diff --git a/internal/config/project.go b/internal/config/project.go index 87848b59..566035bc 100644 --- a/internal/config/project.go +++ b/internal/config/project.go @@ -60,6 +60,7 @@ type ProjectConfig struct { OrgID string `json:"orgID,omitempty"` StopHookMaxAttempts int `json:"stopHookMaxAttempts,omitempty"` Environment *envspec.Environment `json:"environment,omitempty"` + BundleSync bool `json:"bundleSync,omitempty"` } // LoadProjectConfig reads .chunk/config.json from workDir. diff --git a/internal/gitutil/gitutil.go b/internal/gitutil/gitutil.go index 5bf011c6..07a11f18 100644 --- a/internal/gitutil/gitutil.go +++ b/internal/gitutil/gitutil.go @@ -102,6 +102,39 @@ func GeneratePatch(base string) (string, error) { return string(out), nil } +// HeadRef returns the SHA of the current HEAD commit in the repo at cwd. +func HeadRef(cwd string) (string, error) { + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = cwd + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("resolve HEAD: %w", err) + } + sha := strings.TrimSpace(string(out)) + if sha == "" { + return "", fmt.Errorf("resolve HEAD: empty output") + } + return sha, nil +} + +// CreateBundle creates a git bundle from base..HEAD in the repo at cwd and returns the raw bytes. +// If base is empty, the full history up to HEAD is bundled. +func CreateBundle(base, cwd string) ([]byte, error) { + var args []string + if base == "" { + args = []string{"bundle", "create", "-", "HEAD"} + } else { + args = []string{"bundle", "create", "-", base + "..HEAD"} + } + cmd := exec.Command("git", args...) + cmd.Dir = cwd + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("create bundle: %w", err) + } + return out, nil +} + func splitNonEmpty(s string) []string { if s == "" { return nil diff --git a/internal/sidecar/active.go b/internal/sidecar/active.go index 7d48bb31..54b6588d 100644 --- a/internal/sidecar/active.go +++ b/internal/sidecar/active.go @@ -13,9 +13,10 @@ import ( // ActiveSidecar holds the currently active sidecar for a project. type ActiveSidecar struct { - SidecarID string `json:"sidecar_id"` - Name string `json:"name,omitempty"` - Workspace string `json:"workspace,omitempty"` + SidecarID string `json:"sidecar_id"` + Name string `json:"name,omitempty"` + Workspace string `json:"workspace,omitempty"` + LastSyncedRef string `json:"last_synced_ref,omitempty"` } // sidecarFileName returns the name of the sidecar state file. When sessionID diff --git a/internal/sidecar/sidecar.go b/internal/sidecar/sidecar.go index 19c64c94..0a5bbadf 100644 --- a/internal/sidecar/sidecar.go +++ b/internal/sidecar/sidecar.go @@ -3,6 +3,7 @@ package sidecar import ( "context" "fmt" + "io" "os" "strings" @@ -47,7 +48,9 @@ func AddSSHKey(ctx context.Context, client *circleci.Client, sidecarID, publicKe } // SSH opens a session and either runs a command or starts an interactive shell. -func SSH(ctx context.Context, client *circleci.Client, sidecarID, identityFile, authSock string, args []string, envVars map[string]string, io iostream.Streams) error { +// stdin is forwarded to the remote command when non-nil; callers should pass +// os.Stdin when the process stdin is a pipe, nil otherwise. +func SSH(ctx context.Context, client *circleci.Client, sidecarID, identityFile, authSock string, args []string, envVars map[string]string, streams iostream.Streams, stdin io.Reader) error { session, err := OpenSession(ctx, client, sidecarID, identityFile, authSock) if err != nil { return err @@ -58,16 +61,16 @@ func SSH(ctx context.Context, client *circleci.Client, sidecarID, identityFile, } command := ShellJoin(args) - result, err := ExecOverSSH(ctx, session, command, nil, envVars) + result, err := ExecOverSSH(ctx, session, command, stdin, envVars) if err != nil { return err } if result.Stdout != "" { - _, _ = fmt.Fprint(io.Out, result.Stdout) + _, _ = fmt.Fprint(streams.Out, result.Stdout) } if result.Stderr != "" { - _, _ = fmt.Fprint(io.Err, result.Stderr) + _, _ = fmt.Fprint(streams.Err, result.Stderr) } if result.ExitCode != 0 { return fmt.Errorf("%q exited with status %d", command, result.ExitCode) diff --git a/internal/sidecar/sync.go b/internal/sidecar/sync.go index 5d491c9b..afb5b88e 100644 --- a/internal/sidecar/sync.go +++ b/internal/sidecar/sync.go @@ -1,6 +1,7 @@ package sidecar import ( + "bytes" "context" "errors" "fmt" @@ -103,6 +104,151 @@ func Sync(ctx context.Context, return nil } +// BundleSync synchronises local commits and working-tree changes to a sidecar +// using git bundle, without requiring the branch to be pushed to GitHub. +// +// On first sync (no LastSyncedRef) a full bundle of HEAD is sent. On subsequent +// syncs only commits since the last synced ref are bundled (incremental). In +// both cases any uncommitted working-tree changes are applied on top as a patch. +func BundleSync(ctx context.Context, + client *circleci.Client, sidecarID, identityFile, authSock, workdir, cwd string, status iostream.StatusFunc) error { + + session, err := OpenSession(ctx, client, sidecarID, identityFile, authSock) + if err != nil { + return err + } + + _, repo, err := gitremote.DetectOrgAndRepo(cwd) + if err != nil { + return fmt.Errorf("bundle sync: %w", err) + } + + repoPath := ResolveWorkspace(ctx, workdir, repo) + if err := persistWorkspace(ctx, repoPath); err != nil { + status(iostream.LevelWarn, fmt.Sprintf("Could not save workspace: %v", err)) + } + + active, err := LoadActive(ctx) + if err != nil { + return fmt.Errorf("bundle sync: load active sidecar: %w", err) + } + if active == nil { + active = &ActiveSidecar{SidecarID: sidecarID} + } + lastRef := active.LastSyncedRef + if active.SidecarID != sidecarID { + lastRef = "" // sidecar changed; force full bundle + } + + headRef, err := gitutil.HeadRef(cwd) + if err != nil { + return fmt.Errorf("bundle sync: %w", err) + } + + // Ensure the workspace directory exists on the sidecar. + parentDir := filepath.Dir(repoPath) + if result, err := ExecOverSSH(ctx, session, "mkdir -p "+ShellEscape(parentDir), nil, nil); err != nil { + return fmt.Errorf("bundle sync: mkdir: %w", err) + } else if result.ExitCode != 0 { + return fmt.Errorf("bundle sync: mkdir -p %s: %s", parentDir, result.Stderr) + } + + // Init the repo on the sidecar if it's not already there. + testResult, err := ExecOverSSH(ctx, session, "test -d "+ShellEscape(repoPath), nil, nil) + if err != nil { + return fmt.Errorf("bundle sync: check repo dir: %w", err) + } + if testResult.ExitCode != 0 { + initCmd := fmt.Sprintf("git init %s && git -C %s commit --allow-empty -m init", + ShellEscape(repoPath), ShellEscape(repoPath)) + if result, err := ExecOverSSH(ctx, session, initCmd, nil, nil); err != nil { + return fmt.Errorf("bundle sync: git init: %w", err) + } else if result.ExitCode != 0 { + return fmt.Errorf("bundle sync: git init: %s", result.Stderr) + } + lastRef = "" // force full bundle for a fresh repo + } + + resetCmd := fmt.Sprintf("git -C %s reset --hard HEAD", ShellEscape(repoPath)) + cleanCmd := fmt.Sprintf("git -C %s clean -fd", ShellEscape(repoPath)) + + // Sync commits: skip the bundle when already up-to-date, otherwise send and fetch. + if lastRef == headRef { + status(iostream.LevelInfo, "No new commits since last sync.") + } else { + if err := sendBundle(ctx, session, lastRef, cwd, repo, repoPath, status); err != nil { + return err + } + } + + // Reset and clean the remote working tree after the bundle fetch (or no-op sync). + if result, err := ExecOverSSH(ctx, session, resetCmd, nil, nil); err != nil { + return fmt.Errorf("bundle sync: reset: %w", err) + } else if result.ExitCode != 0 { + return fmt.Errorf("bundle sync: reset: %s", result.Stderr) + } + if result, err := ExecOverSSH(ctx, session, cleanCmd, nil, nil); err != nil { + return fmt.Errorf("bundle sync: clean: %w", err) + } else if result.ExitCode != 0 { + return fmt.Errorf("bundle sync: clean: %s", result.Stderr) + } + + // Apply any uncommitted working-tree changes as a patch on top. + patch, err := gitutil.GeneratePatch(headRef) + if err != nil { + return fmt.Errorf("bundle sync: %w", err) + } + if patch != "" { + status(iostream.LevelInfo, fmt.Sprintf("Applying working-tree changes (%d bytes)...", len(patch))) + applyCmd := fmt.Sprintf("git -C %s apply", ShellEscape(repoPath)) + if result, err := ExecOverSSH(ctx, session, applyCmd, strings.NewReader(patch), nil); err != nil { + return fmt.Errorf("bundle sync: apply patch: %w", err) + } else if result.ExitCode != 0 { + return fmt.Errorf("bundle sync: apply patch: %s", result.Stderr) + } + } + + // Persist the synced ref. + active.LastSyncedRef = headRef + if err := SaveActive(ctx, *active); err != nil { + status(iostream.LevelWarn, fmt.Sprintf("Could not save last synced ref: %v", err)) + } + + status(iostream.LevelDone, "Synced") + return nil +} + +// sendBundle creates and transfers a git bundle (full or incremental) to the +// sidecar, then fetches it into the remote repo. +func sendBundle(ctx context.Context, session *Session, lastRef, cwd, repo, repoPath string, status iostream.StatusFunc) error { + label := "incremental bundle" + if lastRef == "" { + label = "full bundle" + } + + bundle, err := gitutil.CreateBundle(lastRef, cwd) + if err != nil { + return fmt.Errorf("bundle sync: %w", err) + } + status(iostream.LevelInfo, fmt.Sprintf("Sending %s (%d bytes)...", label, len(bundle))) + + bundlePath := fmt.Sprintf("/tmp/chunk-sync-%s.bundle", repo) + writeCmd := "cat > " + ShellEscape(bundlePath) + if result, err := ExecOverSSH(ctx, session, writeCmd, bytes.NewReader(bundle), nil); err != nil { + return fmt.Errorf("bundle sync: write bundle: %w", err) + } else if result.ExitCode != 0 { + return fmt.Errorf("bundle sync: write bundle: %s", result.Stderr) + } + + fetchCmd := fmt.Sprintf("git -C %s fetch %s HEAD:HEAD --update-head-ok", ShellEscape(repoPath), ShellEscape(bundlePath)) + if result, err := ExecOverSSH(ctx, session, fetchCmd, nil, nil); err != nil { + return fmt.Errorf("bundle sync: fetch: %w", err) + } else if result.ExitCode != 0 { + return fmt.Errorf("bundle sync: fetch: %s", result.Stderr) + } + return nil +} + var errApplyFailed = errors.New("apply failed") func syncWorkspace(ctx context.Context, status iostream.StatusFunc, org, repo, repoPath string, session *Session) error {