From aaa8d4719a8f8c291d2891f6bd5caf521119c854 Mon Sep 17 00:00:00 2001 From: Neha Sherpa Date: Tue, 3 Feb 2026 17:13:39 -0800 Subject: [PATCH 1/2] feat: Add GitHub App authentication for git cloning and releases Implements GitHub App authentication to enable authenticated access to private repositories and releases. This provides better security and higher rate limits compared to personal access tokens. Key features: - JWT generation with RSA signing for GitHub App authentication - Installation token management with 1-hour expiration and auto-refresh - Token caching to minimize GitHub API calls - Credential injection into git clone/fetch operations - Support for both git strategy and github-releases strategy - Falls back to system credentials if GitHub App not configured Implementation details: - New internal/githubapp package for authentication logic - Updated gitclone package to support credential providers - Modified git commands to inject tokens into GitHub URLs - Added GitHub App configuration blocks to both strategies - Updated configuration examples in cachew.hcl Usage: Configure GitHub App credentials (app ID, private key path, and installation IDs per org) in the git and github-releases strategy blocks to enable authenticated access. --- cachew.hcl | 11 +- cmd/cachewd/main.go | 29 ++-- docker/Dockerfile | 2 +- go.mod | 1 + go.sum | 2 + internal/config/config.go | 16 +- internal/gitclone/command.go | 51 ++++++- internal/gitclone/command_test.go | 4 +- internal/gitclone/manager.go | 73 +++++---- internal/gitclone/manager_test.go | 12 +- internal/githubapp/config.go | 90 +++++++++++ internal/githubapp/jwt.go | 78 ++++++++++ internal/githubapp/tokens.go | 178 ++++++++++++++++++++++ internal/strategy/git/bundle_test.go | 1 + internal/strategy/git/git.go | 35 ++++- internal/strategy/git/git_test.go | 1 + internal/strategy/git/integration_test.go | 1 + internal/strategy/git/snapshot_test.go | 1 + internal/strategy/git/spool_test.go | 1 + internal/strategy/github_releases.go | 65 +++++--- internal/strategy/github_releases_test.go | 7 +- internal/strategy/hermit_test.go | 3 +- 22 files changed, 571 insertions(+), 91 deletions(-) create mode 100644 internal/githubapp/config.go create mode 100644 internal/githubapp/jwt.go create mode 100644 internal/githubapp/tokens.go diff --git a/cachew.hcl b/cachew.hcl index 1f3d981..00362de 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -1,7 +1,3 @@ -# strategy git {} -# strategy docker {} -# strategy hermit {} - # Artifactory caching proxy strategy # artifactory "example.jfrog.io" { # target = "https://example.jfrog.io" @@ -16,8 +12,15 @@ git-clone { mirror-root = "./state/git-mirrors" } +github-app { + app-id = "${GITHUB_APP_ID}" + private-key-path = "${GITHUB_APP_PRIVATE_KEY_PATH}" + installations-json = "${GITHUB_APP_INSTALLATIONS}" +} + metrics {} + git { bundle-interval = "24h" snapshot-interval = "24h" diff --git a/cmd/cachewd/main.go b/cmd/cachewd/main.go index d8cc11e..d4c0ae4 100644 --- a/cmd/cachewd/main.go +++ b/cmd/cachewd/main.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "os" - "strings" "time" "github.com/alecthomas/chroma/v2/quick" @@ -19,6 +18,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/config" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" @@ -35,6 +35,7 @@ type GlobalConfig struct { LoggingConfig logging.Config `hcl:"log,block"` MetricsConfig metrics.Config `hcl:"metrics,block"` GitCloneConfig gitclone.Config `hcl:"git-clone,block"` + GithubAppConfig githubapp.Config `embed:"" hcl:"github-app,block" prefix:"github-app-"` } type CLI struct { @@ -51,13 +52,16 @@ func main() { ast, err := hcl.Parse(cli.Config) kctx.FatalIfErrorf(err) + // Expand environment variables in HCL (e.g., ${GITHUB_APP_ID}) + config.ExpandVars(ast, config.ParseEnvars()) + globalConfigHCL, providersConfigHCL := config.Split[GlobalConfig](ast) // Load global config. var globalConfig GlobalConfig globalSchema, err := hcl.Schema(&globalConfig) kctx.FatalIfErrorf(err) - config.InjectEnvars(globalSchema, globalConfigHCL, "CACHEW", parseEnvars()) + config.InjectEnvars(globalSchema, globalConfigHCL, "CACHEW", config.ParseEnvars()) err = hcl.UnmarshalAST(globalConfigHCL, &globalConfig, hcl.HydratedImplicitBlocks(true)) kctx.FatalIfErrorf(err) @@ -66,10 +70,11 @@ func main() { // Start initialising managerProvider := gitclone.NewManagerProvider(ctx, globalConfig.GitCloneConfig) + tokenManagerProvider := githubapp.NewTokenManagerProvider(globalConfig.GithubAppConfig, logger) scheduler := jobscheduler.New(ctx, globalConfig.SchedulerConfig) - cr, sr := newRegistries(globalConfig.URL, scheduler, managerProvider) + cr, sr := newRegistries(globalConfig.URL, scheduler, managerProvider, tokenManagerProvider) // Commands switch { //nolint:gocritic @@ -100,7 +105,7 @@ func main() { kctx.FatalIfErrorf(err) } -func newRegistries(cachewURL string, scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider) (*cache.Registry, *strategy.Registry) { +func newRegistries(cachewURL string, scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider, tokenManagerProvider githubapp.TokenManagerProvider) (*cache.Registry, *strategy.Registry) { cr := cache.NewRegistry() cache.RegisterMemory(cr) cache.RegisterDisk(cr) @@ -109,10 +114,10 @@ func newRegistries(cachewURL string, scheduler jobscheduler.Scheduler, cloneMana sr := strategy.NewRegistry() strategy.RegisterAPIV1(sr) strategy.RegisterArtifactory(sr) - strategy.RegisterGitHubReleases(sr) + strategy.RegisterGitHubReleases(sr, tokenManagerProvider) strategy.RegisterHermit(sr, cachewURL) strategy.RegisterHost(sr) - git.Register(sr, scheduler, cloneManagerProvider) + git.Register(sr, scheduler, cloneManagerProvider, tokenManagerProvider) gomod.Register(sr, cloneManagerProvider) return cr, sr @@ -144,7 +149,7 @@ func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, prov _, _ = w.Write([]byte("OK")) //nolint:errcheck }) - if err := config.Load(ctx, cr, sr, providersConfigHCL, mux, parseEnvars()); err != nil { + if err := config.Load(ctx, cr, sr, providersConfigHCL, mux, config.ParseEnvars()); err != nil { return nil, fmt.Errorf("load config: %w", err) } @@ -176,13 +181,3 @@ func newServer(ctx context.Context, mux *http.ServeMux, bind string, metricsConf }, } } - -func parseEnvars() map[string]string { - envars := map[string]string{} - for _, env := range os.Environ() { - if key, value, ok := strings.Cut(env, "="); ok { - envars[key] = value - } - } - return envars -} diff --git a/docker/Dockerfile b/docker/Dockerfile index b7bcb2b..90ce866 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ ARG TARGETARCH SHELL ["/bin/sh", "-o", "pipefail", "-c"] # Install runtime dependencies for git operations and TLS -RUN apk add --no-cache ca-certificates git tzdata && \ +RUN apk add --no-cache ca-certificates git git-daemon tzdata && \ addgroup -g 1000 cachew && \ adduser -D -u 1000 -G cachew cachew diff --git a/go.mod b/go.mod index d38e485..f442f5c 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.5 require ( github.com/alecthomas/hcl/v2 v2.6.0 github.com/alecthomas/kong v1.13.0 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/goproxy/goproxy v0.25.0 github.com/lmittmann/tint v1.1.2 github.com/minio/minio-go/v7 v7.0.97 diff --git a/go.sum b/go.sum index c097480..225bc64 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= diff --git a/internal/config/config.go b/internal/config/config.go index ad85116..1cb8d18 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -97,7 +97,7 @@ func Load( vars map[string]string, ) error { logger := logging.FromContext(ctx) - expandVars(ast, vars) + ExpandVars(ast, vars) strategyCandidates := []*hcl.Block{ // Always enable the default API strategy @@ -142,7 +142,19 @@ func Load( return nil } -func expandVars(ast *hcl.AST, vars map[string]string) { +// ParseEnvars returns a map of all environment variables. +func ParseEnvars() map[string]string { + envars := make(map[string]string) + for _, env := range os.Environ() { + if key, value, ok := strings.Cut(env, "="); ok { + envars[key] = value + } + } + return envars +} + +// ExpandVars expands environment variable references in HCL strings and heredocs. +func ExpandVars(ast *hcl.AST, vars map[string]string) { _ = hcl.Visit(ast, func(node hcl.Node, next func() error) error { //nolint:errcheck attr, ok := node.(*hcl.Attribute) if ok { diff --git a/internal/gitclone/command.go b/internal/gitclone/command.go index 31a3a17..391470f 100644 --- a/internal/gitclone/command.go +++ b/internal/gitclone/command.go @@ -5,14 +5,24 @@ package gitclone import ( "bufio" "context" + neturl "net/url" "os/exec" "strings" "github.com/alecthomas/errors" ) -func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, error) { - configArgs, err := getInsteadOfDisableArgsForURL(ctx, url) +func gitCommand(ctx context.Context, repoURL string, credentialProvider CredentialProvider, args ...string) (*exec.Cmd, error) { + modifiedURL := repoURL + if credentialProvider != nil && strings.Contains(repoURL, "github.com") { + token, err := credentialProvider.GetTokenForURL(ctx, repoURL) + if err == nil && token != "" { + modifiedURL = injectTokenIntoURL(repoURL, token) + } + // If error getting token, fall back to original URL (system credentials) + } + + configArgs, err := getInsteadOfDisableArgsForURL(ctx, repoURL) if err != nil { return nil, errors.Wrap(err, "get insteadOf disable args") } @@ -23,8 +33,41 @@ func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, err } allArgs = append(allArgs, args...) - cmd := exec.CommandContext(ctx, "git", allArgs...) - return cmd, nil + // Replace URL in args if it was modified for authentication + if modifiedURL != repoURL { + for i, arg := range allArgs { + if arg == repoURL { + allArgs[i] = modifiedURL + } + } + } + + return exec.CommandContext(ctx, "git", allArgs...), nil +} + +// Converts https://github.com/org/repo to https://x-access-token:TOKEN@github.com/org/repo +func injectTokenIntoURL(rawURL, token string) string { + if token == "" { + return rawURL + } + + u, err := neturl.Parse(rawURL) + if err != nil { + return rawURL + } + + // Only inject token for GitHub URLs + if !strings.Contains(u.Host, "github.com") { + return rawURL + } + + // Upgrade http to https for security + if u.Scheme == "http" { + u.Scheme = "https" + } + + u.User = neturl.UserPassword("x-access-token", token) + return u.String() } func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]string, error) { diff --git a/internal/gitclone/command_test.go b/internal/gitclone/command_test.go index e700a21..12bc2d3 100644 --- a/internal/gitclone/command_test.go +++ b/internal/gitclone/command_test.go @@ -45,7 +45,7 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) { func TestGitCommand(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "https://github.com/user/repo", "version") + cmd, err := gitCommand(ctx, "https://github.com/user/repo", nil, "version") assert.NoError(t, err) assert.NotZero(t, cmd) @@ -59,7 +59,7 @@ func TestGitCommand(t *testing.T) { func TestGitCommandWithEmptyURL(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "", "version") + cmd, err := gitCommand(ctx, "", nil, "version") assert.NoError(t, err) assert.NotZero(t, cmd) diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go index 3df5bec..172050e 100644 --- a/internal/gitclone/manager.go +++ b/internal/gitclone/manager.go @@ -58,23 +58,30 @@ type Config struct { RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"` } +// CredentialProvider provides credentials for git operations. +type CredentialProvider interface { + GetTokenForURL(ctx context.Context, url string) (string, error) +} + type Repository struct { - mu sync.RWMutex - config Config - state State - path string - upstreamURL string - lastFetch time.Time - lastRefCheck time.Time - refCheckValid bool - fetchSem chan struct{} + mu sync.RWMutex + config Config + state State + path string + upstreamURL string + lastFetch time.Time + lastRefCheck time.Time + refCheckValid bool + fetchSem chan struct{} + credentialProvider CredentialProvider } type Manager struct { - config Config - gitTuningConfig GitTuningConfig - clones map[string]*Repository - clonesMu sync.RWMutex + config Config + gitTuningConfig GitTuningConfig + clones map[string]*Repository + clonesMu sync.RWMutex + credentialProvider CredentialProvider } // ManagerProvider is a function that lazily creates a singleton Manager. @@ -82,11 +89,11 @@ type ManagerProvider func() (*Manager, error) func NewManagerProvider(ctx context.Context, config Config) ManagerProvider { return sync.OnceValues(func() (*Manager, error) { - return NewManager(ctx, config) + return NewManager(ctx, config, nil) }) } -func NewManager(ctx context.Context, config Config) (*Manager, error) { +func NewManager(ctx context.Context, config Config, credentialProvider CredentialProvider) (*Manager, error) { if config.MirrorRoot == "" { return nil, errors.New("mirror-root is required") } @@ -109,9 +116,10 @@ func NewManager(ctx context.Context, config Config) (*Manager, error) { "ref_check_interval", config.RefCheckInterval) return &Manager{ - config: config, - gitTuningConfig: DefaultGitTuningConfig(), - clones: make(map[string]*Repository), + config: config, + gitTuningConfig: DefaultGitTuningConfig(), + clones: make(map[string]*Repository), + credentialProvider: credentialProvider, }, nil } @@ -138,11 +146,12 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor clonePath := m.clonePathForURL(upstreamURL) repo = &Repository{ - state: StateEmpty, - config: m.config, - path: clonePath, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), + state: StateEmpty, + config: m.config, + path: clonePath, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + credentialProvider: m.credentialProvider, } gitDir := filepath.Join(clonePath, ".git") @@ -205,10 +214,12 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { upstreamURL := "https://" + host + "/" + repoPath repo := &Repository{ - state: StateReady, - path: path, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), + state: StateReady, + config: m.config, + path: path, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + credentialProvider: m.credentialProvider, } repo.fetchSem <- struct{}{} @@ -315,7 +326,7 @@ func (r *Repository) executeClone(ctx context.Context) error { r.upstreamURL, r.path, } - cmd, err := gitCommand(ctx, r.upstreamURL, args...) + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, args...) if err != nil { return errors.Wrap(err, "create git command") } @@ -331,7 +342,7 @@ func (r *Repository) executeClone(ctx context.Context) error { return errors.Wrapf(err, "configure fetch refspec: %s", string(output)) } - cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path, + cmd, err = gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())), @@ -370,7 +381,7 @@ func (r *Repository) Fetch(ctx context.Context) error { config := DefaultGitTuningConfig() // #nosec G204 - r.path is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path, + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())), @@ -460,7 +471,7 @@ func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) { // #nosec G204 - r.upstreamURL is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, "ls-remote", r.upstreamURL) + cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "ls-remote", r.upstreamURL) if err != nil { return nil, errors.Wrap(err, "create git command") } diff --git a/internal/gitclone/manager_test.go b/internal/gitclone/manager_test.go index 0793e30..439633e 100644 --- a/internal/gitclone/manager_test.go +++ b/internal/gitclone/manager_test.go @@ -24,7 +24,7 @@ func TestNewManager(t *testing.T) { RefCheckInterval: 10 * time.Second, } - manager, err := NewManager(ctx, config) + manager, err := NewManager(ctx, config, nil) assert.NoError(t, err) assert.NotZero(t, manager) assert.Equal(t, tmpDir, manager.config.MirrorRoot) @@ -37,7 +37,7 @@ func TestNewManager_RequiresRootDir(t *testing.T) { RefCheckInterval: 10 * time.Second, } - _, err := NewManager(ctx, config) + _, err := NewManager(ctx, config, nil) assert.Error(t, err) assert.Contains(t, err.Error(), "mirror-root is required") } @@ -51,7 +51,7 @@ func TestManager_GetOrCreate(t *testing.T) { RefCheckInterval: 10 * time.Second, } - manager, err := NewManager(ctx, config) + manager, err := NewManager(ctx, config, nil) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -77,7 +77,7 @@ func TestManager_GetOrCreate_ExistingClone(t *testing.T) { RefCheckInterval: 10 * time.Second, } - manager, err := NewManager(ctx, config) + manager, err := NewManager(ctx, config, nil) assert.NoError(t, err) repoPath := filepath.Join(tmpDir, "github.com", "user", "repo") @@ -102,7 +102,7 @@ func TestManager_Get(t *testing.T) { RefCheckInterval: 10 * time.Second, } - manager, err := NewManager(ctx, config) + manager, err := NewManager(ctx, config, nil) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -127,7 +127,7 @@ func TestManager_DiscoverExisting(t *testing.T) { RefCheckInterval: 10 * time.Second, } - manager, err := NewManager(ctx, config) + manager, err := NewManager(ctx, config, nil) assert.NoError(t, err) repos := []string{ diff --git a/internal/githubapp/config.go b/internal/githubapp/config.go new file mode 100644 index 0000000..e1cd607 --- /dev/null +++ b/internal/githubapp/config.go @@ -0,0 +1,90 @@ +// Package githubapp provides GitHub App authentication and token management. +package githubapp + +import ( + "encoding/json" + "log/slog" + "time" + + "github.com/alecthomas/errors" +) + +type Config struct { + AppID string `hcl:"app-id" help:"GitHub App ID"` + PrivateKeyPath string `hcl:"private-key-path" help:"Path to GitHub App private key (PEM format)"` + InstallationsJSON string `hcl:"installations-json" help:"JSON string mapping org names to installation IDs"` +} + +// Installations maps organization names to GitHub App installation IDs. +type Installations struct { + appID string + privateKeyPath string + orgs map[string]string +} + +// NewInstallations creates an Installations instance from config. +func NewInstallations(config Config, logger *slog.Logger) (*Installations, error) { + if config.InstallationsJSON == "" { + return nil, errors.New("installations-json is required") + } + + var orgs map[string]string + if err := json.Unmarshal([]byte(config.InstallationsJSON), &orgs); err != nil { + logger.Error("Failed to parse installations-json", + "error", err, + "installations_json", config.InstallationsJSON) + return nil, errors.Wrap(err, "parse installations-json") + } + + if len(orgs) == 0 { + return nil, errors.New("installations-json must contain at least one organization") + } + + logger.Info("GitHub App config initialized", + "app_id", config.AppID, + "private_key_path", config.PrivateKeyPath, + "installations", len(orgs)) + + return &Installations{ + appID: config.AppID, + privateKeyPath: config.PrivateKeyPath, + orgs: orgs, + }, nil +} + +func (i *Installations) IsConfigured() bool { + return i != nil && i.appID != "" && i.privateKeyPath != "" && len(i.orgs) > 0 +} + +func (i *Installations) GetInstallationID(org string) string { + if i == nil || i.orgs == nil { + return "" + } + return i.orgs[org] +} + +func (i *Installations) AppID() string { + if i == nil { + return "" + } + return i.appID +} + +func (i *Installations) PrivateKeyPath() string { + if i == nil { + return "" + } + return i.privateKeyPath +} + +type TokenCacheConfig struct { + RefreshBuffer time.Duration // How early to refresh before expiration + JWTExpiration time.Duration // GitHub allows max 10 minutes +} + +func DefaultTokenCacheConfig() TokenCacheConfig { + return TokenCacheConfig{ + RefreshBuffer: 5 * time.Minute, + JWTExpiration: 10 * time.Minute, + } +} diff --git a/internal/githubapp/jwt.go b/internal/githubapp/jwt.go new file mode 100644 index 0000000..a2b5edb --- /dev/null +++ b/internal/githubapp/jwt.go @@ -0,0 +1,78 @@ +package githubapp + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" + "time" + + "github.com/alecthomas/errors" + "github.com/golang-jwt/jwt/v5" +) + +type JWTGenerator struct { + appID string + privateKey *rsa.PrivateKey + expiration time.Duration +} + +func NewJWTGenerator(appID, privateKeyPath string, expiration time.Duration) (*JWTGenerator, error) { + privateKey, err := loadPrivateKey(privateKeyPath) + if err != nil { + return nil, errors.Wrap(err, "load private key") + } + + return &JWTGenerator{ + appID: appID, + privateKey: privateKey, + expiration: expiration, + }, nil +} + +func (g *JWTGenerator) GenerateJWT() (string, error) { + now := time.Now() + + claims := jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(g.expiration)), + Issuer: g.appID, + } + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + signedToken, err := token.SignedString(g.privateKey) + if err != nil { + return "", errors.Wrap(err, "sign JWT") + } + + return signedToken, nil +} + +func loadPrivateKey(path string) (*rsa.PrivateKey, error) { + keyData, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrapf(err, "read private key file: %s", path) + } + + block, _ := pem.Decode(keyData) + if block == nil { + return nil, errors.Errorf("failed to decode PEM block from private key file: %s", path) + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err == nil { + return privateKey, nil + } + + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.Wrap(err, "parse private key (tried both PKCS1 and PKCS8)") + } + + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, errors.Errorf("private key is not RSA (type: %T)", key) + } + + return rsaKey, nil +} diff --git a/internal/githubapp/tokens.go b/internal/githubapp/tokens.go new file mode 100644 index 0000000..51b4a6e --- /dev/null +++ b/internal/githubapp/tokens.go @@ -0,0 +1,178 @@ +package githubapp + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "sync" + "time" + + "github.com/alecthomas/errors" + + "github.com/block/cachew/internal/logging" +) + +// TokenManagerProvider is a function that lazily creates a singleton TokenManager. +type TokenManagerProvider func() (*TokenManager, error) + +// NewTokenManagerProvider creates a provider that lazily initializes a TokenManager. +func NewTokenManagerProvider(config Config, logger *slog.Logger) TokenManagerProvider { + return sync.OnceValues(func() (*TokenManager, error) { + if config.AppID == "" || config.PrivateKeyPath == "" || config.InstallationsJSON == "" { + return nil, nil // Not configured, return nil without error + } + + installations, err := NewInstallations(config, logger) + if err != nil { + return nil, errors.Wrap(err, "create installations") + } + + return NewTokenManager(installations, DefaultTokenCacheConfig()) + }) +} + +type TokenManager struct { + installations *Installations + cacheConfig TokenCacheConfig + jwtGenerator *JWTGenerator + httpClient *http.Client + + mu sync.RWMutex + tokens map[string]*cachedToken +} + +type cachedToken struct { + token string + expiresAt time.Time +} + +func NewTokenManager(installations *Installations, cacheConfig TokenCacheConfig) (*TokenManager, error) { + if !installations.IsConfigured() { + return nil, errors.New("GitHub App not configured") + } + + jwtGenerator, err := NewJWTGenerator(installations.AppID(), installations.PrivateKeyPath(), cacheConfig.JWTExpiration) + if err != nil { + return nil, errors.Wrap(err, "create JWT generator") + } + + return &TokenManager{ + installations: installations, + cacheConfig: cacheConfig, + jwtGenerator: jwtGenerator, + httpClient: http.DefaultClient, + tokens: make(map[string]*cachedToken), + }, nil +} + +func (tm *TokenManager) GetTokenForOrg(ctx context.Context, org string) (string, error) { + if tm == nil { + return "", errors.New("token manager not initialized") + } + logger := logging.FromContext(ctx).With(slog.String("org", org)) + + installationID := tm.installations.GetInstallationID(org) + if installationID == "" { + return "", errors.Errorf("no GitHub App installation configured for org: %s", org) + } + + tm.mu.RLock() + cached, exists := tm.tokens[org] + tm.mu.RUnlock() + + if exists && time.Now().Add(tm.cacheConfig.RefreshBuffer).Before(cached.expiresAt) { + logger.DebugContext(ctx, "Using cached GitHub App token") + return cached.token, nil + } + + logger.DebugContext(ctx, "Fetching new GitHub App installation token", + slog.String("installation_id", installationID)) + + token, expiresAt, err := tm.fetchInstallationToken(ctx, installationID) + if err != nil { + return "", errors.Wrap(err, "fetch installation token") + } + + tm.mu.Lock() + tm.tokens[org] = &cachedToken{ + token: token, + expiresAt: expiresAt, + } + tm.mu.Unlock() + + logger.InfoContext(ctx, "GitHub App token refreshed", + slog.Time("expires_at", expiresAt)) + + return token, nil +} + +func (tm *TokenManager) GetTokenForURL(ctx context.Context, url string) (string, error) { + if tm == nil { + return "", errors.New("token manager not initialized") + } + org, err := extractOrgFromURL(url) + if err != nil { + return "", err + } + + return tm.GetTokenForOrg(ctx, org) +} + +func (tm *TokenManager) fetchInstallationToken(ctx context.Context, installationID string) (string, time.Time, error) { + jwt, err := tm.jwtGenerator.GenerateJWT() + if err != nil { + return "", time.Time{}, errors.Wrap(err, "generate JWT") + } + + url := fmt.Sprintf("https://api.github.com/app/installations/%s/access_tokens", installationID) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) + if err != nil { + return "", time.Time{}, errors.Wrap(err, "create request") + } + + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Authorization", "Bearer "+jwt) + req.Header.Set("X-Github-Api-Version", "2022-11-28") + + resp, err := tm.httpClient.Do(req) + if err != nil { + return "", time.Time{}, errors.Wrap(err, "execute request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + return "", time.Time{}, errors.Errorf("GitHub API returned status %d", resp.StatusCode) + } + + var result struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", time.Time{}, errors.Wrap(err, "decode response") + } + + return result.Token, result.ExpiresAt, nil +} + +func extractOrgFromURL(url string) (string, error) { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + url = strings.TrimPrefix(url, "git@") + + if !strings.HasPrefix(url, "github.com/") && !strings.HasPrefix(url, "github.com:") { + return "", errors.Errorf("not a GitHub URL: %s", url) + } + url = strings.TrimPrefix(url, "github.com/") + url = strings.TrimPrefix(url, "github.com:") + + parts := strings.Split(url, "/") + if len(parts) < 1 || parts[0] == "" { + return "", errors.Errorf("cannot extract org from URL: %s", url) + } + + return parts[0], nil +} diff --git a/internal/strategy/git/bundle_test.go b/internal/strategy/git/bundle_test.go index 5d21fdf..bdcfb3c 100644 --- a/internal/strategy/git/bundle_test.go +++ b/internal/strategy/git/bundle_test.go @@ -11,6 +11,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/git" diff --git a/internal/strategy/git/git.go b/internal/strategy/git/git.go index 17de96e..57cffc3 100644 --- a/internal/strategy/git/git.go +++ b/internal/strategy/git/git.go @@ -21,14 +21,15 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy" ) -func Register(r *strategy.Registry, scheduler jobscheduler.Scheduler, cloneManager gitclone.ManagerProvider) { +func Register(r *strategy.Registry, scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider, tokenManagerProvider githubapp.TokenManagerProvider) { strategy.Register(r, "git", "Caches Git repositories, including bundle and tarball snapshots.", func(ctx context.Context, config Config, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { - return New(ctx, config, scheduler, cache, mux, cloneManager) + return New(ctx, config, scheduler, cache, mux, cloneManagerProvider, tokenManagerProvider) }) } @@ -47,6 +48,7 @@ type Strategy struct { scheduler jobscheduler.Scheduler spoolsMu sync.Mutex spools map[string]*RepoSpools + tokenManager *githubapp.TokenManager } func New( @@ -56,9 +58,21 @@ func New( cache cache.Cache, mux strategy.Mux, cloneManagerProvider gitclone.ManagerProvider, + tokenManagerProvider githubapp.TokenManagerProvider, ) (*Strategy, error) { logger := logging.FromContext(ctx) + // Get GitHub App token manager if configured + tokenManager, err := tokenManagerProvider() + if err != nil { + return nil, errors.Wrap(err, "create token manager") + } + if tokenManager != nil { + logger.InfoContext(ctx, "Using GitHub App authentication for git strategy") + } else { + logger.WarnContext(ctx, "GitHub App not configured, using system git credentials") + } + cloneManager, err := cloneManagerProvider() if err != nil { return nil, errors.Wrap(err, "failed to create clone manager") @@ -75,6 +89,7 @@ func New( ctx: ctx, scheduler: scheduler.WithQueuePrefix("git"), spools: make(map[string]*RepoSpools), + tokenManager: tokenManager, } existing, err := s.cloneManager.DiscoverExisting(ctx) @@ -97,6 +112,22 @@ func New( req.URL.Host = req.PathValue("host") req.URL.Path = "/" + req.PathValue("path") req.Host = req.URL.Host + + // Inject GitHub App authentication for github.com requests + if s.tokenManager != nil && req.URL.Host == "github.com" { + // Extract org from path (e.g., /squareup/blox.git/...) + parts := strings.Split(strings.TrimPrefix(req.URL.Path, "/"), "/") + if len(parts) >= 1 && parts[0] != "" { + org := parts[0] + token, err := s.tokenManager.GetTokenForOrg(req.Context(), org) + if err == nil && token != "" { + // Inject token as Basic auth with "x-access-token" username + req.SetBasicAuth("x-access-token", token) + logger.DebugContext(req.Context(), "Injecting GitHub App auth into upstream request", + slog.String("org", org)) + } + } + } }, Transport: s.httpClient.Transport, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { diff --git a/internal/strategy/git/git_test.go b/internal/strategy/git/git_test.go index 03200db..c94a331 100644 --- a/internal/strategy/git/git_test.go +++ b/internal/strategy/git/git_test.go @@ -11,6 +11,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/git" diff --git a/internal/strategy/git/integration_test.go b/internal/strategy/git/integration_test.go index 41dea51..ad63320 100644 --- a/internal/strategy/git/integration_test.go +++ b/internal/strategy/git/integration_test.go @@ -20,6 +20,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/git" diff --git a/internal/strategy/git/snapshot_test.go b/internal/strategy/git/snapshot_test.go index 594971a..6b37a3d 100644 --- a/internal/strategy/git/snapshot_test.go +++ b/internal/strategy/git/snapshot_test.go @@ -11,6 +11,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/gitclone" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/git" diff --git a/internal/strategy/git/spool_test.go b/internal/strategy/git/spool_test.go index a9cf3db..b2e2af3 100644 --- a/internal/strategy/git/spool_test.go +++ b/internal/strategy/git/spool_test.go @@ -12,6 +12,7 @@ import ( "github.com/alecthomas/assert/v2" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/strategy/git" ) diff --git a/internal/strategy/github_releases.go b/internal/strategy/github_releases.go index fa2da49..5add55b 100644 --- a/internal/strategy/github_releases.go +++ b/internal/strategy/github_releases.go @@ -11,37 +11,51 @@ import ( "github.com/alecthomas/errors" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/handler" ) -func RegisterGitHubReleases(r *Registry) { - Register(r, "github-releases", "Caches public and authenticated GitHub releases.", NewGitHubReleases) +func RegisterGitHubReleases(r *Registry, tokenManagerProvider githubapp.TokenManagerProvider) { + Register(r, "github-releases", "Caches public and authenticated GitHub releases.", func(ctx context.Context, config GitHubReleasesConfig, cache cache.Cache, mux Mux) (*GitHubReleases, error) { + return NewGitHubReleases(ctx, config, cache, mux, tokenManagerProvider) + }) } type GitHubReleasesConfig struct { - Token string `hcl:"token" help:"GitHub token for authentication."` + Token string `hcl:"token,optional" help:"GitHub token for authentication."` PrivateOrgs []string `hcl:"private-orgs" help:"List of private GitHub organisations."` } // The GitHubReleases strategy fetches private (and public) release binaries from GitHub. type GitHubReleases struct { - config GitHubReleasesConfig - cache cache.Cache - client *http.Client + config GitHubReleasesConfig + cache cache.Cache + client *http.Client + tokenManager *githubapp.TokenManager } // NewGitHubReleases creates a [Strategy] that fetches private (and public) release binaries from GitHub. -func NewGitHubReleases(ctx context.Context, config GitHubReleasesConfig, cache cache.Cache, mux Mux) (*GitHubReleases, error) { - s := &GitHubReleases{ - config: config, - cache: cache, - client: http.DefaultClient, - } +func NewGitHubReleases(ctx context.Context, config GitHubReleasesConfig, cache cache.Cache, mux Mux, tokenManagerProvider githubapp.TokenManagerProvider) (*GitHubReleases, error) { logger := logging.FromContext(ctx) - if config.Token == "" { - logger.WarnContext(ctx, "No token configured for github-releases strategy") + + // Get GitHub App token manager if configured + tokenManager, err := tokenManagerProvider() + if err != nil { + return nil, errors.Wrap(err, "create token manager") + } + if tokenManager != nil { + logger.InfoContext(ctx, "Using GitHub App authentication for github-releases strategy") + } else if config.Token == "" { + logger.WarnContext(ctx, "No authentication configured for github-releases strategy") + } + + s := &GitHubReleases{ + config: config, + cache: cache, + client: http.DefaultClient, + tokenManager: tokenManager, } // eg. https://github.com/alecthomas/chroma/releases/download/v2.21.1/chroma-2.21.1-darwin-amd64.tar.gz h := handler.New(s.client, cache). @@ -68,16 +82,31 @@ var _ Strategy = (*GitHubReleases)(nil) func (g *GitHubReleases) String() string { return "github-releases" } // newGitHubRequest creates a new HTTP request with GitHub API headers and authentication. -func (g *GitHubReleases) newGitHubRequest(ctx context.Context, url, accept string) (*http.Request, error) { +func (g *GitHubReleases) newGitHubRequest(ctx context.Context, url, accept, org string) (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, errors.Wrap(err, "create request") } req.Header.Set("Accept", accept) req.Header.Set("X-Github-Api-Version", "2022-11-28") - if g.config.Token != "" { + + // Try GitHub App authentication first, fall back to static token + if g.tokenManager != nil && org != "" { + token, err := g.tokenManager.GetTokenForOrg(ctx, org) + if err != nil { + logging.FromContext(ctx).WarnContext(ctx, "Failed to get GitHub App token, falling back to static token", + slog.String("org", org), + slog.String("error", err.Error())) + } else if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + } + + // Fall back to static token if GitHub App not used or failed + if req.Header.Get("Authorization") == "" && g.config.Token != "" { req.Header.Set("Authorization", "Bearer "+g.config.Token) } + return req, nil } @@ -107,7 +136,7 @@ func (g *GitHubReleases) downloadRelease(ctx context.Context, org, repo, release // Use GitHub API to get release info and find the asset logger.DebugContext(ctx, "Using GitHub API for private release") apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", org, repo, release) - req, err := g.newGitHubRequest(ctx, apiURL, "application/vnd.github+json") + req, err := g.newGitHubRequest(ctx, apiURL, "application/vnd.github+json", org) if err != nil { return nil, httputil.Errorf(http.StatusInternalServerError, "create API request") } @@ -148,7 +177,7 @@ func (g *GitHubReleases) downloadRelease(ctx context.Context, org, repo, release logger.DebugContext(ctx, "Found asset in release", slog.String("asset_url", assetURL)) // Create request for the asset download - req, err = g.newGitHubRequest(ctx, assetURL, "application/octet-stream") + req, err = g.newGitHubRequest(ctx, assetURL, "application/octet-stream", org) if err != nil { return nil, httputil.Errorf(http.StatusInternalServerError, "create asset request failed: %w", err) } diff --git a/internal/strategy/github_releases_test.go b/internal/strategy/github_releases_test.go index 1671481..81865b0 100644 --- a/internal/strategy/github_releases_test.go +++ b/internal/strategy/github_releases_test.go @@ -14,6 +14,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy" ) @@ -126,7 +127,7 @@ func setupTest(t *testing.T, config strategy.GitHubReleasesConfig) (*mockGitHubS t.Cleanup(func() { memCache.Close() }) mux := http.NewServeMux() - _, err = strategy.NewGitHubReleases(ctx, config, memCache, mux) + _, err = strategy.NewGitHubReleases(ctx, config, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) assert.NoError(t, err) return mock, mux, ctx @@ -241,7 +242,7 @@ func TestGitHubReleasesNoToken(t *testing.T) { defer memCache.Close() mux := http.NewServeMux() - gh, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux) + gh, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) assert.NoError(t, err) assert.Equal(t, "github-releases", gh.String()) } @@ -255,7 +256,7 @@ func TestGitHubReleasesString(t *testing.T) { mux := http.NewServeMux() gh, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{ Token: "test-token", - }, memCache, mux) + }, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) assert.NoError(t, err) assert.Equal(t, "github-releases", gh.String()) diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go index 55f7355..b8362f8 100644 --- a/internal/strategy/hermit_test.go +++ b/internal/strategy/hermit_test.go @@ -12,6 +12,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy" ) @@ -94,7 +95,7 @@ func TestHermitGitHubRelease(t *testing.T) { mux, ctx, memCache := setupHermitTest(t) - _, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux) + _, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) assert.NoError(t, err) req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", nil) From 58df2a648d8dae031c7df24a8114b89f519c701b Mon Sep 17 00:00:00 2001 From: Neha Sherpa Date: Mon, 9 Feb 2026 11:21:09 -0800 Subject: [PATCH 2/2] fix: Address comments --- cachew.hcl | 7 +++--- cmd/cachewd/main.go | 24 ++++++++++++++------ internal/config/config.go | 15 ++----------- internal/gitclone/command.go | 13 ++++++----- internal/gitclone/command_test.go | 14 ++++++++++-- internal/gitclone/manager.go | 12 +++++----- internal/githubapp/config.go | 27 ++++++----------------- internal/githubapp/tokens.go | 2 +- internal/strategy/git/bundle_test.go | 8 +++---- internal/strategy/git/git_test.go | 12 +++++----- internal/strategy/git/integration_test.go | 16 +++++++------- internal/strategy/git/snapshot_test.go | 8 +++---- internal/strategy/git/spool_test.go | 1 - internal/strategy/github_releases_test.go | 6 ++--- internal/strategy/gomod/gomod_test.go | 2 +- internal/strategy/hermit_test.go | 2 +- 16 files changed, 83 insertions(+), 86 deletions(-) diff --git a/cachew.hcl b/cachew.hcl index 00362de..dd0ef91 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -13,9 +13,10 @@ git-clone { } github-app { - app-id = "${GITHUB_APP_ID}" - private-key-path = "${GITHUB_APP_PRIVATE_KEY_PATH}" - installations-json = "${GITHUB_APP_INSTALLATIONS}" + # Uncomment and add: + # app-id = "app-id-value" (Can also be passed via setting envar CACHEW_GITHUB_APP_APP_ID) + # private-key-path = "private-key-path-value" (Can also be passed via envar CACHEW_GITHUB_APP_PRIVATE_KEY_PATH) + # installations = { "myorg" : "installation-id" } } metrics {} diff --git a/cmd/cachewd/main.go b/cmd/cachewd/main.go index d4c0ae4..8500782 100644 --- a/cmd/cachewd/main.go +++ b/cmd/cachewd/main.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "os" + "strings" "time" "github.com/alecthomas/chroma/v2/quick" @@ -35,7 +36,7 @@ type GlobalConfig struct { LoggingConfig logging.Config `hcl:"log,block"` MetricsConfig metrics.Config `hcl:"metrics,block"` GitCloneConfig gitclone.Config `hcl:"git-clone,block"` - GithubAppConfig githubapp.Config `embed:"" hcl:"github-app,block" prefix:"github-app-"` + GithubAppConfig githubapp.Config `embed:"" hcl:"github-app,block,optional" prefix:"github-app-"` } type CLI struct { @@ -52,16 +53,13 @@ func main() { ast, err := hcl.Parse(cli.Config) kctx.FatalIfErrorf(err) - // Expand environment variables in HCL (e.g., ${GITHUB_APP_ID}) - config.ExpandVars(ast, config.ParseEnvars()) - globalConfigHCL, providersConfigHCL := config.Split[GlobalConfig](ast) // Load global config. var globalConfig GlobalConfig globalSchema, err := hcl.Schema(&globalConfig) kctx.FatalIfErrorf(err) - config.InjectEnvars(globalSchema, globalConfigHCL, "CACHEW", config.ParseEnvars()) + config.InjectEnvars(globalSchema, globalConfigHCL, "CACHEW", parseEnvars()) err = hcl.UnmarshalAST(globalConfigHCL, &globalConfig, hcl.HydratedImplicitBlocks(true)) kctx.FatalIfErrorf(err) @@ -69,8 +67,10 @@ func main() { logger, ctx := logging.Configure(ctx, globalConfig.LoggingConfig) // Start initialising - managerProvider := gitclone.NewManagerProvider(ctx, globalConfig.GitCloneConfig) tokenManagerProvider := githubapp.NewTokenManagerProvider(globalConfig.GithubAppConfig, logger) + tokenManager, err := tokenManagerProvider() + kctx.FatalIfErrorf(err) + managerProvider := gitclone.NewManagerProvider(ctx, globalConfig.GitCloneConfig, tokenManager) scheduler := jobscheduler.New(ctx, globalConfig.SchedulerConfig) @@ -149,7 +149,7 @@ func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, prov _, _ = w.Write([]byte("OK")) //nolint:errcheck }) - if err := config.Load(ctx, cr, sr, providersConfigHCL, mux, config.ParseEnvars()); err != nil { + if err := config.Load(ctx, cr, sr, providersConfigHCL, mux, parseEnvars()); err != nil { return nil, fmt.Errorf("load config: %w", err) } @@ -181,3 +181,13 @@ func newServer(ctx context.Context, mux *http.ServeMux, bind string, metricsConf }, } } + +func parseEnvars() map[string]string { + envars := map[string]string{} + for _, env := range os.Environ() { + if key, value, ok := strings.Cut(env, "="); ok { + envars[key] = value + } + } + return envars +} diff --git a/internal/config/config.go b/internal/config/config.go index 1cb8d18..52762cf 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -97,7 +97,7 @@ func Load( vars map[string]string, ) error { logger := logging.FromContext(ctx) - ExpandVars(ast, vars) + expandVars(ast, vars) strategyCandidates := []*hcl.Block{ // Always enable the default API strategy @@ -142,19 +142,8 @@ func Load( return nil } -// ParseEnvars returns a map of all environment variables. -func ParseEnvars() map[string]string { - envars := make(map[string]string) - for _, env := range os.Environ() { - if key, value, ok := strings.Cut(env, "="); ok { - envars[key] = value - } - } - return envars -} - // ExpandVars expands environment variable references in HCL strings and heredocs. -func ExpandVars(ast *hcl.AST, vars map[string]string) { +func expandVars(ast *hcl.AST, vars map[string]string) { _ = hcl.Visit(ast, func(node hcl.Node, next func() error) error { //nolint:errcheck attr, ok := node.(*hcl.Attribute) if ok { diff --git a/internal/gitclone/command.go b/internal/gitclone/command.go index 391470f..e003a7e 100644 --- a/internal/gitclone/command.go +++ b/internal/gitclone/command.go @@ -5,17 +5,18 @@ package gitclone import ( "bufio" "context" - neturl "net/url" + "net/url" "os/exec" "strings" "github.com/alecthomas/errors" ) -func gitCommand(ctx context.Context, repoURL string, credentialProvider CredentialProvider, args ...string) (*exec.Cmd, error) { +func (r *Repository) gitCommand(ctx context.Context, args ...string) (*exec.Cmd, error) { + repoURL := r.upstreamURL modifiedURL := repoURL - if credentialProvider != nil && strings.Contains(repoURL, "github.com") { - token, err := credentialProvider.GetTokenForURL(ctx, repoURL) + if r.credentialProvider != nil && strings.Contains(repoURL, "github.com") { + token, err := r.credentialProvider.GetTokenForURL(ctx, repoURL) if err == nil && token != "" { modifiedURL = injectTokenIntoURL(repoURL, token) } @@ -51,7 +52,7 @@ func injectTokenIntoURL(rawURL, token string) string { return rawURL } - u, err := neturl.Parse(rawURL) + u, err := url.Parse(rawURL) if err != nil { return rawURL } @@ -66,7 +67,7 @@ func injectTokenIntoURL(rawURL, token string) string { u.Scheme = "https" } - u.User = neturl.UserPassword("x-access-token", token) + u.User = url.UserPassword("x-access-token", token) return u.String() } diff --git a/internal/gitclone/command_test.go b/internal/gitclone/command_test.go index 12bc2d3..c5757ce 100644 --- a/internal/gitclone/command_test.go +++ b/internal/gitclone/command_test.go @@ -45,7 +45,12 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) { func TestGitCommand(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "https://github.com/user/repo", nil, "version") + repo := &Repository{ + upstreamURL: "https://github.com/user/repo", + credentialProvider: nil, + } + + cmd, err := repo.gitCommand(ctx, "version") assert.NoError(t, err) assert.NotZero(t, cmd) @@ -59,7 +64,12 @@ func TestGitCommand(t *testing.T) { func TestGitCommandWithEmptyURL(t *testing.T) { ctx := context.Background() - cmd, err := gitCommand(ctx, "", nil, "version") + repo := &Repository{ + upstreamURL: "", + credentialProvider: nil, + } + + cmd, err := repo.gitCommand(ctx, "version") assert.NoError(t, err) assert.NotZero(t, cmd) diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go index 172050e..cbde6a8 100644 --- a/internal/gitclone/manager.go +++ b/internal/gitclone/manager.go @@ -87,9 +87,9 @@ type Manager struct { // ManagerProvider is a function that lazily creates a singleton Manager. type ManagerProvider func() (*Manager, error) -func NewManagerProvider(ctx context.Context, config Config) ManagerProvider { +func NewManagerProvider(ctx context.Context, config Config, credentialProvider CredentialProvider) ManagerProvider { return sync.OnceValues(func() (*Manager, error) { - return NewManager(ctx, config, nil) + return NewManager(ctx, config, credentialProvider) }) } @@ -326,7 +326,7 @@ func (r *Repository) executeClone(ctx context.Context) error { r.upstreamURL, r.path, } - cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, args...) + cmd, err := r.gitCommand(ctx, args...) if err != nil { return errors.Wrap(err, "create git command") } @@ -342,7 +342,7 @@ func (r *Repository) executeClone(ctx context.Context) error { return errors.Wrapf(err, "configure fetch refspec: %s", string(output)) } - cmd, err = gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, + cmd, err = r.gitCommand(ctx, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())), @@ -381,7 +381,7 @@ func (r *Repository) Fetch(ctx context.Context) error { config := DefaultGitTuningConfig() // #nosec G204 - r.path is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path, + cmd, err := r.gitCommand(ctx, "-C", r.path, "-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer), "-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit), "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())), @@ -471,7 +471,7 @@ func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) { // #nosec G204 - r.upstreamURL is controlled by us - cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "ls-remote", r.upstreamURL) + cmd, err := r.gitCommand(ctx, "ls-remote", r.upstreamURL) if err != nil { return nil, errors.Wrap(err, "create git command") } diff --git a/internal/githubapp/config.go b/internal/githubapp/config.go index e1cd607..ee3d03b 100644 --- a/internal/githubapp/config.go +++ b/internal/githubapp/config.go @@ -2,7 +2,6 @@ package githubapp import ( - "encoding/json" "log/slog" "time" @@ -10,9 +9,9 @@ import ( ) type Config struct { - AppID string `hcl:"app-id" help:"GitHub App ID"` - PrivateKeyPath string `hcl:"private-key-path" help:"Path to GitHub App private key (PEM format)"` - InstallationsJSON string `hcl:"installations-json" help:"JSON string mapping org names to installation IDs"` + AppID string `hcl:"app-id,optional" help:"GitHub App ID"` + PrivateKeyPath string `hcl:"private-key-path,optional" help:"Path to GitHub App private key (PEM format)"` + Installations map[string]string `hcl:"installations,optional" help:"Mapping of org names to installation IDs"` } // Installations maps organization names to GitHub App installation IDs. @@ -24,31 +23,19 @@ type Installations struct { // NewInstallations creates an Installations instance from config. func NewInstallations(config Config, logger *slog.Logger) (*Installations, error) { - if config.InstallationsJSON == "" { - return nil, errors.New("installations-json is required") - } - - var orgs map[string]string - if err := json.Unmarshal([]byte(config.InstallationsJSON), &orgs); err != nil { - logger.Error("Failed to parse installations-json", - "error", err, - "installations_json", config.InstallationsJSON) - return nil, errors.Wrap(err, "parse installations-json") - } - - if len(orgs) == 0 { - return nil, errors.New("installations-json must contain at least one organization") + if len(config.Installations) == 0 { + return nil, errors.New("installations is required") } logger.Info("GitHub App config initialized", "app_id", config.AppID, "private_key_path", config.PrivateKeyPath, - "installations", len(orgs)) + "installations", len(config.Installations)) return &Installations{ appID: config.AppID, privateKeyPath: config.PrivateKeyPath, - orgs: orgs, + orgs: config.Installations, }, nil } diff --git a/internal/githubapp/tokens.go b/internal/githubapp/tokens.go index 51b4a6e..9eefaec 100644 --- a/internal/githubapp/tokens.go +++ b/internal/githubapp/tokens.go @@ -21,7 +21,7 @@ type TokenManagerProvider func() (*TokenManager, error) // NewTokenManagerProvider creates a provider that lazily initializes a TokenManager. func NewTokenManagerProvider(config Config, logger *slog.Logger) TokenManagerProvider { return sync.OnceValues(func() (*TokenManager, error) { - if config.AppID == "" || config.PrivateKeyPath == "" || config.InstallationsJSON == "" { + if config.AppID == "" || config.PrivateKeyPath == "" || len(config.Installations) == 0 { return nil, nil // Not configured, return nil without error } diff --git a/internal/strategy/git/bundle_test.go b/internal/strategy/git/bundle_test.go index bdcfb3c..915b7ff 100644 --- a/internal/strategy/git/bundle_test.go +++ b/internal/strategy/git/bundle_test.go @@ -23,7 +23,7 @@ func TestBundleHTTPEndpoint(t *testing.T) { cloneManager := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, - }) + }, nil) memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{}) assert.NoError(t, err) @@ -31,7 +31,7 @@ func TestBundleHTTPEndpoint(t *testing.T) { _, err = git.New(ctx, git.Config{ BundleInterval: 24 * time.Hour, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cloneManager) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cloneManager, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) // Create a fake bundle in the cache @@ -106,11 +106,11 @@ func TestBundleInterval(t *testing.T) { cloneManager := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, - }) + }, nil) s, err := git.New(ctx, git.Config{ BundleInterval: tt.bundleInterval, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cloneManager) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cloneManager, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) assert.NotZero(t, s) diff --git a/internal/strategy/git/git_test.go b/internal/strategy/git/git_test.go index c94a331..cc37699 100644 --- a/internal/strategy/git/git_test.go +++ b/internal/strategy/git/git_test.go @@ -67,8 +67,8 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := newTestMux() - cm := gitclone.NewManagerProvider(ctx, tt.config) - s, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm) + cm := gitclone.NewManagerProvider(ctx, tt.config, nil) + s, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil if tt.wantError != "" { assert.Error(t, err) assert.Contains(t, err.Error(), tt.wantError) @@ -151,8 +151,8 @@ func TestNewWithExistingCloneOnDisk(t *testing.T) { cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, FetchInterval: 15, - }) - s, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm) + }, nil) + s, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) assert.NotZero(t, s) } @@ -175,8 +175,8 @@ func TestIntegrationWithMockUpstream(t *testing.T) { cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, FetchInterval: 15, - }) - _, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm) + }, nil) + _, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) // Verify handlers exist diff --git a/internal/strategy/git/integration_test.go b/internal/strategy/git/integration_test.go index ad63320..e6b6d88 100644 --- a/internal/strategy/git/integration_test.go +++ b/internal/strategy/git/integration_test.go @@ -61,9 +61,9 @@ func TestIntegrationGitCloneViaProxy(t *testing.T) { gc := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: clonesDir, FetchInterval: 15, - }) + }, nil) mux := http.NewServeMux() - strategy, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc) + strategy, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) assert.NotZero(t, strategy) @@ -140,10 +140,10 @@ func TestIntegrationGitFetchViaProxy(t *testing.T) { gc := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: clonesDir, FetchInterval: 15, - }) + }, nil) mux := http.NewServeMux() - _, err = git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc) + _, err = git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) server := testServerWithLogging(ctx, mux) @@ -222,8 +222,8 @@ func TestIntegrationPushForwardsToUpstream(t *testing.T) { gc := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: clonesDir, FetchInterval: 15, - }) - _, err = git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc) + }, nil) + _, err = git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) server := testServerWithLogging(ctx, mux) @@ -316,8 +316,8 @@ func TestIntegrationSpoolReusesDuringClone(t *testing.T) { gc := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: clonesDir, FetchInterval: 15, - }) - strategy, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc) + }, nil) + strategy, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, gc, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) strategy.SetHTTPTransport(&countingTransport{ diff --git a/internal/strategy/git/snapshot_test.go b/internal/strategy/git/snapshot_test.go index 6b37a3d..52b6085 100644 --- a/internal/strategy/git/snapshot_test.go +++ b/internal/strategy/git/snapshot_test.go @@ -27,10 +27,10 @@ func TestSnapshotHTTPEndpoint(t *testing.T) { cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, - }) + }, nil) _, err = git.New(ctx, git.Config{ SnapshotInterval: 24 * time.Hour, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cm) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) // Create a fake snapshot in the cache @@ -101,10 +101,10 @@ func TestSnapshotInterval(t *testing.T) { cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, - }) + }, nil) s, err := git.New(ctx, git.Config{ SnapshotInterval: tt.snapshotInterval, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cm) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) assert.NotZero(t, s) }) diff --git a/internal/strategy/git/spool_test.go b/internal/strategy/git/spool_test.go index b2e2af3..a9cf3db 100644 --- a/internal/strategy/git/spool_test.go +++ b/internal/strategy/git/spool_test.go @@ -12,7 +12,6 @@ import ( "github.com/alecthomas/assert/v2" - "github.com/block/cachew/internal/githubapp" "github.com/block/cachew/internal/strategy/git" ) diff --git a/internal/strategy/github_releases_test.go b/internal/strategy/github_releases_test.go index 81865b0..ef8e795 100644 --- a/internal/strategy/github_releases_test.go +++ b/internal/strategy/github_releases_test.go @@ -127,7 +127,7 @@ func setupTest(t *testing.T, config strategy.GitHubReleasesConfig) (*mockGitHubS t.Cleanup(func() { memCache.Close() }) mux := http.NewServeMux() - _, err = strategy.NewGitHubReleases(ctx, config, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) + _, err = strategy.NewGitHubReleases(ctx, config, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) return mock, mux, ctx @@ -242,7 +242,7 @@ func TestGitHubReleasesNoToken(t *testing.T) { defer memCache.Close() mux := http.NewServeMux() - gh, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) + gh, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) assert.Equal(t, "github-releases", gh.String()) } @@ -256,7 +256,7 @@ func TestGitHubReleasesString(t *testing.T) { mux := http.NewServeMux() gh, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{ Token: "test-token", - }, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) + }, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) assert.Equal(t, "github-releases", gh.String()) diff --git a/internal/strategy/gomod/gomod_test.go b/internal/strategy/gomod/gomod_test.go index a89436a..d50edb7 100644 --- a/internal/strategy/gomod/gomod_test.go +++ b/internal/strategy/gomod/gomod_test.go @@ -177,7 +177,7 @@ func setupGoModTest(t *testing.T) (*mockGoModServer, *http.ServeMux, context.Con cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: t.TempDir(), - }) + }, nil) assert.NoError(t, err) mux := http.NewServeMux() _, err = gomod.New(ctx, gomod.Config{ diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go index b8362f8..90baf9a 100644 --- a/internal/strategy/hermit_test.go +++ b/internal/strategy/hermit_test.go @@ -95,7 +95,7 @@ func TestHermitGitHubRelease(t *testing.T) { mux, ctx, memCache := setupHermitTest(t) - _, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) + _, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, memCache, mux, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil assert.NoError(t, err) req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", nil)