feat: Add GitHub App authentication for git cloning and releases

cachew.hcl

@@ -1,7 +1,3 @@
-# strategy git {}
-# strategy docker {}
-# strategy hermit {}
-
 # Artifactory caching proxy strategy
 # artifactory "example.jfrog.io" {
 #   target = "https://example.jfrog.io"
@@ -16,8 +12,16 @@ git-clone {
     mirror-root = "./state/git-mirrors"
 }
 
+github-app {
+  # Uncomment and add:
+  # app-id = "app-id-value" (Can also be passed via setting envar CACHEW_GITHUB_APP_APP_ID)
+  # private-key-path = "private-key-path-value" (Can also be passed via envar CACHEW_GITHUB_APP_PRIVATE_KEY_PATH)
+  # installations = { "myorg" : "installation-id" }
+}
+
 metrics {}
 
+
 git {
   bundle-interval = "24h"
   snapshot-interval = "24h"

cmd/cachewd/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/block/cachew/internal/cache"
 	"github.com/block/cachew/internal/config"
 	"github.com/block/cachew/internal/gitclone"
+	"github.com/block/cachew/internal/githubapp"
 	"github.com/block/cachew/internal/httputil"
 	"github.com/block/cachew/internal/jobscheduler"
 	"github.com/block/cachew/internal/logging"
@@ -35,6 +36,7 @@ type GlobalConfig struct {
 	LoggingConfig   logging.Config      `hcl:"log,block"`
 	MetricsConfig   metrics.Config      `hcl:"metrics,block"`
 	GitCloneConfig  gitclone.Config     `hcl:"git-clone,block"`
+	GithubAppConfig githubapp.Config    `embed:"" hcl:"github-app,block,optional" prefix:"github-app-"`
 }
 
 type CLI struct {
@@ -65,11 +67,14 @@ func main() {
 	logger, ctx := logging.Configure(ctx, globalConfig.LoggingConfig)
 
 	// Start initialising
-	managerProvider := gitclone.NewManagerProvider(ctx, globalConfig.GitCloneConfig)
+	tokenManagerProvider := githubapp.NewTokenManagerProvider(globalConfig.GithubAppConfig, logger)
+	tokenManager, err := tokenManagerProvider()
+	kctx.FatalIfErrorf(err)
+	managerProvider := gitclone.NewManagerProvider(ctx, globalConfig.GitCloneConfig, tokenManager)
 
 	scheduler := jobscheduler.New(ctx, globalConfig.SchedulerConfig)
 
-	cr, sr := newRegistries(globalConfig.URL, scheduler, managerProvider)
+	cr, sr := newRegistries(globalConfig.URL, scheduler, managerProvider, tokenManagerProvider)
 
 	// Commands
 	switch { //nolint:gocritic
@@ -100,7 +105,7 @@ func main() {
 	kctx.FatalIfErrorf(err)
 }
 
-func newRegistries(cachewURL string, scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider) (*cache.Registry, *strategy.Registry) {
+func newRegistries(cachewURL string, scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider, tokenManagerProvider githubapp.TokenManagerProvider) (*cache.Registry, *strategy.Registry) {
 	cr := cache.NewRegistry()
 	cache.RegisterMemory(cr)
 	cache.RegisterDisk(cr)
@@ -109,10 +114,10 @@ func newRegistries(cachewURL string, scheduler jobscheduler.Scheduler, cloneMana
 	sr := strategy.NewRegistry()
 	strategy.RegisterAPIV1(sr)
 	strategy.RegisterArtifactory(sr)
-	strategy.RegisterGitHubReleases(sr)
+	strategy.RegisterGitHubReleases(sr, tokenManagerProvider)
 	strategy.RegisterHermit(sr, cachewURL)
 	strategy.RegisterHost(sr)
-	git.Register(sr, scheduler, cloneManagerProvider)
+	git.Register(sr, scheduler, cloneManagerProvider, tokenManagerProvider)
 	gomod.Register(sr, cloneManagerProvider)
 
 	return cr, sr

docker/Dockerfile

@@ -6,7 +6,7 @@ ARG TARGETARCH
 SHELL ["/bin/sh", "-o", "pipefail", "-c"]
 
 # Install runtime dependencies for git operations and TLS
-RUN apk add --no-cache ca-certificates git tzdata && \
+RUN apk add --no-cache ca-certificates git git-daemon tzdata && \
   addgroup -g 1000 cachew && \
   adduser -D -u 1000 -G cachew cachew
 

go.mod

@@ -5,6 +5,7 @@ go 1.25.5
 require (
 	github.com/alecthomas/hcl/v2 v2.6.0
 	github.com/alecthomas/kong v1.13.0
+	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/goproxy/goproxy v0.25.0
 	github.com/lmittmann/tint v1.1.2
 	github.com/minio/minio-go/v7 v7.0.97

go.sum

@@ -35,6 +35,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=

internal/config/config.go

@@ -142,6 +142,7 @@ func Load(
 	return nil
 }
 
+// ExpandVars expands environment variable references in HCL strings and heredocs.
 func expandVars(ast *hcl.AST, vars map[string]string) {
 	_ = hcl.Visit(ast, func(node hcl.Node, next func() error) error { //nolint:errcheck
 		attr, ok := node.(*hcl.Attribute)

internal/gitclone/command.go

@@ -5,14 +5,25 @@ package gitclone
 import (
 	"bufio"
 	"context"
+	"net/url"
 	"os/exec"
 	"strings"
 
 	"github.com/alecthomas/errors"
 )
 
-func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, error) {
-	configArgs, err := getInsteadOfDisableArgsForURL(ctx, url)
+func (r *Repository) gitCommand(ctx context.Context, args ...string) (*exec.Cmd, error) {
+	repoURL := r.upstreamURL
+	modifiedURL := repoURL
+	if r.credentialProvider != nil && strings.Contains(repoURL, "github.com") {
+		token, err := r.credentialProvider.GetTokenForURL(ctx, repoURL)
+		if err == nil && token != "" {
+			modifiedURL = injectTokenIntoURL(repoURL, token)
+		}
+		// If error getting token, fall back to original URL (system credentials)
+	}
+
+	configArgs, err := getInsteadOfDisableArgsForURL(ctx, repoURL)
 	if err != nil {
 		return nil, errors.Wrap(err, "get insteadOf disable args")
 	}
@@ -23,8 +34,41 @@ func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, err
 	}
 	allArgs = append(allArgs, args...)
 
-	cmd := exec.CommandContext(ctx, "git", allArgs...)
-	return cmd, nil
+	// Replace URL in args if it was modified for authentication
+	if modifiedURL != repoURL {
+		for i, arg := range allArgs {
+			if arg == repoURL {
+				allArgs[i] = modifiedURL
+			}
+		}
+	}
+
+	return exec.CommandContext(ctx, "git", allArgs...), nil
+}
+
+// Converts https://github.com/org/repo to https://x-access-token:TOKEN@github.com/org/repo
+func injectTokenIntoURL(rawURL, token string) string {
+	if token == "" {
+		return rawURL
+	}
+
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+
+	// Only inject token for GitHub URLs
+	if !strings.Contains(u.Host, "github.com") {
+		return rawURL
+	}
+
+	// Upgrade http to https for security
+	if u.Scheme == "http" {
+		u.Scheme = "https"
+	}
+
+	u.User = url.UserPassword("x-access-token", token)
+	return u.String()
 }
 
 func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]string, error) {

internal/gitclone/command_test.go

@@ -45,7 +45,12 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) {
 func TestGitCommand(t *testing.T) {
 	ctx := context.Background()
 
-	cmd, err := gitCommand(ctx, "https://github.com/user/repo", "version")
+	repo := &Repository{
+		upstreamURL:        "https://github.com/user/repo",
+		credentialProvider: nil,
+	}
+
+	cmd, err := repo.gitCommand(ctx, "version")
 	assert.NoError(t, err)
 
 	assert.NotZero(t, cmd)
@@ -59,7 +64,12 @@ func TestGitCommand(t *testing.T) {
 func TestGitCommandWithEmptyURL(t *testing.T) {
 	ctx := context.Background()
 
-	cmd, err := gitCommand(ctx, "", "version")
+	repo := &Repository{
+		upstreamURL:        "",
+		credentialProvider: nil,
+	}
+
+	cmd, err := repo.gitCommand(ctx, "version")
 	assert.NoError(t, err)
 
 	assert.NotZero(t, cmd)

internal/gitclone/manager.go

@@ -58,35 +58,42 @@ type Config struct {
 	RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"`
 }
 
+// CredentialProvider provides credentials for git operations.
+type CredentialProvider interface {
+	GetTokenForURL(ctx context.Context, url string) (string, error)
+}
+
 type Repository struct {
-	mu            sync.RWMutex
-	config        Config
-	state         State
-	path          string
-	upstreamURL   string
-	lastFetch     time.Time
-	lastRefCheck  time.Time
-	refCheckValid bool
-	fetchSem      chan struct{}
+	mu                 sync.RWMutex
+	config             Config
+	state              State
+	path               string
+	upstreamURL        string
+	lastFetch          time.Time
+	lastRefCheck       time.Time
+	refCheckValid      bool
+	fetchSem           chan struct{}
+	credentialProvider CredentialProvider
 }
 
 type Manager struct {
-	config          Config
-	gitTuningConfig GitTuningConfig
-	clones          map[string]*Repository
-	clonesMu        sync.RWMutex
+	config             Config
+	gitTuningConfig    GitTuningConfig
+	clones             map[string]*Repository
+	clonesMu           sync.RWMutex
+	credentialProvider CredentialProvider
 }
 
 // ManagerProvider is a function that lazily creates a singleton Manager.
 type ManagerProvider func() (*Manager, error)
 
-func NewManagerProvider(ctx context.Context, config Config) ManagerProvider {
+func NewManagerProvider(ctx context.Context, config Config, credentialProvider CredentialProvider) ManagerProvider {
 	return sync.OnceValues(func() (*Manager, error) {
-		return NewManager(ctx, config)
+		return NewManager(ctx, config, credentialProvider)
 	})
 }
 
-func NewManager(ctx context.Context, config Config) (*Manager, error) {
+func NewManager(ctx context.Context, config Config, credentialProvider CredentialProvider) (*Manager, error) {
 	if config.MirrorRoot == "" {
 		return nil, errors.New("mirror-root is required")
 	}
@@ -109,9 +116,10 @@ func NewManager(ctx context.Context, config Config) (*Manager, error) {
 		"ref_check_interval", config.RefCheckInterval)
 
 	return &Manager{
-		config:          config,
-		gitTuningConfig: DefaultGitTuningConfig(),
-		clones:          make(map[string]*Repository),
+		config:             config,
+		gitTuningConfig:    DefaultGitTuningConfig(),
+		clones:             make(map[string]*Repository),
+		credentialProvider: credentialProvider,
 	}, nil
 }
 
@@ -138,11 +146,12 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor
 	clonePath := m.clonePathForURL(upstreamURL)
 
 	repo = &Repository{
-		state:       StateEmpty,
-		config:      m.config,
-		path:        clonePath,
-		upstreamURL: upstreamURL,
-		fetchSem:    make(chan struct{}, 1),
+		state:              StateEmpty,
+		config:             m.config,
+		path:               clonePath,
+		upstreamURL:        upstreamURL,
+		fetchSem:           make(chan struct{}, 1),
+		credentialProvider: m.credentialProvider,
 	}
 
 	gitDir := filepath.Join(clonePath, ".git")
@@ -205,10 +214,12 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
 		upstreamURL := "https://" + host + "/" + repoPath
 
 		repo := &Repository{
-			state:       StateReady,
-			path:        path,
-			upstreamURL: upstreamURL,
-			fetchSem:    make(chan struct{}, 1),
+			state:              StateReady,
+			config:             m.config,
+			path:               path,
+			upstreamURL:        upstreamURL,
+			fetchSem:           make(chan struct{}, 1),
+			credentialProvider: m.credentialProvider,
 		}
 		repo.fetchSem <- struct{}{}
 
@@ -315,7 +326,7 @@ func (r *Repository) executeClone(ctx context.Context) error {
 		r.upstreamURL, r.path,
 	}
 
-	cmd, err := gitCommand(ctx, r.upstreamURL, args...)
+	cmd, err := r.gitCommand(ctx, args...)
 	if err != nil {
 		return errors.Wrap(err, "create git command")
 	}
@@ -331,7 +342,7 @@ func (r *Repository) executeClone(ctx context.Context) error {
 		return errors.Wrapf(err, "configure fetch refspec: %s", string(output))
 	}
 
-	cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path,
+	cmd, err = r.gitCommand(ctx, "-C", r.path,
 		"-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer),
 		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit),
 		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())),
@@ -370,7 +381,7 @@ func (r *Repository) Fetch(ctx context.Context) error {
 	config := DefaultGitTuningConfig()
 
 	// #nosec G204 - r.path is controlled by us
-	cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path,
+	cmd, err := r.gitCommand(ctx, "-C", r.path,
 		"-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer),
 		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit),
 		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())),
@@ -460,7 +471,7 @@ func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error
 
 func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) {
 	// #nosec G204 - r.upstreamURL is controlled by us
-	cmd, err := gitCommand(ctx, r.upstreamURL, "ls-remote", r.upstreamURL)
+	cmd, err := r.gitCommand(ctx, "ls-remote", r.upstreamURL)
 	if err != nil {
 		return nil, errors.Wrap(err, "create git command")
 	}