chore: Update and refactor Git client (argoproj-labs#283)

Signed-off-by: jannfis <[email protected]>

Author: jannfis

ext/git/client.go

@@ -1,18 +1,43 @@
 package git
 
 import (
+	"context"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"strconv"
 	"strings"
+	"time"
+
+	gocache "github.com/patrickmn/go-cache"
 
 	argoio "github.com/argoproj/gitops-engine/pkg/utils/io"
+	"github.com/bradleyfalzon/ghinstallation"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/argoproj/argo-cd/v2/common"
+
 	certutil "github.com/argoproj/argo-cd/v2/util/cert"
 )
 
+// In memory cache for storing github APP api token credentials
+var (
+	githubAppTokenCache *gocache.Cache
+)
+
+func init() {
+	githubAppCredsExp := common.GithubAppCredsExpirationDuration
+	if exp := os.Getenv(common.EnvGithubAppCredsExpirationDuration); exp != "" {
+		if qps, err := strconv.Atoi(exp); err != nil {
+			githubAppCredsExp = time.Duration(qps) * time.Minute
+		}
+	}
+
+	githubAppTokenCache = gocache.New(githubAppCredsExp, 1*time.Minute)
+}
+
 type Creds interface {
 	Environ() (io.Closer, []string, error)
 }
@@ -25,13 +50,26 @@ func (c NopCloser) Close() error {
 	return nil
 }
 
+var _ Creds = NopCreds{}
+
 type NopCreds struct {
 }
 
 func (c NopCreds) Environ() (io.Closer, []string, error) {
 	return NopCloser{}, nil, nil
 }
 
+var _ io.Closer = NopCloser{}
+
+type GenericHTTPSCreds interface {
+	HasClientCert() bool
+	GetClientCertData() string
+	GetClientCertKey() string
+	Environ() (io.Closer, []string, error)
+}
+
+var _ GenericHTTPSCreds = HTTPSCreds{}
+
 // HTTPS creds implementation
 type HTTPSCreds struct {
 	// Username for authentication
@@ -44,15 +82,18 @@ type HTTPSCreds struct {
 	clientCertData string
 	// Client certificate key to use
 	clientCertKey string
+	// HTTP/HTTPS proxy used to access repository
+	proxy string
 }
 
-func NewHTTPSCreds(username string, password string, clientCertData string, clientCertKey string, insecure bool) HTTPSCreds {
+func NewHTTPSCreds(username string, password string, clientCertData string, clientCertKey string, insecure bool, proxy string) GenericHTTPSCreds {
 	return HTTPSCreds{
 		username,
 		password,
 		insecure,
 		clientCertData,
 		clientCertKey,
+		proxy,
 	}
 }
 
@@ -71,7 +112,7 @@ func (c HTTPSCreds) Environ() (io.Closer, []string, error) {
 	// In case the repo is configured for using a TLS client cert, we need to make
 	// sure git client will use it. The certificate's key must not be password
 	// protected.
-	if c.clientCertData != "" && c.clientCertKey != "" {
+	if c.HasClientCert() {
 		var certFile, keyFile *os.File
 
 		// We need to actually create two temp files, one for storing cert data and
@@ -116,6 +157,18 @@ func (c HTTPSCreds) Environ() (io.Closer, []string, error) {
 	return httpCloser, env, nil
 }
 
+func (g HTTPSCreds) HasClientCert() bool {
+	return g.clientCertData != "" && g.clientCertKey != ""
+}
+
+func (c HTTPSCreds) GetClientCertData() string {
+	return c.clientCertData
+}
+
+func (c HTTPSCreds) GetClientCertKey() string {
+	return c.clientCertKey
+}
+
 // SSH implementation
 type SSHCreds struct {
 	sshPrivateKey string
@@ -179,3 +232,144 @@ func (c SSHCreds) Environ() (io.Closer, []string, error) {
 	env = append(env, []string{fmt.Sprintf("GIT_SSH_COMMAND=%s", strings.Join(args, " "))}...)
 	return sshPrivateKeyFile(file.Name()), env, nil
 }
+
+// GitHubAppCreds to authenticate as GitHub application
+type GitHubAppCreds struct {
+	appID          int64
+	appInstallId   int64
+	privateKey     string
+	baseURL        string
+	repoURL        string
+	clientCertData string
+	clientCertKey  string
+	insecure       bool
+	proxy          string
+}
+
+// NewGitHubAppCreds provide github app credentials
+func NewGitHubAppCreds(appID int64, appInstallId int64, privateKey string, baseURL string, repoURL string, clientCertData string, clientCertKey string, insecure bool) GenericHTTPSCreds {
+	return GitHubAppCreds{appID: appID, appInstallId: appInstallId, privateKey: privateKey, baseURL: baseURL, repoURL: repoURL, clientCertData: clientCertData, clientCertKey: clientCertKey, insecure: insecure}
+}
+
+func (g GitHubAppCreds) Environ() (io.Closer, []string, error) {
+	token, err := g.getAccessToken()
+	if err != nil {
+		return NopCloser{}, nil, err
+	}
+
+	env := []string{fmt.Sprintf("GIT_ASKPASS=%s", "git-ask-pass.sh"), "GIT_USERNAME=x-access-token", fmt.Sprintf("GIT_PASSWORD=%s", token)}
+	httpCloser := authFilePaths(make([]string, 0))
+
+	// GIT_SSL_NO_VERIFY is used to tell git not to validate the server's cert at
+	// all.
+	if g.insecure {
+		env = append(env, "GIT_SSL_NO_VERIFY=true")
+	}
+
+	// In case the repo is configured for using a TLS client cert, we need to make
+	// sure git client will use it. The certificate's key must not be password
+	// protected.
+	if g.HasClientCert() {
+		var certFile, keyFile *os.File
+
+		// We need to actually create two temp files, one for storing cert data and
+		// another for storing the key. If we fail to create second fail, the first
+		// must be removed.
+		certFile, err := ioutil.TempFile(argoio.TempDir, "")
+		if err == nil {
+			defer certFile.Close()
+			keyFile, err = ioutil.TempFile(argoio.TempDir, "")
+			if err != nil {
+				removeErr := os.Remove(certFile.Name())
+				if removeErr != nil {
+					log.Errorf("Could not remove previously created tempfile %s: %v", certFile.Name(), removeErr)
+				}
+				return NopCloser{}, nil, err
+			}
+			defer keyFile.Close()
+		} else {
+			return NopCloser{}, nil, err
+		}
+
+		// We should have both temp files by now
+		httpCloser = authFilePaths([]string{certFile.Name(), keyFile.Name()})
+
+		_, err = certFile.WriteString(g.clientCertData)
+		if err != nil {
+			httpCloser.Close()
+			return NopCloser{}, nil, err
+		}
+		// GIT_SSL_CERT is the full path to a client certificate to be used
+		env = append(env, fmt.Sprintf("GIT_SSL_CERT=%s", certFile.Name()))
+
+		_, err = keyFile.WriteString(g.clientCertKey)
+		if err != nil {
+			httpCloser.Close()
+			return NopCloser{}, nil, err
+		}
+		// GIT_SSL_KEY is the full path to a client certificate's key to be used
+		env = append(env, fmt.Sprintf("GIT_SSL_KEY=%s", keyFile.Name()))
+
+	}
+	return httpCloser, env, nil
+}
+
+// getAccessToken fetches GitHub token using the app id, install id, and private key.
+// the token is then cached for re-use.
+func (g GitHubAppCreds) getAccessToken() (string, error) {
+	// Timeout
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+
+	// Compute hash of creds for lookup in cache
+	h := sha256.New()
+	_, err := h.Write([]byte(fmt.Sprintf("%s %d %d %s", g.privateKey, g.appID, g.appInstallId, g.baseURL)))
+	if err != nil {
+		return "", err
+	}
+	key := fmt.Sprintf("%x", h.Sum(nil))
+
+	// Check cache for GitHub transport which helps fetch an API token
+	t, found := githubAppTokenCache.Get(key)
+	if found {
+		itr := t.(*ghinstallation.Transport)
+		// This method caches the token and if it's expired retrieves a new one
+		return itr.Token(ctx)
+	}
+
+	// GitHub API url
+	baseUrl := "https://api.github.com"
+	if g.baseURL != "" {
+		baseUrl = strings.TrimSuffix(g.baseURL, "/")
+	}
+
+	// Create a new GitHub transport
+	c := GetRepoHTTPClient(baseUrl, g.insecure, g, g.proxy)
+	itr, err := ghinstallation.New(c.Transport,
+		g.appID,
+		g.appInstallId,
+		[]byte(g.privateKey),
+	)
+	if err != nil {
+		return "", err
+	}
+
+	itr.BaseURL = baseUrl
+
+	// Add transport to cache
+	githubAppTokenCache.Set(key, itr, time.Minute*60)
+
+	return itr.Token(ctx)
+}
+
+func (g GitHubAppCreds) HasClientCert() bool {
+	return g.clientCertData != "" && g.clientCertKey != ""
+}
+
+func (g GitHubAppCreds) GetClientCertData() string {
+	return g.clientCertData
+}
+
+func (g GitHubAppCreds) GetClientCertKey() string {
+	return g.clientCertKey
+}

ext/git/creds.go

@@ -26,6 +26,7 @@ var (
 	commitSHARegex = regexp.MustCompile("^[0-9A-Fa-f]{40}$")
 	sshURLRegex    = regexp.MustCompile("^(ssh://)?([^/:]*?)@[^@]+$")
 	httpsURLRegex  = regexp.MustCompile("^(https://).*")
+	httpURLRegex   = regexp.MustCompile("^(http://).*")
 )
 
 // IsCommitSHA returns whether or not a string is a 40 character SHA-1
@@ -84,9 +85,14 @@ func IsHTTPSURL(url string) bool {
 	return httpsURLRegex.MatchString(url)
 }
 
+// IsHTTPURL returns true if supplied URL is HTTP URL
+func IsHTTPURL(url string) bool {
+	return httpURLRegex.MatchString(url)
+}
+
 // TestRepo tests if a repo exists and is accessible with the given credentials
-func TestRepo(repo string, creds Creds, insecure bool, enableLfs bool) error {
-	clnt, err := NewClient(repo, creds, insecure, enableLfs)
+func TestRepo(repo string, creds Creds, insecure bool, enableLfs bool, proxy string) error {
+	clnt, err := NewClient(repo, creds, insecure, enableLfs, proxy)
 	if err != nil {
 		return err
 	}