Merge pull request #420 from spring-financial-group/partial-clone-funcs

feat: add function for sparse & partial clone of cluster repo

Author: jenkins-x-bot

pkg/gitclient/helpers.go

@@ -331,17 +331,9 @@ func NthTag(g Interface, dir string, n int) (string, string, error) {
 
 // CloneToDir clones the git repository to either the given directory or create a temporary
 func CloneToDir(g Interface, gitURL, dir string) (string, error) {
-	var err error
-	if dir != "" {
-		err = os.MkdirAll(dir, util.DefaultWritePermissions)
-		if err != nil {
-			return "", fmt.Errorf("failed to create directory %s: %w", dir, err)
-		}
-	} else {
-		dir, err = os.MkdirTemp("", "jx-git-")
-		if err != nil {
-			return "", fmt.Errorf("failed to create temporary directory: %w", err)
-		}
+	dir, err := createDir(dir)
+	if err != nil {
+		return "", err
 	}
 
 	log.Logger().Debugf("cloning %s to directory %s", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir))
@@ -354,23 +346,40 @@ func CloneToDir(g Interface, gitURL, dir string) (string, error) {
 	return dir, nil
 }
 
+// PartialCloneToDir Partially clones the git repository to either the given directory or create a temporary one
+// Alternative to SparseCloneToDir when git provider does not support sparse-checkout
+// sparseCheckoutPatterns not supported
+// If shallow is true the clone is made with --depth=1
+func PartialCloneToDir(g Interface, gitURL, dir string, shallow bool) (string, error) {
+	dir, err := createDir(dir)
+	if err != nil {
+		return "", err
+	}
+
+	log.Logger().Debugf("initiating partial clone %s to directory %s", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir))
+
+	parentDir := filepath.Dir(dir)
+	partialCloneArgs := []string{"clone", "--filter=blob:none"}
+	if shallow {
+		log.Logger().Debugf("setting clone depth to 1")
+		partialCloneArgs = append(partialCloneArgs, "--depth=1")
+	}
+	_, err = g.Command(parentDir, append(partialCloneArgs, gitURL, dir)...)
+	if err != nil {
+		return "", fmt.Errorf("failed to partially clone repository %s to directory: %s: %w", gitURL, dir, err)
+	}
+	return dir, nil
+}
+
 // SparseCloneToDir clones the git repository sparsely to either the given directory or create a temporary on.
 // SparseCheckoutPatterns are checked out interpreted as in .gitignore. If no sparseCheckoutPatterns are given the files
 // directly under the root of the repository are checked out.
 // NOTE: This functionality is experimental and also the behaviour may vary between different git servers.
 // If shallow is true the clone is made with --depth=1
 func SparseCloneToDir(g Interface, gitURL, dir string, shallow bool, sparseCheckoutPatterns ...string) (string, error) {
-	var err error
-	if dir != "" {
-		err = os.MkdirAll(dir, util.DefaultWritePermissions)
-		if err != nil {
-			return "", fmt.Errorf("failed to create directory %s: %w", dir, err)
-		}
-	} else {
-		dir, err = os.MkdirTemp("", "jx-git-")
-		if err != nil {
-			return "", fmt.Errorf("failed to create temporary directory: %w", err)
-		}
+	dir, err := createDir(dir)
+	if err != nil {
+		return "", err
 	}
 
 	log.Logger().Debugf("cloning %s to directory %s sparsely", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir))
@@ -566,6 +575,24 @@ func CheckoutRemoteBranch(g Interface, dir string, branch string) error {
 	return Checkout(g, dir, branch)
 }
 
+// createDir creates input directory if it does not exist, or creates a temporary directory
+// createDir creates the input directory if it does not exist, or creates a temporary directory
+func createDir(dir string) (string, error) {
+	var err error
+	if dir != "" {
+		err = os.MkdirAll(dir, util.DefaultWritePermissions)
+		if err != nil {
+			return "", fmt.Errorf("failed to create directory %s: %w", dir, err)
+		}
+	} else {
+		dir, err = os.MkdirTemp("", "jx-git-")
+		if err != nil {
+			return "", fmt.Errorf("failed to create temporary directory: %w", err)
+		}
+	}
+	return dir, nil
+}
+
 // GetLatestCommitMessage returns the latest git commit message
 func GetLatestCommitMessage(g Interface, dir string) (string, error) {
 	return g.Command(dir, "log", "-1", "--pretty=%B")

pkg/requirements/requirements.go

@@ -64,23 +64,9 @@ func GetRequirementsAndGit(g gitclient.Interface, gitURL string) (*jxcore.Requir
 func CloneClusterRepo(g gitclient.Interface, gitURL string) (string, error) {
 	// if we have a kubernetes secret with git auth mounted to the filesystem when running in cluster
 	// we need to turn it into a git credentials file see https://git-scm.com/docs/git-credential-store
-	secretMountPath := os.Getenv(credentialhelper.GIT_SECRET_MOUNT_PATH)
-	if secretMountPath != "" {
-		err := credentialhelper.WriteGitCredentialFromSecretMount()
-		if err != nil {
-			return "", fmt.Errorf("failed to write git credentials file for secret %s : %w", secretMountPath, err)
-		}
-
-		gitURL, err = AddUserPasswordToURLFromDir(gitURL, secretMountPath)
-		if err != nil {
-			return "", fmt.Errorf("failed to add username and password to git URL: %w", err)
-		}
-	} else {
-		if kube.IsInCluster() {
-			log.Logger().Warnf("no $GIT_SECRET_MOUNT_PATH environment variable set")
-		} else {
-			log.Logger().Debugf("no $GIT_SECRET_MOUNT_PATH environment variable set")
-		}
+	gitURL, err := gitCredsFromCluster(gitURL)
+	if err != nil {
+		return "", err
 	}
 
 	// clone cluster repo to a temp dir and load the requirements
@@ -91,6 +77,33 @@ func CloneClusterRepo(g gitclient.Interface, gitURL string) (string, error) {
 	return dir, nil
 }
 
+// PartialCloneClusterRepo clones the cluster repo to a temporary directory and returns the directory path
+// Attempts a sparse clone first, falling back to a partial clone without checkout patterns, then a default clone
+func PartialCloneClusterRepo(g gitclient.Interface, gitURL string, shallow bool, sparseCheckoutPatterns ...string) (string, error) {
+	gitURL, err := gitCredsFromCluster(gitURL)
+	if err != nil {
+		return "", err
+	}
+	// Attempt sparse clone first
+	dir, err := gitclient.SparseCloneToDir(g, gitURL, "", shallow, sparseCheckoutPatterns...)
+	if err != nil {
+		log.Logger().Warnf("failed sparse clone of cluster git repo %s: %v", gitURL, err)
+		log.Logger().Warnf("falling back to partial clone without checkout patterns")
+		// If sparse clone fails, fall back to partial clone
+		dir, err = gitclient.PartialCloneToDir(g, gitURL, "", shallow)
+		if err != nil {
+			log.Logger().Warnf("failed partial clone of cluster git repo %s: %v", gitURL, err)
+			log.Logger().Warnf("falling back to default clone, without checkout patterns")
+			dir, err = gitclient.CloneToDir(g, gitURL, "")
+			if err != nil {
+				return "", fmt.Errorf("failed to clone cluster git repo %s: %w", gitURL, err)
+			}
+		}
+		return dir, nil
+	}
+	return dir, nil
+}
+
 // AddUserPasswordToURLFromDir loads the username and password files from the given directory and adds them to the URL if they are found
 func AddUserPasswordToURLFromDir(gitURL, path string) (string, error) {
 	username, err := loadFile(filepath.Join(path, "username"))
@@ -107,6 +120,28 @@ func AddUserPasswordToURLFromDir(gitURL, path string) (string, error) {
 	return gitURL, nil
 }
 
+func gitCredsFromCluster(gitURL string) (string, error) {
+	secretMountPath := os.Getenv(credentialhelper.GIT_SECRET_MOUNT_PATH)
+	if secretMountPath != "" {
+		err := credentialhelper.WriteGitCredentialFromSecretMount()
+		if err != nil {
+			return "", fmt.Errorf("failed to write git credentials file for secret %s : %w", secretMountPath, err)
+		}
+
+		gitURL, err = AddUserPasswordToURLFromDir(gitURL, secretMountPath)
+		if err != nil {
+			return "", fmt.Errorf("failed to add username and password to git URL: %w", err)
+		}
+	} else {
+		if kube.IsInCluster() {
+			log.Logger().Warnf("no $GIT_SECRET_MOUNT_PATH environment variable set")
+		} else {
+			log.Logger().Debugf("no $GIT_SECRET_MOUNT_PATH environment variable set")
+		}
+	}
+	return gitURL, nil
+}
+
 func loadFile(path string) (string, error) {
 	exists, err := files.FileExists(path)
 	if err != nil {