[Go] Go-Git 并发Clone大量仓库并更新到最新#

import (
	"fmt"
	gogit "github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
	"log"
	"os"
	"runtime"
	"service/util"
	"strings"
	"sync"
	"testing"
)

func GitClone(gitClonePath string) {

	gitUrl := strings.ReplaceAll(gitClonePath, "git clone ", "")
	filePath := strings.ReplaceAll(gitUrl, "git@github.com:", "")
	filePath = strings.ReplaceAll(filePath, ".git", "")

	fmt.Println("---Start Clone And Pull ", filePath)

  // 读取id rsa key
	sshPath := os.Getenv("HOME") + "/.ssh/id_rsa"
	key, _ := os.ReadFile(sshPath)

	publicKey, err := ssh.NewPublicKeys("git", []byte(key), "")
	if err != nil {
		log.Fatalf("get ssh auth public key error")
	}

	_, err = gogit.PlainClone(filePath, false, &gogit.CloneOptions{
		Auth:     publicKey,
		URL:      gitClonePath,
		Progress: os.Stdout,
		//ReferenceName: plumbing.ReferenceName("refs/heads/develop"),
		//RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
		//SingleBranch:  true,
	})

	if err != nil {
		fmt.Println(err, gitUrl)
	}
	fmt.Println("clone success ", gitUrl)

	// Clone than git pull
	r, err := gogit.PlainOpen(filePath)
	if err != nil {
		fmt.Println(err, gitUrl)
	} else {
		// Get the working directory
		w, err := r.Worktree()
		if err != nil {
			fmt.Println(err, gitUrl)
		} else {
			// Pull from origin
			err = w.Pull(&gogit.PullOptions{
				RemoteName: "origin",
				Auth:       publicKey,
				Progress:   os.Stdout,
			})
			if err != nil {
				fmt.Println(err, gitUrl)
			}
		}
	}
	fmt.Println("---End Clone And Pull ", filePath)
}

//读取json 反序列为 指定T类型
func ReadJSON[T interface{}](fileName string) (t T) {
	var json = jsoniter.ConfigCompatibleWithStandardLibrary
	json.Unmarshal(ReadJSONBytes(fileName), &t)
	return t
}

func TestReadAllGithubThanCloneAll(t *testing.T) {

	gitCloneUrls := util.ReadJSON[[]string]("文件1.json")

	//go chan 并发30线程
	maxWorkerCount := 30
	queue := make(chan string, maxWorkerCount)
	runtime.GOMAXPROCS(runtime.NumCPU())
	wg := sync.WaitGroup{}
	for i := 0; i < maxWorkerCount; i++ {
		go func() {
			defer wg.Done()
			wg.Add(1)
			for v := range queue {
				GitClone(v)
			}
		}()
	}

	for _, v := range gitCloneUrls {
		queue <- v
	}
	close(queue)
	wg.Wait()

}