Clone All Repositories in Your Self-hosted GitLab

Published: at 02:08 PM

DO NOT USE this tool in a corporate environment, as it may trigger the company’s security alerts.

Table of contents

Open Table of contents

Introduction

After we change to a new computer and there is no project code locally, we need to clone all the project code again. At this time, if your project code is hosted on GitLab, you can use Golang to write a small tool to automatically clone all the project code.

I have an idea, which is to get all the project information through the GitLab API, and then use Golang to call the system command git clone to clone the code of all repositories.

Prerequisites

Step 1: Get GitLab Token

First, you need to get the GitLab token. You can get it from User Settings -> Access Tokens -> Create a personal access token or visit https://{your gitlab instance}/-/user_settings/personal_access_tokens.

Step 2: Write Golang Code

You can copy the code in Code Snippet to a file named main.go.

Step 3: Run the Code

This tool will clone repositories to the current directory. You should make sure that the current directory is clean and there is no project code in it. To run the code, you can use the following command:

go mod init gitlab-clone
go mod tidy
go run main.go

Because I am afraid that it will be over the limit of your GitLab API, I have not used the goroutine to clone the repositories concurrently. If you have a lot of repositories, you can modify the code to clone the repositories concurrently.

Code Snippet

var (
  GitlabToken string = ""
  GitlabURL   string = "http://gitlab.yourdomain.com"
)

type Project struct {
  Id   int    `json:"id"`
  Name string `json:"name"`
  Path string `json:"path_with_namespace"`
  Repo string `json:"web_url"`
}

func INFO(msg string) {
  msg = "INFO: " + msg
  fmt.Println("\033[32m" + msg + "\033[0m")
}

func ERROR(msg string) {
  msg = "ERROR: " + msg
  fmt.Println("\033[31m" + msg + "\033[0m")
}

func main() {
  if len(GitlabURL) == 0 {
    fmt.Print("Please enter your Gitlab URL: ")
    fmt.Scanln(&GitlabURL)
  }
  if len(GitlabToken) == 0 {
    fmt.Print("Please enter your Gitlab Token: ")
    fmt.Scanln(&GitlabToken)
  }
  GitlabURL = strings.Trim(GitlabURL, "/")
  GitlabURL = strings.Trim(GitlabURL, " ")

  var (
    err      error
    projects []Project
  )

  for i := 1; ; i++ {
    var p []Project
    p, err = Projects(i, 50)
    if err != nil {
      ERROR("Error getting projects: " + err.Error())
      break
    }
    if len(p) == 0 {
      break
    }
    projects = append(projects, p...)
  }

  for _, project := range projects {
    for i := 0; i < 3; i++ {
      err = Clone(project)
      if err != nil {
        ERROR("Error cloning repo: " + err.Error() + " [" + project.Path + "]" + " retrying..." + fmt.Sprint(i+1))
        time.Sleep(time.Second)
        continue
      }
      break
    }
  }

  INFO("All repos cloned successfully!")
}

func Projects(page int, per_page int) ([]Project, error) {
  url := fmt.Sprintf("%s%s?page=%d&per_page=%d", GitlabURL, "/api/v4/projects", page, per_page)
  req, err := http.NewRequest("GET", url, nil)
  if err != nil {
    ERROR("Error creating request: " + err.Error())
    return nil, err
  }
  req.Header.Add("PRIVATE-TOKEN", GitlabToken)
  resp, err := http.DefaultClient.Do(req)
  if err != nil {
    ERROR("Error sending request: " + err.Error())
    return nil, err
  }
  defer resp.Body.Close()
  body, err := io.ReadAll(resp.Body)
  if err != nil {
    ERROR("Error reading response: " + err.Error())
    return nil, err
  }
  var projects []Project
  err = json.Unmarshal(body, &projects)
  if err != nil {
    ERROR("Error parsing response: " + err.Error())
    return nil, err
  }
  return projects, nil
}

func Clone(project Project) error {
  var (
    err error
    repo string
    sp []string
    targetFolder string
    repoURL string
  )
  repo = project.Repo
  sp = strings.Split(repo, "/")
  if len(sp) < 2 {
    ERROR("Invalid repo URL: " + repo)
    return fmt.Errorf("Invalid repo URL: " + repo)
  }
  targetFolder := strings.Replace(project.Path, GitlabURL, "", 1)
  // remove folder if exists
  if _, err := os.Stat(targetFolder); !os.IsNotExist(err) {
    os.RemoveAll(targetFolder)
  }
  // create folder
  if _, err := os.Stat(targetFolder); os.IsNotExist(err) {
    os.Mkdir(targetFolder, os.ModePerm)
  }
  // add token to repo URL
  repoURL = strings.Replace(repo, "https://", "https://oauth2:"+GitlabToken+"@", 1)
  repoURL = repoURL + ".git"
  cmd := fmt.Sprintf("git clone %s %s", repoURL, targetFolder)
  c := exec.Command("sh", "-c", cmd).Run()
  if c != nil {
    ERROR("Error cloning repo: " + c.Error())
    return c
  }
  INFO("Cloned repo: name=" + project.Name + ", path=" + project.Path + ", repo=" + project.Repo)
  return nil
}