mirror of
https://github.com/taigrr/github2mr.git
synced 2026-04-02 03:09:09 -07:00
Initial release of github2mr.
This is the initial release which is fully-functional and works for myself, self-hosted github enterprise installations, and privately hosted gitbucket installs. It has not been tested against other systems (gogs, gitea, etc), but reports of success/failure or patches would be most welcome.
This commit is contained in:
Steve Kemp
437
main.go
Normal file
437
main.go
Normal file
@@ -0,0 +1,437 @@
|
||||
// This is a trivial application which will output a dump of repositories
|
||||
// which are hosted upon github, or some other host which uses a
|
||||
// compatible API.
|
||||
//
|
||||
// It relies upon having an access-token for authentication.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/google/go-github/v29/github"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
var (
|
||||
//
|
||||
// Context for all calls
|
||||
//
|
||||
ctx context.Context
|
||||
|
||||
//
|
||||
// The actual github client
|
||||
//
|
||||
client *github.Client
|
||||
|
||||
//
|
||||
// The token to use for accessing the remote host.
|
||||
//
|
||||
// This is required because gitbucket prefers to see
|
||||
//
|
||||
// Authorization: token SECRET-TOKEN
|
||||
//
|
||||
// Instead of:
|
||||
//
|
||||
// Authorization: bearer SECRET-TOKEN
|
||||
//
|
||||
oauthToken = &oauth2.Token{}
|
||||
|
||||
//
|
||||
// The number of repos to fetch from the API at a time.
|
||||
//
|
||||
pageSize = 50
|
||||
|
||||
//
|
||||
// Our version number, set for release-builds.
|
||||
//
|
||||
version = "unreleased"
|
||||
)
|
||||
|
||||
// Login accepts the address of a github endpoint, and a corresponding
|
||||
// token to authenticate with.
|
||||
//
|
||||
// We use the login to get the user-information which confirms
|
||||
// that the login was correct.
|
||||
func Login(api string, token string) error {
|
||||
|
||||
// Setup context
|
||||
ctx = context.Background()
|
||||
|
||||
// Setup token
|
||||
ts := oauth2.StaticTokenSource(oauthToken)
|
||||
tc := oauth2.NewClient(ctx, ts)
|
||||
|
||||
// Create the API-client
|
||||
client = github.NewClient(tc)
|
||||
|
||||
// If the user is using a custom URL which doesn't have the
|
||||
// versioned API-suffix add it. This appears to be necessary.
|
||||
if api != "https://api.github.com/" {
|
||||
if !strings.HasSuffix(api, "/api/v3/") {
|
||||
if !strings.HasSuffix(api, "/") {
|
||||
api += "/"
|
||||
}
|
||||
api += "api/v3/"
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the URL for sanity, and update the client with it
|
||||
url, err := url.Parse(api)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client.BaseURL = url
|
||||
|
||||
// Fetch user-information about the user who we are logging in as.
|
||||
user, _, err := client.Users.Get(ctx, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Ensure we have a login
|
||||
if *user.Login == "" {
|
||||
return fmt.Errorf("we failed to find our username, which suggests our login failed")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getPersonalRepos returns all the personal repositories which
|
||||
// belong to our user.
|
||||
func getPersonalRepos(fetch string) ([]*github.Repository, error) {
|
||||
|
||||
var results []*github.Repository
|
||||
|
||||
// Fetch in pages
|
||||
opt := &github.RepositoryListOptions{
|
||||
ListOptions: github.ListOptions{PerPage: pageSize},
|
||||
Type: fetch,
|
||||
}
|
||||
|
||||
// Loop until we're done.
|
||||
for {
|
||||
repos, resp, err := client.Repositories.List(ctx, "", opt)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
results = append(results, repos...)
|
||||
if resp.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
opt.Page = resp.NextPage
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
}
|
||||
|
||||
// getOrganizationalRepositores finds all the organizations the
|
||||
// user is a member of, then fetches their repositories
|
||||
func getOrganizationalRepositores(fetch string) ([]*github.Repository, error) {
|
||||
|
||||
var results []*github.Repository
|
||||
|
||||
// Get the organizations the user is a member of.
|
||||
orgs, _, err := client.Organizations.List(ctx, "", nil)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
|
||||
// Fetch in pages
|
||||
opt := &github.RepositoryListByOrgOptions{
|
||||
ListOptions: github.ListOptions{PerPage: pageSize},
|
||||
Type: fetch,
|
||||
}
|
||||
|
||||
// For each organization we want to get their repositories.
|
||||
for _, org := range orgs {
|
||||
|
||||
// Loop forever getting the repositories
|
||||
for {
|
||||
|
||||
repos, resp, err := client.Repositories.ListByOrg(ctx, *org.Login, opt)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
results = append(results, repos...)
|
||||
if resp.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
opt.Page = resp.NextPage
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
//
|
||||
// Entry-point
|
||||
//
|
||||
func main() {
|
||||
|
||||
//
|
||||
// Parse flags
|
||||
//
|
||||
api := flag.String("api", "https://api.github.com/", "The API end-point to use for the remote git-host.")
|
||||
authHeader := flag.Bool("auth-header-token", false, "Use an authorization-header including 'token' rather than 'bearer'.\nThis is required for gitbucket, and perhaps other systems.")
|
||||
exclude := flag.String("exclude", "", "Comma-separated list of repositories to exclude.")
|
||||
getOrgs := flag.String("organizations", "all", "Which organizational repositories to fetch.\nValid values are 'public', 'private', 'none', or 'all'.")
|
||||
getPersonal := flag.String("personal", "all", "Which personal repositories to fetch.\nValid values are 'public', 'private', 'none', or 'all'.")
|
||||
http := flag.Bool("http", false, "Generate HTTP-based clones rather than SSH-based ones.")
|
||||
output := flag.String("output", "", "Write output to the named file, instead of printing to STDOUT.")
|
||||
prefix := flag.String("prefix", "", "The prefix beneath which to store the repositories upon the current system.")
|
||||
token := flag.String("token", "", "The API token used to authenticate to the remote API-host.")
|
||||
versionCmd := flag.Bool("version", false, "Report upon our version, and terminate.")
|
||||
flag.Parse()
|
||||
|
||||
//
|
||||
// Showing only the version?
|
||||
//
|
||||
if *versionCmd {
|
||||
fmt.Printf("github2mr %s\n", version)
|
||||
return
|
||||
}
|
||||
|
||||
//
|
||||
// Validate the repository-types
|
||||
//
|
||||
if *getPersonal != "all" &&
|
||||
*getPersonal != "none" &&
|
||||
*getPersonal != "public" &&
|
||||
*getPersonal != "private" {
|
||||
fmt.Fprintf(os.Stderr, "Valid settings are 'public', 'private', 'none', or 'all'\n")
|
||||
return
|
||||
}
|
||||
if *getOrgs != "all" &&
|
||||
*getOrgs != "none" &&
|
||||
*getOrgs != "public" &&
|
||||
*getOrgs != "private" {
|
||||
fmt.Fprintf(os.Stderr, "Valid settings are 'public', 'private', 'none', or 'all'\n")
|
||||
return
|
||||
}
|
||||
|
||||
//
|
||||
// Get the authentication token supplied via the flag, falling back
|
||||
// to the environment if nothing has been specified.
|
||||
//
|
||||
tok := *token
|
||||
if tok == "" {
|
||||
// Fallback
|
||||
tok = os.Getenv("GITHUB_TOKEN")
|
||||
|
||||
if tok == "" {
|
||||
fmt.Printf("Please specify your github token!\n")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Populate our global OAUTH token with the supplied value.
|
||||
//
|
||||
oauthToken.AccessToken = tok
|
||||
|
||||
//
|
||||
// Allow setting the authorization header-type, if required.
|
||||
//
|
||||
if *authHeader {
|
||||
oauthToken.TokenType = "token"
|
||||
}
|
||||
|
||||
//
|
||||
// Login and confirm that this worked.
|
||||
//
|
||||
err := Login(*api, tok)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Login error - is your token set/correct? %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
//
|
||||
// Fetch details of all "personal" repositories, unless we're not
|
||||
// supposed to.
|
||||
//
|
||||
var personal []*github.Repository
|
||||
if *getPersonal != "none" {
|
||||
personal, err = getPersonalRepos(*getPersonal)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to fetch personal repository list: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Fetch details of all organizational repositories, unless we're
|
||||
// not supposed to.
|
||||
//
|
||||
var orgs []*github.Repository
|
||||
if *getOrgs != "none" {
|
||||
orgs, err = getOrganizationalRepositores(*getOrgs)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to fetch organizational repositories: %s\n",
|
||||
err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// If the prefix is not set then create a default.
|
||||
//
|
||||
// This will be of the form:
|
||||
//
|
||||
// ~/Repos/github.com/x/y
|
||||
// ~/Repos/git.example.com/x/y
|
||||
// ~/Repos/git.steve.fi/x/y
|
||||
//
|
||||
// i.e "~/Repos/${git host}/${owner}/${path}
|
||||
//
|
||||
// (${git host} comes from the remote API host.)
|
||||
//
|
||||
repoPrefix := *prefix
|
||||
if repoPrefix == "" {
|
||||
|
||||
// Get the hostname
|
||||
url, _ := url.Parse(*api)
|
||||
host := url.Hostname()
|
||||
|
||||
// Handle the obvious case
|
||||
if host == "api.github.com" {
|
||||
host = "github.com"
|
||||
}
|
||||
|
||||
// Generate a prefix
|
||||
repoPrefix = os.Getenv("HOME") + "/Repos/" + host
|
||||
}
|
||||
|
||||
//
|
||||
// Combine the results of the repositories we've found.
|
||||
//
|
||||
var all []*github.Repository
|
||||
all = append(all, personal...)
|
||||
all = append(all, orgs...)
|
||||
|
||||
//
|
||||
// Sort the list, based upon the full-name.
|
||||
//
|
||||
sort.Slice(all[:], func(i, j int) bool {
|
||||
|
||||
// Case-insensitive sorting.
|
||||
a := strings.ToLower(*all[i].FullName)
|
||||
b := strings.ToLower(*all[j].FullName)
|
||||
|
||||
return a < b
|
||||
})
|
||||
|
||||
//
|
||||
// Repos we're excluding
|
||||
//
|
||||
excluded := strings.Split(*exclude, ",")
|
||||
|
||||
//
|
||||
// Structure we use for template expansion
|
||||
//
|
||||
type Repo struct {
|
||||
// Prefix-directory for local clones.
|
||||
Prefix string
|
||||
|
||||
// Name of the repository "owner/repo-name".
|
||||
Name string
|
||||
|
||||
// Source to clone from http/ssh-based.
|
||||
Source string
|
||||
}
|
||||
|
||||
//
|
||||
// Repos we will output
|
||||
//
|
||||
var repos []*Repo
|
||||
|
||||
//
|
||||
// Now format the repositories we've discovered.
|
||||
//
|
||||
for _, repo := range all {
|
||||
|
||||
//
|
||||
// The clone-type is configurable
|
||||
//
|
||||
clone := *repo.SSHURL
|
||||
if *http {
|
||||
clone = *repo.CloneURL
|
||||
}
|
||||
|
||||
//
|
||||
// Should we exclude this entry?
|
||||
//
|
||||
skip := false
|
||||
for _, exc := range excluded {
|
||||
|
||||
exc = strings.TrimSpace(exc)
|
||||
|
||||
if len(exc) > 0 && strings.Contains(strings.ToLower(clone), strings.ToLower(exc)) {
|
||||
skip = true
|
||||
}
|
||||
}
|
||||
|
||||
// Skipped
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
|
||||
repos = append(repos, &Repo{Prefix: repoPrefix,
|
||||
Name: *repo.FullName,
|
||||
Source: clone})
|
||||
}
|
||||
|
||||
//
|
||||
// Load the template we'll use for formatting the output
|
||||
//
|
||||
tmpl := `# Generated by github2mr - {{len .}} repositories
|
||||
|
||||
{{range .}}
|
||||
[{{.Prefix}}/{{.Name}}]
|
||||
checkout = git clone {{.Source}}
|
||||
{{end}}
|
||||
`
|
||||
|
||||
//
|
||||
// Parse the template and execute it.
|
||||
//
|
||||
var out bytes.Buffer
|
||||
t := template.Must(template.New("tmpl").Parse(tmpl))
|
||||
err = t.Execute(&out, repos)
|
||||
|
||||
//
|
||||
// If there were errors we're done.
|
||||
//
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error interpolating template:%s\n", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
//
|
||||
// Show the results, or write to the specified file as appropriate
|
||||
//
|
||||
if *output != "" {
|
||||
file, err := os.Create(*output)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to open %s:%s\n", *output, err.Error())
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
file.Write(out.Bytes())
|
||||
} else {
|
||||
fmt.Println(out.String())
|
||||
}
|
||||
|
||||
//
|
||||
// All done.
|
||||
//
|
||||
}
|
||||
Reference in New Issue
Block a user