commit a63e03b549f008fb1799828769f245c71592c71e Author: Steve Kemp Date: Fri Jan 17 20:05:39 2020 +0200 Initial release of github2mr. This is the initial release which is fully-functional and works for myself, self-hosted github enterprise installations, and privately hosted gitbucket installs. It has not been tested against other systems (gogs, gitea, etc), but reports of success/failure or patches would be most welcome. diff --git a/.github/build b/.github/build new file mode 100755 index 0000000..e373112 --- /dev/null +++ b/.github/build @@ -0,0 +1,43 @@ +#!/bin/bash + +# The basename of our binary +BASE="github2mr" + +# Setup an output directory - creating if missing +cur=$(pwd) +OUTPUT="${cur}/bin" +if [ ! -d "${OUTPUT}" ]; then + mkdir -p "${OUTPUT}" +fi + +# We build on multiple platforms/archs +BUILD_PLATFORMS="linux darwin freebsd" +BUILD_ARCHS="amd64 386" + +# For each platform. +for OS in ${BUILD_PLATFORMS[@]}; do + + # For each arch + for ARCH in ${BUILD_ARCHS[@]}; do + + # Setup a suffix for the binary + SUFFIX="${OS}" + + # i386 is better than 386 + if [ "$ARCH" = "386" ]; then + SUFFIX="${SUFFIX}-i386" + else + SUFFIX="${SUFFIX}-${ARCH}" + fi + + echo "Building for ${OS} [${ARCH}] -> ${BASE}-${SUFFIX}" + + # Run the build + export GOARCH=${ARCH} + export GOOS=${OS} + export CGO_ENABLED=0 + + # Build the main-binary + go build -ldflags "-X main.version=$(git describe --tags 2>/dev/null || echo 'master')" -o "${OUTPUT}/${BASE}-${SUFFIX}" + done +done diff --git a/.github/run-tests.sh b/.github/run-tests.sh new file mode 100755 index 0000000..0ea02c2 --- /dev/null +++ b/.github/run-tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Install tools to test our code-quality. +go get -u golang.org/x/lint/golint + +# Failures cause aborts +set -e + +# Run the linter +echo "Launching linter .." +golint -set_exit_status ./... +echo "Completed linter .." + +# Run the vet-checker. +echo "Launching go vet check .." +go vet ./... +echo "Completed go vet check .." + diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..691f8f8 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,10 @@ +on: pull_request +name: Pull Request +jobs: + test: + name: Run tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Test + uses: skx/github-action-tester@master diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..af6a41a --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,13 @@ +on: + push: + branches: + - master +name: Push Event +jobs: + test: + name: Run tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Test + uses: skx/github-action-tester@master diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f094e8c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,19 @@ +on: + release: + types: [created] +name: Handle Release +jobs: + upload: + name: Upload + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@master + - name: Generate the artifacts + uses: skx/github-action-build@master + - name: Upload the artifacts + uses: skx/github-action-publish-binaries@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + args: bin/*-* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e660fd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..eee6ce9 --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ + +* [github2mr](#github2mr) + * [Brief mr Example](#brief-mr-example) +* [Installation](#installation) +* [Configuration / Usage](#configuration--usage) + * [Other Git Hosts](#other-git-hosts) +* [Github Setup](#github-setup) + + + + +# github2mr + +Many [Github](https://github.com/) users have a large number of repositories upon which they work. This application allows you to dump all your repository details into a configuration file for [myrepos](https://myrepos.branchable.com/). + +The myrepos package, containing a binary named `mr`, is a _wonderful_ tool that lets you apply operations to multiple repositories at once given a suitable configuration. + +The end result of using `mr` and `github2mr` is that you should be able to clone all your remote github repositories, and update them easily with only a couple of commands which is great for when you work/develop/live on multiple machines. + + +## Brief `mr` Example + +Let us pretend I'm moving to a new machine; first of all I export the list of all my remote repositories to a configuration file using _this_ tool: + + github2mr > ~/Repos/.mrconfig.github + +* **NOTE**: The first time you create a new configuration file you will need to mark it as being trusted, because it is possible for configuration files to contain arbitrary shell-commands. + * Mark the configuration file as trusted by adding it's name to `~/.mrtrust`: + * `echo ~/Repos/.mrconfig.github >> ~/.mrtrust` + +Now that we've populated a configuration-file we can tell `mr` to checkout each of those repositories: + + mr --jobs 8 --config ~/Repos/.mrconfig.github + +Later in the week I can update all the repositories which have been cloned, pulling in any remote changes that have been made from other systems: + + mr --jobs 8 --config ~/Repos/.mrconfig.github update + +**NOTE**: If you prefer you can just use `update` all the time, `mr` will checkout a repository if it is missing as part of the `update` process. I'm using distinct flags here for clarity. Please read the `mr`-manpage to look at the commands it understands. + + +# Installation + +You should be able to install this application using the standard golang approach: + + $ go get github.com/skx/github2mr + +If you prefer you can [download the latest binary](http://github.com/skx/github2mr/releases) release, for various systems. + + + + +# Configuration / Usage + +Once installed you'll need to configure your github token, which you can generate from [withing your github settings](https://github.com/settings/tokens). + +you can either pass the token as an argument to the tool (via `github2mr -token=xxxxx`), or store it in the environment in the variable GITHUB_TOKEN: + + $ export GITHUB_TOKEN=xxxxx + $ github2mr [options] + +You can run `github2mr -help` to see available options, but in brief: + +* You can choose a default prefix to clone your repositories to. + * By default all repositories will be located at `~/Repos/${git_host}`. +* You can exclude all-organizational repositories. + * Or the reverse, ignoring all personal-repositories. +* You can exclude repositories by name. +* You can default to cloning repositories via HTTP, instead of SSH. + + +## Other Git Hosts + +This tool can be configured to point at other systems which use the same +API as the public-facing Github site. + +To use it against a self-hosted Github Enterprise installation, for example, +simply specify the URL: + + $ export GITHUB_TOKEN=xxxxx + $ github2mr -api=https://git.example.com/ [options] + +It has also been tested against an installation of [gitbucket](https://github.com/gitbucket/gitbucket) which can be configured a similar way - however in this case you'll find that you receive an error "401 bad credentials" unless you add the `-auth-header-token` flag: + + $ export GITHUB_TOKEN=xxxxx + $ github2mr -api=https://git.example.com/ -auth-header-token + +This seems to be related to the OAUTH header the library I'm using sends, by default it will send a HTTP request looking like this: + +``` +GET /api/v3/users/skx/repos HTTP/1.1 +Host: localhost:9999 +User-Agent: go-github +Accept: application/vnd.github.mercy-preview+json +Authorization: Bearer SECRET-TOKEN +Accept-Encoding: gzip +``` + +Notice that the value of the `Authorization`-header begins with `Bearer`? Gitbucket prefers to see `Authorization: token SECRET-VALUE-HERE`. + + + + +# Github Setup + +This repository is configured to run tests upon every commit, and when +pull-requests are created/updated. The testing is carried out via +[.github/run-tests.sh](.github/run-tests.sh) which is used by the +[github-action-tester](https://github.com/skx/github-action-tester) action. + +Releases are automated in a similar fashion via [.github/build](.github/build), +and the [github-action-publish-binaries](https://github.com/skx/github-action-publish-binaries) action. + + +Steve +-- diff --git a/TODO b/TODO new file mode 100644 index 0000000..a3bf394 --- /dev/null +++ b/TODO @@ -0,0 +1,21 @@ +* [x] Work for github personal user - i.e. me + +* [x] Work for organizations. + +* [x] Work against https://git.steve.org.uk/ + +* [-] Work with user/password (+/- 2FA). + +* [x] Allow filtering on name of repos, to cheat at excluding orgs. + +* [x] Allow limiting "public", "private", or "all" repos. + +* [x] Announce. + +* [x] Squash and release. + +* [x] Setup CI + +* [ ] Add decent README.md + +* [ ] Add tests (hard; since we're just injesting. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..843bfc6 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/skx/github2mr + +go 1.13 + +require ( + github.com/google/go-github/v29 v29.0.2 + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..762b1ae --- /dev/null +++ b/go.sum @@ -0,0 +1,22 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts= +github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..f7b4d7c --- /dev/null +++ b/main.go @@ -0,0 +1,437 @@ +// This is a trivial application which will output a dump of repositories +// which are hosted upon github, or some other host which uses a +// compatible API. +// +// It relies upon having an access-token for authentication. + +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "net/url" + "os" + "sort" + "strings" + "text/template" + + "github.com/google/go-github/v29/github" + "golang.org/x/oauth2" +) + +var ( + // + // Context for all calls + // + ctx context.Context + + // + // The actual github client + // + client *github.Client + + // + // The token to use for accessing the remote host. + // + // This is required because gitbucket prefers to see + // + // Authorization: token SECRET-TOKEN + // + // Instead of: + // + // Authorization: bearer SECRET-TOKEN + // + oauthToken = &oauth2.Token{} + + // + // The number of repos to fetch from the API at a time. + // + pageSize = 50 + + // + // Our version number, set for release-builds. + // + version = "unreleased" +) + +// Login accepts the address of a github endpoint, and a corresponding +// token to authenticate with. +// +// We use the login to get the user-information which confirms +// that the login was correct. +func Login(api string, token string) error { + + // Setup context + ctx = context.Background() + + // Setup token + ts := oauth2.StaticTokenSource(oauthToken) + tc := oauth2.NewClient(ctx, ts) + + // Create the API-client + client = github.NewClient(tc) + + // If the user is using a custom URL which doesn't have the + // versioned API-suffix add it. This appears to be necessary. + if api != "https://api.github.com/" { + if !strings.HasSuffix(api, "/api/v3/") { + if !strings.HasSuffix(api, "/") { + api += "/" + } + api += "api/v3/" + } + } + + // Parse the URL for sanity, and update the client with it + url, err := url.Parse(api) + if err != nil { + return err + } + client.BaseURL = url + + // Fetch user-information about the user who we are logging in as. + user, _, err := client.Users.Get(ctx, "") + if err != nil { + return err + } + + // Ensure we have a login + if *user.Login == "" { + return fmt.Errorf("we failed to find our username, which suggests our login failed") + } + + return nil +} + +// getPersonalRepos returns all the personal repositories which +// belong to our user. +func getPersonalRepos(fetch string) ([]*github.Repository, error) { + + var results []*github.Repository + + // Fetch in pages + opt := &github.RepositoryListOptions{ + ListOptions: github.ListOptions{PerPage: pageSize}, + Type: fetch, + } + + // Loop until we're done. + for { + repos, resp, err := client.Repositories.List(ctx, "", opt) + if err != nil { + return results, err + } + results = append(results, repos...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + + return results, nil + +} + +// getOrganizationalRepositores finds all the organizations the +// user is a member of, then fetches their repositories +func getOrganizationalRepositores(fetch string) ([]*github.Repository, error) { + + var results []*github.Repository + + // Get the organizations the user is a member of. + orgs, _, err := client.Organizations.List(ctx, "", nil) + if err != nil { + return results, err + } + + // Fetch in pages + opt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: pageSize}, + Type: fetch, + } + + // For each organization we want to get their repositories. + for _, org := range orgs { + + // Loop forever getting the repositories + for { + + repos, resp, err := client.Repositories.ListByOrg(ctx, *org.Login, opt) + if err != nil { + return results, err + } + results = append(results, repos...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + } + + return results, nil +} + +// +// Entry-point +// +func main() { + + // + // Parse flags + // + api := flag.String("api", "https://api.github.com/", "The API end-point to use for the remote git-host.") + authHeader := flag.Bool("auth-header-token", false, "Use an authorization-header including 'token' rather than 'bearer'.\nThis is required for gitbucket, and perhaps other systems.") + exclude := flag.String("exclude", "", "Comma-separated list of repositories to exclude.") + getOrgs := flag.String("organizations", "all", "Which organizational repositories to fetch.\nValid values are 'public', 'private', 'none', or 'all'.") + getPersonal := flag.String("personal", "all", "Which personal repositories to fetch.\nValid values are 'public', 'private', 'none', or 'all'.") + http := flag.Bool("http", false, "Generate HTTP-based clones rather than SSH-based ones.") + output := flag.String("output", "", "Write output to the named file, instead of printing to STDOUT.") + prefix := flag.String("prefix", "", "The prefix beneath which to store the repositories upon the current system.") + token := flag.String("token", "", "The API token used to authenticate to the remote API-host.") + versionCmd := flag.Bool("version", false, "Report upon our version, and terminate.") + flag.Parse() + + // + // Showing only the version? + // + if *versionCmd { + fmt.Printf("github2mr %s\n", version) + return + } + + // + // Validate the repository-types + // + if *getPersonal != "all" && + *getPersonal != "none" && + *getPersonal != "public" && + *getPersonal != "private" { + fmt.Fprintf(os.Stderr, "Valid settings are 'public', 'private', 'none', or 'all'\n") + return + } + if *getOrgs != "all" && + *getOrgs != "none" && + *getOrgs != "public" && + *getOrgs != "private" { + fmt.Fprintf(os.Stderr, "Valid settings are 'public', 'private', 'none', or 'all'\n") + return + } + + // + // Get the authentication token supplied via the flag, falling back + // to the environment if nothing has been specified. + // + tok := *token + if tok == "" { + // Fallback + tok = os.Getenv("GITHUB_TOKEN") + + if tok == "" { + fmt.Printf("Please specify your github token!\n") + return + } + } + + // + // Populate our global OAUTH token with the supplied value. + // + oauthToken.AccessToken = tok + + // + // Allow setting the authorization header-type, if required. + // + if *authHeader { + oauthToken.TokenType = "token" + } + + // + // Login and confirm that this worked. + // + err := Login(*api, tok) + if err != nil { + fmt.Fprintf(os.Stderr, "Login error - is your token set/correct? %s\n", err.Error()) + return + } + + // + // Fetch details of all "personal" repositories, unless we're not + // supposed to. + // + var personal []*github.Repository + if *getPersonal != "none" { + personal, err = getPersonalRepos(*getPersonal) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to fetch personal repository list: %s\n", err.Error()) + return + } + } + + // + // Fetch details of all organizational repositories, unless we're + // not supposed to. + // + var orgs []*github.Repository + if *getOrgs != "none" { + orgs, err = getOrganizationalRepositores(*getOrgs) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to fetch organizational repositories: %s\n", + err.Error()) + return + } + } + + // + // If the prefix is not set then create a default. + // + // This will be of the form: + // + // ~/Repos/github.com/x/y + // ~/Repos/git.example.com/x/y + // ~/Repos/git.steve.fi/x/y + // + // i.e "~/Repos/${git host}/${owner}/${path} + // + // (${git host} comes from the remote API host.) + // + repoPrefix := *prefix + if repoPrefix == "" { + + // Get the hostname + url, _ := url.Parse(*api) + host := url.Hostname() + + // Handle the obvious case + if host == "api.github.com" { + host = "github.com" + } + + // Generate a prefix + repoPrefix = os.Getenv("HOME") + "/Repos/" + host + } + + // + // Combine the results of the repositories we've found. + // + var all []*github.Repository + all = append(all, personal...) + all = append(all, orgs...) + + // + // Sort the list, based upon the full-name. + // + sort.Slice(all[:], func(i, j int) bool { + + // Case-insensitive sorting. + a := strings.ToLower(*all[i].FullName) + b := strings.ToLower(*all[j].FullName) + + return a < b + }) + + // + // Repos we're excluding + // + excluded := strings.Split(*exclude, ",") + + // + // Structure we use for template expansion + // + type Repo struct { + // Prefix-directory for local clones. + Prefix string + + // Name of the repository "owner/repo-name". + Name string + + // Source to clone from http/ssh-based. + Source string + } + + // + // Repos we will output + // + var repos []*Repo + + // + // Now format the repositories we've discovered. + // + for _, repo := range all { + + // + // The clone-type is configurable + // + clone := *repo.SSHURL + if *http { + clone = *repo.CloneURL + } + + // + // Should we exclude this entry? + // + skip := false + for _, exc := range excluded { + + exc = strings.TrimSpace(exc) + + if len(exc) > 0 && strings.Contains(strings.ToLower(clone), strings.ToLower(exc)) { + skip = true + } + } + + // Skipped + if skip { + continue + } + + repos = append(repos, &Repo{Prefix: repoPrefix, + Name: *repo.FullName, + Source: clone}) + } + + // + // Load the template we'll use for formatting the output + // + tmpl := `# Generated by github2mr - {{len .}} repositories + +{{range .}} +[{{.Prefix}}/{{.Name}}] +checkout = git clone {{.Source}} +{{end}} +` + + // + // Parse the template and execute it. + // + var out bytes.Buffer + t := template.Must(template.New("tmpl").Parse(tmpl)) + err = t.Execute(&out, repos) + + // + // If there were errors we're done. + // + if err != nil { + fmt.Fprintf(os.Stderr, "Error interpolating template:%s\n", err.Error()) + return + } + + // + // Show the results, or write to the specified file as appropriate + // + if *output != "" { + file, err := os.Create(*output) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to open %s:%s\n", *output, err.Error()) + return + } + defer file.Close() + file.Write(out.Bytes()) + } else { + fmt.Println(out.String()) + } + + // + // All done. + // +}