Initial release of github2mr.

This is the initial release which is fully-functional and works
for myself, self-hosted github enterprise installations, and
privately hosted gitbucket installs.

It has not been tested against other systems (gogs, gitea, etc),
but reports of success/failure or patches would be most welcome.

.github/build

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# The basename of our binary
+BASE="github2mr"
+
+# Setup an output directory - creating if missing
+cur=$(pwd)
+OUTPUT="${cur}/bin"
+if [ ! -d "${OUTPUT}" ]; then
+    mkdir -p "${OUTPUT}"
+fi
+
+# We build on multiple platforms/archs
+BUILD_PLATFORMS="linux darwin freebsd"
+BUILD_ARCHS="amd64 386"
+
+# For each platform.
+for OS in ${BUILD_PLATFORMS[@]}; do
+
+    # For each arch
+    for ARCH in ${BUILD_ARCHS[@]}; do
+
+        # Setup a suffix for the binary
+        SUFFIX="${OS}"
+
+        # i386 is better than 386
+        if [ "$ARCH" = "386" ]; then
+            SUFFIX="${SUFFIX}-i386"
+        else
+            SUFFIX="${SUFFIX}-${ARCH}"
+        fi
+
+        echo "Building for ${OS} [${ARCH}] -> ${BASE}-${SUFFIX}"
+
+        # Run the build
+        export GOARCH=${ARCH}
+        export GOOS=${OS}
+        export CGO_ENABLED=0
+
+        # Build the main-binary
+        go build -ldflags "-X main.version=$(git describe --tags 2>/dev/null || echo 'master')" -o "${OUTPUT}/${BASE}-${SUFFIX}"
+    done
+done

.github/run-tests.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Install tools to test our code-quality.
+go get -u golang.org/x/lint/golint
+
+# Failures cause aborts
+set -e
+
+# Run the linter
+echo "Launching linter .."
+golint -set_exit_status ./...
+echo "Completed linter .."
+
+# Run the vet-checker.
+echo "Launching go vet check .."
+go vet ./...
+echo "Completed go vet check .."
+

.github/workflows/pull_request.yml

@@ -0,0 +1,10 @@
+on: pull_request
+name: Pull Request
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - name: Test
+      uses: skx/github-action-tester@master

.github/workflows/push.yml

@@ -0,0 +1,13 @@
+on:
+  push:
+    branches:
+    - master
+name: Push Event
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - name: Test
+      uses: skx/github-action-tester@master

.github/workflows/release.yml

@@ -0,0 +1,19 @@
+on:
+  release:
+      types: [created]
+name: Handle Release
+jobs:
+  upload:
+    name: Upload
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the repository
+      uses: actions/checkout@master
+    - name: Generate the artifacts
+      uses: skx/github-action-build@master
+    - name: Upload the artifacts
+      uses: skx/github-action-publish-binaries@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        args: bin/*-*

.gitignore

@@ -0,0 +1 @@
+bin/

README.md

@@ -0,0 +1,116 @@
+
+* [github2mr](#github2mr)
+  * [Brief mr Example](#brief-mr-example)
+* [Installation](#installation)
+* [Configuration / Usage](#configuration--usage)
+  * [Other Git Hosts](#other-git-hosts)
+* [Github Setup](#github-setup)
+
+
+
+
+# github2mr
+
+Many [Github](https://github.com/) users have a large number of repositories upon which they work.  This application allows you to dump all your repository details into a configuration file for [myrepos](https://myrepos.branchable.com/).
+
+The myrepos package, containing a binary named `mr`, is a _wonderful_ tool that lets you apply operations to multiple repositories at once given a suitable configuration.
+
+The end result of using `mr` and `github2mr` is that you should be able to clone all your remote github repositories, and update them easily with only a couple of commands which is great for when you work/develop/live on multiple machines.
+
+
+## Brief `mr` Example
+
+Let us pretend I'm moving to a new machine; first of all I export the list of all my remote repositories to a configuration file using _this_ tool:
+
+    github2mr > ~/Repos/.mrconfig.github
+
+* **NOTE**: The first time you create a new configuration file you will need to mark it as being trusted, because it is possible for configuration files to contain arbitrary shell-commands.
+  * Mark the configuration file as trusted by adding it's name to `~/.mrtrust`:
+      * `echo ~/Repos/.mrconfig.github >> ~/.mrtrust`
+
+Now that we've populated a configuration-file we can tell `mr` to checkout each of those repositories:
+
+    mr --jobs 8 --config ~/Repos/.mrconfig.github
+
+Later in the week I can update all the repositories which have been cloned, pulling in any remote changes that have been made from other systems:
+
+    mr --jobs 8 --config ~/Repos/.mrconfig.github update
+
+**NOTE**: If you prefer you can just use `update` all the time, `mr` will checkout a repository if it is missing as part of the `update` process.  I'm using distinct flags here for clarity.  Please read the `mr`-manpage to look at the commands it understands.
+
+
+# Installation
+
+You should be able to install this application using the standard golang approach:
+
+    $ go get github.com/skx/github2mr
+
+If you prefer you can [download the latest binary](http://github.com/skx/github2mr/releases) release, for various systems.
+
+
+
+
+# Configuration / Usage
+
+Once installed you'll need to configure your github token, which you can generate from [withing your github settings](https://github.com/settings/tokens).
+
+you can either pass the token as an argument to the tool (via `github2mr -token=xxxxx`), or store it in the environment in the variable GITHUB_TOKEN:
+
+     $ export GITHUB_TOKEN=xxxxx
+     $ github2mr [options]
+
+You can run `github2mr -help` to see available options, but in brief:
+
+* You can choose a default prefix to clone your repositories to.
+  * By default all repositories will be located at `~/Repos/${git_host}`.
+* You can exclude all-organizational repositories.
+  * Or the reverse, ignoring all personal-repositories.
+* You can exclude repositories by name.
+* You can default to cloning repositories via HTTP, instead of SSH.
+
+
+## Other Git Hosts
+
+This tool can be configured to point at other systems which use the same
+API as the public-facing Github site.
+
+To use it against a self-hosted Github Enterprise installation, for example,
+simply specify the URL:
+
+     $ export GITHUB_TOKEN=xxxxx
+     $ github2mr -api=https://git.example.com/ [options]
+
+It has also been tested against an installation of [gitbucket](https://github.com/gitbucket/gitbucket) which can be configured a similar way - however in this case you'll find that you receive an error "401 bad credentials" unless you add the `-auth-header-token` flag:
+
+      $ export GITHUB_TOKEN=xxxxx
+      $ github2mr -api=https://git.example.com/ -auth-header-token
+
+This seems to be related to the OAUTH header the library I'm using sends, by default it will send a HTTP request looking like this:
+
+```
+GET /api/v3/users/skx/repos HTTP/1.1
+Host: localhost:9999
+User-Agent: go-github
+Accept: application/vnd.github.mercy-preview+json
+Authorization: Bearer SECRET-TOKEN
+Accept-Encoding: gzip
+```
+
+Notice that the value of the `Authorization`-header begins with `Bearer`?  Gitbucket prefers to see `Authorization: token SECRET-VALUE-HERE`.
+
+
+
+
+# Github Setup
+
+This repository is configured to run tests upon every commit, and when
+pull-requests are created/updated.  The testing is carried out via
+[.github/run-tests.sh](.github/run-tests.sh) which is used by the
+[github-action-tester](https://github.com/skx/github-action-tester) action.
+
+Releases are automated in a similar fashion via [.github/build](.github/build),
+and the [github-action-publish-binaries](https://github.com/skx/github-action-publish-binaries) action.
+
+
+Steve
+--

TODO

@@ -0,0 +1,21 @@
+* [x] Work for github personal user - i.e. me
+
+* [x] Work for organizations.
+
+* [x] Work against https://git.steve.org.uk/
+
+* [-] Work with user/password (+/- 2FA).
+
+* [x] Allow filtering on name of repos, to cheat at excluding orgs.
+
+* [x] Allow limiting "public", "private", or "all" repos.
+
+* [x] Announce.
+
+* [x] Squash and release.
+
+* [x] Setup CI
+
+* [ ] Add decent README.md
+
+* [ ] Add tests (hard; since we're just injesting.

go.mod

@@ -0,0 +1,8 @@
+module github.com/skx/github2mr
+
+go 1.13
+
+require (
+	github.com/google/go-github/v29 v29.0.2
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+)

go.sum

@@ -0,0 +1,22 @@
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
+github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

main.go

@@ -0,0 +1,437 @@
+// This is a trivial application which will output a dump of repositories
+// which are hosted upon github, or some other host which uses a
+// compatible API.
+//
+// It relies upon having an access-token for authentication.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"text/template"
+
+	"github.com/google/go-github/v29/github"
+	"golang.org/x/oauth2"
+)
+
+var (
+	//
+	// Context for all calls
+	//
+	ctx context.Context
+
+	//
+	// The actual github client
+	//
+	client *github.Client
+
+	//
+	// The token to use for accessing the remote host.
+	//
+	// This is required because gitbucket prefers to see
+	//
+	//     Authorization: token SECRET-TOKEN
+	//
+	// Instead of:
+	//
+	//     Authorization: bearer SECRET-TOKEN
+	//
+	oauthToken = &oauth2.Token{}
+
+	//
+	// The number of repos to fetch from the API at a time.
+	//
+	pageSize = 50
+
+	//
+	// Our version number, set for release-builds.
+	//
+	version = "unreleased"
+)
+
+// Login accepts the address of a github endpoint, and a corresponding
+// token to authenticate with.
+//
+// We use the login to get the user-information which confirms
+// that the login was correct.
+func Login(api string, token string) error {
+
+	// Setup context
+	ctx = context.Background()
+
+	// Setup token
+	ts := oauth2.StaticTokenSource(oauthToken)
+	tc := oauth2.NewClient(ctx, ts)
+
+	// Create the API-client
+	client = github.NewClient(tc)
+
+	// If the user is using a custom URL which doesn't have the
+	// versioned API-suffix add it.  This appears to be necessary.
+	if api != "https://api.github.com/" {
+		if !strings.HasSuffix(api, "/api/v3/") {
+			if !strings.HasSuffix(api, "/") {
+				api += "/"
+			}
+			api += "api/v3/"
+		}
+	}
+
+	// Parse the URL for sanity, and update the client with it
+	url, err := url.Parse(api)
+	if err != nil {
+		return err
+	}
+	client.BaseURL = url
+
+	// Fetch user-information about the user who we are logging in as.
+	user, _, err := client.Users.Get(ctx, "")
+	if err != nil {
+		return err
+	}
+
+	// Ensure we have a login
+	if *user.Login == "" {
+		return fmt.Errorf("we failed to find our username, which suggests our login failed")
+	}
+
+	return nil
+}
+
+// getPersonalRepos returns all the personal repositories which
+// belong to our user.
+func getPersonalRepos(fetch string) ([]*github.Repository, error) {
+
+	var results []*github.Repository
+
+	// Fetch in pages
+	opt := &github.RepositoryListOptions{
+		ListOptions: github.ListOptions{PerPage: pageSize},
+		Type:        fetch,
+	}
+
+	// Loop until we're done.
+	for {
+		repos, resp, err := client.Repositories.List(ctx, "", opt)
+		if err != nil {
+			return results, err
+		}
+		results = append(results, repos...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	return results, nil
+
+}
+
+// getOrganizationalRepositores finds all the organizations the
+// user is a member of, then fetches their repositories
+func getOrganizationalRepositores(fetch string) ([]*github.Repository, error) {
+
+	var results []*github.Repository
+
+	// Get the organizations the user is a member of.
+	orgs, _, err := client.Organizations.List(ctx, "", nil)
+	if err != nil {
+		return results, err
+	}
+
+	// Fetch in pages
+	opt := &github.RepositoryListByOrgOptions{
+		ListOptions: github.ListOptions{PerPage: pageSize},
+		Type:        fetch,
+	}
+
+	// For each organization we want to get their repositories.
+	for _, org := range orgs {
+
+		// Loop forever getting the repositories
+		for {
+
+			repos, resp, err := client.Repositories.ListByOrg(ctx, *org.Login, opt)
+			if err != nil {
+				return results, err
+			}
+			results = append(results, repos...)
+			if resp.NextPage == 0 {
+				break
+			}
+			opt.Page = resp.NextPage
+		}
+	}
+
+	return results, nil
+}
+
+//
+// Entry-point
+//
+func main() {
+
+	//
+	// Parse flags
+	//
+	api := flag.String("api", "https://api.github.com/", "The API end-point to use for the remote git-host.")
+	authHeader := flag.Bool("auth-header-token", false, "Use an authorization-header including 'token' rather than 'bearer'.\nThis is required for gitbucket, and perhaps other systems.")
+	exclude := flag.String("exclude", "", "Comma-separated list of repositories to exclude.")
+	getOrgs := flag.String("organizations", "all", "Which organizational repositories to fetch.\nValid values are 'public', 'private', 'none', or 'all'.")
+	getPersonal := flag.String("personal", "all", "Which personal repositories to fetch.\nValid values are 'public', 'private', 'none', or 'all'.")
+	http := flag.Bool("http", false, "Generate HTTP-based clones rather than SSH-based ones.")
+	output := flag.String("output", "", "Write output to the named file, instead of printing to STDOUT.")
+	prefix := flag.String("prefix", "", "The prefix beneath which to store the repositories upon the current system.")
+	token := flag.String("token", "", "The API token used to authenticate to the remote API-host.")
+	versionCmd := flag.Bool("version", false, "Report upon our version, and terminate.")
+	flag.Parse()
+
+	//
+	// Showing only the version?
+	//
+	if *versionCmd {
+		fmt.Printf("github2mr %s\n", version)
+		return
+	}
+
+	//
+	// Validate the repository-types
+	//
+	if *getPersonal != "all" &&
+		*getPersonal != "none" &&
+		*getPersonal != "public" &&
+		*getPersonal != "private" {
+		fmt.Fprintf(os.Stderr, "Valid settings are 'public', 'private', 'none', or 'all'\n")
+		return
+	}
+	if *getOrgs != "all" &&
+		*getOrgs != "none" &&
+		*getOrgs != "public" &&
+		*getOrgs != "private" {
+		fmt.Fprintf(os.Stderr, "Valid settings are 'public', 'private', 'none', or 'all'\n")
+		return
+	}
+
+	//
+	// Get the authentication token supplied via the flag, falling back
+	// to the environment if nothing has been specified.
+	//
+	tok := *token
+	if tok == "" {
+		// Fallback
+		tok = os.Getenv("GITHUB_TOKEN")
+
+		if tok == "" {
+			fmt.Printf("Please specify your github token!\n")
+			return
+		}
+	}
+
+	//
+	// Populate our global OAUTH token with the supplied value.
+	//
+	oauthToken.AccessToken = tok
+
+	//
+	// Allow setting the authorization header-type, if required.
+	//
+	if *authHeader {
+		oauthToken.TokenType = "token"
+	}
+
+	//
+	// Login and confirm that this worked.
+	//
+	err := Login(*api, tok)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Login error - is your token set/correct? %s\n", err.Error())
+		return
+	}
+
+	//
+	// Fetch details of all "personal" repositories, unless we're not
+	// supposed to.
+	//
+	var personal []*github.Repository
+	if *getPersonal != "none" {
+		personal, err = getPersonalRepos(*getPersonal)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to fetch personal repository list: %s\n", err.Error())
+			return
+		}
+	}
+
+	//
+	// Fetch details of all organizational repositories, unless we're
+	// not supposed to.
+	//
+	var orgs []*github.Repository
+	if *getOrgs != "none" {
+		orgs, err = getOrganizationalRepositores(*getOrgs)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to fetch organizational repositories: %s\n",
+				err.Error())
+			return
+		}
+	}
+
+	//
+	// If the prefix is not set then create a default.
+	//
+	// This will be of the form:
+	//
+	//    ~/Repos/github.com/x/y
+	//    ~/Repos/git.example.com/x/y
+	//    ~/Repos/git.steve.fi/x/y
+	//
+	// i.e "~/Repos/${git host}/${owner}/${path}
+	//
+	// (${git host} comes from the remote API host.)
+	//
+	repoPrefix := *prefix
+	if repoPrefix == "" {
+
+		// Get the hostname
+		url, _ := url.Parse(*api)
+		host := url.Hostname()
+
+		// Handle the obvious case
+		if host == "api.github.com" {
+			host = "github.com"
+		}
+
+		// Generate a prefix
+		repoPrefix = os.Getenv("HOME") + "/Repos/" + host
+	}
+
+	//
+	// Combine the results of the repositories we've found.
+	//
+	var all []*github.Repository
+	all = append(all, personal...)
+	all = append(all, orgs...)
+
+	//
+	// Sort the list, based upon the full-name.
+	//
+	sort.Slice(all[:], func(i, j int) bool {
+
+		// Case-insensitive sorting.
+		a := strings.ToLower(*all[i].FullName)
+		b := strings.ToLower(*all[j].FullName)
+
+		return a < b
+	})
+
+	//
+	// Repos we're excluding
+	//
+	excluded := strings.Split(*exclude, ",")
+
+	//
+	// Structure we use for template expansion
+	//
+	type Repo struct {
+		// Prefix-directory for local clones.
+		Prefix string
+
+		// Name of the repository "owner/repo-name".
+		Name string
+
+		// Source to clone from http/ssh-based.
+		Source string
+	}
+
+	//
+	// Repos we will output
+	//
+	var repos []*Repo
+
+	//
+	// Now format the repositories we've discovered.
+	//
+	for _, repo := range all {
+
+		//
+		// The clone-type is configurable
+		//
+		clone := *repo.SSHURL
+		if *http {
+			clone = *repo.CloneURL
+		}
+
+		//
+		// Should we exclude this entry?
+		//
+		skip := false
+		for _, exc := range excluded {
+
+			exc = strings.TrimSpace(exc)
+
+			if len(exc) > 0 && strings.Contains(strings.ToLower(clone), strings.ToLower(exc)) {
+				skip = true
+			}
+		}
+
+		// Skipped
+		if skip {
+			continue
+		}
+
+		repos = append(repos, &Repo{Prefix: repoPrefix,
+			Name:   *repo.FullName,
+			Source: clone})
+	}
+
+	//
+	// Load the template we'll use for formatting the output
+	//
+	tmpl := `# Generated by github2mr - {{len .}} repositories
+
+{{range .}}
+[{{.Prefix}}/{{.Name}}]
+checkout = git clone {{.Source}}
+{{end}}
+`
+
+	//
+	// Parse the template and execute it.
+	//
+	var out bytes.Buffer
+	t := template.Must(template.New("tmpl").Parse(tmpl))
+	err = t.Execute(&out, repos)
+
+	//
+	// If there were errors we're done.
+	//
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error interpolating template:%s\n", err.Error())
+		return
+	}
+
+	//
+	// Show the results, or write to the specified file as appropriate
+	//
+	if *output != "" {
+		file, err := os.Create(*output)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to open %s:%s\n", *output, err.Error())
+			return
+		}
+		defer file.Close()
+		file.Write(out.Bytes())
+	} else {
+		fmt.Println(out.String())
+	}
+
+	//
+	// All done.
+	//
+}