tools: update offsets tool to work with go versions 1.7 - 1.10

Older go versions (1.7.x) specify a fixed page size (_PageSize const) as
part of their runtime whereas newer go versions populate the page size at
runtime.

The kernel asm bootstrap code was written with go 1.8 in mind. As a
result it attempts to populate the page size manually which obviously
breaks compilation in go 1.7.

The offsets tool has been updated to emit the special def
"SKIP_PAGESIZE_SETUP" when running under go 1.7 which allows us to
perform conditional compilation of the page setup code inside the
bootstrap asm code.

fixup

Makefile

@@ -22,7 +22,6 @@ FUZZ_PKG_LIST := src/gopheros/device/acpi/aml
 ifeq ($(OS), Linux)
 export SHELL := /bin/bash -o pipefail
 
-
 LD := ld
 AS := nasm
 
@@ -105,7 +104,7 @@ $(BUILD_DIR)/go_asm_offsets.inc:
 	@mkdir -p $(BUILD_DIR)
 
 	@echo "[tools:offsets] calculating OS/arch-specific offsets for g, m and stack structs"
-	@GOPATH=$(GOPATH) $(GO) run tools/offsets/offsets.go -target-os $(GOOS) -target-arch $(GOARCH) -go-binary $(GO) -out $@
+	@GOROOT=$(GOROOT) GOPATH=$(GOPATH) $(GO) run tools/offsets/offsets.go -target-os $(GOOS) -target-arch $(GOARCH) -go-binary $(GO) -out $@
 
 $(BUILD_DIR)/arch/$(ARCH)/asm/%.o: src/arch/$(ARCH)/asm/%.s
 	@mkdir -p $(shell dirname $@)

src/arch/x86_64/asm/rt0_64.s

@@ -75,11 +75,13 @@ _rt0_64_entry:
 _rt0_64_setup_go_runtime_structs:
 	%include "go_asm_offsets.inc" ; generated by tools/offsets
 
+	%ifndef SKIP_PAGESIZE_SETUP
 	; The Go allocator expects this symbol to be set to the system page size
-	; As the kernel bypass osinit() this needs to be set here.
+	; As the kernel bypasses osinit() this needs to be manually set here.
 	extern runtime.physPageSize
 	mov rax, runtime.physPageSize
 	mov qword [rax], 0x1000 ; 4096
+	%endif
 
 	; Setup r0_g stack limits using the reserved stack
 	extern stack_top

tools/offsets/offsets.go

@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -30,6 +32,7 @@ func genBuildScript(targetOS, targetArch, goBinary, workDir string) ([]byte, err
 	// rebuild the runtime packages.
 	cmd := exec.Command(goBinary, "build", "-a", "-n")
 	cmd.Dir = workDir
+	cmd.Env = append(cmd.Env, fmt.Sprintf("GOROOT=%s", os.Getenv("GOROOT")))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("GOOS=%s", targetOS))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("GOARCH=%s", targetArch))
 	out, err := cmd.CombinedOutput()
@@ -41,13 +44,18 @@ func genBuildScript(targetOS, targetArch, goBinary, workDir string) ([]byte, err
 }
 
 func patchBuildScript(script []byte, workDir, targetOS, targetArch, goBinary string) ([]byte, error) {
-	lines := strings.Split(string(script), "\n")
+	// Replace $WORK with the workDir location. This is required for executing
+	// build scripts generated by go 1.10
+	lines := strings.Split(
+		strings.Replace(string(script), "$WORK", workDir, -1),
+		"\n",
+	)
 
 	// Inject os/arch and workdir to the top of the build file
 	header := []string{
+		fmt.Sprintf("export GOROOT=%s", os.Getenv("GOROOT")),
 		fmt.Sprintf("export GOOS=%s", targetOS),
 		fmt.Sprintf("export GOARCH=%s", targetArch),
-		fmt.Sprintf("WORK=%q", workDir),
 		fmt.Sprintf("alias pack='%s tool pack'", goBinary),
 	}
 	lines = append(header, lines...)
@@ -58,7 +66,7 @@ func patchBuildScript(script []byte, workDir, targetOS, targetArch, goBinary str
 	var stopOnNextComment bool
 	for lineIndex := 0; lineIndex < len(lines); lineIndex++ {
 		// Ignore empty comments
-		if lines[lineIndex] == "#" {
+		if strings.TrimSpace(lines[lineIndex]) == "#" || strings.Contains(lines[lineIndex], "# import") {
 			continue
 		}
 
@@ -97,8 +105,32 @@ func execBuildScript(script []byte, workDir string) error {
 }
 
 func genAsmIncludes(workDir string) ([]byte, error) {
-	headers, err := ioutil.ReadFile(fmt.Sprintf("%s/runtime/_obj/go_asm.h", workDir))
+	// Find all generated go_asm.h files and concat their conentents
-	if err != nil {
+	var (
+		allHeaders, headers []byte
+	)
+
+	if err := filepath.Walk(workDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if filepath.Base(path) != "go_asm.h" {
+			return nil
+		}
+
+		if headers, err = ioutil.ReadFile(path); err != nil {
+			return err
+		}
+
+		allHeaders = append(allHeaders, '\n')
+		allHeaders = append(allHeaders, headers...)
+		return nil
+	}); err != nil {
 		return nil, err
 	}
 
@@ -106,7 +138,7 @@ func genAsmIncludes(workDir string) ([]byte, error) {
 	includes = append(includes, "; vim: set ft=nasm :\n")
 	includes = append(includes, fmt.Sprintf("; generated by tools/offsets at %v\n", time.Now()))
 
-	for _, line := range strings.Split(string(headers), "\n") {
+	for _, line := range strings.Split(string(allHeaders), "\n") {
 		line = strings.TrimPrefix(line, "#define ")
 
 		// We are only interested in the offsets for the g, m and stack structures
@@ -131,6 +163,16 @@ func genAsmIncludes(workDir string) ([]byte, error) {
 		}
 	}
 
+	// In go 1.7.x, the page size is given by the _PageSize constant whereas in
+	// newer go versions it is specified at runtime and needs to be manually set
+	// by our asm bootstrap code.
+	if strings.Contains(runtime.Version(), "go1.7") {
+		includes = append(includes,
+			"; go 1.7 runtime uses a fixed 4k page size for our target arch so our bootstrap code does not need to do any extra work to set it up",
+			"%define SKIP_PAGESIZE_SETUP 1",
+		)
+	}
+
 	return []byte(strings.Join(includes, "\n")), nil
 }