Merge pull request #66 from achilleasa/ensure-kernel-builds-with-different-go-versions

Ensure kernel builds with different go versions

.travis.yml

@@ -1,9 +1,29 @@
 language: go
 sudo: required
+addons:
+  apt:
+    packages:
+    - nasm
+    - binutils-2.26
+# Compile kernel with various go versions
 go:
+  - 1.7.x
+  - 1.8.x
+  - 1.9.x
+  - 1.10.x
   - 1.x
-script:
+stage: build kernel
-  - make lint
+before_install:
-  - make collect-coverage
+  - export PATH=/usr/lib/binutils-2.26/bin:${PATH}
-after_success:
+script: make kernel
-  - bash <(curl -s https://codecov.io/bash)
+
+# Run the tests against the latest go version
+jobs:
+  include:
+    - stage: run tests
+      go: 1.x
+      script:
+        - make lint
+        - make collect-coverage
+      after_success:
+        - bash <(curl -s https://codecov.io/bash)

BUILD.md

@@ -11,7 +11,7 @@ To compile gopher-os wheh running on Linux you need a fairly recent version of:
 - grub
 - nasm 
 - gcc (for GNU ld)
-- go (1.6+; recommended: 1.8)
+- go 1.7+
 
 The above dependencies can be installed using the appropriate package manager 
 for each particular Linux distribution.

Makefile

@@ -22,7 +22,6 @@ FUZZ_PKG_LIST := src/gopheros/device/acpi/aml
 ifeq ($(OS), Linux)
 export SHELL := /bin/bash -o pipefail
 
-
 LD := ld
 AS := nasm
 
@@ -62,11 +61,13 @@ go.o:
 	    -e "1s|^|export GOOS=$(GOOS)\n|" \
 	    -e "1s|^|export GOARCH=$(GOARCH)\n|" \
 	    -e "1s|^|export GOROOT=$(GOROOT)\n|" \
-	    -e "1s|^|WORK='$(BUILD_ABS_DIR)'\n|" \
+	    -e "1s|^|export CGO_ENABLED=0\n|" \
 	    -e "1s|^|alias pack='$(GO) tool pack'\n|" \
 	    -e "/^mv/d" \
+	    -e "/\/buildid/d" \
 	    -e "s|-extld|-tmpdir='$(BUILD_ABS_DIR)' -linkmode=external -extldflags='-nostartfiles -nodefaultlibs -nostdlib -r' -extld|g" \
-	    | sh 2>&1 | sed -e "s/^/  | /g"
+	    -e 's|$$WORK|$(BUILD_ABS_DIR)|g' \
+            | sh 2>&1 |  sed -e "s/^/  | /g"
 
 	@# build/go.o is a elf32 object file but all go symbols are unexported. Our
 	@# asm entrypoint code needs to know the address to 'main.main' so we use
@@ -105,7 +106,7 @@ $(BUILD_DIR)/go_asm_offsets.inc:
 	@mkdir -p $(BUILD_DIR)
 
 	@echo "[tools:offsets] calculating OS/arch-specific offsets for g, m and stack structs"
-	@GOPATH=$(GOPATH) $(GO) run tools/offsets/offsets.go -target-os $(GOOS) -target-arch $(GOARCH) -go-binary $(GO) -out $@
+	@GOROOT=$(GOROOT) GOPATH=$(GOPATH) $(GO) run tools/offsets/offsets.go -target-os $(GOOS) -target-arch $(GOARCH) -go-binary $(GO) -out $@
 
 $(BUILD_DIR)/arch/$(ARCH)/asm/%.o: src/arch/$(ARCH)/asm/%.s
 	@mkdir -p $(shell dirname $@)

README.md

@@ -4,6 +4,17 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/achilleasa/gopher-os)](https://goreportcard.com/report/github.com/achilleasa/gopher-os)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 
+| Go 1.7.x            | Go 1.8.x            | Go 1.9.x            | Go 1.10.x            | Go 1.x            |
+|---------------------|---------------------|---------------------|----------------------|-------------------|
+| [![go 1.7.x][1]][6] | [![go 1.8.x][2]][6] | [![Go 1.9.x][3]][6] | [![go 1.10.x][4]][6] | [![go 1.x][5]][6] |
+
+[1]: https://travis-matrix-badges.herokuapp.com/repos/achilleasa/gopher-os/branches/master/1
+[2]: https://travis-matrix-badges.herokuapp.com/repos/achilleasa/gopher-os/branches/master/2
+[3]: https://travis-matrix-badges.herokuapp.com/repos/achilleasa/gopher-os/branches/master/3
+[4]: https://travis-matrix-badges.herokuapp.com/repos/achilleasa/gopher-os/branches/master/4
+[5]: https://travis-matrix-badges.herokuapp.com/repos/achilleasa/gopher-os/branches/master/5
+[6]: https://travis-ci.org/achilleasa/gopher-os
+
 The goal of this project is to build a 64-bit POSIX-compliant tick-less kernel
 with a Linux-compatible syscall implementation using [Go](https://golang.org). 
 

src/arch/x86_64/asm/cgo_stubs.s

@@ -6,6 +6,7 @@ bits 64
 global x_cgo_callers
 global x_cgo_init
 global x_cgo_mmap
+global x_cgo_munmap
 global x_cgo_notify_runtime_init_done
 global x_cgo_sigaction
 global x_cgo_thread_start
@@ -17,6 +18,7 @@ global _cgo_yield
 x_cgo_callers:
 x_cgo_init:
 x_cgo_mmap:
+x_cgo_munmap:
 x_cgo_notify_runtime_init_done:
 x_cgo_sigaction:
 x_cgo_thread_start:

src/arch/x86_64/asm/rt0_64.s

@@ -75,11 +75,13 @@ _rt0_64_entry:
 _rt0_64_setup_go_runtime_structs:
 	%include "go_asm_offsets.inc" ; generated by tools/offsets
 
+	%ifndef SKIP_PAGESIZE_SETUP
 	; The Go allocator expects this symbol to be set to the system page size
-	; As the kernel bypass osinit() this needs to be set here.
+	; As the kernel bypasses osinit() this needs to be manually set here.
 	extern runtime.physPageSize
 	mov rax, runtime.physPageSize
 	mov qword [rax], 0x1000 ; 4096
+	%endif
 
 	; Setup r0_g stack limits using the reserved stack
 	extern stack_top

src/gopheros/kernel/goruntime/bootstrap.go

@@ -27,27 +27,6 @@ var (
 	prngSeed = 0xdeadc0de
 )
 
-//go:linkname algInit runtime.alginit
-func algInit()
-
-//go:linkname modulesInit runtime.modulesinit
-func modulesInit()
-
-//go:linkname typeLinksInit runtime.typelinksinit
-func typeLinksInit()
-
-//go:linkname itabsInit runtime.itabsinit
-func itabsInit()
-
-//go:linkname mallocInit runtime.mallocinit
-func mallocInit()
-
-//go:linkname mSysStatInc runtime.mSysStatInc
-func mSysStatInc(*uint64, uintptr)
-
-//go:linkname procResize runtime.procresize
-func procResize(int32) uintptr
-
 // initGoPackages is an alias to main.init which recursively calls the init()
 // methods in all imported packages. Unless this function is called, things like
 // package errors will not be properly initialized causing various problems when
@@ -184,7 +163,7 @@ func getRandomData(r []byte) {
 func Init() *kernel.Error {
 	mallocInitFn()
 	algInitFn()       // setup hash implementation for map keys
-	modulesInitFn()   // provides activeModules
+	modulesInitFn()   // provides activeModules (go 1.8+)
 	typeLinksInitFn() // uses maps, activeModules
 	itabsInitFn()     // uses activeModules
 

src/gopheros/kernel/goruntime/bootstrap_go17.go

@@ -0,0 +1,30 @@
+// +build go1.7,!go1.8
+
+package goruntime
+
+import (
+	_ "unsafe" // required for go:linkname
+)
+
+//go:linkname algInit runtime.alginit
+func algInit()
+
+//go:linkname typeLinksInit runtime.typelinksinit
+func typeLinksInit()
+
+//go:linkname itabsInit runtime.itabsinit
+func itabsInit()
+
+//go:linkname mallocInit runtime.mallocinit
+func mallocInit()
+
+//go:linkname mSysStatInc runtime.mSysStatInc
+func mSysStatInc(*uint64, uintptr)
+
+//go:linkname procResize runtime.procresize
+func procResize(int32) uintptr
+
+// modulesInit is defined on go1.8 so just declare an empty
+// stub for go 1.7 to keep the compiler happy.
+func modulesInit() {
+}

src/gopheros/kernel/goruntime/bootstrap_go18+.go

@@ -0,0 +1,28 @@
+// +build go1.8
+
+package goruntime
+
+import (
+	_ "unsafe" // required for go:linkname
+)
+
+//go:linkname algInit runtime.alginit
+func algInit()
+
+//go:linkname modulesInit runtime.modulesinit
+func modulesInit()
+
+//go:linkname typeLinksInit runtime.typelinksinit
+func typeLinksInit()
+
+//go:linkname itabsInit runtime.itabsinit
+func itabsInit()
+
+//go:linkname mallocInit runtime.mallocinit
+func mallocInit()
+
+//go:linkname mSysStatInc runtime.mSysStatInc
+func mSysStatInc(*uint64, uintptr)
+
+//go:linkname procResize runtime.procresize
+func procResize(int32) uintptr

tools/offsets/offsets.go

@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -30,6 +32,7 @@ func genBuildScript(targetOS, targetArch, goBinary, workDir string) ([]byte, err
 	// rebuild the runtime packages.
 	cmd := exec.Command(goBinary, "build", "-a", "-n")
 	cmd.Dir = workDir
+	cmd.Env = append(cmd.Env, fmt.Sprintf("GOROOT=%s", os.Getenv("GOROOT")))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("GOOS=%s", targetOS))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("GOARCH=%s", targetArch))
 	out, err := cmd.CombinedOutput()
@@ -41,13 +44,18 @@ func genBuildScript(targetOS, targetArch, goBinary, workDir string) ([]byte, err
 }
 
 func patchBuildScript(script []byte, workDir, targetOS, targetArch, goBinary string) ([]byte, error) {
-	lines := strings.Split(string(script), "\n")
+	// Replace $WORK with the workDir location. This is required for executing
+	// build scripts generated by go 1.10
+	lines := strings.Split(
+		strings.Replace(string(script), "$WORK", workDir, -1),
+		"\n",
+	)
 
 	// Inject os/arch and workdir to the top of the build file
 	header := []string{
+		fmt.Sprintf("export GOROOT=%s", os.Getenv("GOROOT")),
 		fmt.Sprintf("export GOOS=%s", targetOS),
 		fmt.Sprintf("export GOARCH=%s", targetArch),
-		fmt.Sprintf("WORK=%q", workDir),
 		fmt.Sprintf("alias pack='%s tool pack'", goBinary),
 	}
 	lines = append(header, lines...)
@@ -58,7 +66,7 @@ func patchBuildScript(script []byte, workDir, targetOS, targetArch, goBinary str
 	var stopOnNextComment bool
 	for lineIndex := 0; lineIndex < len(lines); lineIndex++ {
 		// Ignore empty comments
-		if lines[lineIndex] == "#" {
+		if strings.TrimSpace(lines[lineIndex]) == "#" || strings.Contains(lines[lineIndex], "# import") {
 			continue
 		}
 
@@ -97,8 +105,32 @@ func execBuildScript(script []byte, workDir string) error {
 }
 
 func genAsmIncludes(workDir string) ([]byte, error) {
-	headers, err := ioutil.ReadFile(fmt.Sprintf("%s/runtime/_obj/go_asm.h", workDir))
+	// Find all generated go_asm.h files and concat their conentents
-	if err != nil {
+	var (
+		allHeaders, headers []byte
+	)
+
+	if err := filepath.Walk(workDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if filepath.Base(path) != "go_asm.h" {
+			return nil
+		}
+
+		if headers, err = ioutil.ReadFile(path); err != nil {
+			return err
+		}
+
+		allHeaders = append(allHeaders, '\n')
+		allHeaders = append(allHeaders, headers...)
+		return nil
+	}); err != nil {
 		return nil, err
 	}
 
@@ -106,7 +138,7 @@ func genAsmIncludes(workDir string) ([]byte, error) {
 	includes = append(includes, "; vim: set ft=nasm :\n")
 	includes = append(includes, fmt.Sprintf("; generated by tools/offsets at %v\n", time.Now()))
 
-	for _, line := range strings.Split(string(headers), "\n") {
+	for _, line := range strings.Split(string(allHeaders), "\n") {
 		line = strings.TrimPrefix(line, "#define ")
 
 		// We are only interested in the offsets for the g, m and stack structures
@@ -131,6 +163,16 @@ func genAsmIncludes(workDir string) ([]byte, error) {
 		}
 	}
 
+	// In go 1.7.x, the page size is given by the _PageSize constant whereas in
+	// newer go versions it is specified at runtime and needs to be manually set
+	// by our asm bootstrap code.
+	if strings.Contains(runtime.Version(), "go1.7") {
+		includes = append(includes,
+			"; go 1.7 runtime uses a fixed 4k page size for our target arch so our bootstrap code does not need to do any extra work to set it up",
+			"%define SKIP_PAGESIZE_SETUP 1",
+		)
+	}
+
 	return []byte(strings.Join(includes, "\n")), nil
 }