Setup a more granular PDT based on the ELF image section addresses

src/gopheros/kernel/kmain/kmain.go

@@ -33,6 +33,7 @@ func Kmain(multibootInfoPtr, kernelStart, kernelEnd, kernelPageOffset uintptr) {
 	var err *kernel.Error
 	if err = allocator.Init(kernelStart, kernelEnd); err != nil {
 		panic(err)
+	} else if err = vmm.Init(kernelPageOffset); err != nil {
 		panic(err)
 	} else if err = goruntime.Init(); err != nil {
 		panic(err)

src/gopheros/kernel/mem/vmm/vmm.go

@@ -3,10 +3,12 @@ package vmm
 import (
 	"gopheros/kernel"
 	"gopheros/kernel/cpu"
+	"gopheros/kernel/hal/multiboot"
 	"gopheros/kernel/irq"
 	"gopheros/kernel/kfmt"
 	"gopheros/kernel/mem"
 	"gopheros/kernel/mem/pmm"
+	"unsafe"
 )
 
 var (
@@ -18,6 +20,8 @@ var (
 	// inlined by the compiler.
 	handleExceptionWithCodeFn = irq.HandleExceptionWithCode
 	readCR2Fn                 = cpu.ReadCR2
+	translateFn               = Translate
+	visitElfSectionsFn        = multiboot.VisitElfSections
 
 	errUnrecoverableFault = &kernel.Error{Module: "vmm", Message: "page/gpf fault"}
 )
@@ -142,9 +146,13 @@ func reserveZeroedFrame() *kernel.Error {
 	return nil
 }
 
-// Init initializes the vmm system and installs paging-related exception
+// Init initializes the vmm system, creates a granular PDT for the kernel and
-// handlers.
+// installs paging-related exception handlers.
-func Init() *kernel.Error {
+func Init(kernelPageOffset uintptr) *kernel.Error {
+	if err := setupPDTForKernel(kernelPageOffset); err != nil {
+		return err
+	}
+
 	if err := reserveZeroedFrame(); err != nil {
 		return err
 	}
@@ -153,3 +161,92 @@ func Init() *kernel.Error {
 	handleExceptionWithCodeFn(irq.GPFException, generalProtectionFaultHandler)
 	return nil
 }
+
+// setupPDTForKernel queries the multiboot package for the ELF sections that
+// correspond to the loaded kernel image and establishes a new granular PDT for
+// the kernel's VMA using the appropriate flags (e.g. NX for data sections, RW
+// for writable sections e.t.c).
+func setupPDTForKernel(kernelPageOffset uintptr) *kernel.Error {
+	var pdt PageDirectoryTable
+
+	// Allocate frame for the page directory and initialize it
+	pdtFrame, err := frameAllocator()
+	if err != nil {
+		return err
+	}
+
+	if err = pdt.Init(pdtFrame); err != nil {
+		return err
+	}
+
+	// Query the ELF sections of the kernel image and establish mappings
+	// for each one using the appropriate flags
+	pageSizeMinus1 := uint64(mem.PageSize - 1)
+	var visitor = func(_ string, secFlags multiboot.ElfSectionFlag, secAddress uintptr, secSize uint64) {
+		// Bail out if we have encountered an error; also ignore sections
+		// not using the kernel's VMA
+		if err != nil || secAddress < kernelPageOffset {
+			return
+		}
+
+		flags := FlagPresent
+
+		if (secFlags & multiboot.ElfSectionExecutable) == 0 {
+			flags |= FlagNoExecute
+		}
+
+		if (secFlags & multiboot.ElfSectionWritable) != 0 {
+			flags |= FlagRW
+		}
+
+		// We assume that all sections are page-aligned by the linker script
+		curPage := PageFromAddress(secAddress)
+		curFrame := pmm.Frame((secAddress - kernelPageOffset) >> mem.PageShift)
+		endFrame := curFrame + pmm.Frame(((secSize+pageSizeMinus1) & ^pageSizeMinus1)>>mem.PageShift)
+		for ; curFrame < endFrame; curFrame, curPage = curFrame+1, curPage+1 {
+			if err = pdt.Map(curPage, curFrame, flags); err != nil {
+				return
+			}
+		}
+	}
+
+	// Use the noescape hack to prevent the compiler from leaking the visitor
+	// function literal to the heap.
+	visitElfSectionsFn(
+		*(*multiboot.ElfSectionVisitor)(noEscape(unsafe.Pointer(&visitor))),
+	)
+
+	// If an error occurred while maping the ELF sections bail out
+	if err != nil {
+		return err
+	}
+
+	// Ensure that any pages mapped by the memory allocator using
+	// EarlyReserveRegion are copied to the new page directory.
+	for rsvAddr := earlyReserveLastUsed; rsvAddr < tempMappingAddr; rsvAddr += uintptr(mem.PageSize) {
+		page := PageFromAddress(rsvAddr)
+
+		frameAddr, err := translateFn(rsvAddr)
+		if err != nil {
+			return err
+		}
+
+		if err = pdt.Map(page, pmm.Frame(frameAddr>>mem.PageShift), FlagPresent|FlagRW); err != nil {
+			return err
+		}
+	}
+
+	// Activate the new PDT. After this point, the identify mapping for the
+	// physical memory addresses where the kernel is loaded becomes invalid.
+	pdt.Activate()
+
+	return nil
+}
+
+// noEscape hides a pointer from escape analysis. This function is copied over
+// from runtime/stubs.go
+//go:nosplit
+func noEscape(p unsafe.Pointer) unsafe.Pointer {
+	x := uintptr(p)
+	return unsafe.Pointer(x ^ 0)
+}

src/gopheros/kernel/mem/vmm/vmm_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"gopheros/kernel"
 	"gopheros/kernel/cpu"
+	"gopheros/kernel/hal/multiboot"
 	"gopheros/kernel/irq"
 	"gopheros/kernel/kfmt"
 	"gopheros/kernel/mem"
@@ -165,7 +166,7 @@ func TestNonRecoverablePageFault(t *testing.T) {
 	}
 }
 
-func TestGPtHandler(t *testing.T) {
+func TestGPFHandler(t *testing.T) {
 	defer func() {
 		readCR2Fn = cpu.ReadCR2
 	}()
@@ -191,6 +192,9 @@ func TestGPtHandler(t *testing.T) {
 func TestInit(t *testing.T) {
 	defer func() {
 		frameAllocator = nil
+		activePDTFn = cpu.ActivePDT
+		switchPDTFn = cpu.SwitchPDT
+		translateFn = Translate
 		mapTemporaryFn = MapTemporary
 		unmapFn = Unmap
 		handleExceptionWithCodeFn = irq.HandleExceptionWithCode
@@ -199,6 +203,8 @@ func TestInit(t *testing.T) {
 	// reserve space for an allocated page
 	reservedPage := make([]byte, mem.PageSize)
 
+	multiboot.SetInfoPtr(uintptr(unsafe.Pointer(&emptyInfoData[0])))
+
 	t.Run("success", func(t *testing.T) {
 		// fill page with junk
 		for i := 0; i < len(reservedPage); i++ {
@@ -209,11 +215,15 @@ func TestInit(t *testing.T) {
 			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
 			return pmm.Frame(addr >> mem.PageShift), nil
 		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		switchPDTFn = func(_ uintptr) {}
 		unmapFn = func(p Page) *kernel.Error { return nil }
 		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
 		handleExceptionWithCodeFn = func(_ irq.ExceptionNum, _ irq.ExceptionHandlerWithCode) {}
 
-		if err := Init(); err != nil {
+		if err := Init(0); err != nil {
 			t.Fatal(err)
 		}
 
@@ -225,15 +235,45 @@ func TestInit(t *testing.T) {
 		}
 	})
 
+	t.Run("setupPDT fails", func(t *testing.T) {
+		expErr := &kernel.Error{Module: "test", Message: "out of memory"}
+
+		// Allow the PDT allocation to succeed and then return an error when
+		// trying to allocate the blank fram
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			return pmm.InvalidFrame, expErr
+		})
+
+		if err := Init(0); err != expErr {
+			t.Fatalf("expected error: %v; got %v", expErr, err)
+		}
+	})
+
 	t.Run("blank page allocation error", func(t *testing.T) {
 		expErr := &kernel.Error{Module: "test", Message: "out of memory"}
 
-		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) { return pmm.InvalidFrame, expErr })
+		// Allow the PDT allocation to succeed and then return an error when
+		// trying to allocate the blank fram
+		var allocCount int
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			defer func() { allocCount++ }()
+
+			if allocCount == 0 {
+				addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+				return pmm.Frame(addr >> mem.PageShift), nil
+			}
+
+			return pmm.InvalidFrame, expErr
+		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		switchPDTFn = func(_ uintptr) {}
 		unmapFn = func(p Page) *kernel.Error { return nil }
 		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
 		handleExceptionWithCodeFn = func(_ irq.ExceptionNum, _ irq.ExceptionHandlerWithCode) {}
 
-		if err := Init(); err != expErr {
+		if err := Init(0); err != expErr {
 			t.Fatalf("expected error: %v; got %v", expErr, err)
 		}
 	})
@@ -245,12 +285,207 @@ func TestInit(t *testing.T) {
 			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
 			return pmm.Frame(addr >> mem.PageShift), nil
 		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		switchPDTFn = func(_ uintptr) {}
 		unmapFn = func(p Page) *kernel.Error { return nil }
 		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), expErr }
 		handleExceptionWithCodeFn = func(_ irq.ExceptionNum, _ irq.ExceptionHandlerWithCode) {}
 
-		if err := Init(); err != expErr {
+		if err := Init(0); err != expErr {
 			t.Fatalf("expected error: %v; got %v", expErr, err)
 		}
 	})
 }
+
+func TestSetupPDTForKernel(t *testing.T) {
+	defer func() {
+		frameAllocator = nil
+		activePDTFn = cpu.ActivePDT
+		switchPDTFn = cpu.SwitchPDT
+		translateFn = Translate
+		mapFn = Map
+		mapTemporaryFn = MapTemporary
+		unmapFn = Unmap
+		earlyReserveLastUsed = tempMappingAddr
+	}()
+
+	// reserve space for an allocated page
+	reservedPage := make([]byte, mem.PageSize)
+
+	multiboot.SetInfoPtr(uintptr(unsafe.Pointer(&emptyInfoData[0])))
+
+	t.Run("map kernel sections", func(t *testing.T) {
+		defer func() { visitElfSectionsFn = multiboot.VisitElfSections }()
+
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		switchPDTFn = func(_ uintptr) {}
+		translateFn = func(_ uintptr) (uintptr, *kernel.Error) { return 0xbadf00d000, nil }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
+		visitElfSectionsFn = func(v multiboot.ElfSectionVisitor) {
+			v(".debug", 0, 0, uint64(mem.PageSize>>1)) // address < VMA; should be ignored
+			v(".text", multiboot.ElfSectionExecutable, 0xbadc0ffee, uint64(mem.PageSize>>1))
+			v(".data", multiboot.ElfSectionWritable, 0xbadc0ffee, uint64(mem.PageSize))
+			v(".rodata", 0, 0xbadc0ffee, uint64(mem.PageSize<<1))
+		}
+		mapCount := 0
+		mapFn = func(page Page, frame pmm.Frame, flags PageTableEntryFlag) *kernel.Error {
+			defer func() { mapCount++ }()
+
+			var expFlags PageTableEntryFlag
+
+			switch mapCount {
+			case 0:
+				expFlags = FlagPresent
+			case 1:
+				expFlags = FlagPresent | FlagNoExecute | FlagRW
+			case 2, 3:
+				expFlags = FlagPresent | FlagNoExecute
+			}
+
+			if (flags & expFlags) != expFlags {
+				t.Errorf("[map call %d] expected flags to be %d; got %d", mapCount, expFlags, flags)
+			}
+
+			return nil
+		}
+
+		if err := setupPDTForKernel(0x123); err != nil {
+			t.Fatal(err)
+		}
+
+		if exp := 4; mapCount != exp {
+			t.Errorf("expected Map to be called %d times; got %d", exp, mapCount)
+		}
+	})
+
+	t.Run("map of kernel sections fials", func(t *testing.T) {
+		defer func() { visitElfSectionsFn = multiboot.VisitElfSections }()
+		expErr := &kernel.Error{Module: "test", Message: "map failed"}
+
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		switchPDTFn = func(_ uintptr) {}
+		translateFn = func(_ uintptr) (uintptr, *kernel.Error) { return 0xbadf00d000, nil }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
+		visitElfSectionsFn = func(v multiboot.ElfSectionVisitor) {
+			v(".text", multiboot.ElfSectionExecutable, 0xbadc0ffee, uint64(mem.PageSize>>1))
+		}
+		mapFn = func(page Page, frame pmm.Frame, flags PageTableEntryFlag) *kernel.Error {
+			return expErr
+		}
+
+		if err := setupPDTForKernel(0); err != expErr {
+			t.Fatalf("expected error: %v; got %v", expErr, err)
+		}
+	})
+
+	t.Run("copy allocator reservations to PDT", func(t *testing.T) {
+		earlyReserveLastUsed = tempMappingAddr - uintptr(mem.PageSize)
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		switchPDTFn = func(_ uintptr) {}
+		translateFn = func(_ uintptr) (uintptr, *kernel.Error) { return 0xbadf00d000, nil }
+		unmapFn = func(p Page) *kernel.Error { return nil }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
+		mapFn = func(page Page, frame pmm.Frame, flags PageTableEntryFlag) *kernel.Error {
+			if exp := PageFromAddress(earlyReserveLastUsed); page != exp {
+				t.Errorf("expected Map to be called with page %d; got %d", exp, page)
+			}
+
+			if exp := pmm.Frame(0xbadf00d000 >> mem.PageShift); frame != exp {
+				t.Errorf("expected Map to be called with frame %d; got %d", exp, frame)
+			}
+
+			if flags&(FlagPresent|FlagRW) != (FlagPresent | FlagRW) {
+				t.Error("expected Map to be called FlagPresent | FlagRW")
+			}
+			return nil
+		}
+
+		if err := setupPDTForKernel(0); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("pdt init fails", func(t *testing.T) {
+		expErr := &kernel.Error{Module: "test", Message: "translate failed"}
+
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		activePDTFn = func() uintptr { return 0 }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return 0, expErr }
+
+		if err := setupPDTForKernel(0); err != expErr {
+			t.Fatalf("expected error: %v; got %v", expErr, err)
+		}
+	})
+
+	t.Run("translation fails for page in reserved address space", func(t *testing.T) {
+		expErr := &kernel.Error{Module: "test", Message: "translate failed"}
+
+		earlyReserveLastUsed = tempMappingAddr - uintptr(mem.PageSize)
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		translateFn = func(_ uintptr) (uintptr, *kernel.Error) {
+			return 0, expErr
+		}
+
+		if err := setupPDTForKernel(0); err != expErr {
+			t.Fatalf("expected error: %v; got %v", expErr, err)
+		}
+	})
+
+	t.Run("map fails for page in reserved address space", func(t *testing.T) {
+		expErr := &kernel.Error{Module: "test", Message: "map failed"}
+
+		earlyReserveLastUsed = tempMappingAddr - uintptr(mem.PageSize)
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		activePDTFn = func() uintptr {
+			return uintptr(unsafe.Pointer(&reservedPage[0]))
+		}
+		translateFn = func(_ uintptr) (uintptr, *kernel.Error) { return 0xbadf00d000, nil }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
+		mapFn = func(page Page, frame pmm.Frame, flags PageTableEntryFlag) *kernel.Error { return expErr }
+
+		if err := setupPDTForKernel(0); err != expErr {
+			t.Fatalf("expected error: %v; got %v", expErr, err)
+		}
+	})
+}
+
+var (
+	emptyInfoData = []byte{
+		0, 0, 0, 0, // size
+		0, 0, 0, 0, // reserved
+		0, 0, 0, 0, // tag with type zero and length zero
+		0, 0, 0, 0,
+	}
+)