diff --git a/kernel/mem/vmm/constants_amd64.go b/kernel/mem/vmm/constants_amd64.go index 0f849b6..5e3b95d 100644 --- a/kernel/mem/vmm/constants_amd64.go +++ b/kernel/mem/vmm/constants_amd64.go @@ -12,6 +12,12 @@ const ( // address pointed to by a page table entry. For this particular architecture, // bits 12-51 contain the physical memory address. ptePhysPageMask = uintptr(0x000ffffffffff000) + + // tempMappingAddr is a reserved virtual page address used for + // temporary physical page mappings (e.g. when mapping inactive PDT + // pages). For amd64 this address uses the following table indices: + // 510, 511, 511, 511. + tempMappingAddr = uintptr(0Xffffff7ffffff000) ) var ( diff --git a/kernel/mem/vmm/map.go b/kernel/mem/vmm/map.go new file mode 100644 index 0000000..bd8781c --- /dev/null +++ b/kernel/mem/vmm/map.go @@ -0,0 +1,117 @@ +package vmm + +import ( + "unsafe" + + "github.com/achilleasa/gopher-os/kernel" + "github.com/achilleasa/gopher-os/kernel/mem" + "github.com/achilleasa/gopher-os/kernel/mem/pmm" +) + +var ( + // nextAddrFn is used by used by tests to override the nextTableAddr + // calculations used by Map. When compiling the kernel this function + // will be automatically inlined. + nextAddrFn = func(entryAddr uintptr) uintptr { + return entryAddr + } + + // flushTLBEntryFn is used by tests to override calls to flushTLBEntry + // which will cause a fault if called in user-mode. + flushTLBEntryFn = flushTLBEntry + + errNoHugePageSupport = &kernel.Error{Module: "vmm", Message: "huge pages are not supported"} +) + +// FrameAllocator is a function that can allocate physical frames of the specified order. +type FrameAllocator func(mem.PageOrder) (pmm.Frame, *kernel.Error) + +// Map establishes a mapping between a virtual page and a physical memory frame +// using the currently active page directory table. Calls to Map will use the +// supplied physical frame allocator to initialize missing page tables at each +// paging level supported by the MMU. +func Map(page Page, frame pmm.Frame, flags PageTableEntryFlag, allocFn FrameAllocator) *kernel.Error { + var err *kernel.Error + + walk(page.Address(), func(pteLevel uint8, pte *pageTableEntry) bool { + // If we reached the last level all we need to do is to map the + // frame in place and flag it as present and flush its TLB entry + if pteLevel == pageLevels-1 { + *pte = 0 + pte.SetFrame(frame) + pte.SetFlags(FlagPresent | flags) + flushTLBEntryFn(page.Address()) + return true + } + + if pte.HasFlags(FlagHugePage) { + err = errNoHugePageSupport + return false + } + + // Next table does not yet exist; we need to allocate a + // physical frame for it map it and clear its contents. + if !pte.HasFlags(FlagPresent) { + var newTableFrame pmm.Frame + newTableFrame, err = allocFn(mem.PageOrder(0)) + if err != nil { + return false + } + + *pte = 0 + pte.SetFrame(newTableFrame) + pte.SetFlags(FlagPresent | FlagRW) + + // The next pte entry becomes available but we need to + // make sure that the new page is properly cleared + nextTableAddr := (uintptr(unsafe.Pointer(pte)) << pageLevelBits[pteLevel+1]) + mem.Memset(nextAddrFn(nextTableAddr), 0, mem.PageSize) + } + + return true + }) + + return err +} + +// MapTemporary establishes a temporary RW mapping of a physical memory frame +// to a fixed virtual address overwriting any previous mapping. The temporary +// mapping mechanism is primarily used by the kernel to access and initialize +// inactive page tables. +func MapTemporary(frame pmm.Frame, allocFn FrameAllocator) (Page, *kernel.Error) { + if err := Map(PageFromAddress(tempMappingAddr), frame, FlagRW, allocFn); err != nil { + return 0, err + } + + return PageFromAddress(tempMappingAddr), nil +} + +// Unmap removes a mapping previously installed via a call to Map or MapTemporary. +func Unmap(page Page) *kernel.Error { + var err *kernel.Error + + walk(page.Address(), func(pteLevel uint8, pte *pageTableEntry) bool { + // If we reached the last level all we need to do is to set the + // page as non-present and flush its TLB entry + if pteLevel == pageLevels-1 { + pte.ClearFlags(FlagPresent) + flushTLBEntryFn(page.Address()) + return true + } + + // Next table is not present; this is an invalid mapping + if !pte.HasFlags(FlagPresent) { + err = ErrInvalidMapping + return false + } + + if pte.HasFlags(FlagHugePage) { + err = errNoHugePageSupport + return false + } + + return true + }) + + return err +} diff --git a/kernel/mem/vmm/map_test.go b/kernel/mem/vmm/map_test.go new file mode 100644 index 0000000..ee03a3e --- /dev/null +++ b/kernel/mem/vmm/map_test.go @@ -0,0 +1,251 @@ +package vmm + +import ( + "runtime" + "testing" + "unsafe" + + "github.com/achilleasa/gopher-os/kernel" + "github.com/achilleasa/gopher-os/kernel/mem" + "github.com/achilleasa/gopher-os/kernel/mem/pmm" +) + +func TestNextAddrFn(t *testing.T) { + // Dummy test to keep coverage happy + if exp, got := uintptr(123), nextAddrFn(uintptr(123)); exp != got { + t.Fatalf("expected nextAddrFn to return %v; got %v", exp, got) + } +} + +func TestMapTemporaryAmd64(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("test requires amd64 runtime; skipping") + } + + defer func(origPtePtr func(uintptr) unsafe.Pointer, origNextAddrFn func(uintptr) uintptr, origFlushTLBEntryFn func(uintptr)) { + ptePtrFn = origPtePtr + nextAddrFn = origNextAddrFn + flushTLBEntryFn = origFlushTLBEntryFn + }(ptePtrFn, nextAddrFn, flushTLBEntryFn) + + var physPages [pageLevels][mem.PageSize >> mem.PointerShift]pageTableEntry + nextPhysPage := 0 + + // allocFn returns pages from index 1; we keep index 0 for the P4 entry + allocFn := func(_ mem.PageOrder) (pmm.Frame, *kernel.Error) { + nextPhysPage++ + pageAddr := unsafe.Pointer(&physPages[nextPhysPage][0]) + return pmm.Frame(uintptr(pageAddr) >> mem.PageShift), nil + } + + pteCallCount := 0 + ptePtrFn = func(entry uintptr) unsafe.Pointer { + pteCallCount++ + // The last 12 bits encode the page table offset in bytes + // which we need to convert to a uint64 entry + pteIndex := (entry & uintptr(mem.PageSize-1)) >> mem.PointerShift + return unsafe.Pointer(&physPages[pteCallCount-1][pteIndex]) + } + + nextAddrFn = func(entry uintptr) uintptr { + return uintptr(unsafe.Pointer(&physPages[nextPhysPage][0])) + } + + flushTLBEntryCallCount := 0 + flushTLBEntryFn = func(uintptr) { + flushTLBEntryCallCount++ + } + + // The temporary mappin address breaks down to: + // p4 index: 510 + // p3 index: 511 + // p2 index: 511 + // p1 index: 511 + frame := pmm.Frame(123) + levelIndices := []uint{510, 511, 511, 511} + + page, err := MapTemporary(frame, allocFn) + if err != nil { + t.Fatal(err) + } + + if got := page.Address(); got != tempMappingAddr { + t.Fatalf("expected temp mapping virtual address to be %x; got %x", tempMappingAddr, got) + } + + for level, physPage := range physPages { + pte := physPage[levelIndices[level]] + if !pte.HasFlags(FlagPresent | FlagRW) { + t.Errorf("[pte at level %d] expected entry to have FlagPresent and FlagRW set", level) + } + + switch { + case level < pageLevels-1: + if exp, got := pmm.Frame(uintptr(unsafe.Pointer(&physPages[level+1][0]))>>mem.PageShift), pte.Frame(); got != exp { + t.Errorf("[pte at level %d] expected entry frame to be %d; got %d", level, exp, got) + } + default: + // The last pte entry should point to frame + if got := pte.Frame(); got != frame { + t.Errorf("[pte at level %d] expected entry frame to be %d; got %d", level, frame, got) + } + } + } + + if exp := 1; flushTLBEntryCallCount != exp { + t.Errorf("expected flushTLBEntry to be called %d times; got %d", exp, flushTLBEntryCallCount) + } +} + +func TestMapTemporaryErrorsAmd64(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("test requires amd64 runtime; skipping") + } + + defer func(origPtePtr func(uintptr) unsafe.Pointer, origNextAddrFn func(uintptr) uintptr, origFlushTLBEntryFn func(uintptr)) { + ptePtrFn = origPtePtr + nextAddrFn = origNextAddrFn + flushTLBEntryFn = origFlushTLBEntryFn + }(ptePtrFn, nextAddrFn, flushTLBEntryFn) + + var physPages [pageLevels][mem.PageSize >> mem.PointerShift]pageTableEntry + + // The reserved virt address uses the following page level indices: 510, 511, 511, 511 + p4Index := 510 + frame := pmm.Frame(123) + + t.Run("encounter huge page", func(t *testing.T) { + physPages[0][p4Index].SetFlags(FlagPresent | FlagHugePage) + + ptePtrFn = func(entry uintptr) unsafe.Pointer { + // The last 12 bits encode the page table offset in bytes + // which we need to convert to a uint64 entry + pteIndex := (entry & uintptr(mem.PageSize-1)) >> mem.PointerShift + return unsafe.Pointer(&physPages[0][pteIndex]) + } + + if _, err := MapTemporary(frame, nil); err != errNoHugePageSupport { + t.Fatalf("expected to get errNoHugePageSupport; got %v", err) + } + }) + + t.Run("allocFn returns an error", func(t *testing.T) { + physPages[0][p4Index] = 0 + + expErr := &kernel.Error{Module: "test", Message: "out of memory"} + + allocFn := func(_ mem.PageOrder) (pmm.Frame, *kernel.Error) { + return 0, expErr + } + + if _, err := MapTemporary(frame, allocFn); err != expErr { + t.Fatalf("got unexpected error %v", err) + } + }) +} + +func TestUnmapAmd64(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("test requires amd64 runtime; skipping") + } + + defer func(origPtePtr func(uintptr) unsafe.Pointer, origFlushTLBEntryFn func(uintptr)) { + ptePtrFn = origPtePtr + flushTLBEntryFn = origFlushTLBEntryFn + }(ptePtrFn, flushTLBEntryFn) + + var ( + physPages [pageLevels][mem.PageSize >> mem.PointerShift]pageTableEntry + frame = pmm.Frame(123) + ) + + // Emulate a page mapped to virtAddr 0 across all page levels + for level := 0; level < pageLevels; level++ { + physPages[level][0].SetFlags(FlagPresent | FlagRW) + if level < pageLevels-1 { + physPages[level][0].SetFrame(pmm.Frame(uintptr(unsafe.Pointer(&physPages[level+1][0])) >> mem.PageShift)) + } else { + physPages[level][0].SetFrame(frame) + + } + } + + pteCallCount := 0 + ptePtrFn = func(entry uintptr) unsafe.Pointer { + pteCallCount++ + return unsafe.Pointer(&physPages[pteCallCount-1][0]) + } + + flushTLBEntryCallCount := 0 + flushTLBEntryFn = func(uintptr) { + flushTLBEntryCallCount++ + } + + if err := Unmap(PageFromAddress(0)); err != nil { + t.Fatal(err) + } + + for level, physPage := range physPages { + pte := physPage[0] + + switch { + case level < pageLevels-1: + if !pte.HasFlags(FlagPresent) { + t.Errorf("[pte at level %d] expected entry to retain have FlagPresent set", level) + } + if exp, got := pmm.Frame(uintptr(unsafe.Pointer(&physPages[level+1][0]))>>mem.PageShift), pte.Frame(); got != exp { + t.Errorf("[pte at level %d] expected entry frame to still be %d; got %d", level, exp, got) + } + default: + if pte.HasFlags(FlagPresent) { + t.Errorf("[pte at level %d] expected entry not to have FlagPresent set", level) + } + + // The last pte entry should still point to frame + if got := pte.Frame(); got != frame { + t.Errorf("[pte at level %d] expected entry frame to be %d; got %d", level, frame, got) + } + } + } + + if exp := 1; flushTLBEntryCallCount != exp { + t.Errorf("expected flushTLBEntry to be called %d times; got %d", exp, flushTLBEntryCallCount) + } +} + +func TestUnmapErrorsAmd64(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("test requires amd64 runtime; skipping") + } + + defer func(origPtePtr func(uintptr) unsafe.Pointer, origNextAddrFn func(uintptr) uintptr, origFlushTLBEntryFn func(uintptr)) { + ptePtrFn = origPtePtr + nextAddrFn = origNextAddrFn + flushTLBEntryFn = origFlushTLBEntryFn + }(ptePtrFn, nextAddrFn, flushTLBEntryFn) + + var physPages [pageLevels][mem.PageSize >> mem.PointerShift]pageTableEntry + + t.Run("encounter huge page", func(t *testing.T) { + physPages[0][0].SetFlags(FlagPresent | FlagHugePage) + + ptePtrFn = func(entry uintptr) unsafe.Pointer { + // The last 12 bits encode the page table offset in bytes + // which we need to convert to a uint64 entry + pteIndex := (entry & uintptr(mem.PageSize-1)) >> mem.PointerShift + return unsafe.Pointer(&physPages[0][pteIndex]) + } + + if err := Unmap(PageFromAddress(0)); err != errNoHugePageSupport { + t.Fatalf("expected to get errNoHugePageSupport; got %v", err) + } + }) + + t.Run("virtual address not mapped", func(t *testing.T) { + physPages[0][0].ClearFlags(FlagPresent) + + if err := Unmap(PageFromAddress(0)); err != ErrInvalidMapping { + t.Fatalf("expected to get ErrInvalidMapping; got %v", err) + } + }) +} diff --git a/kernel/mem/vmm/tlb.go b/kernel/mem/vmm/tlb.go new file mode 100644 index 0000000..703e597 --- /dev/null +++ b/kernel/mem/vmm/tlb.go @@ -0,0 +1,11 @@ +package vmm + +// flushTLBEntry flushes a TLB entry for a particular virtual address. +func flushTLBEntry(virtAddr uintptr) + +// switchPDT sets the root page table directory to point to the specified +// physical address and flushes the TLB. +func switchPDT(pdtPhysAddr uintptr) + +// activePDT returns the physical address of the currently active page table. +func activePDT() uintptr diff --git a/kernel/mem/vmm/tlb_amd64.s b/kernel/mem/vmm/tlb_amd64.s new file mode 100644 index 0000000..0f12a43 --- /dev/null +++ b/kernel/mem/vmm/tlb_amd64.s @@ -0,0 +1,15 @@ + #include "textflag.h" + +TEXT ·flushTLBEntry(SB),NOSPLIT,$0 + INVLPG virtAddr+0(FP) + RET + +TEXT ·switchPDT(SB),NOSPLIT,$0 + // loading CR3 also triggers a TLB flush + MOVQ pdtPhysAddr+0(FP), CR3 + RET + +TEXT ·activePDT(SB),NOSPLIT,$0 + MOVQ CR3, AX + MOVQ AX, ret+0(FP) + RET