Merge pull request #30 from achilleasa/implement-copy-on-write-pages

Implement copy on write pages

arch/x86_64/asm/rt0_32.s

@@ -304,9 +304,9 @@ _rt0_enter_long_mode:
 	or eax, (1 << 8) | (1<<11)
 	wrmsr
 
-	; Finally enable paging
+	; Finally enable paging (bit 31) and user/kernel page write protection (bit 16)
 	mov eax, cr0
-	or eax, 1 << 31
+	or eax, (1 << 31) | (1<<16)
 	mov cr0, eax
 
 	; We are in 32-bit compatibility submode. We need to load a 64bit GDT 

kernel/mem/mem.go

@@ -27,3 +27,23 @@ func Memset(addr uintptr, value byte, size Size) {
 		copy(target[index:], target[:index])
 	}
 }
+
+// Memcopy copies size bytes from src to dst.
+func Memcopy(src, dst uintptr, size Size) {
+	if size == 0 {
+		return
+	}
+
+	srcSlice := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
+		Len:  int(size),
+		Cap:  int(size),
+		Data: src,
+	}))
+	dstSlice := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
+		Len:  int(size),
+		Cap:  int(size),
+		Data: dst,
+	}))
+
+	copy(dstSlice, srcSlice)
+}

kernel/mem/mem_test.go

@@ -25,3 +25,28 @@ func TestMemset(t *testing.T) {
 		}
 	}
 }
+
+func TestMemcopy(t *testing.T) {
+	// memcopy with a 0 size should be a no-op
+	Memcopy(uintptr(0), uintptr(0), 0)
+
+	var (
+		src = make([]byte, PageSize)
+		dst = make([]byte, PageSize)
+	)
+	for i := 0; i < len(src); i++ {
+		src[i] = byte(i % 256)
+	}
+
+	Memcopy(
+		uintptr(unsafe.Pointer(&src[0])),
+		uintptr(unsafe.Pointer(&dst[0])),
+		PageSize,
+	)
+
+	for i := 0; i < len(src); i++ {
+		if got := dst[i]; got != src[i] {
+			t.Errorf("value mismatch between src and dst at index %d", i)
+		}
+	}
+}

kernel/mem/vmm/constants_amd64.go

@@ -80,6 +80,10 @@ const (
 	// for this page when the swapping page tables by updating the CR3 register.
 	FlagGlobal
 
+	// FlagCopyOnWrite is used to implement copy-on-write functionality. This
+	// flag and FlagRW are mutually exclusive.
+	FlagCopyOnWrite = 1 << 9
+
 	// FlagNoExecute if set, indicates that a page contains non-executable code.
 	FlagNoExecute = 1 << 63
 )

kernel/mem/vmm/map.go

@@ -9,7 +9,34 @@ import (
 	"github.com/achilleasa/gopher-os/kernel/mem/pmm"
 )
 
+// ReservedZeroedFrame is a special zero-cleared frame allocated by the
+// vmm package's Init function. The purpose of this frame is to assist
+// in implementing on-demand memory allocation when mapping it in
+// conjunction with the CopyOnWrite flag. Here is an example of how it
+// can be used:
+//
+//  func ReserveOnDemand(start vmm.Page, pageCount int) *kernel.Error {
+//    var err *kernel.Error
+//    mapFlags := vmm.FlagPresent|vmm.FlagCopyOnWrite
+//    for page := start; pageCount > 0; pageCount, page = pageCount-1, page+1 {
+//       if err = vmm.Map(page, vmm.ReservedZeroedFrame, mapFlags); err != nil {
+//         return err
+//       }
+//    }
+//    return nil
+//  }
+//
+// In the above example, page mappings are set up for the requested number of
+// pages but no physical memory is reserved for their contents. A write to any
+// of the above pages will trigger a page-fault causing a new frame to be
+// allocated, cleared (the blank frame is copied to the new frame) and
+// installed in-place with RW permissions.
+var ReservedZeroedFrame pmm.Frame
+
 var (
+	// protectReservedZeroedPage is set to true to prevent mapping to
+	protectReservedZeroedPage bool
+
 	// nextAddrFn is used by used by tests to override the nextTableAddr
 	// calculations used by Map. When compiling the kernel this function
 	// will be automatically inlined.
@@ -21,14 +48,21 @@ var (
 	// which will cause a fault if called in user-mode.
 	flushTLBEntryFn = cpu.FlushTLBEntry
 
-	errNoHugePageSupport = &kernel.Error{Module: "vmm", Message: "huge pages are not supported"}
+	errNoHugePageSupport           = &kernel.Error{Module: "vmm", Message: "huge pages are not supported"}
+	errAttemptToRWMapReservedFrame = &kernel.Error{Module: "vmm", Message: "reserved blank frame cannot be mapped with a RW flag"}
 )
 
 // Map establishes a mapping between a virtual page and a physical memory frame
 // using the currently active page directory table. Calls to Map will use the
 // supplied physical frame allocator to initialize missing page tables at each
 // paging level supported by the MMU.
+//
+// Attempts to map ReservedZeroedFrame with a RW flag will result in an error.
 func Map(page Page, frame pmm.Frame, flags PageTableEntryFlag) *kernel.Error {
+	if protectReservedZeroedPage && frame == ReservedZeroedFrame && (flags&FlagRW) != 0 {
+		return errAttemptToRWMapReservedFrame
+	}
+
 	var err *kernel.Error
 
 	walk(page.Address(), func(pteLevel uint8, pte *pageTableEntry) bool {
@@ -37,7 +71,7 @@ func Map(page Page, frame pmm.Frame, flags PageTableEntryFlag) *kernel.Error {
 		if pteLevel == pageLevels-1 {
 			*pte = 0
 			pte.SetFrame(frame)
-			pte.SetFlags(FlagPresent | flags)
+			pte.SetFlags(flags)
 			flushTLBEntryFn(page.Address())
 			return true
 		}
@@ -76,8 +110,14 @@ func Map(page Page, frame pmm.Frame, flags PageTableEntryFlag) *kernel.Error {
 // to a fixed virtual address overwriting any previous mapping. The temporary
 // mapping mechanism is primarily used by the kernel to access and initialize
 // inactive page tables.
+//
+// Attempts to map ReservedZeroedFrame will result in an error.
 func MapTemporary(frame pmm.Frame) (Page, *kernel.Error) {
-	if err := Map(PageFromAddress(tempMappingAddr), frame, FlagRW); err != nil {
+	if protectReservedZeroedPage && frame == ReservedZeroedFrame {
+		return 0, errAttemptToRWMapReservedFrame
+	}
+
+	if err := Map(PageFromAddress(tempMappingAddr), frame, FlagPresent|FlagRW); err != nil {
 		return 0, err
 	}
 

kernel/mem/vmm/map_test.go

@@ -144,6 +144,24 @@ func TestMapTemporaryErrorsAmd64(t *testing.T) {
 			t.Fatalf("got unexpected error %v", err)
 		}
 	})
+
+	t.Run("map BlankReservedFrame RW", func(t *testing.T) {
+		defer func() { protectReservedZeroedPage = false }()
+
+		protectReservedZeroedPage = true
+		if err := Map(Page(0), ReservedZeroedFrame, FlagRW); err != errAttemptToRWMapReservedFrame {
+			t.Fatalf("expected errAttemptToRWMapReservedFrame; got: %v", err)
+		}
+	})
+
+	t.Run("temp-map BlankReservedFrame RW", func(t *testing.T) {
+		defer func() { protectReservedZeroedPage = false }()
+
+		protectReservedZeroedPage = true
+		if _, err := MapTemporary(ReservedZeroedFrame); err != errAttemptToRWMapReservedFrame {
+			t.Fatalf("expected errAttemptToRWMapReservedFrame; got: %v", err)
+		}
+	})
 }
 
 func TestUnmapAmd64(t *testing.T) {

kernel/mem/vmm/vmm.go

@@ -5,6 +5,7 @@ import (
 	"github.com/achilleasa/gopher-os/kernel/cpu"
 	"github.com/achilleasa/gopher-os/kernel/irq"
 	"github.com/achilleasa/gopher-os/kernel/kfmt/early"
+	"github.com/achilleasa/gopher-os/kernel/mem"
 	"github.com/achilleasa/gopher-os/kernel/mem/pmm"
 )
 
@@ -30,7 +31,58 @@ func SetFrameAllocator(allocFn FrameAllocatorFn) {
 }
 
 func pageFaultHandler(errorCode uint64, frame *irq.Frame, regs *irq.Regs) {
-	early.Printf("\nPage fault while accessing address: 0x%16x\nReason: ", readCR2Fn())
+	var (
+		faultAddress = uintptr(readCR2Fn())
+		faultPage    = PageFromAddress(faultAddress)
+		pageEntry    *pageTableEntry
+	)
+
+	// Lookup entry for the page where the fault occurred
+	walk(faultPage.Address(), func(pteLevel uint8, pte *pageTableEntry) bool {
+		nextIsPresent := pte.HasFlags(FlagPresent)
+
+		if pteLevel == pageLevels-1 && nextIsPresent {
+			pageEntry = pte
+		}
+
+		// Abort walk if the next page table entry is missing
+		return nextIsPresent
+	})
+
+	// CoW is supported for RO pages with the CoW flag set
+	if pageEntry != nil && !pageEntry.HasFlags(FlagRW) && pageEntry.HasFlags(FlagCopyOnWrite) {
+		var (
+			copy    pmm.Frame
+			tmpPage Page
+			err     *kernel.Error
+		)
+
+		if copy, err = frameAllocator(); err != nil {
+			nonRecoverablePageFault(faultAddress, errorCode, frame, regs, err)
+		} else if tmpPage, err = mapTemporaryFn(copy); err != nil {
+			nonRecoverablePageFault(faultAddress, errorCode, frame, regs, err)
+		} else {
+			// Copy page contents, mark as RW and remove CoW flag
+			mem.Memcopy(faultPage.Address(), tmpPage.Address(), mem.PageSize)
+			unmapFn(tmpPage)
+
+			// Update mapping to point to the new frame, flag it as RW and
+			// remove the CoW flag
+			pageEntry.ClearFlags(FlagCopyOnWrite)
+			pageEntry.SetFlags(FlagPresent | FlagRW)
+			pageEntry.SetFrame(copy)
+			flushTLBEntryFn(faultPage.Address())
+
+			// Fault recovered; retry the instruction that caused the fault
+			return
+		}
+	}
+
+	nonRecoverablePageFault(faultAddress, errorCode, frame, regs, nil)
+}
+
+func nonRecoverablePageFault(faultAddress uintptr, errorCode uint64, frame *irq.Frame, regs *irq.Regs, err *kernel.Error) {
+	early.Printf("\nPage fault while accessing address: 0x%16x\nReason: ", faultAddress)
 	switch {
 	case errorCode == 0:
 		early.Printf("read from non-present page")
@@ -55,7 +107,7 @@ func pageFaultHandler(errorCode uint64, frame *irq.Frame, regs *irq.Regs) {
 	frame.Print()
 
 	// TODO: Revisit this when user-mode tasks are implemented
-	panicFn(nil)
+	panicFn(err)
 }
 
 func generalProtectionFaultHandler(_ uint64, frame *irq.Frame, regs *irq.Regs) {
@@ -68,11 +120,35 @@ func generalProtectionFaultHandler(_ uint64, frame *irq.Frame, regs *irq.Regs) {
 	panicFn(nil)
 }
 
+// reserveZeroedFrame reserves a physical frame to be used together with
+// FlagCopyOnWrite for lazy allocation requests.
+func reserveZeroedFrame() *kernel.Error {
+	var (
+		err      *kernel.Error
+		tempPage Page
+	)
+
+	if ReservedZeroedFrame, err = frameAllocator(); err != nil {
+		return err
+	} else if tempPage, err = mapTemporaryFn(ReservedZeroedFrame); err != nil {
+		return err
+	}
+	mem.Memset(tempPage.Address(), 0, mem.PageSize)
+	unmapFn(tempPage)
+
+	// From this point on, ReservedZeroedFrame cannot be mapped with a RW flag
+	protectReservedZeroedPage = true
+	return nil
+}
+
 // Init initializes the vmm system and installs paging-related exception
 // handlers.
 func Init() *kernel.Error {
+	if err := reserveZeroedFrame(); err != nil {
+		return err
+	}
+
 	handleExceptionWithCodeFn(irq.PageFaultException, pageFaultHandler)
 	handleExceptionWithCodeFn(irq.GPFException, generalProtectionFaultHandler)
-
 	return nil
 }

kernel/mem/vmm/vmm_test.go

@@ -11,12 +11,98 @@ import (
 	"github.com/achilleasa/gopher-os/kernel/driver/video/console"
 	"github.com/achilleasa/gopher-os/kernel/hal"
 	"github.com/achilleasa/gopher-os/kernel/irq"
+	"github.com/achilleasa/gopher-os/kernel/mem"
+	"github.com/achilleasa/gopher-os/kernel/mem/pmm"
 )
 
-func TestPageFaultHandler(t *testing.T) {
+func TestRecoverablePageFault(t *testing.T) {
-	defer func() {
+	var (
+		frame       irq.Frame
+		regs        irq.Regs
+		panicCalled bool
+		pageEntry   pageTableEntry
+		origPage    = make([]byte, mem.PageSize)
+		clonedPage  = make([]byte, mem.PageSize)
+		err         = &kernel.Error{Module: "test", Message: "something went wrong"}
+	)
+
+	defer func(origPtePtr func(uintptr) unsafe.Pointer) {
+		ptePtrFn = origPtePtr
 		panicFn = kernel.Panic
 		readCR2Fn = cpu.ReadCR2
+		frameAllocator = nil
+		mapTemporaryFn = MapTemporary
+		unmapFn = Unmap
+		flushTLBEntryFn = cpu.FlushTLBEntry
+	}(ptePtrFn)
+
+	specs := []struct {
+		pteFlags   PageTableEntryFlag
+		allocError *kernel.Error
+		mapError   *kernel.Error
+		expPanic   bool
+	}{
+		// Missing pge
+		{0, nil, nil, true},
+		// Page is present but CoW flag not set
+		{FlagPresent, nil, nil, true},
+		// Page is present but both CoW and RW flags set
+		{FlagPresent | FlagRW | FlagCopyOnWrite, nil, nil, true},
+		// Page is present with CoW flag set but allocating a page copy fails
+		{FlagPresent | FlagCopyOnWrite, err, nil, true},
+		// Page is present with CoW flag set but mapping the page copy fails
+		{FlagPresent | FlagCopyOnWrite, nil, err, true},
+		// Page is present with CoW flag set
+		{FlagPresent | FlagCopyOnWrite, nil, nil, false},
+	}
+
+	mockTTY()
+
+	panicFn = func(_ *kernel.Error) {
+		panicCalled = true
+	}
+
+	ptePtrFn = func(entry uintptr) unsafe.Pointer { return unsafe.Pointer(&pageEntry) }
+	readCR2Fn = func() uint64 { return uint64(uintptr(unsafe.Pointer(&origPage[0]))) }
+	unmapFn = func(_ Page) *kernel.Error { return nil }
+	flushTLBEntryFn = func(_ uintptr) {}
+
+	for specIndex, spec := range specs {
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), spec.mapError }
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&clonedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), spec.allocError
+		})
+
+		for i := 0; i < len(origPage); i++ {
+			origPage[i] = byte(i % 256)
+			clonedPage[i] = 0
+		}
+
+		panicCalled = false
+		pageEntry = 0
+		pageEntry.SetFlags(spec.pteFlags)
+
+		pageFaultHandler(2, &frame, &regs)
+
+		if spec.expPanic != panicCalled {
+			t.Errorf("[spec %d] expected panic %t; got %t", specIndex, spec.expPanic, panicCalled)
+		}
+
+		if !spec.expPanic {
+			for i := 0; i < len(origPage); i++ {
+				if origPage[i] != clonedPage[i] {
+					t.Errorf("[spec %d] expected clone page to be a copy of the original page; mismatch at index %d", specIndex, i)
+				}
+			}
+		}
+	}
+
+}
+
+func TestNonRecoverablePageFault(t *testing.T) {
+	defer func() {
+		panicFn = kernel.Panic
 	}()
 
 	specs := []struct {
@@ -71,10 +157,6 @@ func TestPageFaultHandler(t *testing.T) {
 		frame irq.Frame
 	)
 
-	readCR2Fn = func() uint64 {
-		return 0xbadf00d000
-	}
-
 	panicCalled := false
 	panicFn = func(_ *kernel.Error) {
 		panicCalled = true
@@ -84,7 +166,7 @@ func TestPageFaultHandler(t *testing.T) {
 		fb := mockTTY()
 		panicCalled = false
 
-		pageFaultHandler(spec.errCode, &frame, &regs)
+		nonRecoverablePageFault(0xbadf00d000, spec.errCode, &frame, &regs, nil)
 		if got := readTTY(fb); !strings.Contains(got, spec.expReason) {
 			t.Errorf("[spec %d] expected reason %q; got output:\n%q", specIndex, spec.expReason, got)
 			continue
@@ -131,14 +213,69 @@ func TestGPtHandler(t *testing.T) {
 
 func TestInit(t *testing.T) {
 	defer func() {
+		frameAllocator = nil
+		mapTemporaryFn = MapTemporary
+		unmapFn = Unmap
 		handleExceptionWithCodeFn = irq.HandleExceptionWithCode
 	}()
 
-	handleExceptionWithCodeFn = func(_ irq.ExceptionNum, _ irq.ExceptionHandlerWithCode) {}
+	// reserve space for an allocated page
+	reservedPage := make([]byte, mem.PageSize)
 
-	if err := Init(); err != nil {
+	t.Run("success", func(t *testing.T) {
-		t.Fatal(err)
+		// fill page with junk
-	}
+		for i := 0; i < len(reservedPage); i++ {
+			reservedPage[i] = byte(i % 256)
+		}
+
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		unmapFn = func(p Page) *kernel.Error { return nil }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
+		handleExceptionWithCodeFn = func(_ irq.ExceptionNum, _ irq.ExceptionHandlerWithCode) {}
+
+		if err := Init(); err != nil {
+			t.Fatal(err)
+		}
+
+		// reserved page should be zeroed
+		for i := 0; i < len(reservedPage); i++ {
+			if reservedPage[i] != 0 {
+				t.Errorf("expected reserved page to be zeroed; got byte %d at index %d", reservedPage[i], i)
+			}
+		}
+	})
+
+	t.Run("blank page allocation error", func(t *testing.T) {
+		expErr := &kernel.Error{Module: "test", Message: "out of memory"}
+
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) { return pmm.InvalidFrame, expErr })
+		unmapFn = func(p Page) *kernel.Error { return nil }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), nil }
+		handleExceptionWithCodeFn = func(_ irq.ExceptionNum, _ irq.ExceptionHandlerWithCode) {}
+
+		if err := Init(); err != expErr {
+			t.Fatalf("expected error: %v; got %v", expErr, err)
+		}
+	})
+
+	t.Run("blank page mapping error", func(t *testing.T) {
+		expErr := &kernel.Error{Module: "test", Message: "map failed"}
+
+		SetFrameAllocator(func() (pmm.Frame, *kernel.Error) {
+			addr := uintptr(unsafe.Pointer(&reservedPage[0]))
+			return pmm.Frame(addr >> mem.PageShift), nil
+		})
+		unmapFn = func(p Page) *kernel.Error { return nil }
+		mapTemporaryFn = func(f pmm.Frame) (Page, *kernel.Error) { return Page(f), expErr }
+		handleExceptionWithCodeFn = func(_ irq.ExceptionNum, _ irq.ExceptionHandlerWithCode) {}
+
+		if err := Init(); err != expErr {
+			t.Fatalf("expected error: %v; got %v", expErr, err)
+		}
+	})
 }
 
 func readTTY(fb []byte) string {