Implement better vunmap() functionality; Enable NULL address (0 MFN) handling

arch/x86/pagetables.c

@@ -23,6 +23,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <console.h>
+#include <errno.h>
 #include <ktf.h>
 #include <multiboot2.h>
 #include <page.h>
@@ -196,16 +197,22 @@ void dump_user_pagetable_va(void *va) {
     dump_pagetable_va(&user_cr3, va);
 }
 
+static inline void clean_pagetable(void *tab) {
+    for (pgentry_t *e = tab; e < (pgentry_t *) (tab + PAGE_SIZE); e++)
+        set_pgentry(e, MFN_INVALID, PT_NO_FLAGS);
+}
+
 static mfn_t get_cr3_mfn(cr3_t *cr3_entry) {
     void *cr3_mapped = NULL;
 
     if (mfn_invalid(cr3_entry->mfn)) {
         frame_t *frame = get_free_frame();
         BUG_ON(!frame);
+        frame->flags.pagetable = 1;
 
         cr3_entry->mfn = frame->mfn;
         cr3_mapped = tmp_map_mfn(cr3_entry->mfn);
-        memset(cr3_mapped, 0, PAGE_SIZE);
+        clean_pagetable(cr3_mapped);
     }
 
     return cr3_entry->mfn;
@@ -244,11 +251,12 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag
     if (mfn_invalid(mfn)) {
         frame_t *frame = get_free_frame();
         BUG_ON(!frame);
+        frame->flags.pagetable = 1;
 
         mfn = frame->mfn;
         set_pgentry(entry, mfn, flags);
         tab = tmp_map_mfn(mfn);
-        memset(tab, 0, PAGE_SIZE);
+        clean_pagetable(tab);
     }
     else {
         /* Page table already exists but its flags may conflict with our. Maybe fixup */
@@ -258,6 +266,10 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag
     return mfn;
 }
 
+/* This function returns NULL when failed to map a non-NULL virtual address,
+ * MAP_FAILED when failed to map a NULL (0x0) virtual address and otherwise
+ * it returns the same virtual address passed as argument.
+ */
 static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
                    unsigned long l4_flags,
@@ -267,10 +279,8 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order,
     mfn_t l1t_mfn, l2t_mfn, l3t_mfn;
     pgentry_t *tab, *entry;
 
-    if (!va || (_ul(va) & ~PAGE_ORDER_TO_MASK(order)) || !is_canon_va(va))
-        return NULL;
-
-    dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order);
+    if ((_ul(va) & ~PAGE_ORDER_TO_MASK(order)) || !is_canon_va(va))
+        return va ? NULL : MAP_FAILED;
 
 #if defined(__x86_64__)
     l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3_ptr), l4_table_index(va), l4_flags);
@@ -314,6 +324,7 @@ void *vmap_kern(void *va, mfn_t mfn, unsigned int order,
                 unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
     unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order);
 
+    dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order);
     spin_lock(&vmap_lock);
     va = _vmap(&cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags);
     spin_unlock(&vmap_lock);
@@ -327,6 +338,7 @@ void *vmap_user(void *va, mfn_t mfn, unsigned int order,
                 unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
     unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order);
 
+    dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order);
     spin_lock(&vmap_lock);
     va = _vmap(&user_cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags);
     spin_unlock(&vmap_lock);
@@ -352,7 +364,96 @@ static void map_tmp_mapping_entry(void) {
     _tmp_mapping_entry = paddr_to_virt_kern(_paddr(entry));
 }
 
+static int _vunmap(cr3_t *cr3_ptr, void *va, mfn_t *mfn, unsigned int *order) {
+    pgentry_t *tab;
+    mfn_t _mfn;
+    unsigned int _order;
+    pgentry_t *entry;
+    bool present;
+
+    if (mfn_invalid(cr3_ptr->mfn))
+        return -EINVAL;
+
+    tab = tmp_map_mfn(cr3_ptr->mfn);
+#if defined(__x86_64__)
+    pml4_t *l4e = l4_table_entry((pml4_t *) tab, va);
+    if (mfn_invalid(l4e->mfn) || !l4e->P)
+        return -ENOENT;
+
+    tab = tmp_map_mfn(l4e->mfn);
+#endif
+    pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va);
+    if (l3e->PS) {
+        _mfn = l3e->mfn;
+        _order = PAGE_ORDER_1G;
+        entry = &l3e->entry;
+        present = l3e->P;
+        goto done;
+    }
+
+    if (mfn_invalid(l3e->mfn) || !l3e->P)
+        return -ENOENT;
+
+    tab = tmp_map_mfn(l3e->mfn);
+    pde_t *l2e = l2_table_entry((pde_t *) tab, va);
+    if (l2e->PS) {
+        _mfn = l2e->mfn;
+        _order = PAGE_ORDER_2M;
+        entry = &l2e->entry;
+        present = l2e->P;
+        goto done;
+    }
+
+    if (mfn_invalid(l2e->mfn) || !l2e->P)
+        return -ENOENT;
+
+    tab = tmp_map_mfn(l2e->mfn);
+    pte_t *l1e = l1_table_entry((pte_t *) tab, va);
+    _mfn = l1e->mfn;
+    _order = PAGE_ORDER_4K;
+    entry = &l1e->entry;
+    present = l1e->P;
+
+done:
+    if (mfn)
+        *mfn = _mfn;
+    if (order)
+        *order = _order;
+    set_pgentry(entry, MFN_INVALID, PT_NO_FLAGS);
+    if (present)
+        invlpg(va);
+
+    return 0;
+}
+
+int vunmap_kern(void *va, mfn_t *mfn, unsigned int *order) {
+    int err;
+
+    dprintk("%s: va: 0x%p (cr3: 0x%p)\n", __func__, va, &cr3);
+    spin_lock(&vmap_lock);
+    err = _vunmap(&cr3, va, mfn, order);
+    spin_unlock(&vmap_lock);
+    return err;
+}
+
+int vunmap_user(void *va, mfn_t *mfn, unsigned int *order) {
+    int err;
+
+    dprintk("%s: va: 0x%p (cr3: 0x%p)\n", __func__, va, &cr3);
+    spin_lock(&vmap_lock);
+    err = _vunmap(&user_cr3, va, mfn, order);
+    spin_unlock(&vmap_lock);
+    return err;
+}
+
+static inline void init_cr3(cr3_t *cr3_ptr) {
+    memset(cr3_ptr, 0, sizeof(*cr3_ptr));
+    cr3_ptr->mfn = MFN_INVALID;
+}
+
 void init_pagetables(void) {
+    init_cr3(&cr3);
+    init_cr3(&user_cr3);
     init_tmp_mapping();
 
     for_each_memory_range (r) {

arch/x86/traps.c

@@ -52,6 +52,7 @@ static void init_tss(percpu_t *percpu) {
 #if defined(__i386__)
     percpu->tss_df.iopb = sizeof(percpu->tss_df);
     percpu->tss_df.esp0 = _ul(get_free_page_top(GFP_KERNEL));
+    BUG_ON(!percpu->tss_df.esp0);
     percpu->tss_df.ss = __KERN_DS;
     percpu->tss_df.ds = __KERN_DS;
     percpu->tss_df.es = __KERN_DS;
@@ -68,11 +69,14 @@ static void init_tss(percpu_t *percpu) {
 
     /* FIXME */
     percpu->tss.esp0 = _ul(get_free_page_top(GFP_KERNEL));
+    BUG_ON(!percpu->tss.esp0);
     percpu->tss.ss0 = __KERN_DS;
     percpu->tss.cr3 = _ul(cr3.reg);
 #elif defined(__x86_64__)
     percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL | GFP_USER));
+    BUG_ON(!percpu->tss.rsp0);
     percpu->tss.ist[0] = _ul(get_free_page_top(GFP_KERNEL | GFP_USER));
+    BUG_ON(!percpu->tss.ist[0]);
 #endif
     percpu->tss.iopb = sizeof(percpu->tss);
 

common/acpi.c

@@ -156,7 +156,7 @@ static void acpi_table_unmap_pages(void *addr, unsigned mapped_pages) {
     mfn_t mfn = virt_to_mfn(addr);
 
     for (unsigned i = 0; i < mapped_pages; i++, mfn++) {
-        vunmap_kern(mfn_to_virt_kern(mfn), PAGE_ORDER_4K);
+        vunmap_kern(mfn_to_virt_kern(mfn), NULL, NULL);
     }
 }
 

common/percpu.c

@@ -65,4 +65,4 @@ void for_each_percpu(void (*func)(percpu_t *percpu)) {
 
     list_for_each_entry (percpu, &percpu_frames, list)
         func(percpu);
-}
+}

common/sched.c

@@ -118,8 +118,11 @@ static int prepare_task(task_t *task, const char *name, task_func_t func, void *
     task->func = func;
     task->arg = arg;
     task->type = type;
-    if (task->type == TASK_TYPE_USER)
+    if (task->type == TASK_TYPE_USER) {
         task->stack = get_free_page_top(GFP_USER);
+        if (!task->stack)
+            return -ENOMEM;
+    }
     set_task_state(task, TASK_STATE_READY);
     return ESUCCESS;
 }

common/setup.c

@@ -117,10 +117,19 @@ static __always_inline void zero_bss(void) {
 void zap_boot_mappings(void) {
     for_each_memory_range (r) {
         if (r->base == VIRT_IDENT_BASE && IS_INIT_SECTION(r->name)) {
+            unsigned int order = PAGE_ORDER_4K;
+
             memset(r->start, 0, r->end - r->start);
-            for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) {
-                vunmap_kern(mfn_to_virt(mfn), PAGE_ORDER_4K);
-                reclaim_frame(mfn, PAGE_ORDER_4K);
+            for (void *va = r->start; va < r->end; va += ORDER_TO_SIZE(order)) {
+                mfn_t mfn;
+
+                if (vunmap_kern(va, &mfn, &order)) {
+                    /* FIXME: Use warning */
+                    printk("Unable to unmap kernel boot mapping at %p\n", va);
+                    order = PAGE_ORDER_4K;
+                    continue;
+                }
+                reclaim_frame(mfn, order);
             }
         }
     }

common/usermode.c

@@ -64,9 +64,15 @@ long syscall_handler(long syscall_nr, long arg1, long arg2, long arg3, long arg4
 
     case SYSCALL_MUNMAP: {
         void *va = _ptr(arg1);
-        unsigned int order = _u(arg2);
+        mfn_t mfn;
+        unsigned int order;
+        int err;
+
+        err = vunmap_user(va, &mfn, &order);
+        if (err)
+            return err;
 
-        vunmap_user(va, order);
+        put_free_frames(mfn, order);
         return 0;
     }
 
@@ -220,8 +226,8 @@ static inline long __user_text sys_mmap(void *va, unsigned long order) {
     return syscall2(SYSCALL_MMAP, _ul(va), order);
 }
 
-static inline long __user_text sys_munmap(void *va, unsigned long order) {
-    return syscall2(SYSCALL_MUNMAP, _ul(va), order);
+static inline long __user_text sys_munmap(void *va) {
+    return syscall1(SYSCALL_MUNMAP, _ul(va));
 }
 
 void __user_text exit(unsigned long exit_code) {
@@ -240,6 +246,6 @@ void *__user_text mmap(void *va, unsigned long order) {
     return _ptr(sys_mmap(va, order));
 }
 
-void __user_text munmap(void *va, unsigned long order) {
-    sys_munmap(va, order);
+int __user_text munmap(void *va) {
+    return sys_munmap(va);
 }

drivers/acpi/acpica/osl.c

@@ -339,7 +339,7 @@ void AcpiOsUnmapMemory(void *LogicalAddress, ACPI_SIZE Length) {
         if (--frame->refcount > 0)
             continue;
 
-        vunmap_kern(mfn_to_virt_map(mfn), PAGE_ORDER_4K);
+        vunmap_kern(mfn_to_virt_map(mfn), NULL, NULL);
         list_unlink(&frame->list);
         kfree(frame);
     }

include/arch/x86/page.h

@@ -160,8 +160,10 @@ typedef unsigned long mfn_t;
 
 #define _paddr(addr) ((paddr_t) _ul(addr))
 
-#define PADDR_INVALID (0UL)
-#define MFN_INVALID   (0UL)
+#define PADDR_INVALID (~0x0UL)
+#define MFN_INVALID   (paddr_to_mfn(PADDR_INVALID))
+
+#define MAP_FAILED ((void *) 1)
 
 #define IS_ADDR_SPACE_VA(va, as) (_ul(va) >= (as))
 
@@ -181,6 +183,8 @@ extern void *vmap_user(void *va, mfn_t mfn, unsigned int order,
                        unsigned long l3_flags, unsigned long l2_flags,
                        unsigned long l1_flags);
 
+extern int vunmap_kern(void *va, mfn_t *mfn, unsigned int *order);
+extern int vunmap_user(void *va, mfn_t *mfn, unsigned int *order);
 extern void pat_set_type(pat_field_t field, pat_memory_type_t type);
 extern pat_memory_type_t pat_get_type(pat_field_t field);
 
@@ -247,14 +251,6 @@ static inline mfn_t virt_to_mfn(const void *va) {
     return paddr_to_mfn(virt_to_paddr(va));
 }
 
-static inline void vunmap_kern(void *va, unsigned int order) {
-    vmap_kern(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
-}
-
-static inline void vunmap_user(void *va, unsigned int order) {
-    vmap_user(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
-}
-
 static inline void *kmap(mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
                          unsigned long l4_flags,

include/mm/vmm.h

@@ -38,7 +38,7 @@ typedef enum gfp_flags gfp_flags_t;
 /* External definitions */
 
 extern void *get_free_pages(unsigned int order, gfp_flags_t flags);
-extern void put_pages(void *page, unsigned int order);
+extern void put_pages(void *page);
 
 /* Static definitions */
 
@@ -55,11 +55,11 @@ static inline void *get_free_page_top(gfp_flags_t flags) {
 }
 
 static inline void put_page(void *page) {
-    put_pages(page, PAGE_ORDER_4K);
+    put_pages(page);
 }
 
 static inline void put_page_top(void *page) {
-    put_pages(page - PAGE_SIZE, PAGE_ORDER_4K);
+    put_pages(page - PAGE_SIZE);
 }
 
 #endif /* KTF_VMM_H */

include/usermode.h

@@ -64,7 +64,7 @@ extern void init_usermode(percpu_t *percpu);
 extern void __user_text exit(unsigned long exit_code);
 extern void __user_text printf(const char *fmt, ...);
 extern void *__user_text mmap(void *va, unsigned long order);
-extern void __user_text munmap(void *va, unsigned long order);
+extern int __user_text munmap(void *va);
 extern bool __user_text syscall_mode(syscall_mode_t);
 
 #endif /* __ASSEMBLY__ */