Add basic support for user land address space

arch/x86/pagetables.c

@@ -31,6 +31,7 @@
 #include <string.h>
 
 cr3_t cr3;
+cr3_t user_cr3;
 
 static inline const char *dump_pte_flags(char *buf, size_t size, pte_t pte) {
     /* clang-format off */
@@ -152,26 +153,26 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag
     return mfn;
 }
 
-void *vmap(void *va, mfn_t mfn, unsigned int order,
+void *_vmap(cr3_t *cr3, void *va, mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
-           unsigned long l4_flags,
+            unsigned long l4_flags,
 #endif
-           unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
+            unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
     static spinlock_t lock = SPINLOCK_INIT;
     mfn_t l1t_mfn, l2t_mfn, l3t_mfn;
     pgentry_t *tab, *entry;
 
-    if (!va || _ul(va) & ~PAGE_MASK)
+    if (!va || (_ul(va) & ~PAGE_ORDER_TO_MASK(order)))
         return NULL;
 
     dprintk("%s: va: %p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order);
 
     spin_lock(&lock);
 
 #if defined(__x86_64__)
-    l3t_mfn = get_pgentry_mfn(get_cr3_mfn(&cr3), l4_table_index(va), l4_flags);
+    l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3), l4_table_index(va), l4_flags);
 #else
-    l3t_mfn = get_cr3_mfn(&cr3);
+    l3t_mfn = get_cr3_mfn(cr3);
 #endif
 
     if (order == PAGE_ORDER_1G) {
@@ -201,8 +202,25 @@ void *vmap(void *va, mfn_t mfn, unsigned int order,
     return va;
 }
 
-void vunmap(void *va, unsigned int order) {
-    vmap(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
+void *vmap_kern(void *va, mfn_t mfn, unsigned int order,
+#if defined(__x86_64__)
+                unsigned long l4_flags,
+#endif
+                unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
+    unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order);
+
+    return _vmap(&cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags);
+}
+
+void *vmap_user(void *va, mfn_t mfn, unsigned int order,
+#if defined(__x86_64__)
+                unsigned long l4_flags,
+#endif
+                unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
+    unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order);
+
+    return _vmap(&user_cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags,
+                 l1_flags);
 }
 
 void init_pagetables(void) {
@@ -217,8 +235,12 @@ void init_pagetables(void) {
                 kmap_4k(mfn, r->flags);
             break;
         case VIRT_USER_BASE:
-            for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++)
-                vmap_4k(mfn_to_virt_user(mfn), mfn, r->flags);
+            for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) {
+                void *va = mfn_to_virt_user(mfn);
+
+                vmap_4k(va, mfn, r->flags);
+                vmap_user_4k(va, mfn, r->flags);
+            }
             break;
         default:
             break;

arch/x86/traps.c

@@ -77,7 +77,7 @@ static void init_tss(percpu_t *percpu) {
     percpu->tss.ss0 = __KERN_DS;
     percpu->tss.cr3 = _ul(cr3.reg);
 #elif defined(__x86_64__)
-    percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL));
+    percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL | GFP_USER));
     percpu->tss.ist[0] = _ul(get_free_page_top(GFP_KERNEL));
 #endif
     percpu->tss.iopb = sizeof(percpu->tss);
@@ -124,7 +124,7 @@ void init_traps(unsigned int cpu) {
 
     BUG_ON(!percpu);
 
-    percpu->idt = get_free_page(GFP_KERNEL);
+    percpu->idt = get_free_page(GFP_KERNEL | GFP_USER);
     BUG_ON(!percpu->idt);
 
     percpu->idt_ptr.size = (sizeof(percpu->idt) * MAX_INT) - 1;

common/setup.c

@@ -105,7 +105,7 @@ void zap_boot_mappings(void) {
                 memset(r->start, 0, r->end - r->start);
 
             for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) {
-                vunmap(mfn_to_virt(mfn), PAGE_ORDER_4K);
+                vunmap_kern(mfn_to_virt(mfn), PAGE_ORDER_4K);
                 reclaim_frame(mfn, PAGE_ORDER_4K);
             }
         }

drivers/acpi/acpica/osl.c

@@ -294,7 +294,8 @@ void *AcpiOsMapMemory(ACPI_PHYSICAL_ADDRESS PhysicalAddress, ACPI_SIZE Length) {
         void *_va;
 
         if (!frame) {
-            _va = mmap_4k(mfn, L1_PROT);
+            _va = vmap_kern(mfn_to_virt_map(mfn), mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT,
+                            L2_PROT, L1_PROT);
             if (!_va) {
                 spin_unlock(&map_lock);
                 return NULL;
@@ -327,7 +328,7 @@ void AcpiOsUnmapMemory(void *LogicalAddress, ACPI_SIZE Length) {
         if (--frame->refcount > 0)
             continue;
 
-        vunmap(mfn_to_virt_map(mfn), PAGE_ORDER_4K);
+        vunmap_kern(mfn_to_virt_map(mfn), PAGE_ORDER_4K);
         list_unlink(&frame->list);
         kfree(frame);
     }
@@ -351,7 +352,6 @@ typedef struct osd_exec_cb_wrapper osd_exec_cb_wrapper_t;
 
 unsigned long _osd_exec_cb_wrapper(void *arg) {
     osd_exec_cb_wrapper_t *cb = arg;
-
     cb->Function(cb->Context);
     return 0;
 }

include/arch/x86/asm-macros.h

@@ -152,6 +152,13 @@
 #endif
 .endm
 
+.macro SET_CR3 val
+    push %_ASM_AX
+    mov (\val), %_ASM_AX
+    mov %_ASM_AX, %cr3
+    pop %_ASM_AX
+.endm
+
 #define GLOBAL(name) \
     .global name;    \
 name:

include/arch/x86/page.h

@@ -45,6 +45,8 @@
 #define MAX_PAGE_ORDER     PAGE_ORDER_1G
 #define PAGE_ORDER_INVALID (-1)
 
+#define PAGE_ORDER_TO_MASK(order) (~((PAGE_SIZE << (order)) - 1))
+
 #define _PAGE_PRESENT  0x0001
 #define _PAGE_RW       0x0002
 #define _PAGE_USER     0x0004
@@ -153,17 +155,23 @@ typedef unsigned long mfn_t;
 #define PADDR_INVALID (0UL)
 #define MFN_INVALID   (0UL)
 
-#define IS_ADDR_SPACE_VA(va, as) ((_ul(va) & (as)) == (as))
+#define IS_ADDR_SPACE_VA(va, as) (_ul(va) >= (as))
 
 /* External declarations */
 
-extern void *vmap(void *va, mfn_t mfn, unsigned int order,
+extern void *vmap_kern(void *va, mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
-                  unsigned long l4_flags,
+                       unsigned long l4_flags,
 #endif
-                  unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags);
+                       unsigned long l3_flags, unsigned long l2_flags,
+                       unsigned long l1_flags);
 
-extern void vunmap(void *va, unsigned int order);
+extern void *vmap_user(void *va, mfn_t mfn, unsigned int order,
+#if defined(__x86_64__)
+                       unsigned long l4_flags,
+#endif
+                       unsigned long l3_flags, unsigned long l2_flags,
+                       unsigned long l1_flags);
 
 extern void pat_set_type(pat_field_t field, pat_memory_type_t type);
 extern pat_memory_type_t pat_get_type(pat_field_t field);
@@ -210,6 +218,7 @@ static inline void *mfn_to_virt(mfn_t mfn) { return paddr_to_virt(mfn << PAGE_SH
 static inline paddr_t virt_to_paddr(const void *va) {
     paddr_t pa = (paddr_t) va;
 
+    /* Order matters here */
     if (IS_ADDR_SPACE_VA(va, VIRT_KERNEL_BASE))
         return pa - VIRT_KERNEL_BASE;
     if (IS_ADDR_SPACE_VA(va, VIRT_KERNEL_MAP))
@@ -224,68 +233,70 @@ static inline mfn_t virt_to_mfn(const void *va) {
     return paddr_to_mfn(virt_to_paddr(va));
 }
 
-static inline void *kmap(mfn_t mfn, unsigned int order,
-#if defined(__x86_64__)
-                         unsigned long l4_flags,
-#endif
-                         unsigned long l3_flags, unsigned long l2_flags,
-                         unsigned long l1_flags) {
-    return vmap(mfn_to_virt_kern(mfn), mfn, order,
-#if defined(__x86_64__)
-                l4_flags,
-#endif
-                l3_flags, l2_flags, l1_flags);
+static inline void vunmap_kern(void *va, unsigned int order) {
+    vmap_kern(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
+}
+
+static inline void vunmap_user(void *va, unsigned int order) {
+    vmap_user(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
 }
 
-static inline void *mmap(mfn_t mfn, unsigned int order,
+static inline void *kmap(mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
                          unsigned long l4_flags,
 #endif
                          unsigned long l3_flags, unsigned long l2_flags,
                          unsigned long l1_flags) {
-    return vmap(mfn_to_virt_map(mfn), mfn, order,
+    return vmap_kern(mfn_to_virt_kern(mfn), mfn, order,
 #if defined(__x86_64__)
-                l4_flags,
+                     l4_flags,
 #endif
-                l3_flags, l2_flags, l1_flags);
+                     l3_flags, l2_flags, l1_flags);
 }
 
 static inline void *vmap_1g(void *va, mfn_t mfn, unsigned long l3_flags) {
-    return vmap(va, mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
+    return vmap_kern(va, mfn, PAGE_ORDER_1G, L4_PROT, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
 }
 
 static inline void *vmap_2m(void *va, mfn_t mfn, unsigned long l2_flags) {
-    return vmap(va, mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags,
-                PT_NO_FLAGS);
+    return vmap_kern(va, mfn, PAGE_ORDER_2M, L4_PROT, L3_PROT, l2_flags, PT_NO_FLAGS);
 }
 
 static inline void *vmap_4k(void *va, mfn_t mfn, unsigned long l1_flags) {
-    return vmap(va, mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER,
-                l1_flags);
+    return vmap_kern(va, mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, l1_flags);
 }
 
 static inline void *kmap_1g(mfn_t mfn, unsigned long l3_flags) {
-    return kmap(mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
+    return kmap(mfn, PAGE_ORDER_1G, L4_PROT, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
 }
 
 static inline void *kmap_2m(mfn_t mfn, unsigned long l2_flags) {
-    return kmap(mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags, PT_NO_FLAGS);
+    return kmap(mfn, PAGE_ORDER_2M, L4_PROT, L3_PROT, l2_flags, PT_NO_FLAGS);
 }
 
 static inline void *kmap_4k(mfn_t mfn, unsigned long l1_flags) {
-    return kmap(mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, l1_flags);
+    return kmap(mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, l1_flags);
 }
 
-static inline void *mmap_1g(mfn_t mfn, unsigned long l3_flags) {
-    return mmap(mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
+static inline void *vmap_user_1g(void *va, mfn_t mfn, unsigned long l3_flags) {
+    unsigned long user = l3_flags & _PAGE_USER;
+
+    return vmap_user(va, mfn, PAGE_ORDER_1G, L4_PROT | user, l3_flags | user, PT_NO_FLAGS,
+                     PT_NO_FLAGS);
 }
 
-static inline void *mmap_2m(mfn_t mfn, unsigned long l2_flags) {
-    return mmap(mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags, PT_NO_FLAGS);
+static inline void *vmap_user_2m(void *va, mfn_t mfn, unsigned long l2_flags) {
+    unsigned long user = l2_flags & _PAGE_USER;
+
+    return vmap_user(va, mfn, PAGE_ORDER_2M, L4_PROT | user, L3_PROT | user,
+                     l2_flags | user, PT_NO_FLAGS);
 }
 
-static inline void *mmap_4k(mfn_t mfn, unsigned long l1_flags) {
-    return mmap(mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, l1_flags);
+static inline void *vmap_user_4k(void *va, mfn_t mfn, unsigned long l1_flags) {
+    unsigned long user = l1_flags & _PAGE_USER;
+
+    return vmap_user(va, mfn, PAGE_ORDER_4K, L4_PROT | user, L3_PROT | user,
+                     L2_PROT | user, l1_flags);
 }
 
 #endif /* __ASSEMBLY__ */

include/arch/x86/pagetable.h

@@ -137,6 +137,7 @@ union cr3 {
 typedef union cr3 cr3_t;
 
 extern cr3_t cr3;
+extern cr3_t user_cr3;
 
 typedef unsigned int pt_index_t;
 

include/mm/vmm.h

@@ -55,4 +55,8 @@ static inline void *get_free_page_top(uint32_t flags) {
 
 static inline void put_page(void *page) { put_pages(page, PAGE_ORDER_4K); }
 
+static inline void put_page_top(void *page) {
+    put_pages(page - PAGE_SIZE, PAGE_ORDER_4K);
+}
+
 #endif /* KTF_VMM_H */

mm/vmm.c

@@ -40,24 +40,41 @@ void *get_free_pages(unsigned int order, uint32_t flags) {
 
     if (!frame)
         return NULL;
-
     mfn = frame->mfn;
 
-    if (flags & GFP_IDENT)
-        va = vmap(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
-    if (flags & GFP_USER)
-        va = vmap(mfn_to_virt_user(mfn), mfn, order, L4_PROT_USER, L3_PROT_USER,
+    if (flags == GFP_USER) {
+        va = vmap_kern(mfn_to_virt_user(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                       L1_PROT);
+        vmap_user(mfn_to_virt_user(mfn), mfn, order, L4_PROT_USER, L3_PROT_USER,
                   L2_PROT_USER, L1_PROT_USER);
-    if (flags & GFP_KERNEL)
+    }
+
+    if (flags & GFP_IDENT) {
+        va = vmap_kern(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+        if (flags & GFP_USER)
+            vmap_user(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+    }
+
+    if (flags & GFP_KERNEL) {
         va = kmap(mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
-    if (flags & GFP_KERNEL_MAP)
-        va = mmap(mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+        if (flags & GFP_USER)
+            vmap_user(mfn_to_virt_kern(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                      L1_PROT);
+    }
+
+    if (flags & GFP_KERNEL_MAP) {
+        va = vmap_kern(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                       L1_PROT);
+        if (flags & GFP_USER)
+            vmap_user(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                      L1_PROT);
+    }
 
     return va;
 }
 
 void put_pages(void *page, unsigned int order) {
     /* FIXME: unmap all mappings */
-    vunmap(page, order);
+    vunmap_kern(page, order);
     put_free_frames(virt_to_mfn(page), order);
 }