diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index cf3da855..3088e0c2 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -31,6 +31,7 @@ #include cr3_t cr3; +cr3_t user_cr3; static inline const char *dump_pte_flags(char *buf, size_t size, pte_t pte) { /* clang-format off */ @@ -152,16 +153,16 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag return mfn; } -void *vmap(void *va, mfn_t mfn, unsigned int order, +void *_vmap(cr3_t *cr3, void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) - unsigned long l4_flags, + unsigned long l4_flags, #endif - unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { + unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { static spinlock_t lock = SPINLOCK_INIT; mfn_t l1t_mfn, l2t_mfn, l3t_mfn; pgentry_t *tab, *entry; - if (!va || _ul(va) & ~PAGE_MASK) + if (!va || (_ul(va) & ~PAGE_ORDER_TO_MASK(order))) return NULL; dprintk("%s: va: %p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order); @@ -169,9 +170,9 @@ void *vmap(void *va, mfn_t mfn, unsigned int order, spin_lock(&lock); #if defined(__x86_64__) - l3t_mfn = get_pgentry_mfn(get_cr3_mfn(&cr3), l4_table_index(va), l4_flags); + l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3), l4_table_index(va), l4_flags); #else - l3t_mfn = get_cr3_mfn(&cr3); + l3t_mfn = get_cr3_mfn(cr3); #endif if (order == PAGE_ORDER_1G) { @@ -201,8 +202,25 @@ void *vmap(void *va, mfn_t mfn, unsigned int order, return va; } -void vunmap(void *va, unsigned int order) { - vmap(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS); +void *vmap_kern(void *va, mfn_t mfn, unsigned int order, +#if defined(__x86_64__) + unsigned long l4_flags, +#endif + unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { + unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order); + + return _vmap(&cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags); +} + +void *vmap_user(void *va, mfn_t mfn, unsigned int order, +#if defined(__x86_64__) + unsigned long l4_flags, +#endif + unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { + unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order); + + return _vmap(&user_cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, + l1_flags); } void init_pagetables(void) { @@ -217,8 +235,12 @@ void init_pagetables(void) { kmap_4k(mfn, r->flags); break; case VIRT_USER_BASE: - for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) - vmap_4k(mfn_to_virt_user(mfn), mfn, r->flags); + for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) { + void *va = mfn_to_virt_user(mfn); + + vmap_4k(va, mfn, r->flags); + vmap_user_4k(va, mfn, r->flags); + } break; default: break; diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 5791a6c0..f2733d3e 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -77,7 +77,7 @@ static void init_tss(percpu_t *percpu) { percpu->tss.ss0 = __KERN_DS; percpu->tss.cr3 = _ul(cr3.reg); #elif defined(__x86_64__) - percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL)); + percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL | GFP_USER)); percpu->tss.ist[0] = _ul(get_free_page_top(GFP_KERNEL)); #endif percpu->tss.iopb = sizeof(percpu->tss); @@ -124,7 +124,7 @@ void init_traps(unsigned int cpu) { BUG_ON(!percpu); - percpu->idt = get_free_page(GFP_KERNEL); + percpu->idt = get_free_page(GFP_KERNEL | GFP_USER); BUG_ON(!percpu->idt); percpu->idt_ptr.size = (sizeof(percpu->idt) * MAX_INT) - 1; diff --git a/common/setup.c b/common/setup.c index 4eb74006..42cbdaf9 100644 --- a/common/setup.c +++ b/common/setup.c @@ -105,7 +105,7 @@ void zap_boot_mappings(void) { memset(r->start, 0, r->end - r->start); for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) { - vunmap(mfn_to_virt(mfn), PAGE_ORDER_4K); + vunmap_kern(mfn_to_virt(mfn), PAGE_ORDER_4K); reclaim_frame(mfn, PAGE_ORDER_4K); } } diff --git a/drivers/acpi/acpica/osl.c b/drivers/acpi/acpica/osl.c index 9d372a00..23511159 100644 --- a/drivers/acpi/acpica/osl.c +++ b/drivers/acpi/acpica/osl.c @@ -294,7 +294,8 @@ void *AcpiOsMapMemory(ACPI_PHYSICAL_ADDRESS PhysicalAddress, ACPI_SIZE Length) { void *_va; if (!frame) { - _va = mmap_4k(mfn, L1_PROT); + _va = vmap_kern(mfn_to_virt_map(mfn), mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, + L2_PROT, L1_PROT); if (!_va) { spin_unlock(&map_lock); return NULL; @@ -327,7 +328,7 @@ void AcpiOsUnmapMemory(void *LogicalAddress, ACPI_SIZE Length) { if (--frame->refcount > 0) continue; - vunmap(mfn_to_virt_map(mfn), PAGE_ORDER_4K); + vunmap_kern(mfn_to_virt_map(mfn), PAGE_ORDER_4K); list_unlink(&frame->list); kfree(frame); } @@ -351,7 +352,6 @@ typedef struct osd_exec_cb_wrapper osd_exec_cb_wrapper_t; unsigned long _osd_exec_cb_wrapper(void *arg) { osd_exec_cb_wrapper_t *cb = arg; - cb->Function(cb->Context); return 0; } diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h index 6b946d57..acdb7edd 100644 --- a/include/arch/x86/asm-macros.h +++ b/include/arch/x86/asm-macros.h @@ -152,6 +152,13 @@ #endif .endm +.macro SET_CR3 val + push %_ASM_AX + mov (\val), %_ASM_AX + mov %_ASM_AX, %cr3 + pop %_ASM_AX +.endm + #define GLOBAL(name) \ .global name; \ name: diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h index 2a324579..d9206398 100644 --- a/include/arch/x86/page.h +++ b/include/arch/x86/page.h @@ -45,6 +45,8 @@ #define MAX_PAGE_ORDER PAGE_ORDER_1G #define PAGE_ORDER_INVALID (-1) +#define PAGE_ORDER_TO_MASK(order) (~((PAGE_SIZE << (order)) - 1)) + #define _PAGE_PRESENT 0x0001 #define _PAGE_RW 0x0002 #define _PAGE_USER 0x0004 @@ -153,17 +155,23 @@ typedef unsigned long mfn_t; #define PADDR_INVALID (0UL) #define MFN_INVALID (0UL) -#define IS_ADDR_SPACE_VA(va, as) ((_ul(va) & (as)) == (as)) +#define IS_ADDR_SPACE_VA(va, as) (_ul(va) >= (as)) /* External declarations */ -extern void *vmap(void *va, mfn_t mfn, unsigned int order, +extern void *vmap_kern(void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) - unsigned long l4_flags, + unsigned long l4_flags, #endif - unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags); + unsigned long l3_flags, unsigned long l2_flags, + unsigned long l1_flags); -extern void vunmap(void *va, unsigned int order); +extern void *vmap_user(void *va, mfn_t mfn, unsigned int order, +#if defined(__x86_64__) + unsigned long l4_flags, +#endif + unsigned long l3_flags, unsigned long l2_flags, + unsigned long l1_flags); extern void pat_set_type(pat_field_t field, pat_memory_type_t type); extern pat_memory_type_t pat_get_type(pat_field_t field); @@ -210,6 +218,7 @@ static inline void *mfn_to_virt(mfn_t mfn) { return paddr_to_virt(mfn << PAGE_SH static inline paddr_t virt_to_paddr(const void *va) { paddr_t pa = (paddr_t) va; + /* Order matters here */ if (IS_ADDR_SPACE_VA(va, VIRT_KERNEL_BASE)) return pa - VIRT_KERNEL_BASE; if (IS_ADDR_SPACE_VA(va, VIRT_KERNEL_MAP)) @@ -224,68 +233,70 @@ static inline mfn_t virt_to_mfn(const void *va) { return paddr_to_mfn(virt_to_paddr(va)); } -static inline void *kmap(mfn_t mfn, unsigned int order, -#if defined(__x86_64__) - unsigned long l4_flags, -#endif - unsigned long l3_flags, unsigned long l2_flags, - unsigned long l1_flags) { - return vmap(mfn_to_virt_kern(mfn), mfn, order, -#if defined(__x86_64__) - l4_flags, -#endif - l3_flags, l2_flags, l1_flags); +static inline void vunmap_kern(void *va, unsigned int order) { + vmap_kern(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS); +} + +static inline void vunmap_user(void *va, unsigned int order) { + vmap_user(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS); } -static inline void *mmap(mfn_t mfn, unsigned int order, +static inline void *kmap(mfn_t mfn, unsigned int order, #if defined(__x86_64__) unsigned long l4_flags, #endif unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { - return vmap(mfn_to_virt_map(mfn), mfn, order, + return vmap_kern(mfn_to_virt_kern(mfn), mfn, order, #if defined(__x86_64__) - l4_flags, + l4_flags, #endif - l3_flags, l2_flags, l1_flags); + l3_flags, l2_flags, l1_flags); } static inline void *vmap_1g(void *va, mfn_t mfn, unsigned long l3_flags) { - return vmap(va, mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS); + return vmap_kern(va, mfn, PAGE_ORDER_1G, L4_PROT, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS); } static inline void *vmap_2m(void *va, mfn_t mfn, unsigned long l2_flags) { - return vmap(va, mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags, - PT_NO_FLAGS); + return vmap_kern(va, mfn, PAGE_ORDER_2M, L4_PROT, L3_PROT, l2_flags, PT_NO_FLAGS); } static inline void *vmap_4k(void *va, mfn_t mfn, unsigned long l1_flags) { - return vmap(va, mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, - l1_flags); + return vmap_kern(va, mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, l1_flags); } static inline void *kmap_1g(mfn_t mfn, unsigned long l3_flags) { - return kmap(mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS); + return kmap(mfn, PAGE_ORDER_1G, L4_PROT, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS); } static inline void *kmap_2m(mfn_t mfn, unsigned long l2_flags) { - return kmap(mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags, PT_NO_FLAGS); + return kmap(mfn, PAGE_ORDER_2M, L4_PROT, L3_PROT, l2_flags, PT_NO_FLAGS); } static inline void *kmap_4k(mfn_t mfn, unsigned long l1_flags) { - return kmap(mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, l1_flags); + return kmap(mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, l1_flags); } -static inline void *mmap_1g(mfn_t mfn, unsigned long l3_flags) { - return mmap(mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS); +static inline void *vmap_user_1g(void *va, mfn_t mfn, unsigned long l3_flags) { + unsigned long user = l3_flags & _PAGE_USER; + + return vmap_user(va, mfn, PAGE_ORDER_1G, L4_PROT | user, l3_flags | user, PT_NO_FLAGS, + PT_NO_FLAGS); } -static inline void *mmap_2m(mfn_t mfn, unsigned long l2_flags) { - return mmap(mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags, PT_NO_FLAGS); +static inline void *vmap_user_2m(void *va, mfn_t mfn, unsigned long l2_flags) { + unsigned long user = l2_flags & _PAGE_USER; + + return vmap_user(va, mfn, PAGE_ORDER_2M, L4_PROT | user, L3_PROT | user, + l2_flags | user, PT_NO_FLAGS); } -static inline void *mmap_4k(mfn_t mfn, unsigned long l1_flags) { - return mmap(mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, l1_flags); +static inline void *vmap_user_4k(void *va, mfn_t mfn, unsigned long l1_flags) { + unsigned long user = l1_flags & _PAGE_USER; + + return vmap_user(va, mfn, PAGE_ORDER_4K, L4_PROT | user, L3_PROT | user, + L2_PROT | user, l1_flags); } #endif /* __ASSEMBLY__ */ diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index e70eb2a7..3d7f7866 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -137,6 +137,7 @@ union cr3 { typedef union cr3 cr3_t; extern cr3_t cr3; +extern cr3_t user_cr3; typedef unsigned int pt_index_t; diff --git a/include/mm/vmm.h b/include/mm/vmm.h index 80f68a17..e8e48c21 100644 --- a/include/mm/vmm.h +++ b/include/mm/vmm.h @@ -55,4 +55,8 @@ static inline void *get_free_page_top(uint32_t flags) { static inline void put_page(void *page) { put_pages(page, PAGE_ORDER_4K); } +static inline void put_page_top(void *page) { + put_pages(page - PAGE_SIZE, PAGE_ORDER_4K); +} + #endif /* KTF_VMM_H */ diff --git a/mm/vmm.c b/mm/vmm.c index 6d86b21d..fad57929 100644 --- a/mm/vmm.c +++ b/mm/vmm.c @@ -40,24 +40,41 @@ void *get_free_pages(unsigned int order, uint32_t flags) { if (!frame) return NULL; - mfn = frame->mfn; - if (flags & GFP_IDENT) - va = vmap(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); - if (flags & GFP_USER) - va = vmap(mfn_to_virt_user(mfn), mfn, order, L4_PROT_USER, L3_PROT_USER, + if (flags == GFP_USER) { + va = vmap_kern(mfn_to_virt_user(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, + L1_PROT); + vmap_user(mfn_to_virt_user(mfn), mfn, order, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, L1_PROT_USER); - if (flags & GFP_KERNEL) + } + + if (flags & GFP_IDENT) { + va = vmap_kern(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); + if (flags & GFP_USER) + vmap_user(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); + } + + if (flags & GFP_KERNEL) { va = kmap(mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); - if (flags & GFP_KERNEL_MAP) - va = mmap(mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); + if (flags & GFP_USER) + vmap_user(mfn_to_virt_kern(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, + L1_PROT); + } + + if (flags & GFP_KERNEL_MAP) { + va = vmap_kern(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, + L1_PROT); + if (flags & GFP_USER) + vmap_user(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, + L1_PROT); + } return va; } void put_pages(void *page, unsigned int order) { /* FIXME: unmap all mappings */ - vunmap(page, order); + vunmap_kern(page, order); put_free_frames(virt_to_mfn(page), order); }