diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index 9a0e9c5e..3c0bd2b9 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -23,6 +23,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include +#include #include #include #include @@ -196,16 +197,22 @@ void dump_user_pagetable_va(void *va) { dump_pagetable_va(&user_cr3, va); } +static inline void clean_pagetable(void *tab) { + for (pgentry_t *e = tab; e < (pgentry_t *) (tab + PAGE_SIZE); e++) + set_pgentry(e, MFN_INVALID, PT_NO_FLAGS); +} + static mfn_t get_cr3_mfn(cr3_t *cr3_entry) { void *cr3_mapped = NULL; if (mfn_invalid(cr3_entry->mfn)) { frame_t *frame = get_free_frame(); BUG_ON(!frame); + frame->flags.pagetable = 1; cr3_entry->mfn = frame->mfn; cr3_mapped = tmp_map_mfn(cr3_entry->mfn); - memset(cr3_mapped, 0, PAGE_SIZE); + clean_pagetable(cr3_mapped); } return cr3_entry->mfn; @@ -244,11 +251,12 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag if (mfn_invalid(mfn)) { frame_t *frame = get_free_frame(); BUG_ON(!frame); + frame->flags.pagetable = 1; mfn = frame->mfn; set_pgentry(entry, mfn, flags); tab = tmp_map_mfn(mfn); - memset(tab, 0, PAGE_SIZE); + clean_pagetable(tab); } else { /* Page table already exists but its flags may conflict with our. Maybe fixup */ @@ -258,6 +266,10 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag return mfn; } +/* This function returns NULL when failed to map a non-NULL virtual address, + * MAP_FAILED when failed to map a NULL (0x0) virtual address and otherwise + * it returns the same virtual address passed as argument. + */ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) unsigned long l4_flags, @@ -267,10 +279,8 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, mfn_t l1t_mfn, l2t_mfn, l3t_mfn; pgentry_t *tab, *entry; - if (!va || (_ul(va) & ~PAGE_ORDER_TO_MASK(order)) || !is_canon_va(va)) - return NULL; - - dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order); + if ((_ul(va) & ~PAGE_ORDER_TO_MASK(order)) || !is_canon_va(va)) + return va ? NULL : MAP_FAILED; #if defined(__x86_64__) l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3_ptr), l4_table_index(va), l4_flags); @@ -314,6 +324,7 @@ void *vmap_kern(void *va, mfn_t mfn, unsigned int order, unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order); + dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order); spin_lock(&vmap_lock); va = _vmap(&cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags); spin_unlock(&vmap_lock); @@ -327,6 +338,7 @@ void *vmap_user(void *va, mfn_t mfn, unsigned int order, unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order); + dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order); spin_lock(&vmap_lock); va = _vmap(&user_cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags); spin_unlock(&vmap_lock); @@ -352,7 +364,96 @@ static void map_tmp_mapping_entry(void) { _tmp_mapping_entry = paddr_to_virt_kern(_paddr(entry)); } +static int _vunmap(cr3_t *cr3_ptr, void *va, mfn_t *mfn, unsigned int *order) { + pgentry_t *tab; + mfn_t _mfn; + unsigned int _order; + pgentry_t *entry; + bool present; + + if (mfn_invalid(cr3_ptr->mfn)) + return -EINVAL; + + tab = tmp_map_mfn(cr3_ptr->mfn); +#if defined(__x86_64__) + pml4_t *l4e = l4_table_entry((pml4_t *) tab, va); + if (mfn_invalid(l4e->mfn) || !l4e->P) + return -ENOENT; + + tab = tmp_map_mfn(l4e->mfn); +#endif + pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va); + if (l3e->PS) { + _mfn = l3e->mfn; + _order = PAGE_ORDER_1G; + entry = &l3e->entry; + present = l3e->P; + goto done; + } + + if (mfn_invalid(l3e->mfn) || !l3e->P) + return -ENOENT; + + tab = tmp_map_mfn(l3e->mfn); + pde_t *l2e = l2_table_entry((pde_t *) tab, va); + if (l2e->PS) { + _mfn = l2e->mfn; + _order = PAGE_ORDER_2M; + entry = &l2e->entry; + present = l2e->P; + goto done; + } + + if (mfn_invalid(l2e->mfn) || !l2e->P) + return -ENOENT; + + tab = tmp_map_mfn(l2e->mfn); + pte_t *l1e = l1_table_entry((pte_t *) tab, va); + _mfn = l1e->mfn; + _order = PAGE_ORDER_4K; + entry = &l1e->entry; + present = l1e->P; + +done: + if (mfn) + *mfn = _mfn; + if (order) + *order = _order; + set_pgentry(entry, MFN_INVALID, PT_NO_FLAGS); + if (present) + invlpg(va); + + return 0; +} + +int vunmap_kern(void *va, mfn_t *mfn, unsigned int *order) { + int err; + + dprintk("%s: va: 0x%p (cr3: 0x%p)\n", __func__, va, &cr3); + spin_lock(&vmap_lock); + err = _vunmap(&cr3, va, mfn, order); + spin_unlock(&vmap_lock); + return err; +} + +int vunmap_user(void *va, mfn_t *mfn, unsigned int *order) { + int err; + + dprintk("%s: va: 0x%p (cr3: 0x%p)\n", __func__, va, &cr3); + spin_lock(&vmap_lock); + err = _vunmap(&user_cr3, va, mfn, order); + spin_unlock(&vmap_lock); + return err; +} + +static inline void init_cr3(cr3_t *cr3_ptr) { + memset(cr3_ptr, 0, sizeof(*cr3_ptr)); + cr3_ptr->mfn = MFN_INVALID; +} + void init_pagetables(void) { + init_cr3(&cr3); + init_cr3(&user_cr3); init_tmp_mapping(); for_each_memory_range (r) { diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 8b689ffc..20ddc858 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -52,6 +52,7 @@ static void init_tss(percpu_t *percpu) { #if defined(__i386__) percpu->tss_df.iopb = sizeof(percpu->tss_df); percpu->tss_df.esp0 = _ul(get_free_page_top(GFP_KERNEL)); + BUG_ON(!percpu->tss_df.esp0); percpu->tss_df.ss = __KERN_DS; percpu->tss_df.ds = __KERN_DS; percpu->tss_df.es = __KERN_DS; @@ -68,11 +69,14 @@ static void init_tss(percpu_t *percpu) { /* FIXME */ percpu->tss.esp0 = _ul(get_free_page_top(GFP_KERNEL)); + BUG_ON(!percpu->tss.esp0); percpu->tss.ss0 = __KERN_DS; percpu->tss.cr3 = _ul(cr3.reg); #elif defined(__x86_64__) percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL | GFP_USER)); + BUG_ON(!percpu->tss.rsp0); percpu->tss.ist[0] = _ul(get_free_page_top(GFP_KERNEL | GFP_USER)); + BUG_ON(!percpu->tss.ist[0]); #endif percpu->tss.iopb = sizeof(percpu->tss); diff --git a/common/acpi.c b/common/acpi.c index 68f8cc38..1919c70f 100644 --- a/common/acpi.c +++ b/common/acpi.c @@ -156,7 +156,7 @@ static void acpi_table_unmap_pages(void *addr, unsigned mapped_pages) { mfn_t mfn = virt_to_mfn(addr); for (unsigned i = 0; i < mapped_pages; i++, mfn++) { - vunmap_kern(mfn_to_virt_kern(mfn), PAGE_ORDER_4K); + vunmap_kern(mfn_to_virt_kern(mfn), NULL, NULL); } } diff --git a/common/percpu.c b/common/percpu.c index c77d0935..cad7eeb3 100644 --- a/common/percpu.c +++ b/common/percpu.c @@ -65,4 +65,4 @@ void for_each_percpu(void (*func)(percpu_t *percpu)) { list_for_each_entry (percpu, &percpu_frames, list) func(percpu); -} \ No newline at end of file +} diff --git a/common/sched.c b/common/sched.c index 34e9e94d..f3ac044e 100644 --- a/common/sched.c +++ b/common/sched.c @@ -118,8 +118,11 @@ static int prepare_task(task_t *task, const char *name, task_func_t func, void * task->func = func; task->arg = arg; task->type = type; - if (task->type == TASK_TYPE_USER) + if (task->type == TASK_TYPE_USER) { task->stack = get_free_page_top(GFP_USER); + if (!task->stack) + return -ENOMEM; + } set_task_state(task, TASK_STATE_READY); return ESUCCESS; } diff --git a/common/setup.c b/common/setup.c index 39e9dbb7..7f8d2341 100644 --- a/common/setup.c +++ b/common/setup.c @@ -117,10 +117,19 @@ static __always_inline void zero_bss(void) { void zap_boot_mappings(void) { for_each_memory_range (r) { if (r->base == VIRT_IDENT_BASE && IS_INIT_SECTION(r->name)) { + unsigned int order = PAGE_ORDER_4K; + memset(r->start, 0, r->end - r->start); - for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) { - vunmap_kern(mfn_to_virt(mfn), PAGE_ORDER_4K); - reclaim_frame(mfn, PAGE_ORDER_4K); + for (void *va = r->start; va < r->end; va += ORDER_TO_SIZE(order)) { + mfn_t mfn; + + if (vunmap_kern(va, &mfn, &order)) { + /* FIXME: Use warning */ + printk("Unable to unmap kernel boot mapping at %p\n", va); + order = PAGE_ORDER_4K; + continue; + } + reclaim_frame(mfn, order); } } } diff --git a/common/usermode.c b/common/usermode.c index cadab6a5..38724584 100644 --- a/common/usermode.c +++ b/common/usermode.c @@ -64,9 +64,15 @@ long syscall_handler(long syscall_nr, long arg1, long arg2, long arg3, long arg4 case SYSCALL_MUNMAP: { void *va = _ptr(arg1); - unsigned int order = _u(arg2); + mfn_t mfn; + unsigned int order; + int err; + + err = vunmap_user(va, &mfn, &order); + if (err) + return err; - vunmap_user(va, order); + put_free_frames(mfn, order); return 0; } @@ -220,8 +226,8 @@ static inline long __user_text sys_mmap(void *va, unsigned long order) { return syscall2(SYSCALL_MMAP, _ul(va), order); } -static inline long __user_text sys_munmap(void *va, unsigned long order) { - return syscall2(SYSCALL_MUNMAP, _ul(va), order); +static inline long __user_text sys_munmap(void *va) { + return syscall1(SYSCALL_MUNMAP, _ul(va)); } void __user_text exit(unsigned long exit_code) { @@ -240,6 +246,6 @@ void *__user_text mmap(void *va, unsigned long order) { return _ptr(sys_mmap(va, order)); } -void __user_text munmap(void *va, unsigned long order) { - sys_munmap(va, order); +int __user_text munmap(void *va) { + return sys_munmap(va); } diff --git a/drivers/acpi/acpica/osl.c b/drivers/acpi/acpica/osl.c index e33e8dc1..d4fbd46d 100644 --- a/drivers/acpi/acpica/osl.c +++ b/drivers/acpi/acpica/osl.c @@ -339,7 +339,7 @@ void AcpiOsUnmapMemory(void *LogicalAddress, ACPI_SIZE Length) { if (--frame->refcount > 0) continue; - vunmap_kern(mfn_to_virt_map(mfn), PAGE_ORDER_4K); + vunmap_kern(mfn_to_virt_map(mfn), NULL, NULL); list_unlink(&frame->list); kfree(frame); } diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h index 06532eb2..edb7c668 100644 --- a/include/arch/x86/page.h +++ b/include/arch/x86/page.h @@ -160,8 +160,10 @@ typedef unsigned long mfn_t; #define _paddr(addr) ((paddr_t) _ul(addr)) -#define PADDR_INVALID (0UL) -#define MFN_INVALID (0UL) +#define PADDR_INVALID (~0x0UL) +#define MFN_INVALID (paddr_to_mfn(PADDR_INVALID)) + +#define MAP_FAILED ((void *) 1) #define IS_ADDR_SPACE_VA(va, as) (_ul(va) >= (as)) @@ -181,6 +183,8 @@ extern void *vmap_user(void *va, mfn_t mfn, unsigned int order, unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags); +extern int vunmap_kern(void *va, mfn_t *mfn, unsigned int *order); +extern int vunmap_user(void *va, mfn_t *mfn, unsigned int *order); extern void pat_set_type(pat_field_t field, pat_memory_type_t type); extern pat_memory_type_t pat_get_type(pat_field_t field); @@ -247,14 +251,6 @@ static inline mfn_t virt_to_mfn(const void *va) { return paddr_to_mfn(virt_to_paddr(va)); } -static inline void vunmap_kern(void *va, unsigned int order) { - vmap_kern(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS); -} - -static inline void vunmap_user(void *va, unsigned int order) { - vmap_user(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS); -} - static inline void *kmap(mfn_t mfn, unsigned int order, #if defined(__x86_64__) unsigned long l4_flags, diff --git a/include/mm/vmm.h b/include/mm/vmm.h index 114d4117..3d14b4e8 100644 --- a/include/mm/vmm.h +++ b/include/mm/vmm.h @@ -38,7 +38,7 @@ typedef enum gfp_flags gfp_flags_t; /* External definitions */ extern void *get_free_pages(unsigned int order, gfp_flags_t flags); -extern void put_pages(void *page, unsigned int order); +extern void put_pages(void *page); /* Static definitions */ @@ -55,11 +55,11 @@ static inline void *get_free_page_top(gfp_flags_t flags) { } static inline void put_page(void *page) { - put_pages(page, PAGE_ORDER_4K); + put_pages(page); } static inline void put_page_top(void *page) { - put_pages(page - PAGE_SIZE, PAGE_ORDER_4K); + put_pages(page - PAGE_SIZE); } #endif /* KTF_VMM_H */ diff --git a/include/usermode.h b/include/usermode.h index 9f41145a..40bc133b 100644 --- a/include/usermode.h +++ b/include/usermode.h @@ -64,7 +64,7 @@ extern void init_usermode(percpu_t *percpu); extern void __user_text exit(unsigned long exit_code); extern void __user_text printf(const char *fmt, ...); extern void *__user_text mmap(void *va, unsigned long order); -extern void __user_text munmap(void *va, unsigned long order); +extern int __user_text munmap(void *va); extern bool __user_text syscall_mode(syscall_mode_t); #endif /* __ASSEMBLY__ */ diff --git a/mm/pmm.c b/mm/pmm.c index 0a18cd0f..10f0c605 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -77,13 +77,15 @@ static frames_array_t *new_frames_array(void) { if (!boot_flags.virt) { frame_t *frame = get_free_frame(); + if (!frame) + goto error; array = (frames_array_t *) mfn_to_virt_kern(frame->mfn); } - else + else { array = get_free_page(GFP_KERNEL); - - if (!array) - panic("PMM: Unable to allocate new page for frame array"); + if (!array) + goto error; + } dprintk("%s: allocated new frames array: %p\n", __func__, array); @@ -91,6 +93,9 @@ static frames_array_t *new_frames_array(void) { total_free_frames += array->meta.free_count; return array; +error: + panic("PMM: Unable to allocate new page for frame array"); + UNREACHABLE(); } static void del_frames_array(frames_array_t *array) { @@ -539,7 +544,7 @@ frame_t *get_free_frames(unsigned int order) { void put_free_frames(mfn_t mfn, unsigned int order) { frame_t *frame; - BUG_ON(mfn_invalid(mfn) || order > MAX_PAGE_ORDER); + ASSERT(order <= MAX_PAGE_ORDER); spin_lock(&lock); frame = find_mfn_frame(busy_frames, mfn, order); diff --git a/mm/slab.c b/mm/slab.c index ba391c14..185c0739 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -140,7 +140,7 @@ meta_slab_t *slab_meta_alloc() { * Allocate a 4K page */ free_page = get_free_page(GFP_KERNEL_MAP); - if (free_page == NULL) { + if (!free_page) { dprintk("slab_meta_alloc failed, not enough free pages\n"); return NULL; } @@ -159,7 +159,7 @@ meta_slab_t *slab_meta_alloc() { ret = initialize_slab(meta_slab_page); if (ret != ESUCCESS) { dprintk("initialize_slab in slab_meta_alloc failed\n"); - put_pages(free_page, PAGE_ORDER_4K); + put_pages(free_page); return NULL; } @@ -234,7 +234,7 @@ static void *ktf_alloc(size_t size) { */ free_page = get_free_page(GFP_KERNEL_MAP); - if (free_page == NULL) { + if (!free_page) { dprintk("ktf_alloc failed, not enough free pages\n"); slab_free(META_SLAB_PAGE_ENTRY(meta_slab), meta_slab); alloc = NULL; @@ -250,7 +250,7 @@ static void *ktf_alloc(size_t size) { if (ret != ESUCCESS) { dprintk("initialize_slab failed\n"); - put_pages(free_page, PAGE_ORDER_4K); + put_pages(free_page); alloc = NULL; goto out; } @@ -306,7 +306,7 @@ static void ktf_free(void *ptr) { * meta slab free */ list_unlink(&slab->list); - put_pages(slab->slab_base, PAGE_ORDER_4K); + put_pages(slab->slab_base); meta_slab_page = META_SLAB_PAGE_ENTRY(slab); slab_free(meta_slab_page, slab); /* @@ -320,7 +320,7 @@ static void ktf_free(void *ptr) { meta_slab_page->slab_base); list_unlink(&meta_slab_page->list); memset(meta_slab_page, 0, PAGE_SIZE); - put_pages(meta_slab_page, PAGE_ORDER_4K); + put_pages(meta_slab_page); } } spin_unlock(&slab_mm_lock); diff --git a/mm/vmm.c b/mm/vmm.c index b40fe62c..ba7bc65b 100644 --- a/mm/vmm.c +++ b/mm/vmm.c @@ -30,18 +30,23 @@ #include #include +/* Used by higher level mmap_range() functions - must be taken before vmap_lock */ +static spinlock_t mmap_lock = SPINLOCK_INIT; + void *get_free_pages(unsigned int order, gfp_flags_t flags) { - frame_t *frame = get_free_frames(order); + frame_t *frame; void *va = NULL; mfn_t mfn; if (!boot_flags.virt) panic("Unable to use %s() before final page tables are set", __func__); + frame = get_free_frames(order); if (!frame) - return NULL; + return va; mfn = frame->mfn; + spin_lock(&mmap_lock); if (flags == GFP_USER) { va = vmap_kern(mfn_to_virt_user(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); @@ -69,12 +74,17 @@ void *get_free_pages(unsigned int order, gfp_flags_t flags) { vmap_user(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); } + spin_unlock(&mmap_lock); return va; } -void put_pages(void *page, unsigned int order) { - /* FIXME: unmap all mappings */ - vunmap_kern(page, order); - put_free_frames(virt_to_mfn(page), order); -} +void put_pages(void *page) { + unsigned int order; + mfn_t mfn; + + spin_lock(&mmap_lock); + BUG_ON(vunmap_kern(page, &mfn, &order)); + spin_unlock(&mmap_lock); + put_free_frames(mfn, order); +} \ No newline at end of file diff --git a/smp/mptables.c b/smp/mptables.c index 5b9f3316..e3c8baf3 100644 --- a/smp/mptables.c +++ b/smp/mptables.c @@ -89,9 +89,11 @@ static mpf_t *get_mpf_addr(void) { return NULL; sysm_addr = kmap_4k(paddr_to_mfn(_paddr(range.end) - KB(1)), L1_PROT_RO); - ptr = find_mpf(sysm_addr, sysm_addr + KB(1)); - if (ptr) - return ptr; + if (sysm_addr) { + ptr = find_mpf(sysm_addr, sysm_addr + KB(1)); + if (ptr) + return ptr; + } return find_mpf(paddr_to_virt_kern(BIOS_ROM_ADDR_START), paddr_to_virt_kern(BIOS_ROM_ADDR_START + KB(64))); diff --git a/smp/smp.c b/smp/smp.c index 312d3915..66b7dd53 100644 --- a/smp/smp.c +++ b/smp/smp.c @@ -81,6 +81,7 @@ static __text_init void boot_cpu(cpu_t *cpu) { return; ap_new_sp = get_free_pages_top(PAGE_ORDER_2M, GFP_KERNEL_MAP); + BUG_ON(!ap_new_sp); ap_cpuid = cpu->id; ap_callin = false; smp_wmb(); diff --git a/tests/unittests.c b/tests/unittests.c index 8a6c379c..82d114b8 100644 --- a/tests/unittests.c +++ b/tests/unittests.c @@ -98,7 +98,12 @@ static unsigned long __user_text test_user_task_func2(void *arg) { va = mmap(_ptr(0xfff80000), PAGE_ORDER_4K); printf(USTR("mmap: %lx\n"), _ul(va)); + if (munmap(va) != 0) { + printf(USTR("ERROR: munmap failed\n")); + ud2(); + } + va = mmap(_ptr(0xfff80000), PAGE_ORDER_4K); memset(va, 0xcc, 0x1000); ((void (*)(void)) va)();