diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 20ddc858..a88d69e4 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -130,7 +130,7 @@ void init_traps(const cpu_t *cpu) { BUG_ON(!percpu); - percpu->idt = get_free_page(GFP_KERNEL | GFP_USER); + percpu->idt = get_free_page(GFP_KERNEL_MAP | GFP_USER); BUG_ON(!percpu->idt); percpu->idt_ptr.size = (sizeof(idt_entry_t) * MAX_INT) - 1; diff --git a/common/percpu.c b/common/percpu.c index cad7eeb3..f1439073 100644 --- a/common/percpu.c +++ b/common/percpu.c @@ -50,7 +50,7 @@ percpu_t *get_percpu_page(unsigned int cpu) { /* Per CPU page must be identity mapped, * because GDT descriptor has 32-bit base. */ - percpu = get_free_page(GFP_IDENT | GFP_KERNEL | GFP_USER); + percpu = get_free_page(GFP_IDENT | GFP_KERNEL_MAP | GFP_USER); BUG_ON(!percpu); memset(percpu, 0, PAGE_SIZE); diff --git a/mm/pmm.c b/mm/pmm.c index 089d9101..949a4349 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -41,6 +41,7 @@ static frames_array_t early_frames; static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; +#define MIN_NUM_4K_FRAMES 2 static size_t frames_count[MAX_PAGE_ORDER + 1]; static spinlock_t lock = SPINLOCK_INIT; @@ -64,6 +65,30 @@ static inline void init_frame(frame_t *frame) { frame->flags.free = true; } +static inline frame_t *reserve_frame(frame_t *frame) { + if (!frame) + return NULL; + + if (frame->refcount++ == 0) { + list_unlink(&frame->list); + list_add(&frame->list, &busy_frames[frame->order]); + } + + return frame; +} + +static inline bool return_frame(frame_t *frame) { + ASSERT(is_frame_used(frame)); + + if (--frame->refcount == 0) { + list_unlink(&frame->list); + list_add(&frame->list, &free_frames[frame->order]); + return true; + } + + return false; +} + static inline void init_frames_array(frames_array_t *array) { memset(array, 0, sizeof(*array)); array->meta.free_count = ARRAY_SIZE(array->frames); @@ -74,15 +99,16 @@ static inline void init_frames_array(frames_array_t *array) { static frames_array_t *new_frames_array(void) { frames_array_t *array; + frame_t *frame; - if (!boot_flags.virt) { - frame_t *frame = get_free_frame(); - if (!frame) - goto error; + frame = reserve_frame(get_first_frame(free_frames, PAGE_ORDER_4K)); + if (!frame) + goto error; + + if (!boot_flags.virt) array = (frames_array_t *) mfn_to_virt_kern(frame->mfn); - } else { - array = get_free_page(GFP_KERNEL); + array = vmap_kern_4k(mfn_to_virt_map(frame->mfn), frame->mfn, L1_PROT); if (!array) goto error; } @@ -383,30 +409,6 @@ void init_pmm(void) { display_frames(); } -static inline frame_t *reserve_frame(frame_t *frame) { - if (!frame) - return NULL; - - if (frame->refcount++ == 0) { - list_unlink(&frame->list); - list_add(&frame->list, &busy_frames[frame->order]); - } - - return frame; -} - -static inline bool return_frame(frame_t *frame) { - ASSERT(is_frame_used(frame)); - - if (--frame->refcount == 0) { - list_unlink(&frame->list); - list_add(&frame->list, &free_frames[frame->order]); - return true; - } - - return false; -} - static frame_t *_find_mfn_frame(list_head_t *list, mfn_t mfn, unsigned int order) { frame_t *frame; @@ -509,31 +511,6 @@ static frame_t *find_larger_frame(list_head_t *list, unsigned int order) { return NULL; } -/* Reserves and returns the first free frame fulfilling - * the condition specified by the callback. - * This function does not split larger frames. - */ -frame_t *get_free_frames_cond(free_frames_cond_t cb) { - spin_lock(&lock); - for_each_order (order) { - frame_t *frame; - - if (list_is_empty(&free_frames[order])) - continue; - - list_for_each_entry (frame, &free_frames[order], list) { - if (cb(frame)) { - reserve_frame(frame); - spin_unlock(&lock); - return frame; - } - } - } - spin_unlock(&lock); - - return NULL; -} - static inline void relink_frame_to_order(frame_t *frame, unsigned int new_order) { ASSERT(new_order <= MAX_PAGE_ORDER); @@ -596,6 +573,50 @@ static void merge_frames(frame_t *first) { merge_frames(first); } +static inline bool enough_4k_frames(void) { + frame_t *frame; + int count = 0; + + list_for_each_entry (frame, &free_frames[PAGE_ORDER_4K], list) { + if (++count >= MIN_NUM_4K_FRAMES) + return true; + } + + return false; +} + +static void try_create_4k_frames(void) { + while (!enough_4k_frames()) { + frame_t *frame = find_larger_frame(free_frames, PAGE_ORDER_4K); + if (!frame) + panic("No more frames available to create 4K frames"); + split_frame(frame); + } +} + +/* Reserves and returns the first free frame fulfilling + * the condition specified by the callback. + * This function does not split larger frames. + */ +frame_t *get_free_frames_cond(free_frames_cond_t cb) { + spin_lock(&lock); + try_create_4k_frames(); + for_each_order (order) { + frame_t *frame; + + list_for_each_entry (frame, &free_frames[order], list) { + if (cb(frame)) { + reserve_frame(frame); + spin_unlock(&lock); + return frame; + } + } + } + spin_unlock(&lock); + + return NULL; +} + frame_t *get_free_frames(unsigned int order) { frame_t *frame; @@ -603,7 +624,11 @@ frame_t *get_free_frames(unsigned int order) { return NULL; spin_lock(&lock); + if (order == PAGE_ORDER_4K) + try_create_4k_frames(); + while (list_is_empty(&free_frames[order])) { + BUG_ON(order == PAGE_ORDER_4K); frame = find_larger_frame(free_frames, order); if (!frame) { spin_unlock(&lock); @@ -643,7 +668,9 @@ void map_frames_array(void) { list_for_each_entry (array, &frames, list) { mfn_t mfn = virt_to_mfn(array); + void *va = IS_ADDR_SPACE_VA(array, VIRT_KERNEL_BASE) ? mfn_to_virt_kern(mfn) + : mfn_to_virt_map(mfn); - BUG_ON(!vmap_kern_4k(mfn_to_virt_kern(mfn), mfn, L1_PROT)); + BUG_ON(!vmap_kern_4k(va, mfn, L1_PROT)); } }