diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 96406e09..f20c5ed5 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -60,6 +60,12 @@ typedef struct frames_array frames_array_t; typedef bool (*free_frames_cond_t)(frame_t *free_frame); +#define ORDER_TO_SIZE(order) (PAGE_SIZE << (order)) + +#define FIRST_FRAME_SIBLING(mfn, order) ((mfn) % (1UL << (order)) == 0) +#define NEXT_MFN(mfn, order) ((mfn) + (1UL << (order))) +#define PREV_MFN(mfn, order) ((mfn) - (1UL << (order))) + /* External definitions */ extern void display_frames_count(void); @@ -80,6 +86,17 @@ static inline bool paddr_invalid(paddr_t pa) { static inline bool mfn_invalid(mfn_t mfn) { return paddr_invalid(mfn_to_paddr(mfn)); } +static inline bool has_frames(list_head_t *frames, unsigned int order) { + return !(order > MAX_PAGE_ORDER || list_is_empty(&frames[order])); +} + +static inline frame_t *get_first_frame(list_head_t *frames, unsigned int order) { + if (!has_frames(frames, order)) + return NULL; + + return list_first_entry(&frames[order], frame_t, list); +} + static inline frame_t *get_free_frame(void) { return get_free_frames(PAGE_ORDER_4K); } static inline void put_free_frame(mfn_t mfn) { return put_free_frames(mfn, PAGE_ORDER_4K); diff --git a/mm/pmm.c b/mm/pmm.c index 70550a13..34d6f27d 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -172,8 +172,8 @@ static inline frame_t *take_frame(frame_t *frame, frames_array_t *array) { return frame; } -static inline frame_t *put_frame(frame_t *frame, frames_array_t *array) { - ASSERT(!frame); +static inline frame_t *put_frames_array_entry(frame_t *frame, frames_array_t *array) { + BUG_ON(is_frame_free(frame)); if (!array) array = find_frames_array(frame); @@ -204,11 +204,15 @@ static inline frame_t *get_frames_array_entry(void) { return NULL; } -static inline void put_frames_array_entry(frame_t *frame) { - if (is_frame_free(frame)) - return; +static inline void destroy_frame(frame_t *frame) { + BUG_ON(is_frame_used(frame)); + + if (frame) { + list_unlink(&frame->list); + frames_count[frame->order]--; - put_frame(frame, NULL); + put_frames_array_entry(frame, NULL); + } } static inline frame_t *new_frame(mfn_t mfn, unsigned int order) { @@ -233,8 +237,18 @@ static inline void add_frame(mfn_t mfn, unsigned int order) { list_add_tail(&frame->list, &free_frames[order]); } +static inline unsigned int find_max_avail_order(size_t size) { + for (unsigned int order = MAX_PAGE_ORDER; order > PAGE_ORDER_4K; order--) { + if (ORDER_TO_SIZE(order) <= size) + return order; + } + + return PAGE_ORDER_4K; +} + static size_t process_memory_range(unsigned index) { paddr_t start, end, cur; + unsigned int max_order; addr_range_t range; size_t size; @@ -257,31 +271,27 @@ static size_t process_memory_range(unsigned index) { add_early_frame(paddr_to_mfn(cur), PAGE_ORDER_4K); else add_frame(paddr_to_mfn(cur), PAGE_ORDER_4K); - cur += (PAGE_SIZE << PAGE_ORDER_4K); + cur += ORDER_TO_SIZE(PAGE_ORDER_4K); } - /* Add initial 2M frames and align to 1G. */ - while (cur % PAGE_SIZE_1G && cur + PAGE_SIZE_2M <= end) { - add_frame(paddr_to_mfn(cur), PAGE_ORDER_2M); - cur += (PAGE_SIZE << PAGE_ORDER_2M); - } + max_order = find_max_avail_order(end - cur); - /* Add all remaining 1G frames. */ - while (cur + PAGE_SIZE_1G <= end) { - add_frame(paddr_to_mfn(cur), PAGE_ORDER_1G); - cur += (PAGE_SIZE << PAGE_ORDER_1G); + /* Add all available max_order frames. */ + while (cur + ORDER_TO_SIZE(max_order) <= end) { + add_frame(paddr_to_mfn(cur), max_order); + cur += ORDER_TO_SIZE(max_order); } /* Add all remaining 2M frames. */ while (cur + PAGE_SIZE_2M <= end) { add_frame(paddr_to_mfn(cur), PAGE_ORDER_2M); - cur += (PAGE_SIZE << PAGE_ORDER_2M); + cur += ORDER_TO_SIZE(PAGE_ORDER_2M); } /* Add all remaining 4K frames. */ while (cur < end) { add_frame(paddr_to_mfn(cur), PAGE_ORDER_4K); - cur += (PAGE_SIZE << PAGE_ORDER_4K); + cur += ORDER_TO_SIZE(PAGE_ORDER_4K); } if (cur != end) { @@ -360,33 +370,61 @@ static inline frame_t *reserve_frame(frame_t *frame) { return frame; } -static inline frame_t *return_frame(frame_t *frame) { +static inline bool return_frame(frame_t *frame) { if (!is_frame_used(frame)) panic("PMM: trying to return unused frame: %p\n", frame); if (--frame->refcount == 0) { list_unlink(&frame->list); list_add(&frame->list, &free_frames[frame->order]); + return true; } - return frame; + return false; +} + +static frame_t *find_mfn_frame(list_head_t *frames, mfn_t mfn, unsigned int order) { + frame_t *frame; + + if (!has_frames(frames, order)) + return NULL; + + list_for_each_entry (frame, &frames[order], list) { + if (frame->mfn == mfn) + return frame; + } + + return NULL; +} + +static frame_t *find_larger_frame(list_head_t *frames, unsigned int order) { + while (++order <= MAX_PAGE_ORDER) { + frame_t *frame = get_first_frame(frames, order); + + if (frame) + return frame; + } + + return NULL; } /* Reserves and returns the first free frame fulfilling - * the condition specified by the callback + * the condition specified by the callback. + * This function does not split larger frames. */ frame_t *get_free_frames_cond(free_frames_cond_t cb) { - frame_t *frame; - spin_lock(&lock); for_each_order (order) { + frame_t *frame; + if (list_is_empty(&free_frames[order])) continue; list_for_each_entry (frame, &free_frames[order], list) { if (cb(frame)) { + reserve_frame(frame); spin_unlock(&lock); - return reserve_frame(frame); + return frame; } } } @@ -395,6 +433,68 @@ frame_t *get_free_frames_cond(free_frames_cond_t cb) { return NULL; } +static inline void relink_frame_to_order(frame_t *frame, unsigned int new_order) { + BUG_ON(new_order > MAX_PAGE_ORDER); + + list_unlink(&frame->list); + frames_count[frame->order]--; + + frame->order = new_order; + + list_add_tail(&frame->list, &free_frames[frame->order]); + frames_count[frame->order]++; +} + +static void split_frame(frame_t *frame) { + BUG_ON(!frame); + + if (opt_debug) { + printk("PMM: Splitting frame:\n"); + display_frame(frame); + } + + /* First sibling frame */ + relink_frame_to_order(frame, frame->order - 1); + + /* Create new frame entry for the second sibling frame */ + add_frame(NEXT_MFN(frame->mfn, frame->order), frame->order); +} + +static void merge_frames(frame_t *first) { + frame_t *second; + + BUG_ON(!first); + + if (FIRST_FRAME_SIBLING(first->mfn, first->order + 1)) { + mfn_t next_mfn = NEXT_MFN(first->mfn, first->order); + second = find_mfn_frame(free_frames, next_mfn, first->order); + } + else { + /* Second frame sibling */ + mfn_t prev_mfn = PREV_MFN(first->mfn, first->order); + second = first; + first = find_mfn_frame(free_frames, prev_mfn, first->order); + } + + if (!first || !second) + return; + + if (opt_debug) { + printk("PMM: Merging frames:\n"); + display_frame(first); + display_frame(second); + } + + /* Make the first sibling a higher order frame */ + relink_frame_to_order(first, first->order + 1); + + /* Destroy the second sibling frame */ + destroy_frame(second); + + /* Try to merge higher order frames */ + merge_frames(first); +} + frame_t *get_free_frames(unsigned int order) { frame_t *frame; @@ -402,13 +502,16 @@ frame_t *get_free_frames(unsigned int order) { return NULL; spin_lock(&lock); - if (list_is_empty(&free_frames[order])) { - /* FIXME: Add page split */ - spin_unlock(&lock); - return NULL; + while (list_is_empty(&free_frames[order])) { + frame = find_larger_frame(free_frames, order); + if (!frame) { + spin_unlock(&lock); + return NULL; + } + split_frame(frame); } - frame = reserve_frame(list_first_entry(&free_frames[order], frame_t, list)); + frame = reserve_frame(get_first_frame(free_frames, order)); spin_unlock(&lock); return frame; @@ -417,24 +520,18 @@ frame_t *get_free_frames(unsigned int order) { void put_free_frames(mfn_t mfn, unsigned int order) { frame_t *frame; - BUG_ON(mfn_invalid(mfn)); - - if (order > MAX_PAGE_ORDER) - return; + BUG_ON(mfn_invalid(mfn) || order > MAX_PAGE_ORDER); spin_lock(&lock); - list_for_each_entry (frame, &busy_frames[order], list) { - if (frame->mfn == mfn) { - /* FIXME: Maintain order wrt mfn value */ - /* FIXME: Add frame merge */ - return_frame(frame); - spin_unlock(&lock); - return; - } - } - spin_unlock(&lock); + frame = find_mfn_frame(busy_frames, mfn, order); + if (!frame) + panic("PMM: unable to find frame: %lx, order: %u among busy frames\n", mfn, + order); - panic("PMM: unable to find frame: %x in busy frames list\n"); + if (return_frame(frame)) + merge_frames(frame); + + spin_unlock(&lock); } void map_used_memory(void) {