[mm/page] multi-list page manager

[mm/page] page debugger
[libcpu/aarch64] hugepage support

Author: polarvid

components/drivers/virtio/virtio_net.c

@@ -11,6 +11,7 @@
 #include <rthw.h>
 #include <rtthread.h>
 #include <cpuport.h>
+#include <mm_aspace.h>
 
 #ifdef RT_USING_VIRTIO_NET
 
@@ -106,7 +107,7 @@ static struct pbuf *virtio_net_rx(rt_device_t dev)
 #ifdef RT_USING_SMP
             level = rt_spin_lock_irqsave(&virtio_dev->spinlock);
 #endif
-            rt_memcpy(p->payload, (void *)VIRTIO_PA2VA(queue_rx->desc[id].addr), len);
+            rt_memcpy(p->payload, (void *)queue_rx->desc[id].addr - PV_OFFSET, len);
 
             queue_rx->used_idx++;
 

components/lwp/arch/aarch64/cortex-a/lwp_arch.c

@@ -26,7 +26,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
 {
     size_t *mmu_table;
 
-    mmu_table = (size_t *)rt_pages_alloc(0);
+    mmu_table = (size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
     if (!mmu_table)
     {
         return -RT_ENOMEM;

components/lwp/arch/risc-v/rv64/lwp_arch.c

@@ -91,7 +91,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
 {
     rt_ubase_t *mmu_table;
 
-    mmu_table = (rt_ubase_t *)rt_pages_alloc(0);
+    mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
     if (!mmu_table)
     {
         return -RT_ENOMEM;

components/lwp/arch/x86/i386/lwp_arch.c

@@ -82,7 +82,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
 {
     rt_size_t *mmu_table;
 
-    mmu_table = (rt_size_t *)rt_pages_alloc(0);
+    mmu_table = (rt_size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
     if (!mmu_table)
     {
         return -1;

components/lwp/lwp.c

@@ -9,6 +9,7 @@
  * 2018-11-02     heyuanjie    fix complie error in iar
  * 2021-02-03     lizhirui     add 64-bit arch support and riscv64 arch support
  * 2021-08-26     linzhenxing  add lwp_setcwd\lwp_getcwd
+ * 2023-02-20     wangxiaoyao  inv icache before new app startup
  */
 
 #include <rthw.h>
@@ -1097,6 +1098,18 @@ static void _lwp_thread_entry(void *parameter)
         icache_invalid_all();
     }
 
+    /**
+     * without ASID support, it will be a special case when trying to run application
+     * and exit multiple times and a same page frame allocated to it bound to
+     * different text segment. Then we are in a situation where icache contains
+     * out-of-dated data and must be handle by the running core itself.
+     * with ASID support, this should be a rare case that ASID & page frame both
+     * identical to previous running application.
+     *
+     * For a new application loaded into memory, icache are seen as empty. And there
+     * should be nothing in the icache entry to match. So this icache invalidation
+     * operation should have barely influence.
+     */
     rt_hw_icache_invalidate_all();
 
 #ifdef ARCH_MM_MMU

components/lwp/lwp_shm.c

@@ -6,6 +6,7 @@
  * Change Logs:
  * Date           Author       Notes
  * 2019-10-12     Jesven       first version
+ * 2023-02-20     wangxiaoyao  adapt to mm
  */
 #include <rthw.h>
 #include <rtthread.h>
@@ -17,8 +18,6 @@
 
 #include <lwp_user_mm.h>
 #include <mmu.h>
-#include <mm_aspace.h>
-#include <mm_flag.h>
 
 /* the kernel structure to represent a share-memory */
 struct lwp_shm_struct
@@ -140,7 +139,7 @@ static int _lwp_shmget(size_t key, size_t size, int create)
 
         /* allocate pages up to 2's exponent to cover the required size */
         bit = rt_page_bits(size);
-        page_addr = rt_pages_alloc(bit);           /* virtual address */
+        page_addr = rt_pages_alloc_ext(bit, PAGE_ANY_AVAILABLE);   /* virtual address */
         if (!page_addr)
         {
             goto err;

components/lwp/lwp_syscall.c

@@ -1861,7 +1861,7 @@ static char *_insert_args(int new_argc, char *new_argv[], struct lwp_args_info *
     {
         goto quit;
     }
-    page = rt_pages_alloc(0); /* 1 page */
+    page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
     if (!page)
     {
         goto quit;
@@ -2065,7 +2065,7 @@ int load_ldso(struct rt_lwp *lwp, char *exec_name, char *const argv[], char *con
         }
     }
 
-    page = rt_pages_alloc(0); /* 1 page */
+    page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
     if (!page)
     {
         SET_ERRNO(ENOMEM);
@@ -2252,7 +2252,7 @@ sysret_t sys_execve(const char *path, char *const argv[], char *const envp[])
         SET_ERRNO(EINVAL);
         goto quit;
     }
-    page = rt_pages_alloc(0); /* 1 page */
+    page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
     if (!page)
     {
         SET_ERRNO(ENOMEM);

components/lwp/lwp_user_mm.c

@@ -10,6 +10,7 @@
  * 2021-02-12     lizhirui     add 64-bit support for lwp_brk
  * 2021-02-19     lizhirui     add riscv64 support for lwp_user_accessable and lwp_get_from_user
  * 2021-06-07     lizhirui     modify user space bound check
+ * 2022-12-25     wangxiaoyao  adapt to new mm
  */
 
 #include <rtthread.h>
@@ -130,7 +131,7 @@ static void _user_do_page_fault(struct rt_varea *varea,
 
             if (!(varea->flag & MMF_TEXT))
             {
-                void *cp = rt_pages_alloc(0);
+                void *cp = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
                 if (cp)
                 {
                     memcpy(cp, vaddr, ARCH_PAGE_SIZE);

components/mm/mm_aspace.c

@@ -134,24 +134,16 @@ static int _do_named_map(rt_aspace_t aspace, void *vaddr, rt_size_t length,
     int err = RT_EOK;
 
     /* it's ensured by caller that (void*)end will not overflow */
-    void *end = vaddr + length;
     void *phyaddr = (void *)(offset << MM_PAGE_SHIFT);
-    while (vaddr != end)
+
+    void *ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, length, attr);
+    if (ret == RT_NULL)
     {
-        /* TODO try to map with huge TLB, when flag & HUGEPAGE */
-        rt_size_t pgsz = ARCH_PAGE_SIZE;
-        void *ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, pgsz, attr);
-        if (ret == RT_NULL)
-        {
-            err = -RT_ERROR;
-            break;
-        }
-        vaddr += pgsz;
-        phyaddr += pgsz;
+        err = -RT_ERROR;
     }
 
     if (err == RT_EOK)
-        rt_hw_tlb_invalidate_range(aspace, end - length, length, ARCH_PAGE_SIZE);
+        rt_hw_tlb_invalidate_range(aspace, vaddr, length, ARCH_PAGE_SIZE);
 
     return err;
 }
@@ -164,7 +156,7 @@ rt_inline void _do_page_fault(struct rt_aspace_fault_msg *msg, rt_size_t off,
     msg->fault_vaddr = vaddr;
     msg->fault_op = MM_FAULT_OP_READ;
     msg->fault_type = MM_FAULT_TYPE_PAGE_FAULT;
-    msg->response.status = -1;
+    msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE;
     msg->response.vaddr = 0;
     msg->response.size = 0;
 
@@ -180,9 +172,9 @@ int _varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg)
          * the page returned by handler is not checked
          * cause no much assumption can make on it
          */
-        void *store = msg->response.vaddr;
+        char *store = msg->response.vaddr;
         rt_size_t store_sz = msg->response.size;
-        if (msg->fault_vaddr + store_sz > varea->start + varea->size)
+        if ((char *)msg->fault_vaddr + store_sz > (char *)varea->start + varea->size)
         {
             LOG_W("%s: too much (0x%lx) of buffer on vaddr %p is provided",
                     __func__, store_sz, msg->fault_vaddr);
@@ -232,9 +224,9 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start,
     int err = RT_EOK;
 
     /* it's ensured by caller that start & size ara page-aligned */
-    void *end = start + size;
-    void *vaddr = start;
-    rt_size_t off = varea->offset + ((start - varea->start) >> ARCH_PAGE_SHIFT);
+    char *end = (char *)start + size;
+    char *vaddr = start;
+    rt_size_t off = varea->offset + ((vaddr - (char *)varea->start) >> ARCH_PAGE_SHIFT);
 
     while (vaddr != end)
     {
@@ -243,8 +235,10 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start,
         _do_page_fault(&msg, off, vaddr, varea->mem_obj, varea);
 
         if (_varea_map_with_msg(varea, &msg))
+        {
+            err = -RT_ENOMEM;
             break;
-
+        }
         /**
          * It's hard to identify the mapping pattern on a customized handler
          * So we terminate the prefetch process on that case
@@ -386,7 +380,7 @@ rt_varea_t _varea_create(void *start, rt_size_t size)
 }
 
 #define _IS_OVERFLOW(start, length) ((length) > (0ul - (uintptr_t)(start)))
-#define _IS_OVERSIZE(start, length, limit_s, limit_sz) (((length) + (rt_size_t)((start) - (limit_start))) > (limit_size))
+#define _IS_OVERSIZE(start, length, limit_s, limit_sz) (((length) + (rt_size_t)((char *)(start) - (char *)(limit_start))) > (limit_size))
 
 static inline int _not_in_range(void *start, rt_size_t length,
                                 void *limit_start, rt_size_t limit_size)
@@ -642,7 +636,7 @@ int rt_aspace_unmap(rt_aspace_t aspace, void *addr)
     if (_not_in_range(addr, 1, aspace->start, aspace->size))
     {
         LOG_I("%s: %lx not in range of aspace[%lx:%lx]", __func__, addr,
-              aspace->start, aspace->start + aspace->size);
+              aspace->start, (char *)aspace->start + aspace->size);
         return -RT_EINVAL;
     }
 
@@ -658,7 +652,7 @@ static inline void *_lower(void *a, void *b)
 
 static inline void *_align(void *va, rt_ubase_t align_mask)
 {
-    return (void *)((rt_ubase_t)(va + ~align_mask) & align_mask);
+    return (void *)((rt_ubase_t)((char *)va + ~align_mask) & align_mask);
 }
 
 static void *_ascending_search(rt_varea_t varea, rt_size_t req_size,
@@ -667,17 +661,17 @@ static void *_ascending_search(rt_varea_t varea, rt_size_t req_size,
     void *ret = RT_NULL;
     while (varea && varea->start < limit.end)
     {
-        void *candidate = varea->start + varea->size;
+        char *candidate = (char *)varea->start + varea->size;
         candidate = _align(candidate, align_mask);
 
-        if (candidate > limit.end || limit.end - candidate + 1 < req_size)
+        if (candidate > (char *)limit.end || (char *)limit.end - candidate + 1 < req_size)
             break;
 
         rt_varea_t nx_va = ASPACE_VAREA_NEXT(varea);
         if (nx_va)
         {
             rt_size_t gap_size =
-                _lower(limit.end, nx_va->start - 1) - candidate + 1;
+                (char *)_lower(limit.end, (char *)nx_va->start - 1) - candidate + 1;
             if (gap_size >= req_size)
             {
                 ret = candidate;
@@ -703,15 +697,15 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
     rt_varea_t varea = _aspace_bst_search_exceed(aspace, limit.start);
     if (varea)
     {
-        void *candidate = _align(limit.start, align_mask);
-        rt_size_t gap_size = varea->start - candidate;
+        char *candidate = _align(limit.start, align_mask);
+        rt_size_t gap_size = (char *)varea->start - candidate;
         if (gap_size >= req_size)
         {
             rt_varea_t former = _aspace_bst_search(aspace, limit.start);
             if (former)
             {
-                candidate = _align(former->start + former->size, align_mask);
-                gap_size = varea->start - candidate;
+                candidate = _align((char *)former->start + former->size, align_mask);
+                gap_size = (char *)varea->start - candidate;
 
                 if (gap_size >= req_size)
                     va = candidate;
@@ -730,12 +724,12 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
     }
     else
     {
-        void *candidate;
+        char *candidate;
         rt_size_t gap_size;
 
         candidate = limit.start;
         candidate = _align(candidate, align_mask);
-        gap_size = limit.end - candidate + 1;
+        gap_size = (char *)limit.end - candidate + 1;
 
         if (gap_size >= req_size)
             va = candidate;
@@ -750,7 +744,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
 {
     rt_varea_t varea = NULL;
     void *va = RT_NULL;
-    struct _mm_range limit = {limit_start, limit_start + limit_size - 1};
+    struct _mm_range limit = {limit_start, (char *)limit_start + limit_size - 1};
 
     rt_ubase_t align_mask = ~0ul;
     if (flags & MMF_REQUEST_ALIGN)
@@ -762,7 +756,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
     {
         /* if prefer and free, just return the prefer region */
         prefer = _align(prefer, align_mask);
-        struct _mm_range range = {prefer, prefer + req_size - 1};
+        struct _mm_range range = {prefer, (char *)prefer + req_size - 1};
         varea = _aspace_bst_search_overlap(aspace, range);
 
         if (!varea)
@@ -780,7 +774,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
             if (va == RT_NULL)
             {
                 /* rewind to first range */
-                limit.end = varea->start - 1;
+                limit.end = (char *)varea->start - 1;
                 va = _find_head_and_asc_search(aspace, req_size, align_mask,
                                                limit);
             }
@@ -798,7 +792,7 @@ int rt_aspace_load_page(rt_aspace_t aspace, void *addr, rt_size_t npage)
 {
     int err = RT_EOK;
     rt_varea_t varea;
-    void *end = addr + (npage << ARCH_PAGE_SHIFT);
+    char *end = (char *)addr + (npage << ARCH_PAGE_SHIFT);
 
     WR_LOCK(aspace);
     varea = _aspace_bst_search(aspace, addr);
@@ -809,7 +803,7 @@ int rt_aspace_load_page(rt_aspace_t aspace, void *addr, rt_size_t npage)
         LOG_W("%s: varea not exist", __func__);
         err = -RT_ENOENT;
     }
-    else if (addr >= end || (rt_size_t)addr & ARCH_PAGE_MASK ||
+    else if ((char *)addr >= end || (rt_size_t)addr & ARCH_PAGE_MASK ||
              _not_in_range(addr, npage << ARCH_PAGE_SHIFT, varea->start,
                            varea->size))
     {
@@ -938,12 +932,12 @@ static int _dump(rt_varea_t varea, void *arg)
 {
     if (varea->mem_obj && varea->mem_obj->get_name)
     {
-        rt_kprintf("[%p - %p] %s\n", varea->start, varea->start + varea->size,
+        rt_kprintf("[%p - %p] %s\n", varea->start, (char *)varea->start + varea->size,
                    varea->mem_obj->get_name(varea));
     }
     else
     {
-        rt_kprintf("[%p - %p] phy-map\n", varea->start, varea->start + varea->size);
+        rt_kprintf("[%p - %p] phy-map\n", varea->start, (char *)varea->start + varea->size);
         rt_kprintf("\t\\_ paddr = %p\n",  varea->offset << MM_PAGE_SHIFT);
     }
     return 0;

components/mm/mm_fault.c

@@ -28,7 +28,7 @@
 
 static int _fetch_page(rt_varea_t varea, struct rt_aspace_fault_msg *msg)
 {
-    int err;
+    int err = UNRECOVERABLE;
     msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE;
     msg->response.vaddr = 0;
     msg->response.size = 0;
@@ -104,7 +104,7 @@ int rt_aspace_fault_try_fix(struct rt_aspace_fault_msg *msg)
         if (varea)
         {
             void *pa = rt_hw_mmu_v2p(aspace, msg->fault_vaddr);
-            msg->off = (msg->fault_vaddr - varea->start) >> ARCH_PAGE_SHIFT;
+            msg->off = ((char *)msg->fault_vaddr - (char *)varea->start) >> ARCH_PAGE_SHIFT;
 
             /* permission checked by fault op */
             switch (msg->fault_op)

components/mm/mm_object.c

@@ -70,7 +70,7 @@ void rt_varea_pgmgr_pop(rt_varea_t varea, void *vaddr, rt_size_t size)
 static void on_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg)
 {
     void *page;
-    page = rt_pages_alloc(0);
+    page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
 
     if (!page)
     {