From 0407dbfe10024a188a6b9cf58a352e679b50bbdd Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 28 May 2023 21:00:57 +0200
Subject: [PATCH 01/16] fix: Pass callback to toggle_directory to notify about
 async results

---
 lua/neo-tree/sources/filesystem/init.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index df85b9c8..8157038d 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -393,7 +393,7 @@ M.setup = function(config, global_config)
 end
 
 ---Expands or collapses the current node.
-M.toggle_directory = function(state, node, path_to_reveal, skip_redraw, recursive)
+M.toggle_directory = function(state, node, path_to_reveal, skip_redraw, recursive, callback)
   local tree = state.tree
   if not node then
     node = tree:get_node()
@@ -406,7 +406,7 @@ M.toggle_directory = function(state, node, path_to_reveal, skip_redraw, recursiv
     local id = node:get_id()
     state.explicitly_opened_directories[id] = true
     renderer.position.set(state, nil)
-    fs_scan.get_items(state, id, path_to_reveal, nil, false, recursive)
+    fs_scan.get_items(state, id, path_to_reveal, callback, false, recursive)
   elseif node:has_children() then
     local updated = false
     if node:is_expanded() then

From df452165bb0d7ff5c6eebac43a1835d21e213ddf Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 28 May 2023 21:01:43 +0200
Subject: [PATCH 02/16] Extract methods for better readability

---
 .../sources/filesystem/lib/fs_scan.lua        | 77 +++++++++++--------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index da35e039..909f8621 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -362,35 +362,9 @@ M.get_items_async = function(state, parent_id, path_to_reveal, callback)
   M.get_items(state, parent_id, path_to_reveal, callback, true)
 end
 
-M.get_items = function(state, parent_id, path_to_reveal, callback, async, recursive)
-  if state.async_directory_scan == "always" then
-    async = true
-  elseif state.async_directory_scan == "never" then
-    async = false
-  elseif type(async) == "nil" then
-    async = (state.async_directory_scan == "auto") or state.async_directory_scan
-  end
-
-  if not parent_id then
-    M.stop_watchers(state)
-  end
-  local context = file_items.create_context()
-  context.state = state
-  context.parent_id = parent_id
-  context.path_to_reveal = path_to_reveal
-  context.recursive = recursive
-  context.callback = callback
-
-  -- Create root folder
-  local root = file_items.create_item(context, parent_id or state.path, "directory")
-  root.name = vim.fn.fnamemodify(root.path, ":~")
-  root.loaded = true
-  root.search_pattern = state.search_pattern
-  context.root = root
-  context.folders[root.path] = root
-  state.default_expanded_nodes = state.force_open_folders or { state.path }
-
-  if state.search_pattern then
+local handle_search_pattern = function (context)
+    local state = context.state
+    local root = context.root
     local search_opts = {
       filtered_items = state.filtered_items,
       find_command = state.find_command,
@@ -417,9 +391,12 @@ M.get_items = function(state, parent_id, path_to_reveal, callback, async, recurs
       -- Use the external command because the plenary search is slow
       filter_external.find_files(search_opts)
     end
-  else
-    -- In the case of a refresh or navigating up, we need to make sure that all
-    -- open folders are loaded.
+end
+
+local handle_refresh_or_up = function (context, async)
+    local parent_id = context.parent_id
+    local path_to_reveal = context.path_to_reveal
+    local state = context.state
     local path = parent_id or state.path
     context.paths_to_load = {}
     if parent_id == nil then
@@ -473,6 +450,42 @@ M.get_items = function(state, parent_id, path_to_reveal, callback, async, recurs
     else
       sync_scan(context, path)
     end
+end
+
+M.get_items = function(state, parent_id, path_to_reveal, callback, async, recursive) 
+  if state.async_directory_scan == "always" then
+    async = true
+  elseif state.async_directory_scan == "never" then
+    async = false
+  elseif type(async) == "nil" then
+    async = (state.async_directory_scan == "auto") or state.async_directory_scan
+  end
+
+  if not parent_id then
+    M.stop_watchers(state)
+  end
+  local context = file_items.create_context()
+  context.state = state
+  context.parent_id = parent_id
+  context.path_to_reveal = path_to_reveal
+  context.recursive = recursive
+  context.callback = callback
+
+  -- Create root folder
+  local root = file_items.create_item(context, parent_id or state.path, "directory")
+  root.name = vim.fn.fnamemodify(root.path, ":~")
+  root.loaded = true
+  root.search_pattern = state.search_pattern
+  context.root = root
+  context.folders[root.path] = root
+  state.default_expanded_nodes = state.force_open_folders or { state.path }
+
+  if state.search_pattern then
+    handle_search_pattern(context)
+  else
+    -- In the case of a refresh or navigating up, we need to make sure that all
+    -- open folders are loaded.
+    handle_refresh_or_up(context, async)
   end
 end
 

From f3ac4a911fc12f190f20fe70065b92f69693dc26 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 28 May 2023 21:10:03 +0200
Subject: [PATCH 03/16] fix: Handle correctly recursive traversals when deep
 strategy used

---
 lua/neo-tree/sources/filesystem/lib/fs_scan.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index 909f8621..0da62ff4 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -190,7 +190,8 @@ local function scan_dir_sync(context, path)
       if
         grandchild_nodes == nil
         or #grandchild_nodes == 0
-        or #grandchild_nodes == 1 and grandchild_nodes[1].type == "directory"
+        or (#grandchild_nodes == 1 and grandchild_nodes[1].type == "directory")
+        or context.recursive
       then
         scan_dir_sync(context, child.path)
       end

From fea2e5bbb8179eaf41a043df5bb4d6fa0ec88ec0 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sat, 3 Jun 2023 11:39:07 +0200
Subject: [PATCH 04/16] Rewrite recursive expand to use plenary.async

---
 lua/neo-tree/git/ignored.lua                  |   2 +-
 lua/neo-tree/sources/common/commands.lua      |  46 +++----
 lua/neo-tree/sources/filesystem/commands.lua  |   5 +-
 lua/neo-tree/sources/filesystem/init.lua      |  73 +++++++++++
 .../sources/filesystem/lib/fs_scan.lua        | 123 +++++++++++++++---
 5 files changed, 197 insertions(+), 52 deletions(-)

diff --git a/lua/neo-tree/git/ignored.lua b/lua/neo-tree/git/ignored.lua
index 2898cf4f..97af6a14 100644
--- a/lua/neo-tree/git/ignored.lua
+++ b/lua/neo-tree/git/ignored.lua
@@ -118,7 +118,7 @@ M.mark_ignored = function(state, items, callback)
         on_exit = function(self, code, _)
           local result
           if code ~= 0 then
-            log.debug("Failed to load ignored files for", state.path, ":", self:stderr_result())
+            log.debug("Failed to load ignored files for", folder, ":", self:stderr_result())
             result = {}
           else
             result = self:result()
diff --git a/lua/neo-tree/sources/common/commands.lua b/lua/neo-tree/sources/common/commands.lua
index ab948e4a..b8230f16 100644
--- a/lua/neo-tree/sources/common/commands.lua
+++ b/lua/neo-tree/sources/common/commands.lua
@@ -2,6 +2,7 @@
 
 local vim = vim
 local fs_actions = require("neo-tree.sources.filesystem.lib.fs_actions")
+local fs = require("neo-tree.sources.filesystem")
 local utils = require("neo-tree.utils")
 local renderer = require("neo-tree.ui.renderer")
 local events = require("neo-tree.events")
@@ -10,6 +11,7 @@ local popups = require("neo-tree.ui.popups")
 local log = require("neo-tree.log")
 local help = require("neo-tree.sources.common.help")
 local Preview = require("neo-tree.sources.common.preview")
+local async = require("plenary.async")
 
 ---Gets the node parent folder
 ---@param state table to look for nodes
@@ -108,35 +110,23 @@ M.add_directory = function(state, callback)
   fs_actions.create_directory(in_directory, callback, using_root_directory)
 end
 
-M.expand_all_nodes = function(state, toggle_directory)
-  if toggle_directory == nil then
-    toggle_directory = function(_, node)
-      node:expand()
-    end
-  end
-  --state.explicitly_opened_directories = state.explicitly_opened_directories or {}
-
-  local expand_node
-  expand_node = function(node)
-    local id = node:get_id()
-    if node.type == "directory" and not node:is_expanded() then
-      toggle_directory(state, node)
-      node = state.tree:get_node(id)
-    end
-    local children = state.tree:get_nodes(id)
-    if children then
-      for _, child in ipairs(children) do
-        if child.type == "directory" then
-          expand_node(child)
-        end
+---Expand all nodes
+---@param state table The state of the source
+---@param node table A node to expand
+M.expand_all_nodes = function(state, node)
+  log.debug("Expanding all nodes under " .. node:get_id())
+  local task = function ()
+    fs.expand_directory(state, node)
+  end
+  async.run(
+      task,
+      function ()
+        log.debug("All nodes expanded - redrawing")
+        vim.schedule_wrap(function()
+          renderer.redraw(state)
+        end)
       end
-    end
-  end
-
-  for _, node in ipairs(state.tree:get_nodes()) do
-    expand_node(node)
-  end
-  renderer.redraw(state)
+    )
 end
 
 M.close_node = function(state, callback)
diff --git a/lua/neo-tree/sources/filesystem/commands.lua b/lua/neo-tree/sources/filesystem/commands.lua
index 160ca604..00ad89e3 100644
--- a/lua/neo-tree/sources/filesystem/commands.lua
+++ b/lua/neo-tree/sources/filesystem/commands.lua
@@ -68,10 +68,7 @@ M.delete_visual = function(state, selected_nodes)
 end
 
 M.expand_all_nodes = function(state)
-  local toggle_dir_no_redraw = function(_state, node)
-    fs.toggle_directory(_state, node, nil, true, true)
-  end
-  cc.expand_all_nodes(state, toggle_dir_no_redraw)
+  cc.expand_all_nodes(state, state.tree:get_node(state.path))
 end
 
 ---Shows the filter input, which will filter the tree.
diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index 8157038d..95c0c91b 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -10,6 +10,7 @@ local log = require("neo-tree.log")
 local manager = require("neo-tree.sources.manager")
 local git = require("neo-tree.git")
 local glob = require("neo-tree.sources.filesystem.lib.globtopattern")
+local async = require("plenary.async")
 
 local M = {
   name = "filesystem",
@@ -428,4 +429,76 @@ M.toggle_directory = function(state, node, path_to_reveal, skip_redraw, recursiv
   end
 end
 
+--- Recursively expand all loaded nodes under the given node
+--- returns table with all discovered nodes that need to be loaded 
+---@param node table a node to expand
+---@param state table current state of the source
+---@return table discovered nodes that need to be loaded
+local function expand_loaded(node, state)
+    local function rec(_node, to_load)
+      if _node.loaded == false then
+        table.insert(to_load, _node)
+      else
+        if not _node:is_expanded() then
+          _node:expand()
+          state.explicitly_opened_directories[_node:get_id()] = true
+        end
+        local children = state.tree:get_nodes(_node:get_id())
+        log.debug("Expanding childrens of " .. _node:get_id())
+        for _, child in ipairs(children) do
+          if child.type == "directory" then
+             rec(child, to_load)
+          else
+            log.trace("Child: " .. child.name .. " is not a directory, skipping")
+          end
+        end
+      end
+    end
+
+    local to_load = {}
+    rec(node, to_load)
+    return to_load
+end
+
+--- Recursively expands all nodes under the given node
+--- loading nodes if necessary.
+--- asyn method
+---@param node table a node to expand
+---@param state table current state of the source
+local function expand_and_load(node, state)
+    local function rec(to_load, progress)
+      local to_load_current = expand_loaded(node, state)
+      for _,v in ipairs(to_load_current) do
+        table.insert(to_load, v)
+      end
+      if progress <= #to_load then
+        M.expand_directory(state, to_load[progress])
+        rec(to_load, progress + 1)
+      end
+    end
+    rec({}, 1)
+end
+
+---Expands given node recursively loading all descendant nodes if needed
+---async method
+---@param state table current state of the source
+---@param node table a node to expand
+M.expand_directory = function(state, node)
+  log.debug("Expanding directory " .. node:get_id())
+  if node.type ~= "directory" then
+    return
+  end
+  state.explicitly_opened_directories = state.explicitly_opened_directories or {}
+  if node.loaded == false then
+    local id = node:get_id()
+    state.explicitly_opened_directories[id] = true
+    renderer.position.set(state, nil)
+    fs_scan.get_dir_items_async(state, id, true)
+    -- ignore results as we know here that all descendant nodes have been already loaded
+    expand_loaded(node ,state)
+  else
+    expand_and_load(node, state)
+  end
+end
+
 return M
diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index 0da62ff4..fe3cfbae 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -106,6 +106,26 @@ local render_context = function(context)
   context = nil
 end
 
+local job_complete_async = function(context)
+  local state = context.state
+  local parent_id = context.parent_id
+  if #context.all_items == 0 then
+    log.info("No items, skipping git ignored/status lookups")
+  elseif state.filtered_items.hide_gitignored or state.enable_git_status then
+    local mark_ignored_async = async.wrap(function (_state, _all_items, _callback)
+      git.mark_ignored(_state, _all_items, _callback)
+    end, 3)
+    local all_items = mark_ignored_async(state, context.all_items)
+
+    if parent_id then
+      vim.list_extend(state.git_ignored, all_items)
+    else
+      state.git_ignored = all_items
+    end
+  end
+  return context
+end
+
 local job_complete = function(context)
   local state = context.state
   local parent_id = context.parent_id
@@ -135,8 +155,8 @@ local job_complete = function(context)
         state.git_ignored = all_items
       end
     end
+    render_context(context)
   end
-  render_context(context)
 end
 
 local function create_node(context, node)
@@ -199,26 +219,34 @@ local function scan_dir_sync(context, path)
   end
 end
 
-local function scan_dir_async(context, path, callback)
-  get_children_async(path, function(children)
-    for _, child in ipairs(children) do
-      create_node(context, child)
-      if child.type == "directory" then
-        local grandchild_nodes = get_children_sync(child.path)
-        if
-          grandchild_nodes == nil
-          or #grandchild_nodes == 0
-          or #grandchild_nodes == 1 and grandchild_nodes[1].type == "directory"
-        then
-          scan_dir_sync(context, child.path)
-        end
+local function scan_dir_async(context, path)
+  log.debug("scan_dir_async - start " .. path)
+
+  local get_children = async.wrap(function (callback)
+    return get_children_async(path, callback)
+  end, 1)
+
+  local children = get_children()
+  for _, child in ipairs(children) do
+    create_node(context, child)
+    if child.type == "directory" then
+      local grandchild_nodes = get_children_sync(child.path)
+      if
+        grandchild_nodes == nil
+        or #grandchild_nodes == 0
+        or #grandchild_nodes == 1 and grandchild_nodes[1].type == "directory"
+      then
+        scan_dir_sync(context, child.path)
       end
     end
-    process_node(context, path)
-    callback(path)
-  end)
+  end
+
+  log.debug("scan_dir_async - finish " .. path)
+  process_node(context, path)
+  return path
 end
 
+
 -- async_scan scans all the directories in context.paths_to_load
 -- and adds them as items to render in the UI.
 local function async_scan(context, path)
@@ -229,7 +257,9 @@ local function async_scan(context, path)
     local scan_tasks = {}
     for _, p in ipairs(context.paths_to_load) do
       local scan_task = async.wrap(function(callback)
-        scan_dir_async(context, p, callback)
+        async.run(function ()
+            scan_dir_async(context, p)
+        end, callback)
       end, 1)
       table.insert(scan_tasks, scan_task)
     end
@@ -471,7 +501,6 @@ M.get_items = function(state, parent_id, path_to_reveal, callback, async, recurs
   context.path_to_reveal = path_to_reveal
   context.recursive = recursive
   context.callback = callback
-
   -- Create root folder
   local root = file_items.create_item(context, parent_id or state.path, "directory")
   root.name = vim.fn.fnamemodify(root.path, ":~")
@@ -490,6 +519,62 @@ M.get_items = function(state, parent_id, path_to_reveal, callback, async, recurs
   end
 end
 
+-- async method
+M.get_dir_items_async = function(state, parent_id, recursive)
+  local context = file_items.create_context()
+  context.state = state
+  context.parent_id = parent_id
+  context.path_to_reveal = nil
+  context.recursive = recursive
+  context.callback = nil
+  context.paths_to_load = {}
+
+  -- Create root folder
+  local root = file_items.create_item(context, parent_id or state.path, "directory")
+  root.name = vim.fn.fnamemodify(root.path, ":~")
+  root.loaded = true
+  root.search_pattern = state.search_pattern
+  context.root = root
+  context.folders[root.path] = root
+  state.default_expanded_nodes = state.force_open_folders or { state.path }
+
+  local filtered_items = state.filtered_items or {}
+  context.is_a_never_show_file = function(fname)
+    if fname then
+      local _, name = utils.split_path(fname)
+      if name then
+        if filtered_items.never_show and filtered_items.never_show[name] then
+          return true
+        end
+        if utils.is_filtered_by_pattern(filtered_items.never_show_by_pattern, fname, name) then
+          return true
+        end
+      end
+    end
+    return false
+  end
+  table.insert(context.paths_to_load, parent_id)
+ 
+  local scan_tasks = {}
+  for _, p in ipairs(context.paths_to_load) do
+     local scan_task = function ()
+       scan_dir_async(context, p)
+     end
+     table.insert(scan_tasks, scan_task)
+  end
+  async.util.join(scan_tasks)
+
+  job_complete_async(context)
+
+  local finalize =  async.wrap(function (_context, _callback)
+    vim.schedule(function ()
+        render_context(_context)
+        _callback()
+    end)
+  end, 2)
+  finalize(context)
+end
+
 M.stop_watchers = function(state)
   if state.use_libuv_file_watcher and state.tree then
     -- We are loaded a new root or refreshing, unwatch any folders that were

From e534a342257ec9adfbafdfe644aae86c135a0d9a Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sat, 3 Jun 2023 23:03:06 +0200
Subject: [PATCH 05/16] Recursively load files from filesystem

---
 lua/neo-tree/sources/filesystem/init.lua        | 9 +++------
 lua/neo-tree/sources/filesystem/lib/fs_scan.lua | 4 ++--
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index 95c0c91b..280dea78 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -462,7 +462,7 @@ end
 
 --- Recursively expands all nodes under the given node
 --- loading nodes if necessary.
---- asyn method
+--- async method
 ---@param node table a node to expand
 ---@param state table current state of the source
 local function expand_and_load(node, state)
@@ -492,13 +492,10 @@ M.expand_directory = function(state, node)
   if node.loaded == false then
     local id = node:get_id()
     state.explicitly_opened_directories[id] = true
-    renderer.position.set(state, nil)
+    renderer.position.set(state, nil) -- todo should not be called recursively
     fs_scan.get_dir_items_async(state, id, true)
-    -- ignore results as we know here that all descendant nodes have been already loaded
-    expand_loaded(node ,state)
-  else
-    expand_and_load(node, state)
   end
+  expand_and_load(node, state)
 end
 
 return M
diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index fe3cfbae..936661fc 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -236,13 +236,13 @@ local function scan_dir_async(context, path)
         or #grandchild_nodes == 0
         or #grandchild_nodes == 1 and grandchild_nodes[1].type == "directory"
       then
-        scan_dir_sync(context, child.path)
+        scan_dir_async(context, child.path)
       end
     end
   end
 
-  log.debug("scan_dir_async - finish " .. path)
   process_node(context, path)
+  log.debug("scan_dir_async - finish " .. path)
   return path
 end
 

From e448258ceac523616b78320a5ea7a5f45efdfac6 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sat, 3 Jun 2023 23:27:49 +0200
Subject: [PATCH 06/16] Remove unused variable

---
 lua/neo-tree/sources/filesystem/init.lua | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index 280dea78..7efc6e38 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -10,7 +10,6 @@ local log = require("neo-tree.log")
 local manager = require("neo-tree.sources.manager")
 local git = require("neo-tree.git")
 local glob = require("neo-tree.sources.filesystem.lib.globtopattern")
-local async = require("plenary.async")
 
 local M = {
   name = "filesystem",

From 714ea7bdb7d864d7031d04b6182c1904b9ef56b4 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 4 Jun 2023 12:37:34 +0200
Subject: [PATCH 07/16] Change is_loaded to loaded

---
 lua/neo-tree/ui/renderer.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lua/neo-tree/ui/renderer.lua b/lua/neo-tree/ui/renderer.lua
index 2072b564..a60b3c70 100644
--- a/lua/neo-tree/ui/renderer.lua
+++ b/lua/neo-tree/ui/renderer.lua
@@ -1198,7 +1198,7 @@ M.show_nodes = function(sourceItems, state, parentId, callback)
             if node.id == parent.id then
               item.name = parent.name .. utils.path_separator .. item.name
               item.level = level - 1
-              item.is_loaded = utils.truthy(item.children)
+              item.loaded = utils.truthy(item.children)
               siblings[i] = NuiTree.Node(item, item.children)
               break
             end

From b5a72242925026dba8de79b279ecf71e2c4fd434 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 4 Jun 2023 14:06:55 +0200
Subject: [PATCH 08/16] Fix using multiple fs calls to scan the same directory

---
 lua/neo-tree/sources/common/commands.lua        | 4 +---
 lua/neo-tree/sources/filesystem/init.lua        | 4 +++-
 lua/neo-tree/sources/filesystem/lib/fs_scan.lua | 6 ++++--
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/lua/neo-tree/sources/common/commands.lua b/lua/neo-tree/sources/common/commands.lua
index b8230f16..0f82d0cd 100644
--- a/lua/neo-tree/sources/common/commands.lua
+++ b/lua/neo-tree/sources/common/commands.lua
@@ -122,9 +122,7 @@ M.expand_all_nodes = function(state, node)
       task,
       function ()
         log.debug("All nodes expanded - redrawing")
-        vim.schedule_wrap(function()
-          renderer.redraw(state)
-        end)
+        renderer.redraw(state)
       end
     )
 end
diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index 7efc6e38..82fd50af 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -493,8 +493,10 @@ M.expand_directory = function(state, node)
     state.explicitly_opened_directories[id] = true
     renderer.position.set(state, nil) -- todo should not be called recursively
     fs_scan.get_dir_items_async(state, id, true)
+    expand_loaded(node, state)
+  else
+    expand_and_load(node, state)
   end
-  expand_and_load(node, state)
 end
 
 return M
diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index 936661fc..616d5e62 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -219,6 +219,7 @@ local function scan_dir_sync(context, path)
   end
 end
 
+--- async method
 local function scan_dir_async(context, path)
   log.debug("scan_dir_async - start " .. path)
 
@@ -234,7 +235,8 @@ local function scan_dir_async(context, path)
       if
         grandchild_nodes == nil
         or #grandchild_nodes == 0
-        or #grandchild_nodes == 1 and grandchild_nodes[1].type == "directory"
+        or (#grandchild_nodes == 1 and grandchild_nodes[1].type == "directory")
+        or context.recursive
       then
         scan_dir_async(context, child.path)
       end
@@ -520,7 +522,7 @@ M.get_items = function(state, parent_id, path_to_reveal, callback, async, recurs
 end
 
 -- async method
-M.get_dir_items_async = function(state, parent_id, recursive)
+M.get_dir_items_async = function(state, parent_id, recursive, expand)
   local context = file_items.create_context()
   context.state = state
   context.parent_id = parent_id

From a0d6cb3ebbfa09e28eaafac4fda944a0f1b276ed Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 4 Jun 2023 14:11:29 +0200
Subject: [PATCH 09/16] Move rendere.save_pos level up so it won't be called
 multiple times

---
 lua/neo-tree/sources/common/commands.lua | 1 +
 lua/neo-tree/sources/filesystem/init.lua | 7 +++----
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lua/neo-tree/sources/common/commands.lua b/lua/neo-tree/sources/common/commands.lua
index 0f82d0cd..9030d79c 100644
--- a/lua/neo-tree/sources/common/commands.lua
+++ b/lua/neo-tree/sources/common/commands.lua
@@ -115,6 +115,7 @@ end
 ---@param node table A node to expand
 M.expand_all_nodes = function(state, node)
   log.debug("Expanding all nodes under " .. node:get_id())
+  renderer.position.set(state, nil)
   local task = function ()
     fs.expand_directory(state, node)
   end
diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index 82fd50af..77e5251a 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -478,11 +478,11 @@ local function expand_and_load(node, state)
     rec({}, 1)
 end
 
----Expands given node recursively loading all descendant nodes if needed
----async method
+--- Expands given node recursively loading all descendant nodes if needed
+--- async method
 ---@param state table current state of the source
 ---@param node table a node to expand
-M.expand_directory = function(state, node)
+M.expand_directory_recursively = function(state, node)
   log.debug("Expanding directory " .. node:get_id())
   if node.type ~= "directory" then
     return
@@ -491,7 +491,6 @@ M.expand_directory = function(state, node)
   if node.loaded == false then
     local id = node:get_id()
     state.explicitly_opened_directories[id] = true
-    renderer.position.set(state, nil) -- todo should not be called recursively
     fs_scan.get_dir_items_async(state, id, true)
     expand_loaded(node, state)
   else

From 3da0ba23df949951b453993b4e61af51c7cb7751 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 4 Jun 2023 15:24:47 +0200
Subject: [PATCH 10/16] Fix after rename

---
 lua/neo-tree/sources/common/commands.lua | 2 +-
 lua/neo-tree/sources/filesystem/init.lua | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lua/neo-tree/sources/common/commands.lua b/lua/neo-tree/sources/common/commands.lua
index 9030d79c..8cd839be 100644
--- a/lua/neo-tree/sources/common/commands.lua
+++ b/lua/neo-tree/sources/common/commands.lua
@@ -117,7 +117,7 @@ M.expand_all_nodes = function(state, node)
   log.debug("Expanding all nodes under " .. node:get_id())
   renderer.position.set(state, nil)
   local task = function ()
-    fs.expand_directory(state, node)
+    fs.expand_directory_recursively(state, node)
   end
   async.run(
       task,
diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index 77e5251a..f4a92004 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -471,7 +471,7 @@ local function expand_and_load(node, state)
         table.insert(to_load, v)
       end
       if progress <= #to_load then
-        M.expand_directory(state, to_load[progress])
+        M.expand_directory_recursively(state, to_load[progress])
         rec(to_load, progress + 1)
       end
     end

From c8ff77a6eb5c8e9e95c7a76846ba3a914edea59a Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 4 Jun 2023 16:34:59 +0200
Subject: [PATCH 11/16] Extract node_expander and restore common/command
 interface

---
 lua/neo-tree/sources/common/commands.lua      | 14 ++-
 lua/neo-tree/sources/common/node_expander.lua | 85 +++++++++++++++++++
 lua/neo-tree/sources/filesystem/commands.lua  |  4 +-
 lua/neo-tree/sources/filesystem/init.lua      | 76 ++---------------
 .../sources/filesystem/lib/fs_scan.lua        |  2 +-
 5 files changed, 107 insertions(+), 74 deletions(-)
 create mode 100644 lua/neo-tree/sources/common/node_expander.lua

diff --git a/lua/neo-tree/sources/common/commands.lua b/lua/neo-tree/sources/common/commands.lua
index 8cd839be..6955bcc7 100644
--- a/lua/neo-tree/sources/common/commands.lua
+++ b/lua/neo-tree/sources/common/commands.lua
@@ -2,7 +2,6 @@
 
 local vim = vim
 local fs_actions = require("neo-tree.sources.filesystem.lib.fs_actions")
-local fs = require("neo-tree.sources.filesystem")
 local utils = require("neo-tree.utils")
 local renderer = require("neo-tree.ui.renderer")
 local events = require("neo-tree.events")
@@ -12,6 +11,7 @@ local log = require("neo-tree.log")
 local help = require("neo-tree.sources.common.help")
 local Preview = require("neo-tree.sources.common.preview")
 local async = require("plenary.async")
+local node_expander = require("neo-tree.sources.common.node_expander")
 
 ---Gets the node parent folder
 ---@param state table to look for nodes
@@ -113,11 +113,19 @@ end
 ---Expand all nodes
 ---@param state table The state of the source
 ---@param node table A node to expand
-M.expand_all_nodes = function(state, node)
+M.expand_all_nodes = function(state, node, source_expander)
+  if node == nil then
+    node = state.tree:get_node(state.path)
+  end
   log.debug("Expanding all nodes under " .. node:get_id())
+  if source_expander == nil then
+    source_expander = node_expander.default_expander
+  end
+
   renderer.position.set(state, nil)
+
   local task = function ()
-    fs.expand_directory_recursively(state, node)
+    node_expander.expand_directory_recursively(state, node, source_expander)
   end
   async.run(
       task,
diff --git a/lua/neo-tree/sources/common/node_expander.lua b/lua/neo-tree/sources/common/node_expander.lua
new file mode 100644
index 00000000..96c34b2b
--- /dev/null
+++ b/lua/neo-tree/sources/common/node_expander.lua
@@ -0,0 +1,85 @@
+local log = require("neo-tree.log")
+
+local M = {}
+
+--- Recursively expand all loaded nodes under the given node
+--- returns table with all discovered nodes that need to be loaded 
+---@param node table a node to expand
+---@param state table current state of the source
+---@return table discovered nodes that need to be loaded
+local function expand_loaded(node, state, node_expander)
+    local function rec(current_node, to_load)
+      if node_expander.should_prefetch(current_node) then
+        log.trace("Node " .. current_node:get_id() .. "not loaded, saving for later")
+        table.insert(to_load, current_node)
+      else
+        if not current_node:is_expanded() then
+          current_node:expand()
+          state.explicitly_opened_directories[current_node:get_id()] = true
+        end
+        local children = state.tree:get_nodes(current_node:get_id())
+        log.debug("Expanding childrens of " .. current_node:get_id())
+        for _, child in ipairs(children) do
+          if child.type == "directory" then
+             rec(child, to_load)
+          else
+            log.trace("Child: " .. child.name .. " is not a directory, skipping")
+          end
+        end
+      end
+    end
+
+    local to_load = {}
+    rec(node, to_load)
+    return to_load
+end
+
+--- Recursively expands all nodes under the given node
+--- loading nodes if necessary.
+--- async method
+---@param node table a node to expand
+---@param state table current state of the source
+local function expand_and_load(node, state, node_expander)
+    local function rec(to_load, progress)
+      local to_load_current = expand_loaded(node, state, node_expander)
+      for _,v in ipairs(to_load_current) do
+        table.insert(to_load, v)
+      end
+      if progress <= #to_load then
+        M.expand_directory_recursively(state, to_load[progress], node_expander)
+        rec(to_load, progress + 1)
+      end
+    end
+    rec({}, 1)
+end
+
+--- Expands given node recursively loading all descendant nodes if needed
+--- async method
+---@param state table current state of the source
+---@param node table a node to expand
+M.expand_directory_recursively = function(state, node, node_expander)
+  log.debug("Expanding directory " .. node:get_id())
+  if node.type ~= "directory" then
+    return
+  end
+  state.explicitly_opened_directories = state.explicitly_opened_directories or {}
+  if node_expander.should_prefetch(node) then
+    local id = node:get_id()
+    state.explicitly_opened_directories[id] = true
+    node_expander.prefetch(state, node)
+    expand_loaded(node, state, node_expander)
+  else
+    expand_and_load(node, state, node_expander)
+  end
+end
+
+M.default_expander = {
+  prefetch = function (state, node)
+    log.debug("Default expander prefetch does nothing")
+  end,
+  should_prefetch = function (node)
+    return false
+  end
+}
+
+return M
diff --git a/lua/neo-tree/sources/filesystem/commands.lua b/lua/neo-tree/sources/filesystem/commands.lua
index 00ad89e3..7b1aa558 100644
--- a/lua/neo-tree/sources/filesystem/commands.lua
+++ b/lua/neo-tree/sources/filesystem/commands.lua
@@ -67,8 +67,8 @@ M.delete_visual = function(state, selected_nodes)
   cc.delete_visual(state, selected_nodes, utils.wrap(refresh, state))
 end
 
-M.expand_all_nodes = function(state)
-  cc.expand_all_nodes(state, state.tree:get_node(state.path))
+M.expand_all_nodes = function(state, node)
+  cc.expand_all_nodes(state, node, fs.expander)
 end
 
 ---Shows the filter input, which will filter the tree.
diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index f4a92004..4d46c452 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -428,74 +428,14 @@ M.toggle_directory = function(state, node, path_to_reveal, skip_redraw, recursiv
   end
 end
 
---- Recursively expand all loaded nodes under the given node
---- returns table with all discovered nodes that need to be loaded 
----@param node table a node to expand
----@param state table current state of the source
----@return table discovered nodes that need to be loaded
-local function expand_loaded(node, state)
-    local function rec(_node, to_load)
-      if _node.loaded == false then
-        table.insert(to_load, _node)
-      else
-        if not _node:is_expanded() then
-          _node:expand()
-          state.explicitly_opened_directories[_node:get_id()] = true
-        end
-        local children = state.tree:get_nodes(_node:get_id())
-        log.debug("Expanding childrens of " .. _node:get_id())
-        for _, child in ipairs(children) do
-          if child.type == "directory" then
-             rec(child, to_load)
-          else
-            log.trace("Child: " .. child.name .. " is not a directory, skipping")
-          end
-        end
-      end
-    end
-
-    local to_load = {}
-    rec(node, to_load)
-    return to_load
-end
-
---- Recursively expands all nodes under the given node
---- loading nodes if necessary.
---- async method
----@param node table a node to expand
----@param state table current state of the source
-local function expand_and_load(node, state)
-    local function rec(to_load, progress)
-      local to_load_current = expand_loaded(node, state)
-      for _,v in ipairs(to_load_current) do
-        table.insert(to_load, v)
-      end
-      if progress <= #to_load then
-        M.expand_directory_recursively(state, to_load[progress])
-        rec(to_load, progress + 1)
-      end
-    end
-    rec({}, 1)
-end
-
---- Expands given node recursively loading all descendant nodes if needed
---- async method
----@param state table current state of the source
----@param node table a node to expand
-M.expand_directory_recursively = function(state, node)
-  log.debug("Expanding directory " .. node:get_id())
-  if node.type ~= "directory" then
-    return
-  end
-  state.explicitly_opened_directories = state.explicitly_opened_directories or {}
-  if node.loaded == false then
-    local id = node:get_id()
-    state.explicitly_opened_directories[id] = true
-    fs_scan.get_dir_items_async(state, id, true)
-    expand_loaded(node, state)
-  else
-    expand_and_load(node, state)
+M.expander = {
+  prefetch = function (state, node)
+    log.debug("Running fs prefetch for: " .. node:get_id())
+    fs_scan.get_dir_items_async(state, node:get_id(), true)
+  end,
+  should_prefetch = function (node)
+    return not node.loaded
   end
-end
+}
 
 return M
diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index 616d5e62..ca8bca69 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -522,7 +522,7 @@ M.get_items = function(state, parent_id, path_to_reveal, callback, async, recurs
 end
 
 -- async method
-M.get_dir_items_async = function(state, parent_id, recursive, expand)
+M.get_dir_items_async = function(state, parent_id, recursive)
   local context = file_items.create_context()
   context.state = state
   context.parent_id = parent_id

From 280b224b9c95a3f7ed2e198612ace1df424085e2 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 4 Jun 2023 16:41:47 +0200
Subject: [PATCH 12/16] Move default value to filesystem/commands

---
 lua/neo-tree/sources/common/commands.lua     | 3 ---
 lua/neo-tree/sources/filesystem/commands.lua | 3 +++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lua/neo-tree/sources/common/commands.lua b/lua/neo-tree/sources/common/commands.lua
index 6955bcc7..dffa6586 100644
--- a/lua/neo-tree/sources/common/commands.lua
+++ b/lua/neo-tree/sources/common/commands.lua
@@ -114,9 +114,6 @@ end
 ---@param state table The state of the source
 ---@param node table A node to expand
 M.expand_all_nodes = function(state, node, source_expander)
-  if node == nil then
-    node = state.tree:get_node(state.path)
-  end
   log.debug("Expanding all nodes under " .. node:get_id())
   if source_expander == nil then
     source_expander = node_expander.default_expander
diff --git a/lua/neo-tree/sources/filesystem/commands.lua b/lua/neo-tree/sources/filesystem/commands.lua
index 7b1aa558..b1a18154 100644
--- a/lua/neo-tree/sources/filesystem/commands.lua
+++ b/lua/neo-tree/sources/filesystem/commands.lua
@@ -68,6 +68,9 @@ M.delete_visual = function(state, selected_nodes)
 end
 
 M.expand_all_nodes = function(state, node)
+  if node == nil then
+    node = state.tree:get_node(state.path)
+  end
   cc.expand_all_nodes(state, node, fs.expander)
 end
 

From ced3d8dae0221426cfc9fceec1ce299658db7999 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Mon, 5 Jun 2023 11:04:43 +0200
Subject: [PATCH 13/16] Rename expander to prefetcher and add method docs

---
 lua/neo-tree/sources/common/commands.lua      |  9 +++----
 lua/neo-tree/sources/common/node_expander.lua | 24 ++++++++++---------
 lua/neo-tree/sources/filesystem/commands.lua  |  2 +-
 lua/neo-tree/sources/filesystem/init.lua      |  2 +-
 4 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/lua/neo-tree/sources/common/commands.lua b/lua/neo-tree/sources/common/commands.lua
index dffa6586..f5dfc561 100644
--- a/lua/neo-tree/sources/common/commands.lua
+++ b/lua/neo-tree/sources/common/commands.lua
@@ -113,16 +113,17 @@ end
 ---Expand all nodes
 ---@param state table The state of the source
 ---@param node table A node to expand
-M.expand_all_nodes = function(state, node, source_expander)
+---@param prefetcher table an object with two methods `prefetch(state, node)` and `should_prefetch(node) => boolean`
+M.expand_all_nodes = function(state, node, prefetcher)
   log.debug("Expanding all nodes under " .. node:get_id())
-  if source_expander == nil then
-    source_expander = node_expander.default_expander
+  if prefetcher == nil then
+    prefetcher = node_expander.default_prefetcher
   end
 
   renderer.position.set(state, nil)
 
   local task = function ()
-    node_expander.expand_directory_recursively(state, node, source_expander)
+    node_expander.expand_directory_recursively(state, node, prefetcher)
   end
   async.run(
       task,
diff --git a/lua/neo-tree/sources/common/node_expander.lua b/lua/neo-tree/sources/common/node_expander.lua
index 96c34b2b..3f657faa 100644
--- a/lua/neo-tree/sources/common/node_expander.lua
+++ b/lua/neo-tree/sources/common/node_expander.lua
@@ -7,9 +7,9 @@ local M = {}
 ---@param node table a node to expand
 ---@param state table current state of the source
 ---@return table discovered nodes that need to be loaded
-local function expand_loaded(node, state, node_expander)
+local function expand_loaded(node, state, prefetcher)
     local function rec(current_node, to_load)
-      if node_expander.should_prefetch(current_node) then
+      if prefetcher.should_prefetch(current_node) then
         log.trace("Node " .. current_node:get_id() .. "not loaded, saving for later")
         table.insert(to_load, current_node)
       else
@@ -39,14 +39,14 @@ end
 --- async method
 ---@param node table a node to expand
 ---@param state table current state of the source
-local function expand_and_load(node, state, node_expander)
+local function expand_and_load(node, state, prefetcher)
     local function rec(to_load, progress)
-      local to_load_current = expand_loaded(node, state, node_expander)
+      local to_load_current = expand_loaded(node, state, prefetcher)
       for _,v in ipairs(to_load_current) do
         table.insert(to_load, v)
       end
       if progress <= #to_load then
-        M.expand_directory_recursively(state, to_load[progress], node_expander)
+        M.expand_directory_recursively(state, to_load[progress], prefetcher)
         rec(to_load, progress + 1)
       end
     end
@@ -54,26 +54,28 @@ local function expand_and_load(node, state, node_expander)
 end
 
 --- Expands given node recursively loading all descendant nodes if needed
+--- Nodes will be loaded using given prefetcher
 --- async method
 ---@param state table current state of the source
 ---@param node table a node to expand
-M.expand_directory_recursively = function(state, node, node_expander)
+---@param prefetcher table an object with two methods `prefetch(state, node)` and `should_prefetch(node) => boolean`
+M.expand_directory_recursively = function(state, node, prefetcher)
   log.debug("Expanding directory " .. node:get_id())
   if node.type ~= "directory" then
     return
   end
   state.explicitly_opened_directories = state.explicitly_opened_directories or {}
-  if node_expander.should_prefetch(node) then
+  if prefetcher.should_prefetch(node) then
     local id = node:get_id()
     state.explicitly_opened_directories[id] = true
-    node_expander.prefetch(state, node)
-    expand_loaded(node, state, node_expander)
+    prefetcher.prefetch(state, node)
+    expand_loaded(node, state, prefetcher)
   else
-    expand_and_load(node, state, node_expander)
+    expand_and_load(node, state, prefetcher)
   end
 end
 
-M.default_expander = {
+M.default_prefetcher = {
   prefetch = function (state, node)
     log.debug("Default expander prefetch does nothing")
   end,
diff --git a/lua/neo-tree/sources/filesystem/commands.lua b/lua/neo-tree/sources/filesystem/commands.lua
index b1a18154..f2d242b2 100644
--- a/lua/neo-tree/sources/filesystem/commands.lua
+++ b/lua/neo-tree/sources/filesystem/commands.lua
@@ -71,7 +71,7 @@ M.expand_all_nodes = function(state, node)
   if node == nil then
     node = state.tree:get_node(state.path)
   end
-  cc.expand_all_nodes(state, node, fs.expander)
+  cc.expand_all_nodes(state, node, fs.prefetcher)
 end
 
 ---Shows the filter input, which will filter the tree.
diff --git a/lua/neo-tree/sources/filesystem/init.lua b/lua/neo-tree/sources/filesystem/init.lua
index 4d46c452..1f89651a 100644
--- a/lua/neo-tree/sources/filesystem/init.lua
+++ b/lua/neo-tree/sources/filesystem/init.lua
@@ -428,7 +428,7 @@ M.toggle_directory = function(state, node, path_to_reveal, skip_redraw, recursiv
   end
 end
 
-M.expander = {
+M.prefetcher = {
   prefetch = function (state, node)
     log.debug("Running fs prefetch for: " .. node:get_id())
     fs_scan.get_dir_items_async(state, node:get_id(), true)

From 2db83a6faf278be6009def101c5a7c773f359e5b Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sun, 11 Jun 2023 14:34:52 +0200
Subject: [PATCH 14/16] Simplify algorithm

---
 lua/neo-tree/sources/common/node_expander.lua | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/lua/neo-tree/sources/common/node_expander.lua b/lua/neo-tree/sources/common/node_expander.lua
index 3f657faa..c04f736c 100644
--- a/lua/neo-tree/sources/common/node_expander.lua
+++ b/lua/neo-tree/sources/common/node_expander.lua
@@ -34,23 +34,18 @@ local function expand_loaded(node, state, prefetcher)
     return to_load
 end
 
---- Recursively expands all nodes under the given node
---- loading nodes if necessary.
+--- Recursively expands all nodes under the given node collecting all unloaded nodes
+--- Then run prefetcher on all unloaded nodes. Finally, expand loded nodes. 
 --- async method
 ---@param node table a node to expand
 ---@param state table current state of the source
 local function expand_and_load(node, state, prefetcher)
-    local function rec(to_load, progress)
-      local to_load_current = expand_loaded(node, state, prefetcher)
-      for _,v in ipairs(to_load_current) do
-        table.insert(to_load, v)
+      local to_load = expand_loaded(node, state, prefetcher)
+      for _, _node in ipairs(to_load) do
+        prefetcher.prefetch(state, _node)
+        -- no need to handle results as prefetch is recursive
+        expand_loaded(_node, state, prefetcher)
       end
-      if progress <= #to_load then
-        M.expand_directory_recursively(state, to_load[progress], prefetcher)
-        rec(to_load, progress + 1)
-      end
-    end
-    rec({}, 1)
 end
 
 --- Expands given node recursively loading all descendant nodes if needed

From 8dc93bad2b9a6fcb23cf20293eb40af45eb9100a Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sat, 8 Jul 2023 08:05:31 +0200
Subject: [PATCH 15/16] Refactor: Use async function in async stack

---
 lua/neo-tree/sources/filesystem/lib/fs_scan.lua | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index ca8bca69..c1bcad45 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -11,9 +11,6 @@ local git = require("neo-tree.git")
 local events = require("neo-tree.events")
 local async = require("plenary.async")
 
-local Path = require("plenary.path")
-local os_sep = Path.path.sep
-
 local M = {}
 
 local on_directory_loaded = function(context, dir_path)
@@ -223,15 +220,15 @@ end
 local function scan_dir_async(context, path)
   log.debug("scan_dir_async - start " .. path)
 
-  local get_children = async.wrap(function (callback)
-    return get_children_async(path, callback)
-  end, 1)
+  local get_children = async.wrap(function (_path, callback)
+    return get_children_async(_path, callback)
+  end, 2)
 
-  local children = get_children()
+  local children = get_children(path)
   for _, child in ipairs(children) do
     create_node(context, child)
     if child.type == "directory" then
-      local grandchild_nodes = get_children_sync(child.path)
+      local grandchild_nodes = get_children(child.path)
       if
         grandchild_nodes == nil
         or #grandchild_nodes == 0

From 722aa3d8f11076031f28ff4b0889b3067c8e0393 Mon Sep 17 00:00:00 2001
From: ghostbuster91 <ghostbuster91@users.noreply.github.com>
Date: Sat, 8 Jul 2023 08:13:44 +0200
Subject: [PATCH 16/16] Refactor: Simplify starting multiple async tasks

---
 lua/neo-tree/sources/filesystem/lib/fs_scan.lua | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
index c1bcad45..366ec78f 100644
--- a/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
+++ b/lua/neo-tree/sources/filesystem/lib/fs_scan.lua
@@ -255,11 +255,9 @@ local function async_scan(context, path)
   if scan_mode == "deep" then
     local scan_tasks = {}
     for _, p in ipairs(context.paths_to_load) do
-      local scan_task = async.wrap(function(callback)
-        async.run(function ()
-            scan_dir_async(context, p)
-        end, callback)
-      end, 1)
+      local scan_task = function ()
+        scan_dir_async(context, p)
+      end
       table.insert(scan_tasks, scan_task)
     end