gh-81441: shutil.rmtree() FileNotFoundError race condition

Lib/shutil.py

@@ -541,13 +541,17 @@ def _rmtree_unsafe(path, onerror):
     try:
         with os.scandir(path) as scandir_it:
             entries = list(scandir_it)
+    except FileNotFoundError:
+        entries = []
     except OSError:
         onerror(os.scandir, path, sys.exc_info())
         entries = []
     for entry in entries:
         fullname = entry.path
         try:
             is_dir = entry.is_dir(follow_symlinks=False)
+        except FileNotFoundError:
+            continue
         except OSError:
             is_dir = False
         if is_dir:
@@ -557,17 +561,23 @@ def _rmtree_unsafe(path, onerror):
                     # a directory with a symlink after the call to
                     # os.scandir or entry.is_dir above.
                     raise OSError("Cannot call rmtree on a symbolic link")
+            except FileNotFoundError:
+                continue
             except OSError:
                 onerror(os.path.islink, fullname, sys.exc_info())
                 continue
             _rmtree_unsafe(fullname, onerror)
         else:
             try:
                 os.unlink(fullname)
+            except FileNotFoundError:
+                continue
             except OSError:
                 onerror(os.unlink, fullname, sys.exc_info())
     try:
         os.rmdir(path)
+    except FileNotFoundError:
+        pass
     except OSError:
         onerror(os.rmdir, path, sys.exc_info())
 
@@ -576,6 +586,8 @@ def _rmtree_safe_fd(topfd, path, onerror):
     try:
         with os.scandir(topfd) as scandir_it:
             entries = list(scandir_it)
+    except FileNotFoundError:
+        return
     except OSError as err:
         err.filename = path
         onerror(os.scandir, path, sys.exc_info())
@@ -584,19 +596,25 @@ def _rmtree_safe_fd(topfd, path, onerror):
         fullname = os.path.join(path, entry.name)
         try:
             is_dir = entry.is_dir(follow_symlinks=False)
+        except FileNotFoundError:
+            continue
         except OSError:
             is_dir = False
         else:
             if is_dir:
                 try:
                     orig_st = entry.stat(follow_symlinks=False)
                     is_dir = stat.S_ISDIR(orig_st.st_mode)
+                except FileNotFoundError:
+                    continue
                 except OSError:
                     onerror(os.lstat, fullname, sys.exc_info())
                     continue
         if is_dir:
             try:
                 dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd)
+            except FileNotFoundError:
+                continue
             except OSError:
                 onerror(os.open, fullname, sys.exc_info())
             else:
@@ -605,6 +623,8 @@ def _rmtree_safe_fd(topfd, path, onerror):
                         _rmtree_safe_fd(dirfd, fullname, onerror)
                         try:
                             os.rmdir(entry.name, dir_fd=topfd)
+                        except FileNotFoundError:
+                            continue
                         except OSError:
                             onerror(os.rmdir, fullname, sys.exc_info())
                     else:
@@ -621,6 +641,8 @@ def _rmtree_safe_fd(topfd, path, onerror):
         else:
             try:
                 os.unlink(entry.name, dir_fd=topfd)
+            except FileNotFoundError:
+                continue
             except OSError:
                 onerror(os.unlink, fullname, sys.exc_info())
 
@@ -646,6 +668,12 @@ def onerror(*args):
     elif onerror is None:
         def onerror(*args):
             raise
+    try:
+        if not os.path.exists(path):
+            raise FileNotFoundError("Cannot call rmtree on a non-existent path")
+    except:
+        onerror(os.path.exists, path, sys.exc_info())
+        return
     if _use_fd_functions:
         # While the unsafe rmtree works fine on bytes, the fd based does not.
         if isinstance(path, bytes):

Lib/test/test_shutil.py

@@ -336,6 +336,109 @@ def raiser(fn, *args, **kwargs):
         finally:
             os.lstat = orig_lstat
 
+    def test_rmtree_deleted_file_race_condition(self):
+        # bpo-37260
+        #
+        # Test that a file deleted after it is enumerated
+        # by scandir() but before unlink() is called
+        # doesn't generate any errors.
+        def _onerror(fn, path, exc_info):
+            if path == paths[0]:
+                os.chmod(paths[0], mode)
+                os.rmdir(paths[0])
+                os.unlink(paths[1])
+                pass
+            else:
+                raise
+
+        paths = [os.path.join(TESTFN, 'foo'),
+                 os.path.join(TESTFN, 'bar')]
+        os.mkdir(TESTFN)
+        os.mkdir(paths[0])
+        write_file((TESTFN, 'bar'), 'bar')
+        mode = os.stat(paths[0]).st_mode
+        os.chmod(paths[0], 0)
+
+        try:
+            shutil.rmtree(TESTFN, onerror=_onerror)
+        except:
+            # test failed, so cleanup artifacts
+            try:
+                os.chmod(paths[0], mode)
+            except:
+                pass
+
+            shutil.rmtree(TESTFN)
+
+    def test_rmtree_deleted_dir_race_condition(self):
+        # bpo-37260
+        #
+        # Test that a directory deleted after it is enumerated
+        # by scandir() but before rmdr() is called
+        # doesn't generate any errors.
+        def _onerror(fn, path, exc_info):
+            if path == paths[0]:
+                os.chmod(paths[0], mode)
+                os.rmdir(paths[0])
+                os.rmdir(paths[1])
+                pass
+            else:
+                raise
+
+        paths = [os.path.join(TESTFN, 'foo'),
+                 os.path.join(TESTFN, 'bar')]
+        os.mkdir(TESTFN)
+        os.mkdir(paths[0])
+        os.mkdir(paths[1])
+        mode = os.stat(paths[0]).st_mode
+        os.chmod(paths[0], 0)
+
+        try:
+            shutil.rmtree(TESTFN, onerror=_onerror)
+        except:
+            # test failed, so cleanup artifacts
+            try:
+                os.chmod(paths[0], mode)
+            except:
+                pass
+
+            shutil.rmtree(TESTFN)
+
+    @support.skip_unless_symlink
+    def test_rmtree_deleted_symlink_race_condition(self):
+        # bpo-37260
+        #
+        # Test that a symlink deleted after it is enumerated
+        # by scandir() but before unlink() is called
+        # doesn't generate any errors.
+        def _onerror(fn, path, exc_info):
+            if path == paths[0]:
+                os.chmod(paths[0], mode)
+                os.rmdir(paths[0])
+                os.unlink(paths[1])
+                pass
+            else:
+                raise
+
+        paths = [os.path.join(TESTFN, 'foo'),
+                 os.path.join(TESTFN, 'bar')]
+        os.mkdir(TESTFN)
+        os.mkdir(paths[0])
+        os.symlink('foo', paths[1])
+        mode = os.stat(paths[0]).st_mode
+        os.chmod(paths[0], 0)
+
+        try:
+            shutil.rmtree(TESTFN, onerror=_onerror)
+        except:
+            # test failed, so cleanup artifacts
+            try:
+                os.chmod(paths[0], mode)
+            except:
+                pass
+
+            shutil.rmtree(TESTFN)
+
     @support.skip_unless_symlink
     def test_copymode_follow_symlinks(self):
         tmp_dir = self.mkdtemp()

Misc/NEWS.d/next/Library/2019-06-14-22-37-32.bpo-37260.oecdIf.rst

@@ -0,0 +1 @@
+Fixed a race condition in shutil.rmtree() in which directory entries removed by another process or thread while shutil.rmtree() is running can cause shutil.rmtree() to raise FileNotFoundError.  Patch by Jeffrey Kintscher.