From 917df7d0775d615c5a7592a8264e933a0d7f5c93 Mon Sep 17 00:00:00 2001 From: Bert Peters Date: Thu, 28 Nov 2024 18:45:54 +0100 Subject: [PATCH 1/8] Avoid unbounded growth SpooledTempfile.writelines --- Lib/tempfile.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b5a15f7b72c872..854c00dd4f10a6 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -848,10 +848,17 @@ def write(self, s): return rv def writelines(self, iterable): - file = self._file - rv = file.writelines(iterable) - self._check(file) - return rv + if self._max_size == 0 or self._rolled: + return self._file.writelines(iterable) + + it = iter(iterable) + + for line in it: + self.write(line) + if self._rolled: + break + + return self._file.writelines(it) def detach(self): return self._file.detach() From 5ad82d48fcc16850f27420de1debfae60a126ffa Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:29:22 +0000 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst diff --git a/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst new file mode 100644 index 00000000000000..b99be671502e8d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst @@ -0,0 +1 @@ +Avoid unbounded buffering for :class:`tempfile.SpooledTemporaryFile`. Previously, when the :meth:`tempfile.SpooledTemporaryFile.writelines` method was called, disk-spillover was only checked after the lines iterator had been exhausted. This is now done while writing instead. From b4dd28cf43d27ad6e2eb84676a92bc0334f7a6ec Mon Sep 17 00:00:00 2001 From: Bert Peters Date: Fri, 29 Nov 2024 09:00:18 +0100 Subject: [PATCH 3/8] Ensure that SpooledTemporaryFile rolls over in writelines --- Lib/test/test_tempfile.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index a5e182cef23dc5..a9c48f3dd8c2e8 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1288,6 +1288,18 @@ def test_writelines(self): buf = f.read() self.assertEqual(buf, b'xyz') + def test_writelines_rollover(self): + # Verify writelines rolls over before exhausting the iterator + f = self.do_create(max_size=2) + + def it(): + yield b'xy' + self.assertFalse(f._rolled) + yield b'z' + self.assertTrue(f._rolled) + + f.writelines(it()) + def test_writelines_sequential(self): # A SpooledTemporaryFile should hold exactly max_size bytes, and roll # over afterward From 07d2a662f7819ec9478951780b57d9eb718941a0 Mon Sep 17 00:00:00 2001 From: Bert Peters Date: Sat, 30 Nov 2024 12:28:14 +0100 Subject: [PATCH 4/8] =?UTF-8?q?fixup!=20=F0=9F=93=9C=F0=9F=A4=96=20Added?= =?UTF-8?q?=20by=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst index b99be671502e8d..d26858af5666b2 100644 --- a/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst +++ b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst @@ -1 +1 @@ -Avoid unbounded buffering for :class:`tempfile.SpooledTemporaryFile`. Previously, when the :meth:`tempfile.SpooledTemporaryFile.writelines` method was called, disk-spillover was only checked after the lines iterator had been exhausted. This is now done while writing instead. +Avoid unbounded buffering for :meth:`tempfile.SpooledTemporaryFile.writelines`. Previously, disk spillover was only checked after the lines iterator had been exhausted. This is now done after each line is written. From 9177f34baf62de69682c27a62266d91ffd43151d Mon Sep 17 00:00:00 2001 From: Bert Peters Date: Sat, 30 Nov 2024 12:43:37 +0100 Subject: [PATCH 5/8] fixup! Ensure that SpooledTemporaryFile rolls over in writelines --- Lib/test/test_tempfile.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index a9c48f3dd8c2e8..31982ae6ea2aab 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1299,6 +1299,22 @@ def it(): self.assertTrue(f._rolled) f.writelines(it()) + pos = f.seek(0) + self.assertEqual(pos, 0) + buf = f.read() + self.assertEqual(buf, b'xyz') + + def test_writelines_fast_path(self): + f = self.do_create(max_size=2) + f.write(b'abc') + self.assertTrue(f._rolled) + + f.writelines([b'd', b'e', b'f']) + pos = f.seek(0) + self.assertEqual(pos, 0) + buf = f.read() + self.assertEqual(buf, b'abcdef') + def test_writelines_sequential(self): # A SpooledTemporaryFile should hold exactly max_size bytes, and roll From d685dfb3e26ee92000aa7672441c7fc8568ff3e6 Mon Sep 17 00:00:00 2001 From: Bert Peters Date: Sun, 1 Dec 2024 10:18:54 +0100 Subject: [PATCH 6/8] =?UTF-8?q?fixup!=20fixup!=20=F0=9F=93=9C=F0=9F=A4=96?= =?UTF-8?q?=20Added=20by=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst index d26858af5666b2..55a8b341420262 100644 --- a/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst +++ b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst @@ -1 +1 @@ -Avoid unbounded buffering for :meth:`tempfile.SpooledTemporaryFile.writelines`. Previously, disk spillover was only checked after the lines iterator had been exhausted. This is now done after each line is written. +Avoid unbounded buffering for :meth:`!tempfile.SpooledTemporaryFile.writelines`. Previously, disk spillover was only checked after the lines iterator had been exhausted. This is now done after each line is written. From b50e03401c0d4bb17f1e8bfd191d648a19ac8cc4 Mon Sep 17 00:00:00 2001 From: Bert Peters Date: Sat, 1 Mar 2025 10:30:53 +0100 Subject: [PATCH 7/8] =?UTF-8?q?fixup!=20fixup!=20fixup!=20=F0=9F=93=9C?= =?UTF-8?q?=F0=9F=A4=96=20Added=20by=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst index 55a8b341420262..029c348918e0d1 100644 --- a/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst +++ b/Misc/NEWS.d/next/Security/2024-11-28-20-29-21.gh-issue-127371.PeEhUd.rst @@ -1 +1,3 @@ -Avoid unbounded buffering for :meth:`!tempfile.SpooledTemporaryFile.writelines`. Previously, disk spillover was only checked after the lines iterator had been exhausted. This is now done after each line is written. +Avoid unbounded buffering for :meth:`!tempfile.SpooledTemporaryFile.writelines`. +Previously, disk spillover was only checked after the lines iterator had been +exhausted. This is now done after each line is written. From ee6db45090a7ecadf61416f120bd7d04502feba6 Mon Sep 17 00:00:00 2001 From: Bert Peters Date: Tue, 4 Mar 2025 21:37:20 +0100 Subject: [PATCH 8/8] fixup! fixup! Ensure that SpooledTemporaryFile rolls over in writelines --- Lib/tempfile.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 09340ca1a2d55d..cadb0bed3cce3b 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -872,13 +872,10 @@ def writelines(self, iterable): return self._file.writelines(iterable) it = iter(iterable) - for line in it: self.write(line) if self._rolled: - break - - return self._file.writelines(it) + return self._file.writelines(it) def detach(self): return self._file.detach()