Fully implement PEP 527

Author: di

tests/unit/forklift/test_legacy.py

@@ -786,7 +786,10 @@ def test_fails_invalid_version(self, pyramid_config, pyramid_request, version):
         resp = excinfo.value
 
         assert resp.status_code == 400
-        assert resp.status == "400 Unknown protocol version."
+        assert (
+            resp.status
+            == "400 Invalid value for filetype. Error: Use a known file type."
+        )
 
     @pytest.mark.parametrize(
         ("post_data", "message"),
@@ -2485,106 +2488,6 @@ def storage_service_store(path, file_path, *, meta):
             )
         ]
 
-    def test_upload_succeeds_with_legacy_ext(
-        self, tmpdir, monkeypatch, pyramid_config, db_request, metrics
-    ):
-        monkeypatch.setattr(tempfile, "tempdir", str(tmpdir))
-
-        pyramid_config.testing_securitypolicy(userid=1)
-
-        user = UserFactory.create()
-        EmailFactory.create(user=user)
-        project = ProjectFactory.create(allow_legacy_files=True)
-        release = ReleaseFactory.create(project=project, version="1.0")
-        RoleFactory.create(user=user, project=project)
-
-        filename = "{}-{}.tar.bz2".format(project.name, release.version)
-
-        db_request.user = user
-        db_request.remote_addr = "10.10.10.30"
-        db_request.user_agent = "warehouse-tests/6.6.6"
-        db_request.POST = MultiDict(
-            {
-                "metadata_version": "1.2",
-                "name": project.name,
-                "version": release.version,
-                "filetype": "sdist",
-                "pyversion": "source",
-                "md5_digest": _TAR_BZ2_PKG_MD5,
-                "content": pretend.stub(
-                    filename=filename,
-                    file=io.BytesIO(_TAR_BZ2_PKG_TESTDATA),
-                    type="application/tar",
-                ),
-            }
-        )
-
-        def storage_service_store(path, file_path, *, meta):
-            with open(file_path, "rb") as fp:
-                assert fp.read() == _TAR_BZ2_PKG_TESTDATA
-
-        storage_service = pretend.stub(store=storage_service_store)
-        db_request.find_service = lambda svc, name=None, context=None: {
-            IFileStorage: storage_service,
-            IMetricsService: metrics,
-        }.get(svc)
-
-        monkeypatch.setattr(legacy, "_is_valid_dist_file", lambda *a, **kw: True)
-
-        resp = legacy.file_upload(db_request)
-
-        assert resp.status_code == 200
-
-    def test_upload_succeeds_with_legacy_type(
-        self, tmpdir, monkeypatch, pyramid_config, db_request, metrics
-    ):
-        monkeypatch.setattr(tempfile, "tempdir", str(tmpdir))
-
-        pyramid_config.testing_securitypolicy(userid=1)
-
-        user = UserFactory.create()
-        EmailFactory.create(user=user)
-        project = ProjectFactory.create(allow_legacy_files=True)
-        release = ReleaseFactory.create(project=project, version="1.0")
-        RoleFactory.create(user=user, project=project)
-
-        filename = "{}-{}.tar.gz".format(project.name, release.version)
-
-        db_request.user = user
-        db_request.remote_addr = "10.10.10.30"
-        db_request.user_agent = "warehouse-tests/6.6.6"
-        db_request.POST = MultiDict(
-            {
-                "metadata_version": "1.2",
-                "name": project.name,
-                "version": release.version,
-                "filetype": "bdist_dumb",
-                "pyversion": "3.5",
-                "md5_digest": _TAR_GZ_PKG_MD5,
-                "content": pretend.stub(
-                    filename=filename,
-                    file=io.BytesIO(_TAR_GZ_PKG_TESTDATA),
-                    type="application/tar",
-                ),
-            }
-        )
-
-        def storage_service_store(path, file_path, *, meta):
-            with open(file_path, "rb") as fp:
-                assert fp.read() == _TAR_GZ_PKG_TESTDATA
-
-        storage_service = pretend.stub(store=storage_service_store)
-        db_request.find_service = lambda svc, name=None, context=None: {
-            IFileStorage: storage_service,
-            IMetricsService: metrics,
-        }.get(svc)
-
-        monkeypatch.setattr(legacy, "_is_valid_dist_file", lambda *a, **kw: True)
-
-        resp = legacy.file_upload(db_request)
-
-        assert resp.status_code == 200
-
     @pytest.mark.parametrize("plat", ["linux_x86_64", "linux_x86_64.win32"])
     def test_upload_fails_with_unsupported_wheel_plat(
         self, monkeypatch, pyramid_config, db_request, plat

warehouse/forklift/legacy.py

@@ -141,11 +141,7 @@ def _valid_platform_tag(platform_tag):
 _error_message_order = ["metadata_version", "name", "version"]
 
 
-_dist_file_regexes = {
-    # True/False is for legacy or not.
-    True: re.compile(r".+?\.(exe|tar\.gz|bz2|rpm|deb|zip|tgz|egg|dmg|msi|whl)$", re.I),
-    False: re.compile(r".+?\.(tar\.gz|zip|whl|egg)$", re.I),
-}
+_dist_file_re = re.compile(r".+?\.(tar\.gz|zip|whl|egg)$", re.I)
 
 
 _wheel_file_re = re.compile(
@@ -464,16 +460,7 @@ class MetadataForm(forms.Form):
         validators=[
             wtforms.validators.DataRequired(),
             wtforms.validators.AnyOf(
-                [
-                    "bdist_dmg",
-                    "bdist_dumb",
-                    "bdist_egg",
-                    "bdist_msi",
-                    "bdist_rpm",
-                    "bdist_wheel",
-                    "bdist_wininst",
-                    "sdist",
-                ],
+                ["bdist_egg", "bdist_wheel", "sdist",],
                 message="Use a known file type.",
             ),
         ]
@@ -1150,7 +1137,7 @@ def file_upload(request):
         )
 
     # Make sure the filename ends with an allowed extension.
-    if _dist_file_regexes[project.allow_legacy_files].search(filename) is None:
+    if _dist_file_re.search(filename) is None:
         raise _exc_with_message(
             HTTPBadRequest,
             "Invalid file extension: Use .egg, .tar.gz, .whl or .zip "
@@ -1172,16 +1159,6 @@ def file_upload(request):
     ):
         raise _exc_with_message(HTTPBadRequest, "Invalid distribution file.")
 
-    # Ensure that the package filetype is allowed.
-    # TODO: Once PEP 527 is completely implemented we should be able to delete
-    #       this and just move it into the form itself.
-    if not project.allow_legacy_files and form.filetype.data not in {
-        "sdist",
-        "bdist_wheel",
-        "bdist_egg",
-    }:
-        raise _exc_with_message(HTTPBadRequest, "Unknown type of file.")
-
     # The project may or may not have a file size specified on the project, if
     # it does then it may or may not be smaller or larger than our global file
     # size limits.

warehouse/migrations/versions/b265ed9eeb8a_fully_deprecate_legacy_distribution_.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fully deprecate legacy distribution types
+
+Revision ID: b265ed9eeb8a
+Revises: 5c029d9ef925
+Create Date: 2020-03-12 17:51:08.447903
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "b265ed9eeb8a"
+down_revision = "5c029d9ef925"
+
+
+def upgrade():
+    op.drop_column("projects", "allow_legacy_files")
+
+
+def downgrade():
+    op.add_column(
+        "projects",
+        sa.Column(
+            "allow_legacy_files",
+            sa.BOOLEAN(),
+            server_default=sa.text("false"),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )

warehouse/packaging/models.py

@@ -122,7 +122,6 @@ class Project(SitemapMixin, db.Model):
     has_docs = Column(Boolean)
     upload_limit = Column(Integer, nullable=True)
     last_serial = Column(Integer, nullable=False, server_default=sql.text("0"))
-    allow_legacy_files = Column(Boolean, nullable=False, server_default=sql.false())
     zscore = Column(Float, nullable=True)
 
     total_size = Column(BigInteger, server_default=sql.text("0"))