Skip to content

Commit 9914be2

Browse files
DarkaMaulDarkaMaul
committed
Merge branch 'refs/heads/main' into dm/store-attestations
# Conflicts: # warehouse/forklift/legacy.py
2 parents 7a10776 + b419f32 commit 9914be2

File tree

7 files changed

+295
-18
lines changed

7 files changed

+295
-18
lines changed

‎.github/workflows/ci.yml

+38
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,41 @@ jobs:
9393
key: ${{ runner.os }}-mypy-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
9494
- name: Run ${{ matrix.name }}
9595
run: ${{ matrix.command }}
96+
97+
check_db:
98+
name: Check Database Consistency
99+
needs: build
100+
runs-on: depot-ubuntu-22.04-arm
101+
continue-on-error: true
102+
container:
103+
image: registry.depot.dev/rltf7cln5v:${{ needs.build.outputs.buildId }}
104+
credentials:
105+
username: x-token
106+
password: ${{ needs.build.outputs.token }}
107+
services:
108+
postgres:
109+
image: postgres:16.1
110+
ports:
111+
- 5432:5432
112+
env:
113+
POSTGRES_DB: warehouse
114+
POSTGRES_HOST_AUTH_METHOD: trust # never do this in production!
115+
POSTGRES_INITDB_ARGS: '--no-sync --set fsync=off --set full_page_writes=off'
116+
# Set health checks to wait until postgres has started
117+
options: --health-cmd "pg_isready --username=postgres --dbname=postgres" --health-interval 10s --health-timeout 5s --health-retries 5
118+
steps:
119+
- name: Check out repository
120+
uses: actions/checkout@v4
121+
- name: Dotenv Action
122+
# We need to load the environment variables to run the CLI
123+
id: dotenv
124+
uses: falti/dotenv-action@v1
125+
with:
126+
path: dev/environment
127+
export-variables: true
128+
keys-case: upper
129+
- name: Check Database
130+
run: bin/db-check
131+
env:
132+
# override the hostname set in `dev/environment`
133+
DATABASE_URL: 'postgresql+psycopg://postgres@postgres/warehouse'

‎Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ inittuf: .state/db-migrated
145145
runmigrations: .state/docker-build-base
146146
docker compose run --rm web python -m warehouse db upgrade head
147147

148+
checkdb: .state/docker-build-base
149+
docker compose run --rm web bin/db-check
150+
148151
reindex: .state/docker-build-base
149152
docker compose run --rm web python -m warehouse search reindex
150153

@@ -168,4 +171,4 @@ purge: stop clean
168171
stop:
169172
docker compose stop
170173

171-
.PHONY: default build serve resetdb initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations
174+
.PHONY: default build serve resetdb initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations checkdb

‎bin/db-check

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# When on GitHub Actions, if a label is present on the PR, skip the check.
5+
if [ -n "$GITHUB_ACTIONS" ]; then
6+
if [ -n "$GITHUB_EVENT_PATH" ]; then
7+
if jq -e '.pull_request.labels | map(.name) | index("skip-db-check")' < "$GITHUB_EVENT_PATH" > /dev/null; then
8+
echo "Skipping database check due to 'skip-db-check' label"
9+
exit 0
10+
fi
11+
fi
12+
fi
13+
14+
# Check for outstanding database migrations
15+
python -m warehouse db upgrade head
16+
python -m warehouse db check

‎tests/unit/cli/test_db.py

+13
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import warehouse.db
2525

2626
from warehouse.cli.db.branches import branches
27+
from warehouse.cli.db.check import check
2728
from warehouse.cli.db.current import current
2829
from warehouse.cli.db.downgrade import downgrade
2930
from warehouse.cli.db.heads import heads
@@ -299,6 +300,18 @@ def test_upgrade_command(monkeypatch, cli, pyramid_config):
299300
assert alembic_upgrade.calls == [pretend.call(alembic_config, "foo")]
300301

301302

303+
def test_check_command(monkeypatch, cli, pyramid_config):
304+
alembic_check = pretend.call_recorder(lambda config: None)
305+
monkeypatch.setattr(alembic.command, "check", alembic_check)
306+
307+
alembic_config = pretend.stub(attributes={})
308+
pyramid_config.alembic_config = lambda: alembic_config
309+
310+
result = cli.invoke(check, obj=pyramid_config)
311+
assert result.exit_code == 0
312+
assert alembic_check.calls == [pretend.call(alembic_config)]
313+
314+
302315
def test_dbml_command(monkeypatch, cli):
303316
generate_dbml_file = pretend.call_recorder(lambda tables, path: None)
304317
monkeypatch.setattr(warehouse.cli.db.dbml, "generate_dbml_file", generate_dbml_file)

‎tests/unit/forklift/test_legacy.py

+151-11
Original file line numberDiff line numberDiff line change
@@ -1961,7 +1961,9 @@ def test_upload_fails_with_existing_filename_diff_content(
19611961
),
19621962
}
19631963
)
1964-
1964+
blake2_256_digest = hashlib.blake2b(
1965+
file_content.getvalue(), digest_size=256 // 8
1966+
).hexdigest()
19651967
db_request.db.add(
19661968
FileFactory.create(
19671969
release=release,
@@ -1985,7 +1987,9 @@ def test_upload_fails_with_existing_filename_diff_content(
19851987
assert db_request.help_url.calls == [pretend.call(_anchor="file-name-reuse")]
19861988
assert resp.status_code == 400
19871989
assert resp.status == (
1988-
"400 File already exists. See /the/help/url/ for more information."
1990+
f"400 File already exists ({filename!r}, "
1991+
f"with blake2_256 hash {blake2_256_digest!r}). "
1992+
"See /the/help/url/ for more information."
19891993
)
19901994

19911995
def test_upload_fails_with_diff_filename_same_blake2(
@@ -2017,15 +2021,16 @@ def test_upload_fails_with_diff_filename_same_blake2(
20172021
}
20182022
)
20192023

2024+
blake2_256_digest = hashlib.blake2b(
2025+
file_content.getvalue(), digest_size=256 // 8
2026+
).hexdigest()
20202027
db_request.db.add(
20212028
FileFactory.create(
20222029
release=release,
20232030
filename=filename,
20242031
md5_digest=hashlib.md5(file_content.getvalue()).hexdigest(),
20252032
sha256_digest=hashlib.sha256(file_content.getvalue()).hexdigest(),
2026-
blake2_256_digest=hashlib.blake2b(
2027-
file_content.getvalue(), digest_size=256 // 8
2028-
).hexdigest(),
2033+
blake2_256_digest=blake2_256_digest,
20292034
path="source/{name[0]}/{name}/{filename}".format(
20302035
name=project.name, filename=filename
20312036
),
@@ -2041,7 +2046,9 @@ def test_upload_fails_with_diff_filename_same_blake2(
20412046
assert db_request.help_url.calls == [pretend.call(_anchor="file-name-reuse")]
20422047
assert resp.status_code == 400
20432048
assert resp.status == (
2044-
"400 File already exists. See /the/help/url/ for more information."
2049+
f"400 File already exists ({db_request.POST['content'].filename!r}, "
2050+
f"with blake2_256 hash {blake2_256_digest!r}). "
2051+
"See /the/help/url/ for more information."
20452052
)
20462053

20472054
@pytest.mark.parametrize(
@@ -4536,12 +4543,145 @@ def test_missing_trailing_slash_redirect(pyramid_request):
45364543
"https://github.com",
45374544
False,
45384545
),
4539-
( # Publisher URL is None
4540-
"https://github.com/owner/project",
4541-
None,
4546+
],
4547+
)
4548+
def test_verify_url_with_trusted_publisher(url, publisher_url, expected):
4549+
assert legacy._verify_url_with_trusted_publisher(url, publisher_url) == expected
4550+
4551+
4552+
@pytest.mark.parametrize(
4553+
("url", "project_name", "project_normalized_name", "expected"),
4554+
[
4555+
( # PyPI /project/ case
4556+
"https://pypi.org/project/myproject",
4557+
"myproject",
4558+
"myproject",
4559+
True,
4560+
),
4561+
( # PyPI /p/ case
4562+
"https://pypi.org/p/myproject",
4563+
"myproject",
4564+
"myproject",
4565+
True,
4566+
),
4567+
( # pypi.python.org /project/ case
4568+
"https://pypi.python.org/project/myproject",
4569+
"myproject",
4570+
"myproject",
4571+
True,
4572+
),
4573+
( # pypi.python.org /p/ case
4574+
"https://pypi.python.org/p/myproject",
4575+
"myproject",
4576+
"myproject",
4577+
True,
4578+
),
4579+
( # python.org/pypi/ case
4580+
"https://python.org/pypi/myproject",
4581+
"myproject",
4582+
"myproject",
4583+
True,
4584+
),
4585+
( # PyPI /project/ case
4586+
"https://pypi.org/project/myproject",
4587+
"myproject",
4588+
"myproject",
4589+
True,
4590+
),
4591+
( # Normalized name differs from URL
4592+
"https://pypi.org/project/my_project",
4593+
"my_project",
4594+
"my-project",
4595+
True,
4596+
),
4597+
( # Normalized name same as URL
4598+
"https://pypi.org/project/my-project",
4599+
"my_project",
4600+
"my-project",
4601+
True,
4602+
),
4603+
( # Trailing slash
4604+
"https://pypi.org/project/myproject/",
4605+
"myproject",
4606+
"myproject",
4607+
True,
4608+
),
4609+
( # Domains are case insensitive
4610+
"https://PyPI.org/project/myproject",
4611+
"myproject",
4612+
"myproject",
4613+
True,
4614+
),
4615+
( # Paths are case-sensitive
4616+
"https://pypi.org/Project/myproject",
4617+
"myproject",
4618+
"myproject",
4619+
False,
4620+
),
4621+
( # Wrong domain
4622+
"https://example.com/project/myproject",
4623+
"myproject",
4624+
"myproject",
4625+
False,
4626+
),
4627+
( # Wrong path
4628+
"https://pypi.org/something/myproject",
4629+
"myproject",
4630+
"myproject",
4631+
False,
4632+
),
4633+
( # Path has extra components
4634+
"https://pypi.org/something/myproject/something",
4635+
"myproject",
4636+
"myproject",
4637+
False,
4638+
),
4639+
( # Wrong package name
4640+
"https://pypi.org/project/otherproject",
4641+
"myproject",
4642+
"myproject",
4643+
False,
4644+
),
4645+
( # Similar package name
4646+
"https://pypi.org/project/myproject",
4647+
"myproject2",
4648+
"myproject2",
4649+
False,
4650+
),
4651+
( # Similar package name
4652+
"https://pypi.org/project/myproject2",
4653+
"myproject",
4654+
"myproject",
45424655
False,
45434656
),
45444657
],
45454658
)
4546-
def test_verify_url(url, publisher_url, expected):
4547-
assert legacy._verify_url(url, publisher_url) == expected
4659+
def test_verify_url_pypi(url, project_name, project_normalized_name, expected):
4660+
assert (
4661+
legacy._verify_url_pypi(url, project_name, project_normalized_name) == expected
4662+
)
4663+
4664+
4665+
def test_verify_url():
4666+
# `_verify_url` is just a helper function that calls `_verify_url_pypi` and
4667+
# `_verify_url_with_trusted_publisher`, where the actual verification logic lives.
4668+
assert legacy._verify_url(
4669+
url="https://pypi.org/project/myproject/",
4670+
publisher_url=None,
4671+
project_name="myproject",
4672+
project_normalized_name="myproject",
4673+
)
4674+
4675+
assert legacy._verify_url(
4676+
url="https://github.com/org/myproject/issues",
4677+
publisher_url="https://github.com/org/myproject",
4678+
project_name="myproject",
4679+
project_normalized_name="myproject",
4680+
)
4681+
4682+
assert not legacy._verify_url(
4683+
url="example.com",
4684+
publisher_url="https://github.com/or/myproject",
4685+
project_name="myproject",
4686+
project_normalized_name="myproject",
4687+
)

‎warehouse/cli/db/check.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License.
12+
13+
import alembic.command
14+
import click
15+
16+
from warehouse.cli.db import db
17+
18+
19+
@db.command()
20+
@click.pass_obj
21+
def check(config):
22+
"""
23+
Check if autogenerate will create new operations.
24+
"""
25+
alembic.command.check(config.alembic_config())

0 commit comments

Comments
 (0)