diff --git a/warehouse/accounts/views.py b/warehouse/accounts/views.py index 31d21b9909c8..842502d11bbc 100644 --- a/warehouse/accounts/views.py +++ b/warehouse/accounts/views.py @@ -77,9 +77,14 @@ from warehouse.events.tags import EventTag from warehouse.metrics.interfaces import IMetricsService from warehouse.oidc.forms import DeletePublisherForm +from warehouse.oidc.forms.buildkite import PendingBuildkitePublisherForm from warehouse.oidc.forms.github import PendingGitHubPublisherForm from warehouse.oidc.interfaces import TooManyOIDCRegistrations -from warehouse.oidc.models import PendingGitHubPublisher, PendingOIDCPublisher +from warehouse.oidc.models import ( + PendingBuildkitePublisher, + PendingGitHubPublisher, + PendingOIDCPublisher, +) from warehouse.organizations.interfaces import IOrganizationService from warehouse.organizations.models import OrganizationRole, OrganizationRoleType from warehouse.packaging.models import ( @@ -1488,6 +1493,13 @@ def _check_ratelimits(self): ) ) + @property + def pending_buildkite_publisher_form(self): + return PendingBuildkitePublisherForm( + self.request.POST, + project_factory=self.project_factory + ) + @property def pending_github_publisher_form(self): return PendingGitHubPublisherForm( @@ -1499,6 +1511,8 @@ def pending_github_publisher_form(self): @property def default_response(self): return { + "default_publisher_name": "github", + "pending_buildkite_publisher_form": self.pending_buildkite_publisher_form, "pending_github_publisher_form": self.pending_github_publisher_form, } @@ -1516,11 +1530,147 @@ def manage_publishing(self): return self.default_response + @view_config( + route_name="manage.account.publishing.buildkite", + request_method="POST", + request_param=PendingBuildkitePublisherForm.__params__, + ) + def add_pending_buildkite_oidc_publisher(self): + response = self.default_response + response["default_publisher_name"] = "buildkite" + + if self.request.flags.disallow_oidc(AdminFlagValue.DISALLOW_BUILDKITE_OIDC): + self.request.session.flash( + self.request._( + "Buildkite-based trusted publishing is temporarily disabled. " + "See https://pypi.org/help#admin-intervention for details." + ), + queue="error", + ) + return response + + self.metrics.increment( + "warehouse.oidc.add_pending_publisher.attempt", tags=["publisher:Buildkite"] + ) + + if not self.request.user.has_primary_verified_email: + self.request.session.flash( + self.request._( + "You must have a verified email in order to register a " + "pending trusted publisher. " + "See https://pypi.org/help#openid-connect for details." + ), + queue="error", + ) + return response + + # Separately from having permission to register pending OIDC publishers, + # we limit users to no more than 3 pending publishers at once. + if len(self.request.user.pending_oidc_publishers) >= 3: + self.request.session.flash( + self.request._( + "You can't register more than 3 pending trusted " + "publishers at once." + ), + queue="error", + ) + return response + + try: + self._check_ratelimits() + except TooManyOIDCRegistrations as exc: + self.metrics.increment( + "warehouse.oidc.add_pending_publisher.ratelimited", + tags=["publisher:Buildkite"], + ) + return HTTPTooManyRequests( + self.request._( + "There have been too many attempted trusted publisher " + "registrations. Try again later." + ), + retry_after=exc.resets_in.total_seconds(), + ) + + self._hit_ratelimits() + + response = response + form = response["pending_buildkite_publisher_form"] + + if not form.validate(): + self.request.session.flash( + self.request._("The trusted publisher could not be registered"), + queue="error", + ) + return response + + publisher_already_exists = ( + self.request.db.query(PendingBuildkitePublisher) + .filter_by( + organization_slug=form.organization_slug.data, + pipeline_slug=form.pipeline_slug.data, + ) + .first() + is not None + ) + + if publisher_already_exists: + self.request.session.flash( + self.request._( + "This trusted publisher has already been registered. " + "Please contact PyPI's admins if this wasn't intentional." + ), + queue="error", + ) + return response + + pending_publisher = PendingBuildkitePublisher( + project_name=form.project_name.data, + added_by=self.request.user, + organization_slug=form.organization_slug.data, + pipeline_slug=form.pipeline_slug.data, + build_branch=form.build_branch.data, + build_tag=form.build_tag.data, + step_key=form.step_key.data, + ) + + self.request.db.add(pending_publisher) + self.request.db.flush() # To get the new ID + + self.request.user.record_event( + tag=EventTag.Account.PendingOIDCPublisherAdded, + request=self.request, + additional={ + "project": pending_publisher.project_name, + "publisher": pending_publisher.publisher_name, + "id": str(pending_publisher.id), + "specifier": str(pending_publisher), + "url": pending_publisher.publisher_url(), + "submitted_by": self.request.user.username, + }, + ) + + self.request.session.flash( + self.request._( + "Registered a new pending publisher to create " + f"the project '{pending_publisher.project_name}'." + ), + queue="success", + ) + + self.metrics.increment( + "warehouse.oidc.add_pending_publisher.ok", tags=["publisher:Buildkite"] + ) + + return HTTPSeeOther(self.request.route_path("manage.account.publishing")) + @view_config( request_method="POST", request_param=PendingGitHubPublisherForm.__params__, ) def add_pending_github_oidc_publisher(self): + response = self.default_response + response["default_publisher_name"] = "github" + if self.request.flags.disallow_oidc(AdminFlagValue.DISALLOW_GITHUB_OIDC): self.request.session.flash( self.request._( @@ -1529,7 +1679,7 @@ def add_pending_github_oidc_publisher(self): ), queue="error", ) - return self.default_response + return response self.metrics.increment( "warehouse.oidc.add_pending_publisher.attempt", tags=["publisher:GitHub"] diff --git a/warehouse/admin/flags.py b/warehouse/admin/flags.py index 17e39fb2e600..1451d36cc62f 100644 --- a/warehouse/admin/flags.py +++ b/warehouse/admin/flags.py @@ -25,6 +25,7 @@ class AdminFlagValue(enum.Enum): DISALLOW_NEW_UPLOAD = "disallow-new-upload" DISALLOW_NEW_USER_REGISTRATION = "disallow-new-user-registration" DISALLOW_OIDC = "disallow-oidc" + DISALLOW_BUILDKITE_OIDC = "disallow-buildkite-oidc" DISALLOW_GITHUB_OIDC = "disallow-github-oidc" DISALLOW_GOOGLE_OIDC = "disallow-google-oidc" READ_ONLY = "read-only" diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index 341a3d0330c3..1f22b1cea936 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -101,9 +101,14 @@ ) from warehouse.metrics.interfaces import IMetricsService from warehouse.oidc.forms import DeletePublisherForm +from warehouse.oidc.forms.buildkite import BuildkitePublisherForm from warehouse.oidc.forms.github import GitHubPublisherForm from warehouse.oidc.interfaces import TooManyOIDCRegistrations -from warehouse.oidc.models import GitHubPublisher, OIDCPublisher +from warehouse.oidc.models import ( + BuildkitePublisher, + GitHubPublisher, + OIDCPublisher, +) from warehouse.organizations.interfaces import IOrganizationService from warehouse.organizations.models import ( OrganizationProject, @@ -1216,6 +1221,12 @@ def _check_ratelimits(self): ) ) + @property + def buildkite_publisher_form(self): + return BuildkitePublisherForm( + self.request.POST, + ) + @property def github_publisher_form(self): return GitHubPublisherForm( @@ -1227,6 +1238,8 @@ def github_publisher_form(self): def default_response(self): return { "project": self.project, + "default_publisher_name": "github", + "buildkite_publisher_form": self.buildkite_publisher_form, "github_publisher_form": self.github_publisher_form, } @@ -1243,11 +1256,133 @@ def manage_project_oidc_publishers(self): return self.default_response + @view_config( + route_name="manage.project.settings.publishing.buildkite", + request_method="POST", + request_param=BuildkitePublisherForm.__params__, + ) + def add_buildkite_oidc_publisher(self): + response = self.default_response + response["default_publisher_name"] = "buildkite" + + if self.request.flags.disallow_oidc(AdminFlagValue.DISALLOW_BUILDKITE_OIDC): + self.request.session.flash( + self.request._( + "Buildkite-based trusted publishing is temporarily disabled. " + "See https://pypi.org/help#admin-intervention for details." + ), + queue="error", + ) + return response + + self.metrics.increment( + "warehouse.oidc.add_publisher.attempt", tags=["publisher:Buildkite"] + ) + + try: + self._check_ratelimits() + except TooManyOIDCRegistrations as exc: + self.metrics.increment( + "warehouse.oidc.add_publisher.ratelimited", tags=["publisher:Buildkite"] + ) + return HTTPTooManyRequests( + self.request._( + "There have been too many attempted trusted publisher " + "registrations. Try again later." + ), + retry_after=exc.resets_in.total_seconds(), + ) + + self._hit_ratelimits() + + form = response["buildkite_publisher_form"] + + if not form.validate(): + self.request.session.flash( + self.request._("The trusted publisher could not be registered"), + queue="error", + ) + return response + + # Buildkite OIDC publishers are unique on the tuple of + # (repository_name, repository_owner, workflow_filename, environment), + # so we check for an already registered one before creating. + publisher = ( + self.request.db.query(BuildkitePublisher) + .filter( + BuildkitePublisher.organization_slug == form.organization_slug.data, + BuildkitePublisher.pipeline_slug == form.pipeline_slug.data, + ) + .one_or_none() + ) + if publisher is None: + publisher = BuildkitePublisher( + organization_slug=form.organization_slug.data, + pipeline_slug=form.pipeline_slug.data, + build_branch=form.build_branch.data, + build_tag=form.build_tag.data, + step_key=form.step_key.data, + ) + + self.request.db.add(publisher) + + # Each project has a unique set of OIDC publishers; the same + # publisher can't be registered to the project more than once. + if publisher in self.project.oidc_publishers: + self.request.session.flash( + self.request._( + f"{publisher} is already registered with {self.project.name}" + ), + queue="error", + ) + return response + + for user in self.project.users: + send_trusted_publisher_added_email( + self.request, + user, + project_name=self.project.name, + publisher=publisher, + ) + + self.project.oidc_publishers.append(publisher) + + self.project.record_event( + tag=EventTag.Project.OIDCPublisherAdded, + request=self.request, + additional={ + "publisher": publisher.publisher_name, + "id": str(publisher.id), + "specifier": str(publisher), + "url": publisher.publisher_url(), + "submitted_by": self.request.user.username, + }, + ) + + self.request.session.flash( + f"Added {publisher} in {publisher.publisher_url()} to {self.project.name}", + queue="success", + ) + + self.metrics.increment( + "warehouse.oidc.add_publisher.ok", tags=["publisher:Buildkite"] + ) + + return HTTPSeeOther( + self.request.route_path( + "manage.project.settings.publishing", + project_name=self.project.name, + ) + ) + @view_config( request_method="POST", request_param=GitHubPublisherForm.__params__, ) def add_github_oidc_publisher(self): + response = self.default_response + response["default_publisher_name"] = "github" + if self.request.flags.disallow_oidc(AdminFlagValue.DISALLOW_GITHUB_OIDC): self.request.session.flash( self.request._( @@ -1256,7 +1391,7 @@ def add_github_oidc_publisher(self): ), queue="error", ) - return self.default_response + return response self.metrics.increment( "warehouse.oidc.add_publisher.attempt", tags=["publisher:GitHub"] @@ -1278,7 +1413,7 @@ def add_github_oidc_publisher(self): self._hit_ratelimits() - response = self.default_response + response = response form = response["github_publisher_form"] if not form.validate(): diff --git a/warehouse/migrations/versions/8c83f0a1d70e_add_buildkite_oidc_models.py b/warehouse/migrations/versions/8c83f0a1d70e_add_buildkite_oidc_models.py new file mode 100644 index 000000000000..a66a3a696b7f --- /dev/null +++ b/warehouse/migrations/versions/8c83f0a1d70e_add_buildkite_oidc_models.py @@ -0,0 +1,95 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Add Buildkite OIDC models + +Revision ID: 8c83f0a1d70e +Revises: 186f076eb60b +Create Date: 2023-10-27 09:20:18.030874 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "8c83f0a1d70e" +down_revision = "186f076eb60b" + +# Note: It is VERY important to ensure that a migration does not lock for a +# long period of time and to ensure that each individual migration does +# not break compatibility with the *previous* version of the code base. +# This is because the migrations will be ran automatically as part of the +# deployment process, but while the previous version of the code is still +# up and running. Thus backwards incompatible changes must be broken up +# over multiple migrations inside of multiple pull requests in order to +# phase them in over multiple deploys. +# +# By default, migrations cannot wait more than 4s on acquiring a lock +# and each individual statement cannot take more than 5s. This helps +# prevent situations where a slow migration takes the entire site down. +# +# If you need to increase this timeout for a migration, you can do so +# by adding: +# +# op.execute("SET statement_timeout = 5000") +# op.execute("SET lock_timeout = 4000") +# +# To whatever values are reasonable for this migration as part of your +# migration. + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "buildkite_oidc_publishers", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("organization_slug", sa.String(), nullable=False), + sa.Column("pipeline_slug", sa.String(), nullable=False), + sa.Column("build_branch", sa.String(), nullable=False), + sa.Column("build_tag", sa.String(), nullable=False), + sa.Column("step_key", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["oidc_publishers.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "organization_slug", "pipeline_slug", name="_buildkite_oidc_publisher_uc" + ), + ) + op.create_table( + "pending_buildkite_oidc_publishers", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("organization_slug", sa.String(), nullable=False), + sa.Column("pipeline_slug", sa.String(), nullable=False), + sa.Column("build_branch", sa.String(), nullable=False), + sa.Column("build_tag", sa.String(), nullable=False), + sa.Column("step_key", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["pending_oidc_publishers.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "organization_slug", + "pipeline_slug", + name="_pending_buildkite_oidc_publisher_uc", + ), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("pending_buildkite_oidc_publishers") + op.drop_table("buildkite_oidc_publishers") + # ### end Alembic commands ### diff --git a/warehouse/oidc/__init__.py b/warehouse/oidc/__init__.py index 15452ce8871c..8a75be16fd3a 100644 --- a/warehouse/oidc/__init__.py +++ b/warehouse/oidc/__init__.py @@ -15,7 +15,11 @@ from warehouse.oidc.interfaces import IOIDCPublisherService from warehouse.oidc.services import OIDCPublisherServiceFactory from warehouse.oidc.tasks import compute_oidc_metrics -from warehouse.oidc.utils import GITHUB_OIDC_ISSUER_URL, GOOGLE_OIDC_ISSUER_URL +from warehouse.oidc.utils import ( + BUILDKITE_OIDC_ISSUER_URL, + GITHUB_OIDC_ISSUER_URL, + GOOGLE_OIDC_ISSUER_URL, +) def includeme(config): @@ -23,6 +27,15 @@ def includeme(config): config.registry.settings["oidc.backend"] ) + config.register_service_factory( + OIDCPublisherServiceFactory( + publisher="buildkite", + issuer_url=BUILDKITE_OIDC_ISSUER_URL, + service_class=oidc_publisher_service_class, + ), + IOIDCPublisherService, + name="buildkite", + ) config.register_service_factory( OIDCPublisherServiceFactory( publisher="github", @@ -47,6 +60,7 @@ def includeme(config): auth = config.get_settings().get("auth.domain") config.add_route("oidc.audience", "/_/oidc/audience", domain=auth) + config.add_route("oidc.buildkite.mint_token", "/_/oidc/buildkite/mint-token", domain=auth) config.add_route("oidc.github.mint_token", "/_/oidc/github/mint-token", domain=auth) # Compute OIDC metrics periodically diff --git a/warehouse/oidc/forms/buildkite.py b/warehouse/oidc/forms/buildkite.py new file mode 100644 index 000000000000..f870860b3a7f --- /dev/null +++ b/warehouse/oidc/forms/buildkite.py @@ -0,0 +1,88 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import requests +import sentry_sdk +import wtforms + +from warehouse import forms +from warehouse.i18n import localize as _ +from warehouse.utils.project import PROJECT_NAME_RE + +_VALID_BUILDKITE_SLUG = re.compile(r"^[a-zA-Z0-9]+[a-zA-Z0-9\-]*$") + + +class BuildkitePublisherBase(forms.Form): + __params__ = [ + "organization_slug", + "pipeline_slug", + "build_branch", + "build_tag", + "step_key", + ] + + organization_slug = wtforms.StringField( + validators=[ + wtforms.validators.InputRequired( + message=_("Specify Buildkite organization slug"), + ), + wtforms.validators.Regexp( + _VALID_BUILDKITE_SLUG, message=_("Invalid pipeline slug") + ), + ] + ) + + pipeline_slug = wtforms.StringField( + validators=[ + wtforms.validators.InputRequired( + message=_("Specify Buildkite pipeline slug"), + ), + wtforms.validators.Regexp( + _VALID_BUILDKITE_SLUG, message=_("Invalid pipeline slug") + ), + ] + ) + + build_branch = wtforms.StringField(validators=[wtforms.validators.Optional()]) + build_tag = wtforms.StringField(validators=[wtforms.validators.Optional()]) + step_key = wtforms.StringField(validators=[wtforms.validators.Optional()]) + + +class PendingBuildkitePublisherForm(BuildkitePublisherBase): + __params__ = BuildkitePublisherBase.__params__ + ["project_name"] + + project_name = wtforms.StringField( + validators=[ + wtforms.validators.InputRequired(message=_("Specify project name")), + wtforms.validators.Regexp( + PROJECT_NAME_RE, message=_("Invalid project name") + ), + ] + ) + + def __init__(self, *args, project_factory, **kwargs): + super().__init__(*args, **kwargs) + self._project_factory = project_factory + + def validate_project_name(self, field): + project_name = field.data + + if project_name in self._project_factory: + raise wtforms.validators.ValidationError( + _("This project name is already in use") + ) + + +class BuildkitePublisherForm(BuildkitePublisherBase): + pass diff --git a/warehouse/oidc/models/__init__.py b/warehouse/oidc/models/__init__.py index 3d5d8bd33158..8329cbfa8883 100644 --- a/warehouse/oidc/models/__init__.py +++ b/warehouse/oidc/models/__init__.py @@ -11,14 +11,17 @@ # limitations under the License. from warehouse.oidc.models._core import OIDCPublisher, PendingOIDCPublisher +from warehouse.oidc.models.buildkite import BuildkitePublisher, PendingBuildkitePublisher from warehouse.oidc.models.github import GitHubPublisher, PendingGitHubPublisher from warehouse.oidc.models.google import GooglePublisher, PendingGooglePublisher __all__ = [ "OIDCPublisher", "PendingOIDCPublisher", + "PendingBuildkitePublisher", "PendingGitHubPublisher", "PendingGooglePublisher", + "BuildkitePublisher", "GitHubPublisher", "GooglePublisher", ] diff --git a/warehouse/oidc/models/buildkite.py b/warehouse/oidc/models/buildkite.py new file mode 100644 index 000000000000..ac40207a81e5 --- /dev/null +++ b/warehouse/oidc/models/buildkite.py @@ -0,0 +1,174 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from sqlalchemy import ForeignKey, String, UniqueConstraint +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Query, mapped_column +from sqlalchemy.sql.expression import func, literal + +from warehouse.oidc.errors import InvalidPublisherError +from warehouse.oidc.interfaces import SignedClaims +from warehouse.oidc.models._core import ( + CheckClaimCallable, + OIDCPublisher, + PendingOIDCPublisher, + check_claim_binary, +) + + +def _check_sub(ground_truth, signed_claim, _all_signed_claims): + # We expect a string formatted as follows: + # organization:ORG:pipeline:PIPELINE:[...] + # where [...] is a concatenation of other job context metadata. We + # currently lack the ground context to verify that additional metadata, so + # we limit our verification to just the ORG and PIPELINE components. + + # Defensive: Buildkite should never give us an empty subject. + if not signed_claim: + return False + + return signed_claim.startswith(f"{ground_truth}:") + +class BuildkitePublisherMixin: + """ + Common functionality for both pending and concrete Buildkite OIDC publishers. + """ + + organization_slug = mapped_column(String, nullable=False) + pipeline_slug = mapped_column(String, nullable=False) + build_branch = mapped_column(String, nullable=False) + build_tag = mapped_column(String, nullable=False) + step_key = mapped_column(String, nullable=False) + + __required_verifiable_claims__: dict[str, CheckClaimCallable[Any]] = { + "sub": _check_sub, + "organization_slug": check_claim_binary(str.__eq__), + "pipeline_slug": check_claim_binary(str.__eq__), + } + + __required_unverifiable_claims__: set[str] = { + "build_number", + "job_id", + "agent_id", + } + + __optional_verifiable_claims__: dict[str, CheckClaimCallable[Any]] = { + "build_branch": check_claim_binary(str.__eq__), + "build_tag": check_claim_binary(str.__eq__), + "step_key": check_claim_binary(str.__eq__), + } + + __unchecked_claims__ = { + "organization_id", + "pipeline_id", + "build_commit", + } + + @staticmethod + def __lookup_all__(klass, signed_claims: SignedClaims) -> Query | None: + return ( + Query(klass) + .filter_by( + organization_slug=signed_claims["organization_slug"], + pipeline_slug=signed_claims["pipeline_slug"], + ) + ) + + __lookup_strategies__ = [ + __lookup_all__, + ] + + @property + def publisher_name(self): + return "Buildkite" + + @property + def sub(self): + return f"organization:{self.organization_slug}:pipeline:{self.pipeline_slug}" + + def publisher_url(self, claims=None): + base = f"https://buildkite.com/{self.organization_slug}/{self.pipeline_slug}" + + build_number = claims.get("build_number") if claims else None + job_id = claims.get("job_id") if claims else None + + if build_number and job_id: + return f"{base}/builds/{build_number}#{job_id}" + elif build_number: + return f"{base}/builds/{build_number}" + return base + + def __str__(self): + return f"{self.organization_slug}/{self.pipeline_slug}" + + +class BuildkitePublisher(BuildkitePublisherMixin, OIDCPublisher): + __tablename__ = "buildkite_oidc_publishers" + __mapper_args__ = {"polymorphic_identity": "buildkite_oidc_publishers"} + __table_args__ = ( + UniqueConstraint( + "organization_slug", + "pipeline_slug", + name="_buildkite_oidc_publisher_uc", + ), + ) + + id = mapped_column( + UUID(as_uuid=True), ForeignKey(OIDCPublisher.id), primary_key=True + ) + + +class PendingBuildkitePublisher(BuildkitePublisherMixin, PendingOIDCPublisher): + __tablename__ = "pending_buildkite_oidc_publishers" + __mapper_args__ = {"polymorphic_identity": "pending_buildkite_oidc_publishers"} + __table_args__ = ( + UniqueConstraint( + "organization_slug", + "pipeline_slug", + name="_pending_buildkite_oidc_publisher_uc", + ), + ) + + id = mapped_column( + UUID(as_uuid=True), ForeignKey(PendingOIDCPublisher.id), primary_key=True + ) + + def reify(self, session): + """ + Returns a `BuildkitePublisher` for this `PendingBuildkitePublisher`, + deleting the `PendingBuildkitePublisher` in the process. + """ + + maybe_publisher = ( + session.query(BuildkitePublisher) + .filter( + BuildkitePublisher.organization_slug == self.organization_slug, + BuildkitePublisher.pipeline_slug == self.pipeline_slug, + BuildkitePublisher.build_branch == self.build_branch, + BuildkitePublisher.build_tag == self.build_tag, + BuildkitePublisher.step_key == self.step_key, + ) + .one_or_none() + ) + + publisher = maybe_publisher or BuildkitePublisher( + organization_slug=self.organization_slug, + pipeline_slug=self.pipeline_slug, + build_branch=self.build_branch, + build_tag=self.build_tag, + step_key=self.step_key, + ) + + session.delete(self) + return publisher diff --git a/warehouse/oidc/utils.py b/warehouse/oidc/utils.py index 8cdef771214a..a347e84b4d82 100644 --- a/warehouse/oidc/utils.py +++ b/warehouse/oidc/utils.py @@ -19,20 +19,28 @@ from warehouse.oidc.errors import InvalidPublisherError from warehouse.oidc.interfaces import SignedClaims from warehouse.oidc.models import ( + BuildkitePublisher, GitHubPublisher, GooglePublisher, OIDCPublisher, + PendingBuildkitePublisher, PendingGitHubPublisher, PendingGooglePublisher, ) from warehouse.oidc.models._core import OIDCPublisherMixin +BUILDKITE_OIDC_ISSUER_URL = "https://agent.buildkite.com" GITHUB_OIDC_ISSUER_URL = "https://token.actions.githubusercontent.com" GOOGLE_OIDC_ISSUER_URL = "https://accounts.google.com" -OIDC_ISSUER_URLS = {GITHUB_OIDC_ISSUER_URL, GOOGLE_OIDC_ISSUER_URL} +OIDC_ISSUER_URLS = { + BUILDKITE_OIDC_ISSUER_URL, + GITHUB_OIDC_ISSUER_URL, + GOOGLE_OIDC_ISSUER_URL, +} OIDC_PUBLISHER_CLASSES: dict[str, dict[bool, type[OIDCPublisherMixin]]] = { + BUILDKITE_OIDC_ISSUER_URL: {False: BuildkitePublisher, True: PendingBuildkitePublisher}, GITHUB_OIDC_ISSUER_URL: {False: GitHubPublisher, True: PendingGitHubPublisher}, GOOGLE_OIDC_ISSUER_URL: {False: GooglePublisher, True: PendingGooglePublisher}, } diff --git a/warehouse/oidc/views.py b/warehouse/oidc/views.py index 251e13677c1c..3a847e3cb4ca 100644 --- a/warehouse/oidc/views.py +++ b/warehouse/oidc/views.py @@ -64,6 +64,163 @@ def oidc_audience(request): return {"audience": audience} +@view_config( + route_name="oidc.buildkite.mint_token", + require_methods=["POST"], + renderer="json", + require_csrf=False, + has_translations=True, +) +def mint_token_from_buildkite_oidc(request): + def _invalid(errors): + request.response.status = 422 + print(f"422: {repr(errors)}") + return {"message": "Token request failed", "errors": errors} + + if request.flags.disallow_oidc(AdminFlagValue.DISALLOW_BUILDKITE_OIDC): + print("Buildkite OIDC disallowed") + return _invalid( + errors=[ + { + "code": "not-enabled", + "description": ( + "Buildkite-based trusted publishing functionality not enabled" + ), + } + ] + ) + + try: + payload = TokenPayload.parse_raw(request.body) + unverified_jwt = payload.token + except ValidationError as exc: + return _invalid(errors=[{"code": "invalid-payload", "description": str(exc)}]) + + oidc_service = request.find_service(IOIDCPublisherService, name="buildkite") + claims = oidc_service.verify_jwt_signature(unverified_jwt) + if not claims: + return _invalid( + errors=[ + {"code": "invalid-token", "description": "malformed or invalid token"} + ] + ) + + # First, try to find a pending publisher. + try: + pending_publisher = oidc_service.find_publisher(claims, pending=True) + factory = ProjectFactory(request) + + # If the project already exists, this pending publisher is no longer + # valid and needs to be removed. + # NOTE: This is mostly a sanity check, since we dispose of invalidated + # pending publishers below. + if pending_publisher.project_name in factory: + request.db.delete(pending_publisher) + return _invalid( + errors=[ + { + "code": "invalid-pending-publisher", + "description": "valid token, but project already exists", + } + ] + ) + + # Create the new project, and reify the pending publisher against it. + project_service = request.find_service(IProjectService) + new_project = project_service.create_project( + pending_publisher.project_name, + pending_publisher.added_by, + request, + ratelimited=False, + ) + oidc_service.reify_pending_publisher(pending_publisher, new_project) + + # Successfully converting a pending publisher into a normal publisher + # is a positive signal, so we reset the associated ratelimits. + ratelimiters = _ratelimiters(request) + ratelimiters["user.oidc"].clear(pending_publisher.added_by.id) + ratelimiters["ip.oidc"].clear(request.remote_addr) + + # There might be other pending publishers for the same project name, + # which we've now invalidated by creating the project. These would + # be disposed of on use, but we explicitly dispose of them here while + # also sending emails to their owners. + stale_pending_publishers = ( + request.db.query(PendingOIDCPublisher) + .filter( + func.normalize_pep426_name(PendingOIDCPublisher.project_name) + == func.normalize_pep426_name(pending_publisher.project_name) + ) + .all() + ) + for stale_publisher in stale_pending_publishers: + send_pending_trusted_publisher_invalidated_email( + request, + stale_publisher.added_by, + project_name=stale_publisher.project_name, + ) + request.db.delete(stale_publisher) + except InvalidPublisherError: + # If the claim set isn't valid for a pending publisher, it's OK, we + # will try finding a regular publisher + pass + + # We either don't have a pending OIDC publisher, or we *did* + # have one and we've just converted it. Either way, look for a full publisher + # to actually do the macaroon minting with. + try: + publisher = oidc_service.find_publisher(claims, pending=False) + except InvalidPublisherError as e: + return _invalid( + errors=[ + { + "code": "invalid-publisher", + "description": f"valid token, but no corresponding publisher ({e})", + } + ] + ) + + # At this point, we've verified that the given JWT is valid for the given + # project. All we need to do is mint a new token. + # NOTE: For OIDC-minted API tokens, the Macaroon's description string + # is purely an implementation detail and is not displayed to the user. + macaroon_service = request.find_service(IMacaroonService, context=None) + not_before = int(time.time()) + expires_at = not_before + 900 + serialized, dm = macaroon_service.create_macaroon( + request.domain, + ( + f"OpenID token: {str(publisher)} " + f"({datetime.fromtimestamp(not_before).isoformat()})" + ), + [ + caveats.OIDCPublisher( + oidc_publisher_id=str(publisher.id), + ), + caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]), + caveats.Expiration(expires_at=expires_at, not_before=not_before), + ], + oidc_publisher_id=publisher.id, + additional={ + "oidc": { + "build_branch": claims.get("build_branch"), + "build_commit": claims.get("build_commit"), + }, + }, + ) + for project in publisher.projects: + project.record_event( + tag=EventTag.Project.ShortLivedAPITokenAdded, + request=request, + additional={ + "expires": expires_at, + "publisher_name": publisher.publisher_name, + "publisher_url": publisher.publisher_url(), + }, + ) + return {"success": True, "token": serialized} + + @view_config( route_name="oidc.github.mint_token", require_methods=["POST"], @@ -71,7 +228,7 @@ def oidc_audience(request): require_csrf=False, has_translations=True, ) -def mint_token_from_oidc(request): +def mint_token_from_github_oidc(request): def _invalid(errors): request.response.status = 422 return {"message": "Token request failed", "errors": errors} diff --git a/warehouse/routes.py b/warehouse/routes.py index b7cea0c669c6..e084e625dd34 100644 --- a/warehouse/routes.py +++ b/warehouse/routes.py @@ -206,6 +206,9 @@ def includeme(config): config.add_route( "manage.account.publishing", "/manage/account/publishing/", domain=warehouse ) + config.add_route( + "manage.account.publishing.buildkite", "/manage/account/publishing/buildkite/", domain=warehouse + ) config.add_route( "manage.account.two-factor", "/manage/account/two-factor/", domain=warehouse ) @@ -384,6 +387,13 @@ def includeme(config): traverse="/{project_name}", domain=warehouse, ) + config.add_route( + "manage.project.settings.publishing.buildkite", + "/manage/project/{project_name}/settings/publishing/buildkite/", + factory="warehouse.packaging.models:ProjectFactory", + traverse="/{project_name}", + domain=warehouse, + ) config.add_route( "manage.project.remove_organization_project", "/manage/project/{project_name}/remove_organization_project/", diff --git a/warehouse/templates/manage/account/publishing.html b/warehouse/templates/manage/account/publishing.html index 3545fee01ca6..698326c870db 100644 --- a/warehouse/templates/manage/account/publishing.html +++ b/warehouse/templates/manage/account/publishing.html @@ -22,7 +22,116 @@ {{ oidc_title() }} {% endblock %} -{% macro github_form(request, pending_github_pubisher_form) %} +{% macro buildkite_form(request, pending_buildkite_publisher_form) %} + {{ form_error_anchor(pending_github_publisher_form) }} +
+ + {{ form_errors(pending_buildkite_publisher_form) }} +
+ + {{ pending_buildkite_publisher_form.project_name(placeholder=gettext("project name"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", aria_describedby="project_name-errors") }} +

+ {% trans %}The project (on PyPI) that will be created when this publisher is used{% endtrans %} +

+
+ {{ field_errors(pending_buildkite_publisher_form.project_name) }} +
+
+
+ + {{ pending_buildkite_publisher_form.organization_slug(placeholder=gettext("acme-inc"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", aria_describedby="organization_slug-errors") }} +

+ {% trans %}The slug of the Buildkite organization that owns the pipeline{% endtrans %} +

+
+ {{ field_errors(pending_buildkite_publisher_form.organization_slug) }} +
+
+
+ + {{ pending_buildkite_publisher_form.pipeline_slug(placeholder=gettext("widgets-client"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", **{"aria-describedby":"pipeline_slug-errors"}) }} +

+ {% trans %}The slug of the Buildkite pipeline that contains the publishing steps{% endtrans %} +

+
+ {{ field_errors(pending_buildkite_publisher_form.pipeline_slug) }} +
+
+
+ + {{ pending_buildkite_publisher_form.build_branch(placeholder=gettext("main"), class_="form-group__field", autocomplete="off", **{"aria-describedby":"build_branch-errors"}) }} +

+ {% trans %}Restrict publishing to a specific branch by supplying a branch name{% endtrans %} +

+
+ {{ field_errors(pending_buildkite_publisher_form.build_branch) }} +
+
+
+ + {{ pending_buildkite_publisher_form.build_tag(placeholder=gettext("v1.x"), class_="form-group__field", autocomplete="off", **{"aria-describedby":"build_tag-errors"}) }} +

+ {% trans %}Restrict publishing to a specific tag by supplying a tag name{% endtrans %} +

+
+ {{ field_errors(pending_buildkite_publisher_form.build_tag) }} +
+
+
+ + {{ pending_buildkite_publisher_form.step_key(placeholder=gettext("publish-python-package"), class_="form-group__field", autocomplete="off", **{"aria-describedby":"step_key-errors"}) }} +

+ {% trans href="https://buildkite.com/docs/pipelines/command-step#agents" %} + Restrict publishing to a specific step in your pipeline by supplying a step key + {% endtrans %} +

+
+ {{ field_errors(pending_buildkite_publisher_form.step_key) }} +
+
+
+ +
+
+{% endmacro %} + +{% macro github_form(request, pending_github_publisher_form) %} {{ form_error_anchor(pending_github_publisher_form) }}
@@ -209,12 +318,13 @@

{% trans %}Add a new pending publisher{% endtrans %}

{% if request.user.has_two_factor %} - {% set publishers = [("GitHub", github_form(request, pending_github_publisher_form))] %} + {% set publishers = [("GitHub", github_form(request, pending_github_publisher_form)), ("Buildkite", buildkite_form(request, pending_buildkite_publisher_form))] %} + {% set default_publisher_index = (publishers | map(attribute=0) | map("lower") | list).index(default_publisher_name) %} -
+
{% for publisher_name, _ in publishers %} - {% endfor %} diff --git a/warehouse/templates/manage/project/publishing.html b/warehouse/templates/manage/project/publishing.html index e952c531d720..4c9a73b725cb 100644 --- a/warehouse/templates/manage/project/publishing.html +++ b/warehouse/templates/manage/project/publishing.html @@ -60,92 +60,207 @@

{% trans %}Manage current publishers{% endtrans %}

{% trans %}Add a new publisher{% endtrans %}

{% if request.user.has_two_factor %} -

GitHub

+
+
+ -

- {% trans href="https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect" %} - Read more about GitHub Actions's OpenID Connect support here. - {% endtrans %} -

- - {{ form_error_anchor(github_publisher_form) }} - - - {{ form_errors(github_publisher_form) }} -
- - {{ github_publisher_form.owner(placeholder=gettext("owner"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", aria_describedby="owner-errors") }} -

- {% trans %}The GitHub organization name or GitHub username that owns the repository{% endtrans %} -

-
- {{ field_errors(github_publisher_form.owner) }} -
-
-
- - {{ github_publisher_form.repository(placeholder=gettext("repository"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", **{"aria-describedby":"repository-errors"}) }} -

- {% trans %}The name of the GitHub repository that contains the publishing workflow{% endtrans %} -

-
- {{ field_errors(github_publisher_form.repository) }} -
+
-
- - {{ github_publisher_form.workflow_filename(placeholder=gettext("workflow.yml"), class_="form-group__field", autocomplete="off", **{"aria-describedby":"workflow_filename-errors"}) }} -

- {% trans %}The filename of the publishing workflow. This file should exist in the .github/workflows/ directory in the repository configured above.{% endtrans %} + +

+

+ {% trans href="https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect" %} + Read more about GitHub Actions's OpenID Connect support here. + {% endtrans %}

-
- {{ field_errors(github_publisher_form.workflow_filename) }} -
+ + {{ form_error_anchor(github_publisher_form) }} + + + {{ form_errors(github_publisher_form) }} +
+ + {{ github_publisher_form.owner(placeholder=gettext("owner"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", aria_describedby="owner-errors") }} +

+ {% trans %}The GitHub organization name or GitHub username that owns the repository{% endtrans %} +

+
+ {{ field_errors(github_publisher_form.owner) }} +
+
+
+ + {{ github_publisher_form.repository(placeholder=gettext("repository"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", **{"aria-describedby":"repository-errors"}) }} +

+ {% trans %}The name of the GitHub repository that contains the publishing workflow{% endtrans %} +

+
+ {{ field_errors(github_publisher_form.repository) }} +
+
+
+ + {{ github_publisher_form.workflow_filename(placeholder=gettext("workflow.yml"), class_="form-group__field", autocomplete="off", **{"aria-describedby":"workflow_filename-errors"}) }} +

+ {% trans %}The filename of the publishing workflow. This file should exist in the .github/workflows/ directory in the repository configured above.{% endtrans %} +

+
+ {{ field_errors(github_publisher_form.workflow_filename) }} +
+
+
+ + {{ github_publisher_form.environment(placeholder=gettext("release"), class_="form-group__field", autocomplete="off", **{"aria-describedby":"environment-errors"}) }} +

+ {% trans href="https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment" %} + The name of the GitHub Actions environment + that the above workflow uses for publishing. This should be + configured under the repository's settings. While not required, a + dedicated publishing environment is strongly + encouraged, especially if your repository has + maintainers with commit access who shouldn't have PyPI publishing + access. + {% endtrans %} +

+
+ {{ field_errors(github_publisher_form.environment) }} +
+
+
+ +
+
-
- - {{ github_publisher_form.environment(placeholder=gettext("release"), class_="form-group__field", autocomplete="off", **{"aria-describedby":"environment-errors"}) }} -

- {% trans href="https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment" %} - The name of the GitHub Actions environment - that the above workflow uses for publishing. This should be - configured under the repository's settings. While not required, a - dedicated publishing environment is strongly - encouraged, especially if your repository has - maintainers with commit access who shouldn't have PyPI publishing - access. + +

+

+ {% trans href="https://buildkite.com/docs/agent/v3/cli-oidc#request-oidc-token" %} + Read more about Buildkite's OpenID Connect support here. {% endtrans %}

-
- {{ field_errors(github_publisher_form.environment) }} -
-
-
- + +
+ + {{ form_errors(buildkite_publisher_form) }} +
+ + {{ buildkite_publisher_form.organization_slug(placeholder=gettext("acme-inc"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", aria_describedby="organization_slug-errors") }} +

+ {% trans %}The slug of the Buildkite organization that owns the pipeline{% endtrans %} +

+
+ {{ field_errors(buildkite_publisher_form.organization_slug) }} +
+
+
+ + {{ buildkite_publisher_form.pipeline_slug(placeholder=gettext("widgets-client"), autocomplete="off", autocapitalize="off", spellcheck="false", class_="form-group__field", aria_describedby="pipeline_slug-errors") }} +

+ {% trans %}The slug of the Buildkite pipeline that contains the publishing steps{% endtrans %} +

+
+ {{ field_errors(buildkite_publisher_form.pipeline_slug) }} +
+
+
+ + {{ buildkite_publisher_form.build_branch(placeholder=gettext("main"), class_="form-group__field", autocomplete="off", aria_describedby="build_branch-errors") }} +

+ {% trans %}Restrict publishing to a specific branch by supplying a branch name{% endtrans %} +

+
+ {{ field_errors(buildkite_publisher_form.build_branch) }} +
+
+
+ + {{ buildkite_publisher_form.build_tag(placeholder=gettext("main"), class_="form-group__field", autocomplete="off", aria_describedby="build_tag-errors") }} +

+ {% trans %}Restrict publishing to a specific tag by supplying a tag name{% endtrans %} +

+
+ {{ field_errors(buildkite_publisher_form.build_tag) }} +
+
+
+ + {{ buildkite_publisher_form.step_key(placeholder=gettext("publish-python-package"), class_="form-group__field", autocomplete="off", aria_describedby="step_key-errors") }} +

+ {% trans href="https://buildkite.com/docs/pipelines/command-step#agents" %} + Restrict publishing to a specific step in your pipeline by supplying a step key + {% endtrans %} +

+
+ {{ field_errors(buildkite_publisher_form.build_branch) }} +
+
+
+ +
+
- +
{% else %}{# user has not enabled 2FA #}