Replace conflicting repository-service-tuf dep (#16098)

* Replace conflicting repository-service-tuf dep

Previously, `repository-service-tuf` (i.e. the RSTUF cli) was used to
bootstrap an RSTUF repo for development. This PR re-implements the
relevant parts of the cli locally in Warehouse and removes the
`repository-service-tuf` dependency, which conflicts with other
dependencies.

Change details
- Add lightweight RSTUF API client library (can be re-used for #15815)
- Add local `warehouse tuf bootstrap` cli subcommand, to wraps lib calls
- Invoke local cli via `make inittuf`
- Remove dependency

supersedes #15958 (cc @facutuesca @woodruffw)

Signed-off-by: Lukas Puehringer <[email protected]>

* Make payload arg in tuf cli "lazy"

Other than the regular click File, the LazyFile also has the "name"
attribute, when passing stdin via "-".

We print the name on success.

Signed-off-by: Lukas Puehringer <[email protected]>

* Add minimal unittest for TUF bootstrap cli

Signed-off-by: Lukas Puehringer <[email protected]>

* Add unit tests for RSTUF API client lib

Signed-off-by: Lukas Puehringer <[email protected]>

---------

Signed-off-by: Lukas Puehringer <[email protected]>

Author: lukpueh

Makefile

@@ -128,7 +128,7 @@ initdb: .state/docker-build-base .state/db-populated
 inittuf: .state/db-migrated
 	docker compose up -d rstuf-api
 	docker compose up -d rstuf-worker
-	docker compose run --rm web rstuf admin ceremony -b -u -f dev/rstuf/bootstrap.json --api-server http://rstuf-api
+	docker compose run --rm web python -m warehouse tuf bootstrap dev/rstuf/bootstrap.json --api-server http://rstuf-api
 
 runmigrations: .state/docker-build-base
 	docker compose run --rm web python -m warehouse db upgrade head

requirements/dev.txt

@@ -3,4 +3,3 @@ hupper>=1.9
 pip-tools>=1.0
 pyramid_debugtoolbar>=2.5
 pip-api
-repository-service-tuf

tests/unit/cli/test_tuf.py

@@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from pretend import call, call_recorder
+
+from warehouse.cli import tuf
+
+
+class TestTUF:
+    def test_bootstrap(self, cli, monkeypatch):
+        task_id = "123456"
+        server = "rstuf.api"
+        payload = ["foo"]
+
+        post = call_recorder(lambda *a: task_id)
+        wait = call_recorder(lambda *a: None)
+        monkeypatch.setattr(tuf, "post_bootstrap", post)
+        monkeypatch.setattr(tuf, "wait_for_success", wait)
+
+        result = cli.invoke(
+            tuf.bootstrap, args=["--api-server", server, "-"], input=json.dumps(payload)
+        )
+
+        assert result.exit_code == 0
+
+        assert post.calls == [call(server, payload)]
+        assert wait.calls == [call(server, task_id)]

tests/unit/tuf/test_tuf.py

@@ -0,0 +1,90 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from pretend import call, call_recorder, stub
+
+from warehouse import tuf
+
+
+class TestTUF:
+    server = "rstuf.api"
+    task_id = "123456"
+
+    def test_get_task_state(self, monkeypatch):
+        state = "SUCCESS"
+
+        resp_json = {"data": {"state": state}}
+        resp = stub(
+            raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json)
+        )
+        get = call_recorder(lambda *a: resp)
+        monkeypatch.setattr(tuf.requests, "get", get)
+
+        result = tuf.get_task_state(self.server, self.task_id)
+
+        assert result == state
+        assert get.calls == [call(f"{self.server}/api/v1/task?task_id={self.task_id}")]
+
+    def test_post_bootstrap(self, monkeypatch):
+        payload = ["foo"]
+
+        resp_json = {"data": {"task_id": self.task_id}}
+        resp = stub(
+            raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json)
+        )
+        post = call_recorder(lambda *a, **kw: resp)
+        monkeypatch.setattr(tuf.requests, "post", post)
+
+        # Test success
+        result = tuf.post_bootstrap(self.server, payload)
+
+        assert result == self.task_id
+        assert post.calls == [call(f"{self.server}/api/v1/bootstrap", json=payload)]
+
+        # Test fail with incomplete response json
+        del resp_json["data"]
+        with pytest.raises(tuf.RSTUFError):
+            tuf.post_bootstrap(self.server, payload)
+
+    def test_wait_for_success(self, monkeypatch):
+        get_task_state = call_recorder(lambda *a: "SUCCESS")
+        monkeypatch.setattr(tuf, "get_task_state", get_task_state)
+        tuf.wait_for_success(self.server, self.task_id)
+
+        assert get_task_state.calls == [call(self.server, self.task_id)]
+
+    @pytest.mark.parametrize(
+        "state, iterations",
+        [
+            ("PENDING", 20),
+            ("RUNNING", 20),
+            ("RECEIVED", 20),
+            ("STARTED", 20),
+            ("FAILURE", 1),
+            ("ERRORED", 1),
+            ("REVOKED", 1),
+            ("REJECTED", 1),
+            ("bogus", 1),
+        ],
+    )
+    def test_wait_for_success_error(self, state, iterations, monkeypatch):
+        monkeypatch.setattr(tuf.time, "sleep", lambda *a: None)
+
+        get_task_state = call_recorder(lambda *a: state)
+        monkeypatch.setattr(tuf, "get_task_state", get_task_state)
+
+        with pytest.raises(tuf.RSTUFError):
+            tuf.wait_for_success(self.server, self.task_id)
+
+        assert get_task_state.calls == [call(self.server, self.task_id)] * iterations

warehouse/cli/tuf.py

@@ -0,0 +1,33 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from warehouse.cli import warehouse
+from warehouse.tuf import post_bootstrap, wait_for_success
+
+
+@warehouse.group()
+def tuf():
+    """Manage TUF."""
+
+
+@tuf.command()
+@click.argument("payload", type=click.File("rb", lazy=True), required=True)
+@click.option("--api-server", required=True)
+def bootstrap(payload, api_server):
+    """Use payload file to bootstrap RSTUF server."""
+    task_id = post_bootstrap(api_server, json.load(payload))
+    wait_for_success(api_server, task_id)
+    print(f"Bootstrap completed using `{payload.name}`. 🔐 🎉")

warehouse/tuf/__init__.py

@@ -0,0 +1,74 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+RSTUF API client library
+"""
+
+import time
+
+from typing import Any
+
+import requests
+
+
+class RSTUFError(Exception):
+    pass
+
+
+def get_task_state(server: str, task_id: str) -> str:
+    resp = requests.get(f"{server}/api/v1/task?task_id={task_id}")
+    resp.raise_for_status()
+    return resp.json()["data"]["state"]
+
+
+def post_bootstrap(server: str, payload: Any) -> str:
+    resp = requests.post(f"{server}/api/v1/bootstrap", json=payload)
+    resp.raise_for_status()
+
+    # TODO: Ask upstream to not return 200 on error
+    resp_json = resp.json()
+    resp_data = resp_json.get("data")
+    if not resp_data:
+        raise RSTUFError(f"Error in RSTUF job: {resp_json}")
+
+    return resp_data["task_id"]
+
+
+def wait_for_success(server: str, task_id: str):
+    """Poll RSTUF task state API until success or error."""
+
+    retries = 20
+    delay = 1
+
+    for _ in range(retries):
+        state = get_task_state(server, task_id)
+
+        match state:
+            case "SUCCESS":
+                break
+
+            case "PENDING" | "RUNNING" | "RECEIVED" | "STARTED":
+                time.sleep(delay)
+                continue
+
+            case "FAILURE":
+                raise RSTUFError("RSTUF job failed, please check payload and retry")
+
+            case "ERRORED" | "REVOKED" | "REJECTED":
+                raise RSTUFError("RSTUF internal problem, please check RSTUF health")
+
+            case _:
+                raise RSTUFError(f"RSTUF job returned unexpected state: {state}")
+
+    else:
+        raise RSTUFError("RSTUF job failed, please check payload and retry")