♻️ Is922/select default wallet and pricing plan in the backend part 2 (#4869)

Co-authored-by: Pedro Crespo <[email protected]>

Author: matusdrobuliak66

packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py

@@ -1,5 +1,6 @@
 from typing import Any, ClassVar, TypeAlias
 
+from models_library.resource_tracker import HardwareInfo, PricingInfo
 from pydantic import BaseModel, ByteSize, Field
 
 from ..services import ServicePortKey
@@ -46,6 +47,14 @@ class DynamicServiceCreate(ServiceDetails):
         default=None,
         description="contains information about the wallet used to bill the running service",
     )
+    pricing_info: PricingInfo | None = Field(
+        default=None,
+        description="contains pricing information (ex. pricing plan and unit ids)",
+    )
+    hardware_info: HardwareInfo | None = Field(
+        default=None,
+        description="contains harware information (ex. aws_ec2_instances)",
+    )
 
     class Config:
         schema_extra: ClassVar[dict[str, Any]] = {
@@ -62,6 +71,8 @@ class Config:
                     "examples"
                 ][0],
                 "wallet_info": WalletInfo.Config.schema_extra["examples"][0],
+                "pricing_info": PricingInfo.Config.schema_extra["examples"][0],
+                "hardware_info": HardwareInfo.Config.schema_extra["examples"][0],
             }
         }
 

packages/models-library/src/models_library/resource_tracker.py

@@ -1,7 +1,7 @@
 from enum import auto
-from typing import TypeAlias
+from typing import Any, ClassVar, NamedTuple, TypeAlias
 
-from pydantic import PositiveInt
+from pydantic import BaseModel, PositiveInt
 
 from .utils.enums import StrAutoEnum
 
@@ -37,3 +37,37 @@ class CreditClassification(StrAutoEnum):
 
 class PricingPlanClassification(StrAutoEnum):
     TIER = auto()
+
+
+class PricingInfo(BaseModel):
+    pricing_plan_id: PricingPlanId
+    pricing_unit_id: PricingUnitId
+    pricing_unit_cost_id: PricingUnitCostId
+
+    class Config:
+        schema_extra: ClassVar[dict[str, Any]] = {
+            "examples": [
+                {"pricing_plan_id": 1, "pricing_unit_id": 1, "pricing_unit_cost_id": 1}
+            ]
+        }
+
+
+class HardwareInfo(BaseModel):
+    aws_ec2_instances: list[str]
+
+    class Config:
+        schema_extra: ClassVar[dict[str, Any]] = {
+            "examples": [{"aws_ec2_instances": ["c6a.4xlarge"]}]
+        }
+
+
+class PricingAndHardwareInfoTuple(NamedTuple):
+    pricing_plan_id: PricingPlanId
+    pricing_unit_id: PricingUnitId
+    current_cost_per_unit_id: PricingUnitCostId
+    aws_ec2_instances: list[str]
+
+
+class PricingPlanAndUnitIdsTuple(NamedTuple):
+    pricing_plan_id: PricingPlanId
+    pricing_unit_id: PricingUnitId

packages/postgres-database/src/simcore_postgres_database/migration/versions/5c62b190e124_migration_of_aws_ec2_instances_data_in_.py

@@ -0,0 +1,32 @@
+"""migration of aws_ec2_instances data in pricing units
+
+Revision ID: 5c62b190e124
+Revises: 7777d181dc1f
+Create Date: 2023-10-17 05:15:29.780925+00:00
+
+"""
+from alembic import op
+from simcore_postgres_database.models.resource_tracker_pricing_units import (
+    resource_tracker_pricing_units,
+)
+
+# revision identifiers, used by Alembic.
+revision = "5c62b190e124"
+down_revision = "7777d181dc1f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # One time migration to populate specific info with some reasonable value, it will be changed manually based on concrete needs
+    op.execute(
+        resource_tracker_pricing_units.update().values(
+            specific_info={"aws_ec2_instances": ["t3.medium"]}
+        )
+    )
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py

@@ -16,6 +16,7 @@
 from models_library.callbacks_mapping import CallbacksMapping
 from models_library.generated_models.docker_rest_api import ContainerState, Status2
 from models_library.projects_nodes_io import NodeID
+from models_library.resource_tracker import HardwareInfo, PricingInfo
 from models_library.service_settings_labels import (
     DynamicSidecarServiceLabels,
     PathMappingsLabel,
@@ -427,6 +428,14 @@ def endpoint(self) -> AnyHttpUrl:
         default=None,
         description="contains information about the wallet used to bill the running service",
     )
+    pricing_info: PricingInfo | None = Field(
+        default=None,
+        description="contains pricing information so we know what is the cost of running of the service",
+    )
+    hardware_info: HardwareInfo | None = Field(
+        default=None,
+        description="contains harware information so we know on which hardware to run the service",
+    )
 
     @property
     def get_proxy_endpoint(self) -> AnyHttpUrl:
@@ -485,6 +494,8 @@ def from_http_request(
             "request_simcore_user_agent": request_simcore_user_agent,
             "dynamic_sidecar": {"service_removal_state": {"can_save": can_save}},
             "wallet_info": service.wallet_info,
+            "pricing_info": service.pricing_info,
+            "hardware_info": service.hardware_info,
         }
         if run_id:
             obj_dict["run_id"] = run_id

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py

@@ -15,7 +15,6 @@
 
 from .....core.settings import DynamicSidecarSettings
 from .....models.dynamic_services_scheduler import SchedulerData
-from .....modules.resource_usage_client import ResourceUsageApi
 from .....utils.db import get_repository
 from ....db.repositories.groups_extra_properties import GroupsExtraPropertiesRepository
 from ....db.repositories.projects import ProjectsRepository
@@ -132,14 +131,9 @@ async def progress_create_containers(
     if scheduler_data.wallet_info:
         wallet_id = scheduler_data.wallet_info.wallet_id
         wallet_name = scheduler_data.wallet_info.wallet_name
-        resource_usage_api = ResourceUsageApi.get_from_state(app)
-        (
-            pricing_plan_id,
-            pricing_unit_id,
-            pricing_unit_cost_id,
-        ) = await resource_usage_api.get_default_service_pricing_plan_and_pricing_unit(
-            scheduler_data.product_name, scheduler_data.key, scheduler_data.version
-        )
+        pricing_plan_id = scheduler_data.pricing_info.pricing_plan_id
+        pricing_unit_id = scheduler_data.pricing_info.pricing_unit_id
+        pricing_unit_cost_id = scheduler_data.pricing_info.pricing_unit_cost_id
 
     metrics_params = CreateServiceMetricsAdditionalParams(
         wallet_id=wallet_id,

services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py

@@ -12,6 +12,7 @@
 from models_library.aiodocker_api import AioDockerServiceSpec
 from models_library.callbacks_mapping import CallbacksMapping
 from models_library.docker import to_simcore_runtime_docker_label_key
+from models_library.resource_tracker import HardwareInfo, PricingInfo
 from models_library.service_settings_labels import (
     SimcoreServiceLabels,
     SimcoreServiceSettingsLabel,
@@ -153,6 +154,8 @@ def expected_dynamic_sidecar_spec(
                     "request_simcore_user_agent": request_simcore_user_agent,
                     "restart_policy": "on-inputs-downloaded",
                     "wallet_info": WalletInfo.Config.schema_extra["examples"][0],
+                    "pricing_info": PricingInfo.Config.schema_extra["examples"][0],
+                    "hardware_info": HardwareInfo.Config.schema_extra["examples"][0],
                     "service_name": "dy-sidecar_75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                     "service_port": 65534,
                     "service_resources": {

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py

@@ -19,7 +19,7 @@ class PricingUnitCostsDB(BaseModel):
     valid_from: datetime
     valid_to: datetime | None
     created: datetime
-    comment: str
+    comment: str | None
     modified: datetime
 
     class Config:

services/web/server/src/simcore_service_webserver/catalog/_handlers.py

@@ -332,13 +332,13 @@ async def get_service_pricing_plan(request: Request):
     ctx = CatalogRequestContext.create(request)
     path_params = parse_request_path_parameters_as(ServicePathParams, request)
 
-    service_pricing_plan: ServicePricingPlanGet = (
-        await get_default_service_pricing_plan(
-            app=request.app,
-            product_name=ctx.product_name,
-            service_key=path_params.service_key,
-            service_version=path_params.service_version,
-        )
+    service_pricing_plan = await get_default_service_pricing_plan(
+        app=request.app,
+        product_name=ctx.product_name,
+        service_key=path_params.service_key,
+        service_version=path_params.service_version,
     )
 
-    return envelope_json_response(service_pricing_plan)
+    return envelope_json_response(
+        parse_obj_as(ServicePricingPlanGet, service_pricing_plan)
+    )

services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py

@@ -12,6 +12,7 @@
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeIDStr
 from models_library.rabbitmq_messages import ProgressRabbitMessageProject, ProgressType
+from models_library.resource_tracker import HardwareInfo, PricingInfo
 from models_library.services import ServicePortKey
 from models_library.services_resources import (
     ServiceResourcesDict,
@@ -96,6 +97,8 @@ async def run_dynamic_service(
     simcore_user_agent: str,
     service_resources: ServiceResourcesDict,
     wallet_info: WalletInfo | None,
+    pricing_info: PricingInfo | None,
+    hardware_info: HardwareInfo | None,
 ) -> DataType:
     """
     Requests to run (i.e. create and start) a dynamic service:
@@ -115,6 +118,8 @@ async def run_dynamic_service(
             service_resources
         ),
         "wallet_info": wallet_info,
+        "pricing_info": pricing_info,
+        "hardware_info": hardware_info,
     }
 
     headers = {

services/web/server/src/simcore_service_webserver/director_v2/_handlers.py

@@ -21,6 +21,7 @@
     GroupExtraPropertiesRepo,
 )
 from simcore_service_webserver.db.plugin import get_database_engine
+from simcore_service_webserver.users.exceptions import UserDefaultWalletNotFoundError
 
 from .._constants import RQ_PRODUCT_KEY
 from .._meta import API_VTAG as VTAG
@@ -114,9 +115,7 @@ async def start_computation(request: web.Request) -> web.Response:
                 preference_class=user_preferences_api.PreferredWalletIdFrontendUserPreference,
             )
             if user_default_wallet_preference is None:
-                raise ValueError(
-                    "User does not have default wallet - this should not happen"
-                )
+                raise UserDefaultWalletNotFoundError(uid=req_ctx.user_id)
             project_wallet_id = parse_obj_as(
                 WalletID, user_default_wallet_preference.value
             )
@@ -198,6 +197,8 @@ async def start_computation(request: web.Request) -> web.Response:
             reason=exc.reason,
             http_error_cls=get_http_error(exc.status) or web.HTTPServiceUnavailable,
         )
+    except UserDefaultWalletNotFoundError as exc:
+        return create_error_response(exc, http_error_cls=web.HTTPNotFound)
 
 
 @routes.post(f"/{VTAG}/computations/{{project_id}}:stop", name="stop_computation")

services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py

@@ -59,6 +59,7 @@
 from ._nodes_api import NodeScreenshot, get_node_screenshots
 from .db import ProjectDBAPI
 from .exceptions import (
+    DefaultPricingUnitNotFoundError,
     NodeNotFoundError,
     ProjectNodeResourcesInsufficientRightsError,
     ProjectNodeResourcesInvalidError,
@@ -79,6 +80,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse:
             ProjectNotFoundError,
             NodeNotFoundError,
             UserDefaultWalletNotFoundError,
+            DefaultPricingUnitNotFoundError,
         ) as exc:
             raise web.HTTPNotFound(reason=f"{exc}") from exc
 
@@ -173,9 +175,9 @@ async def get_node(request: web.Request) -> web.Response:
 
         if "data" not in service_data:
             # dynamic-service NODE STATE
-            assert (
+            assert (  # nosec
                 parse_obj_as(NodeGet | NodeGetIdle, service_data) is not None
-            )  # nosec
+            )
             return envelope_json_response(service_data)
 
         # LEGACY-service NODE STATE

services/web/server/src/simcore_service_webserver/projects/_project_nodes_pricing_unit_handlers.py

@@ -6,6 +6,7 @@
 import logging
 
 from aiohttp import web
+from models_library.api_schemas_webserver.resource_usage import PricingUnitGet
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID
 from models_library.resource_tracker import PricingPlanId, PricingUnitId
@@ -83,7 +84,14 @@ async def get_project_node_pricing_unit(request: web.Request):
     pricing_unit_get = await rut_api.get_pricing_plan_unit(
         request.app, req_ctx.product_name, pricing_plan_id, pricing_unit_id
     )
-    return envelope_json_response(pricing_unit_get)
+    webserver_pricing_unit_get = PricingUnitGet(
+        pricing_unit_id=pricing_unit_get.pricing_unit_id,
+        unit_name=pricing_unit_get.unit_name,
+        unit_extra_info=pricing_unit_get.unit_extra_info,
+        current_cost_per_unit=pricing_unit_get.current_cost_per_unit,
+        default=pricing_unit_get.default,
+    )
+    return envelope_json_response(webserver_pricing_unit_get)
 
 
 class _ProjectNodePricingUnitPathParams(BaseModel):

services/web/server/src/simcore_service_webserver/projects/db.py

@@ -18,7 +18,11 @@
 from models_library.projects_comments import CommentID, ProjectsCommentsDB
 from models_library.projects_nodes import Node
 from models_library.projects_nodes_io import NodeID, NodeIDStr
-from models_library.resource_tracker import PricingPlanId, PricingUnitId
+from models_library.resource_tracker import (
+    PricingPlanAndUnitIdsTuple,
+    PricingPlanId,
+    PricingUnitId,
+)
 from models_library.users import UserID
 from models_library.utils.fastapi_encoders import jsonable_encoder
 from models_library.wallets import WalletDB, WalletID
@@ -802,7 +806,7 @@ async def get_project_node_pricing_unit_id(
         self,
         project_uuid: ProjectID,
         node_uuid: NodeID,
-    ) -> tuple | None:
+    ) -> PricingPlanAndUnitIdsTuple | None:
         async with self.engine.acquire() as conn:
             result = await conn.execute(
                 sa.select(
@@ -823,9 +827,7 @@ async def get_project_node_pricing_unit_id(
             )
             row = await result.fetchone()
             if row:
-                return parse_obj_as(
-                    tuple[PricingPlanId, PricingUnitId], (row[0], row[1])
-                )
+                return PricingPlanAndUnitIdsTuple(row[0], row[1])
             return None
 
     async def connect_pricing_unit_to_project_node(

services/web/server/src/simcore_service_webserver/projects/exceptions.py

@@ -106,3 +106,14 @@ class ProjectNodeResourcesInvalidError(BaseProjectError):
 
 class ProjectNodeResourcesInsufficientRightsError(BaseProjectError):
     ...
+
+
+class DefaultPricingUnitNotFoundError(BaseProjectError):
+    """Node was not found in project"""
+
+    def __init__(self, project_uuid: str, node_uuid: str):
+        super().__init__(
+            f"Default pricing unit not found for node {node_uuid} in project {project_uuid}"
+        )
+        self.node_uuid = node_uuid
+        self.project_uuid = project_uuid