✨ Allow more scicrunch RRIDs as classifiers (#2110)

Allows adding any identifier scicrunch (besides actual resources) as classifiers by using the scicrunch resolver entrypoint.

Author: pcrespov

services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v2.py

@@ -1,34 +1,32 @@
 import asyncio
 import logging
+from collections import deque
 from pathlib import Path
 from typing import Optional
-from collections import deque
 
 from aiohttp import web
-from aiopg.sa.result import ResultProxy, RowProxy
 from aiopg.sa.engine import SAConnection
-
-from simcore_postgres_database.models.scicrunch_resources import scicrunch_resources
+from aiopg.sa.result import ResultProxy, RowProxy
 from servicelib.pools import non_blocking_process_pool_executor
+from simcore_postgres_database.models.scicrunch_resources import scicrunch_resources
+from simcore_service_webserver.catalog_client import get_service
+from simcore_service_webserver.projects.projects_api import get_project_for_user
+from simcore_service_webserver.projects.projects_exceptions import ProjectsException
+from simcore_service_webserver.scicrunch.db import ResearchResourceRepository
 
 from ..exceptions import ExporterException
-from .formatter_v1 import FormatterV1
 from .base_formatter import BaseFormatter
+from .formatter_v1 import FormatterV1
 from .sds import write_sds_directory_content
-from .sds.xlsx.templates.submission import SubmissionDocumentParams
-from .sds.xlsx.templates.dataset_description import DatasetDescriptionParams
 from .sds.xlsx.templates.code_description import (
-    CodeDescriptionParams,
     CodeDescriptionModel,
-    RRIDEntry,
+    CodeDescriptionParams,
     InputsEntryModel,
     OutputsEntryModel,
+    RRIDEntry,
 )
-
-from simcore_service_webserver.projects.projects_exceptions import ProjectsException
-from simcore_service_webserver.projects.projects_api import get_project_for_user
-from simcore_service_webserver.catalog_client import get_service
-from simcore_service_webserver.scicrunch.scicrunch_db import ResearchResourceRepository
+from .sds.xlsx.templates.dataset_description import DatasetDescriptionParams
+from .sds.xlsx.templates.submission import SubmissionDocumentParams
 
 log = logging.getLogger(__name__)
 

services/web/server/src/simcore_service_webserver/groups_classifiers.py

@@ -18,7 +18,7 @@
 from simcore_postgres_database.models.classifiers import group_classifiers
 
 from .constants import APP_DB_ENGINE_KEY
-from .scicrunch.scicrunch_db import ResearchResourceRepository
+from .scicrunch.db import ResearchResourceRepository
 from .scicrunch.service_client import SciCrunch
 
 logger = logging.getLogger(__name__)
@@ -124,7 +124,7 @@ async def build_rrids_tree_view(app, tree_view_mode="std") -> Dict[str, Any]:
                 classifier=resource.rrid,
                 display_name=resource.name.title(),
                 short_description=resource.description,
-                url=scicrunch.get_rrid_link(resource.rrid),
+                url=scicrunch.get_resolver_web_url(resource.rrid),
             )
 
             node = validated_item.display_name.replace(":", " ")

services/web/server/src/simcore_service_webserver/groups_handlers.py

@@ -14,9 +14,10 @@
     UserInsufficientRightsError,
 )
 from .login.decorators import RQT_USERID_KEY, login_required
-from .scicrunch.scicrunch_db import ResearchResourceRepository
-from .scicrunch.scicrunch_models import ResearchResource, ResourceHit
-from .scicrunch.service_client import InvalidRRID, SciCrunch, ScicrunchError
+from .scicrunch.db import ResearchResourceRepository
+from .scicrunch.errors import ScicrunchError
+from .scicrunch.models import ResearchResource, ResourceHit
+from .scicrunch.service_client import InvalidRRID, SciCrunch
 from .security_decorators import permission_required
 from .users_exceptions import UserNotFoundError
 

services/web/server/src/simcore_service_webserver/scicrunch/__init__.py

@@ -1,7 +1,7 @@
 """
    Submodule to interact with K-Core's https://scicrunch.org service
     - client to validate and get info about RRIDs via scicrunch's API (service_client)
-    - keeps validated RRIDs in pg-database (scicrunch_db)
+    - keeps validated RRIDs in pg-database (scicrunch.db)
     - define models for all interfaces: scicrunch API, postgres DB and webserver API (scicrunch_models)
 
    NOTE: should have no dependencies with other modules in this service

services/web/server/src/simcore_service_webserver/scicrunch/_config.py

@@ -6,15 +6,16 @@
 
 class SciCrunchSettings(BaseSettings):
 
-    api_base_url: HttpUrl = Field(
+    SCICRUNCH_API_BASE_URL: HttpUrl = Field(
         f"{SCICRUNCH_DEFAULT_URL}/api/1",
         description="Base url to scicrunch API's entrypoint",
     )
 
     # NOTE: Login in https://scicrunch.org and get API Key under My Account -> API Keys
     # WARNING: this needs to be setup in osparc-ops before deploying
-    api_key: SecretStr
+    SCICRUNCH_API_KEY: SecretStr
 
-    class Config:
-        case_sensitive = False
-        env_prefix = "SCICRUNCH_"
+    SCICRUNCH_RESOLVER_BASE_URL: HttpUrl = Field(
+        f"{SCICRUNCH_DEFAULT_URL}/resolver",
+        description="Base url to scicrunch resolver entrypoint",
+    )

services/web/server/src/simcore_service_webserver/scicrunch/_resolver.py

@@ -0,0 +1,110 @@
+"""
+    Layer to interact withscicrunch service resolver API
+    SEE https://scicrunch.org/resolver
+
+"""
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from aiohttp import ClientSession
+from pydantic import Field
+from pydantic.main import BaseModel
+from pydantic.types import NonNegativeInt
+
+from ._config import SciCrunchSettings
+
+logger = logging.getLogger(__name__)
+
+
+# MODELS ---------------------------
+
+# This is a partial model from the resolver response
+# that extracts the information we are interested
+# NOTE: this model was deduced by trial-and-error
+#
+class ItemInfo(BaseModel):
+    description: str = ""
+    name: str
+    identifier: str
+
+
+class RRIDInfo(BaseModel):
+    is_unique: bool = True
+    proper_citation: str = Field(..., alias="properCitation")
+
+
+class HitSource(BaseModel):
+    item: ItemInfo
+    rrid: RRIDInfo
+
+    def flatten_dict(self) -> Dict[str, Any]:
+        """Used as an output"""
+        return {**self.item.dict(), **self.rrid.dict()}
+
+
+class HitDetail(BaseModel):
+    source: HitSource = Field(..., alias="_source")
+
+
+class Hits(BaseModel):
+    total: NonNegativeInt
+    hits: List[HitDetail]
+
+
+class ResolverInfo(BaseModel):
+    uri: str
+    timestamp: datetime
+
+
+class ResolverResponseBody(BaseModel):
+    hits: Hits
+    resolver: ResolverInfo
+
+
+class ResolvedItem(BaseModel):
+    """ Result model for resolve_rrid """
+
+    description: str
+    name: str
+    identifier: str
+    is_unique: bool
+    proper_citation: str
+
+
+# REQUESTS --------------------------------
+
+
+async def resolve_rrid(
+    identifier: str, client: ClientSession, settings: SciCrunchSettings
+) -> Optional[ResolvedItem]:
+    """
+    Provides a API to access to results as provided by this web https://scicrunch.org/resolver
+
+    """
+    # Example https://scicrunch.org/resolver/RRID:AB_90755.json
+    identifier = identifier.strip()
+    url = f"{settings.SCICRUNCH_RESOLVER_BASE_URL}/{identifier}.json"
+
+    async with client.get(url, raise_for_status=True) as resp:
+        body = await resp.json()
+
+    # process and simplify response
+    resolved = ResolverResponseBody.parse_obj(body)
+    if resolved.hits.total == 0:
+        return None
+
+    hit = resolved.hits.hits[0].source
+
+    if resolved.hits.total > 1:
+        logger.warning(
+            "Multiple hits (%d) for '%s'. Returning first",
+            resolved.hits.total,
+            identifier,
+        )
+    else:
+        assert resolved.hits.total == 1  # nosec
+
+    output = ResolvedItem.parse_obj(hit.flatten_dict())
+    return output

services/web/server/src/simcore_service_webserver/scicrunch/_rest.py

@@ -0,0 +1,122 @@
+"""
+   Functions and models to query scicrunch service REST API (https://scicrunch.org/api/)
+
+    - http client for API requests
+    - Error handling:
+        - translates network errors
+        - translates request error codes
+
+    Free functions with raw request scicrunch.org API
+    - client request context
+    - raise_for_status=True -> Raise an aiohttp.ClientResponseError if the response status is 400 or higher
+    - validates response and prunes using pydantic models
+
+    SEE test_scicrunch_service_api.py
+"""
+
+import logging
+from typing import Any, Dict, List, Union
+
+from aiohttp import ClientSession
+from pydantic import BaseModel, Field
+from yarl import URL
+
+from ._config import SciCrunchSettings
+from .models import ResourceHit
+
+logger = logging.getLogger(__name__)
+
+
+# MODELS --
+#
+# NOTE: These models are a trucated version of the data payload for a scicrunch response.#
+# NOTE: Examples of complete responses can be found in test_scicrunch.py::mock_scicrunch_service_api
+#
+
+
+class FieldItem(BaseModel):
+    field_name: str = Field(..., alias="field")
+    required: bool
+    value: Union[str, None, List[Any]] = None
+
+
+class ResourceView(BaseModel):
+    resource_fields: List[FieldItem] = Field([], alias="fields")
+    version: int
+    curation_status: str
+    last_curated_version: int
+    scicrunch_id: str
+
+    @classmethod
+    def from_response_payload(cls, payload: Dict):
+        assert payload["success"] == True  # nosec
+        return cls(**payload["data"])
+
+    @property
+    def is_curated(self) -> bool:
+        return self.curation_status.lower() == "curated"
+
+    def _get_field(self, fieldname: str):
+        for field in self.resource_fields:
+            if field.field_name == fieldname:
+                return field.value
+        raise ValueError(f"Cannot file expected field {fieldname}")
+
+    def get_name(self):
+        return str(self._get_field("Resource Name"))
+
+    def get_description(self):
+        return str(self._get_field("Description"))
+
+    def get_resource_url(self):
+        return URL(str(self._get_field("Resource URL")))
+
+
+class ListOfResourceHits(BaseModel):
+    __root__: List[ResourceHit]
+
+
+# REQUESTS
+
+
+async def get_all_versions(
+    unprefixed_rrid: str, client: ClientSession, settings: SciCrunchSettings
+) -> List[Dict[str, Any]]:
+    async with client.get(
+        f"{settings.SCICRUNCH_API_BASE_URL}/resource/versions/all/{unprefixed_rrid}",
+        params={"key": settings.SCICRUNCH_API_KEY.get_secret_value()},
+        raise_for_status=True,
+    ) as resp:
+        body = await resp.json()
+        return body.get("data") if body.get("success") else []
+
+
+async def get_resource_fields(
+    rrid: str, client: ClientSession, settings: SciCrunchSettings
+) -> ResourceView:
+    async with client.get(
+        f"{settings.SCICRUNCH_API_BASE_URL}/resource/fields/view/{rrid}",
+        params={"key": settings.SCICRUNCH_API_KEY.get_secret_value()},
+        raise_for_status=True,
+    ) as resp:
+        body = await resp.json()
+
+        assert body.get("success")  # nosec
+        return ResourceView(**body.get("data", {}))
+
+
+async def autocomplete_by_name(
+    guess_name: str, client: ClientSession, settings: SciCrunchSettings
+) -> ListOfResourceHits:
+    async with client.get(
+        f"{settings.SCICRUNCH_API_BASE_URL}/resource/fields/autocomplete",
+        params={
+            "key": settings.SCICRUNCH_API_KEY.get_secret_value(),
+            "field": "Resource Name",
+            "value": guess_name.strip(),
+        },
+        raise_for_status=True,
+    ) as resp:
+        body = await resp.json()
+        assert body.get("success")  # nosec
+        return ListOfResourceHits.parse_obj(body.get("data", []))

services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_db.py renamed to services/web/server/src/simcore_service_webserver/scicrunch/db.py

@@ -12,7 +12,7 @@
 from simcore_postgres_database.models.scicrunch_resources import scicrunch_resources
 from sqlalchemy.dialects.postgresql import insert as sa_pg_insert
 
-from .scicrunch_models import ResearchResource, ResearchResourceAtdB
+from .models import ResearchResource, ResearchResourceAtdB
 
 logger = logging.getLogger(__name__)