Added FI support.

Co-Authored-By: Raphael Gomes Santos <[email protected]>

Author: srfiorini

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 ign-*
+*.patch
 
 #IDE
 .idea/

hkpy/hkbase/hkrepository.py

@@ -9,10 +9,13 @@
 import copy
 import json
 import requests
+from urllib.parse import quote
 from io import TextIOWrapper, BufferedReader, BufferedIOBase
 
 from . import HKTransaction, generate_id, constants
 from ..hklib import hkfy, HKEntity, HKContext
+from ..hklib.fi.fi import FI
+from ..hklib.node import HKDataNode
 from ..oops import HKBError, HKpyError
 from ..utils import response_validator
 from ..common.result_set import ResultSet
@@ -73,29 +76,96 @@ def add_entities(self, entities: Union[HKEntity, List[HKEntity]], transaction: O
 
         url = f'{self.base._repository_uri}/{self.name}/entity/'
 
+        filtered_data_objects = []
+
         if not isinstance(entities, (list,tuple)):
             entities = [entities]
-
+        
         if isinstance(entities[0], HKEntity):
             entities = [x.to_dict() for x in entities]
         elif isinstance(entities[0], dict):
             pass
-            # entities = list(map(hkfy, entities))
         else:
             raise ValueError
 
+        data_entities, entities = self.filter_data_entities(entities)
+        if data_entities:
+            self.add_data_entities(data_entities)
+
         headers = copy.deepcopy(self._headers)
         headers['Content-Type'] = 'application/json'
 
         response = requests.put(url=url, data=json.dumps(entities), headers=headers)
         response_validator(response=response)
 
-    def filter_entities(self, filter_: Union[str, Dict]) -> List[HKEntity]:
+    def filter_data_entities(self, entities: Union[HKEntity, List[HKEntity]]):
+        """ filter the entities with raw data.
+        Return two lists:
+            dataentities: entities with raw data.
+            nondataentites: entities without raw data.
+        
+        Parameters
+        ----------
+        entities : (Union[HKDataNode, List[HKDataNode]]) entity or list of entities
+        transaction : (Optional[HKTransaction]) connection transaction
+
+        Returns
+        -------
+        dataentities: (List[HKEntity]) list of entities with raw data
+        nondataentites: (List[HKEntity]) list of entities without raw data
+        """
+        
+        dataentities = []
+        nondataentites = []
+        for entity in entities:
+            if 'raw_data' in entity:
+                dataentities.append(entity)
+            else:
+                nondataentites.append(entity)
+        return dataentities, nondataentites
+
+    def add_data_entities(self, dataentities: Union[HKDataNode, List[HKDataNode]], transaction: Optional[HKTransaction]=None) -> None:
+        """ Add entities with raw data to repository.
+        
+        Parameters
+        ----------
+        entities : (Union[HKDataNode, List[HKDataNode]]) data node or list of data nodes
+        transaction : (Optional[HKTransaction]) connection transaction
+        """
+
+        url = f'{self.base._repository_uri}/{self.name}/entity/'
+
+        if not dataentities:
+            return
+
+        if not isinstance(dataentities, (list,tuple)):
+            dataentities = [dataentities]
+
+        if isinstance(dataentities[0], HKEntity):
+            dataentities = [x.to_dict() for x in entities]
+        elif isinstance(dataentities[0], dict):
+            pass
+        else:
+            raise ValueError
+
+        files = {}
+        data = {}
+        for entity in dataentities:
+            file = entity.pop('raw_data')
+            files[entity['id']] = (entity['id'], file, entity['properties']['mimeType']) 
+            data[entity['id']] = json.dumps(entity)
+
+        response = requests.put(url=url, data=data, files=files)
+        response_validator(response=response)
+
+
+    def filter_entities(self, filter_: Union[str, Dict], bring_raw_data: Optional[bool]=False) -> List[HKEntity]:
         """ Get entities filtered by a css filter or json filter.
 
         Parameters
         ----------
         filter_ : (Union[str, Dict]) retrieval filter
+        bring_raw_data: Optional[bool] flag to define whether raw data should be fetched or not
 
         Returns
         -------
@@ -135,15 +205,20 @@ def check_list(the_filter, depth=0):
         except Exception as err:
             raise HKBError(message='Could not retrieve the entities.', error=err)
 
-        return [hkfy(entity) for entity in data.values()]
+        entities = [hkfy(entity) for entity in data.values()]
+        if bring_raw_data:
+            entities = self.retrieve_raw_data_from_data_entities(entities)
+
+        return entities
 
-    def get_entities(self, ids: List[Union[str, Dict]]) -> List[HKEntity]:
+    def get_entities(self, ids: List[Union[str, Dict]], bring_raw_data: Optional[bool]=False) -> List[HKEntity]:
         """ Get entities by an array of ids, where the id can be a string or an object containing remote information of
          virtual entities.
 
         Parameters
         ----------
         ids : List[Union[str, Dict]] entities identifiers
+        bring_raw_data: Optional[bool] flag to define whether raw data should be fetched or not
 
         Returns
         -------
@@ -163,7 +238,32 @@ def get_entities(self, ids: List[Union[str, Dict]]) -> List[HKEntity]:
         except Exception as err:
             raise HKBError(message='Could not retrieve the entities.', error=err)
 
-        return [hkfy(entity) for entity in data.values()]
+        entities = [hkfy(entity) for entity in data.values()]
+        if bring_raw_data:
+            entities = self.retrieve_raw_data_from_data_entities(entities)
+
+        return entities
+
+    def retrieve_raw_data_from_data_entities(self, entities: List[HKEntity]) -> List[HKEntity]:
+        """ Retrive data from storage from data entities 
+
+        Obs.: This is probably temporary. Probably the data will come from hkbase directly
+
+        Parameters
+        ----------
+        ids : List[HKEntity] entities
+
+        Returns
+        -------
+        (List[HKEntity]) list of data entities filled with their raw data
+        """
+
+        for i in range(len(entities)):
+            if 'mimeType' in entities[i].properties:
+                raw_data = self.get_object(entities[i].id_)
+                entities[i] = HKDataNode(raw_data, id_=entities[i].id_, parent=entities[i].parent, properties=entities[i].properties, metaproperties=entities[i].metaproperties)
+                
+        return entities
 
     def delete_entities(self, ids: Optional[Union[str, List[str], HKEntity, List[HKEntity]]] = None, transaction: Optional[HKTransaction]=None) -> None:
         """ Delete entities from the repository using their ids.
@@ -175,15 +275,12 @@ def delete_entities(self, ids: Optional[Union[str, List[str], HKEntity, List[HKE
 
         url = f'{self.base._repository_uri}/{self.name}/entity/'
 
-        if ids is None or len(ids) == 0:
-            ids = {}
-        else:
-            if not isinstance(ids, (list,tuple)):
-                ids = [ids]
-
-            if isinstance(ids[0], HKEntity):
-                ids = [x.id_ for x in ids]
-
+        if not isinstance(ids, (list,tuple)):
+            ids = [ids]
+        
+        if isinstance(ids[0], HKEntity):
+            ids = [x.id_ for x in ids]
+            
         response = requests.delete(url=url, data=json.dumps(ids), headers=self._headers)
         response_validator(response=response)
 
@@ -216,7 +313,9 @@ def import_data(self, fd: Union[str, TextIOWrapper], datatype: constants.Content
             raise HKpyError(message='Data formaat not supported.')
 
         if 'as_hk' in options and options['as_hk'] == True:
-            self.add_entities(entities=json.loads(fd))
+            entities = json.loads(fd)
+            entities = list(map(hkfy, entities.values()))
+            self.add_entities(list(entities))
 
         else:
             url = f'{self.base._repository_uri}/{self.name}/rdf'
@@ -323,6 +422,10 @@ def add_object(self, object_: [str, TextIOWrapper, BufferedReader, BufferedIOBas
         elif isinstance(object_, str) and os.path.isfile(object_):
             with open(object_ ,'rb') as f:
                 object_ = f.read()
+        elif isinstance(object_, str):
+            pass # do nothing
+        elif isinstance(object_, bytes):
+            pass # do nothing
         else:
             raise HKpyError(message='Object not valid.')
 
@@ -475,3 +578,36 @@ def _build_sparql_result(self, data) -> Union[SPARQLResultSet, bool]:
         if not ('results' in data and 'bindings' in data['results'] and 'head' in data):
             raise HKpyError(f'The given data is not of the expected format')
         return SPARQLResultSet(data)
+
+    def resolve_fi(self, fi: FI, raw = False) -> Union[bytes, List[HKEntity]] :
+        """
+        Resolve a full FI. Set raw to True in order to get raw output. Otherwise the function will try to
+        interpret the response (i.e. into hk objects)
+        """
+        quoted_fi = quote(fi.__str__(), safe="");
+        url = f'{self.base._repository_uri}/{self.name}/fi/{quoted_fi}'
+
+        response = requests.get(url=url, headers=self._headers)
+        _, data = response_validator(response=response, content='.')
+
+        if raw:
+            return data
+
+        # some extra response data converter
+        content_type = response.headers['Content-Type']
+        if content_type.startswith('hyperknowledge/graph'):
+            return hkfy(response.json())
+        elif content_type.startswith('hyperknowledge/node'):
+            return hkfy(response.json())
+        elif content_type.startswith('text'):
+            return response.content.decode()
+        else:
+            return data
+
+    def persist_fi(self, fi):
+
+        quoted_fi = quote(fi.__str__(), safe="");
+        url = f'{self.base._repository_uri}/{self.name}/fi/{quoted_fi}'
+
+        response = requests.put(url=url, headers=self._headers)
+        response_validator(response=response)

hkpy/hklib/__init__.py

@@ -11,7 +11,8 @@
 from .entity import HKEntity
 from .connector import HKConnector
 from .link import HKLink
-from .node import HKNode, HKContext, HKReferenceNode, HKTrail, HKAnyNode
+from .node import HKNode, HKContext, HKReferenceNode, HKTrail, HKAnyNode, HKDataNode
+from .graphbuilder import HKGraphBuilder
 
 def hkfy(entity: Union[str, Dict]) -> HKEntity:
     """ Convert an entity in string or dict format to a HKEntity object.
@@ -43,7 +44,7 @@ def hkfy(entity: Union[str, Dict]) -> HKEntity:
                 hke = HKNode(id_=entity['id'], parent=entity.get('parent'))
             elif entity['type'] == constants.HKType.REFERENCENODE:
                 ref = entity['ref'] if 'ref' in entity else None
-                hke = HKReferenceNode(id_=entity['id'], ref=ref, parent=entity.get('parent'))       
+                hke = HKReferenceNode(id_=entity['id'], ref=ref, parent=entity.get('parent'))
             if 'interfaces' in entity:
                 hke.interfaces = entity['interfaces']
 
@@ -52,6 +53,14 @@ def hkfy(entity: Union[str, Dict]) -> HKEntity:
 
         elif entity['type'] == constants.HKType.ANCHOR:
             raise NotImplementedError
+        elif entity['type'] == constants.HKType.GRAPH:
+            hke = HKGraph()
+            entities = []
+            for entity_type, graph_entity in entity.items():
+                if entity_type != 'type':
+                    for eid, e in graph_entity.items():
+                        entities.append(hkfy(e))
+            hke.add_entities(entities)
 
         if 'properties' in entity:
             hke.add_properties(properties=entity['properties'])

hkpy/hklib/entity.py

@@ -55,6 +55,9 @@ def add_properties(self, **kwargs) -> None:
         else:
             self.properties.update(kwargs)
 
+    def set_property(self, property, value):
+        self.properties[property] = value
+
     def add_metaproperties(self, **kwargs) -> None:
         """ Add metaproperties in the HKEntity.
 

hkpy/hklib/fi/__init__.py

@@ -0,0 +1,7 @@
+
+
+
+from .fi import FI
+from .fianchor import FIAnchor
+from .fioperator import FIOperator
+from .fiparser import parse_anchor, parse_fi, parse_id