fix: Fix JSON and UUID type checking, add JSON test, consolidate setup.py

cloudquery/sdk/scalar/scalar_factory.py

@@ -1,17 +1,18 @@
 import pyarrow as pa
-from .scalar import ScalarInvalidTypeError
+
+from cloudquery.sdk.types import UUIDType, JSONType
 from .binary import Binary
 from .bool import Bool
 from .date32 import Date32
 from .date64 import Date64
 from .float import Float
 from .int import Int
 from .list import List
+from .scalar import ScalarInvalidTypeError
 from .string import String
 from .timestamp import Timestamp
 from .uint import Uint
 from .uuid import UUID
-from cloudquery.sdk.types import UUIDType, JSONType
 
 
 class ScalarFactory:
@@ -85,9 +86,9 @@ def new_scalar(self, dt: pa.DataType):
         #     return ()
         elif dt_id == pa.types.lib.Type_TIMESTAMP:
             return Timestamp()
-        elif dt == UUIDType:
+        elif dt == UUIDType():
             return UUID()
-        elif dt == JSONType:
+        elif dt == JSONType():
             return String()
         else:
             raise ScalarInvalidTypeError("Invalid type {} for scalar".format(dt))

cloudquery/sdk/scheduler/scheduler.py

@@ -1,18 +1,19 @@
-from typing import List, Generator, Any
 import queue
-import time
 import structlog
+import time
+import traceback
+from concurrent import futures
 from enum import Enum
-from cloudquery.sdk.schema import Table, Resource
+from typing import Generator
+from typing import List, Generator, Any
+
 from cloudquery.sdk.message import (
     SyncMessage,
     SyncInsertMessage,
     SyncMigrateTableMessage,
 )
-from concurrent import futures
-from typing import Generator
+from cloudquery.sdk.schema import Table, Resource
 from .table_resolver import TableResolver
-import traceback
 
 QUEUE_PER_WORKER = 100
 
@@ -103,11 +104,13 @@ def resolve_table(
                 )
             total_resources = 0
             for item in resolver.resolve(client, parent_item):
+                print("item", item)
                 try:
                     resource = self.resolve_resource(
                         resolver, client, parent_item, item
                     )
                 except Exception as e:
+                    print("exception", e)
                     self._logger.error(
                         "failed to resolve resource",
                         table=resolver.table.name,
@@ -181,6 +184,7 @@ def sync(
         finished_table_resolvers = 0
         while True:
             message = res.get()
+            print("got message", message)
             if type(message) == TableResolverStarted:
                 total_table_resolvers += message.count
                 if total_table_resolvers == finished_table_resolvers:
@@ -192,4 +196,5 @@ def sync(
                     break
                 continue
             yield message
-        thread.shutdown()
+        print("shutting down thread")
+        thread.shutdown(wait=True)

requirements.txt

@@ -1,21 +1 @@
-cloudquery-plugin-pb==0.0.14
-exceptiongroup==1.1.2
-black==23.7.0
-grpcio==1.56.2
-grpcio-tools==1.56.2
-iniconfig==2.0.0
-Jinja2==3.1.2
-MarkupSafe==2.1.3
-numpy==1.25.1
-packaging==23.1
-pandas==2.0.3
-pluggy==1.2.0
-protobuf==4.23.4
-pyarrow==12.0.1
-pytest==7.4.0
-python-dateutil==2.8.2
-pytz==2023.3
-six==1.16.0
-structlog==23.1.0
-tomli==2.0.1
-tzdata==2023.3
+.

setup.py

@@ -11,9 +11,26 @@
 
 dependencies = [
     "cloudquery-plugin-pb==0.0.14",
-    "pyarrow==12.0.1",
+    "exceptiongroup==1.1.2",
+    "black==23.7.0",
+    "grpcio==1.56.2",
+    "grpcio-tools==1.56.2",
+    "iniconfig==2.0.0",
     "Jinja2==3.1.2",
+    "MarkupSafe==2.1.3",
+    "numpy==1.25.1",
+    "packaging==23.1",
+    "pandas==2.0.3",
+    "pluggy==1.2.0",
+    "protobuf==4.23.4",
+    "pyarrow==12.0.1",
+    "pytest==7.4.0",
+    "python-dateutil==2.8.2",
+    "pytz==2023.3",
+    "six==1.16.0",
     "structlog==23.1.0",
+    "tomli==2.0.1",
+    "tzdata==2023.3",
 ]
 url = "https://github.com/cloudquery/plugin-sdk-python"
 

tests/types/json.py

@@ -0,0 +1,7 @@
+from cloudquery.sdk.types import JSONType
+
+
+def test_json_type():
+    j = JSONType()
+    # test equality
+    assert j == JSONType()