feat: Add package command (#84)

Co-authored-by: erezrokah <[email protected]>

Author: maaarcelino

.github/workflows/unittests.yml

@@ -13,6 +13,9 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - # Required for the package command tests to work
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.gitignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
+
+.DS_Store

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Copy the code and install dependencies
+COPY requirements.txt .
+COPY setup.cfg .
+COPY setup.py .
+COPY cloudquery cloudquery
+COPY main.py .
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+EXPOSE 7777
+
+ENTRYPOINT ["python3", "main.py"]
+
+CMD ["serve", "--address", "[::]:7777", "--log-format", "json", "--log-level", "info"]

cloudquery/sdk/internal/memdb/memdb.py

@@ -3,16 +3,75 @@
 from cloudquery.sdk import schema
 from typing import List, Generator, Dict
 import pyarrow as pa
+from cloudquery.sdk.types import JSONType
 
 NAME = "memdb"
 VERSION = "development"
 
 
 class MemDB(plugin.Plugin):
     def __init__(self) -> None:
-        super().__init__(NAME, VERSION)
+        super().__init__(
+            NAME, VERSION, opts=plugin.plugin.Options(team="cloudquery", kind="source")
+        )
         self._db: Dict[str, pa.RecordBatch] = {}
-        self._tables: Dict[str, schema.Table] = {}
+        self._tables: Dict[str, schema.Table] = {
+            "table_1": schema.Table(
+                name="table_1",
+                columns=[
+                    schema.Column(
+                        name="name",
+                        type=pa.string(),
+                        primary_key=True,
+                        not_null=True,
+                        unique=True,
+                    ),
+                    schema.Column(
+                        name="id",
+                        type=pa.string(),
+                        primary_key=True,
+                        not_null=True,
+                        unique=True,
+                        incremental_key=True,
+                    ),
+                ],
+                title="Table 1",
+                description="Test Table 1",
+                is_incremental=True,
+                relations=[
+                    schema.Table(
+                        name="table_1_relation_1",
+                        columns=[
+                            schema.Column(
+                                name="name",
+                                type=pa.string(),
+                                primary_key=True,
+                                not_null=True,
+                                unique=True,
+                            ),
+                            schema.Column(name="data", type=JSONType()),
+                        ],
+                        title="Table 1 Relation 1",
+                        description="Test Table 1 Relation 1",
+                    )
+                ],
+            ),
+            "table_2": schema.Table(
+                name="table_2",
+                columns=[
+                    schema.Column(
+                        name="name",
+                        type=pa.string(),
+                        primary_key=True,
+                        not_null=True,
+                        unique=True,
+                    ),
+                    schema.Column(name="id", type=pa.string()),
+                ],
+                title="Table 2",
+                description="Test Table 2",
+            ),
+        }
 
     def get_tables(self, options: plugin.TableOptions = None) -> List[plugin.Table]:
         tables = list(self._tables.values())

cloudquery/sdk/plugin/plugin.py

@@ -29,10 +29,32 @@ class SyncOptions:
     backend_options: BackendOptions = None
 
 
+@dataclass
+class BuildTarget:
+    os: str = None
+    arch: str = None
+
+
+@dataclass
+class Options:
+    dockerfile: str = None
+    build_targets: List[BuildTarget] = None
+    team: str = None
+    kind: str = None
+
+
 class Plugin:
-    def __init__(self, name: str, version: str) -> None:
+    def __init__(self, name: str, version: str, opts: Options = None) -> None:
         self._name = name
         self._version = version
+        self._opts = Options() if opts is None else opts
+        if self._opts.dockerfile is None:
+            self._opts.dockerfile = "Dockerfile"
+        if self._opts.build_targets is None:
+            self._opts.build_targets = [
+                BuildTarget("linux", "amd64"),
+                BuildTarget("linux", "arm64"),
+            ]
 
     def init(self, spec: bytes, no_connection: bool = False) -> None:
         pass
@@ -46,6 +68,18 @@ def name(self) -> str:
     def version(self) -> str:
         return self._version
 
+    def team(self) -> str:
+        return self._opts.team
+
+    def kind(self) -> str:
+        return self._opts.kind
+
+    def dockerfile(self) -> str:
+        return self._opts.dockerfile
+
+    def build_targets(self) -> List[BuildTarget]:
+        return self._opts.build_targets
+
     def get_tables(self, options: TableOptions) -> List[Table]:
         raise NotImplementedError()
 

cloudquery/sdk/schema/table.py

@@ -151,6 +151,8 @@ def filter_dfs_func(tt: List[Table], include, exclude, skip_dependent_tables: bo
     filtered_tables = []
     for t in tt:
         filtered_table = copy.deepcopy(t)
+        for r in filtered_table.relations:
+            r.parent = filtered_table
         filtered_table = _filter_dfs_impl(
             filtered_table, False, include, exclude, skip_dependent_tables
         )