diff --git a/cloudquery/sdk/docs/__init__.py b/cloudquery/sdk/docs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/cloudquery/sdk/docs/generator.py b/cloudquery/sdk/docs/generator.py deleted file mode 100644 index 7bc569e..0000000 --- a/cloudquery/sdk/docs/generator.py +++ /dev/null @@ -1,145 +0,0 @@ -import json -import os -import pathlib -import re -from typing import List - -import jinja2 - -from cloudquery.sdk.docs.markdown_templates import ALL_TABLES, ALL_TABLES_ENTRY, TABLE -from cloudquery.sdk.schema import Table - - -class JsonTable: - def __init__(self): - self.name = "" - self.title = "" - self.description = "" - self.columns = [] - self.relations = [] - - def to_dict(self): - return { - "name": self.name, - "title": self.title, - "description": self.description, - "columns": [col.to_dict() for col in self.columns], - "relations": [rel.to_dict() for rel in self.relations], - } - - -class JsonColumn: - def __init__(self): - self.name = "" - self.type = "" - self.is_primary_key = None - self.is_incremental_key = None - - def to_dict(self): - return { - "name": self.name, - "type": self.type, - "is_primary_key": self.is_primary_key, - "is_incremental_key": self.is_incremental_key, - } - - -class Generator: - def __init__(self, plugin_name: str, tables: List[Table]): - self._plugin_name = plugin_name - self._tables = sorted(tables) - - def generate(self, directory: str, format: str): - os.makedirs(directory, exist_ok=True) - if format == "markdown": - self._generate_markdown(directory) - elif format == "json": - self._generate_json(directory) - - def _generate_json(self, directory: str): - json_tables = self._jsonify_tables(self._tables) - buffer = bytes(json.dumps(json_tables, indent=2, ensure_ascii=False), "utf-8") - output_path = pathlib.Path(directory) / "__tables.json" - with output_path.open("wb") as f: - f.write(buffer) - return None - - def _jsonify_tables(self, tables): - json_tables = [] - for table in tables: - json_columns = [] - for col in table.columns: - json_column = JsonColumn() - json_column.name = col.name - json_column.type = str(col.type) - json_column.is_primary_key = col.primary_key - json_column.is_incremental_key = col.incremental_key - json_columns.append(json_column.__dict__) - - json_table = JsonTable() - json_table.name = table.name - json_table.title = table.title - json_table.description = table.description - json_table.columns = json_columns - json_table.relations = self._jsonify_tables(table.relations) - json_tables.append(json_table.__dict__) - - return json_tables - - def _generate_markdown(self, directory: str): - env = jinja2.Environment() - env.globals["indent_to_depth"] = self._indent_to_depth - env.globals["all_tables_entry"] = self._all_tables_entry - all_tables_template = env.from_string(ALL_TABLES) - rendered_all_tables = all_tables_template.render( - plugin_name=self._plugin_name, tables=self._tables - ) - formatted_all_tables = self._format_markdown(rendered_all_tables) - - with open(os.path.join(directory, "README.md"), "w") as f: - f.write(formatted_all_tables) - - for table in self._tables: - self._render_table(directory, env, table) - - def _render_table(self, directory: str, env: jinja2.Environment, table: Table): - table_template = env.from_string(TABLE) - table_md = table_template.render(table=table) - formatted_table_md = self._format_markdown(table_md) - with open(os.path.join(directory, table.name + ".md"), "w") as f: - f.write(formatted_table_md) - for relation in table.relations: - self._render_table(directory, env, relation) - - def _all_tables_entry(self, table: Table): - env = jinja2.Environment() - env.globals["indent_to_depth"] = self._indent_to_depth - env.globals["all_tables_entry"] = self._all_tables_entry - env.globals["indent_table_to_depth"] = self._indent_table_to_depth - entry_template = env.from_string(ALL_TABLES_ENTRY) - return entry_template.render(table=table) - - @staticmethod - def _indent_table_to_depth(table: Table) -> str: - s = "" - t = table - while t.parent is not None: - s += " " - t = t.parent - return s - - @staticmethod - def _indent_to_depth(text: str, depth: int) -> str: - indentation = depth * 4 # You can adjust the number of spaces as needed - lines = text.split("\n") - indented_lines = [(" " * indentation) + line for line in lines] - return "\n".join(indented_lines) - - @staticmethod - def _format_markdown(text: str) -> str: - re_match_newlines = re.compile(r"\n{3,}") - re_match_headers = re.compile(r"(#{1,6}.+)\n+") - - text = re_match_newlines.sub(r"\n\n", text) - text = re_match_headers.sub(r"\1\n\n", text) - return text diff --git a/cloudquery/sdk/docs/markdown_templates.py b/cloudquery/sdk/docs/markdown_templates.py deleted file mode 100644 index e27f421..0000000 --- a/cloudquery/sdk/docs/markdown_templates.py +++ /dev/null @@ -1,58 +0,0 @@ -ALL_TABLES = """# Source Plugin: {{ plugin_name }} -## Tables -{%- for table in tables %} -{{ all_tables_entry(table) }} -{%- endfor %}""" - -ALL_TABLES_ENTRY = """{{ indent_table_to_depth(table) }}- [{{ table.name }}]({{ table.name }}.md){% if table.is_incremental %} (Incremental){% endif %} -{%- for rel in table.relations %} -{{ all_tables_entry(rel) }} -{%- endfor %}""" - -TABLE = """# Table: {{ table.name }} - -This table shows data for {{ table.title }}. - -{{ table.description }} -{% set length = table.primary_keys | length %} -{% if length == 0 %} -This table does not have a primary key. -{% elif length == 1 %} -The primary key for this table is **{{ table.primary_keys[0] }}**. -{% else %} -The composite primary key for this table is ( -{%- for pk in table.primary_keys %} - {{- ", " if loop.index > 1 else "" }}**{{ pk }}** -{%- endfor -%} -). -{% endif %} - -{% if table.is_incremental %} -It supports incremental syncs{% set ik_length = table.incremental_keys | length %} -{%- if ik_length == 1 %} based on the **{{ table.incremental_keys[0] }}** column{% else %} based on the ( -{%- for ik in table.incremental_keys %} - {{- ", " if loop.index > 1 else "" }}**{{ ik }}** -{%- endfor -%} -) columns{% endif %}. -{%- endif %} - -{% if table.relations or table.parent %} -## Relations -{% endif %} -{% if table.parent %} -This table depends on [{{ table.parent.name }}]({{ table.parent.name }}.md). -{% endif %} -{% if table.relations %} -The following tables depend on {{ table.name }}: -{% for rel in table.relations %} - - [{{ rel.name }}]({{ rel.name }}.md) -{%- endfor %} -{% endif %} - -## Columns -| Name | Type | -| ------------- | ------------- | -{%- for column in table.columns %} -|{{ column.name }}{% if column.primary_key %} (PK){% endif %}{% if column.incremental_key %} (Incremental Key){% endif %}|`{{ column.type }}`| -{%- endfor %} -""" diff --git a/cloudquery/sdk/serve/plugin.py b/cloudquery/sdk/serve/plugin.py index f298c9a..23f56cc 100644 --- a/cloudquery/sdk/serve/plugin.py +++ b/cloudquery/sdk/serve/plugin.py @@ -10,12 +10,9 @@ from cloudquery.plugin_v3 import plugin_pb2_grpc from structlog import wrap_logger -from cloudquery.sdk.docs.generator import Generator from cloudquery.sdk.internal.servers.discovery_v1.discovery import DiscoveryServicer from cloudquery.sdk.internal.servers.plugin_v3 import PluginServicer -from cloudquery.sdk.plugin.plugin import Plugin, TableOptions - -DOC_FORMATS = ["json", "markdown"] +from cloudquery.sdk.plugin.plugin import Plugin _IS_WINDOWS = sys.platform == "win32" @@ -136,34 +133,10 @@ def run(self, args): help="network to serve on. can be tcp or unix", ) - doc_parser = subparsers.add_parser( - "doc", - formatter_class=argparse.RawTextHelpFormatter, - help="Generate documentation for tables", - description="""Generate documentation for tables. - -If format is markdown, a destination directory will be created (if necessary) containing markdown files. -Example: -doc ./output - -If format is JSON, a destination directory will be created (if necessary) with a single json file called __tables.json. -Example: -doc --format json . -""", - ) - doc_parser.add_argument("directory", type=str) - doc_parser.add_argument( - "--format", - type=str, - default="json", - help="output format. one of: {}".format(",".join(DOC_FORMATS)), - ) parsed_args = parser.parse_args(args) if parsed_args.command == "serve": self._serve(parsed_args) - elif parsed_args.command == "doc": - self._generate_docs(parsed_args) else: parser.print_help() sys.exit(1) @@ -185,16 +158,3 @@ def _serve(self, args): def stop(self): self._server.stop(5) - - def _generate_docs(self, args): - generator = Generator( - self._plugin.name(), - self._plugin.get_tables( - options=TableOptions( - tables=["*"], - skip_tables=[], - skip_dependent_tables=False, - ) - ), - ) - generator.generate(args.directory, args.format) diff --git a/tests/docs/__init__.py b/tests/docs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/docs/snapshots/README.md b/tests/docs/snapshots/README.md deleted file mode 100644 index 572188e..0000000 --- a/tests/docs/snapshots/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Source Plugin: test_plugin - -## Tables - -- [test_table](test_table.md) -- [test_table_composite_pk](test_table_composite_pk.md) (Incremental) -- [test_table_relations](test_table_relations.md) (Incremental) - - [test_table_child](test_table_child.md) - - [test_table_grandchild](test_table_grandchild.md) \ No newline at end of file diff --git a/tests/docs/snapshots/test_table.md b/tests/docs/snapshots/test_table.md deleted file mode 100644 index a00dc43..0000000 --- a/tests/docs/snapshots/test_table.md +++ /dev/null @@ -1,12 +0,0 @@ -# Table: test_table - -This table shows data for Test Table. - -The primary key for this table is **string**. - -## Columns - -| Name | Type | -| ------------- | ------------- | -|string (PK)|`string`| -|int32|`int32`| \ No newline at end of file diff --git a/tests/docs/snapshots/test_table_child.md b/tests/docs/snapshots/test_table_child.md deleted file mode 100644 index f06e9ee..0000000 --- a/tests/docs/snapshots/test_table_child.md +++ /dev/null @@ -1,20 +0,0 @@ -# Table: test_table_child - -This table shows data for Child Table. - -The primary key for this table is **pk1**. - -## Relations - -This table depends on [test_table_relations](test_table_relations.md). - -The following tables depend on test_table_child: - - - [test_table_grandchild](test_table_grandchild.md) - -## Columns - -| Name | Type | -| ------------- | ------------- | -|pk1 (PK)|`string`| -|fk1|`string`| \ No newline at end of file diff --git a/tests/docs/snapshots/test_table_composite_pk.md b/tests/docs/snapshots/test_table_composite_pk.md deleted file mode 100644 index 3ebead8..0000000 --- a/tests/docs/snapshots/test_table_composite_pk.md +++ /dev/null @@ -1,15 +0,0 @@ -# Table: test_table_composite_pk - -This table shows data for Composite PKs. - -The composite primary key for this table is (**pk1**, **pk2**). - -It supports incremental syncs based on the (**pk1**, **pk2**) columns. - -## Columns - -| Name | Type | -| ------------- | ------------- | -|pk1 (PK) (Incremental Key)|`string`| -|pk2 (PK) (Incremental Key)|`string`| -|int32|`int32`| \ No newline at end of file diff --git a/tests/docs/snapshots/test_table_grandchild.md b/tests/docs/snapshots/test_table_grandchild.md deleted file mode 100644 index 3568be9..0000000 --- a/tests/docs/snapshots/test_table_grandchild.md +++ /dev/null @@ -1,15 +0,0 @@ -# Table: test_table_grandchild - -This table shows data for Grandchild Table. - -The primary key for this table is **pk1**. - -## Relations - -This table depends on [test_table_child](test_table_child.md). - -## Columns - -| Name | Type | -| ------------- | ------------- | -|pk1 (PK)|`string`| \ No newline at end of file diff --git a/tests/docs/snapshots/test_table_relations.md b/tests/docs/snapshots/test_table_relations.md deleted file mode 100644 index b1e4664..0000000 --- a/tests/docs/snapshots/test_table_relations.md +++ /dev/null @@ -1,19 +0,0 @@ -# Table: test_table_relations - -This table shows data for Table Relations. - -The primary key for this table is **pk1**. - -It supports incremental syncs based on the () columns. - -## Relations - -The following tables depend on test_table_relations: - - - [test_table_child](test_table_child.md) - -## Columns - -| Name | Type | -| ------------- | ------------- | -|pk1 (PK)|`string`| \ No newline at end of file diff --git a/tests/docs/test_generator.py b/tests/docs/test_generator.py deleted file mode 100644 index 2bcae3f..0000000 --- a/tests/docs/test_generator.py +++ /dev/null @@ -1,105 +0,0 @@ -import glob -import os -import unittest -from tempfile import TemporaryDirectory - -import pyarrow as pa - -from cloudquery.sdk.docs.generator import Generator -from cloudquery.sdk.schema import Table, Column - -dirname = os.path.dirname(__file__) -SNAPSHOT_DIRECTORY = os.path.join(dirname, "snapshots") - - -def read_snapshot(name): - with open(os.path.join(SNAPSHOT_DIRECTORY, name)) as f: - return f.read() - - -def update_snapshot(name, content): - with open(os.path.join(SNAPSHOT_DIRECTORY, name), "w") as f: - f.write(content) - - -class T(unittest.TestCase): - def test_docs_generator_markdown(self): - tables = [ - Table( - name="test_table", - title="Test Table", - columns=[ - Column("string", pa.string(), primary_key=True), - Column("int32", pa.int32()), - ], - ), - Table( - name="test_table_composite_pk", - title="Composite PKs", - is_incremental=True, - columns=[ - Column("pk1", pa.string(), primary_key=True, incremental_key=True), - Column("pk2", pa.string(), primary_key=True, incremental_key=True), - Column("int32", pa.int32()), - ], - ), - Table( - name="test_table_relations", - title="Table Relations", - is_incremental=True, - columns=[ - Column("pk1", pa.string(), primary_key=True), - ], - relations=[ - Table( - name="test_table_child", - title="Child Table", - columns=[ - Column("pk1", pa.string(), primary_key=True), - Column("fk1", pa.string()), - ], - relations=[ - Table( - name="test_table_grandchild", - title="Grandchild Table", - columns=[Column("pk1", pa.string(), primary_key=True)], - ) - ], - ) - ], - ), - ] - - # set parent relations - tables[2].relations[0].parent = tables[2] - tables[2].relations[0].relations[0].parent = tables[2].relations[0] - - gen = Generator("test_plugin", tables) - with TemporaryDirectory() as d: - gen.generate(d, format="markdown") - - files = glob.glob(os.path.join(d, "*.md")) - file_names = [os.path.basename(f) for f in files] - assert sorted(file_names) == sorted( - [ - "README.md", - "test_table_composite_pk.md", - "test_table.md", - "test_table_relations.md", - "test_table_child.md", - "test_table_grandchild.md", - ] - ) - - updated_snapshots = False - for file_name in file_names: - with self.subTest(msg=file_name): - with open(os.path.join(d, file_name)) as f: - content = f.read() - try: - snapshot = read_snapshot(file_name) - self.assertEqual(snapshot, content) - except FileNotFoundError: - update_snapshot(file_name, content) - updated_snapshots = True - assert not updated_snapshots, "Updated snapshots" diff --git a/tests/scheduler/table_resolver.py b/tests/scheduler/table_resolver.py index 3a11575..b89e20b 100644 --- a/tests/scheduler/table_resolver.py +++ b/tests/scheduler/table_resolver.py @@ -7,7 +7,7 @@ @dataclass -class TestItem: +class Item: test_column2: int = 1 @@ -20,9 +20,7 @@ def test_table_resolver_resolve_column(): ], ) resource_dict = Resource(table=test_table, parent=None, item={"test_column": 1}) - resource_obj = Resource( - table=test_table, parent=None, item=TestItem(test_column2=2) - ) + resource_obj = Resource(table=test_table, parent=None, item=Item(test_column2=2)) test_resolver = TableResolver(table=test_table, child_resolvers=[]) test_resolver.resolve_column( client=None, resource=resource_dict, column_name="test_column"