feat: Add package command (#194)

Fixes cloudquery/cloudquery-issues#790

Author: erezrokah

Diff for: .github/workflows/ci.yml

@@ -19,12 +19,16 @@ jobs:
           distribution: "temurin"
           java-version: "18"
           cache: "gradle"
+      - # Required for the package command tests to work
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
       - name: Validate Gradle wrapper
         uses: gradle/wrapper-validation-action@63d15e7a1e697b1de6f3ba0507106f89100c8518
       - name: Build package
         run: ./gradlew build
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_ACTOR: ${{ github.actor }}
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v4
         if: success() || failure() # always run even if the previous step fails

Diff for: .gitignore

@@ -32,3 +32,5 @@ build
 # Intellij
 .idea
 .cq
+
+dist

Diff for: Dockerfile

@@ -0,0 +1,21 @@
+FROM --platform=$BUILDPLATFORM gradle:8.3-jdk20 as build
+ARG GITHUB_ACTOR
+ARG GITHUB_TOKEN
+
+WORKDIR /code
+
+COPY . .
+
+RUN gradle jar --no-daemon
+
+FROM eclipse-temurin:20-jre
+
+COPY --from=build /code/lib/build/libs/*.jar /app/app.jar
+
+EXPOSE 7777
+
+ENV _JAVA_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED"
+
+ENTRYPOINT ["java", "-jar", "/app/app.jar"]
+
+CMD [ "serve", "--address", "localhost:7777", "--log-format", "json", "--log-level", "info" ]

Diff for: docs/overview.md

@@ -0,0 +1,3 @@
+# MemDB Plugin
+
+Test docs for the MemDB plugin.

Diff for: lib/build.gradle

@@ -40,6 +40,7 @@ dependencies {
     implementation 'io.cloudquery:plugin-pb-java:0.0.15'
     implementation 'org.apache.arrow:arrow-memory-core:12.0.1'
     implementation 'org.apache.arrow:arrow-vector:12.0.1'
+    implementation 'commons-io:commons-io:2.15.1'
 
     implementation "com.fasterxml.jackson.core:jackson-core:2.16.1"
     implementation "com.fasterxml.jackson.core:jackson-annotations:2.16.1"
@@ -77,6 +78,17 @@ java {
     }
 }
 
+jar {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    manifest {
+        attributes "Main-Class": "io.cloudquery.MainClass"
+    }
+
+    from {
+        configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
+    }
+}
+
 publishing {
     repositories {
         maven {

Diff for: lib/src/main/java/io/cloudquery/memdb/MemDB.java

@@ -5,6 +5,7 @@
 import io.cloudquery.plugin.ClientNotInitializedException;
 import io.cloudquery.plugin.NewClientOptions;
 import io.cloudquery.plugin.Plugin;
+import io.cloudquery.plugin.PluginKind;
 import io.cloudquery.plugin.TableOutputStream;
 import io.cloudquery.scheduler.Scheduler;
 import io.cloudquery.schema.ClientMeta;
@@ -88,6 +89,8 @@ public void resolve(
 
   public MemDB() {
     super("memdb", "0.0.1");
+    setTeam("cloudquery");
+    setKind(PluginKind.Source);
   }
 
   @Override
@@ -144,12 +147,14 @@ public void close() {
 
   @Override
   public ClientMeta newClient(String spec, NewClientOptions options) throws Exception {
-    this.spec = Spec.fromJSON(spec);
     this.allTables = getTables();
     Tables.transformTables(this.allTables);
     for (Table table : this.allTables) {
       table.addCQIDs();
     }
+    if (!options.isNoConnection()) {
+      this.spec = Spec.fromJSON(spec);
+    }
     return new MemDBClient();
   }
 }

Diff for: lib/src/main/java/io/cloudquery/plugin/BuildArch.java

@@ -0,0 +1,16 @@
+package io.cloudquery.plugin;
+
+public enum BuildArch {
+  AMD64("amd64"),
+  ARM64("arm64");
+
+  public final String arch;
+
+  private BuildArch(String arch) {
+    this.arch = arch;
+  }
+
+  public String toString() {
+    return this.arch;
+  }
+}

Diff for: lib/src/main/java/io/cloudquery/plugin/BuildOS.java

@@ -0,0 +1,17 @@
+package io.cloudquery.plugin;
+
+public enum BuildOS {
+  Windows("windows"),
+  Linux("linux"),
+  Darwin("darwin");
+
+  public final String os;
+
+  private BuildOS(String os) {
+    this.os = os;
+  }
+
+  public String toString() {
+    return this.os;
+  }
+}

Diff for: lib/src/main/java/io/cloudquery/plugin/BuildTarget.java

@@ -0,0 +1,12 @@
+package io.cloudquery.plugin;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+@Getter
+public class BuildTarget {
+  @NonNull protected final BuildOS os;
+  @NonNull protected final BuildArch arch;
+}

Diff for: lib/src/main/java/io/cloudquery/plugin/Plugin.java

@@ -17,8 +17,21 @@
 public abstract class Plugin {
   @NonNull protected final String name;
   @NonNull protected final String version;
+
   @Setter protected Logger logger;
   @Setter protected String jsonSchema;
+
+  @Setter protected String team;
+  @Setter protected PluginKind kind;
+  @Setter protected String dockerfile = "Dockerfile";
+
+  @Setter
+  protected BuildTarget[] buildTargets =
+      new BuildTarget[] {
+        new BuildTarget(BuildOS.Linux, BuildArch.ARM64),
+        new BuildTarget(BuildOS.Linux, BuildArch.AMD64),
+      };
+
   protected ClientMeta client;
 
   public void init(String spec, NewClientOptions options) throws Exception {

Diff for: lib/src/main/java/io/cloudquery/plugin/PluginKind.java

@@ -0,0 +1,16 @@
+package io.cloudquery.plugin;
+
+public enum PluginKind {
+  Source("source"),
+  Destination("destination");
+
+  public final String kind;
+
+  private PluginKind(String kind) {
+    this.kind = kind;
+  }
+
+  public String toString() {
+    return this.kind;
+  }
+}

Diff for: lib/src/main/java/io/cloudquery/schema/Table.java

@@ -9,7 +9,6 @@
 import io.cloudquery.schema.Column.ColumnBuilder;
 import io.cloudquery.transformers.TransformerException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,7 +31,7 @@ public interface Transform {
   public static List<Table> flattenTables(List<Table> tables) {
     Map<String, Table> flattenMap = new LinkedHashMap<>();
     for (Table table : tables) {
-      Table newTable = table.toBuilder().relations(Collections.emptyList()).build();
+      Table newTable = table.toBuilder().build();
       flattenMap.put(newTable.name, newTable);
       for (Table child : flattenTables(table.getRelations())) {
         flattenMap.put(child.name, child);