From 598b45d864d7dc346b1e1067b7cf52551389bd32 Mon Sep 17 00:00:00 2001 From: spieden Date: Wed, 22 Nov 2023 20:29:22 -0800 Subject: [PATCH 1/5] fix: add date types to ArrowHelper --- .../io/cloudquery/helper/ArrowHelper.java | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java index e3a3a9e..f76e089 100644 --- a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java +++ b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java @@ -20,25 +20,7 @@ import java.util.Objects; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.FixedSizeBinaryVector; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.LargeVarBinaryVector; -import org.apache.arrow.vector.LargeVarCharVector; -import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TimeStampVector; -import org.apache.arrow.vector.TinyIntVector; -import org.apache.arrow.vector.UInt1Vector; -import org.apache.arrow.vector.UInt2Vector; -import org.apache.arrow.vector.UInt4Vector; -import org.apache.arrow.vector.UInt8Vector; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; @@ -139,6 +121,14 @@ private static void setVectorData(FieldVector vector, Object data) { jsonVector.setSafe(0, (byte[]) data); return; } + if (vector instanceof DateDayVector dayDateVector) { + dayDateVector.set(0, (int) data); + return; + } + if (vector instanceof DateMilliVector dateMilliVector) { + dateMilliVector.set(0, (long) data); + return; + } throw new IllegalArgumentException("Unsupported vector type: " + vector.getClass()); } From a6d7ef5fd3dabb9920fbb00f894b82937c1dacd1 Mon Sep 17 00:00:00 2001 From: spieden Date: Wed, 22 Nov 2023 20:33:32 -0800 Subject: [PATCH 2/5] fix: undo imports reformat --- .../io/cloudquery/helper/ArrowHelper.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java index f76e089..a4854ff 100644 --- a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java +++ b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java @@ -20,7 +20,27 @@ import java.util.Objects; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.*; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.LargeVarBinaryVector; +import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; From 7cac3026e81ab378b9739d874d2666ed7025df46 Mon Sep 17 00:00:00 2001 From: spieden Date: Fri, 24 Nov 2023 16:52:16 -0800 Subject: [PATCH 3/5] fix: add tests for date as days since epoch, remove days as milliseconds since epoch --- .../main/java/io/cloudquery/helper/ArrowHelper.java | 4 ---- .../java/io/cloudquery/helper/ArrowHelperTest.java | 13 +++++++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java index a4854ff..fc2166e 100644 --- a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java +++ b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java @@ -145,10 +145,6 @@ private static void setVectorData(FieldVector vector, Object data) { dayDateVector.set(0, (int) data); return; } - if (vector instanceof DateMilliVector dateMilliVector) { - dateMilliVector.set(0, (long) data); - return; - } throw new IllegalArgumentException("Unsupported vector type: " + vector.getClass()); } diff --git a/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java b/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java index ff52029..aa3917c 100644 --- a/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java +++ b/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java @@ -15,8 +15,11 @@ import io.cloudquery.schema.Resource; import io.cloudquery.schema.Table; import java.io.IOException; +import java.time.LocalDate; import java.util.List; import java.util.Map; + +import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -41,7 +44,8 @@ public class ArrowHelperTest { .primaryKey(true) .build(), Column.builder().name("string_column2").type(ArrowType.Utf8.INSTANCE).build(), - Column.builder().name("boolean_column").type(ArrowType.Bool.INSTANCE).build())) + Column.builder().name("boolean_column").type(ArrowType.Bool.INSTANCE).build(), + Column.builder().name("date_days_column").type(new ArrowType.Date(DateUnit.DAY)).build())) .build(); @Test @@ -69,6 +73,9 @@ public void testToArrowSchema() { CQ_EXTENSION_PRIMARY_KEY, "false")); + assertEquals(arrowSchema.getFields().get(2).getName(), "boolean_column"); + assertEquals(arrowSchema.getFields().get(3).getName(), "date_days_column"); + assertEquals( arrowSchema.getCustomMetadata(), Map.of( @@ -84,7 +91,8 @@ public void testFromArrowSchema() { List fields = List.of( Field.nullable("string_column1", ArrowType.Utf8.INSTANCE), - Field.nullable("string_column2", ArrowType.Utf8.INSTANCE)); + Field.nullable("string_column2", ArrowType.Utf8.INSTANCE), + Field.nullable("date_days_column", new ArrowType.Date(DateUnit.DAY))); Schema schema = new Schema(fields, Map.of(CQ_TABLE_NAME, "table1")); @@ -120,6 +128,7 @@ public void testRoundTripResourceEncoding() throws Exception { Resource resource = Resource.builder().table(TEST_TABLE).build(); resource.set("string_column1", "test_data"); resource.set("string_column2", "test_data2"); + resource.set("date_days_column", (int) LocalDate.parse("2023-11-24").toEpochDay()); resource.set("boolean_column", true); Assertions.assertDoesNotThrow( From 0f8c23e7c62724dc4fe9fc17c0e017b81b8deb32 Mon Sep 17 00:00:00 2001 From: spieden Date: Fri, 24 Nov 2023 16:53:09 -0800 Subject: [PATCH 4/5] fix: unused import --- lib/src/main/java/io/cloudquery/helper/ArrowHelper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java index fc2166e..7c99307 100644 --- a/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java +++ b/lib/src/main/java/io/cloudquery/helper/ArrowHelper.java @@ -23,7 +23,6 @@ import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; From adcea5d9e0dab5b42efea4127f56b8e74cdf98b9 Mon Sep 17 00:00:00 2001 From: spieden Date: Sat, 25 Nov 2023 19:08:18 -0800 Subject: [PATCH 5/5] fix: formatting --- lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java b/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java index aa3917c..4a63eb3 100644 --- a/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java +++ b/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java @@ -18,7 +18,6 @@ import java.time.LocalDate; import java.util.List; import java.util.Map; - import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -45,7 +44,10 @@ public class ArrowHelperTest { .build(), Column.builder().name("string_column2").type(ArrowType.Utf8.INSTANCE).build(), Column.builder().name("boolean_column").type(ArrowType.Bool.INSTANCE).build(), - Column.builder().name("date_days_column").type(new ArrowType.Date(DateUnit.DAY)).build())) + Column.builder() + .name("date_days_column") + .type(new ArrowType.Date(DateUnit.DAY)) + .build())) .build(); @Test