From d0771b82e99562adb244b73965365b034227fa2c Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Fri, 15 Nov 2019 17:55:50 +0100 Subject: [PATCH 01/32] wip --- lib/myxql/geometry.ex | 7 +++ lib/myxql/protocol/values.ex | 85 ++++++++++++++++++++++++++--- test/myxql/protocol/values_test.exs | 12 ++++ 3 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 lib/myxql/geometry.ex diff --git a/lib/myxql/geometry.ex b/lib/myxql/geometry.ex new file mode 100644 index 00000000..566d3a72 --- /dev/null +++ b/lib/myxql/geometry.ex @@ -0,0 +1,7 @@ +defmodule MyXQL.Geometry.Point do + defstruct [:x, :y] +end + +defmodule MyXQL.Geometry.Multipoint do + defstruct [:points] +end diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 4e989f7c..73cb9ee1 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -83,6 +83,7 @@ defmodule MyXQL.Protocol.Values do defp column_def_to_type(column_def(type: :mysql_type_string)), do: :binary defp column_def_to_type(column_def(type: :mysql_type_bit, length: length)), do: {:bit, length} defp column_def_to_type(column_def(type: :mysql_type_null)), do: :null + defp column_def_to_type(column_def(type: :mysql_type_geometry)), do: :geometry # Text values @@ -308,7 +309,7 @@ defmodule MyXQL.Protocol.Values do end defp decode_binary_row(<>, null_bitmap, [:binary | t], acc), - do: decode_string_lenenc(r, null_bitmap, t, acc) + do: decode_string_lenenc(r, null_bitmap, t, acc, & &1) defp decode_binary_row(<>, null_bitmap, [:int1 | t], acc), do: decode_int1(r, null_bitmap, t, acc) @@ -361,10 +362,58 @@ defmodule MyXQL.Protocol.Values do defp decode_binary_row(<>, null_bitmap, [{:bit, size} | t], acc), do: decode_bit(r, size, null_bitmap, t, acc) + defp decode_binary_row(<>, null_bitmap, [:geometry | t], acc), + do: decode_string_lenenc(r, null_bitmap, t, acc, &decode_geometry/1) + defp decode_binary_row(<<>>, _null_bitmap, [], acc) do Enum.reverse(acc) end + # this looks similar to WKB [1] but instead of: + # + # <> + # + # we have: + # + # <<0::32, byte_order::8, type::32, ...>> + # + # so seems there's some extra padding in front. Maybe it's a 4-byte srid? + # + # byte_order is allegedly big endian (0x01) but we need to decode floats as little-endian. (lol.) + # + # Looking at [2] + # + # > Values should be stored in internal geometry format, but you can convert them to that + # > format from either Well-Known Text (WKT) or Well-Known Binary (WKB) format. + # + # so yeah, looks like mysql might be storing it in an internal format that is not WKB. + # There's a ST_ToBinary() function that allegedly returns data in WKB so this might be helpful + # for debugging. + # + # [1] https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary + # [2] https://dev.mysql.com/doc/refman/8.0/en/populating-spatial-columns.html + defp decode_geometry(<<0::uint4, 1::uint1, type::uint4, r::bits>>) do + # ^ ^ big endian + # | + # some padding? maybe srid? + decode_geometry(type, r) + end + + defp decode_geometry(1, <>), + do: %MyXQL.Geometry.Point{x: x, y: y} + + defp decode_geometry(4, <>), + do: decode_multipoint(r, size, []) + + # <<1, 1, 0, 0, 0>> in front seems like some kind of header? + defp decode_multipoint(<<1, 1, 0, 0, 0, x::64-signed-little-float, y::64-signed-little-float, r::bits>>, size, acc) do + decode_multipoint(r, size - 1, [{x, y} | acc]) + end + + defp decode_multipoint("", 0, acc) do + %MyXQL.Geometry.Multipoint{points: Enum.reverse(acc)} + end + defp decode_int1(<>, null_bitmap, t, acc), do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) @@ -510,18 +559,36 @@ defmodule MyXQL.Protocol.Values do } end - defp decode_string_lenenc(<>, null_bitmap, t, acc) + defp decode_string_lenenc(<>, null_bitmap, t, acc, decoder) when n < 251, - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) - defp decode_string_lenenc(<<0xFC, n::uint2, v::string(n), r::bits>>, null_bitmap, t, acc), - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + defp decode_string_lenenc( + <<0xFC, n::uint2, v::string(n), r::bits>>, + null_bitmap, + t, + acc, + decoder + ), + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) - defp decode_string_lenenc(<<0xFD, n::uint3, v::string(n), r::bits>>, null_bitmap, t, acc), - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + defp decode_string_lenenc( + <<0xFD, n::uint3, v::string(n), r::bits>>, + null_bitmap, + t, + acc, + decoder + ), + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) - defp decode_string_lenenc(<<0xFE, n::uint8, v::string(n), r::bits>>, null_bitmap, t, acc), - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + defp decode_string_lenenc( + <<0xFE, n::uint8, v::string(n), r::bits>>, + null_bitmap, + t, + acc, + decoder + ), + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) defp decode_json(<>, null_bitmap, t, acc) when n < 251, do: decode_binary_row(r, null_bitmap >>> 1, t, [decode_json(v) | acc]) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 33061615..a560a3e5 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -273,6 +273,18 @@ defmodule MyXQL.Protocol.ValueTest do test "CHAR", c do assert_roundtrip(c, "my_char", "é") end + + if @protocol == :binary do + test "POINT", c do + assert query!(c, "SELECT ST_GeomFromText('POINT(1 2.2)')").rows == [ + [%MyXQL.Geometry.Point{x: 1.0, y: 2.2}] + ] + + assert query!(c, "SELECT ST_GeomFromText('MULTIPOINT(1 1, 2 2)')").rows == [ + [%MyXQL.Geometry.Multipoint{points: [{1.0, 1.0}, {2.0, 2.0}]}] + ] + end + end end end From 293b8b4ee8c57558702216e6e11d718455bf2045 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Sat, 16 Nov 2019 02:10:49 +0100 Subject: [PATCH 02/32] Encode some geometry types --- lib/myxql/protocol/values.ex | 27 ++++++++++++++++++++++++++- test/myxql/protocol/values_test.exs | 25 +++++++++++++++---------- test/test_helper.exs | 4 +++- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 73cb9ee1..0f6d75ae 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -172,6 +172,10 @@ defmodule MyXQL.Protocol.Values do decode_bit(value, size) end + def decode_text_value(value, :geometry) do + decode_geometry(value) + end + # Binary values def encode_binary_value(value) @@ -217,6 +221,19 @@ defmodule MyXQL.Protocol.Values do {:mysql_type_tiny, <<0>>} end + def encode_binary_value(%MyXQL.Geometry.Point{x: x, y: y}) do + encode_geometry(<<1::uint4, x::64-signed-little-float, y::64-signed-little-float>>) + end + + def encode_binary_value(%MyXQL.Geometry.Multipoint{points: points}) do + binary = + Enum.map_join(points, "", fn {x, y} -> + <<1, 1, 0, 0, 0, x::64-signed-little-float, y::64-signed-little-float>> + end) + + encode_geometry(<<4::uint4, length(points)::uint4, binary::binary>>) + end + def encode_binary_value(term) when is_list(term) or is_map(term) do string = json_library().encode!(term) {:mysql_type_var_string, encode_string_lenenc(string)} @@ -226,6 +243,10 @@ defmodule MyXQL.Protocol.Values do raise ArgumentError, "query has invalid parameter #{inspect(other)}" end + defp encode_geometry(binary) do + {:mysql_type_geometry, encode_string_lenenc(<<0::uint4, 1::uint1, binary::binary>>)} + end + ## Time/DateTime # MySQL supports negative time and days, we don't. @@ -406,7 +427,11 @@ defmodule MyXQL.Protocol.Values do do: decode_multipoint(r, size, []) # <<1, 1, 0, 0, 0>> in front seems like some kind of header? - defp decode_multipoint(<<1, 1, 0, 0, 0, x::64-signed-little-float, y::64-signed-little-float, r::bits>>, size, acc) do + defp decode_multipoint( + <<1, 1, 0, 0, 0, x::64-signed-little-float, y::64-signed-little-float, r::bits>>, + size, + acc + ) do decode_multipoint(r, size - 1, [{x, y} | acc]) end diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index a560a3e5..be6afacc 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -274,16 +274,14 @@ defmodule MyXQL.Protocol.ValueTest do assert_roundtrip(c, "my_char", "é") end - if @protocol == :binary do - test "POINT", c do - assert query!(c, "SELECT ST_GeomFromText('POINT(1 2.2)')").rows == [ - [%MyXQL.Geometry.Point{x: 1.0, y: 2.2}] - ] - - assert query!(c, "SELECT ST_GeomFromText('MULTIPOINT(1 1, 2 2)')").rows == [ - [%MyXQL.Geometry.Multipoint{points: [{1.0, 1.0}, {2.0, 2.0}]}] - ] - end + test "POINT", c do + assert_roundtrip(c, "my_point", %MyXQL.Geometry.Point{x: 1.0, y: 2.2}) + end + + test "MULTIPOINT", c do + assert_roundtrip(c, "my_multipoint", %MyXQL.Geometry.Multipoint{ + points: [{1.0, 1.0}, {2.0, 2.0}] + }) end end end @@ -382,6 +380,13 @@ defmodule MyXQL.Protocol.ValueTest do list when is_list(list) -> "'#{Jason.encode!(list)}'" + %MyXQL.Geometry.Point{x: x, y: y} -> + "POINT(#{x}, #{y})" + + %MyXQL.Geometry.Multipoint{points: points} -> + binary = Enum.map_join(points, ", ", fn {x, y} -> "POINT(#{x}, #{y})" end) + "MULTIPOINT(" <> binary <> ")" + %_{} = struct -> "'#{struct}'" diff --git a/test/test_helper.exs b/test/test_helper.exs index eb4db623..7a9333eb 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -124,7 +124,9 @@ defmodule TestHelper do my_mediumblob MEDIUMBLOB, my_longblob LONGBLOB, #{if supports_json?(), do: "my_json JSON,", else: ""} - my_char CHAR + my_char CHAR, + my_point POINT, + my_multipoint MULTIPOINT ); DROP PROCEDURE IF EXISTS single_procedure; From 753ce823362e02e311695f9f84290244985aa889 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Sat, 16 Nov 2019 12:01:26 +0100 Subject: [PATCH 03/32] Use geo library --- lib/myxql/geometry.ex | 7 ----- lib/myxql/protocol/values.ex | 45 +++++------------------------ mix.exs | 1 + mix.lock | 1 + test/myxql/protocol/values_test.exs | 12 ++++---- 5 files changed, 14 insertions(+), 52 deletions(-) delete mode 100644 lib/myxql/geometry.ex diff --git a/lib/myxql/geometry.ex b/lib/myxql/geometry.ex deleted file mode 100644 index 566d3a72..00000000 --- a/lib/myxql/geometry.ex +++ /dev/null @@ -1,7 +0,0 @@ -defmodule MyXQL.Geometry.Point do - defstruct [:x, :y] -end - -defmodule MyXQL.Geometry.Multipoint do - defstruct [:points] -end diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 0f6d75ae..d7a628a2 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -221,18 +221,8 @@ defmodule MyXQL.Protocol.Values do {:mysql_type_tiny, <<0>>} end - def encode_binary_value(%MyXQL.Geometry.Point{x: x, y: y}) do - encode_geometry(<<1::uint4, x::64-signed-little-float, y::64-signed-little-float>>) - end - - def encode_binary_value(%MyXQL.Geometry.Multipoint{points: points}) do - binary = - Enum.map_join(points, "", fn {x, y} -> - <<1, 1, 0, 0, 0, x::64-signed-little-float, y::64-signed-little-float>> - end) - - encode_geometry(<<4::uint4, length(points)::uint4, binary::binary>>) - end + def encode_binary_value(%Geo.Point{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiPoint{} = geo), do: encode_geometry(geo) def encode_binary_value(term) when is_list(term) or is_map(term) do string = json_library().encode!(term) @@ -243,8 +233,9 @@ defmodule MyXQL.Protocol.Values do raise ArgumentError, "query has invalid parameter #{inspect(other)}" end - defp encode_geometry(binary) do - {:mysql_type_geometry, encode_string_lenenc(<<0::uint4, 1::uint1, binary::binary>>)} + defp encode_geometry(geo) do + binary = geo |> Geo.WKB.encode!(:ndr) |> Base.decode16!() + {:mysql_type_geometry, encode_string_lenenc(<<0::uint4, binary::binary>>)} end ## Time/DateTime @@ -413,30 +404,8 @@ defmodule MyXQL.Protocol.Values do # # [1] https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary # [2] https://dev.mysql.com/doc/refman/8.0/en/populating-spatial-columns.html - defp decode_geometry(<<0::uint4, 1::uint1, type::uint4, r::bits>>) do - # ^ ^ big endian - # | - # some padding? maybe srid? - decode_geometry(type, r) - end - - defp decode_geometry(1, <>), - do: %MyXQL.Geometry.Point{x: x, y: y} - - defp decode_geometry(4, <>), - do: decode_multipoint(r, size, []) - - # <<1, 1, 0, 0, 0>> in front seems like some kind of header? - defp decode_multipoint( - <<1, 1, 0, 0, 0, x::64-signed-little-float, y::64-signed-little-float, r::bits>>, - size, - acc - ) do - decode_multipoint(r, size - 1, [{x, y} | acc]) - end - - defp decode_multipoint("", 0, acc) do - %MyXQL.Geometry.Multipoint{points: Enum.reverse(acc)} + defp decode_geometry(<<0::uint4, r::bits>>) do + r |> Base.encode16() |> Geo.WKB.decode!() end defp decode_int1(<>, null_bitmap, t, acc), diff --git a/mix.exs b/mix.exs index ed11af86..d0cd234f 100644 --- a/mix.exs +++ b/mix.exs @@ -51,6 +51,7 @@ defmodule MyXQL.MixProject do {:db_connection, "~> 2.0", db_connection_opts()}, {:decimal, "~> 1.6"}, {:jason, "~> 1.0", optional: true}, + {:geo, "~> 3.3"}, {:binpp, ">= 0.0.0", only: [:dev, :test]}, {:dialyxir, "~> 1.0-rc", only: :dev, runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index fa1cdf0f..31069ca7 100644 --- a/mix.lock +++ b/mix.lock @@ -9,6 +9,7 @@ "earmark": {:hex, :earmark, "1.4.0", "397e750b879df18198afc66505ca87ecf6a96645545585899f6185178433cc09", [:mix], [], "hexpm"}, "erlex": {:hex, :erlex, "0.2.1", "cee02918660807cbba9a7229cae9b42d1c6143b768c781fa6cee1eaf03ad860b", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, + "geo": {:hex, :geo, "3.3.2", "30c7b458bcb0ab1ca73a997b26d22c68643d9ffe1726e475e0759499b6024cff", [:mix], [], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index be6afacc..0d16fc1d 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -275,13 +275,11 @@ defmodule MyXQL.Protocol.ValueTest do end test "POINT", c do - assert_roundtrip(c, "my_point", %MyXQL.Geometry.Point{x: 1.0, y: 2.2}) + assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {1.0, 2.2}}) end test "MULTIPOINT", c do - assert_roundtrip(c, "my_multipoint", %MyXQL.Geometry.Multipoint{ - points: [{1.0, 1.0}, {2.0, 2.0}] - }) + assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{coordinates: [{1.0, 1.0}, {2.0, 2.0}]}) end end end @@ -380,11 +378,11 @@ defmodule MyXQL.Protocol.ValueTest do list when is_list(list) -> "'#{Jason.encode!(list)}'" - %MyXQL.Geometry.Point{x: x, y: y} -> + %Geo.Point{coordinates: {x, y}} -> "POINT(#{x}, #{y})" - %MyXQL.Geometry.Multipoint{points: points} -> - binary = Enum.map_join(points, ", ", fn {x, y} -> "POINT(#{x}, #{y})" end) + %Geo.MultiPoint{coordinates: coordinates} -> + binary = Enum.map_join(coordinates, ", ", fn {x, y} -> "POINT(#{x}, #{y})" end) "MULTIPOINT(" <> binary <> ")" %_{} = struct -> From e3a698ff9e124acf826db565aa57c43f71d38323 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Sat, 16 Nov 2019 13:21:22 +0100 Subject: [PATCH 04/32] Improve test --- test/myxql/protocol/values_test.exs | 68 +++++++++++------------------ 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 0d16fc1d..ad9a545e 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -279,7 +279,9 @@ defmodule MyXQL.Protocol.ValueTest do end test "MULTIPOINT", c do - assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{coordinates: [{1.0, 1.0}, {2.0, 2.0}]}) + assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{ + coordinates: [{1.0, 1.0}, {2.0, 2.0}] + }) end end end @@ -360,48 +362,7 @@ defmodule MyXQL.Protocol.ValueTest do defp insert(%{protocol: :text} = c, fields, values) when is_list(fields) and is_list(values) do fields = Enum.map_join(fields, ", ", &"`#{&1}`") - - values = - Enum.map_join(values, ", ", fn - nil -> - "NULL" - - true -> - "TRUE" - - false -> - "FALSE" - - %DateTime{} = datetime -> - "'#{NaiveDateTime.to_iso8601(datetime)}'" - - list when is_list(list) -> - "'#{Jason.encode!(list)}'" - - %Geo.Point{coordinates: {x, y}} -> - "POINT(#{x}, #{y})" - - %Geo.MultiPoint{coordinates: coordinates} -> - binary = Enum.map_join(coordinates, ", ", fn {x, y} -> "POINT(#{x}, #{y})" end) - "MULTIPOINT(" <> binary <> ")" - - %_{} = struct -> - "'#{struct}'" - - map when is_map(map) -> - "'#{Jason.encode!(map)}'" - - value when is_binary(value) -> - "'#{value}'" - - value when is_bitstring(value) -> - size = bit_size(value) - <> = value - "B'#{Integer.to_string(value, 2)}'" - - value -> - "'#{value}'" - end) + values = Enum.map_join(values, ", ", &encode_text/1) %MyXQL.Result{last_insert_id: id} = query!(c, "INSERT INTO test_types (#{fields}) VALUES (#{values})") @@ -422,6 +383,27 @@ defmodule MyXQL.Protocol.ValueTest do insert(c, [field], [value]) end + defp encode_text(nil), do: "NULL" + defp encode_text(true), do: "TRUE" + defp encode_text(false), do: "FALSE" + defp encode_text(%DateTime{} = datetime), do: "'#{NaiveDateTime.to_iso8601(datetime)}'" + defp encode_text(%Geo.Point{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) + defp encode_text(%_{} = struct), do: "'#{struct}'" + defp encode_text(map) when is_map(map), do: "'#{Jason.encode!(map)}'" + defp encode_text(list) when is_list(list), do: "'#{Jason.encode!(list)}'" + defp encode_text(value) when is_binary(value), do: "'#{value}'" + + defp encode_text(value) when is_bitstring(value) do + size = bit_size(value) + <> = value + "B'#{Integer.to_string(value, 2)}'" + end + + defp encode_text(value), do: "'#{value}'" + + defp encode_text_geo(value), do: "ST_GeomFromText('#{Geo.WKT.encode!(value)}')" + defp get(c, fields, id) when is_list(fields) do fields = Enum.map_join(fields, ", ", &"`#{&1}`") statement = "SELECT #{fields} FROM test_types WHERE id = '#{id}'" From 55e0902dc2d2951d9edb01372100275677fa689f Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Tue, 19 Nov 2019 20:33:59 +0100 Subject: [PATCH 05/32] chore: add elixir_ls to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e8599ac7..16e35365 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ myxql-*.tar /errmsg-utf8.txt /xref_graph.* /config +.elixir_ls From 615ec7961046eb2882e41685027a59783faa45a1 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Tue, 19 Nov 2019 20:35:48 +0100 Subject: [PATCH 06/32] chore: add more Geo types --- lib/myxql/protocol/values.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index d7a628a2..cca235a0 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -222,7 +222,18 @@ defmodule MyXQL.Protocol.Values do end def encode_binary_value(%Geo.Point{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.PointM{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.PointZ{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.PointZM{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.MultiPoint{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.LineString{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.LineStringZ{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiLineString{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiLineStringZ{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.Polygon{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.PolygonZ{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiPolygon{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiPolygonZ{} = geo), do: encode_geometry(geo) def encode_binary_value(term) when is_list(term) or is_map(term) do string = json_library().encode!(term) From 7057c7614e4f4aecfc8dea687e58de60fdaf3090 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Tue, 19 Nov 2019 20:45:34 +0100 Subject: [PATCH 07/32] chore: add Polygon to the tests --- test/myxql/protocol/values_test.exs | 7 +++++++ test/test_helper.exs | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index ad9a545e..0a202d20 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -283,6 +283,12 @@ defmodule MyXQL.Protocol.ValueTest do coordinates: [{1.0, 1.0}, {2.0, 2.0}] }) end + + test "POLYGON", c do + assert_roundtrip(c, "my_polygon", %Geo.Polygon{ + coordinates: [[{30, 10}, {40, 40}, {20, 40}, {10, 20}, {30, 10}]] + }) + end end end @@ -389,6 +395,7 @@ defmodule MyXQL.Protocol.ValueTest do defp encode_text(%DateTime{} = datetime), do: "'#{NaiveDateTime.to_iso8601(datetime)}'" defp encode_text(%Geo.Point{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.Polygon{} = geo), do: encode_text_geo(geo) defp encode_text(%_{} = struct), do: "'#{struct}'" defp encode_text(map) when is_map(map), do: "'#{Jason.encode!(map)}'" defp encode_text(list) when is_list(list), do: "'#{Jason.encode!(list)}'" diff --git a/test/test_helper.exs b/test/test_helper.exs index 7a9333eb..d467ee77 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -126,7 +126,8 @@ defmodule TestHelper do #{if supports_json?(), do: "my_json JSON,", else: ""} my_char CHAR, my_point POINT, - my_multipoint MULTIPOINT + my_multipoint MULTIPOINT, + my_polygon POLYGON ); DROP PROCEDURE IF EXISTS single_procedure; From cdface2f0fe14aaa56832c8e8dce686227d85955 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Tue, 19 Nov 2019 21:20:55 +0100 Subject: [PATCH 08/32] chore: update readme with a Geometry section --- README.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/README.md b/README.md index d2046d40..9aeeb670 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,56 @@ You can customize it to use another library via the `:json_library` configuratio config :myxql, :json_library, SomeJSONModule ``` +## Geometry support + +MyXQL comes with support for PostGIS data type support out of the box via the [Geo](https://github.com/bryanjos/geo/) library, which itself has zero dependencies. Bear in mind: inserting is best done via the `ST_GeomFromText` functions provided by MySQL. The `Geo` library can encode the `%Geo.DATA_TYPE{}` structs to the WKT string that functions expects. + +So you can do the following: + +```elixir + +iex> {:ok, pid} = MyXQL.start_link(username: "root") +iex> MyXQL.query!(pid, "CREATE DATABASE IF NOT EXISTS geo_db") + +# setup some geo fields +iex> {:ok, pid} = MyXQL.start_link(username: "root", database: "geo_db") +iex> MyXQL.query!(pid, "CREATE TABLE IF NOT EXISTS geo (id serial primary key, point POINT, polygon POLYGON)") + +# starting with one of the `Geo` data structs: +iex> point = %Geo.Point{coordinates: {1.0, 2.2}} +iex> MyXQL.query!(pid, "INSERT INTO geo (`point`) VALUES (ST_GeomFromText('#{Geo.WKT.encode!(point)}'))") +%MyXQL.Result{columns: nil, connection_id: 11204,, last_insert_id: 1, num_rows: 1, num_warnings: 0, rows: nil} + +iex> polygon = %Geo.Polygon{coordinates: [[{30, 10}, {40, 40}, {20, 40}, {10, 20}, {30, 10}]]} +iex> MyXQL.query!(pid, "INSERT INTO geo (`polygon`) VALUES (ST_GeomFromText('#{Geo.WKT.encode!(polygon)}'))") + +iex> MyXQL.query!(pid, "SELECT * FROM geo") +%MyXQL.Result{ + columns: ["id", "point", "polygon"], + connection_id: 2078, + last_insert_id: nil, + num_rows: 2, + num_warnings: 0, + rows: [ + [1, %Geo.Point{coordinates: {1.0, 2.2}, properties: %{}, srid: nil}, nil], + [ + 2, + nil, + %Geo.Polygon{ + coordinates: [ + [{30.0, 10.0}, {40.0, 40.0}, {20.0, 40.0}, {10.0, 20.0}, {30.0, 10.0}] + ], + properties: %{}, + srid: nil + } + ] + ] +} +``` + +If you plan on using MyXQL with Ecto, and need to handle user input, you might need to add a (custom Ecto type)[https://hexdocs.pm/ecto/Ecto.Type.html +] in order to construct a valid WKT string. + ## Contributing Run tests: From 2afd2a5ddcab436ff69763dde1bbdd9004a2abf2 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Tue, 19 Nov 2019 21:21:18 +0100 Subject: [PATCH 09/32] chore: add some comments for the Geo section in the code --- lib/myxql/protocol/values.ex | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index cca235a0..e0f4082f 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -392,29 +392,23 @@ defmodule MyXQL.Protocol.Values do Enum.reverse(acc) end - # this looks similar to WKB [1] but instead of: + # Geometry + # ======= # - # <> + # Here, we use the awesome Geo [1] package to help encode and decode the WKT and WKB data in and out of the db. + # There still is some mistery about the first 32-bit integer, which most likely is the srid, judging. # - # we have: + # In the future, it would be nice to completely parse this ourselves, @jeroenbourgois got quite for, but not quite there. + # @wojtekmach then came to the rescue and proposed to use the Geo package, with is a no-dependency drop in. # - # <<0::32, byte_order::8, type::32, ...>> + # For future reference, here are some interesting reads about the Well-Known Text (WKT) and Well-Known Binary (WKB) format used to store the geo data, and references to how MySQL stores these internally. # - # so seems there's some extra padding in front. Maybe it's a 4-byte srid? + # https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary + # https://dev.mysql.com/doc/refman/8.0/en/populating-spatial-columns.html + # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-wkb-format + # https://www.ibm.com/support/knowledgecenter/en/SSEPGG_11.1.0/com.ibm.db2.luw.spatial.topics.doc/doc/rsbp4121.html # - # byte_order is allegedly big endian (0x01) but we need to decode floats as little-endian. (lol.) - # - # Looking at [2] - # - # > Values should be stored in internal geometry format, but you can convert them to that - # > format from either Well-Known Text (WKT) or Well-Known Binary (WKB) format. - # - # so yeah, looks like mysql might be storing it in an internal format that is not WKB. - # There's a ST_ToBinary() function that allegedly returns data in WKB so this might be helpful - # for debugging. - # - # [1] https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary - # [2] https://dev.mysql.com/doc/refman/8.0/en/populating-spatial-columns.html + # Still, all considered, this is a very elegant and readable solution! defp decode_geometry(<<0::uint4, r::bits>>) do r |> Base.encode16() |> Geo.WKB.decode!() end From 9827b7ffc689e24e572f26165f0c6183c380bf18 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Tue, 19 Nov 2019 21:42:49 +0100 Subject: [PATCH 10/32] chore: update readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9aeeb670..c2f1cc48 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,9 @@ config :myxql, :json_library, SomeJSONModule ## Geometry support -MyXQL comes with support for PostGIS data type support out of the box via the [Geo](https://github.com/bryanjos/geo/) library, which itself has zero dependencies. Bear in mind: inserting is best done via the `ST_GeomFromText` functions provided by MySQL. The `Geo` library can encode the `%Geo.DATA_TYPE{}` structs to the WKT string that functions expects. +MyXQL comes with support for PostGIS data type support out of the box via the [Geo](https://github.com/bryanjos/geo/) library, which itself has zero dependencies. + +Bear in mind: inserting is best done via the `ST_GeomFromText` functions provided by MySQL. The `Geo` library can encode the `%Geo.DATA_TYPE{}` structs to the WKT string that functions expects. So you can do the following: @@ -141,7 +143,6 @@ iex> MyXQL.query!(pid, "CREATE DATABASE IF NOT EXISTS geo_db") iex> {:ok, pid} = MyXQL.start_link(username: "root", database: "geo_db") iex> MyXQL.query!(pid, "CREATE TABLE IF NOT EXISTS geo (id serial primary key, point POINT, polygon POLYGON)") -# starting with one of the `Geo` data structs: iex> point = %Geo.Point{coordinates: {1.0, 2.2}} iex> MyXQL.query!(pid, "INSERT INTO geo (`point`) VALUES (ST_GeomFromText('#{Geo.WKT.encode!(point)}'))") %MyXQL.Result{columns: nil, connection_id: 11204,, last_insert_id: 1, num_rows: 1, num_warnings: 0, rows: nil} @@ -173,8 +174,8 @@ iex> MyXQL.query!(pid, "SELECT * FROM geo") } ``` -If you plan on using MyXQL with Ecto, and need to handle user input, you might need to add a (custom Ecto type)[https://hexdocs.pm/ecto/Ecto.Type.html -] in order to construct a valid WKT string. +If you plan on using MyXQL with Ecto, and need to handle user input, you might need to add a [custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html +) in order to construct a valid WKT string. ## Contributing From 9b237c1361ab738de35147972f8306d877933892 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Wed, 20 Nov 2019 01:53:20 +0100 Subject: [PATCH 11/32] Update README.md --- README.md | 89 ++++++++++++++----------------------------------------- 1 file changed, 22 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index c2f1cc48..644ec955 100644 --- a/README.md +++ b/README.md @@ -85,22 +85,23 @@ See [Mariaex Compatibility](https://github.com/elixir-ecto/myxql/blob/master/MAR ## Data representation ``` -MySQL Elixir ------ ------ -NULL nil -bool 1 | 0 -int 42 -float 42.0 -decimal #Decimal<42.0> * -date ~D[2013-10-12] ** -time ~T[00:37:14] -datetime ~N[2013-10-12 00:37:14] **, *** -timestamp #DateTime<2013-10-12 00:37:14Z> *** -json %{"foo" => "bar"} **** -char "é" -text "myxql" -binary <<1, 2, 3>> -bit <<1::size(1), 0::size(1)>> +MySQL Elixir +----- ------ +NULL nil +bool 1 | 0 +int 42 +float 42.0 +decimal #Decimal<42.0> * +date ~D[2013-10-12] ** +time ~T[00:37:14] +datetime ~N[2013-10-12 00:37:14] **, *** +timestamp ~U[2013-10-12 00:37:14Z] *** +json %{"foo" => "bar"} **** +char "é" +text "myxql" +binary <<1, 2, 3>> +bit <<1::size(1), 0::size(1)>> +point, polygon, ... %Geo.Point{coordinates: {0.0, 1.0}}, ... ***** ``` \* See [Decimal](https://github.com/ericmj/decimal) @@ -112,6 +113,11 @@ bit <<1::size(1), 0::size(1)>> \*\*\*\* MySQL added a native JSON type in version 5.7.8, if you're using earlier versions, remember to use TEXT column for your JSON field. +\*\*\*\*\* Encoding/decoding between `Geo.*` structs and the PostGIS EWKB binary format is +done using the [Geo](https://github.com/bryanjos/geo) package. If you're using MyXQL geometry +types with Ecto and need to for example accept EWKT format as user input, consider implementing an +[custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html). + ## JSON support MyXQL comes with JSON support out of the box via the [Jason](https://github.com/michalmuskala/jason) library. To use it, add `:jason` to your dependencies: @@ -126,57 +132,6 @@ You can customize it to use another library via the `:json_library` configuratio config :myxql, :json_library, SomeJSONModule ``` -## Geometry support - -MyXQL comes with support for PostGIS data type support out of the box via the [Geo](https://github.com/bryanjos/geo/) library, which itself has zero dependencies. - -Bear in mind: inserting is best done via the `ST_GeomFromText` functions provided by MySQL. The `Geo` library can encode the `%Geo.DATA_TYPE{}` structs to the WKT string that functions expects. - -So you can do the following: - -```elixir - -iex> {:ok, pid} = MyXQL.start_link(username: "root") -iex> MyXQL.query!(pid, "CREATE DATABASE IF NOT EXISTS geo_db") - -# setup some geo fields -iex> {:ok, pid} = MyXQL.start_link(username: "root", database: "geo_db") -iex> MyXQL.query!(pid, "CREATE TABLE IF NOT EXISTS geo (id serial primary key, point POINT, polygon POLYGON)") - -iex> point = %Geo.Point{coordinates: {1.0, 2.2}} -iex> MyXQL.query!(pid, "INSERT INTO geo (`point`) VALUES (ST_GeomFromText('#{Geo.WKT.encode!(point)}'))") -%MyXQL.Result{columns: nil, connection_id: 11204,, last_insert_id: 1, num_rows: 1, num_warnings: 0, rows: nil} - -iex> polygon = %Geo.Polygon{coordinates: [[{30, 10}, {40, 40}, {20, 40}, {10, 20}, {30, 10}]]} -iex> MyXQL.query!(pid, "INSERT INTO geo (`polygon`) VALUES (ST_GeomFromText('#{Geo.WKT.encode!(polygon)}'))") - -iex> MyXQL.query!(pid, "SELECT * FROM geo") -%MyXQL.Result{ - columns: ["id", "point", "polygon"], - connection_id: 2078, - last_insert_id: nil, - num_rows: 2, - num_warnings: 0, - rows: [ - [1, %Geo.Point{coordinates: {1.0, 2.2}, properties: %{}, srid: nil}, nil], - [ - 2, - nil, - %Geo.Polygon{ - coordinates: [ - [{30.0, 10.0}, {40.0, 40.0}, {20.0, 40.0}, {10.0, 20.0}, {30.0, 10.0}] - ], - properties: %{}, - srid: nil - } - ] - ] -} -``` - -If you plan on using MyXQL with Ecto, and need to handle user input, you might need to add a [custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html -) in order to construct a valid WKT string. - ## Contributing Run tests: From 48d55a679fe12a282fc4c3e6171d2642e1fb4be9 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Wed, 20 Nov 2019 02:00:57 +0100 Subject: [PATCH 12/32] Update values.ex --- lib/myxql/protocol/values.ex | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index e0f4082f..0c2d3a7f 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -393,22 +393,18 @@ defmodule MyXQL.Protocol.Values do end # Geometry - # ======= # - # Here, we use the awesome Geo [1] package to help encode and decode the WKT and WKB data in and out of the db. - # There still is some mistery about the first 32-bit integer, which most likely is the srid, judging. + # MySQL basically uses PostGIS EWKB binary representation (an extension to OpenGIS WKB standard) on + # the wire. The only difference is MySQL has additional 4 bytes just before the EWKB bytes, + # in all observed cases they're always `00 00 00 00`, and our best guess is they're left there + # for future extensibility. # - # In the future, it would be nice to completely parse this ourselves, @jeroenbourgois got quite for, but not quite there. - # @wojtekmach then came to the rescue and proposed to use the Geo package, with is a no-dependency drop in. - # - # For future reference, here are some interesting reads about the Well-Known Text (WKT) and Well-Known Binary (WKB) format used to store the geo data, and references to how MySQL stores these internally. + # For future reference here are some resources that were useful when implementing this: # # https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary # https://dev.mysql.com/doc/refman/8.0/en/populating-spatial-columns.html # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-wkb-format # https://www.ibm.com/support/knowledgecenter/en/SSEPGG_11.1.0/com.ibm.db2.luw.spatial.topics.doc/doc/rsbp4121.html - # - # Still, all considered, this is a very elegant and readable solution! defp decode_geometry(<<0::uint4, r::bits>>) do r |> Base.encode16() |> Geo.WKB.decode!() end From c1da9adf18d8ac7f18b24b816731632aac212e1d Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 06:42:33 +0100 Subject: [PATCH 13/32] chore: remove editor related ignore entry in the .gitignore file --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 16e35365..e8599ac7 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,3 @@ myxql-*.tar /errmsg-utf8.txt /xref_graph.* /config -.elixir_ls From e1dcaab904341e2968d6489f2e8908010666bbda Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 06:47:42 +0100 Subject: [PATCH 14/32] chore: check for db server geo support to run appropriate tests --- test/myxql/protocol/values_test.exs | 3 +++ test/test_helper.exs | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 0a202d20..3035f891 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -274,16 +274,19 @@ defmodule MyXQL.Protocol.ValueTest do assert_roundtrip(c, "my_char", "é") end + @tag geometry: true test "POINT", c do assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {1.0, 2.2}}) end + @tag geometry: true test "MULTIPOINT", c do assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{ coordinates: [{1.0, 1.0}, {2.0, 2.0}] }) end + @tag geometry: true test "POLYGON", c do assert_roundtrip(c, "my_polygon", %Geo.Polygon{ coordinates: [[{30, 10}, {40, 40}, {20, 40}, {10, 20}, {30, 10}]] diff --git a/test/test_helper.exs b/test/test_helper.exs index d467ee77..59aa66ac 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -125,9 +125,9 @@ defmodule TestHelper do my_longblob LONGBLOB, #{if supports_json?(), do: "my_json JSON,", else: ""} my_char CHAR, - my_point POINT, - my_multipoint MULTIPOINT, - my_polygon POLYGON + #{if supports_geometry?(), do: "my_point POINT,", else: ""} + #{if supports_geometry?(), do: "my_mulypoint MULTIPOINT,", else: ""} + #{if supports_geometry?(), do: "my_polygon POLYGON", else: ""} ); DROP PROCEDURE IF EXISTS single_procedure; @@ -181,6 +181,20 @@ defmodule TestHelper do end end + def supports_geometry?() do + sql = + "CREATE TEMPORARY TABLE myxql_test.test_geo (point POINT); SHOW COLUMNS IN myxql_test.test_json" + + case mysql(sql) do + {:ok, result} -> + [%{"Type" => type}] = result + type == "POINT" + + {:error, _} -> + false + end + end + def supports_timestamp_precision?() do case mysql("CREATE TEMPORARY TABLE myxql_test.timestamp_precision (time time(3));") do {:ok, _} -> true @@ -246,6 +260,7 @@ defmodule TestHelper do exclude = [{:ssl, not supports_ssl?()} | exclude] exclude = [{:public_key_exchange, not supports_public_key_exchange?()} | exclude] exclude = [{:json, not supports_json?()} | exclude] + exclude = [{:geometry, not supports_geometry?()} | exclude] exclude = [{:timestamp_precision, not supports_timestamp_precision?()} | exclude] exclude = if mariadb?, do: [{:bit, true} | exclude], else: exclude exclude From db7e56ea202216940a7c3b7d4108ffc73a774a01 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 20:28:27 +0100 Subject: [PATCH 15/32] chore: add more geom tests --- test/myxql/protocol/values_test.exs | 115 +++++++++++++++++++++++++++- test/test_helper.exs | 15 ++-- 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 3035f891..cf9bad84 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -276,9 +276,40 @@ defmodule MyXQL.Protocol.ValueTest do @tag geometry: true test "POINT", c do - assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {1.0, 2.2}}) + assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}}) end + # @tag geometry: true + # test "POINTM", c do + # assert_roundtrip(c, "my_point", %Geo.PointM{coordinates: {1, 1, 1}}) + # end + + # @tag geometry: true + # test "POINTZ", c do + # assert_roundtrip(c, "my_point", %Geo.PointZ{coordinates: {1, 1, 1}}) + # end + + # @tag geometry: true + # test "POINTZM", c do + # assert_roundtrip(c, "my_point", %Geo.PointZM{coordinates: {1, 1, 1, 1}}) + # end + # + @tag geometry: true + test "LINESTRING", c do + assert_roundtrip(c, "my_linestring", %Geo.LineString{ + coordinates: [{30, 10}, {10, 30}, {40, 40}], + srid: 4326 + }) + end + + # @tag geometry: true + # test "LINESTRINGZ", c do + # assert_roundtrip(c, "my_linestring", %Geo.LineStringZ{ + # coordinates: [{30, 10, 3}, {10, 30, 90}, {40, 40, 40}], + # srid: 4326 + # }) + # end + @tag geometry: true test "MULTIPOINT", c do assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{ @@ -286,12 +317,80 @@ defmodule MyXQL.Protocol.ValueTest do }) end + # @tag geometry: true + # test "MULTIPOINTZ", c do + # assert_roundtrip(c, "my_multipoint", %Geo.MultiPointZ{ + # coordinates: [{0, 0, 0}, {20, 20, 20}, {60, 60, 60}] + # }) + # end + + @tag geometry: true + test "MULTILINESTRING", c do + assert_roundtrip(c, "my_multilinestring", %Geo.MultiLineString{ + coordinates: [[{10, 10}, {20, 20}], [{15, 15}, {30, 15}]] + }) + end + + # @tag geometry: true + # test "MULTILINESTRINGZ", c do + # assert_roundtrip(c, "my_multilinestring", %Geo.MultiLineStringZ{ + # coordinates: [[{10, 10, 10}, {20, 20, 20}], [{15, 15, 15}, {30, 15, 10}]] + # }) + # end + @tag geometry: true test "POLYGON", c do assert_roundtrip(c, "my_polygon", %Geo.Polygon{ coordinates: [[{30, 10}, {40, 40}, {20, 40}, {10, 20}, {30, 10}]] }) end + + # @tag geometry: true + # test "POLYGONZ", c do + # assert_roundtrip(c, "my_polygon", %Geo.PolygonZ{ + # coordinates: [ + # [{35, 10}, {45, 45}, {15, 40}, {10, 20}, {35, 10}], + # [{20, 30}, {35, 35}, {30, 20}, {20, 30}] + # ], + # srid: 4326 + # }) + # end + + @tag geometry: true + test "MULTIPOLYGON", c do + assert_roundtrip(c, "my_multipolygon", %Geo.MultiPolygon{ + coordinates: [ + [[{40, 40}, {20, 45}, {45, 30}, {40, 40}]], + [ + [{20, 35}, {10, 30}, {10, 10}, {30, 5}, {45, 20}, {20, 35}], + [{30, 20}, {20, 15}, {20, 25}, {30, 20}] + ] + ], + srid: 4326 + }) + end + + # @tag geometry: true + # test "MULTIPOLYGONZ", c do + # assert_roundtrip(c, "my_multipolygon", %Geo.MultiPolygonZ{ + # coordinates: [ + # [[{40, 40, 40}, {20, 45, 60}, {45, 30, 15}, {40, 40, 40}]], + # [ + # [{20, 35, 48}, {10, 30, 50}, {10, 10, 10}, {30, 5, 18}, {45, 20, 10}, {20, 35, 48}], + # [{30, 20, 10}, {20, 15, 10}, {20, 25, 4}, {30, 20, 10}] + # ] + # ], + # srid: 4326 + # }) + # end + + @tag geometry: true + test "GEOMETRYCOLLECTION", c do + assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{ + geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}, srid: 4326}], + srid: 4326 + }) + end end end @@ -397,8 +496,20 @@ defmodule MyXQL.Protocol.ValueTest do defp encode_text(false), do: "FALSE" defp encode_text(%DateTime{} = datetime), do: "'#{NaiveDateTime.to_iso8601(datetime)}'" defp encode_text(%Geo.Point{} = geo), do: encode_text_geo(geo) - defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.PointZ{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.PointM{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.PointZM{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.LineString{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.LineStringZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.Polygon{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.PolygonZ{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.MultiPointZ{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.MultiLineString{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.MultiLineStringZ{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.MultiPolygon{} = geo), do: encode_text_geo(geo) + # defp encode_text(%Geo.MultiPolygonZ{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.GeometryCollection{} = geo), do: encode_text_geo(geo) defp encode_text(%_{} = struct), do: "'#{struct}'" defp encode_text(map) when is_map(map), do: "'#{Jason.encode!(map)}'" defp encode_text(list) when is_list(list), do: "'#{Jason.encode!(list)}'" diff --git a/test/test_helper.exs b/test/test_helper.exs index 59aa66ac..91cc5719 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -75,6 +75,7 @@ defmodule TestHelper do my_time6 TIME(6), my_datetime3 DATETIME(3), my_datetime6 DATETIME(6), + """ mysql!(""" @@ -124,10 +125,14 @@ defmodule TestHelper do my_mediumblob MEDIUMBLOB, my_longblob LONGBLOB, #{if supports_json?(), do: "my_json JSON,", else: ""} - my_char CHAR, #{if supports_geometry?(), do: "my_point POINT,", else: ""} - #{if supports_geometry?(), do: "my_mulypoint MULTIPOINT,", else: ""} - #{if supports_geometry?(), do: "my_polygon POLYGON", else: ""} + #{if supports_geometry?(), do: "my_linestring LINESTRING,", else: ""} + #{if supports_geometry?(), do: "my_polygon POLYGON,", else: ""} + #{if supports_geometry?(), do: "my_multipoint MULTIPOINT,", else: ""} + #{if supports_geometry?(), do: "my_multilinestring MULTILINESTRING,", else: ""} + #{if supports_geometry?(), do: "my_multipolygon MULTIPOLYGON,", else: ""} + #{if supports_geometry?(), do: "my_geometrycollection GEOMETRYCOLLECTION,", else: ""} + my_char CHAR ); DROP PROCEDURE IF EXISTS single_procedure; @@ -183,12 +188,12 @@ defmodule TestHelper do def supports_geometry?() do sql = - "CREATE TEMPORARY TABLE myxql_test.test_geo (point POINT); SHOW COLUMNS IN myxql_test.test_json" + "CREATE TEMPORARY TABLE myxql_test.test_geo (point POINT); SHOW COLUMNS IN myxql_test.test_geo" case mysql(sql) do {:ok, result} -> [%{"Type" => type}] = result - type == "POINT" + type == "point" {:error, _} -> false From 611773e6d9d6d44ab5e2354a02c762cd827fee8d Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 20:48:18 +0100 Subject: [PATCH 16/32] chore: fix some geo tests --- test/myxql/protocol/values_test.exs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index cf9bad84..579b2d94 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -297,16 +297,14 @@ defmodule MyXQL.Protocol.ValueTest do @tag geometry: true test "LINESTRING", c do assert_roundtrip(c, "my_linestring", %Geo.LineString{ - coordinates: [{30, 10}, {10, 30}, {40, 40}], - srid: 4326 + coordinates: [{30, 10}, {10, 30}, {40, 40}] }) end # @tag geometry: true # test "LINESTRINGZ", c do # assert_roundtrip(c, "my_linestring", %Geo.LineStringZ{ - # coordinates: [{30, 10, 3}, {10, 30, 90}, {40, 40, 40}], - # srid: 4326 + # coordinates: [{30, 10, 3}, {10, 30, 90}, {40, 40, 40}] # }) # end @@ -351,8 +349,7 @@ defmodule MyXQL.Protocol.ValueTest do # coordinates: [ # [{35, 10}, {45, 45}, {15, 40}, {10, 20}, {35, 10}], # [{20, 30}, {35, 35}, {30, 20}, {20, 30}] - # ], - # srid: 4326 + # ] # }) # end @@ -365,8 +362,7 @@ defmodule MyXQL.Protocol.ValueTest do [{20, 35}, {10, 30}, {10, 10}, {30, 5}, {45, 20}, {20, 35}], [{30, 20}, {20, 15}, {20, 25}, {30, 20}] ] - ], - srid: 4326 + ] }) end @@ -379,16 +375,14 @@ defmodule MyXQL.Protocol.ValueTest do # [{20, 35, 48}, {10, 30, 50}, {10, 10, 10}, {30, 5, 18}, {45, 20, 10}, {20, 35, 48}], # [{30, 20, 10}, {20, 15, 10}, {20, 25, 4}, {30, 20, 10}] # ] - # ], - # srid: 4326 + # ] # }) # end @tag geometry: true test "GEOMETRYCOLLECTION", c do assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{ - geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}, srid: 4326}], - srid: 4326 + geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}}] }) end end @@ -503,7 +497,7 @@ defmodule MyXQL.Protocol.ValueTest do # defp encode_text(%Geo.LineStringZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.Polygon{} = geo), do: encode_text_geo(geo) # defp encode_text(%Geo.PolygonZ{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) # defp encode_text(%Geo.MultiPointZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.MultiLineString{} = geo), do: encode_text_geo(geo) # defp encode_text(%Geo.MultiLineStringZ{} = geo), do: encode_text_geo(geo) @@ -523,7 +517,11 @@ defmodule MyXQL.Protocol.ValueTest do defp encode_text(value), do: "'#{value}'" - defp encode_text_geo(value), do: "ST_GeomFromText('#{Geo.WKT.encode!(value)}')" + defp encode_text_geo(value) do + geom = Geo.WKT.encode!(value) + IO.inspect(geom) + "ST_GeomFromText('#{geom}')" + end defp get(c, fields, id) when is_list(fields) do fields = Enum.map_join(fields, ", ", &"`#{&1}`") From fa3284932724683238e54e1462f4fc87c9e6d2a1 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 20:56:26 +0100 Subject: [PATCH 17/32] chore: for geometrycollection only test text for now --- test/myxql/protocol/values_test.exs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 579b2d94..5561483a 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -379,11 +379,13 @@ defmodule MyXQL.Protocol.ValueTest do # }) # end - @tag geometry: true - test "GEOMETRYCOLLECTION", c do - assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{ - geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}}] - }) + if @protocol == :text do + @tag geometry: true + test "GEOMETRYCOLLECTION", c do + assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{ + geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}}] + }) + end end end end @@ -517,11 +519,7 @@ defmodule MyXQL.Protocol.ValueTest do defp encode_text(value), do: "'#{value}'" - defp encode_text_geo(value) do - geom = Geo.WKT.encode!(value) - IO.inspect(geom) - "ST_GeomFromText('#{geom}')" - end + defp encode_text_geo(value), do: "ST_GeomFromText('#{Geo.WKT.encode!(value)}')" defp get(c, fields, id) when is_list(fields) do fields = Enum.map_join(fields, ", ", &"`#{&1}`") From cee851c328c0a6868e669e00105c88fb33126cd5 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Wed, 20 Nov 2019 21:04:02 +0100 Subject: [PATCH 18/32] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 644ec955..d1fd6cfe 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,9 @@ point, polygon, ... %Geo.Point{coordinates: {0.0, 1.0}}, ... ***** \*\*\*\* MySQL added a native JSON type in version 5.7.8, if you're using earlier versions, remember to use TEXT column for your JSON field. -\*\*\*\*\* Encoding/decoding between `Geo.*` structs and the PostGIS EWKB binary format is +\*\*\*\*\* Encoding/decoding between `Geo.*` structs and the OpenGIS WKB binary format is done using the [Geo](https://github.com/bryanjos/geo) package. If you're using MyXQL geometry -types with Ecto and need to for example accept EWKT format as user input, consider implementing an +types with Ecto and need to for example accept a WKT format as user input, consider implementing an [custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html). ## JSON support From f7b0245c16244834d67662e8a786205529d3128c Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:14:28 +0100 Subject: [PATCH 19/32] chore: remove unsupported geo types --- test/myxql/protocol/values_test.exs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 5561483a..c9b0c440 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -492,19 +492,11 @@ defmodule MyXQL.Protocol.ValueTest do defp encode_text(false), do: "FALSE" defp encode_text(%DateTime{} = datetime), do: "'#{NaiveDateTime.to_iso8601(datetime)}'" defp encode_text(%Geo.Point{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.PointZ{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.PointM{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.PointZM{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.LineString{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.LineStringZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.Polygon{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.PolygonZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.MultiPointZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.MultiLineString{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.MultiLineStringZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.MultiPolygon{} = geo), do: encode_text_geo(geo) - # defp encode_text(%Geo.MultiPolygonZ{} = geo), do: encode_text_geo(geo) defp encode_text(%Geo.GeometryCollection{} = geo), do: encode_text_geo(geo) defp encode_text(%_{} = struct), do: "'#{struct}'" defp encode_text(map) when is_map(map), do: "'#{Jason.encode!(map)}'" From 08fbf54472664cb34754764850bcc5302a564327 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:17:15 +0100 Subject: [PATCH 20/32] chore: make the geom encoding more obvious by adding the srid sequence --- lib/myxql/protocol/values.ex | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 0c2d3a7f..dd5ad36c 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -222,18 +222,11 @@ defmodule MyXQL.Protocol.Values do end def encode_binary_value(%Geo.Point{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.PointM{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.PointZ{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.PointZM{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.MultiPoint{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.LineString{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.LineStringZ{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.MultiLineString{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.MultiLineStringZ{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.Polygon{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.PolygonZ{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.MultiPolygon{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.MultiPolygonZ{} = geo), do: encode_geometry(geo) def encode_binary_value(term) when is_list(term) or is_map(term) do string = json_library().encode!(term) @@ -245,8 +238,9 @@ defmodule MyXQL.Protocol.Values do end defp encode_geometry(geo) do - binary = geo |> Geo.WKB.encode!(:ndr) |> Base.decode16!() - {:mysql_type_geometry, encode_string_lenenc(<<0::uint4, binary::binary>>)} + srid = geo.srid || 0 + binary = %{geo | srid: nil} |> Geo.WKB.encode!(:ndr) |> Base.decode16!() + {:mysql_type_geometry, encode_string_lenenc(<>)} end ## Time/DateTime From 520569be284c4c52fa60032a29827fd016ac2b62 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:18:41 +0100 Subject: [PATCH 21/32] chore: add missing encoding for geometrycollection --- lib/myxql/protocol/values.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index dd5ad36c..1aa78464 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -227,6 +227,7 @@ defmodule MyXQL.Protocol.Values do def encode_binary_value(%Geo.MultiLineString{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.Polygon{} = geo), do: encode_geometry(geo) def encode_binary_value(%Geo.MultiPolygon{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.GeometryCollection{} = geo), do: encode_geometry(geo) def encode_binary_value(term) when is_list(term) or is_map(term) do string = json_library().encode!(term) From db78cd639e657e69f9296b70efc74b3a44df4ba8 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:19:12 +0100 Subject: [PATCH 22/32] chore: remove tests for unsupported geo types --- test/myxql/protocol/values_test.exs | 59 ----------------------------- 1 file changed, 59 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index c9b0c440..ddf03d1e 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -279,21 +279,6 @@ defmodule MyXQL.Protocol.ValueTest do assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}}) end - # @tag geometry: true - # test "POINTM", c do - # assert_roundtrip(c, "my_point", %Geo.PointM{coordinates: {1, 1, 1}}) - # end - - # @tag geometry: true - # test "POINTZ", c do - # assert_roundtrip(c, "my_point", %Geo.PointZ{coordinates: {1, 1, 1}}) - # end - - # @tag geometry: true - # test "POINTZM", c do - # assert_roundtrip(c, "my_point", %Geo.PointZM{coordinates: {1, 1, 1, 1}}) - # end - # @tag geometry: true test "LINESTRING", c do assert_roundtrip(c, "my_linestring", %Geo.LineString{ @@ -301,13 +286,6 @@ defmodule MyXQL.Protocol.ValueTest do }) end - # @tag geometry: true - # test "LINESTRINGZ", c do - # assert_roundtrip(c, "my_linestring", %Geo.LineStringZ{ - # coordinates: [{30, 10, 3}, {10, 30, 90}, {40, 40, 40}] - # }) - # end - @tag geometry: true test "MULTIPOINT", c do assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{ @@ -315,13 +293,6 @@ defmodule MyXQL.Protocol.ValueTest do }) end - # @tag geometry: true - # test "MULTIPOINTZ", c do - # assert_roundtrip(c, "my_multipoint", %Geo.MultiPointZ{ - # coordinates: [{0, 0, 0}, {20, 20, 20}, {60, 60, 60}] - # }) - # end - @tag geometry: true test "MULTILINESTRING", c do assert_roundtrip(c, "my_multilinestring", %Geo.MultiLineString{ @@ -329,13 +300,6 @@ defmodule MyXQL.Protocol.ValueTest do }) end - # @tag geometry: true - # test "MULTILINESTRINGZ", c do - # assert_roundtrip(c, "my_multilinestring", %Geo.MultiLineStringZ{ - # coordinates: [[{10, 10, 10}, {20, 20, 20}], [{15, 15, 15}, {30, 15, 10}]] - # }) - # end - @tag geometry: true test "POLYGON", c do assert_roundtrip(c, "my_polygon", %Geo.Polygon{ @@ -343,16 +307,6 @@ defmodule MyXQL.Protocol.ValueTest do }) end - # @tag geometry: true - # test "POLYGONZ", c do - # assert_roundtrip(c, "my_polygon", %Geo.PolygonZ{ - # coordinates: [ - # [{35, 10}, {45, 45}, {15, 40}, {10, 20}, {35, 10}], - # [{20, 30}, {35, 35}, {30, 20}, {20, 30}] - # ] - # }) - # end - @tag geometry: true test "MULTIPOLYGON", c do assert_roundtrip(c, "my_multipolygon", %Geo.MultiPolygon{ @@ -366,19 +320,6 @@ defmodule MyXQL.Protocol.ValueTest do }) end - # @tag geometry: true - # test "MULTIPOLYGONZ", c do - # assert_roundtrip(c, "my_multipolygon", %Geo.MultiPolygonZ{ - # coordinates: [ - # [[{40, 40, 40}, {20, 45, 60}, {45, 30, 15}, {40, 40, 40}]], - # [ - # [{20, 35, 48}, {10, 30, 50}, {10, 10, 10}, {30, 5, 18}, {45, 20, 10}, {20, 35, 48}], - # [{30, 20, 10}, {20, 15, 10}, {20, 25, 4}, {30, 20, 10}] - # ] - # ] - # }) - # end - if @protocol == :text do @tag geometry: true test "GEOMETRYCOLLECTION", c do From 45e551a1946e5db11ad730bd97f086601fcf9a52 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:19:36 +0100 Subject: [PATCH 23/32] chore: fix geometrycollection test --- test/myxql/protocol/values_test.exs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index ddf03d1e..7e6bbc24 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -320,13 +320,11 @@ defmodule MyXQL.Protocol.ValueTest do }) end - if @protocol == :text do - @tag geometry: true - test "GEOMETRYCOLLECTION", c do - assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{ - geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}}] - }) - end + @tag geometry: true + test "GEOMETRYCOLLECTION", c do + assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{ + geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}}] + }) end end end From f0d591b7e74039611a3b953a3cd4444c804fe14f Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Wed, 20 Nov 2019 22:23:52 +0100 Subject: [PATCH 24/32] Update values.ex --- lib/myxql/protocol/values.ex | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 1aa78464..97c573ce 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -387,19 +387,7 @@ defmodule MyXQL.Protocol.Values do Enum.reverse(acc) end - # Geometry - # - # MySQL basically uses PostGIS EWKB binary representation (an extension to OpenGIS WKB standard) on - # the wire. The only difference is MySQL has additional 4 bytes just before the EWKB bytes, - # in all observed cases they're always `00 00 00 00`, and our best guess is they're left there - # for future extensibility. - # - # For future reference here are some resources that were useful when implementing this: - # - # https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary - # https://dev.mysql.com/doc/refman/8.0/en/populating-spatial-columns.html - # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-wkb-format - # https://www.ibm.com/support/knowledgecenter/en/SSEPGG_11.1.0/com.ibm.db2.luw.spatial.topics.doc/doc/rsbp4121.html + # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-internal-format defp decode_geometry(<<0::uint4, r::bits>>) do r |> Base.encode16() |> Geo.WKB.decode!() end From 2c80785dc030a480515ed1694be2594236b3b6df Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:28:44 +0100 Subject: [PATCH 25/32] chore: update mariaex compatibility text, removing the unsupported geo statement --- MARIAEX_COMPATIBILITY.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/MARIAEX_COMPATIBILITY.md b/MARIAEX_COMPATIBILITY.md index 57d28bc9..77ed105f 100644 --- a/MARIAEX_COMPATIBILITY.md +++ b/MARIAEX_COMPATIBILITY.md @@ -38,8 +38,6 @@ Queries: * MyXQL represents bit type `B'101'` as `<<1::1, 0::0, 1::1>>` (`<<5::size(3)>>`), Mariaex represents it as `<<5>>` - * MyXQL does not support geometry types - * MyXQL does not support `:type_names`, `result_types`, `:decode`, `:encode_mapper`, `:decode_mapper`, `:include_table_name`, and `:binary_as` options From 953df7a074abd60ca2fc3c51ff23deabd87d68dc Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:39:15 +0100 Subject: [PATCH 26/32] chore: try to exclude geom on mariadb --- test/test_helper.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index 91cc5719..eac7d1fb 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -193,7 +193,9 @@ defmodule TestHelper do case mysql(sql) do {:ok, result} -> [%{"Type" => type}] = result - type == "point" + IO.puts("geom type -> ") + IO.inspect(type) + String.downcase(type) == "point" {:error, _} -> false From f9ed8ddb9645fddaa9a18a641b5b888fe9e04b06 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:45:07 +0100 Subject: [PATCH 27/32] chore: remove debug statements --- test/test_helper.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index eac7d1fb..e024b3ba 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -193,8 +193,6 @@ defmodule TestHelper do case mysql(sql) do {:ok, result} -> [%{"Type" => type}] = result - IO.puts("geom type -> ") - IO.inspect(type) String.downcase(type) == "point" {:error, _} -> From 6a9a113d845722f4407094a4758eb0795696d593 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:53:03 +0100 Subject: [PATCH 28/32] chore: testing --- test/myxql/protocol/values_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 7e6bbc24..8c5a900e 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -456,11 +456,13 @@ defmodule MyXQL.Protocol.ValueTest do fields = Enum.map_join(fields, ", ", &"`#{&1}`") statement = "SELECT #{fields} FROM test_types WHERE id = '#{id}'" %MyXQL.Result{rows: [values]} = query!(c, statement) + IO.inspect(values) values end defp get(c, field, id) do [value] = get(c, [field], id) + IO.inspect(value) value end From a71d2e23d743bf1ae0a328ca1a4705ac29a6cc76 Mon Sep 17 00:00:00 2001 From: Jeroen Bourgois Date: Wed, 20 Nov 2019 22:58:31 +0100 Subject: [PATCH 29/32] chore: revert logging --- test/myxql/protocol/values_test.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 8c5a900e..7e6bbc24 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -456,13 +456,11 @@ defmodule MyXQL.Protocol.ValueTest do fields = Enum.map_join(fields, ", ", &"`#{&1}`") statement = "SELECT #{fields} FROM test_types WHERE id = '#{id}'" %MyXQL.Result{rows: [values]} = query!(c, statement) - IO.inspect(values) values end defp get(c, field, id) do [value] = get(c, [field], id) - IO.inspect(value) value end From dd9ede587bfd7d1e93ace619fa8a98fa41e338f7 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 21 Nov 2019 01:59:26 +0100 Subject: [PATCH 30/32] Decode SRID --- lib/myxql/protocol/values.ex | 5 +++-- test/myxql/protocol/values_test.exs | 14 +++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 97c573ce..65abeecc 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -388,8 +388,9 @@ defmodule MyXQL.Protocol.Values do end # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-internal-format - defp decode_geometry(<<0::uint4, r::bits>>) do - r |> Base.encode16() |> Geo.WKB.decode!() + defp decode_geometry(<>) do + srid = if srid == 0, do: nil, else: srid + r |> Base.encode16() |> Geo.WKB.decode!() |> Map.put(:srid, srid) end defp decode_int1(<>, null_bitmap, t, acc), diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 7e6bbc24..cdfaafc6 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -279,6 +279,11 @@ defmodule MyXQL.Protocol.ValueTest do assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}}) end + @tag geometry: true + test "POINT with SRID", c do + assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}, srid: 4326}) + end + @tag geometry: true test "LINESTRING", c do assert_roundtrip(c, "my_linestring", %Geo.LineString{ @@ -450,7 +455,14 @@ defmodule MyXQL.Protocol.ValueTest do defp encode_text(value), do: "'#{value}'" - defp encode_text_geo(value), do: "ST_GeomFromText('#{Geo.WKT.encode!(value)}')" + defp encode_text_geo(value) do + if srid = value.srid do + value = %{value | srid: nil} + "ST_GeomFromText('#{Geo.WKT.encode!(value)}', #{srid})" + else + "ST_GeomFromText('#{Geo.WKT.encode!(value)}')" + end + end defp get(c, fields, id) when is_list(fields) do fields = Enum.map_join(fields, ", ", &"`#{&1}`") From 659513d7419f769cabc5142dee2fe2bc37b611b3 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 21 Nov 2019 01:59:54 +0100 Subject: [PATCH 31/32] Simply test setup --- test/test_helper.exs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index e024b3ba..1c673300 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -75,7 +75,16 @@ defmodule TestHelper do my_time6 TIME(6), my_datetime3 DATETIME(3), my_datetime6 DATETIME(6), + """ + geometry = """ + my_point POINT, + my_linestring LINESTRING, + my_polygon POLYGON, + my_multipoint MULTIPOINT, + my_multilinestring MULTILINESTRING, + my_multipolygon MULTIPOLYGON, + my_geometrycollection GEOMETRYCOLLECTION, """ mysql!(""" @@ -125,13 +134,7 @@ defmodule TestHelper do my_mediumblob MEDIUMBLOB, my_longblob LONGBLOB, #{if supports_json?(), do: "my_json JSON,", else: ""} - #{if supports_geometry?(), do: "my_point POINT,", else: ""} - #{if supports_geometry?(), do: "my_linestring LINESTRING,", else: ""} - #{if supports_geometry?(), do: "my_polygon POLYGON,", else: ""} - #{if supports_geometry?(), do: "my_multipoint MULTIPOINT,", else: ""} - #{if supports_geometry?(), do: "my_multilinestring MULTILINESTRING,", else: ""} - #{if supports_geometry?(), do: "my_multipolygon MULTIPOLYGON,", else: ""} - #{if supports_geometry?(), do: "my_geometrycollection GEOMETRYCOLLECTION,", else: ""} + #{if supports_geometry?(), do: geometry, else: ""} my_char CHAR ); From 4d2e8a4206a01f520682500269f9d08e08ddf8e7 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 21 Nov 2019 02:00:14 +0100 Subject: [PATCH 32/32] Update supports_geometry? check --- test/test_helper.exs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index 1c673300..836a1a6d 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -190,17 +190,9 @@ defmodule TestHelper do end def supports_geometry?() do - sql = - "CREATE TEMPORARY TABLE myxql_test.test_geo (point POINT); SHOW COLUMNS IN myxql_test.test_geo" - - case mysql(sql) do - {:ok, result} -> - [%{"Type" => type}] = result - String.downcase(type) == "point" - - {:error, _} -> - false - end + # mysql 5.5 does not have ST_GeomFromText (it has GeomFromText) so we're excluding it + # (even though we could test against it) to keep the test suite simpler + match?({:ok, _}, mysql("SELECT ST_GeomFromText('POINT(0 0)')")) end def supports_timestamp_precision?() do @@ -258,6 +250,12 @@ defmodule TestHelper do [%{"@@version" => version}] = mysql!("select @@version") mariadb? = version =~ ~r"mariadb"i + mariadb_exclude = [ + # for both bit and geometry, inserting with wire protocol does not work for some reason + bit: true, + geometry: true + ] + exclude = for plugin <- supported_auth_plugins, not (plugin in available_auth_plugins) do @@ -270,7 +268,7 @@ defmodule TestHelper do exclude = [{:json, not supports_json?()} | exclude] exclude = [{:geometry, not supports_geometry?()} | exclude] exclude = [{:timestamp_precision, not supports_timestamp_precision?()} | exclude] - exclude = if mariadb?, do: [{:bit, true} | exclude], else: exclude + exclude = if mariadb?, do: mariadb_exclude ++ exclude, else: exclude exclude end end