diff --git a/MARIAEX_COMPATIBILITY.md b/MARIAEX_COMPATIBILITY.md index 57d28bc9..77ed105f 100644 --- a/MARIAEX_COMPATIBILITY.md +++ b/MARIAEX_COMPATIBILITY.md @@ -38,8 +38,6 @@ Queries: * MyXQL represents bit type `B'101'` as `<<1::1, 0::0, 1::1>>` (`<<5::size(3)>>`), Mariaex represents it as `<<5>>` - * MyXQL does not support geometry types - * MyXQL does not support `:type_names`, `result_types`, `:decode`, `:encode_mapper`, `:decode_mapper`, `:include_table_name`, and `:binary_as` options diff --git a/README.md b/README.md index d2046d40..d1fd6cfe 100644 --- a/README.md +++ b/README.md @@ -85,22 +85,23 @@ See [Mariaex Compatibility](https://github.com/elixir-ecto/myxql/blob/master/MAR ## Data representation ``` -MySQL Elixir ------ ------ -NULL nil -bool 1 | 0 -int 42 -float 42.0 -decimal #Decimal<42.0> * -date ~D[2013-10-12] ** -time ~T[00:37:14] -datetime ~N[2013-10-12 00:37:14] **, *** -timestamp #DateTime<2013-10-12 00:37:14Z> *** -json %{"foo" => "bar"} **** -char "é" -text "myxql" -binary <<1, 2, 3>> -bit <<1::size(1), 0::size(1)>> +MySQL Elixir +----- ------ +NULL nil +bool 1 | 0 +int 42 +float 42.0 +decimal #Decimal<42.0> * +date ~D[2013-10-12] ** +time ~T[00:37:14] +datetime ~N[2013-10-12 00:37:14] **, *** +timestamp ~U[2013-10-12 00:37:14Z] *** +json %{"foo" => "bar"} **** +char "é" +text "myxql" +binary <<1, 2, 3>> +bit <<1::size(1), 0::size(1)>> +point, polygon, ... %Geo.Point{coordinates: {0.0, 1.0}}, ... ***** ``` \* See [Decimal](https://github.com/ericmj/decimal) @@ -112,6 +113,11 @@ bit <<1::size(1), 0::size(1)>> \*\*\*\* MySQL added a native JSON type in version 5.7.8, if you're using earlier versions, remember to use TEXT column for your JSON field. +\*\*\*\*\* Encoding/decoding between `Geo.*` structs and the OpenGIS WKB binary format is +done using the [Geo](https://github.com/bryanjos/geo) package. If you're using MyXQL geometry +types with Ecto and need to for example accept a WKT format as user input, consider implementing an +[custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html). + ## JSON support MyXQL comes with JSON support out of the box via the [Jason](https://github.com/michalmuskala/jason) library. To use it, add `:jason` to your dependencies: diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 4e989f7c..65abeecc 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -83,6 +83,7 @@ defmodule MyXQL.Protocol.Values do defp column_def_to_type(column_def(type: :mysql_type_string)), do: :binary defp column_def_to_type(column_def(type: :mysql_type_bit, length: length)), do: {:bit, length} defp column_def_to_type(column_def(type: :mysql_type_null)), do: :null + defp column_def_to_type(column_def(type: :mysql_type_geometry)), do: :geometry # Text values @@ -171,6 +172,10 @@ defmodule MyXQL.Protocol.Values do decode_bit(value, size) end + def decode_text_value(value, :geometry) do + decode_geometry(value) + end + # Binary values def encode_binary_value(value) @@ -216,6 +221,14 @@ defmodule MyXQL.Protocol.Values do {:mysql_type_tiny, <<0>>} end + def encode_binary_value(%Geo.Point{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiPoint{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.LineString{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiLineString{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.Polygon{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.MultiPolygon{} = geo), do: encode_geometry(geo) + def encode_binary_value(%Geo.GeometryCollection{} = geo), do: encode_geometry(geo) + def encode_binary_value(term) when is_list(term) or is_map(term) do string = json_library().encode!(term) {:mysql_type_var_string, encode_string_lenenc(string)} @@ -225,6 +238,12 @@ defmodule MyXQL.Protocol.Values do raise ArgumentError, "query has invalid parameter #{inspect(other)}" end + defp encode_geometry(geo) do + srid = geo.srid || 0 + binary = %{geo | srid: nil} |> Geo.WKB.encode!(:ndr) |> Base.decode16!() + {:mysql_type_geometry, encode_string_lenenc(<>)} + end + ## Time/DateTime # MySQL supports negative time and days, we don't. @@ -308,7 +327,7 @@ defmodule MyXQL.Protocol.Values do end defp decode_binary_row(<>, null_bitmap, [:binary | t], acc), - do: decode_string_lenenc(r, null_bitmap, t, acc) + do: decode_string_lenenc(r, null_bitmap, t, acc, & &1) defp decode_binary_row(<>, null_bitmap, [:int1 | t], acc), do: decode_int1(r, null_bitmap, t, acc) @@ -361,10 +380,19 @@ defmodule MyXQL.Protocol.Values do defp decode_binary_row(<>, null_bitmap, [{:bit, size} | t], acc), do: decode_bit(r, size, null_bitmap, t, acc) + defp decode_binary_row(<>, null_bitmap, [:geometry | t], acc), + do: decode_string_lenenc(r, null_bitmap, t, acc, &decode_geometry/1) + defp decode_binary_row(<<>>, _null_bitmap, [], acc) do Enum.reverse(acc) end + # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-internal-format + defp decode_geometry(<>) do + srid = if srid == 0, do: nil, else: srid + r |> Base.encode16() |> Geo.WKB.decode!() |> Map.put(:srid, srid) + end + defp decode_int1(<>, null_bitmap, t, acc), do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) @@ -510,18 +538,36 @@ defmodule MyXQL.Protocol.Values do } end - defp decode_string_lenenc(<>, null_bitmap, t, acc) + defp decode_string_lenenc(<>, null_bitmap, t, acc, decoder) when n < 251, - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) - defp decode_string_lenenc(<<0xFC, n::uint2, v::string(n), r::bits>>, null_bitmap, t, acc), - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + defp decode_string_lenenc( + <<0xFC, n::uint2, v::string(n), r::bits>>, + null_bitmap, + t, + acc, + decoder + ), + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) - defp decode_string_lenenc(<<0xFD, n::uint3, v::string(n), r::bits>>, null_bitmap, t, acc), - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + defp decode_string_lenenc( + <<0xFD, n::uint3, v::string(n), r::bits>>, + null_bitmap, + t, + acc, + decoder + ), + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) - defp decode_string_lenenc(<<0xFE, n::uint8, v::string(n), r::bits>>, null_bitmap, t, acc), - do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc]) + defp decode_string_lenenc( + <<0xFE, n::uint8, v::string(n), r::bits>>, + null_bitmap, + t, + acc, + decoder + ), + do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc]) defp decode_json(<>, null_bitmap, t, acc) when n < 251, do: decode_binary_row(r, null_bitmap >>> 1, t, [decode_json(v) | acc]) diff --git a/mix.exs b/mix.exs index ed11af86..d0cd234f 100644 --- a/mix.exs +++ b/mix.exs @@ -51,6 +51,7 @@ defmodule MyXQL.MixProject do {:db_connection, "~> 2.0", db_connection_opts()}, {:decimal, "~> 1.6"}, {:jason, "~> 1.0", optional: true}, + {:geo, "~> 3.3"}, {:binpp, ">= 0.0.0", only: [:dev, :test]}, {:dialyxir, "~> 1.0-rc", only: :dev, runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index fa1cdf0f..31069ca7 100644 --- a/mix.lock +++ b/mix.lock @@ -9,6 +9,7 @@ "earmark": {:hex, :earmark, "1.4.0", "397e750b879df18198afc66505ca87ecf6a96645545585899f6185178433cc09", [:mix], [], "hexpm"}, "erlex": {:hex, :erlex, "0.2.1", "cee02918660807cbba9a7229cae9b42d1c6143b768c781fa6cee1eaf03ad860b", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, + "geo": {:hex, :geo, "3.3.2", "30c7b458bcb0ab1ca73a997b26d22c68643d9ffe1726e475e0759499b6024cff", [:mix], [], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/myxql/protocol/values_test.exs b/test/myxql/protocol/values_test.exs index 33061615..cdfaafc6 100644 --- a/test/myxql/protocol/values_test.exs +++ b/test/myxql/protocol/values_test.exs @@ -273,6 +273,64 @@ defmodule MyXQL.Protocol.ValueTest do test "CHAR", c do assert_roundtrip(c, "my_char", "é") end + + @tag geometry: true + test "POINT", c do + assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}}) + end + + @tag geometry: true + test "POINT with SRID", c do + assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}, srid: 4326}) + end + + @tag geometry: true + test "LINESTRING", c do + assert_roundtrip(c, "my_linestring", %Geo.LineString{ + coordinates: [{30, 10}, {10, 30}, {40, 40}] + }) + end + + @tag geometry: true + test "MULTIPOINT", c do + assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{ + coordinates: [{1.0, 1.0}, {2.0, 2.0}] + }) + end + + @tag geometry: true + test "MULTILINESTRING", c do + assert_roundtrip(c, "my_multilinestring", %Geo.MultiLineString{ + coordinates: [[{10, 10}, {20, 20}], [{15, 15}, {30, 15}]] + }) + end + + @tag geometry: true + test "POLYGON", c do + assert_roundtrip(c, "my_polygon", %Geo.Polygon{ + coordinates: [[{30, 10}, {40, 40}, {20, 40}, {10, 20}, {30, 10}]] + }) + end + + @tag geometry: true + test "MULTIPOLYGON", c do + assert_roundtrip(c, "my_multipolygon", %Geo.MultiPolygon{ + coordinates: [ + [[{40, 40}, {20, 45}, {45, 30}, {40, 40}]], + [ + [{20, 35}, {10, 30}, {10, 10}, {30, 5}, {45, 20}, {20, 35}], + [{30, 20}, {20, 15}, {20, 25}, {30, 20}] + ] + ] + }) + end + + @tag geometry: true + test "GEOMETRYCOLLECTION", c do + assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{ + geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}}] + }) + end end end @@ -352,41 +410,7 @@ defmodule MyXQL.Protocol.ValueTest do defp insert(%{protocol: :text} = c, fields, values) when is_list(fields) and is_list(values) do fields = Enum.map_join(fields, ", ", &"`#{&1}`") - - values = - Enum.map_join(values, ", ", fn - nil -> - "NULL" - - true -> - "TRUE" - - false -> - "FALSE" - - %DateTime{} = datetime -> - "'#{NaiveDateTime.to_iso8601(datetime)}'" - - list when is_list(list) -> - "'#{Jason.encode!(list)}'" - - %_{} = struct -> - "'#{struct}'" - - map when is_map(map) -> - "'#{Jason.encode!(map)}'" - - value when is_binary(value) -> - "'#{value}'" - - value when is_bitstring(value) -> - size = bit_size(value) - <> = value - "B'#{Integer.to_string(value, 2)}'" - - value -> - "'#{value}'" - end) + values = Enum.map_join(values, ", ", &encode_text/1) %MyXQL.Result{last_insert_id: id} = query!(c, "INSERT INTO test_types (#{fields}) VALUES (#{values})") @@ -407,6 +431,39 @@ defmodule MyXQL.Protocol.ValueTest do insert(c, [field], [value]) end + defp encode_text(nil), do: "NULL" + defp encode_text(true), do: "TRUE" + defp encode_text(false), do: "FALSE" + defp encode_text(%DateTime{} = datetime), do: "'#{NaiveDateTime.to_iso8601(datetime)}'" + defp encode_text(%Geo.Point{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.LineString{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.Polygon{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.MultiLineString{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.MultiPolygon{} = geo), do: encode_text_geo(geo) + defp encode_text(%Geo.GeometryCollection{} = geo), do: encode_text_geo(geo) + defp encode_text(%_{} = struct), do: "'#{struct}'" + defp encode_text(map) when is_map(map), do: "'#{Jason.encode!(map)}'" + defp encode_text(list) when is_list(list), do: "'#{Jason.encode!(list)}'" + defp encode_text(value) when is_binary(value), do: "'#{value}'" + + defp encode_text(value) when is_bitstring(value) do + size = bit_size(value) + <> = value + "B'#{Integer.to_string(value, 2)}'" + end + + defp encode_text(value), do: "'#{value}'" + + defp encode_text_geo(value) do + if srid = value.srid do + value = %{value | srid: nil} + "ST_GeomFromText('#{Geo.WKT.encode!(value)}', #{srid})" + else + "ST_GeomFromText('#{Geo.WKT.encode!(value)}')" + end + end + defp get(c, fields, id) when is_list(fields) do fields = Enum.map_join(fields, ", ", &"`#{&1}`") statement = "SELECT #{fields} FROM test_types WHERE id = '#{id}'" diff --git a/test/test_helper.exs b/test/test_helper.exs index eb4db623..836a1a6d 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -77,6 +77,16 @@ defmodule TestHelper do my_datetime6 DATETIME(6), """ + geometry = """ + my_point POINT, + my_linestring LINESTRING, + my_polygon POLYGON, + my_multipoint MULTIPOINT, + my_multilinestring MULTILINESTRING, + my_multipolygon MULTIPOLYGON, + my_geometrycollection GEOMETRYCOLLECTION, + """ + mysql!(""" USE myxql_test; @@ -124,6 +134,7 @@ defmodule TestHelper do my_mediumblob MEDIUMBLOB, my_longblob LONGBLOB, #{if supports_json?(), do: "my_json JSON,", else: ""} + #{if supports_geometry?(), do: geometry, else: ""} my_char CHAR ); @@ -178,6 +189,12 @@ defmodule TestHelper do end end + def supports_geometry?() do + # mysql 5.5 does not have ST_GeomFromText (it has GeomFromText) so we're excluding it + # (even though we could test against it) to keep the test suite simpler + match?({:ok, _}, mysql("SELECT ST_GeomFromText('POINT(0 0)')")) + end + def supports_timestamp_precision?() do case mysql("CREATE TEMPORARY TABLE myxql_test.timestamp_precision (time time(3));") do {:ok, _} -> true @@ -233,6 +250,12 @@ defmodule TestHelper do [%{"@@version" => version}] = mysql!("select @@version") mariadb? = version =~ ~r"mariadb"i + mariadb_exclude = [ + # for both bit and geometry, inserting with wire protocol does not work for some reason + bit: true, + geometry: true + ] + exclude = for plugin <- supported_auth_plugins, not (plugin in available_auth_plugins) do @@ -243,8 +266,9 @@ defmodule TestHelper do exclude = [{:ssl, not supports_ssl?()} | exclude] exclude = [{:public_key_exchange, not supports_public_key_exchange?()} | exclude] exclude = [{:json, not supports_json?()} | exclude] + exclude = [{:geometry, not supports_geometry?()} | exclude] exclude = [{:timestamp_precision, not supports_timestamp_precision?()} | exclude] - exclude = if mariadb?, do: [{:bit, true} | exclude], else: exclude + exclude = if mariadb?, do: mariadb_exclude ++ exclude, else: exclude exclude end end