Geometry types (#88)

Author: jeroenbourgois

MARIAEX_COMPATIBILITY.md

@@ -38,8 +38,6 @@ Queries:
 
   * MyXQL represents bit type `B'101'` as `<<1::1, 0::0, 1::1>>` (`<<5::size(3)>>`), Mariaex represents it as `<<5>>`
 
-  * MyXQL does not support geometry types
-
   * MyXQL does not support `:type_names`, `result_types`, `:decode`, `:encode_mapper`,
     `:decode_mapper`, `:include_table_name`, and `:binary_as` options
 

README.md

@@ -85,22 +85,23 @@ See [Mariaex Compatibility](https://github.com/elixir-ecto/myxql/blob/master/MAR
 ## Data representation
 
 ```
-MySQL           Elixir
------           ------
-NULL            nil
-bool            1 | 0
-int             42
-float           42.0
-decimal         #Decimal<42.0> *
-date            ~D[2013-10-12] **
-time            ~T[00:37:14]
-datetime        ~N[2013-10-12 00:37:14] **, ***
-timestamp       #DateTime<2013-10-12 00:37:14Z> ***
-json            %{"foo" => "bar"} ****
-char            "é"
-text            "myxql"
-binary          <<1, 2, 3>>
-bit             <<1::size(1), 0::size(1)>>
+MySQL                Elixir
+-----                ------
+NULL                 nil
+bool                 1 | 0
+int                  42
+float                42.0
+decimal              #Decimal<42.0> *
+date                 ~D[2013-10-12] **
+time                 ~T[00:37:14]
+datetime             ~N[2013-10-12 00:37:14] **, ***
+timestamp            ~U[2013-10-12 00:37:14Z] ***
+json                 %{"foo" => "bar"} ****
+char                 "é"
+text                 "myxql"
+binary               <<1, 2, 3>>
+bit                  <<1::size(1), 0::size(1)>>
+point, polygon, ...  %Geo.Point{coordinates: {0.0, 1.0}}, ... *****
 ```
 
 \* See [Decimal](https://github.com/ericmj/decimal)
@@ -112,6 +113,11 @@ bit             <<1::size(1), 0::size(1)>>
 \*\*\*\* MySQL added a native JSON type in version 5.7.8, if you're using earlier versions,
 remember to use TEXT column for your JSON field.
 
+\*\*\*\*\* Encoding/decoding between `Geo.*` structs and the OpenGIS WKB binary format is
+done using the [Geo](https://github.com/bryanjos/geo) package. If you're using MyXQL geometry
+types with Ecto and need to for example accept a WKT format as user input, consider implementing an
+[custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html).
+
 ## JSON support
 
 MyXQL comes with JSON support out of the box via the [Jason](https://github.com/michalmuskala/jason) library. To use it, add `:jason` to your dependencies:

lib/myxql/protocol/values.ex

@@ -83,6 +83,7 @@ defmodule MyXQL.Protocol.Values do
   defp column_def_to_type(column_def(type: :mysql_type_string)), do: :binary
   defp column_def_to_type(column_def(type: :mysql_type_bit, length: length)), do: {:bit, length}
   defp column_def_to_type(column_def(type: :mysql_type_null)), do: :null
+  defp column_def_to_type(column_def(type: :mysql_type_geometry)), do: :geometry
 
   # Text values
 
@@ -171,6 +172,10 @@ defmodule MyXQL.Protocol.Values do
     decode_bit(value, size)
   end
 
+  def decode_text_value(value, :geometry) do
+    decode_geometry(value)
+  end
+
   # Binary values
 
   def encode_binary_value(value)
@@ -216,6 +221,14 @@ defmodule MyXQL.Protocol.Values do
     {:mysql_type_tiny, <<0>>}
   end
 
+  def encode_binary_value(%Geo.Point{} = geo), do: encode_geometry(geo)
+  def encode_binary_value(%Geo.MultiPoint{} = geo), do: encode_geometry(geo)
+  def encode_binary_value(%Geo.LineString{} = geo), do: encode_geometry(geo)
+  def encode_binary_value(%Geo.MultiLineString{} = geo), do: encode_geometry(geo)
+  def encode_binary_value(%Geo.Polygon{} = geo), do: encode_geometry(geo)
+  def encode_binary_value(%Geo.MultiPolygon{} = geo), do: encode_geometry(geo)
+  def encode_binary_value(%Geo.GeometryCollection{} = geo), do: encode_geometry(geo)
+
   def encode_binary_value(term) when is_list(term) or is_map(term) do
     string = json_library().encode!(term)
     {:mysql_type_var_string, encode_string_lenenc(string)}
@@ -225,6 +238,12 @@ defmodule MyXQL.Protocol.Values do
     raise ArgumentError, "query has invalid parameter #{inspect(other)}"
   end
 
+  defp encode_geometry(geo) do
+    srid = geo.srid || 0
+    binary = %{geo | srid: nil} |> Geo.WKB.encode!(:ndr) |> Base.decode16!()
+    {:mysql_type_geometry, encode_string_lenenc(<<srid::uint4, binary::binary>>)}
+  end
+
   ## Time/DateTime
 
   # MySQL supports negative time and days, we don't.
@@ -308,7 +327,7 @@ defmodule MyXQL.Protocol.Values do
   end
 
   defp decode_binary_row(<<r::bits>>, null_bitmap, [:binary | t], acc),
-    do: decode_string_lenenc(r, null_bitmap, t, acc)
+    do: decode_string_lenenc(r, null_bitmap, t, acc, & &1)
 
   defp decode_binary_row(<<r::bits>>, null_bitmap, [:int1 | t], acc),
     do: decode_int1(r, null_bitmap, t, acc)
@@ -361,10 +380,19 @@ defmodule MyXQL.Protocol.Values do
   defp decode_binary_row(<<r::bits>>, null_bitmap, [{:bit, size} | t], acc),
     do: decode_bit(r, size, null_bitmap, t, acc)
 
+  defp decode_binary_row(<<r::bits>>, null_bitmap, [:geometry | t], acc),
+    do: decode_string_lenenc(r, null_bitmap, t, acc, &decode_geometry/1)
+
   defp decode_binary_row(<<>>, _null_bitmap, [], acc) do
     Enum.reverse(acc)
   end
 
+  # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-internal-format
+  defp decode_geometry(<<srid::uint4, r::bits>>) do
+    srid = if srid == 0, do: nil, else: srid
+    r |> Base.encode16() |> Geo.WKB.decode!() |> Map.put(:srid, srid)
+  end
+
   defp decode_int1(<<v::int1, r::bits>>, null_bitmap, t, acc),
     do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc])
 
@@ -510,18 +538,36 @@ defmodule MyXQL.Protocol.Values do
     }
   end
 
-  defp decode_string_lenenc(<<n::uint1, v::string(n), r::bits>>, null_bitmap, t, acc)
+  defp decode_string_lenenc(<<n::uint1, v::string(n), r::bits>>, null_bitmap, t, acc, decoder)
        when n < 251,
-       do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc])
+       do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc])
 
-  defp decode_string_lenenc(<<0xFC, n::uint2, v::string(n), r::bits>>, null_bitmap, t, acc),
-    do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc])
+  defp decode_string_lenenc(
+         <<0xFC, n::uint2, v::string(n), r::bits>>,
+         null_bitmap,
+         t,
+         acc,
+         decoder
+       ),
+       do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc])
 
-  defp decode_string_lenenc(<<0xFD, n::uint3, v::string(n), r::bits>>, null_bitmap, t, acc),
-    do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc])
+  defp decode_string_lenenc(
+         <<0xFD, n::uint3, v::string(n), r::bits>>,
+         null_bitmap,
+         t,
+         acc,
+         decoder
+       ),
+       do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc])
 
-  defp decode_string_lenenc(<<0xFE, n::uint8, v::string(n), r::bits>>, null_bitmap, t, acc),
-    do: decode_binary_row(r, null_bitmap >>> 1, t, [v | acc])
+  defp decode_string_lenenc(
+         <<0xFE, n::uint8, v::string(n), r::bits>>,
+         null_bitmap,
+         t,
+         acc,
+         decoder
+       ),
+       do: decode_binary_row(r, null_bitmap >>> 1, t, [decoder.(v) | acc])
 
   defp decode_json(<<n::uint1, v::string(n), r::bits>>, null_bitmap, t, acc) when n < 251,
     do: decode_binary_row(r, null_bitmap >>> 1, t, [decode_json(v) | acc])

mix.exs

@@ -51,6 +51,7 @@ defmodule MyXQL.MixProject do
       {:db_connection, "~> 2.0", db_connection_opts()},
       {:decimal, "~> 1.6"},
       {:jason, "~> 1.0", optional: true},
+      {:geo, "~> 3.3"},
       {:binpp, ">= 0.0.0", only: [:dev, :test]},
       {:dialyxir, "~> 1.0-rc", only: :dev, runtime: false},
       {:ex_doc, ">= 0.0.0", only: :dev, runtime: false},

mix.lock

@@ -9,6 +9,7 @@
   "earmark": {:hex, :earmark, "1.4.0", "397e750b879df18198afc66505ca87ecf6a96645545585899f6185178433cc09", [:mix], [], "hexpm"},
   "erlex": {:hex, :erlex, "0.2.1", "cee02918660807cbba9a7229cae9b42d1c6143b768c781fa6cee1eaf03ad860b", [:mix], [], "hexpm"},
   "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
+  "geo": {:hex, :geo, "3.3.2", "30c7b458bcb0ab1ca73a997b26d22c68643d9ffe1726e475e0759499b6024cff", [:mix], [], "hexpm"},
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
   "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},

test/myxql/protocol/values_test.exs

@@ -273,6 +273,64 @@ defmodule MyXQL.Protocol.ValueTest do
       test "CHAR", c do
         assert_roundtrip(c, "my_char", "é")
       end
+
+      @tag geometry: true
+      test "POINT", c do
+        assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}})
+      end
+
+      @tag geometry: true
+      test "POINT with SRID", c do
+        assert_roundtrip(c, "my_point", %Geo.Point{coordinates: {0, 0}, srid: 4326})
+      end
+
+      @tag geometry: true
+      test "LINESTRING", c do
+        assert_roundtrip(c, "my_linestring", %Geo.LineString{
+          coordinates: [{30, 10}, {10, 30}, {40, 40}]
+        })
+      end
+
+      @tag geometry: true
+      test "MULTIPOINT", c do
+        assert_roundtrip(c, "my_multipoint", %Geo.MultiPoint{
+          coordinates: [{1.0, 1.0}, {2.0, 2.0}]
+        })
+      end
+
+      @tag geometry: true
+      test "MULTILINESTRING", c do
+        assert_roundtrip(c, "my_multilinestring", %Geo.MultiLineString{
+          coordinates: [[{10, 10}, {20, 20}], [{15, 15}, {30, 15}]]
+        })
+      end
+
+      @tag geometry: true
+      test "POLYGON", c do
+        assert_roundtrip(c, "my_polygon", %Geo.Polygon{
+          coordinates: [[{30, 10}, {40, 40}, {20, 40}, {10, 20}, {30, 10}]]
+        })
+      end
+
+      @tag geometry: true
+      test "MULTIPOLYGON", c do
+        assert_roundtrip(c, "my_multipolygon", %Geo.MultiPolygon{
+          coordinates: [
+            [[{40, 40}, {20, 45}, {45, 30}, {40, 40}]],
+            [
+              [{20, 35}, {10, 30}, {10, 10}, {30, 5}, {45, 20}, {20, 35}],
+              [{30, 20}, {20, 15}, {20, 25}, {30, 20}]
+            ]
+          ]
+        })
+      end
+
+      @tag geometry: true
+      test "GEOMETRYCOLLECTION", c do
+        assert_roundtrip(c, "my_geometrycollection", %Geo.GeometryCollection{
+          geometries: [%Geo.Point{coordinates: {54.1745659, 15.5398456}}]
+        })
+      end
     end
   end
 
@@ -352,41 +410,7 @@ defmodule MyXQL.Protocol.ValueTest do
 
   defp insert(%{protocol: :text} = c, fields, values) when is_list(fields) and is_list(values) do
     fields = Enum.map_join(fields, ", ", &"`#{&1}`")
-
-    values =
-      Enum.map_join(values, ", ", fn
-        nil ->
-          "NULL"
-
-        true ->
-          "TRUE"
-
-        false ->
-          "FALSE"
-
-        %DateTime{} = datetime ->
-          "'#{NaiveDateTime.to_iso8601(datetime)}'"
-
-        list when is_list(list) ->
-          "'#{Jason.encode!(list)}'"
-
-        %_{} = struct ->
-          "'#{struct}'"
-
-        map when is_map(map) ->
-          "'#{Jason.encode!(map)}'"
-
-        value when is_binary(value) ->
-          "'#{value}'"
-
-        value when is_bitstring(value) ->
-          size = bit_size(value)
-          <<value::size(size)>> = value
-          "B'#{Integer.to_string(value, 2)}'"
-
-        value ->
-          "'#{value}'"
-      end)
+    values = Enum.map_join(values, ", ", &encode_text/1)
 
     %MyXQL.Result{last_insert_id: id} =
       query!(c, "INSERT INTO test_types (#{fields}) VALUES (#{values})")
@@ -407,6 +431,39 @@ defmodule MyXQL.Protocol.ValueTest do
     insert(c, [field], [value])
   end
 
+  defp encode_text(nil), do: "NULL"
+  defp encode_text(true), do: "TRUE"
+  defp encode_text(false), do: "FALSE"
+  defp encode_text(%DateTime{} = datetime), do: "'#{NaiveDateTime.to_iso8601(datetime)}'"
+  defp encode_text(%Geo.Point{} = geo), do: encode_text_geo(geo)
+  defp encode_text(%Geo.LineString{} = geo), do: encode_text_geo(geo)
+  defp encode_text(%Geo.Polygon{} = geo), do: encode_text_geo(geo)
+  defp encode_text(%Geo.MultiPoint{} = geo), do: encode_text_geo(geo)
+  defp encode_text(%Geo.MultiLineString{} = geo), do: encode_text_geo(geo)
+  defp encode_text(%Geo.MultiPolygon{} = geo), do: encode_text_geo(geo)
+  defp encode_text(%Geo.GeometryCollection{} = geo), do: encode_text_geo(geo)
+  defp encode_text(%_{} = struct), do: "'#{struct}'"
+  defp encode_text(map) when is_map(map), do: "'#{Jason.encode!(map)}'"
+  defp encode_text(list) when is_list(list), do: "'#{Jason.encode!(list)}'"
+  defp encode_text(value) when is_binary(value), do: "'#{value}'"
+
+  defp encode_text(value) when is_bitstring(value) do
+    size = bit_size(value)
+    <<value::size(size)>> = value
+    "B'#{Integer.to_string(value, 2)}'"
+  end
+
+  defp encode_text(value), do: "'#{value}'"
+
+  defp encode_text_geo(value) do
+    if srid = value.srid do
+      value = %{value | srid: nil}
+      "ST_GeomFromText('#{Geo.WKT.encode!(value)}', #{srid})"
+    else
+      "ST_GeomFromText('#{Geo.WKT.encode!(value)}')"
+    end
+  end
+
   defp get(c, fields, id) when is_list(fields) do
     fields = Enum.map_join(fields, ", ", &"`#{&1}`")
     statement = "SELECT #{fields} FROM test_types WHERE id = '#{id}'"