Add multi-step input method support for Linux

This implements the Gtk hooks required to support multi-step input
methods on Linux. This builds on the support for composing regions
(preedit region in Gtk terminology) added to TextInputModel in
flutter#21682.

Specifically, the following changes are included:

1. Add handler for TextInput.setMarkedTextRegion framework messages: On
any change to the EditableText in the framework, this message is sent
which provides an updated rect (in the local co-ordinates of the
EditableText) for the composing region. If not in composing mode, the
cursor rect is sent.

2. Add handler for TextInput.setEditableSizeAndTransform framework messages:
On any change to the RenderObject underlying the EditableText, an
updated size for the full EditableText widget, as well as an affine
transform matrix from local co-ordinates to Flutter root co-ordinates is
sent.

3. On either of the above messages, we now recompute and store a cached
copy of the preedit rect in the Gtk window co-ordinates. This rect is
used to inform Gtk of the cursor position, so that it can position any
system IM composing window correctly for on-the-spot composing, such as
is used when inputting Japanese text.

4. Adds handlers for preedit-start, preedit-changed, and preedit-end
signals from Gtk. These are passed on to the TextInputModel.

5. Updates the preedit-commit handler to commit the composing region to
the text or, if not composing, insert new text at the cursor.

6. Updates the handler for TextInput.setEditingState framework messages
to extract the composing range base and extent and pass these on to
TextInputModel.

7. Updates update_editing_state function to set composing base and
extent on text input state updates sent to the framework.

Author: cbracken

shell/platform/linux/fl_text_input_plugin.cc

@@ -22,6 +22,9 @@ static constexpr char kHideMethod[] = "TextInput.hide";
 static constexpr char kUpdateEditingStateMethod[] =
     "TextInputClient.updateEditingState";
 static constexpr char kPerformActionMethod[] = "TextInputClient.performAction";
+static constexpr char kSetEditableSizeAndTransform[] =
+    "TextInput.setEditableSizeAndTransform";
+static constexpr char kSetMarkedTextRect[] = "TextInput.setMarkedTextRect";
 
 static constexpr char kInputActionKey[] = "inputAction";
 static constexpr char kTextInputTypeKey[] = "inputType";
@@ -34,6 +37,8 @@ static constexpr char kSelectionIsDirectionalKey[] = "selectionIsDirectional";
 static constexpr char kComposingBaseKey[] = "composingBase";
 static constexpr char kComposingExtentKey[] = "composingExtent";
 
+static constexpr char kTransform[] = "transform";
+
 static constexpr char kTextAffinityDownstream[] = "TextAffinity.downstream";
 static constexpr char kMultilineInputType[] = "TextInputType.multiline";
 
@@ -57,6 +62,23 @@ struct _FlTextInputPlugin {
   GtkIMContext* im_context;
 
   flutter::TextInputModel* text_model;
+
+  // The owning Flutter view.
+  FlView* view;
+
+  // A 4x4 matrix that maps from `EditableText` local coordinates to the
+  // coordinate system of `PipelineOwner.rootNode`.
+  double editabletext_transform[4][4];
+
+  // The smallest rect, in local coordinates, of the text in the composing
+  // range, or of the caret in the case where there is no current composing
+  // range. This value is updated via `TextInput.setMarkedTextRect` messages
+  // over the text input channel.
+  GdkRectangle composing_rect;
+
+  // A cached copy of the composing (pre-edit in Gtk terminology) rect in
+  // window coordinates.
+  GdkRectangle preedit_rect;
 };
 
 G_DEFINE_TYPE(FlTextInputPlugin, fl_text_input_plugin, G_TYPE_OBJECT)
@@ -99,13 +121,22 @@ static void update_editing_state(FlTextInputPlugin* self) {
   fl_value_set_string_take(value, kSelectionExtentKey,
                            fl_value_new_int(selection.extent()));
 
+  int composing_base = self->text_model->composing()
+                           ? self->text_model->composing_range().base()
+                           : -1;
+  int composing_extent = self->text_model->composing()
+                             ? self->text_model->composing_range().extent()
+                             : -1;
+  fl_value_set_string_take(value, kComposingBaseKey,
+                           fl_value_new_int(composing_base));
+  fl_value_set_string_take(value, kComposingExtentKey,
+                           fl_value_new_int(composing_extent));
+
   // The following keys are not implemented and set to default values.
   fl_value_set_string_take(value, kSelectionAffinityKey,
                            fl_value_new_string(kTextAffinityDownstream));
   fl_value_set_string_take(value, kSelectionIsDirectionalKey,
                            fl_value_new_bool(FALSE));
-  fl_value_set_string_take(value, kComposingBaseKey, fl_value_new_int(-1));
-  fl_value_set_string_take(value, kComposingExtentKey, fl_value_new_int(-1));
 
   fl_value_append(args, value);
 
@@ -138,9 +169,50 @@ static void perform_action(FlTextInputPlugin* self) {
                                   nullptr, perform_action_response_cb, self);
 }
 
+// Signal handler for GtkIMContext::preedit-start
+static void im_preedit_start_cb(FlTextInputPlugin* self) {
+  self->text_model->BeginComposing();
+
+  // Set the top-level window used for system input method windows.
+  GdkWindow* window =
+      gtk_widget_get_window(gtk_widget_get_toplevel(GTK_WIDGET(self->view)));
+  gtk_im_context_set_client_window(self->im_context, window);
+
+  // Set the cursor location in window co-ordinates so that Gtk can position
+  // any system input method windows.
+  gtk_im_context_set_cursor_location(self->im_context, &self->preedit_rect);
+}
+
+// Signal handler for GtkIMContext::preedit-changed
+static void im_preedit_changed_cb(FlTextInputPlugin* self) {
+  // Set the cursor location in window co-ordinates so that Gtk can position
+  // any system input method windows.
+  gtk_im_context_set_cursor_location(self->im_context, &self->preedit_rect);
+
+  gchar* buf = nullptr;
+  gint cursor_offset = 0;
+  gtk_im_context_get_preedit_string(self->im_context, &buf, NULL,
+                                    &cursor_offset);
+  cursor_offset += self->text_model->composing_range().base();
+  self->text_model->UpdateComposingText(buf);
+  self->text_model->SetSelection(TextRange(cursor_offset, cursor_offset));
+
+  update_editing_state(self);
+}
+
 // Signal handler for GtkIMContext::commit
 static void im_commit_cb(FlTextInputPlugin* self, const gchar* text) {
-  self->text_model->AddText(text);
+  if (self->text_model->composing()) {
+    self->text_model->CommitComposing();
+  } else {
+    self->text_model->AddText(text);
+  }
+  update_editing_state(self);
+}
+
+// Signal handler for GtkIMContext::preedit-end
+static void im_preedit_end_cb(FlTextInputPlugin* self) {
+  self->text_model->EndComposing();
   update_editing_state(self);
 }
 
@@ -208,6 +280,8 @@ static FlMethodResponse* set_editing_state(FlTextInputPlugin* self,
                                            FlValue* args) {
   const gchar* text =
       fl_value_get_string(fl_value_lookup_string(args, kTextKey));
+  self->text_model->SetText(text);
+
   int64_t selection_base =
       fl_value_get_int(fl_value_lookup_string(args, kSelectionBaseKey));
   int64_t selection_extent =
@@ -220,6 +294,19 @@ static FlMethodResponse* set_editing_state(FlTextInputPlugin* self,
   self->text_model->SetText(text);
   self->text_model->SetSelection(TextRange(selection_base, selection_extent));
 
+  int64_t composing_base =
+      fl_value_get_int(fl_value_lookup_string(args, kComposingBaseKey));
+  int64_t composing_extent =
+      fl_value_get_int(fl_value_lookup_string(args, kComposingExtentKey));
+  if (composing_base == -1 && composing_extent == -1) {
+    self->text_model->EndComposing();
+  } else {
+    size_t composing_start = std::min(composing_base, composing_extent);
+    size_t cursor_offset = selection_base - composing_start;
+    self->text_model->SetComposingRange(
+        TextRange(composing_base, composing_extent), cursor_offset);
+  }
+
   return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
 }
 
@@ -237,6 +324,70 @@ static FlMethodResponse* hide(FlTextInputPlugin* self) {
   return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
 }
 
+// Update the local copy of the preedit (composing) rect.
+//
+// As text is input by the user, the framework sends two streams of updates
+// over the text input channel: updates to the composing rect (cursor rect when
+// not in IME composing mode) and updates to the matrix transform from local
+// coordinates to Flutter root coordinates. This function is called after each
+// of these updates.
+static void update_preedit_rect(FlTextInputPlugin* self) {
+  // Skip update if not composing to avoid setting to position 0.
+  if (!self->text_model->composing()) {
+    return;
+  }
+
+  // Transform the x, y positions of the cursor from local coordinates to
+  // Flutter view coordinates.
+  gint x = self->composing_rect.x * self->editabletext_transform[0][0] +
+           self->composing_rect.y * self->editabletext_transform[1][0] +
+           self->editabletext_transform[3][0] + self->composing_rect.width;
+  gint y = self->composing_rect.x * self->editabletext_transform[0][1] +
+           self->composing_rect.y * self->editabletext_transform[1][1] +
+           self->editabletext_transform[3][1] + self->composing_rect.height;
+
+  // Transform from Flutter view coordinates to Gtk window coordinates and
+  // write them to preedit_rect.
+  gtk_widget_translate_coordinates(
+      GTK_WIDGET(self->view), gtk_widget_get_toplevel(GTK_WIDGET(self->view)),
+      x, y, &self->preedit_rect.x, &self->preedit_rect.y);
+}
+
+// Handles updates from the framework to the cached coordinate transform of the
+// `EditableText` relative to the root coordinate system.
+static FlMethodResponse* set_editable_size_and_transform(
+    FlTextInputPlugin* self,
+    FlValue* args) {
+  FlValue* transform = fl_value_lookup_string(args, kTransform);
+  size_t transform_len = fl_value_get_length(transform);
+  g_warn_if_fail(transform_len == 16);
+
+  for (size_t i = 0; i < transform_len; ++i) {
+    double val = fl_value_get_float(fl_value_get_list_value(transform, i));
+    self->editabletext_transform[i / 4][i % 4] = val;
+  }
+  update_preedit_rect(self);
+
+  return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
+}
+
+// Handles updates from the framework to the cached composing rect in local
+// coordinates.
+static FlMethodResponse* set_marked_text_rect(FlTextInputPlugin* self,
+                                              FlValue* args) {
+  self->composing_rect.x =
+      fl_value_get_float(fl_value_lookup_string(args, "x"));
+  self->composing_rect.y =
+      fl_value_get_float(fl_value_lookup_string(args, "y"));
+  self->composing_rect.width =
+      fl_value_get_float(fl_value_lookup_string(args, "width"));
+  self->composing_rect.height =
+      fl_value_get_float(fl_value_lookup_string(args, "height"));
+  update_preedit_rect(self);
+
+  return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
+}
+
 // Called when a method call is received from Flutter.
 static void method_call_cb(FlMethodChannel* channel,
                            FlMethodCall* method_call,
@@ -257,6 +408,10 @@ static void method_call_cb(FlMethodChannel* channel,
     response = clear_client(self);
   } else if (strcmp(method, kHideMethod) == 0) {
     response = hide(self);
+  } else if (strcmp(method, kSetEditableSizeAndTransform) == 0) {
+    response = set_editable_size_and_transform(self, args);
+  } else if (strcmp(method, kSetMarkedTextRect) == 0) {
+    response = set_marked_text_rect(self, args);
   } else {
     response = FL_METHOD_RESPONSE(fl_method_not_implemented_response_new());
   }
@@ -267,6 +422,11 @@ static void method_call_cb(FlMethodChannel* channel,
   }
 }
 
+static void view_weak_notify_cb(gpointer user_data, GObject* object) {
+  FlTextInputPlugin* self = FL_TEXT_INPUT_PLUGIN(object);
+  self->view = nullptr;
+}
+
 static void fl_text_input_plugin_dispose(GObject* object) {
   FlTextInputPlugin* self = FL_TEXT_INPUT_PLUGIN(object);
 
@@ -289,6 +449,15 @@ static void fl_text_input_plugin_init(FlTextInputPlugin* self) {
   self->client_id = kClientIdUnset;
   self->im_context = gtk_im_multicontext_new();
   self->input_multiline = FALSE;
+  g_signal_connect_object(self->im_context, "preedit-start",
+                          G_CALLBACK(im_preedit_start_cb), self,
+                          G_CONNECT_SWAPPED);
+  g_signal_connect_object(self->im_context, "preedit-end",
+                          G_CALLBACK(im_preedit_end_cb), self,
+                          G_CONNECT_SWAPPED);
+  g_signal_connect_object(self->im_context, "preedit-changed",
+                          G_CALLBACK(im_preedit_changed_cb), self,
+                          G_CONNECT_SWAPPED);
   g_signal_connect_object(self->im_context, "commit", G_CALLBACK(im_commit_cb),
                           self, G_CONNECT_SWAPPED);
   g_signal_connect_object(self->im_context, "retrieve-surrounding",
@@ -300,7 +469,8 @@ static void fl_text_input_plugin_init(FlTextInputPlugin* self) {
   self->text_model = new flutter::TextInputModel();
 }
 
-FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger) {
+FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger,
+                                            FlView* view) {
   g_return_val_if_fail(FL_IS_BINARY_MESSENGER(messenger), nullptr);
 
   FlTextInputPlugin* self = FL_TEXT_INPUT_PLUGIN(
@@ -311,7 +481,8 @@ FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger) {
       fl_method_channel_new(messenger, kChannelName, FL_METHOD_CODEC(codec));
   fl_method_channel_set_method_call_handler(self->channel, method_call_cb, self,
                                             nullptr);
-
+  self->view = view;
+  g_object_weak_ref(G_OBJECT(view), view_weak_notify_cb, self);
   return self;
 }
 

shell/platform/linux/fl_text_input_plugin.h

@@ -8,6 +8,7 @@
 #include <gdk/gdk.h>
 
 #include "flutter/shell/platform/linux/public/flutter_linux/fl_binary_messenger.h"
+#include "flutter/shell/platform/linux/public/flutter_linux/fl_view.h"
 
 G_BEGIN_DECLS
 
@@ -33,7 +34,8 @@ G_DECLARE_FINAL_TYPE(FlTextInputPlugin,
  *
  * Returns: a new #FlTextInputPlugin.
  */
-FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger);
+FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger,
+                                            FlView* view);
 
 /**
  * fl_text_input_plugin_filter_keypress

shell/platform/linux/fl_view.cc

@@ -161,7 +161,7 @@ static void fl_view_constructed(GObject* object) {
   self->key_event_plugin = fl_key_event_plugin_new(messenger);
   self->mouse_cursor_plugin = fl_mouse_cursor_plugin_new(messenger, self);
   self->platform_plugin = fl_platform_plugin_new(messenger);
-  self->text_input_plugin = fl_text_input_plugin_new(messenger);
+  self->text_input_plugin = fl_text_input_plugin_new(messenger, self);
 }
 
 static void fl_view_set_property(GObject* object,