diff --git a/src/prompt_toolkit/key_binding/vi_state.py b/src/prompt_toolkit/key_binding/vi_state.py
index 10593a82e..6afc17a6a 100644
--- a/src/prompt_toolkit/key_binding/vi_state.py
+++ b/src/prompt_toolkit/key_binding/vi_state.py
@@ -55,6 +55,9 @@ def __init__(self) -> None:
         #: The Vi mode we're currently in to.
         self.__input_mode = InputMode.INSERT
 
+        #: Prevent input modes not listed here
+        self.allowed_input_modes = list(InputMode)
+
         #: Waiting for digraph.
         self.waiting_for_digraph = False
         self.digraph_symbol1: Optional[str] = None  # (None or a symbol.)
@@ -89,7 +92,8 @@ def input_mode(self, value: InputMode) -> None:
             self.operator_func = None
             self.operator_arg = None
 
-        self.__input_mode = value
+        if value in self.allowed_input_modes:
+            self.__input_mode = value
 
     def reset(self) -> None:
         """