Merge pull request #699

Fix/colang 2 runtime issues

Author: drazvan

CHANGELOG-Colang.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 
 * [#672](https://github.com/NVIDIA/NeMo-Guardrails/pull/672) Fixes a event group match bug (e.g. `match $flow_ref.Finished() or $flow_ref.Failed()`)
+* [#699](https://github.com/NVIDIA/NeMo-Guardrails/pull/699) Fix issues with ActionUpdated events and user utterance action extraction.
 
 ## [2.0-beta.2] - 2024-07-25
 

nemoguardrails/actions/llm/generation.py

@@ -145,7 +145,10 @@ def _extract_user_message_example(self, flow: Flow):
         if isinstance(el, SpecOp):
             if el.op == "match":
                 spec = cast(SpecOp, el).spec
-                if spec.name != "UtteranceUserActionFinished":
+                if (
+                    not hasattr(spec, "name")
+                    or spec.name != "UtteranceUserActionFinished"
+                ):
                     return
 
                 if "final_transcript" not in spec.arguments:

nemoguardrails/colang/v2_x/library/avatars.co

@@ -45,13 +45,17 @@ flow user gestured $gesture -> $final_gesture
   $final_gesture = $event.gesture
 
 @meta(user_action=True)
-flow user became present -> $user_id
-  """Wait for user to be detected as present (e.g. camera ROI)."""
+flow user became absent
+  """Wait for user to be detected as absent (e.g. out of camera ROI)."""
   match PresenceUserAction.Finished() as $event
-  $user_id = $event.user_id
+
+@meta(user_action=True)
+flow user became present
+  """Wait for user to be detected as present (e.g. inside camera ROI)."""
+  match PresenceUserAction.Started() as $event
 
 @meta(user_intent=True)
-flow user interrupted bot talking $sentence_length=15
+flow user interrupted bot talking $sentence_length=15 $pattern="stop"
   """Wait for when the user talked while bot is speaking."""
   activate tracking bot talking state
   global $bot_talking_state
@@ -66,7 +70,7 @@ flow user interrupted bot talking $sentence_length=15
       or when bot said something
         break
 
-      if len($transcript) > $sentence_length
+      if len($transcript) > $sentence_length or search($pattern,$transcript)
         log 'bot interrupted by user with: "{$transcript}"'
         return
 
@@ -165,6 +169,12 @@ flow finish all bot actions
   send FinishFlow(flow_id="bot gesture")
   send FinishFlow(flow_id="bot posture")
 
+flow fail all bot actions
+  """Stops all the current bot actions."""
+  send StopFlow(flow_id="_bot_say")
+  send StopFlow(flow_id="bot gesture")
+  send StopFlow(flow_id="bot posture")
+
 flow finish all scene actions
   """Stops all the current scene actions."""
   send FinishFlow(flow_id="scene show choice")
@@ -173,11 +183,13 @@ flow finish all scene actions
   send FinishFlow(flow_id="scene show form")
 
 @loop("bot_interruption")
-flow handling bot talking interruption $mode="inform"
+flow handling bot talking interruption $mode="inform" $stop_pattern="stop|cancel"
   """Handling the bot talking interruption reaction."""
-  user interrupted bot talking
+  user interrupted bot talking $sentence_length=15 $pattern=$stop_pattern
   if $mode == "interrupt"
     finish all bot actions
+  if $mode == "abort"
+    fail all bot actions
   elif $mode == "inform"
     start VisualInformationSceneAction(title="Please wait with talking!", support_prompts=["You should only talk after the avatar."], content=[])
     wait 3.0

nemoguardrails/colang/v2_x/library/core.co

@@ -163,14 +163,19 @@ flow bot refuse to respond
 flow tracking bot talking state
   """tracking bot talking state in global variable $bot_talking_state."""
   global $bot_talking_state
+
+  # TODO: keeping both for backwards compatibility reason. To clean up.
+  global $last_bot_script
   global $last_bot_message
+
   if $bot_talking_state == None
     $bot_talking_state = False
   await bot started saying something
   $bot_talking_state = True
   await bot said something as $bot_said_flow
   $bot_talking_state = False
   $last_bot_script = $bot_said_flow.text
+  $last_bot_message = $bot_said_flow.text
 
 @loop("state_tracking")
 flow tracking user talking state

nemoguardrails/colang/v2_x/library/guardrails.co

@@ -22,12 +22,15 @@ flow _user_said $text -> $event
 
 
 @override
-flow _user_saying -> $event
+flow _user_saying $text -> $event
   """Override core flow for when the user is saying something."""
   global $user_message
   if $text
-    # This matches to a transcript where after some initial characters it finds $text followed optionally by up to two words
-    match UtteranceUserAction.TranscriptUpdated(interim_transcript=regex("(?i).*({$text})((\s*\w+\s*){0,2})\W*$")) as $event
+    if is_regex($text)
+      match UtteranceUserAction.TranscriptUpdated(interim_transcript=$text) as $event
+    else
+      # This matches to a transcript where after some initial characters it finds $text followed optionally by up to two words
+      match UtteranceUserAction.TranscriptUpdated(interim_transcript=regex("(?i).*({$text})((\s*\w+\s*){{0,2}})\W*$")) as $event
   else
     match UtteranceUserAction.TranscriptUpdated() as $event
 

nemoguardrails/colang/v2_x/runtime/flows.py

@@ -264,11 +264,14 @@ def process_event(self, event: ActionEvent) -> None:
     def get_event(self, name: str, arguments: dict) -> ActionEvent:
         """Returns the corresponding action event."""
         if name.endswith("Updated"):
-            split_name = name.rsplit("Updated", 1)
-            if split_name[0] == "":
-                raise ColangSyntaxError(f"Invalid action event {name}!")
-            arguments.update({"event_parameter_name": split_name[0]})
-            name = "Updated"
+            if len(name) > 7:
+                split_name = name.rsplit("Updated", 1)
+                if split_name[0] == "":
+                    raise ColangSyntaxError(f"Invalid action event {name}!")
+                arguments.update({"event_parameter_name": split_name[0]})
+                name = "Updated"
+            else:
+                arguments.update({"event_parameter_name": ""})
         if name not in Action._event_name_map:
             raise ColangSyntaxError(f"Invalid action event {name}!")
         func = getattr(self, Action._event_name_map[name])

nemoguardrails/utils.py

@@ -128,12 +128,15 @@ def _has_property(e: Dict[str, Any], p: Property) -> bool:
     "UtteranceBotAction": ("bot_speech", "replace"),
     "UtteranceUserAction": ("user_speech", "replace"),
     "TimerBotAction": ("time", "parallel"),
+    "FacialGestureBotAction": ("bot_gesture", "override"),
     "GestureBotAction": ("bot_gesture", "override"),
     "FacialGestureBotAction": ("bot_face", "replace"),
     "PostureBotAction": ("bot_posture", "override"),
     "VisualChoiceSceneAction": ("information", "override"),
     "VisualInformationSceneAction": ("information", "override"),
     "VisualFormSceneAction": ("information", "override"),
+    "MotionEffectCameraAction": ("camera_motion_effect", "override"),
+    "ShotCameraAction": ("camera_shot", "override"),
 }
 
 
@@ -142,8 +145,9 @@ def _add_modality_info(event_dict: Dict[str, Any]) -> None:
     for action_name, modality_info in _action_to_modality_info.items():
         modality_name, modality_policy = modality_info
         if action_name in event_dict["type"]:
-            event_dict["action_info_modality"] = modality_name
-            event_dict["action_info_modality_policy"] = modality_policy
+            event_dict.setdefault("action_info_modality", modality_name)
+            event_dict.setdefault("action_info_modality_policy", modality_policy)
+            return
 
 
 def _update_action_properties(event_dict: Dict[str, Any]) -> None: