|
13 | 13 |
|
14 | 14 | import jwt |
15 | 15 |
|
| 16 | +from promptflow.core._errors import MissingRequiredPackage |
16 | 17 | from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException |
17 | 18 | from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client |
18 | 19 | from azure.ai.evaluation._model_configurations import AzureAIProject |
19 | 20 | from azure.core.credentials import TokenCredential |
| 21 | +from azure.core.exceptions import HttpResponseError |
20 | 22 | from azure.core.pipeline.policies import AsyncRetryPolicy |
21 | 23 |
|
22 | 24 | from .constants import ( |
|
26 | 28 | Tasks, |
27 | 29 | _InternalEvaluationMetrics, |
28 | 30 | ) |
29 | | -from .utils import get_harm_severity_level |
| 31 | +from .utils import get_harm_severity_level, retrieve_content_type |
30 | 32 |
|
31 | 33 | try: |
32 | 34 | version = importlib.metadata.version("azure-ai-evaluation") |
@@ -281,7 +283,11 @@ def _parse_content_harm_response( |
281 | 283 | if key == EvaluationMetrics.HATE_FAIRNESS: |
282 | 284 | key = EvaluationMetrics.HATE_UNFAIRNESS |
283 | 285 |
|
284 | | - result: Dict[str, Union[str, float]] = {key: math.nan, key + "_score": math.nan, key + "_reason": ""} |
| 286 | + result: Dict[str, Union[str, float]] = { |
| 287 | + (key.value if hasattr(key, "value") else key): math.nan, |
| 288 | + f"{(key.value if hasattr(key, 'value') else key)}_score": math.nan, |
| 289 | + f"{(key.value if hasattr(key, 'value') else key)}_reason": math.nan, |
| 290 | + } |
285 | 291 |
|
286 | 292 | response = batch_response[0] |
287 | 293 | if metric_name not in response: |
@@ -330,6 +336,8 @@ def _parse_content_harm_response( |
330 | 336 | reason = "" |
331 | 337 |
|
332 | 338 | harm_score = metric_value |
| 339 | + if metric_value == "n/a": |
| 340 | + return result |
333 | 341 | if not math.isnan(metric_value): |
334 | 342 | # int(math.nan) causes a value error, and math.nan is already handled |
335 | 343 | # by get_harm_severity_level |
@@ -465,3 +473,109 @@ async def evaluate_with_rai_service( |
465 | 473 | result = parse_response(annotation_response, metric_name, metric_display_name) |
466 | 474 |
|
467 | 475 | return result |
| 476 | + |
| 477 | + |
| 478 | +def generate_payload_multimodal(content_type: str, messages, metric: str) -> Dict: |
| 479 | + """Generate the payload for the annotation request |
| 480 | + :param content_type: The type of the content representing multimodal or images. |
| 481 | + :type content_type: str |
| 482 | + :param messages: The normalized list of messages to be entered as the "Contents" in the payload. |
| 483 | + :type messages: str |
| 484 | + :param metric: The evaluation metric to use. This determines the task type, and whether a "MetricList" is needed |
| 485 | + in the payload. |
| 486 | + :type metric: str |
| 487 | + :return: The payload for the annotation request. |
| 488 | + :rtype: Dict |
| 489 | + """ |
| 490 | + include_metric = True |
| 491 | + task = Tasks.CONTENT_HARM |
| 492 | + if metric == EvaluationMetrics.PROTECTED_MATERIAL: |
| 493 | + task = Tasks.PROTECTED_MATERIAL |
| 494 | + include_metric = False |
| 495 | + |
| 496 | + if include_metric: |
| 497 | + return { |
| 498 | + "ContentType": content_type, |
| 499 | + "Contents": [{"messages": messages}], |
| 500 | + "AnnotationTask": task, |
| 501 | + "MetricList": [metric], |
| 502 | + } |
| 503 | + return { |
| 504 | + "ContentType": content_type, |
| 505 | + "Contents": [{"messages": messages}], |
| 506 | + "AnnotationTask": task, |
| 507 | + } |
| 508 | + |
| 509 | + |
| 510 | +async def submit_multimodal_request(messages, metric: str, rai_svc_url: str, token: str) -> str: |
| 511 | + """Submit request to Responsible AI service for evaluation and return operation ID |
| 512 | + :param messages: The normalized list of messages to be entered as the "Contents" in the payload. |
| 513 | + :type messages: str |
| 514 | + :param metric: The evaluation metric to use. |
| 515 | + :type metric: str |
| 516 | + :param rai_svc_url: The Responsible AI service URL. |
| 517 | + :type rai_svc_url: str |
| 518 | + :param token: The Azure authentication token. |
| 519 | + :type token: str |
| 520 | + :return: The operation ID. |
| 521 | + :rtype: str |
| 522 | + """ |
| 523 | + ## handle json payload and payload from inference sdk strongly type messages |
| 524 | + if len(messages) > 0 and not isinstance(messages[0], dict): |
| 525 | + try: |
| 526 | + from azure.ai.inference.models import ChatRequestMessage |
| 527 | + except ImportError as ex: |
| 528 | + error_message = ( |
| 529 | + "Please install 'azure-ai-inference' package to use SystemMessage, UserMessage, AssistantMessage" |
| 530 | + ) |
| 531 | + raise MissingRequiredPackage(message=error_message) from ex |
| 532 | + if len(messages) > 0 and isinstance(messages[0], ChatRequestMessage): |
| 533 | + messages = [message.as_dict() for message in messages] |
| 534 | + |
| 535 | + filtered_messages = [message for message in messages if message["role"] != "system"] |
| 536 | + assistant_messages = [message for message in messages if message["role"] == "assistant"] |
| 537 | + content_type = retrieve_content_type(assistant_messages, metric) |
| 538 | + payload = generate_payload_multimodal(content_type, filtered_messages, metric) |
| 539 | + |
| 540 | + ## calling rai service for annotation |
| 541 | + url = rai_svc_url + "/submitannotation" |
| 542 | + headers = get_common_headers(token) |
| 543 | + async with get_async_http_client() as client: |
| 544 | + response = await client.post( # pylint: disable=too-many-function-args,unexpected-keyword-arg |
| 545 | + url, json=payload, headers=headers |
| 546 | + ) |
| 547 | + if response.status_code != 202: |
| 548 | + raise HttpResponseError( |
| 549 | + message=f"Received unexpected HTTP status: {response.status_code} {response.text()}", response=response |
| 550 | + ) |
| 551 | + result = response.json() |
| 552 | + operation_id = result["location"].split("/")[-1] |
| 553 | + return operation_id |
| 554 | + |
| 555 | + |
| 556 | +async def evaluate_with_rai_service_multimodal( |
| 557 | + messages, metric_name: str, project_scope: AzureAIProject, credential: TokenCredential |
| 558 | +): |
| 559 | + """ "Evaluate the content safety of the response using Responsible AI service |
| 560 | + :param messages: The normalized list of messages. |
| 561 | + :type messages: str |
| 562 | + :param metric_name: The evaluation metric to use. |
| 563 | + :type metric_name: str |
| 564 | + :param project_scope: The Azure AI project scope details. |
| 565 | + :type project_scope: Dict |
| 566 | + :param credential: The Azure authentication credential. |
| 567 | + :type credential: |
| 568 | + ~azure.core.credentials.TokenCredential |
| 569 | + :return: The parsed annotation result. |
| 570 | + :rtype: List[List[Dict]] |
| 571 | + """ |
| 572 | + |
| 573 | + # Get RAI service URL from discovery service and check service availability |
| 574 | + token = await fetch_or_reuse_token(credential) |
| 575 | + rai_svc_url = await get_rai_svc_url(project_scope, token) |
| 576 | + await ensure_service_availability(rai_svc_url, token, Tasks.CONTENT_HARM) |
| 577 | + # Submit annotation request and fetch result |
| 578 | + operation_id = await submit_multimodal_request(messages, metric_name, rai_svc_url, token) |
| 579 | + annotation_response = cast(List[Dict], await fetch_result(operation_id, rai_svc_url, credential, token)) |
| 580 | + result = parse_response(annotation_response, metric_name) |
| 581 | + return result |
0 commit comments