|
2259 | 2259 | "max":20,
|
2260 | 2260 | "min":0
|
2261 | 2261 | },
|
| 2262 | + "EvaluationBedrockKnowledgeBaseIdentifiers":{ |
| 2263 | + "type":"list", |
| 2264 | + "member":{"shape":"KnowledgeBaseId"}, |
| 2265 | + "max":1, |
| 2266 | + "min":0 |
| 2267 | + }, |
2262 | 2268 | "EvaluationBedrockModel":{
|
2263 | 2269 | "type":"structure",
|
2264 | 2270 | "required":["modelIdentifier"],
|
2265 | 2271 | "members":{
|
2266 | 2272 | "modelIdentifier":{
|
2267 |
| - "shape":"EvaluationModelIdentifier", |
| 2273 | + "shape":"EvaluationBedrockModelIdentifier", |
2268 | 2274 | "documentation":"<p>The ARN of the Amazon Bedrock model or inference profile specified.</p>"
|
2269 | 2275 | },
|
2270 | 2276 | "inferenceParams":{
|
|
2278 | 2284 | },
|
2279 | 2285 | "documentation":"<p>Contains the ARN of the Amazon Bedrock model or <a href=\"https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html\">inference profile</a> specified in your evaluation job. Each Amazon Bedrock model supports different <code>inferenceParams</code>. To learn more about supported inference parameters for Amazon Bedrock models, see <a href=\"https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html\">Inference parameters for foundation models</a>.</p> <p>The <code>inferenceParams</code> are specified using JSON. To successfully insert JSON as string make sure that all quotations are properly escaped. For example, <code>\"temperature\":\"0.25\"</code> key value pair would need to be formatted as <code>\\\"temperature\\\":\\\"0.25\\\"</code> to successfully accepted in the request.</p>"
|
2280 | 2286 | },
|
| 2287 | + "EvaluationBedrockModelIdentifier":{ |
| 2288 | + "type":"string", |
| 2289 | + "max":2048, |
| 2290 | + "min":1, |
| 2291 | + "pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:((:foundation-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|([0-9]{12}:provisioned-model/[a-z0-9]{12})|([0-9]{12}:imported-model/[a-z0-9]{12})|([0-9]{12}:application-inference-profile/[a-z0-9]{12})|([0-9]{12}:inference-profile/(([a-z-]{2,8}.)[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([0-9]{12}:(default-prompt-router|prompt-router)/[a-zA-Z0-9-:.]+)))|(([a-z]{2}[.]{1})([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|arn:aws(-[^:]+)?:sagemaker:[a-z0-9-]{1,20}:[0-9]{12}:endpoint/[a-z0-9-]{1,63}" |
| 2292 | + }, |
| 2293 | + "EvaluationBedrockModelIdentifiers":{ |
| 2294 | + "type":"list", |
| 2295 | + "member":{"shape":"EvaluationBedrockModelIdentifier"}, |
| 2296 | + "max":2, |
| 2297 | + "min":0 |
| 2298 | + }, |
2281 | 2299 | "EvaluationConfig":{
|
2282 | 2300 | "type":"structure",
|
2283 | 2301 | "members":{
|
|
2370 | 2388 | "documentation":"<p>The configuration details of the inference model for an evaluation job.</p> <p>For automated model evaluation jobs, only a single model is supported.</p> <p>For human-based model evaluation jobs, your annotator can compare the responses for up to two different models.</p>",
|
2371 | 2389 | "union":true
|
2372 | 2390 | },
|
| 2391 | + "EvaluationInferenceConfigSummary":{ |
| 2392 | + "type":"structure", |
| 2393 | + "members":{ |
| 2394 | + "modelConfigSummary":{ |
| 2395 | + "shape":"EvaluationModelConfigSummary", |
| 2396 | + "documentation":"<p>A summary of the models used in an Amazon Bedrock model evaluation job. These resources can be models in Amazon Bedrock or models outside of Amazon Bedrock that you use to generate your own inference response data.</p>" |
| 2397 | + }, |
| 2398 | + "ragConfigSummary":{ |
| 2399 | + "shape":"EvaluationRagConfigSummary", |
| 2400 | + "documentation":"<p>A summary of the RAG resources used in an Amazon Bedrock Knowledge Base evaluation job. These resources can be Knowledge Bases in Amazon Bedrock or RAG sources outside of Amazon Bedrock that you use to generate your own inference response data.</p>" |
| 2401 | + } |
| 2402 | + }, |
| 2403 | + "documentation":"<p>Identifies the models, Knowledge Bases, or other RAG sources evaluated in a model or Knowledge Base evaluation job.</p>" |
| 2404 | + }, |
2373 | 2405 | "EvaluationJobArn":{
|
2374 | 2406 | "type":"string",
|
2375 | 2407 | "max":1011,
|
|
2446 | 2478 | "bedrockModel":{
|
2447 | 2479 | "shape":"EvaluationBedrockModel",
|
2448 | 2480 | "documentation":"<p>Defines the Amazon Bedrock model or inference profile and inference parameters you want used.</p>"
|
| 2481 | + }, |
| 2482 | + "precomputedInferenceSource":{ |
| 2483 | + "shape":"EvaluationPrecomputedInferenceSource", |
| 2484 | + "documentation":"<p>Defines the model used to generate inference response data for a model evaluation job where you provide your own inference response data.</p>" |
2449 | 2485 | }
|
2450 | 2486 | },
|
2451 | 2487 | "documentation":"<p>Defines the models used in the model evaluation job.</p>",
|
2452 | 2488 | "union":true
|
2453 | 2489 | },
|
| 2490 | + "EvaluationModelConfigSummary":{ |
| 2491 | + "type":"structure", |
| 2492 | + "members":{ |
| 2493 | + "bedrockModelIdentifiers":{ |
| 2494 | + "shape":"EvaluationBedrockModelIdentifiers", |
| 2495 | + "documentation":"<p>The Amazon Resource Names (ARNs) of the models used for the evaluation job.</p>" |
| 2496 | + }, |
| 2497 | + "precomputedInferenceSourceIdentifiers":{ |
| 2498 | + "shape":"EvaluationPrecomputedInferenceSourceIdentifiers", |
| 2499 | + "documentation":"<p>A label that identifies the models used for a model evaluation job where you provide your own inference response data.</p>" |
| 2500 | + } |
| 2501 | + }, |
| 2502 | + "documentation":"<p>A summary of the models used in an Amazon Bedrock model evaluation job. These resources can be models in Amazon Bedrock or models outside of Amazon Bedrock that you use to generate your own inference response data.</p>" |
| 2503 | + }, |
2454 | 2504 | "EvaluationModelConfigs":{
|
2455 | 2505 | "type":"list",
|
2456 | 2506 | "member":{"shape":"EvaluationModelConfig"},
|
2457 | 2507 | "max":2,
|
2458 | 2508 | "min":1
|
2459 | 2509 | },
|
2460 |
| - "EvaluationModelIdentifier":{ |
2461 |
| - "type":"string", |
2462 |
| - "max":2048, |
2463 |
| - "min":1, |
2464 |
| - "pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:((:foundation-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|([0-9]{12}:provisioned-model/[a-z0-9]{12})|([0-9]{12}:imported-model/[a-z0-9]{12})|([0-9]{12}:application-inference-profile/[a-z0-9]{12})|([0-9]{12}:inference-profile/(([a-z-]{2,8}.)[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([0-9]{12}:(default-prompt-router|prompt-router)/[a-zA-Z0-9-:.]+)))|(([a-z]{2}[.]{1})([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|arn:aws(-[^:]+)?:sagemaker:[a-z0-9-]{1,20}:[0-9]{12}:endpoint/[a-z0-9-]{1,63}" |
2465 |
| - }, |
2466 |
| - "EvaluationModelIdentifiers":{ |
2467 |
| - "type":"list", |
2468 |
| - "member":{"shape":"EvaluationModelIdentifier"}, |
2469 |
| - "max":2, |
2470 |
| - "min":0 |
2471 |
| - }, |
2472 | 2510 | "EvaluationModelInferenceParams":{
|
2473 | 2511 | "type":"string",
|
2474 | 2512 | "max":1023,
|
|
2486 | 2524 | },
|
2487 | 2525 | "documentation":"<p>The Amazon S3 location where the results of your evaluation job are saved.</p>"
|
2488 | 2526 | },
|
| 2527 | + "EvaluationPrecomputedInferenceSource":{ |
| 2528 | + "type":"structure", |
| 2529 | + "required":["inferenceSourceIdentifier"], |
| 2530 | + "members":{ |
| 2531 | + "inferenceSourceIdentifier":{ |
| 2532 | + "shape":"EvaluationPrecomputedInferenceSourceIdentifier", |
| 2533 | + "documentation":"<p>A label that identifies a model used in a model evaluation job where you provide your own inference response data.</p>" |
| 2534 | + } |
| 2535 | + }, |
| 2536 | + "documentation":"<p>A summary of a model used for a model evaluation job where you provide your own inference response data.</p>" |
| 2537 | + }, |
| 2538 | + "EvaluationPrecomputedInferenceSourceIdentifier":{ |
| 2539 | + "type":"string", |
| 2540 | + "max":256, |
| 2541 | + "min":1, |
| 2542 | + "pattern":"[a-zA-Z0-9]([a-zA-Z0-9._-]){0,255}" |
| 2543 | + }, |
| 2544 | + "EvaluationPrecomputedInferenceSourceIdentifiers":{ |
| 2545 | + "type":"list", |
| 2546 | + "member":{"shape":"EvaluationPrecomputedInferenceSourceIdentifier"}, |
| 2547 | + "max":2, |
| 2548 | + "min":0 |
| 2549 | + }, |
| 2550 | + "EvaluationPrecomputedRagSourceConfig":{ |
| 2551 | + "type":"structure", |
| 2552 | + "members":{ |
| 2553 | + "retrieveSourceConfig":{ |
| 2554 | + "shape":"EvaluationPrecomputedRetrieveSourceConfig", |
| 2555 | + "documentation":"<p>A summary of a RAG source used for a retrieve-only Knowledge Base evaluation job where you provide your own inference response data.</p>" |
| 2556 | + }, |
| 2557 | + "retrieveAndGenerateSourceConfig":{ |
| 2558 | + "shape":"EvaluationPrecomputedRetrieveAndGenerateSourceConfig", |
| 2559 | + "documentation":"<p>A summary of a RAG source used for a retrieve-and-generate Knowledge Base evaluation job where you provide your own inference response data.</p>" |
| 2560 | + } |
| 2561 | + }, |
| 2562 | + "documentation":"<p>A summary of a RAG source used for a Knowledge Base evaluation job where you provide your own inference response data.</p>", |
| 2563 | + "union":true |
| 2564 | + }, |
| 2565 | + "EvaluationPrecomputedRagSourceIdentifier":{ |
| 2566 | + "type":"string", |
| 2567 | + "max":256, |
| 2568 | + "min":1, |
| 2569 | + "pattern":"[a-zA-Z0-9]([a-zA-Z0-9._-]){0,255}" |
| 2570 | + }, |
| 2571 | + "EvaluationPrecomputedRagSourceIdentifiers":{ |
| 2572 | + "type":"list", |
| 2573 | + "member":{"shape":"EvaluationPrecomputedRagSourceIdentifier"}, |
| 2574 | + "max":1, |
| 2575 | + "min":0 |
| 2576 | + }, |
| 2577 | + "EvaluationPrecomputedRetrieveAndGenerateSourceConfig":{ |
| 2578 | + "type":"structure", |
| 2579 | + "required":["ragSourceIdentifier"], |
| 2580 | + "members":{ |
| 2581 | + "ragSourceIdentifier":{ |
| 2582 | + "shape":"EvaluationPrecomputedRagSourceIdentifier", |
| 2583 | + "documentation":"<p>A label that identifies the RAG source used for a retrieve-and-generate Knowledge Base evaluation job where you provide your own inference response data.</p>" |
| 2584 | + } |
| 2585 | + }, |
| 2586 | + "documentation":"<p>A summary of a RAG source used for a retrieve-and-generate Knowledge Base evaluation job where you provide your own inference response data.</p>" |
| 2587 | + }, |
| 2588 | + "EvaluationPrecomputedRetrieveSourceConfig":{ |
| 2589 | + "type":"structure", |
| 2590 | + "required":["ragSourceIdentifier"], |
| 2591 | + "members":{ |
| 2592 | + "ragSourceIdentifier":{ |
| 2593 | + "shape":"EvaluationPrecomputedRagSourceIdentifier", |
| 2594 | + "documentation":"<p>A label that identifies the RAG source used for a retrieve-only Knowledge Base evaluation job where you provide your own inference response data.</p>" |
| 2595 | + } |
| 2596 | + }, |
| 2597 | + "documentation":"<p>A summary of a RAG source used for a retrieve-only Knowledge Base evaluation job where you provide your own inference response data.</p>" |
| 2598 | + }, |
| 2599 | + "EvaluationRagConfigSummary":{ |
| 2600 | + "type":"structure", |
| 2601 | + "members":{ |
| 2602 | + "bedrockKnowledgeBaseIdentifiers":{ |
| 2603 | + "shape":"EvaluationBedrockKnowledgeBaseIdentifiers", |
| 2604 | + "documentation":"<p>The Amazon Resource Names (ARNs) of the Knowledge Base resources used for a Knowledge Base evaluation job where Amazon Bedrock invokes the Knowledge Base for you.</p>" |
| 2605 | + }, |
| 2606 | + "precomputedRagSourceIdentifiers":{ |
| 2607 | + "shape":"EvaluationPrecomputedRagSourceIdentifiers", |
| 2608 | + "documentation":"<p>A label that identifies the RAG sources used for a Knowledge Base evaluation job where you provide your own inference response data.</p>" |
| 2609 | + } |
| 2610 | + }, |
| 2611 | + "documentation":"<p>A summary of the RAG resources used in an Amazon Bedrock Knowledge Base evaluation job. These resources can be Knowledge Bases in Amazon Bedrock or RAG sources outside of Amazon Bedrock that you use to generate your own inference response data.</p>" |
| 2612 | + }, |
2489 | 2613 | "EvaluationRatingMethod":{
|
2490 | 2614 | "type":"string",
|
2491 | 2615 | "max":100,
|
|
2534 | 2658 | "documentation":"<p>The type of task for model evaluation.</p>"
|
2535 | 2659 | },
|
2536 | 2660 | "modelIdentifiers":{
|
2537 |
| - "shape":"EvaluationModelIdentifiers", |
2538 |
| - "documentation":"<p>The Amazon Resource Names (ARNs) of the model(s) used for the evaluation job.</p>" |
| 2661 | + "shape":"EvaluationBedrockModelIdentifiers", |
| 2662 | + "documentation":"<p>The Amazon Resource Names (ARNs) of the model(s) used for the evaluation job.</p>", |
| 2663 | + "deprecated":true, |
| 2664 | + "deprecatedMessage":"Inference identifiers should be retrieved from the inferenceConfigSummary" |
2539 | 2665 | },
|
2540 | 2666 | "ragIdentifiers":{
|
2541 |
| - "shape":"RAGIdentifiers", |
2542 |
| - "documentation":"<p>The Amazon Resource Names (ARNs) of the knowledge base resources used for a knowledge base evaluation job.</p>" |
| 2667 | + "shape":"EvaluationBedrockKnowledgeBaseIdentifiers", |
| 2668 | + "documentation":"<p>The Amazon Resource Names (ARNs) of the knowledge base resources used for a knowledge base evaluation job.</p>", |
| 2669 | + "deprecated":true, |
| 2670 | + "deprecatedMessage":"Inference identifiers should be retrieved from the inferenceConfigSummary" |
2543 | 2671 | },
|
2544 | 2672 | "evaluatorModelIdentifiers":{
|
2545 | 2673 | "shape":"EvaluatorModelIdentifiers",
|
2546 | 2674 | "documentation":"<p>The Amazon Resource Names (ARNs) of the models used to compute the metrics for a knowledge base evaluation job.</p>"
|
2547 | 2675 | },
|
| 2676 | + "inferenceConfigSummary":{ |
| 2677 | + "shape":"EvaluationInferenceConfigSummary", |
| 2678 | + "documentation":"<p>Identifies the models, Knowledge Bases, or other RAG sources evaluated in a model or Knowledge Base evaluation job.</p>" |
| 2679 | + }, |
2548 | 2680 | "applicationType":{
|
2549 | 2681 | "shape":"ApplicationType",
|
2550 | 2682 | "documentation":"<p>Specifies whether the evaluation job is for evaluating a model or evaluating a knowledge base (retrieval and response generation).</p>"
|
|
2586 | 2718 | "type":"string",
|
2587 | 2719 | "max":2048,
|
2588 | 2720 | "min":1,
|
2589 |
| - "pattern":"arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}::foundation-model/[a-z0-9-]{1,63}[.]{1}([a-z0-9-]{1,63}[.]){0,2}[a-z0-9-]{1,63}([:][a-z0-9-]{1,63}){0,2}$|(^[a-z0-9-]+[.][a-z0-9-]+([.][a-z0-9-]+)*(:[a-z0-9-]+)?$)|^[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([a-z0-9-]{1,63}[.]){0,2}[a-z0-9-]{1,63}([:][a-z0-9-]{1,63}){0,2}(/[a-z0-9]{12}|)" |
| 2721 | + "pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:((:foundation-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|([0-9]{12}:inference-profile/(([a-z-]{2,8}.)[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))))$|(^[a-z0-9-]+[.][a-z0-9-]+([.][a-z0-9-]+)*(:[a-z0-9-]+)?$)|^[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([a-z0-9-]{1,63}[.]){0,2}[a-z0-9-]{1,63}([:][a-z0-9-]{1,63}){0,2}(/[a-z0-9]{12}|)" |
2590 | 2722 | },
|
2591 | 2723 | "EvaluatorModelIdentifiers":{
|
2592 | 2724 | "type":"list",
|
|
6159 | 6291 | "type":"string",
|
6160 | 6292 | "max":2048,
|
6161 | 6293 | "min":1,
|
6162 |
| - "pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:(([0-9]{12}:custom-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-:]{1,63}/[a-z0-9]{12}$)|(:foundation-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}$)))|([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.]?[a-z0-9-]{1,63})([:][a-z0-9-]{1,63}){0,2})|(([0-9a-zA-Z][_-]?)+)" |
| 6294 | + "pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:(([0-9]{12}:custom-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-:]{1,63}/[a-z0-9]{12}$)|(:foundation-model/([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.]?[a-z0-9-]{1,63})([:][a-z0-9-]{1,63}){0,2})|(([0-9a-zA-Z][_-]?)+)$)|([0-9]{12}:(inference-profile|application-inference-profile)/[a-zA-Z0-9-:.]+$)))|([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.]?[a-z0-9-]{1,63})([:][a-z0-9-]{1,63}){0,2})|(([0-9a-zA-Z][_-]?)+)" |
6163 | 6295 | },
|
6164 | 6296 | "ModelIdentifier":{
|
6165 | 6297 | "type":"string",
|
|
6750 | 6882 | "knowledgeBaseConfig":{
|
6751 | 6883 | "shape":"KnowledgeBaseConfig",
|
6752 | 6884 | "documentation":"<p>Contains configuration details for knowledge base retrieval and response generation.</p>"
|
| 6885 | + }, |
| 6886 | + "precomputedRagSourceConfig":{ |
| 6887 | + "shape":"EvaluationPrecomputedRagSourceConfig", |
| 6888 | + "documentation":"<p>Contains configuration details about the RAG source used to generate inference response data for a Knowledge Base evaluation job.</p>" |
6753 | 6889 | }
|
6754 | 6890 | },
|
6755 | 6891 | "documentation":"<p>Contains configuration details for retrieval of information and response generation.</p>",
|
6756 | 6892 | "union":true
|
6757 | 6893 | },
|
6758 |
| - "RAGIdentifiers":{ |
6759 |
| - "type":"list", |
6760 |
| - "member":{"shape":"KnowledgeBaseId"}, |
6761 |
| - "max":1, |
6762 |
| - "min":0 |
6763 |
| - }, |
6764 | 6894 | "RAGStopSequences":{
|
6765 | 6895 | "type":"list",
|
6766 | 6896 | "member":{"shape":"RAGStopSequencesMemberString"},
|
|
0 commit comments