カスタムコードベースの評価者

agentcore eval evaluator create \
  --name "MyCodeEvaluator" \
  --level TRACE \
  --lambda-arn "arn:aws:lambda:us-east-1:123456789012:function:my-eval-function" \
  --lambda-timeout 120

from bedrock_agentcore.evaluation.code_based_evaluators import (
    EvaluatorInput,
    EvaluatorOutput,
    code_based_evaluator,
)
import json as _json

@code_based_evaluator()
def json_response_evaluator(input: EvaluatorInput) -> EvaluatorOutput:
    """Check if the agent response in the target trace contains valid JSON."""
    for span in input.session_spans:
        if span.get("traceId") != input.target_trace_id:
            continue
        if span.get("name", "").startswith("Model:") or span.get("name") == "Agent.invoke":
            output = span.get("attributes", {}).get("gen_ai.completion", "")
            try:
                _json.loads(output)
                return EvaluatorOutput(
                    value=1.0,
                    label="Pass",
                    explanation="Response contains valid JSON"
                )
            except (ValueError, TypeError):
                pass

    return EvaluatorOutput(
        value=0.0,
        label="Fail",
        explanation="No valid JSON found in agent response"
    )

import boto3

client = boto3.client('bedrock-agentcore-control')

response = client.create_evaluator(
    evaluatorName="MyCodeEvaluator",
    level="TRACE",
    evaluatorConfig={
        "codeBased": {
            "lambdaConfig": {
                "lambdaArn": "arn:aws:lambda:us-east-1:123456789012:function:my-eval-function",
                "lambdaTimeoutInSeconds": 120
            }
        }
    }
)

print(f"Evaluator ID: {response['evaluatorId']}")
print(f"Evaluator ARN: {response['evaluatorArn']}")

aws bedrock-agentcore-control create-evaluator \
    --evaluator-name 'MyCodeEvaluator' \
    --level TRACE \
    --evaluator-config '{
        "codeBased": {
            "lambdaConfig": {
                "lambdaArn": "arn:aws:lambda:us-east-1:123456789012:function:my-eval-function",
                "lambdaTimeoutInSeconds": 120
            }
        }
    }'

agentcore run eval \
  --runtime "your_runtime_name" \
  --session-id "your_session_id" \
  --evaluator "code-based-evaluator-id"

from bedrock_agentcore.evaluation.client import EvaluationClient

client = EvaluationClient(
    region_name="region"
)

results = client.run(
    evaluator_ids=[
        "code-based-evaluator-id",
    ],
    session_id="session-id",
    log_group_name="log-group-name",
)

import boto3

client = boto3.client('bedrock-agentcore')

response = client.evaluate(
    evaluatorId="code-based-evaluator-id",
    evaluationInput={"sessionSpans": session_span_logs}
)

for result in response["evaluationResults"]:
    if "errorCode" in result:
        print(f"Error: {result['errorCode']} - {result['errorMessage']}")
    else:
        print(f"Label: {result['label']}, Value: {result.get('value')}")
        print(f"Explanation: {result.get('explanation', '')}")

aws bedrock-agentcore evaluate \
    --cli-input-json file://session_span_logs.json

agentcore add online-eval \
  --name "your_config_name" \
  --runtime "your_runtime_name" \
  --evaluator "code-based-evaluator-id" \
  --sampling-rate 1.0 \
  --enable-on-create

このコマンドは、オンライン評価設定をローカル agentcore.json に追加します。agentcore deploy を実行して、 AWS アカウントに作成します。

from bedrock_agentcore_starter_toolkit import Evaluation

eval_client = Evaluation()

config = eval_client.create_online_config(
    config_name="my_online_eval_config",
    agent_id="agent-id",
    sampling_rate=1.0,
    evaluator_list=["code-based-evaluator-id"],
    enable_on_create=True
)

print(f"Config ID: {config['onlineEvaluationConfigId']}")

import boto3

client = boto3.client('bedrock-agentcore-control')

response = client.create_online_evaluation_config(
    onlineEvaluationConfigName="my_online_eval_config",
    rule={"samplingConfig": {"samplingPercentage": 100.0}},
    dataSourceConfig={
        "cloudWatchLogs": {
            "logGroupNames": ["/aws/agentcore/my-agent-traces"],
            "serviceNames": ["my-agent.DEFAULT"]
        }
    },
    evaluators=[{"evaluatorId": "code-based-evaluator-id"}],
    evaluationExecutionRoleArn="arn:aws:iam::account-id:role/AgentCoreEvaluationRole",
    enableOnCreate=True
)

print(f"Config ID: {response['onlineEvaluationConfigId']}")

aws bedrock-agentcore-control create-online-evaluation-config \
    --online-evaluation-config-name "my_online_eval_config" \
    --rule '{"samplingConfig": {"samplingPercentage": 100.0}}' \
    --data-source-config '{"cloudWatchLogs": {"logGroupNames": ["/aws/agentcore/my-agent-traces"], "serviceNames": ["my-agent.DEFAULT"]}}' \
    --evaluators '[{"evaluatorId": "code-based-evaluator-id"}]' \
    --evaluation-execution-role-arn "arn:aws:iam::account-id:role/AgentCoreEvaluationRole" \
    --enable-on-create