Index

A

agent_input (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) agent_inspect module agent_inspect.clients module agent_inspect.clients.azure_openai_client module agent_inspect.clients.litellm_client module agent_inspect.clients.llm_client module agent_inspect.exception module agent_inspect.exception.error_codes module agent_inspect.exception.exception module agent_inspect.metrics module agent_inspect.metrics.adapters module agent_inspect.metrics.adapters.base_adapter module agent_inspect.metrics.adapters.tau2bench_adapter module agent_inspect.metrics.adapters.toolsandbox_adapter module agent_inspect.metrics.constants module agent_inspect.metrics.multi_samples module agent_inspect.metrics.multi_samples.multi_sample_metric module agent_inspect.metrics.multi_samples.pass_at_k module agent_inspect.metrics.multi_samples.pass_hat_k module agent_inspect.metrics.observed module agent_inspect.metrics.observed.latency module agent_inspect.metrics.observed.observed_metric module agent_inspect.metrics.observed.token_count module agent_inspect.metrics.observed.tool_call_count module agent_inspect.metrics.scorer module agent_inspect.metrics.scorer.auc module agent_inspect.metrics.scorer.llm_based_metric module agent_inspect.metrics.scorer.metric module agent_inspect.metrics.scorer.ppt module agent_inspect.metrics.scorer.progress module agent_inspect.metrics.scorer.success module agent_inspect.metrics.scorer.templates module agent_inspect.metrics.scorer.tool_correctness module agent_inspect.metrics.utils module agent_inspect.metrics.utils.expected_tool_validators module agent_inspect.metrics.utils.metrics_utils module agent_inspect.metrics.utils.subgoal_validators module agent_inspect.metrics.utils.trace_validators module agent_inspect.metrics.utils.user_proxy_validators module agent_inspect.metrics.validator module agent_inspect.metrics.validator.exact_match module agent_inspect.metrics.validator.llm_check module

agent_inspect.metrics.validator.regex_match module agent_inspect.metrics.validator.subgoal_completion module agent_inspect.metrics.validator.tool_call_completion module agent_inspect.metrics.validator.validator module agent_inspect.models module agent_inspect.models.llm_payload module agent_inspect.models.llm_response module agent_inspect.models.metrics module agent_inspect.models.metrics.agent_data_sample module agent_inspect.models.metrics.agent_trace module agent_inspect.models.metrics.metric_score module agent_inspect.models.metrics.validation_result module agent_inspect.models.tools module agent_inspect.models.tools.analysis_models module agent_inspect.models.user_proxy module agent_inspect.models.user_proxy.chat module agent_inspect.models.user_proxy.terminating_condition module agent_inspect.tools module agent_inspect.tools.error_analysis module agent_inspect.tools.error_analysis.error_analysis module agent_inspect.tools.error_analysis.llm_constants module agent_inspect.tools.error_analysis.statistic_analysis module agent_inspect.user_proxy module agent_inspect.user_proxy.llm_proxy_agent module agent_inspect.user_proxy.templates module agent_inspect.user_proxy.user_proxy_agent module agent_inspect.user_proxy.utils module agent_response (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) agent_responses (agent_inspect.models.user_proxy.chat.ConversationTurn attribute) (agent_inspect.models.user_proxy.ConversationTurn attribute) agent_run_id (agent_inspect.models.tools.analysis_models.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.analysis_models.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute) agent_thought (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) AgentDialogueTrace (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace) AgentResponse (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace) analyze_batch() (agent_inspect.tools.error_analysis.error_analysis.ErrorAnalysis method) (agent_inspect.tools.error_analysis.ErrorAnalysis method) (agent_inspect.tools.ErrorAnalysis method) analyzed_validations_clustered_by_errors (agent_inspect.models.tools.analysis_models.ErrorAnalysisResult attribute) (agent_inspect.models.tools.ErrorAnalysisResult attribute) AnalyzedSubgoalValidation (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) AUC (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.auc) AUC_CALCULATION_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) AverageLatency (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.latency) AzureOpenAIClient (class in agent_inspect.clients) (class in agent_inspect.clients.azure_openai_client)

B

backoff_handler() (in module agent_inspect.clients.azure_openai_client) (in module agent_inspect.clients.litellm_client) base_error (agent_inspect.models.tools.analysis_models.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.AnalyzedSubgoalValidation attribute)

BaseAdapter (class in agent_inspect.metrics.adapters) (class in agent_inspect.metrics.adapters.base_adapter) BooleanScore (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.metric_score)

C

ChatHistory (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat) check (agent_inspect.models.metrics.agent_data_sample.ToolInputParameter attribute) (agent_inspect.models.metrics.agent_data_sample.ToolOutput attribute) (agent_inspect.models.metrics.ToolInputParameter attribute) (agent_inspect.models.metrics.ToolOutput attribute) (agent_inspect.models.user_proxy.chat.UserProxyMessage attribute) (agent_inspect.models.user_proxy.terminating_condition.TerminatingCondition attribute) (agent_inspect.models.user_proxy.TerminatingCondition attribute) (agent_inspect.models.user_proxy.UserProxyMessage attribute) CLIENT_ERROR_CODE (agent_inspect.exception.ClientComponent attribute) (agent_inspect.exception.error_codes.ClientComponent attribute) CLIENT_REQUEST_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) ClientComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) completed_subgoal_validations (agent_inspect.models.tools.analysis_models.ErrorAnalysisResult attribute) (agent_inspect.models.tools.ErrorAnalysisResult attribute) completion (agent_inspect.models.llm_response.LLMResponse attribute) (agent_inspect.models.LLMResponse attribute) compute() (agent_inspect.metrics.multi_samples.multi_sample_metric.MultiSampleMetric method) (agent_inspect.metrics.multi_samples.MultiSampleMetric method) (agent_inspect.metrics.multi_samples.pass_at_k.PassAtK method) (agent_inspect.metrics.multi_samples.pass_hat_k.PassHatK method) (agent_inspect.metrics.multi_samples.PassAtK method) (agent_inspect.metrics.multi_samples.PassHatK method)

compute_statistic_analysis_result() (agent_inspect.tools.error_analysis.statistic_analysis.StatisticAnalysis static method) (agent_inspect.tools.error_analysis.StatisticAnalysis static method) (agent_inspect.tools.StatisticAnalysis static method) conversation (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute) Conversation (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) conversations (agent_inspect.models.user_proxy.chat.ChatHistory attribute) (agent_inspect.models.user_proxy.ChatHistory attribute) ConversationTurn (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat) convert_payload_to_raw_request() (agent_inspect.clients.azure_openai_client.AzureOpenAIClient method) (agent_inspect.clients.AzureOpenAIClient method) (agent_inspect.clients.litellm_client.LiteLLMClient method) (agent_inspect.clients.LiteLLMClient method) convert_to_agent_trace() (agent_inspect.metrics.adapters.base_adapter.BaseAdapter method) (agent_inspect.metrics.adapters.BaseAdapter method) (agent_inspect.metrics.adapters.tau2bench_adapter.Tau2BenchAdapter method) (agent_inspect.metrics.adapters.Tau2BenchAdapter method) (agent_inspect.metrics.adapters.toolsandbox_adapter.ToolsandboxAdapter method) (agent_inspect.metrics.adapters.ToolsandboxAdapter method)

D

data_sample_id (agent_inspect.models.tools.analysis_models.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.analysis_models.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute)

details (agent_inspect.models.metrics.agent_data_sample.SubGoal attribute) (agent_inspect.models.metrics.SubGoal attribute)

E

EMPTY_PROGRESS_SCORE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) EMPTY_VALIDATION_RESULT (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) ensure_full_stop() (in module agent_inspect.user_proxy.utils) error_message (agent_inspect.models.llm_response.LLMResponse attribute) (agent_inspect.models.LLMResponse attribute) ErrorAnalysis (class in agent_inspect.tools) (class in agent_inspect.tools.error_analysis) (class in agent_inspect.tools.error_analysis.error_analysis) ErrorAnalysisDataSample (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) ErrorAnalysisResult (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) ErrorCode (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) evaluate() (agent_inspect.metrics.observed.AverageLatency method) (agent_inspect.metrics.observed.InputTotalTokenCount method) (agent_inspect.metrics.observed.latency.AverageLatency method) (agent_inspect.metrics.observed.latency.LatencyMetric method) (agent_inspect.metrics.observed.latency.TotalLatency method) (agent_inspect.metrics.observed.observed_metric.ObservedMetric method) (agent_inspect.metrics.observed.ObservedMetric method) (agent_inspect.metrics.observed.OutputTotalTokenCount method) (agent_inspect.metrics.observed.ReasoningTotalTokenCount method) (agent_inspect.metrics.observed.token_count.InputTotalTokenCount method) (agent_inspect.metrics.observed.token_count.OutputTotalTokenCount method) (agent_inspect.metrics.observed.token_count.ReasoningTotalTokenCount method) (agent_inspect.metrics.observed.token_count.TokenConsumptionMetric method) (agent_inspect.metrics.observed.token_count.TotalTokenConsumption method) (agent_inspect.metrics.observed.TokenConsumptionMetric method) (agent_inspect.metrics.observed.tool_call_count.ToolCallCount method) (agent_inspect.metrics.observed.ToolCallCount method) (agent_inspect.metrics.observed.TotalLatency method) (agent_inspect.metrics.observed.TotalTokenConsumption method) (agent_inspect.metrics.scorer.AUC method) (agent_inspect.metrics.scorer.auc.AUC method) (agent_inspect.metrics.scorer.llm_based_metric.LLMBasedMetric method) (agent_inspect.metrics.scorer.LLMBasedMetric method) (agent_inspect.metrics.scorer.Metric method) (agent_inspect.metrics.scorer.metric.Metric method) (agent_inspect.metrics.scorer.PPT method) (agent_inspect.metrics.scorer.ppt.PPT method) (agent_inspect.metrics.scorer.progress.ProgressBasedMetric method) (agent_inspect.metrics.scorer.progress.ProgressScore method) (agent_inspect.metrics.scorer.progress.ProgressScoresThroughTurns method) (agent_inspect.metrics.scorer.ProgressBasedMetric method) (agent_inspect.metrics.scorer.ProgressScore method) (agent_inspect.metrics.scorer.ProgressScoresThroughTurns method) (agent_inspect.metrics.scorer.success.SuccessBasedMetric method) (agent_inspect.metrics.scorer.success.SuccessScore method) (agent_inspect.metrics.scorer.success.SuccessScoreFinalTurn method) (agent_inspect.metrics.scorer.SuccessBasedMetric method) (agent_inspect.metrics.scorer.SuccessScore method) (agent_inspect.metrics.scorer.SuccessScoreFinalTurn method) (agent_inspect.metrics.scorer.tool_correctness.ToolCorrectnessMetric method) (agent_inspect.metrics.scorer.ToolCorrectnessMetric method)

EVALUATION_ERROR_CODE (agent_inspect.exception.error_codes.EvaluationComponent attribute) (agent_inspect.exception.EvaluationComponent attribute) EvaluationComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) EvaluationError, [1] EvaluationSample (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) exact_match() (in module agent_inspect.metrics.validator) (in module agent_inspect.metrics.validator.exact_match) expected_output (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) expected_parameters (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) expected_response (agent_inspect.models.metrics.agent_data_sample.Conversation attribute) (agent_inspect.models.metrics.Conversation attribute) expected_tool_call (agent_inspect.models.metrics.ToolCallValidationResult attribute) (agent_inspect.models.metrics.validation_result.ToolCallValidationResult attribute) expected_tool_calls (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute) ExpectedToolCall (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) ExpectedToolCallValidator (class in agent_inspect.metrics.utils.expected_tool_validators) explanations (agent_inspect.models.metrics.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.ToolCallValidationResult attribute) (agent_inspect.models.metrics.validation_result.ValidationResult attribute) (agent_inspect.models.metrics.ValidationResult attribute)

F

from_id (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute)

G

generate_message_from_chat_history() (agent_inspect.user_proxy.llm_proxy_agent.LLMProxyAgent method) (agent_inspect.user_proxy.user_proxy_agent.UserProxyAgent method) (agent_inspect.user_proxy.UserProxyAgent method) get_auc_score_from_progress_scores() (agent_inspect.metrics.scorer.AUC static method) (agent_inspect.metrics.scorer.auc.AUC static method) get_config_or_default() (in module agent_inspect.metrics.utils.metrics_utils) get_majority_voted_score() (in module agent_inspect.metrics.utils.metrics_utils) get_majority_voted_score_from_judge_responses() (agent_inspect.metrics.validator.Validator method) (agent_inspect.metrics.validator.validator.Validator method)

get_ppt_score_from_progress_scores() (agent_inspect.metrics.scorer.PPT static method) (agent_inspect.metrics.scorer.ppt.PPT static method) get_success_score_from_progress_score() (agent_inspect.metrics.scorer.success.SuccessBasedMetric static method) (agent_inspect.metrics.scorer.SuccessBasedMetric static method) get_success_score_from_validation_results() (agent_inspect.metrics.scorer.success.SuccessBasedMetric static method) (agent_inspect.metrics.scorer.SuccessBasedMetric static method) give_up_handler() (in module agent_inspect.clients.azure_openai_client) (in module agent_inspect.clients.litellm_client)

I

id (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.EvaluationSample attribute) (agent_inspect.models.metrics.Step attribute) (agent_inspect.models.metrics.TurnTrace attribute) (agent_inspect.models.user_proxy.chat.ChatHistory attribute) (agent_inspect.models.user_proxy.chat.ConversationTurn attribute) (agent_inspect.models.user_proxy.ChatHistory attribute) (agent_inspect.models.user_proxy.ConversationTurn attribute) input_token_consumption (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) InputTotalTokenCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) INSUFFICIENT_JUDGE_RESPONSES_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_JSON_DECODE_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute)

INVALID_JUDGE_RESPONSE_FORMAT_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_JUDGE_TRIALS_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_LLM_JUDGE_RESULT_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_USER_MESSAGE_REFLECTION (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_USER_PROXY_RESPONSE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_VALUE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) InvalidInputValueError, [1] is_completed (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.ToolCallValidationResult attribute) (agent_inspect.models.metrics.validation_result.ValidationResult attribute) (agent_inspect.models.metrics.ValidationResult attribute)

J

json_data (agent_inspect.models.user_proxy.chat.ResponseFromAgent attribute) (agent_inspect.models.user_proxy.ResponseFromAgent attribute) judge_expectation (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute)

judge_std (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute)

L

latency_in_ms (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) LatencyMetric (class in agent_inspect.metrics.observed.latency) LiteLLMClient (class in agent_inspect.clients) (class in agent_inspect.clients.litellm_client) llm_check() (in module agent_inspect.metrics.validator) (in module agent_inspect.metrics.validator.llm_check) LLMBasedMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.llm_based_metric)

LLMClient (class in agent_inspect.clients) (class in agent_inspect.clients.llm_client) LLMPayload (class in agent_inspect.models) (class in agent_inspect.models.llm_payload) LLMProxyAgent (class in agent_inspect.user_proxy.llm_proxy_agent) LLMResponse (class in agent_inspect.models) (class in agent_inspect.models.llm_response)

M

make_llm_request() (agent_inspect.clients.azure_openai_client.AzureOpenAIClient method) (agent_inspect.clients.AzureOpenAIClient method) (agent_inspect.clients.litellm_client.LiteLLMClient method) (agent_inspect.clients.LiteLLMClient method) (agent_inspect.clients.llm_client.LLMClient method) (agent_inspect.clients.LLMClient method) make_llm_request_with_retry() (agent_inspect.clients.azure_openai_client.AzureOpenAIClient method) (agent_inspect.clients.AzureOpenAIClient method) (agent_inspect.clients.litellm_client.LiteLLMClient method) (agent_inspect.clients.LiteLLMClient method) make_llm_requests() (agent_inspect.clients.azure_openai_client.AzureOpenAIClient method) (agent_inspect.clients.AzureOpenAIClient method) (agent_inspect.clients.litellm_client.LiteLLMClient method) (agent_inspect.clients.LiteLLMClient method) (agent_inspect.clients.llm_client.LLMClient method) (agent_inspect.clients.LLMClient method) make_request_with_payload() (agent_inspect.clients.azure_openai_client.AzureOpenAIClient method) (agent_inspect.clients.AzureOpenAIClient method) (agent_inspect.clients.litellm_client.LiteLLMClient method) (agent_inspect.clients.LiteLLMClient method) (agent_inspect.clients.llm_client.LLMClient method) (agent_inspect.clients.LLMClient method) make_request_with_payload_using_retry() (agent_inspect.clients.azure_openai_client.AzureOpenAIClient method) (agent_inspect.clients.AzureOpenAIClient method) (agent_inspect.clients.litellm_client.LiteLLMClient method) (agent_inspect.clients.LiteLLMClient method) map_subgoal_validations_to_binary_matrix() (in module agent_inspect.metrics.utils.metrics_utils) match_to_int() (in module agent_inspect.metrics.utils.metrics_utils) MAX_RETRY_EXCEEDED_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) max_tokens (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) message (agent_inspect.models.metrics.agent_data_sample.Conversation attribute) (agent_inspect.models.metrics.Conversation attribute) message_str (agent_inspect.models.user_proxy.chat.UserProxyMessage attribute) (agent_inspect.models.user_proxy.UserProxyMessage attribute) Metric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.metric) MISSING_VALUE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) model (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) module agent_inspect agent_inspect.clients agent_inspect.clients.azure_openai_client agent_inspect.clients.litellm_client agent_inspect.clients.llm_client agent_inspect.exception agent_inspect.exception.error_codes agent_inspect.exception.exception agent_inspect.metrics agent_inspect.metrics.adapters agent_inspect.metrics.adapters.base_adapter agent_inspect.metrics.adapters.tau2bench_adapter agent_inspect.metrics.adapters.toolsandbox_adapter agent_inspect.metrics.constants agent_inspect.metrics.multi_samples agent_inspect.metrics.multi_samples.multi_sample_metric agent_inspect.metrics.multi_samples.pass_at_k agent_inspect.metrics.multi_samples.pass_hat_k agent_inspect.metrics.observed agent_inspect.metrics.observed.latency agent_inspect.metrics.observed.observed_metric agent_inspect.metrics.observed.token_count agent_inspect.metrics.observed.tool_call_count agent_inspect.metrics.scorer agent_inspect.metrics.scorer.auc agent_inspect.metrics.scorer.llm_based_metric agent_inspect.metrics.scorer.metric agent_inspect.metrics.scorer.ppt agent_inspect.metrics.scorer.progress agent_inspect.metrics.scorer.success agent_inspect.metrics.scorer.templates agent_inspect.metrics.scorer.tool_correctness agent_inspect.metrics.utils agent_inspect.metrics.utils.expected_tool_validators agent_inspect.metrics.utils.metrics_utils agent_inspect.metrics.utils.subgoal_validators agent_inspect.metrics.utils.trace_validators agent_inspect.metrics.utils.user_proxy_validators agent_inspect.metrics.validator agent_inspect.metrics.validator.exact_match agent_inspect.metrics.validator.llm_check agent_inspect.metrics.validator.regex_match agent_inspect.metrics.validator.subgoal_completion agent_inspect.metrics.validator.tool_call_completion agent_inspect.metrics.validator.validator agent_inspect.models agent_inspect.models.llm_payload agent_inspect.models.llm_response agent_inspect.models.metrics agent_inspect.models.metrics.agent_data_sample agent_inspect.models.metrics.agent_trace agent_inspect.models.metrics.metric_score agent_inspect.models.metrics.validation_result agent_inspect.models.tools agent_inspect.models.tools.analysis_models agent_inspect.models.user_proxy agent_inspect.models.user_proxy.chat agent_inspect.models.user_proxy.terminating_condition agent_inspect.tools agent_inspect.tools.error_analysis agent_inspect.tools.error_analysis.error_analysis agent_inspect.tools.error_analysis.llm_constants agent_inspect.tools.error_analysis.statistic_analysis agent_inspect.user_proxy agent_inspect.user_proxy.llm_proxy_agent agent_inspect.user_proxy.templates agent_inspect.user_proxy.user_proxy_agent agent_inspect.user_proxy.utils

MultiSampleMetric (class in agent_inspect.metrics.multi_samples) (class in agent_inspect.metrics.multi_samples.multi_sample_metric)

N

name (agent_inspect.models.metrics.agent_data_sample.ToolInputParameter attribute) (agent_inspect.models.metrics.ToolInputParameter attribute)

NumericalScore (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.metric_score)

O

ObservedMetric (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.observed_metric) output_token_consumption (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute)

OutputTotalTokenCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count)

P

parent_ids (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) PassAtK (class in agent_inspect.metrics.multi_samples) (class in agent_inspect.metrics.multi_samples.pass_at_k) PassHatK (class in agent_inspect.metrics.multi_samples) (class in agent_inspect.metrics.multi_samples.pass_hat_k) PPT (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.ppt) PPT_CALCULATION_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute)

ProgressBasedMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.progress) ProgressScore (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.progress) ProgressScoresThroughTurns (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.progress) prompt_sent_to_llmj (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.validation_result.SubGoalValidationResult attribute)

R

reasoning_token_consumption (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) ReasoningTotalTokenCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) regex_match() (in module agent_inspect.metrics.validator) (in module agent_inspect.metrics.validator.regex_match)

response (agent_inspect.models.metrics.agent_trace.AgentResponse attribute) (agent_inspect.models.metrics.AgentResponse attribute) response_str (agent_inspect.models.user_proxy.chat.ResponseFromAgent attribute) (agent_inspect.models.user_proxy.ResponseFromAgent attribute) ResponseFromAgent (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat)

S

score (agent_inspect.models.metrics.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) StatisticAnalysis (class in agent_inspect.tools) (class in agent_inspect.tools.error_analysis) (class in agent_inspect.tools.error_analysis.statistic_analysis) StatisticAnalysisResult (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) status (agent_inspect.models.llm_response.LLMResponse attribute) (agent_inspect.models.LLMResponse attribute) status_code (agent_inspect.models.metrics.agent_trace.AgentResponse attribute) (agent_inspect.models.metrics.AgentResponse attribute) Step (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace) steps (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) stop_sequence (agent_inspect.models.user_proxy.terminating_condition.TerminatingCondition attribute) (agent_inspect.models.user_proxy.TerminatingCondition attribute) structured_output (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) sub_goal (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.validation_result.SubGoalValidationResult attribute) sub_goals (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute)

sub_scores (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) SubGoal (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) subgoal_validation (agent_inspect.models.tools.analysis_models.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.AnalyzedSubgoalValidation attribute) subgoal_validations (agent_inspect.models.tools.analysis_models.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute) SubGoalCompletionValidator (class in agent_inspect.metrics.validator) (class in agent_inspect.metrics.validator.subgoal_completion) SubGoalValidationResult (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.validation_result) SubGoalValidator (class in agent_inspect.metrics.utils.subgoal_validators) SuccessBasedMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.success) SuccessScore (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.success) SuccessScoreFinalTurn (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.success) system_prompt (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute)

T

tally_judge_voting() (in module agent_inspect.metrics.utils.metrics_utils) tally_votes() (in module agent_inspect.metrics.utils.metrics_utils) Tau2BenchAdapter (class in agent_inspect.metrics.adapters) (class in agent_inspect.metrics.adapters.tau2bench_adapter) temperature (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) TerminatingCondition (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.terminating_condition) TokenConsumptionMetric (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) tool (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) (agent_inspect.models.metrics.Step attribute) TOOL_ERROR_CODE (agent_inspect.exception.error_codes.ToolComponent attribute) (agent_inspect.exception.ToolComponent attribute) tool_input_args (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) tool_output (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) ToolCallCompletionValidator (class in agent_inspect.metrics.validator) (class in agent_inspect.metrics.validator.tool_call_completion) ToolCallCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.tool_call_count) ToolCallValidationResult (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.validation_result) ToolComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes)

ToolCorrectnessMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.tool_correctness) ToolError, [1] ToolInputParameter (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) ToolOutput (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) ToolsandboxAdapter (class in agent_inspect.metrics.adapters) (class in agent_inspect.metrics.adapters.toolsandbox_adapter) TotalLatency (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.latency) TotalTokenConsumption (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) TraceValidator (class in agent_inspect.metrics.utils.trace_validators) turn (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.agent_data_sample.SubGoal attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) (agent_inspect.models.metrics.SubGoal attribute) turn_id (agent_inspect.models.metrics.agent_data_sample.Conversation attribute) (agent_inspect.models.metrics.Conversation attribute) turns (agent_inspect.models.metrics.agent_trace.AgentDialogueTrace attribute) (agent_inspect.models.metrics.AgentDialogueTrace attribute) TurnTrace (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace) type (agent_inspect.models.metrics.agent_data_sample.SubGoal attribute) (agent_inspect.models.metrics.SubGoal attribute)

U

UNSUCCESSFUL_LLM_SUMMARIZATION (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) UNSUCCESSFUL_MAJORITY_VOTING (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) UNSUPPORTED_ATTRIBUTION_TYPE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) user_instruction (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute) user_message (agent_inspect.models.user_proxy.chat.ConversationTurn attribute) (agent_inspect.models.user_proxy.ConversationTurn attribute) user_prompt (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute)

USER_PROXY_ERROR_CODE (agent_inspect.exception.error_codes.UserProxyComponent attribute) (agent_inspect.exception.UserProxyComponent attribute) UserProxyAgent (class in agent_inspect.user_proxy) (class in agent_inspect.user_proxy.user_proxy_agent) UserProxyComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) UserProxyError, [1] UserProxyInputValidator (class in agent_inspect.metrics.utils.user_proxy_validators) UserProxyMessage (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat)

V

validate() (agent_inspect.metrics.validator.subgoal_completion.SubGoalCompletionValidator method) (agent_inspect.metrics.validator.SubGoalCompletionValidator method) (agent_inspect.metrics.validator.tool_call_completion.ToolCallCompletionValidator method) (agent_inspect.metrics.validator.ToolCallCompletionValidator method) (agent_inspect.metrics.validator.Validator method) (agent_inspect.metrics.validator.validator.Validator method) validate_dynamic() (agent_inspect.metrics.validator.subgoal_completion.SubGoalCompletionValidator method) (agent_inspect.metrics.validator.SubGoalCompletionValidator method) validate_inputs_for_pass_k_initialisation() (in module agent_inspect.metrics.utils.metrics_utils)

validation_results (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) ValidationResult (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.validation_result) Validator (class in agent_inspect.metrics.validator) (class in agent_inspect.metrics.validator.validator) value (agent_inspect.models.metrics.agent_data_sample.ToolInputParameter attribute) (agent_inspect.models.metrics.agent_data_sample.ToolOutput attribute) (agent_inspect.models.metrics.ToolInputParameter attribute) (agent_inspect.models.metrics.ToolOutput attribute)