Method: googleapis.aiplatform.v1beta1.projects.locations.evaluateInstances

Evaluates instances based on a given metric.

Arguments

Parameters
location

string

Required. The resource name of the Location to evaluate the instances. Format: projects/{project}/locations/{location}
region

string

Required. Region of the HTTP endpoint. For example, if region is set to us-central1, the endpoint https://us-central1-integrations.googleapis.com will be used. See service endpoints.
body

object (GoogleCloudAiplatformV1beta1EvaluateInstancesRequest)

Required.

Raised exceptions

Exceptions
ConnectionError In case of a network problem (such as DNS failure or refused connection).
HttpError If the response status is >= 400 (excluding 429 and 503).
TimeoutError If a long-running operation takes longer to finish than the specified timeout limit.
TypeError If an operation or function receives an argument of the wrong type.
ValueError If an operation or function receives an argument of the right type but an inappropriate value. For example, a negative timeout.

Response

If successful, the response contains an instance of GoogleCloudAiplatformV1beta1EvaluateInstancesResponse.

Subworkflow snippet

Some fields might be optional or required. To identify required fields, refer to the API documentation.

YAML

- evaluateInstances:
    call: googleapis.aiplatform.v1beta1.projects.locations.evaluateInstances
    args:
        location: ...
        region: ...
        body:
            bleuInput:
                instances: ...
                metricSpec: ...
            coherenceInput:
                instance:
                    prediction: ...
                metricSpec:
                    version: ...
            exactMatchInput:
                instances: ...
                metricSpec: ...
            fluencyInput:
                instance:
                    prediction: ...
                metricSpec:
                    version: ...
            fulfillmentInput:
                instance:
                    instruction: ...
                    prediction: ...
                metricSpec:
                    version: ...
            groundednessInput:
                instance:
                    context: ...
                    prediction: ...
                metricSpec:
                    version: ...
            pairwiseQuestionAnsweringQualityInput:
                instance:
                    baselinePrediction: ...
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            pairwiseSummarizationQualityInput:
                instance:
                    baselinePrediction: ...
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            questionAnsweringCorrectnessInput:
                instance:
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            questionAnsweringHelpfulnessInput:
                instance:
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            questionAnsweringQualityInput:
                instance:
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            questionAnsweringRelevanceInput:
                instance:
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            rougeInput:
                instances: ...
                metricSpec:
                    rougeType: ...
                    splitSummaries: ...
                    useStemmer: ...
            safetyInput:
                instance:
                    prediction: ...
                metricSpec:
                    version: ...
            summarizationHelpfulnessInput:
                instance:
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            summarizationQualityInput:
                instance:
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            summarizationVerbosityInput:
                instance:
                    context: ...
                    instruction: ...
                    prediction: ...
                    reference: ...
                metricSpec:
                    useReference: ...
                    version: ...
            toolCallValidInput:
                instances: ...
                metricSpec: ...
            toolNameMatchInput:
                instances: ...
                metricSpec: ...
            toolParameterKeyMatchInput:
                instances: ...
                metricSpec: ...
            toolParameterKvMatchInput:
                instances: ...
                metricSpec:
                    useStrictStringMatch: ...
    result: evaluateInstancesResult

JSON

[
  {
    "evaluateInstances": {
      "call": "googleapis.aiplatform.v1beta1.projects.locations.evaluateInstances",
      "args": {
        "location": "...",
        "region": "...",
        "body": {
          "bleuInput": {
            "instances": "...",
            "metricSpec": "..."
          },
          "coherenceInput": {
            "instance": {
              "prediction": "..."
            },
            "metricSpec": {
              "version": "..."
            }
          },
          "exactMatchInput": {
            "instances": "...",
            "metricSpec": "..."
          },
          "fluencyInput": {
            "instance": {
              "prediction": "..."
            },
            "metricSpec": {
              "version": "..."
            }
          },
          "fulfillmentInput": {
            "instance": {
              "instruction": "...",
              "prediction": "..."
            },
            "metricSpec": {
              "version": "..."
            }
          },
          "groundednessInput": {
            "instance": {
              "context": "...",
              "prediction": "..."
            },
            "metricSpec": {
              "version": "..."
            }
          },
          "pairwiseQuestionAnsweringQualityInput": {
            "instance": {
              "baselinePrediction": "...",
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "pairwiseSummarizationQualityInput": {
            "instance": {
              "baselinePrediction": "...",
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "questionAnsweringCorrectnessInput": {
            "instance": {
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "questionAnsweringHelpfulnessInput": {
            "instance": {
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "questionAnsweringQualityInput": {
            "instance": {
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "questionAnsweringRelevanceInput": {
            "instance": {
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "rougeInput": {
            "instances": "...",
            "metricSpec": {
              "rougeType": "...",
              "splitSummaries": "...",
              "useStemmer": "..."
            }
          },
          "safetyInput": {
            "instance": {
              "prediction": "..."
            },
            "metricSpec": {
              "version": "..."
            }
          },
          "summarizationHelpfulnessInput": {
            "instance": {
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "summarizationQualityInput": {
            "instance": {
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "summarizationVerbosityInput": {
            "instance": {
              "context": "...",
              "instruction": "...",
              "prediction": "...",
              "reference": "..."
            },
            "metricSpec": {
              "useReference": "...",
              "version": "..."
            }
          },
          "toolCallValidInput": {
            "instances": "...",
            "metricSpec": "..."
          },
          "toolNameMatchInput": {
            "instances": "...",
            "metricSpec": "..."
          },
          "toolParameterKeyMatchInput": {
            "instances": "...",
            "metricSpec": "..."
          },
          "toolParameterKvMatchInput": {
            "instances": "...",
            "metricSpec": {
              "useStrictStringMatch": "..."
            }
          }
        }
      },
      "result": "evaluateInstancesResult"
    }
  }
]