refactor(generative_ai/evaluation): migrate vertexai.generative_models to genai SDK

genai/evaluation/noxfile_config.py

@@ -0,0 +1,42 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default TEST_CONFIG_OVERRIDE for python repos.
+
+# You can copy this file into your directory, then it will be imported from
+# the noxfile.py.
+
+# The source of truth:
+# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py
+
+TEST_CONFIG_OVERRIDE = {
+    # You can opt out from the test for specific Python versions.
+    "ignored_versions": ["3.8", "3.9", "3.10", "3.12", "3.13"],
+    # Old samples are opted out of enforcing Python type hints
+    # All new samples should feature them
+    "enforce_type_hints": True,
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    "gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
+    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+    # If you need to use a specific version of pip,
+    # change pip_version_override to the string representation
+    # of the version number, for example, "20.2.4"
+    "pip_version_override": None,
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    "envs": {},
+}

genai/evaluation/pairwise_summarization_quality.py

@@ -0,0 +1,119 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START aiplatform_genai_evaluation_pairwise_summarization_quality]
+
+import os
+
+from google import genai
+from google.genai import types
+
+import pandas as pd
+
+from vertexai.evaluation import (
+    EvalTask,
+    MetricPromptTemplateExamples,
+    PairwiseMetric,
+)
+from vertexai.preview.evaluation import EvalResult
+
+# TODO (developer) set GOOGLE_CLOUD_PROJECT and REGION_ID
+# environment variables before running.
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+LOCATION = os.getenv("REGION_ID")
+BASELINE_MODEL = os.getenv("BASELINE_MODEL", "gemini-2.5-flash")
+CANDIDATE_MODEL = os.getenv("CANDIDATE_MODEL", "gemini-2.5-pro")
+
+PROMPT = """
+    Summarize the text such that a five-year-old can understand.
+
+    # Text
+
+    As part of a comprehensive initiative to tackle urban congestion and foster
+    sustainable urban living, a major city has revealed ambitious plans for an
+    extensive overhaul of its public transportation system. The project aims not
+    only to improve the efficiency and reliability of public transit but also to
+    reduce the city\'s carbon footprint and promote eco-friendly commuting options.
+    City officials anticipate that this strategic investment will enhance
+    accessibility for residents and visitors alike, ushering in a new era of
+    efficient, environmentally conscious urban transportation.
+    """
+
+
+def evaluate_output() -> EvalResult:
+    """
+    Evaluates a candidate model's summarization quality
+    against a baseline model using Vertex AI.
+    """
+
+    baseline_responses = []
+    candidate_responses = []
+
+    genai_client = genai.Client(vertexai=True, project=PROJECT_ID, location=LOCATION)
+
+    baseline_resp = genai_client.models.generate_content(
+        model=BASELINE_MODEL,
+        contents=PROMPT,
+        config=types.GenerateContentConfig(temperature=0.4),
+    )
+    baseline_responses.append(baseline_resp.text)
+
+    candidate_resp = genai_client.models.generate_content(
+        model=CANDIDATE_MODEL,
+        contents=PROMPT,
+        config=types.GenerateContentConfig(temperature=0.4),
+    )
+    candidate_responses.append(candidate_resp.text)
+
+    eval_df = pd.DataFrame(
+        {
+            "prompt": PROMPT,
+            "response": candidate_responses,
+            "baseline_model_response": baseline_responses,
+        }
+    )
+
+    prompt_template = MetricPromptTemplateExamples.get_prompt_template(
+        "pairwise_summarization_quality"
+    )
+
+    pairwise_text_quality = PairwiseMetric(
+        metric="pairwise_summarization_quality",
+        metric_prompt_template=prompt_template,
+    )
+
+    eval_task = EvalTask(
+        dataset=eval_df,
+        metrics=[pairwise_text_quality],
+        experiment="pairwise-benchmark2",
+    )
+
+    comparison_result = eval_task.evaluate()
+
+    pd.set_option("display.max_columns", None)
+    pd.set_option("display.max_colwidth", 250)
+
+    columns_to_print = [
+        "prompt",
+        "baseline_model_response",
+        "response",
+        "pairwise_summarization_quality/pairwise_choice",
+        "pairwise_summarization_quality/explanation",
+    ]
+    print(comparison_result.metrics_table[columns_to_print])
+
+    return comparison_result
+
+
+# [END aiplatform_genai_evaluation_pairwise_summarization_quality]

genai/evaluation/requirements-test.txt

@@ -0,0 +1 @@
+pytest==9.0.3

genai/evaluation/requirements.txt

@@ -0,0 +1,4 @@
+pandas==2.3.3
+google-auth==2.55.2
+google-cloud-aiplatform[evaluation]==1.161.0
+google-genai==2.12.1

genai/evaluation/test_evaluation.py

@@ -0,0 +1,19 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pairwise_summarization_quality
+
+def test_pairwise_evaluation_summarization_quality() -> None:
+    response = pairwise_summarization_quality.evaluate_output()
+    assert response