feat(genai): Add new samples for the Interactions API

genai/batch_prediction/requirements.txt

@@ -1,3 +1,3 @@
 google-cloud-bigquery==3.29.0
 google-cloud-storage==2.19.0
-google-genai==1.42.0
+google-genai==2.29.0

genai/bounding_box/boundingbox_interactions_with_txt_img.py

@@ -0,0 +1,120 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def generate_content() -> str:
+    # [START googlegenaisdk_interactions_boundingbox_with_txt_img]
+    import json
+    import requests
+    from google import genai
+    from google.genai.interactions import (
+        ImageContent,
+        TextContent,
+        TextResponseFormat,
+    )
+    from PIL import Image, ImageColor, ImageDraw
+    from pydantic import BaseModel
+
+    # Helper class to represent a bounding box
+    class BoundingBox(BaseModel):
+        """
+        Represents a bounding box with its 2D coordinates and associated label.
+
+        Attributes:
+            box_2d (list[int]): A list of integers representing the 2D coordinates of the bounding box,
+                                typically in the format [y_min, x_min, y_max, x_max].
+            label (str): A string representing the label or class associated with the object within the bounding box.
+        """
+
+        box_2d: list[int]
+        label: str
+
+    # Helper function to plot bounding boxes on an image
+    def plot_bounding_boxes(image_uri: str, bounding_boxes: list[BoundingBox]) -> None:
+        """
+        Plots bounding boxes on an image with labels, using PIL and normalized coordinates.
+
+        Args:
+            image_uri: The URI of the image file.
+            bounding_boxes: A list of BoundingBox objects. Each box's coordinates are in
+                            normalized [y_min, x_min, y_max, x_max] format.
+        """
+        with Image.open(requests.get(image_uri, stream=True, timeout=10).raw) as im:
+            width, height = im.size
+            draw = ImageDraw.Draw(im)
+
+            colors = list(ImageColor.colormap.keys())
+
+            for i, bbox in enumerate(bounding_boxes):
+                # Scale normalized coordinates to image dimensions
+                abs_y_min = int(bbox.box_2d[0] / 1000 * height)
+                abs_x_min = int(bbox.box_2d[1] / 1000 * width)
+                abs_y_max = int(bbox.box_2d[2] / 1000 * height)
+                abs_x_max = int(bbox.box_2d[3] / 1000 * width)
+
+                color = colors[i % len(colors)]
+
+                # Draw the rectangle using the correct (x, y) pairs
+                draw.rectangle(
+                    ((abs_x_min, abs_y_min), (abs_x_max, abs_y_max)),
+                    outline=color,
+                    width=4,
+                )
+                if bbox.label:
+                    # Position the text at the top-left corner of the box
+                    draw.text((abs_x_min + 8, abs_y_min + 6), bbox.label, fill=color)
+
+            im.show()
+
+    client = genai.Client()
+    image_uri = "https://storage.googleapis.com/generativeai-downloads/images/socks.jpg"
+
+    system_instruction = """
+    Return bounding boxes as an array with labels.
+    Never return masks. Limit to 25 objects.
+    If an object is present multiple times, give each object a unique label
+    according to its distinct characteristics (colors, size, position, etc..).
+    """
+
+    interaction = client.interactions.create(
+        model="gemini-3.8-flash",
+        system_instruction=system_instruction,
+        input=[
+            ImageContent(
+                uri=image_uri,
+                mime_type="image/jpeg",
+            ),
+            TextContent(
+                text="Output the positions of the socks with a face. Label according to position in the image."
+            ),
+        ],
+        response_format=TextResponseFormat(
+            mime_type="application/json",
+            schema={
+                "type": "array",
+                "items": BoundingBox.model_json_schema(),
+            },
+        ),
+    )
+
+    print(interaction.output_text)
+    boxes = [BoundingBox(**b) for b in json.loads(interaction.output_text)]
+    plot_bounding_boxes(image_uri, boxes)
+
+    # [END googlegenaisdk_interactions_boundingbox_with_txt_img]
+    return interaction.output_text
+
+
+if __name__ == "__main__":
+    generate_content()

genai/bounding_box/requirements.txt

@@ -1,2 +1,2 @@
-google-genai==1.42.0
+google-genai==2.29.0
 pillow==12.3.0; python_version >= "3.10"

genai/bounding_box/test_bounding_box_examples.py

@@ -18,6 +18,7 @@
 
 import os
 
+import boundingbox_interactions_with_txt_img
 import boundingbox_with_txt_img
 
 os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True"
@@ -29,3 +30,9 @@
 def test_boundingbox_with_txt_img() -> None:
     response = boundingbox_with_txt_img.generate_content()
     assert response
+
+
+def test_boundingbox_interactions_with_txt_img() -> None:
+    response = boundingbox_interactions_with_txt_img.generate_content()
+    assert response
+

genai/code_execution/codeexecution_interactions_annotateimage_with_txt_gcsimg.py

@@ -0,0 +1,67 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def generate_content() -> bool:
+    # [START googlegenaisdk_interactions_codeexecution_annotateimage_with_txt_gcsimg]
+    import base64
+    import io
+    from PIL import Image
+    from google import genai
+    from google.genai.interactions import CodeExecution, ImageContent, TextContent
+
+    client = genai.Client()
+
+    interaction = client.interactions.create(
+        model="gemini-3.8-flash",
+        input=[
+            ImageContent(
+                uri="https://storage.googleapis.com/cloud-samples-data/generative-ai/image/robotic.jpeg",
+                mime_type="image/jpeg",
+            ),
+            TextContent(
+                text="Annotate on the image with arrows of different colors, which object should go into which bin."
+            ),
+        ],
+        tools=[CodeExecution()],
+    )
+
+    img_count = 0
+    for step in interaction.steps:
+        if step.type == "code_execution_call":
+            print("####################### 1. Generate Python Code #######################")
+            print(step.arguments.code)
+        elif step.type == "code_execution_result":
+            print("####################### 2. Executing Python Code #######################")
+            print(step.result)
+
+    if interaction.output_image:
+        print("####################### 3. Save Output #######################")
+        img_count += 1
+        output_location = f"robotic-annotate-output-{img_count}.jpg"
+        image_bytes = base64.b64decode(interaction.output_image.data)
+        image = Image.open(io.BytesIO(image_bytes))
+        image = image.convert("RGB")
+        image.save(output_location)
+        print(f"Output is saved to {output_location}")
+
+    if interaction.output_text:
+        print(interaction.output_text)
+
+    # [END googlegenaisdk_interactions_codeexecution_annotateimage_with_txt_gcsimg]
+    return True
+
+
+if __name__ == "__main__":
+    generate_content()

genai/code_execution/codeexecution_interactions_barplot_with_txt_img.py

@@ -0,0 +1,68 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def generate_content() -> bool:
+    # [START googlegenaisdk_interactions_codeexecution_barplot_with_txt_img]
+    import base64
+    import io
+    from PIL import Image
+    from google import genai
+    from google.genai.interactions import CodeExecution, ImageContent, TextContent
+
+    client = genai.Client()
+
+    interaction = client.interactions.create(
+        model="gemini-3.8-flash",
+        input=[
+            ImageContent(
+                uri="https://storage.googleapis.com/cloud-samples-data/generative-ai/image/benchmark.jpeg",
+                mime_type="image/jpeg",
+            ),
+            TextContent(
+                text="Make a bar chart of per-category performance, normalize prior SOTA as 1.0 for each task,"
+                "then take average per-category. Plot using matplotlib with nice style."
+            ),
+        ],
+        tools=[CodeExecution()],
+    )
+
+    img_count = 0
+    for step in interaction.steps:
+        if step.type == "code_execution_call":
+            print("####################### 1. Generate Python Code #######################")
+            print(step.arguments.code)
+        elif step.type == "code_execution_result":
+            print("####################### 2. Executing Python Code #######################")
+            print(step.result)
+
+    if interaction.output_image:
+        print("####################### 3. Save Output #######################")
+        img_count += 1
+        output_location = f"output-barplot-{img_count}.jpg"
+        image_bytes = base64.b64decode(interaction.output_image.data)
+        image = Image.open(io.BytesIO(image_bytes))
+        image = image.convert("RGB")
+        image.save(output_location)
+        print(f"Output is saved to {output_location}")
+
+    if interaction.output_text:
+        print(interaction.output_text)
+
+    # [END googlegenaisdk_interactions_codeexecution_barplot_with_txt_img]
+    return True
+
+
+if __name__ == "__main__":
+    generate_content()

genai/code_execution/codeexecution_interactions_cropimage_with_txt_img.py

@@ -0,0 +1,65 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def generate_content() -> bool:
+    # [START googlegenaisdk_interactions_codeexecution_cropimage_with_txt_img]
+    import base64
+    import io
+    import requests
+    from PIL import Image
+    from google import genai
+    from google.genai.interactions import CodeExecution, ImageContent, TextContent
+
+    # Download the input image
+    image_path = "https://storage.googleapis.com/cloud-samples-data/generative-ai/image/chips.jpeg"
+    image_bytes = requests.get(image_path).content
+    b64_image = base64.b64encode(image_bytes).decode("utf-8")
+
+    client = genai.Client()
+
+    interaction = client.interactions.create(
+        model="gemini-3.8-flash",
+        input=[
+            ImageContent(data=b64_image, mime_type="image/jpeg"),
+            TextContent(text="Locate the ESMT chip. What are the numbers on the chip?"),
+        ],
+        tools=[CodeExecution()],
+    )
+
+    for step in interaction.steps:
+        if step.type == "code_execution_call":
+            print("####################### 1. Generate Python Code #######################")
+            print(step.arguments.code)
+        elif step.type == "code_execution_result":
+            print("####################### 2. Executing Python Code #######################")
+            print(step.result)
+
+    if interaction.output_image:
+        print("####################### 3. Save Output #######################")
+        image_data = base64.b64decode(interaction.output_image.data)
+        image = Image.open(io.BytesIO(image_data))
+        output_location = "ESMT-chip-output.jpg"
+        image.save(output_location)
+        print(f"Output is saved to {output_location}")
+
+    if interaction.output_text:
+        print(interaction.output_text)
+
+    # [END googlegenaisdk_interactions_codeexecution_cropimage_with_txt_img]
+    return True
+
+
+if __name__ == "__main__":
+    generate_content()

genai/code_execution/test_codeexecution.py

@@ -16,6 +16,9 @@
 import codeexecution_annotateimage_with_txt_gcsimg
 import codeexecution_barplot_with_txt_img
 import codeexecution_cropimage_with_txt_img
+import codeexecution_interactions_annotateimage_with_txt_gcsimg
+import codeexecution_interactions_barplot_with_txt_img
+import codeexecution_interactions_cropimage_with_txt_img
 
 os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True"
 os.environ["GOOGLE_CLOUD_LOCATION"] = "global"  # "us-central1"
@@ -33,3 +36,16 @@ def test_codeexecution_barplot_with_txt_img() -> None:
 
 def test_codeexecution_cropimage_with_txt_img() -> None:
     assert codeexecution_cropimage_with_txt_img.generate_content()
+
+
+def test_codeexecution_interactions_annotateimage_with_txt_gcsimg() -> None:
+    assert codeexecution_interactions_annotateimage_with_txt_gcsimg.generate_content()
+
+
+def test_codeexecution_interactions_barplot_with_txt_img() -> None:
+    assert codeexecution_interactions_barplot_with_txt_img.generate_content()
+
+
+def test_codeexecution_interactions_cropimage_with_txt_img() -> None:
+    assert codeexecution_interactions_cropimage_with_txt_img.generate_content()
+

genai/content_cache/requirements.txt

@@ -1 +1 @@
-google-genai==1.42.0
+google-genai==2.29.0