From 9bc371665933f9c06846b20913df88bb0a8efd63 Mon Sep 17 00:00:00 2001 From: Stan Date: Fri, 20 Nov 2026 22:01:26 +0500 Subject: [PATCH] docs: HeyGen Decision 1 takes videos and audio - `videos` and `audios` (public https URLs, up to 10 each) on both pages and specs, with a Video and audio section; examples run on dev 2026-11-20 (CC0 clips from MDN's interactive examples) - images up to 100 on Decision 1 too (HeyGen's own limit now) --- .../heygen/decision-1-turbo.json | 20 +++ .../heygen/decision-1-turbo.md | 149 ++++++++++++++++- .../decision-models/heygen/decision-1.json | 24 ++- .../decision-models/heygen/decision-1.md | 153 +++++++++++++++++- 4 files changed, 334 insertions(+), 12 deletions(-) diff --git a/docs/api-references/decision-models/heygen/decision-1-turbo.json b/docs/api-references/decision-models/heygen/decision-1-turbo.json index 8a257ed9..12a7b965 100644 --- a/docs/api-references/decision-models/heygen/decision-1-turbo.json +++ b/docs/api-references/decision-models/heygen/decision-1-turbo.json @@ -291,6 +291,26 @@ "nullable": true }, "description": "Not supported here: This model takes no `assets`: send its images in `images`. A non-empty map is refused." + }, + "videos": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "minItems": 1, + "maxItems": 10, + "description": "Up to 10 videos of the state, in order, each a public https:// URL that answers without a redirect or authentication, with a `video/*` Content-Type — up to 50 MiB each; the model reads sampled frames (a 16-second clip is about 11,000 tokens). Every question sees all of them; their tokens are billed as input tokens." + }, + "audios": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "minItems": 1, + "maxItems": 10, + "description": "Up to 10 audio clips of the state, in order, each a public https:// URL that answers without a redirect or authentication, with a `audio/*` Content-Type — MP3 or WAV; a 5-second voice clip is about 300 tokens. Every question sees all of them; their tokens are billed as input tokens." } }, "required": [ diff --git a/docs/api-references/decision-models/heygen/decision-1-turbo.md b/docs/api-references/decision-models/heygen/decision-1-turbo.md index 6c2a078d..565bf20c 100644 --- a/docs/api-references/decision-models/heygen/decision-1-turbo.md +++ b/docs/api-references/decision-models/heygen/decision-1-turbo.md @@ -1,5 +1,5 @@ --- -description: "Model id: heygen/decision-1-turbo. HeyGen Decision 1 Turbo is the faster, cheaper HeyGen decision model: calibrated yes/no, choice and score answers about text and up to 100 images in one request…" +description: "Model id: heygen/decision-1-turbo. HeyGen Decision 1 Turbo is the faster, cheaper HeyGen decision model: calibrated yes/no, choice and score answers about text, images, video and audio in one request…" --- # HeyGen Decision 1 Turbo @@ -20,11 +20,11 @@ This documentation is valid for the following list of our models: ## Model Overview -HeyGen Decision 1 Turbo is the faster, cheaper HeyGen decision model. Like [Decision 1](decision-1.md) it returns typed answers (yes/no, choice, score) with a calibrated probability for every answer and generates no text, but it takes up to 100 images in one request and has a 262,144-token context window. Up to 400 named questions per request. +HeyGen Decision 1 Turbo is the faster, cheaper HeyGen decision model. Like [Decision 1](decision-1.md) it returns typed answers (yes/no, choice, score) with a calibrated probability for every answer and generates no text, but it is cheaper per token and has a 262,144-token context window. It takes up to 100 images, 10 videos and 10 audio clips in one request. Up to 400 named questions per request. The request is the same as for [HeyGen Decision 1](decision-1.md), so you can switch between them by changing `model`. On the same support ticket below, Turbo used 765 input tokens against 1,002 for Decision 1 and answered in about half a second. -It uses the same `/v1/decisions` request as [Jev](../typesafe/jev.md) and [d1](../liquid-ai/d1.md) for text: `state` and an object of typed `questions`. Images differ: they go in **`images`**, not in `assets` (see [below](#images)). +It uses the same `/v1/decisions` request as [Jev](../typesafe/jev.md) and [d1](../liquid-ai/d1.md) for text: `state` and an object of typed `questions`. Media differ: images go in **`images`**, videos in **`videos`** and audio in **`audios`**, not in `assets` (see [Images](#images) and [Video and audio](#video-and-audio)). {% hint style="success" %} [Create AI/ML API Key](https://aimlapi.com/app/keys) @@ -47,7 +47,7 @@ At the bottom of this page, pick the snippet for your preferred programming lang ▪ **Provide input:** put the content to evaluate in `state` and describe what to decide in `questions`. **4️ (Optional) Tune the request**\ -See the API schema below for the three question types (`noul`, `choice`, `score`), their `criteria`, and the `images` field. +See the API schema below for the three question types (`noul`, `choice`, `score`), their `criteria`, and the `images`, `videos` and `audios` fields. **5️ Run your code**\ Run the updated code in your development environment. @@ -296,3 +296,144 @@ A `choice` option can be `null` when its key says it all, as `cat`, `dog` and `b {% hint style="warning" %} Images are billed as input tokens: 1.5 tokens per 32×32-pixel patch (a 1024×1024 image is 1,536), counted again for every question. The request above billed 3,592 input tokens against 765 for the text-only one. Images are never resized: one outside the limits, or a request over 16 MiB once the images are inlined, is refused with a `400` that says what to change. {% endhint %} + +## Video and audio + +Send up to 10 videos in **`videos`** and up to 10 audio clips in **`audios`**, in order, next to `images` or on their own. Each entry is a public `https://` URL that answers without a redirect or a login, with a `video/*` or `audio/*` Content-Type (a video up to 50 MiB). Base64 `data:` URLs are not taken for video and audio. Refer to them by position ("video 1", "audio 1") in `instructions`. The model reads the images, then the videos, then `state`. + +{% code overflow="wrap" %} +```python +import json, requests + +response = requests.post( + "https://api.aimlapi.com/v1/decisions", + headers={ + "Authorization": "Bearer ", + "Content-Type": "application/json", + }, + json={ + "model": "heygen/decision-1-turbo", + "state": "A clip uploaded to a nature channel.", + "videos": ["https://interactive-examples.mdn.mozilla.net/media/cc0-videos/flower.mp4"], + "questions": { + "scene": { + "type": "choice", + "instructions": "What does video 1 mainly show?", + "criteria": { + "flower": "A flower or plant up close", + "animal": "A mammal or bird", + "city": "Streets or buildings", + "people": "People", + }, + }, + "safe": {"type": "noul", "instructions": "Is video 1 safe to show to all ages?"}, + }, + }, +) + +print(json.dumps(response.json(), indent=2, ensure_ascii=False)) +``` +{% endcode %} + +{% code overflow="wrap" %} +```json +{ + "model": "heygen-decision-1-turbo", + "answers": { + "safe": { + "type": "noul", + "noul": 0.98 + }, + "scene": { + "type": "choice", + "choice": "flower", + "confidence": 0.97, + "probabilities": { + "animal": 0.01, + "city": 0.0, + "flower": 0.98, + "people": 0.01 + } + } + }, + "usage": { + "input_tokens": 5448, + "output_tokens": 0 + }, + "meta": { + "model": "heygen/decision-1-turbo", + "provider": "heygen", + "usage": { + "credits_used": 916, + "usd_spent": 0.0 + } + } +} +``` +{% endcode %} + +Audio goes the same way, in `audios`: + +{% code overflow="wrap" %} +```python +json={ + "model": "heygen/decision-1-turbo", + "state": "A sound uploaded to a sound-effects library.", + "audios": ["https://interactive-examples.mdn.mozilla.net/media/cc0-audio/t-rex-roar.mp3"], + "questions": { + "speech": {"type": "noul", "instructions": "Does someone speak in audio 1?"}, + "sound": { + "type": "choice", + "instructions": "What is audio 1?", + "criteria": { + "speech": "A person talking", + "music": "Music", + "animal": "An animal or creature sound", + "noise": "Machine or background noise", + }, + }, + }, +} +``` +{% endcode %} + +{% code overflow="wrap" %} +```json +{ + "model": "heygen-decision-1-turbo", + "answers": { + "sound": { + "type": "choice", + "choice": "animal", + "confidence": 0.92, + "probabilities": { + "animal": 0.94, + "music": 0.02, + "noise": 0.03, + "speech": 0.02 + } + }, + "speech": { + "type": "noul", + "noul": 0.06 + } + }, + "usage": { + "input_tokens": 261, + "output_tokens": 0 + }, + "meta": { + "model": "heygen/decision-1-turbo", + "provider": "heygen", + "usage": { + "credits_used": 44, + "usd_spent": 0.0 + } + } +} +``` +{% endcode %} + +{% hint style="warning" %} +Video and audio are billed as input tokens and, like images, counted again for every question: the 5-second clip above billed 5,448 input tokens for its two questions (a 16-second clip is about 11,000 per question), and the audio clip 261. A video or audio URL that is not `https://`, or more than 10 of either, is refused with a `400` before the request reaches HeyGen. +{% endhint %} diff --git a/docs/api-references/decision-models/heygen/decision-1.json b/docs/api-references/decision-models/heygen/decision-1.json index 3bff88ee..c6183457 100644 --- a/docs/api-references/decision-models/heygen/decision-1.json +++ b/docs/api-references/decision-models/heygen/decision-1.json @@ -282,8 +282,8 @@ "format": "uri", "description": "A JPEG, PNG or WebP image: a base64 `data:` URL (`data:image/png;base64,…`) or an http(s) URL." }, - "maxItems": 32, - "description": "Up to 32 images of the state, in order: every question sees all of them, so refer to an image by its position. Each one counts as input tokens — 1.5 per 32×32-pixel patch (a 1024×1024 image is 1,536) — again for every question. 16.0 MiB for the whole request once the images are inlined. Images are never resized: one outside these limits is refused with a 400 that says what to change." + "maxItems": 100, + "description": "Up to 100 images of the state, in order: every question sees all of them, so refer to an image by its position. Each one counts as input tokens — 1.5 per 32×32-pixel patch (a 1024×1024 image is 1,536) — again for every question. 16.0 MiB for the whole request once the images are inlined. Images are never resized: one outside these limits is refused with a 400 that says what to change." }, "assets": { "type": "object", @@ -291,6 +291,26 @@ "nullable": true }, "description": "Not supported here: This model takes no `assets`: send its images in `images`. A non-empty map is refused." + }, + "videos": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "minItems": 1, + "maxItems": 10, + "description": "Up to 10 videos of the state, in order, each a public https:// URL that answers without a redirect or authentication, with a `video/*` Content-Type — up to 50 MiB each; the model reads sampled frames (a 16-second clip is about 11,000 tokens). Every question sees all of them; their tokens are billed as input tokens." + }, + "audios": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "minItems": 1, + "maxItems": 10, + "description": "Up to 10 audio clips of the state, in order, each a public https:// URL that answers without a redirect or authentication, with a `audio/*` Content-Type — MP3 or WAV; a 5-second voice clip is about 300 tokens. Every question sees all of them; their tokens are billed as input tokens." } }, "required": [ diff --git a/docs/api-references/decision-models/heygen/decision-1.md b/docs/api-references/decision-models/heygen/decision-1.md index 2cb205b0..5c36ad29 100644 --- a/docs/api-references/decision-models/heygen/decision-1.md +++ b/docs/api-references/decision-models/heygen/decision-1.md @@ -1,5 +1,5 @@ --- -description: "Model id: heygen/decision-1. HeyGen Decision 1 answers typed questions about text and images in one request (yes/no, choice, score) with a calibrated probability for every answer and no generated text…" +description: "Model id: heygen/decision-1. HeyGen Decision 1 answers typed questions about text, images, video and audio in one request (yes/no, choice, score) with a calibrated probability for every answer and no generated text…" --- # HeyGen Decision 1 @@ -20,11 +20,11 @@ This documentation is valid for the following list of our models: ## Model Overview -HeyGen Decision 1 is HeyGen's decision model: instead of generating text it returns typed answers (yes/no, choice, score) with a calibrated probability for every answer. One request can carry up to 400 named questions about the same text and up to 32 images, which makes it a fit for moderation, identity checks, routing, triage and QA. Context window is 131,072 tokens. +HeyGen Decision 1 is HeyGen's decision model: instead of generating text it returns typed answers (yes/no, choice, score) with a calibrated probability for every answer. One request can carry up to 400 named questions about the same text, up to 100 images, 10 videos and 10 audio clips, which makes it a fit for moderation, identity checks, routing, triage and QA. Context window is 131,072 tokens. -[HeyGen Decision 1 Turbo](decision-1-turbo.md) takes the same request and is the faster, cheaper option: on the support ticket below it used 765 input tokens against 1,002 here, and it takes up to 100 images instead of 32. +[HeyGen Decision 1 Turbo](decision-1-turbo.md) takes the same request and is the faster, cheaper option: on the support ticket below it used 765 input tokens against 1,002 here. -It uses the same `/v1/decisions` request as [Jev](../typesafe/jev.md) and [d1](../liquid-ai/d1.md) for text: `state` and an object of typed `questions`. Images differ: they go in **`images`**, not in `assets` (see [below](#images)). +It uses the same `/v1/decisions` request as [Jev](../typesafe/jev.md) and [d1](../liquid-ai/d1.md) for text: `state` and an object of typed `questions`. Media differ: images go in **`images`**, videos in **`videos`** and audio in **`audios`**, not in `assets` (see [Images](#images) and [Video and audio](#video-and-audio)). {% hint style="success" %} [Create AI/ML API Key](https://aimlapi.com/app/keys) @@ -47,7 +47,7 @@ At the bottom of this page, pick the snippet for your preferred programming lang ▪ **Provide input:** put the content to evaluate in `state` and describe what to decide in `questions`. **4️ (Optional) Tune the request**\ -See the API schema below for the three question types (`noul`, `choice`, `score`), their `criteria`, and the `images` field. +See the API schema below for the three question types (`noul`, `choice`, `score`), their `criteria`, and the `images`, `videos` and `audios` fields. **5️ Run your code**\ Run the updated code in your development environment. @@ -218,7 +218,7 @@ Every key you send under `questions` comes back under `answers` with the matchin ## Images -Send up to 32 images in **`images`**, in order. Each entry is a JPEG, PNG or WebP image as an http(s) URL or a base64 `data:` URL (`data:image/png;base64,…`). Every question sees all of them, so refer to an image by its position ("image 1", "image 2") in `instructions`. `assets`, which [Jev 1.14](../typesafe/jev-1.14.md) uses for images, is refused by this model, and there is no `box` question. +Send up to 100 images in **`images`**, in order. Each entry is a JPEG, PNG or WebP image as an http(s) URL or a base64 `data:` URL (`data:image/png;base64,…`). Every question sees all of them, so refer to an image by its position ("image 1", "image 2") in `instructions`. `assets`, which [Jev 1.14](../typesafe/jev-1.14.md) uses for images, is refused by this model, and there is no `box` question. {% code overflow="wrap" %} ```python @@ -296,3 +296,144 @@ A `choice` option can be `null` when its key says it all, as `cat`, `dog` and `b {% hint style="warning" %} Images are billed as input tokens: 1.5 tokens per 32×32-pixel patch (a 1024×1024 image is 1,536), counted again for every question. The request above billed 3,697 input tokens against 1,002 for the text-only one. Images are never resized: one outside the limits, or a request over 16 MiB once the images are inlined, is refused with a `400` that says what to change. {% endhint %} + +## Video and audio + +Send up to 10 videos in **`videos`** and up to 10 audio clips in **`audios`**, in order, next to `images` or on their own. Each entry is a public `https://` URL that answers without a redirect or a login, with a `video/*` or `audio/*` Content-Type (a video up to 50 MiB). Base64 `data:` URLs are not taken for video and audio. Refer to them by position ("video 1", "audio 1") in `instructions`. The model reads the images, then the videos, then `state`. + +{% code overflow="wrap" %} +```python +import json, requests + +response = requests.post( + "https://api.aimlapi.com/v1/decisions", + headers={ + "Authorization": "Bearer ", + "Content-Type": "application/json", + }, + json={ + "model": "heygen/decision-1", + "state": "A clip uploaded to a nature channel.", + "videos": ["https://interactive-examples.mdn.mozilla.net/media/cc0-videos/flower.mp4"], + "questions": { + "scene": { + "type": "choice", + "instructions": "What does video 1 mainly show?", + "criteria": { + "flower": "A flower or plant up close", + "animal": "A mammal or bird", + "city": "Streets or buildings", + "people": "People", + }, + }, + "safe": {"type": "noul", "instructions": "Is video 1 safe to show to all ages?"}, + }, + }, +) + +print(json.dumps(response.json(), indent=2, ensure_ascii=False)) +``` +{% endcode %} + +{% code overflow="wrap" %} +```json +{ + "model": "heygen-decision-1", + "answers": { + "safe": { + "type": "noul", + "noul": 0.98 + }, + "scene": { + "type": "choice", + "choice": "flower", + "confidence": 0.97, + "probabilities": { + "animal": 0.01, + "city": 0.0, + "flower": 0.98, + "people": 0.01 + } + } + }, + "usage": { + "input_tokens": 5448, + "output_tokens": 0 + }, + "meta": { + "model": "heygen/decision-1", + "provider": "heygen", + "usage": { + "credits_used": 1526, + "usd_spent": 0.0 + } + } +} +``` +{% endcode %} + +Audio goes the same way, in `audios`: + +{% code overflow="wrap" %} +```python +json={ + "model": "heygen/decision-1", + "state": "A sound uploaded to a sound-effects library.", + "audios": ["https://interactive-examples.mdn.mozilla.net/media/cc0-audio/t-rex-roar.mp3"], + "questions": { + "speech": {"type": "noul", "instructions": "Does someone speak in audio 1?"}, + "sound": { + "type": "choice", + "instructions": "What is audio 1?", + "criteria": { + "speech": "A person talking", + "music": "Music", + "animal": "An animal or creature sound", + "noise": "Machine or background noise", + }, + }, + }, +} +``` +{% endcode %} + +{% code overflow="wrap" %} +```json +{ + "model": "heygen-decision-1", + "answers": { + "sound": { + "type": "choice", + "choice": "animal", + "confidence": 0.92, + "probabilities": { + "animal": 0.94, + "music": 0.02, + "noise": 0.03, + "speech": 0.02 + } + }, + "speech": { + "type": "noul", + "noul": 0.06 + } + }, + "usage": { + "input_tokens": 261, + "output_tokens": 0 + }, + "meta": { + "model": "heygen/decision-1", + "provider": "heygen", + "usage": { + "credits_used": 74, + "usd_spent": 0.0 + } + } +} +``` +{% endcode %} + +{% hint style="warning" %} +Video and audio are billed as input tokens and, like images, counted again for every question: the 5-second clip above billed 5,448 input tokens for its two questions (a 16-second clip is about 11,000 per question), and the audio clip 261. A video or audio URL that is not `https://`, or more than 10 of either, is refused with a `400` before the request reaches HeyGen. +{% endhint %}