diff --git a/CHANGELOG.md b/CHANGELOG.md index 35852a5..50141a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.7.0 - 2026-11-09 + +Breaking: `extract()` on `Search1API` and `AsyncSearch1API` requires the keyword arguments `prompt` and `response_format`, matching the API. `response_format` is typed as `ExtractResponseFormat` (in `search1api.types`): `{"type": "json_schema", "json_schema": }`. The API never accepted a request without them; it now rejects one with `422` and an `errors` array instead of `400`, which the client raises as `UnprocessableEntityError`. The bundled OpenAPI snapshot records the required fields, the request example and `TARGET_BAD_REQUEST` in `failureCategory`. + ## 0.6.1 - 2026-11-05 The `X-Search1API-Client` request header now reports the installed version (`python/0.6.1`). Since 0.3.0 it had always said `python/0.3.0`. The version lives in `search1api/_version.py` and a test keeps it equal to `pyproject.toml`. No API changes. diff --git a/README.md b/README.md index 54d71be..6fb900b 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,34 @@ Path("screenshot.png").write_bytes(screenshot["data"]) print(screenshot["content_type"], screenshot.get("request_id")) ``` -The clients also support news, crawl, sitemap, trending, extract, usage, and +## Extract + +`extract` pulls structured fields out of one page. Both `prompt` and +`response_format` are required; wrap your JSON Schema as +`{"type": "json_schema", "json_schema": }`: + +```python +response = client.extract( + "https://example.com/product", + prompt="Extract the product name and price.", + response_format={ + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "price": {"type": "number"}, + }, + }, + }, +) + +print(response["results"]) +``` + +A request without either field raises `UnprocessableEntityError` (`422`). + +The clients also support news, crawl, sitemap, trending, usage, and batch operations exposed by the Search1API HTTP API. Requests time out after 30 seconds and retry `429` and transient `5xx` responses twice by default. Authentication, payment, and validation errors are never retried. Deepcrawl diff --git a/openapi/search1api.openapi.json b/openapi/search1api.openapi.json index c83eb36..4b601cd 100644 --- a/openapi/search1api.openapi.json +++ b/openapi/search1api.openapi.json @@ -71,7 +71,7 @@ }, "failureCategory": { "type": "string", - "description": "`/crawl` and `/extract`: why the page could not be read, e.g. `NOT_FOUND`, `GONE`, `FORBIDDEN`, `AUTH_REQUIRED`, `BLOCKED`, `RATE_LIMITED`, `DNS_NOT_FOUND`, `TARGET_UNREACHABLE`, `TIMEOUT`, `QUALITY_CHECK`." + "description": "`/crawl` and `/extract`: why the page could not be read, e.g. `NOT_FOUND`, `GONE`, `FORBIDDEN`, `AUTH_REQUIRED`, `BLOCKED`, `RATE_LIMITED`, `DNS_NOT_FOUND`, `TARGET_UNREACHABLE`, `TARGET_BAD_REQUEST`, `TIMEOUT`, `QUALITY_CHECK`." }, "upstreamStatus": { "type": "integer", @@ -501,18 +501,41 @@ "properties": { "url": { "type": "string", - "format": "uri" + "format": "uri", + "description": "The webpage to extract data from." }, "prompt": { - "type": "string" + "type": "string", + "minLength": 1, + "description": "Natural-language instructions: what to extract from the page, e.g. \"Extract the product name and price.\"" }, "response_format": { "type": "object", - "additionalProperties": {} + "properties": { + "type": { + "type": "string", + "enum": [ + "json_schema" + ] + }, + "json_schema": { + "type": "object", + "additionalProperties": {}, + "description": "The JSON Schema the result must follow, e.g. {\"type\": \"object\", \"properties\": {\"title\": {\"type\": \"string\"}}}. Pass the schema itself, not an object wrapping it under name/schema." + } + }, + "required": [ + "type", + "json_schema" + ], + "additionalProperties": false, + "description": "Shape of the result. Wrap your JSON Schema as {\"type\": \"json_schema\", \"json_schema\": }." } }, "required": [ - "url" + "url", + "prompt", + "response_format" ], "additionalProperties": false }, @@ -3270,7 +3293,7 @@ "post": { "operationId": "extract", "summary": "Extract structured content from a URL", - "description": "Pull structured data out of a single webpage using a natural-language prompt and a JSON schema you supply. Use it when you need specific fields — prices, specifications, contact details — rather than the whole document; when you want the full text, use POST /crawl. Costs 10 credits per request.", + "description": "Pull structured data out of a single webpage using a natural-language prompt and a JSON schema you supply; both are required, and a request missing either returns 422 with the fields to fix. Use it when you need specific fields — prices, specifications, contact details — rather than the whole document; when you want the full text, use POST /crawl. Costs 10 credits per request.", "tags": [ "Crawl" ], @@ -3279,13 +3302,13 @@ "id": "js", "lang": "ts", "label": "TypeScript SDK", - "source": "import { Search1API } from '@search1api/client';\n\nconst client = new Search1API();\nconst response = await client.extract('https://example.com', {\n prompt: 'Extract the page title and description.',\n});\n\nconsole.log(response);" + "source": "import { Search1API } from '@search1api/client';\n\nconst client = new Search1API();\nconst response = await client.extract('https://example.com', {\n prompt: 'Extract the page title and description.',\n responseFormat: {\n type: 'json_schema',\n json_schema: {\n type: 'object',\n properties: {\n title: { type: 'string' },\n description: { type: 'string' },\n },\n },\n },\n});\n\nconsole.log(response.results);" }, { "id": "python", "lang": "python", "label": "Python SDK", - "source": "from search1api import Search1API\n\nclient = Search1API()\nresponse = client.extract(\n \"https://example.com\",\n prompt=\"Extract the page title and description.\",\n)\n\nprint(response)" + "source": "from search1api import Search1API\n\nclient = Search1API()\nresponse = client.extract(\n \"https://example.com\",\n prompt=\"Extract the page title and description.\",\n response_format={\n \"type\": \"json_schema\",\n \"json_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\"type\": \"string\"},\n \"description\": {\"type\": \"string\"},\n },\n },\n },\n)\n\nprint(response[\"results\"])" } ], "responses": { @@ -3411,20 +3434,70 @@ "properties": { "url": { "type": "string", - "format": "uri" + "format": "uri", + "description": "The webpage to extract data from." }, "prompt": { - "type": "string" + "type": "string", + "minLength": 1, + "description": "Natural-language instructions: what to extract from the page, e.g. \"Extract the product name and price.\"" }, "response_format": { "type": "object", - "additionalProperties": {} + "properties": { + "type": { + "type": "string", + "enum": [ + "json_schema" + ] + }, + "json_schema": { + "type": "object", + "additionalProperties": {}, + "description": "The JSON Schema the result must follow, e.g. {\"type\": \"object\", \"properties\": {\"title\": {\"type\": \"string\"}}}. Pass the schema itself, not an object wrapping it under name/schema." + } + }, + "required": [ + "type", + "json_schema" + ], + "additionalProperties": false, + "description": "Shape of the result. Wrap your JSON Schema as {\"type\": \"json_schema\", \"json_schema\": }." } }, "required": [ - "url" + "url", + "prompt", + "response_format" ], "additionalProperties": false + }, + "examples": { + "pageSummary": { + "summary": "Title and description", + "description": "Wrap the JSON Schema of the result as {\"type\": \"json_schema\", \"json_schema\": }.", + "value": { + "url": "https://example.com", + "prompt": "Extract the page title and description.", + "response_format": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": [ + "title" + ] + } + } + } + } } } } diff --git a/pyproject.toml b/pyproject.toml index 3a61c2f..518e812 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "search1api" -version = "0.6.1" +version = "0.7.0" description = "Official Python client for Search1API" readme = "README.md" license = { file = "LICENSE" } diff --git a/src/search1api/_version.py b/src/search1api/_version.py index 951860b..63a07cd 100644 --- a/src/search1api/_version.py +++ b/src/search1api/_version.py @@ -1,3 +1,3 @@ # Sent as ``X-Search1API-Client: python/``. Kept equal to the version # in pyproject.toml by tests/test_client.py; bump both together. -__version__ = "0.6.1" +__version__ = "0.7.0" diff --git a/src/search1api/client.py b/src/search1api/client.py index bdfa0b8..acaab0d 100644 --- a/src/search1api/client.py +++ b/src/search1api/client.py @@ -31,6 +31,7 @@ DeepcrawlFormat, DeepcrawlStatusResponse, ExtractResponse, + ExtractResponseFormat, FeedbackAgent, FeedbackCategory, FeedbackResponse, @@ -476,17 +477,15 @@ def extract( self, url: str, *, - prompt: Optional[str] = None, - response_format: Optional[Mapping[str, Any]] = None, + prompt: str, + response_format: ExtractResponseFormat, ) -> ExtractResponse: return cast( ExtractResponse, self._request_json( "POST", "/extract", - json=_compact( - {"url": url, "prompt": prompt, "response_format": response_format} - ), + json={"url": url, "prompt": prompt, "response_format": response_format}, ), ) @@ -908,17 +907,15 @@ async def extract( self, url: str, *, - prompt: Optional[str] = None, - response_format: Optional[Mapping[str, Any]] = None, + prompt: str, + response_format: ExtractResponseFormat, ) -> ExtractResponse: return cast( ExtractResponse, await self._request_json( "POST", "/extract", - json=_compact( - {"url": url, "prompt": prompt, "response_format": response_format} - ), + json={"url": url, "prompt": prompt, "response_format": response_format}, ), ) diff --git a/src/search1api/types.py b/src/search1api/types.py index ad97a5e..8f710b6 100644 --- a/src/search1api/types.py +++ b/src/search1api/types.py @@ -279,6 +279,14 @@ class TrendingResponse(TypedDict): results: List[TrendingResult] +class ExtractResponseFormat(TypedDict): + """Shape of the extract result: your JSON Schema, wrapped.""" + + type: Literal["json_schema"] + json_schema: Dict[str, Any] + """The JSON Schema of the result itself, not wrapped in ``{name, schema}``.""" + + class ExtractResponse(TypedDict): success: bool extractParameters: Dict[str, str] diff --git a/tests/test_client.py b/tests/test_client.py index 60586df..693dc76 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -12,8 +12,10 @@ InternalServerError, Search1API, Search1APIConfigurationError, + UnprocessableEntityError, __version__, ) +from search1api.types import ExtractResponseFormat def json_response(status_code, body, request): @@ -201,6 +203,101 @@ def handler(request): http_client.close() +EXTRACT_FORMAT: ExtractResponseFormat = { + "type": "json_schema", + "json_schema": {"type": "object", "properties": {"title": {"type": "string"}}}, +} + + +def test_extract_sends_prompt_and_response_format(): + seen = [] + + def handler(request): + seen.append(request) + return json_response( + 200, + { + "success": True, + "extractParameters": {"url": "https://example.com"}, + "results": {"title": "Example Domain"}, + }, + request, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(handler)) + client = Search1API("test-key", client=http_client) + + result = client.extract( + "https://example.com", + prompt="Extract the page title.", + response_format=EXTRACT_FORMAT, + ) + + assert result["results"] == {"title": "Example Domain"} + assert seen[0].url.path == "/extract" + assert json.loads(seen[0].content) == { + "url": "https://example.com", + "prompt": "Extract the page title.", + "response_format": EXTRACT_FORMAT, + } + http_client.close() + + +def test_extract_validation_error_raises_unprocessable_entity(): + def handler(request): + return json_response( + 422, + { + "ok": False, + "error": "Unprocessable Entity", + "message": "prompt is required and cannot be empty", + "errors": [{"field": "prompt", "code": "too_small"}], + }, + request, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(handler)) + client = Search1API("test-key", client=http_client) + + with pytest.raises(UnprocessableEntityError): + client.extract( + "https://example.com", prompt=" ", response_format=EXTRACT_FORMAT + ) + http_client.close() + + +def test_async_extract_sends_prompt_and_response_format(): + async def run(): + seen = [] + + async def handler(request): + seen.append(request) + return json_response( + 200, + { + "success": True, + "extractParameters": {"url": "https://example.com"}, + "results": {"title": "Example Domain"}, + }, + request, + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + client = AsyncSearch1API("test-key", client=http_client) + try: + await client.extract( + "https://example.com", + prompt="Extract the page title.", + response_format=EXTRACT_FORMAT, + ) + finally: + await http_client.aclose() + return seen + + seen = asyncio.run(run()) + assert json.loads(seen[0].content)["response_format"] == EXTRACT_FORMAT + + def test_deepcrawl_starts_and_polls(): responses = [ (202, {"taskId": "task_1", "status": "queued"}),