diff --git a/docs/labelbox/index.rst b/docs/labelbox/index.rst index 39e879191..3e27674d0 100644 --- a/docs/labelbox/index.rst +++ b/docs/labelbox/index.rst @@ -41,6 +41,7 @@ Labelbox Python SDK Documentation ontology_kind organization pagination + performance project project-model-config project-sync diff --git a/docs/labelbox/performance.rst b/docs/labelbox/performance.rst new file mode 100644 index 000000000..b86d5fe98 --- /dev/null +++ b/docs/labelbox/performance.rst @@ -0,0 +1,6 @@ +Performance +=============================================================================================== + +.. automodule:: labelbox.schema.performance + :members: PerformanceMetric, PerformanceMetricSeries, PerformanceTimeBucket, PerformanceScoreBucket, PerformanceHandlingTime, ProjectLabelerPerformance, ProjectReviewerPerformance, WorkspaceLabelerPerformance, PerformanceReportDownload, PerformanceRows, PerformanceInterval, PerformanceDeletedLabels, PerformanceMemberType + :show-inheritance: diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index 08a58f50e..5a73c5088 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -102,6 +102,20 @@ from labelbox.schema.organization import Organization from labelbox.schema.project import Project from labelbox.schema.project_model_config import ProjectModelConfig +from labelbox.schema.performance import ( + PerformanceDeletedLabels, + PerformanceInterval, + PerformanceMemberType, + PerformanceMetric, + PerformanceMetricSeries, + PerformanceReportDownload, + PerformanceRows, + PerformanceScoreBucket, + PerformanceTimeBucket, + ProjectLabelerPerformance, + ProjectReviewerPerformance, + WorkspaceLabelerPerformance, +) from labelbox.schema.project_overview import ( ProjectOverview, ProjectOverviewDetailed, diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 75755afcc..83801cdbc 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -44,6 +44,7 @@ from labelbox.schema.identifiables import DataRowIds, GlobalKeys from labelbox.schema.label_score import LabelScore from labelbox.schema.labeling_frontend import LabelingFrontend +from labelbox.schema import performance as _performance from labelbox.schema.labeling_service_dashboard import LabelingServiceDashboard from labelbox.schema.media_type import ( MediaType, @@ -1861,7 +1862,8 @@ def is_feature_schema_archived( raise ResourceNotFoundError(Ontology, ontology_id) else: raise LabelboxError( - "Failed to get the feature schema archived status." + "Failed to get the feature schema archived status. " + f"The server answered {response.status_code}: {response.text}" ) def get_model_slice(self, slice_id) -> ModelSlice: @@ -2327,6 +2329,213 @@ def get_labeling_service_dashboards( """ return LabelingServiceDashboard.get_all(self, search_query=search_query) + def get_performance_metrics(self) -> List[_performance.PerformanceMetric]: + """Lists the performance metrics that can be queried. + + These are the metrics a project's Performance page draws. Each has a + name to pass to :meth:`Project.get_performance_metric` or + :meth:`get_workspace_performance_metric`, a unit and a definition. + + Returns: + List[PerformanceMetric]: The metrics available to your + organization. + + Example: + >>> for metric in client.get_performance_metrics(): + >>> print(metric.name, metric.unit, metric.description) + """ + return _performance.get_performance_metrics(self) + + def get_workspace_performance_metric( + self, + metric: str, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + interval: Union[ + _performance.PerformanceInterval, str + ] = _performance.PerformanceInterval.DAY, + project_ids: Optional[List[str]] = None, + user_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + use_cache: bool = True, + member_type: Optional[ + Union[_performance.PerformanceMemberType, str] + ] = None, + owner_organization_ids: Optional[List[str]] = None, + ) -> _performance.PerformanceMetricSeries: + """Returns one performance metric across the workspace. + + This is the workspace Monitor's data: every project your organization + owns and every project another organization shares with it, or only + the projects named in ``project_ids``. + + Args: + metric (str): A metric name from :meth:`get_performance_metrics`, + for example ``"labels_created"``. + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + interval (PerformanceInterval or str): The width of each bucket: + day (the default), week, month, quarter or year. + project_ids (Optional[List[str]]): Narrow the report to these + projects. You then need performance access to each of them + instead of the organization-wide permission. + user_ids (Optional[List[str]]): Count only work by these users. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + use_cache (bool): Pass False to recompute instead of using a + result cached within the last few minutes. + member_type (Optional[PerformanceMemberType or str]): Count only + work by members of your own organization (``"internal"``) or + only by members of other organizations, such as a workforce + provider's (``"external"``). The Monitor's Member Type + filter. + owner_organization_ids (Optional[List[str]]): Count only projects + owned by these organizations. Use it to separate the projects + you own from the ones shared with you. + + ``member_type`` and ``owner_organization_ids`` report on other + members, so they need permission to view all performance in every + project the report covers. + + Returns: + PerformanceMetricSeries: The metric as a value per interval. + + Raises: + lbox.exceptions.ResourceNotFoundError: The metric does not exist, + or a project named is not one you can see. + lbox.exceptions.AuthorizationError: You may not view performance + across the organization, or not everyone's in a project the + report covers. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. + + Example: + >>> series = client.get_workspace_performance_metric( + >>> "labels_created", "2026-01-01", "2026-01-31", interval="week") + >>> for bucket in series.buckets: + >>> print(bucket.start.date(), bucket.value) + >>> + >>> # The same, for the workforce provider's members only. + >>> series = client.get_workspace_performance_metric( + >>> "labels_created", "2026-01-01", "2026-01-31", + >>> member_type="external") + """ + return _performance.get_workspace_metric( + self, + metric, + start_date, + end_date, + interval=interval, + project_ids=project_ids, + user_ids=user_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + use_cache=use_cache, + member_type=member_type, + owner_organization_ids=owner_organization_ids, + ) + + def get_workspace_labeler_performance( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + project_ids: Optional[List[str]] = None, + user_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + member_type: Optional[ + Union[_performance.PerformanceMemberType, str] + ] = None, + owner_organization_ids: Optional[List[str]] = None, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = 50, + use_cache: bool = True, + ) -> _performance.PerformanceRows[_performance.WorkspaceLabelerPerformance]: + """Returns per-labeler totals across the workspace. + + This is the workspace Monitor's member table: one row for each + labeler on each project they worked on in the period. It covers the + projects your organization owns and the ones other organizations + share with it. Each row says which organization owns the project + (``project_organization_id``) and whether the labeler is one of your + own members (``member_type``). + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + project_ids (Optional[List[str]]): Narrow the report to these + projects. You then need performance access to each of them + instead of the organization-wide permission. + user_ids (Optional[List[str]]): Return only these users. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + member_type (Optional[PerformanceMemberType or str]): Return only + members of your own organization (``"internal"``) or only + members of other organizations, such as a workforce + provider's (``"external"``). + owner_organization_ids (Optional[List[str]]): Return only + projects owned by these organizations. + sort_by (Optional[str]): A column to sort by, named as the row's + attribute, for example ``"labels_created"``. Without it, rows + are ordered by project, then labeler. Sorting by email is not + offered. + descending (bool): Sort from the highest value down. + page_size (int): Rows fetched per request, at most 100. + use_cache (bool): Pass False to read the report again instead of + using rows read within the last few minutes. Leave it on + while reading a large table, so that every page comes from + the same rows. + + Returns: + PerformanceRows[WorkspaceLabelerPerformance]: One row per labeler + and project, fetched a page at a time as you iterate. Emails are + masked for labelers you may not identify. + + Raises: + lbox.exceptions.ResourceNotFoundError: A project named is not one + you can see. + lbox.exceptions.AuthorizationError: You may not view performance + across the organization. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. + + Example: + >>> rows = client.get_workspace_labeler_performance( + >>> "2026-01-01", "2026-01-31", member_type="external") + >>> print(rows.total) + >>> for row in rows: + >>> print(row.project_name, row.email, row.labels_created) + """ + return _performance.get_workspace_labelers( + self, + start_date, + end_date, + project_ids=project_ids, + user_ids=user_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + member_type=member_type, + owner_organization_ids=owner_organization_ids, + sort_by=sort_by, + descending=descending, + page_size=page_size, + use_cache=use_cache, + ) + def get_task_by_id(self, task_id: str) -> Union[Task, DataUpsertTask]: """ Fetches a task by ID. diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py new file mode 100644 index 000000000..868bef2bb --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -0,0 +1,856 @@ +"""Performance reports. + +The numbers behind a project's Performance page and the workspace Monitor, +read through the Labelbox REST API (``/api/v1/performance``). The methods on +:class:`labelbox.Client` and :class:`labelbox.schema.project.Project` that +return the classes below are thin wrappers over the functions at the bottom +of this module. +""" + +from datetime import date, datetime, timezone +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generic, + Iterable, + Iterator, + List, + Optional, + Type, + TypeVar, + Union, +) +from urllib.parse import quote + +import requests +from google.api_core import exceptions as core_exceptions +from google.api_core import retry +from lbox.exceptions import ( + ApiLimitError, + AuthenticationError, + AuthorizationError, + InternalServerError, + InvalidQueryError, + LabelboxError, + NetworkError, + ResourceNotFoundError, + TimeoutError, +) + +from labelbox.utils import _CamelCaseMixin + +if TYPE_CHECKING: + from labelbox import Client + +# Reports are computed on request and a wide date range can take a while. +_TIMEOUT_SECONDS = 120 +# How long a request is retried for, counted from its first attempt. Twice the +# time one attempt may take: enough to try once more after an attempt that +# timed out, and not to start a third. +_RETRY_DEADLINE_SECONDS = 2 * _TIMEOUT_SECONDS +_DEFAULT_PAGE_SIZE = 50 + +DateLike = Union[date, datetime, str] +"""A calendar day: a ``date``, a ``datetime`` or a string such as ``"2026-01-31"``.""" + + +class PerformanceInterval(str, Enum): + """The width of each bucket of a metric. Buckets follow UTC days.""" + + DAY = "day" + WEEK = "week" + MONTH = "month" + QUARTER = "quarter" + YEAR = "year" + + +class PerformanceDeletedLabels(str, Enum): + """Whether a report counts labels that were later deleted.""" + + INCLUDE = "include" + EXCLUDE = "exclude" + ONLY = "only" + + +class PerformanceMemberType(str, Enum): + """Whose account a member's is, seen from your organization. + + A project can be shared with another organization, such as a workforce + provider, whose members then work in it. ``EXTERNAL`` is a member whose + account belongs to another organization, or an Alignerr account; the + Monitor calls these "Workforce members". ``INTERNAL`` is a member of your + own organization. + + An Alignerr account is external to everyone, Alignerr's own organization + included. + """ + + INTERNAL = "internal" + EXTERNAL = "external" + + +class PerformanceMetric(_CamelCaseMixin): + """A metric that can be queried. + + Attributes: + name (str): The name to pass to the metric methods, for example + ``"labels_created"``. + category (str): ``"throughput"``, ``"efficiency"`` or ``"quality"``. + unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. A + percentage is a fraction from 0 to 1: 0.9 means 90%. + buckets (str): ``"time"`` for one value per interval, ``"score"`` for + a histogram of agreement scores. + description (str): What the metric measures, as the Performance page + explains it. + """ + + name: str + category: str + unit: str + buckets: str + description: str + + +class PerformanceHandlingTime(_CamelCaseMixin): + """Handling time split by activity, in seconds.""" + + creating: float + reviewing: float + reworking: float + + +class PerformanceTimeBucket(_CamelCaseMixin): + """The value of a metric over one interval. + + Attributes: + start (datetime): Start of the interval, inclusive. + end (datetime): End of the interval, exclusive. + value (float): The metric's value, in the unit of its series. + numerator (Optional[float]): For ratios, the amount divided. + denominator (Optional[float]): For ratios, what it was divided by. + modes (Optional[PerformanceHandlingTime]): The split by activity. + Only average handling time metrics have it. + """ + + start: datetime + end: datetime + value: float + numerator: Optional[float] = None + denominator: Optional[float] = None + modes: Optional[PerformanceHandlingTime] = None + + +class PerformanceScoreBucket(_CamelCaseMixin): + """The number of labels whose agreement score falls in one range. + + There are ten equal ranges, from 0 to 0.1 up to 0.9 to 1: fractions, + like every score these reports return. + + Attributes: + range_start (float): Lower bound of the range, from 0 to 0.9. + range_end (float): Upper bound of the range, from 0.1 to 1. + value (float): How many labels scored in the range. + """ + + range_start: float + range_end: float + value: float + + +class PerformanceMetricSeries(_CamelCaseMixin): + """One metric over a period. + + Attributes: + metric (str): The metric's name. + unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. A + percentage is a fraction from 0 to 1: 0.9 means 90%. + bucket_type (str): ``"time"`` or ``"score"``; says which of the two + bucket classes ``buckets`` holds. + buckets (List[Union[PerformanceTimeBucket, PerformanceScoreBucket]]): + The values. + interval (str): The bucket width that was asked for. + start_date (date): First day of the period. + end_date (date): Last day of the period. + generated_at (datetime): When the response was produced. This is not + how fresh the underlying data is. + project_id (Optional[str]): Set on a project's series. + organization_id (Optional[str]): Set on a workspace series. + project_ids (Optional[List[str]]): The projects a workspace series + was narrowed to, if any. + owner_organization_ids (Optional[List[str]]): The owning + organizations a workspace series was narrowed to, if any. + member_type (Optional[str]): The member type a workspace series was + narrowed to, if any: ``"internal"`` or ``"external"``. + skipped_project_ids (Optional[List[str]]): On a workspace series + narrowed by owning organization, the projects of those owners that + are not counted because you may not view everyone's performance + in them. Empty when none were left out. + """ + + metric: str + unit: str + bucket_type: str + buckets: List[Union[PerformanceTimeBucket, PerformanceScoreBucket]] + interval: str + start_date: date + end_date: date + generated_at: datetime + project_id: Optional[str] = None + organization_id: Optional[str] = None + project_ids: Optional[List[str]] = None + owner_organization_ids: Optional[List[str]] = None + member_type: Optional[str] = None + skipped_project_ids: Optional[List[str]] = None + + +class ProjectLabelerPerformance(_CamelCaseMixin): + """One labeler's totals on a project over a period. + + Times are in seconds. Percentages and agreement scores are fractions from + 0 to 1: 0.9 means 90%. ``email`` is masked for labelers the caller may not + identify, exactly as on the Performance page. + + Attributes: + user_id (str) + project_id (str) + email (str) + member_type (Optional[str]): ``"internal"`` for a member of your + organization, ``"external"`` for a member of another one (see + :class:`PerformanceMemberType`). ``None`` if the account no + longer exists. + labels_created (int) + labels_skipped (int) + member_total_time (float) + labeling_time_submitted (float) + labeling_time_skipped (float) + reviews_received (int) + review_time_all (float) + rework_time_all (float) + avg_time_per_label (float) + avg_review_time_all (float) + avg_rework_time_all (float) + avg_benchmark_agreement (float) + avg_consensus_agreement (float) + rework_percentage (float) + approval_percentage (float) + avg_rating (Optional[float]): ``None`` when the labeler has no ratings. + is_removed (bool): True when the labeler is no longer on the project. + """ + + user_id: str + project_id: str + email: str + member_type: Optional[str] = None + labels_created: int + labels_skipped: int + member_total_time: float + labeling_time_submitted: float + labeling_time_skipped: float + reviews_received: int + review_time_all: float + rework_time_all: float + avg_time_per_label: float + avg_review_time_all: float + avg_rework_time_all: float + avg_benchmark_agreement: float + avg_consensus_agreement: float + rework_percentage: float + approval_percentage: float + avg_rating: Optional[float] = None + is_removed: bool = False + + +class ProjectReviewerPerformance(_CamelCaseMixin): + """One reviewer's totals on a project over a period. + + Times are in seconds. Percentages are fractions from 0 to 1: 0.9 means + 90%. ``email`` is masked for reviewers the caller may not identify, + exactly as on the Performance page. + + Attributes: + user_id (str) + project_id (str) + email (str) + member_type (Optional[str]): ``"internal"`` for a member of your + organization, ``"external"`` for a member of another one (see + :class:`PerformanceMemberType`). ``None`` if the account no + longer exists. + labels_reviewed (int) + labels_reworked (int) + member_total_time (float) + avg_review_time (float) + avg_rework_time (float) + review_time (float) + rework_time (float) + rework_percentage (float) + approval_percentage (float) + is_removed (bool): True when the reviewer is no longer on the project. + """ + + user_id: str + project_id: str + email: str + member_type: Optional[str] = None + labels_reviewed: int + labels_reworked: int + member_total_time: float + avg_review_time: float + avg_rework_time: float + review_time: float + rework_time: float + rework_percentage: float + approval_percentage: float + is_removed: bool = False + + +class WorkspaceLabelerPerformance(_CamelCaseMixin): + """One labeler's totals on one project, from the workspace report. + + A labeler who worked on several projects has one of these per project. + Times are in seconds. Percentages and agreement scores are fractions from + 0 to 1: 0.9 means 90%. ``email`` is masked as on the Monitor. + + The report covers the projects your organization owns and the ones other + organizations share with it. ``project_organization_id`` says which is + which, and ``member_type`` whether the labeler is one of your own members. + + Attributes: + user_id (str) + email (str) + member_type (Optional[str]): ``"internal"`` for a member of your + organization, ``"external"`` for a member of another one (see + :class:`PerformanceMemberType`). ``None`` if the account no + longer exists. + project_id (str) + project_name (Optional[str]): ``None`` for a project that can no + longer be found. Its figures are still counted. + project_organization_id (Optional[str]): The organization that owns + the project. It differs from yours for a project shared with you, + and is ``None`` for a project that can no longer be found. + project_organization_name (Optional[str]) + is_project_deleted (bool) + labels_created (int) + labels_skipped (int) + member_total_time (float) + labeling_time_submitted (float) + labeling_time_skipped (float) + reviews_received (int) + review_time_all (float) + rework_time_all (float) + avg_time_per_label (float) + avg_review_time_all (float) + avg_rework_time_all (float) + avg_benchmark_agreement (float) + avg_consensus_agreement (float) + rework_percentage (float) + approval_percentage (float) + is_removed (bool) + """ + + user_id: str + email: str + member_type: Optional[str] = None + project_id: str + project_name: Optional[str] = None + project_organization_id: Optional[str] = None + project_organization_name: Optional[str] = None + is_project_deleted: bool + labels_created: int + labels_skipped: int + member_total_time: float + labeling_time_submitted: float + labeling_time_skipped: float + reviews_received: int + review_time_all: float + rework_time_all: float + avg_time_per_label: float + avg_review_time_all: float + avg_rework_time_all: float + avg_benchmark_agreement: float + avg_consensus_agreement: float + rework_percentage: float + approval_percentage: float + is_removed: bool = False + + +class PerformanceReportDownload(_CamelCaseMixin): + """A link to a project's performance report as a file. + + Attributes: + project_id (str) + url (str): Where to download the file from. Anyone with the link can + open it until it expires. + expires_in_seconds (int): How long the link stays valid. + start_date (date) + end_date (date) + generated_at (datetime) + """ + + project_id: str + url: str + expires_in_seconds: int + start_date: date + end_date: date + generated_at: datetime + + +Row = TypeVar("Row", bound=_CamelCaseMixin) + + +class PerformanceRows(Generic[Row]): + """The rows of a performance table. + + Iterating fetches the table a page at a time, so a large table is never + held in one response. Nothing is requested until the rows, ``total`` or + ``cached_at`` are read. The rows come in a fixed order, so no row is + repeated or missed between pages. + + Iterating a second time reads the table again from its first page, and + ``total`` and ``cached_at`` then describe that reading. + + >>> labelers = project.get_labeler_performance("2026-01-01", "2026-01-31") + >>> labelers.total + 37 + >>> for labeler in labelers: + ... print(labeler.email, labeler.labels_created) + """ + + def __init__( + self, + fetch_page: Callable[[int], Dict[str, Any]], + row_type: Type[Row], + ): + self._fetch_page = fetch_page + self._row_type = row_type + self._first_page: Optional[Dict[str, Any]] = None + self._first_page_iterated = False + + def _page(self, number: int) -> Dict[str, Any]: + if number != 1: + return self._fetch_page(number) + if self._first_page is None: + self._first_page = self._fetch_page(1) + return self._first_page + + @property + def total(self) -> int: + """The number of rows in the whole table.""" + return int(self._page(1)["total"]) + + @property + def cached_at(self) -> Optional[datetime]: + """When the rows were computed, if they were served from a cache.""" + cached_at = self._page(1).get("cachedAt") + return _parse_datetime(cached_at) if cached_at else None + + def __iter__(self) -> Iterator[Row]: + if self._first_page_iterated: + # The first page is kept so that reading ``total`` and then the + # rows costs one request. A later pass must not start from that + # old page and continue with new ones. + self._first_page = None + self._first_page_iterated = True + + number = 1 + seen = 0 + while True: + page = self._page(number) + rows = page["data"] + for row in rows: + yield self._row_type.model_validate(row) + seen += len(rows) + if not rows or seen >= int(page["total"]): + return + number += 1 + + +def _parse_datetime(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _to_day(value: DateLike) -> str: + """The day as the API takes it. Reports follow UTC days.""" + if isinstance(value, datetime): + if value.tzinfo is not None: + value = value.astimezone(timezone.utc) + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + return value + + +def _to_id(value: Any) -> str: + """The ID of a value that is an ID or an SDK object. + + Most SDK objects carry their ID as ``uid``; the pydantic ones, such as + UserGroup, carry it as ``id``. + """ + if isinstance(value, str): + return value + for attribute in ("uid", "id"): + identifier = getattr(value, attribute, None) + if isinstance(identifier, str) and identifier: + return identifier + raise TypeError( + f"Expected an ID or an object that has one, got {type(value).__name__}" + ) + + +def _to_ids(name: str, values: Optional[Iterable[Any]]) -> Optional[str]: + """IDs as one comma-separated parameter. Accepts IDs or SDK objects. + + ``None`` leaves the filter off. An empty list is refused: it is nearly + always a list that came out empty, and leaving the filter off for it + would report on everyone. + """ + if values is None: + return None + if isinstance(values, str): + values = [values] + ids = [_to_id(value) for value in values] + if not ids: + raise ValueError( + f"{name} is empty. Pass None to leave this filter off; " + "an empty list would otherwise report on everyone." + ) + return ",".join(ids) + + +def _to_field(attribute: str) -> str: + """A row attribute as the API spells it: labels_created is labelsCreated. + + A name already in the API's spelling is left as it is. + """ + head, *rest = attribute.split("_") + return head + "".join(part[:1].upper() + part[1:] for part in rest) + + +def _to_query(params: Dict[str, Any]) -> Dict[str, str]: + query: Dict[str, str] = {} + for name, value in params.items(): + if value is None: + continue + if isinstance(value, bool): + query[name] = "true" if value else "false" + elif isinstance(value, Enum): + query[name] = str(value.value) + else: + query[name] = str(value) + return query + + +def _error_message(response: requests.Response) -> str: + try: + body = response.json() + except ValueError: + body = None + if not isinstance(body, dict): + return response.text or f"HTTP {response.status_code}" + + message = body.get("message") or f"HTTP {response.status_code}" + if isinstance(message, list): + message = "; ".join(str(part) for part in message) + details = body.get("details") + if isinstance(details, list) and details: + problems = "; ".join( + f"{detail.get('parameter')} {detail.get('message')}" + for detail in details + if isinstance(detail, dict) + ) + message = f"{message}: {problems}" + return str(message) + + +def _raise_for_response(response: requests.Response) -> None: + message = _error_message(response) + status = response.status_code + if status == 400: + raise InvalidQueryError(message) + if status == 401: + raise AuthenticationError(message) + if status == 403: + raise AuthorizationError(message) + if status == 404: + raise ResourceNotFoundError(message=message) + if status == 429: + raise ApiLimitError(message) + if status >= 500: + raise InternalServerError(message) + raise LabelboxError(message) + + +def _request(client: "Client", path: str, params: Dict[str, str]) -> Any: + try: + response = client.connection.get( + f"{client.rest_endpoint}/performance{path}", + params=params, + timeout=_TIMEOUT_SECONDS, + ) + except requests.exceptions.Timeout as error: + raise TimeoutError(str(error)) + except requests.exceptions.RequestException as error: + raise NetworkError(error) + if response.status_code != requests.codes.ok: + _raise_for_response(response) + return response.json() + + +# Reading a report changes nothing, so a failure of the service or a timeout +# is tried again, with the growing pauses Client.execute retries a query with. +# The deadline is set here because the default (120 seconds) is already spent +# when a request of this length times out, and it would never be retried. +_retry_transient = retry.Retry( + predicate=retry.if_exception_type(InternalServerError, TimeoutError), + deadline=_RETRY_DEADLINE_SECONDS, +) + + +def _get(client: "Client", path: str, **params: Any) -> Dict[str, Any]: + try: + return _retry_transient(_request)(client, path, _to_query(params)) + except core_exceptions.RetryError as error: + if not isinstance(error.cause, LabelboxError): + raise + failure = error.cause + # Out of attempts: report what kept failing, not that retrying did. It is + # raised out here, not inside the handler, because the retry error already + # names the failure as its cause. Chaining the failure back to it would + # make the two point at each other, and some traceback printers follow + # such a chain forever. + raise failure + + +def _segment(value: Any) -> str: + """A value as one path segment, so an ID can never alter the path.""" + return quote(_to_id(value), safe="") + + +def _period(start_date: DateLike, end_date: DateLike) -> Dict[str, str]: + return {"startDate": _to_day(start_date), "endDate": _to_day(end_date)} + + +def get_performance_metrics(client: "Client") -> List[PerformanceMetric]: + body = _get(client, "/metrics") + return [PerformanceMetric.model_validate(metric) for metric in body["data"]] + + +def get_project_metric( + client: "Client", + project_id: str, + metric: str, + start_date: DateLike, + end_date: DateLike, + interval: Union[PerformanceInterval, str], + user_ids: Optional[Iterable[Any]], + user_group_ids: Optional[Iterable[Any]], + batch_ids: Optional[Iterable[Any]], + deleted_labels: Union[PerformanceDeletedLabels, str], +) -> PerformanceMetricSeries: + body = _get( + client, + f"/projects/{_segment(project_id)}/metrics/{_segment(metric)}", + **_period(start_date, end_date), + interval=interval, + userIds=_to_ids("user_ids", user_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), + batchIds=_to_ids("batch_ids", batch_ids), + deletedLabels=deleted_labels, + ) + return PerformanceMetricSeries.model_validate(body) + + +def _table( + client: "Client", + path: str, + row_type: Type[Row], + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]], + batch_ids: Optional[Iterable[Any]], + deleted_labels: Union[PerformanceDeletedLabels, str], + sort_by: Optional[str], + descending: bool, + page_size: int, + **extra: Any, +) -> PerformanceRows[Row]: + params = { + **_period(start_date, end_date), + "userIds": _to_ids("user_ids", user_ids), + "batchIds": _to_ids("batch_ids", batch_ids), + "deletedLabels": deleted_labels, + # Columns are named as the row's attributes; the API spells them in + # camel case. + "sort": _to_field(sort_by) if sort_by else None, + "order": ("desc" if descending else "asc") if sort_by else None, + "perPage": page_size, + **extra, + } + + def fetch_page(number: int) -> Dict[str, Any]: + return _get(client, path, page=number, **params) + + return PerformanceRows(fetch_page, row_type) + + +def get_project_labelers( + client: "Client", + project_id: str, + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = _DEFAULT_PAGE_SIZE, + use_cache: bool = True, +) -> PerformanceRows[ProjectLabelerPerformance]: + return _table( + client, + f"/projects/{_segment(project_id)}/labelers", + ProjectLabelerPerformance, + start_date, + end_date, + user_ids, + batch_ids, + deleted_labels, + sort_by, + descending, + page_size, + userGroupIds=_to_ids("user_group_ids", user_group_ids), + # Only sent to skip the cache, which is on unless asked otherwise. + useCache=None if use_cache else False, + ) + + +def get_project_reviewers( + client: "Client", + project_id: str, + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = _DEFAULT_PAGE_SIZE, +) -> PerformanceRows[ProjectReviewerPerformance]: + return _table( + client, + f"/projects/{_segment(project_id)}/reviewers", + ProjectReviewerPerformance, + start_date, + end_date, + user_ids, + batch_ids, + deleted_labels, + sort_by, + descending, + page_size, + userGroupIds=_to_ids("user_group_ids", user_group_ids), + ) + + +def get_project_report_download( + client: "Client", + project_id: str, + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, +) -> PerformanceReportDownload: + body = _get( + client, + f"/projects/{_segment(project_id)}/download", + **_period(start_date, end_date), + userIds=_to_ids("user_ids", user_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), + batchIds=_to_ids("batch_ids", batch_ids), + deletedLabels=deleted_labels, + ) + return PerformanceReportDownload.model_validate(body) + + +def get_workspace_metric( + client: "Client", + metric: str, + start_date: DateLike, + end_date: DateLike, + interval: Union[PerformanceInterval, str] = PerformanceInterval.DAY, + project_ids: Optional[Iterable[Any]] = None, + user_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, + use_cache: bool = True, + member_type: Optional[Union[PerformanceMemberType, str]] = None, + owner_organization_ids: Optional[Iterable[Any]] = None, +) -> PerformanceMetricSeries: + body = _get( + client, + f"/workspace/metrics/{_segment(metric)}", + **_period(start_date, end_date), + interval=interval, + projectIds=_to_ids("project_ids", project_ids), + userIds=_to_ids("user_ids", user_ids), + batchIds=_to_ids("batch_ids", batch_ids), + deletedLabels=deleted_labels, + memberType=member_type, + ownerOrganizationIds=_to_ids( + "owner_organization_ids", owner_organization_ids + ), + useCache=None if use_cache else False, + ) + return PerformanceMetricSeries.model_validate(body) + + +def get_workspace_labelers( + client: "Client", + start_date: DateLike, + end_date: DateLike, + project_ids: Optional[Iterable[Any]] = None, + user_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, + member_type: Optional[Union[PerformanceMemberType, str]] = None, + owner_organization_ids: Optional[Iterable[Any]] = None, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = _DEFAULT_PAGE_SIZE, + use_cache: bool = True, +) -> PerformanceRows[WorkspaceLabelerPerformance]: + return _table( + client, + "/workspace/labelers", + WorkspaceLabelerPerformance, + start_date, + end_date, + user_ids, + batch_ids, + deleted_labels, + sort_by, + descending, + page_size, + projectIds=_to_ids("project_ids", project_ids), + memberType=member_type, + ownerOrganizationIds=_to_ids( + "owner_organization_ids", owner_organization_ids + ), + useCache=None if use_cache else False, + ) diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index bd51e8b20..83eea92d3 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -53,6 +53,7 @@ LabelingService, LabelingServiceStatus, ) +from labelbox.schema import performance as _performance from labelbox.schema.labeling_service_dashboard import LabelingServiceDashboard from labelbox.schema.media_type import MediaType from labelbox.schema.model_config import ModelConfig @@ -1852,6 +1853,273 @@ def get_overview( return ProjectOverviewDetailed(**overview) + def get_performance_metric( + self, + metric: str, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + interval: Union[ + _performance.PerformanceInterval, str + ] = _performance.PerformanceInterval.DAY, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + ) -> _performance.PerformanceMetricSeries: + """Returns one of this project's performance metrics over a period. + + The values are the ones the chart on the project's Performance page + shows for the same dates and filters. + + Args: + metric (str): A metric name from + :meth:`labelbox.Client.get_performance_metrics`, for example + ``"labels_created"``. + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + interval (PerformanceInterval or str): The width of each bucket: + day (the default), week, month, quarter or year. + user_ids (Optional[List[str]]): Count only work by these users. + user_group_ids (Optional[List[str]]): Count only work by the + current members of these user groups. With ``user_ids`` too, + only users in both are counted. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + + Returns: + PerformanceMetricSeries: The metric as a value per interval, or + per score range for the two distribution metrics. + + Raises: + lbox.exceptions.ResourceNotFoundError: The metric does not exist, + or you cannot view this project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. + + Example: + >>> series = project.get_performance_metric( + >>> "labels_created", "2026-01-01", "2026-01-31") + >>> for bucket in series.buckets: + >>> print(bucket.start.date(), bucket.value) + """ + return _performance.get_project_metric( + self.client, + self.uid, + metric, + start_date, + end_date, + interval, + user_ids, + user_group_ids, + batch_ids, + deleted_labels, + ) + + def get_labeler_performance( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = 50, + use_cache: bool = True, + ) -> _performance.PerformanceRows[_performance.ProjectLabelerPerformance]: + """Returns per-labeler totals for this project over a period. + + This is the labeler table on the project's Performance page. It is + not the same report as :meth:`labeler_performance`, which has no + period and fewer figures. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + user_ids (Optional[List[str]]): Return only these users. + user_group_ids (Optional[List[str]]): Return only the current + members of these user groups. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + sort_by (Optional[str]): A figure of + :class:`ProjectLabelerPerformance` to sort on, for example + ``"labels_created"``. Sorting by email is not offered. + descending (bool): Sort from high to low. Only used with + ``sort_by``. + page_size (int): Rows fetched per request while iterating, up to + 100. + use_cache (bool): Pass False to recompute instead of using a + result cached within the last few minutes. + + Returns: + PerformanceRows[ProjectLabelerPerformance]: The rows, fetched a + page at a time as you iterate. Emails are masked for labelers you + may not identify. + + Raises: + lbox.exceptions.ResourceNotFoundError: You cannot view this + project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. + + Example: + >>> labelers = project.get_labeler_performance( + >>> "2026-01-01", "2026-01-31", + >>> sort_by="labels_created", descending=True) + >>> print(labelers.total) + >>> for labeler in labelers: + >>> print(labeler.email, labeler.labels_created) + """ + return _performance.get_project_labelers( + self.client, + self.uid, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + sort_by=sort_by, + descending=descending, + page_size=page_size, + use_cache=use_cache, + ) + + def get_reviewer_performance( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = 50, + ) -> _performance.PerformanceRows[_performance.ProjectReviewerPerformance]: + """Returns per-reviewer totals for this project over a period. + + This is the reviewer table on the project's Performance page. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + user_ids (Optional[List[str]]): Return only these users. + user_group_ids (Optional[List[str]]): Return only the current + members of these user groups. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + sort_by (Optional[str]): A figure of + :class:`ProjectReviewerPerformance` to sort on, for example + ``"labels_reviewed"``. Sorting by email is not offered. + descending (bool): Sort from high to low. Only used with + ``sort_by``. + page_size (int): Rows fetched per request while iterating, up to + 100. + + Returns: + PerformanceRows[ProjectReviewerPerformance]: The rows, fetched a + page at a time as you iterate. Emails are masked for reviewers + you may not identify. + + Raises: + lbox.exceptions.ResourceNotFoundError: You cannot view this + project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. + + Example: + >>> for reviewer in project.get_reviewer_performance( + >>> "2026-01-01", "2026-01-31"): + >>> print(reviewer.email, reviewer.labels_reviewed) + """ + return _performance.get_project_reviewers( + self.client, + self.uid, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + sort_by=sort_by, + descending=descending, + page_size=page_size, + ) + + def get_performance_report_download( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + ) -> _performance.PerformanceReportDownload: + """Returns a link to this project's performance report as a file. + + This is the file the Download button on the Performance page gives. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + user_ids (Optional[List[str]]): Include only these users. + user_group_ids (Optional[List[str]]): Include only the current + members of these user groups. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + + Returns: + PerformanceReportDownload: The link and how long it stays valid. + Anyone with the link can open the file until it expires. + + Raises: + lbox.exceptions.ResourceNotFoundError: You cannot view this + project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. + + Example: + >>> report = project.get_performance_report_download( + >>> "2026-01-01", "2026-01-31") + >>> print(report.url) + """ + return _performance.get_project_report_download( + self.client, + self.uid, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + ) + def clone(self) -> "Project": """ Clones the current project. diff --git a/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py b/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py index 9a286b55e..96dbe0556 100644 --- a/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py +++ b/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py @@ -1,3 +1,4 @@ +import time import uuid from labelbox import parser from labelbox.data.annotation_types.annotation import ObjectAnnotation @@ -25,7 +26,19 @@ """ +# Every test that creates an MEA prediction import is skipped: the backend +# rejects the createModelErrorAnalysisPredictionImport mutation, which is the +# only thing ModelRun.add_predictions() and MEAPredictionImport.create*() call. +# Tests that only read a model run still work and stay enabled. +mea_prediction_import_removed = pytest.mark.skip( + reason="MEA (Model Error Analysis) prediction import is no longer supported " + "by the backend: the prediction import service rejects every import of " + "that type, so ModelRun.add_predictions() fails with '9 FAILED_PRECONDITION: " + "MEA prediction import has been deprecated' (python-monorepo #2515)." +) + +@mea_prediction_import_removed @pytest.mark.order(1) def test_create_from_objects( model_run_with_data_rows, @@ -54,6 +67,7 @@ def test_create_from_objects( ) +@mea_prediction_import_removed def test_create_from_objects_global_key( client, model_run_with_data_rows, @@ -83,6 +97,7 @@ def test_create_from_objects_global_key( ) +@mea_prediction_import_removed def test_create_from_objects_with_confidence( predictions_with_confidence, model_run_with_data_rows, @@ -118,6 +133,7 @@ def test_create_from_objects_with_confidence( ) +@mea_prediction_import_removed def test_create_from_objects_all_project_labels( model_run_with_all_project_labels, object_predictions_for_annotation_import, @@ -147,24 +163,62 @@ def test_create_from_objects_all_project_labels( ) +def _export_model_run_labels( + model_run: ModelRun, expected_data_row_count: int, timeout_seconds=180 +): + """The first label of each data row of a model run, once the export has them. + + `upsert_labels` returns when the labels are attached to the model run, and + the export catches up a little later: until then a data row comes out with + no label for the run. So export again while any is missing, and fail if + they have not all arrived in time. + """ + deadline = time.time() + timeout_seconds + while True: + export_task = model_run.export() + export_task.wait_till_done() + labels_by_data_row = [ + ( + data_row.json["data_row"]["id"], + data_row.json["experiments"][model_run.model_id]["runs"][ + model_run.uid + ]["labels"], + ) + for data_row in export_task.get_buffered_stream() + ] + if len(labels_by_data_row) >= expected_data_row_count and all( + labels for _, labels in labels_by_data_row + ): + return [ + (data_row_id, labels[0]) + for data_row_id, labels in labels_by_data_row + ] + + missing = [ + data_row_id + for data_row_id, labels in labels_by_data_row + if not labels + ] + assert time.time() < deadline, ( + f"After {timeout_seconds}s the export of model run {model_run.uid} " + f"had {len(labels_by_data_row)} of {expected_data_row_count} data " + f"rows, {len(missing)} of them without a label" + ) + time.sleep(5) + + def test_model_run_project_labels( model_run_with_all_project_labels: ModelRun, model_run_predictions ): model_run = model_run_with_all_project_labels - export_task = model_run.export() - export_task.wait_till_done() - stream = export_task.get_buffered_stream() + expected_data_row_count = len( + set(prediction["dataRow"]["id"] for prediction in model_run_predictions) + ) # exports to list of tuples (data_row_id, label) needed to adapt test to export v2 instead of export v1 since data rows ids are not at label level in export v2. - model_run_exported_labels = [ - ( - data_row.json["data_row"]["id"], - data_row.json["experiments"][model_run.model_id]["runs"][ - model_run.uid - ]["labels"][0], - ) - for data_row in stream - ] + model_run_exported_labels = _export_model_run_labels( + model_run, expected_data_row_count + ) labels_indexed_by_name = {} @@ -196,6 +250,7 @@ def test_model_run_project_labels( ) +@mea_prediction_import_removed def test_create_from_label_objects( model_run_with_data_rows, object_predictions_for_annotation_import, @@ -266,6 +321,7 @@ def test_create_from_label_objects( ) +@mea_prediction_import_removed def test_create_from_local_file( tmp_path, model_run_with_data_rows, @@ -301,6 +357,7 @@ def test_create_from_local_file( ) +@mea_prediction_import_removed def test_predictions_with_custom_metrics( model_run, object_predictions_for_annotation_import, @@ -328,6 +385,7 @@ def test_predictions_with_custom_metrics( ) +@mea_prediction_import_removed def test_get(client, model_run_with_data_rows, annotation_import_test_helpers): name = str(uuid.uuid4()) url = "https://storage.googleapis.com/labelbox-public-bucket/predictions_test_v2.ndjson" @@ -344,6 +402,7 @@ def test_get(client, model_run_with_data_rows, annotation_import_test_helpers): annotation_import.wait_until_done() +@mea_prediction_import_removed @pytest.mark.slow def test_wait_till_done(model_run_predictions, model_run_with_data_rows): name = str(uuid.uuid4()) diff --git a/libs/labelbox/tests/embedding_cleanup.py b/libs/labelbox/tests/embedding_cleanup.py index eed2a0062..ab0da8c72 100644 --- a/libs/labelbox/tests/embedding_cleanup.py +++ b/libs/labelbox/tests/embedding_cleanup.py @@ -5,10 +5,15 @@ from random import uniform from typing import Any, Callable, Iterable, List, Optional -from lbox.exceptions import LabelboxError +from lbox.exceptions import LabelboxError, MalformedQueryException EMBEDDING_NAME_PREFIX_V2 = "sdk-int-ci-v2-" EMBEDDING_CAP_ERROR_SNIPPET = "Max limit of custom embeddings" +# Creating and deleting an embedding are GraphQL mutations, and the API +# reports the cap with the MALFORMED_REQUEST code. The SDK raises that as +# MalformedQueryException, which does not derive from LabelboxError, so +# catching LabelboxError alone lets the cap error through unhealed. +EMBEDDING_CAP_ERROR_TYPES = (LabelboxError, MalformedQueryException) EMBEDDING_STALE_TTL_SECONDS = 12 * 3600 LEGACY_NAME_RE = re.compile(r"^sdk-int-[0-9a-f]{32}$") @@ -40,7 +45,7 @@ def select_stale_embeddings(embeddings: Iterable[Any], now: float) -> List[Any]: return stale_embeddings -def is_embedding_cap_error(error: LabelboxError) -> bool: +def is_embedding_cap_error(error: Exception) -> bool: return EMBEDDING_CAP_ERROR_SNIPPET in str(error) @@ -59,7 +64,7 @@ def create_embedding_with_heal( for attempt in range(1, _MAX_CREATE_ATTEMPTS + 1): try: return create_embedding() - except LabelboxError as error: + except EMBEDDING_CAP_ERROR_TYPES as error: if not is_embedding_cap_error(error): raise @@ -77,8 +82,11 @@ def create_embedding_with_heal( for embedding in stale_embeddings: try: delete_embedding(embedding) - except LabelboxError: + except EMBEDDING_CAP_ERROR_TYPES: # Another worker may have deleted the same stale embedding. + # Several jobs hit the cap at once and sweep the same + # names, and the delete is a GraphQL mutation too, so it + # can fail with either kind of error. pass last_swept_count = len(stale_embeddings) diff --git a/libs/labelbox/tests/integration/test_client_errors.py b/libs/labelbox/tests/integration/test_client_errors.py index b5444eadf..f070387cb 100644 --- a/libs/labelbox/tests/integration/test_client_errors.py +++ b/libs/labelbox/tests/integration/test_client_errors.py @@ -9,6 +9,13 @@ import labelbox.client from labelbox.schema.media_type import MediaType +# Deployed API environments replace the message of every GraphQL validation +# error with this fixed string, so that schema details (field names, "Did you +# mean ..." hints) are not exposed to clients. The error code is unchanged. +# Only an API that does not hide schema details, such as one run locally, +# still returns the detailed message. +MASKED_VALIDATION_ERROR_MESSAGE = "GraphQL validation error" + def test_missing_api_key(): key = os.environ.get(lbox.request_client._LABELBOX_API_KEY, None) @@ -41,7 +48,10 @@ def test_syntax_error(client): def test_semantic_error(client): with pytest.raises(lbox.exceptions.InvalidQueryError) as excinfo: client.execute("query {bbb {id}}", check_naming=False) - assert excinfo.value.message.startswith('Cannot query field "bbb"') + message = excinfo.value.message + assert message == MASKED_VALIDATION_ERROR_MESSAGE or message.startswith( + 'Cannot query field "bbb"' + ) def test_timeout_error(client, project): @@ -59,11 +69,24 @@ def test_timeout_error(client, project): def test_query_complexity_error(client): - with pytest.raises(lbox.exceptions.ValidationFailedError) as excinfo: + # The complexity limit is a validation rule, so where validation messages + # are masked its "Query complexity limit exceeded" message is replaced as + # well. The client then has nothing left to tell it apart from any other + # validation failure and raises InvalidQueryError; ValidationFailedError is + # only reachable against an API that does not hide schema details. + with pytest.raises( + ( + lbox.exceptions.InvalidQueryError, + lbox.exceptions.ValidationFailedError, + ) + ) as excinfo: client.execute( "{projects {datasets {dataRows {labels {id}}}}}", check_naming=False ) - assert excinfo.value.message == "Query complexity limit exceeded" + if isinstance(excinfo.value, lbox.exceptions.ValidationFailedError): + assert excinfo.value.message == "Query complexity limit exceeded" + else: + assert excinfo.value.message == MASKED_VALIDATION_ERROR_MESSAGE def test_resource_not_found_error(client): diff --git a/libs/labelbox/tests/integration/test_data_rows.py b/libs/labelbox/tests/integration/test_data_rows.py index ba3c108d5..99cc051bb 100644 --- a/libs/labelbox/tests/integration/test_data_rows.py +++ b/libs/labelbox/tests/integration/test_data_rows.py @@ -1,5 +1,6 @@ import json import os +import time import uuid from datetime import datetime, timezone from tempfile import NamedTemporaryFile @@ -1034,6 +1035,38 @@ def _wait_for_task(task, timeout_seconds=600): ), f"Task {task.uid} did not complete within {timeout_seconds}s: {task.status}" +def _create_data_row_once_global_key_is_released( + dataset, data_row_payload, global_key, timeout_seconds=180 +): + """Create a data row with the global key of a data row deleted just before. + + The key is not free the moment `delete()` returns: the import looks global + keys up in the catalog search index, and the deleted data row leaves that + index a little later. Until it has, the import reports the key as a + duplicate. So repeat the import while that is the only thing it reports, + and fail on any other error or if the key is not released in time. + """ + still_held = f"Duplicate global key: '{global_key}'" + sleep_time = 5 + deadline = time.time() + timeout_seconds + while True: + task = dataset.create_data_rows([data_row_payload]) + _wait_for_task(task) + + failed_data_rows = task.failed_data_rows + if not failed_data_rows: + return task + + messages = [row["message"] for row in failed_data_rows] + assert messages == [still_held], f"Unexpected errors: {messages}" + if time.time() >= deadline: + raise AssertionError( + f"Global key '{global_key}' was still reported as a duplicate " + f"{timeout_seconds}s after its data row was deleted" + ) + time.sleep(sleep_time) + + def test_data_row_delete_and_create_with_same_global_key( client, dataset, sample_image ): @@ -1065,8 +1098,9 @@ def test_data_row_delete_and_create_with_same_global_key( client.get_data_row(new_data_row_id).delete() # should successfully insert new datarow now - task = dataset.create_data_rows([data_row_payload]) - _wait_for_task(task) + task = _create_data_row_once_global_key_is_released( + dataset, data_row_payload, global_key_1 + ) assert task.result[0]["global_key"] == global_key_1 diff --git a/libs/labelbox/tests/integration/test_label.py b/libs/labelbox/tests/integration/test_label.py index 54e929efc..8de6c5184 100644 --- a/libs/labelbox/tests/integration/test_label.py +++ b/libs/labelbox/tests/integration/test_label.py @@ -41,12 +41,15 @@ def test_label_update(configured_project_with_label): assert label.label == "something else" -def test_label_bulk_deletion(configured_project_with_label): +def test_label_bulk_deletion(configured_project_with_label, label_helpers): project, _, _, _ = configured_project_with_label for _ in range(2): # only run twice, already have one label in the fixture project.create_label() + # A finished label import is not listed by `project.labels()` right away + # (the fixture waits for its own label the same way), so wait for all three + label_helpers.wait_for_labels(project, 3) labels = project.labels() l1 = next(labels) l2 = next(labels) diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index a1bb23d52..dd31ed33e 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -95,18 +95,18 @@ def delete_tag(tag_id: str): assert tagA.uid is not None tags = org.get_resource_tags() - lenA = len(tags) - assert lenA > 0 + assert tagA.uid in {tag.uid for tag in tags} tagB = client.get_organization().create_resource_tag(tagB) assert tagB.text == textB assert "#" + tagB.color == colorB assert tagB.uid is not None + # Look for the two tags themselves, not for a longer list: other runs of + # this suite create and delete tags in the same organization meanwhile, so + # the number of tags can go either way between two readings. tags = client.get_organization().get_resource_tags() - lenB = len(tags) - assert lenB > 0 - assert lenB > lenA + assert {tagA.uid, tagB.uid} <= {tag.uid for tag in tags} project_resource_tag = client.get_project( p1.uid diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py new file mode 100644 index 000000000..3075957df --- /dev/null +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -0,0 +1,1299 @@ +from datetime import date, datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import requests +from google.api_core import retry +from lbox.exceptions import ( + ApiLimitError, + AuthenticationError, + AuthorizationError, + InternalServerError, + InvalidQueryError, + LabelboxError, + NetworkError, + ResourceNotFoundError, + TimeoutError, +) + +from labelbox import ( + Client, + PerformanceDeletedLabels, + PerformanceInterval, + PerformanceMemberType, + PerformanceScoreBucket, + PerformanceTimeBucket, + Project, +) +from labelbox.schema import performance +from labelbox.schema.user_group import UserGroup + +BASE = "https://api.labelbox.com/api/v1/performance" + +# The schedule the SDK ships with. The tests below swap in a quick one. +SHIPPED_RETRY = performance._retry_transient + +# Responses as the REST API returns them. +SERIES = { + "projectId": "project-1", + "metric": "labels_created", + "unit": "count", + "bucketType": "time", + "buckets": [ + { + "start": "2026-01-01T00:00:00.000Z", + "end": "2026-01-02T00:00:00.000Z", + "value": 4, + "numerator": None, + "denominator": None, + "modes": None, + } + ], + "interval": "day", + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "generatedAt": "2026-01-08T09:30:00.000Z", +} + + +def labeler(user_id): + return { + "userId": user_id, + "projectId": "project-1", + "email": f"{user_id}@example.com", + "labelsCreated": 12, + "labelsSkipped": 1, + "memberTotalTime": 3600, + "labelingTimeSubmitted": 3000, + "labelingTimeSkipped": 100, + "reviewsReceived": 4, + "reviewTimeAll": 300, + "reworkTimeAll": 200, + "avgTimePerLabel": 250, + "avgReviewTimeAll": 75, + "avgReworkTimeAll": 50, + "avgBenchmarkAgreement": 0.9, + "avgConsensusAgreement": 0.8, + "reworkPercentage": 0.1, + "approvalPercentage": 0.9, + "avgRating": 4.5, + "isRemoved": False, + } + + +def table(rows, total, page=1, cached_at=None): + return { + "projectId": "project-1", + "data": rows, + "page": page, + "perPage": 50, + "total": total, + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "cachedAt": cached_at, + "generatedAt": "2026-01-08T09:30:00.000Z", + } + + +REVIEWER = { + "userId": "user-9", + "projectId": "project-1", + "email": "user-9@example.com", + "labelsReviewed": 20, + "labelsReworked": 2, + "memberTotalTime": 1800, + "avgReviewTime": 60, + "avgReworkTime": 30, + "reviewTime": 1200, + "reworkTime": 600, + "reworkPercentage": 0.1, + "approvalPercentage": 0.9, + "isRemoved": False, +} + + +def workspace_labeler(user_id, project_id, **fields): + return { + **{k: v for k, v in labeler(user_id).items() if k != "avgRating"}, + "projectId": project_id, + "memberType": "internal", + "projectName": "Project one", + "projectOrganizationId": "org-1", + "projectOrganizationName": "Org one", + "isProjectDeleted": False, + **fields, + } + + +def workspace_table(rows, total, page=1, cached_at=None): + return { + "organizationId": "org-1", + "projectIds": None, + "ownerOrganizationIds": None, + "memberType": None, + "data": rows, + "page": page, + "perPage": 50, + "total": total, + "cachedAt": cached_at, + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "generatedAt": "2026-01-08T09:30:00.000Z", + } + + +def response(body, status=200): + reply = Mock() + reply.status_code = status + reply.json.return_value = body + reply.text = str(body) + return reply + + +@pytest.fixture(autouse=True) +def quick_retries(monkeypatch): + # The real schedule waits seconds between attempts, for up to two minutes. + monkeypatch.setattr( + performance, + "_retry_transient", + retry.Retry( + predicate=retry.if_exception_type( + InternalServerError, TimeoutError + ), + initial=0.001, + maximum=0.001, + deadline=0.05, + ), + ) + + +@pytest.fixture +def client(): + client = Client(api_key="api_key") + client.connection.get = Mock(return_value=response(SERIES)) + return client + + +@pytest.fixture +def project(client): + # The methods only read the project's client and ID. + return SimpleNamespace(client=client, uid="project-1") + + +def sent(client, call=-1): + """The URL and query string of a request the client made.""" + args, kwargs = client.connection.get.call_args_list[call] + return args[0], kwargs["params"] + + +class TestMetricCatalog: + def test_lists_the_metrics(self, client): + client.connection.get.return_value = response( + { + "data": [ + { + "name": "labels_created", + "category": "throughput", + "unit": "count", + "buckets": "time", + "description": "Labels created in the period.", + } + ] + } + ) + + metrics = client.get_performance_metrics() + + assert sent(client) == (f"{BASE}/metrics", {}) + assert [metric.name for metric in metrics] == ["labels_created"] + assert metrics[0].unit == "count" + assert metrics[0].description == "Labels created in the period." + + +class TestProjectMetric: + def test_asks_for_the_metric_over_the_period(self, client, project): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/metrics/labels_created" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "interval": "day", + "deletedLabels": "include", + } + + def test_returns_a_value_per_interval(self, client, project): + series = Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert series.metric == "labels_created" + assert series.unit == "count" + assert series.bucket_type == "time" + assert series.project_id == "project-1" + assert series.start_date == date(2026, 1, 1) + assert series.end_date == date(2026, 1, 7) + assert series.generated_at == datetime( + 2026, 1, 8, 9, 30, tzinfo=timezone.utc + ) + (bucket,) = series.buckets + assert isinstance(bucket, PerformanceTimeBucket) + assert bucket.start == datetime(2026, 1, 1, tzinfo=timezone.utc) + assert bucket.end == datetime(2026, 1, 2, tzinfo=timezone.utc) + assert bucket.value == 4 + assert bucket.modes is None + + def test_returns_score_ranges_for_a_distribution(self, client, project): + client.connection.get.return_value = response( + { + **SERIES, + "metric": "consensus_distribution", + "bucketType": "score", + "buckets": [ + {"rangeStart": 0, "rangeEnd": 10, "value": 3}, + {"rangeStart": 10, "rangeEnd": 20, "value": 9}, + ], + } + ) + + series = Project.get_performance_metric( + project, "consensus_distribution", "2026-01-01", "2026-01-07" + ) + + assert series.bucket_type == "score" + assert all( + isinstance(bucket, PerformanceScoreBucket) + for bucket in series.buckets + ) + assert [ + (b.range_start, b.range_end, b.value) for b in series.buckets + ] == [ + (0, 10, 3), + (10, 20, 9), + ] + + def test_carries_the_ratio_and_the_split_by_activity(self, client, project): + client.connection.get.return_value = response( + { + **SERIES, + "metric": "aht_done_labels", + "unit": "seconds", + "buckets": [ + { + "start": "2026-01-01T00:00:00.000Z", + "end": "2026-01-02T00:00:00.000Z", + "value": 90, + "numerator": 900, + "denominator": 10, + "modes": { + "creating": 60, + "reviewing": 20, + "reworking": 10, + }, + } + ], + } + ) + + (bucket,) = Project.get_performance_metric( + project, "aht_done_labels", "2026-01-01", "2026-01-07" + ).buckets + + assert (bucket.value, bucket.numerator, bucket.denominator) == ( + 90, + 900, + 10, + ) + assert bucket.modes.creating == 60 + assert bucket.modes.reviewing == 20 + assert bucket.modes.reworking == 10 + + def test_sends_the_filters(self, client, project): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + interval=PerformanceInterval.WEEK, + user_ids=["user-1", "user-2"], + user_group_ids=["group-a"], + batch_ids=["batch-1"], + deleted_labels=PerformanceDeletedLabels.EXCLUDE, + ) + + _, params = sent(client) + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "interval": "week", + "userIds": "user-1,user-2", + "userGroupIds": "group-a", + "batchIds": "batch-1", + "deletedLabels": "exclude", + } + + def test_takes_plain_strings_in_place_of_the_enums(self, client, project): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + interval="month", + deleted_labels="only", + ) + + _, params = sent(client) + assert params["interval"] == "month" + assert params["deletedLabels"] == "only" + + def test_accepts_sdk_objects_where_ids_are_expected(self, client, project): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_ids=[SimpleNamespace(uid="user-1"), "user-2"], + batch_ids="batch-1", + ) + + _, params = sent(client) + assert params["userIds"] == "user-1,user-2" + assert params["batchIds"] == "batch-1" + + def test_accepts_user_group_objects(self, client, project): + # UserGroup is a pydantic model whose ID is `id`, not `uid`. + groups = [ + UserGroup(client=client, id="group-a", name="Vendor A"), + UserGroup(client=client, id="group-b", name="Vendor B"), + ] + + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_group_ids=groups, + ) + + _, params = sent(client) + assert params["userGroupIds"] == "group-a,group-b" + + @pytest.mark.parametrize( + "not_an_id", + [42, None, SimpleNamespace(name="no id"), SimpleNamespace(id="")], + ) + def test_refuses_a_filter_value_that_has_no_id( + self, client, project, not_an_id + ): + with pytest.raises(TypeError, match="Expected an ID"): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_ids=["user-1", not_an_id], + ) + + client.connection.get.assert_not_called() + + def test_leaves_a_filter_out_when_it_is_not_given(self, client, project): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_ids=None, + ) + + _, params = sent(client) + assert "userIds" not in params + + @pytest.mark.parametrize( + "argument", + ["user_ids", "user_group_ids", "batch_ids"], + ) + @pytest.mark.parametrize("empty", [[], (), set(), iter([])]) + def test_refuses_an_empty_filter_instead_of_reporting_on_everyone( + self, client, project, argument, empty + ): + with pytest.raises(ValueError, match=f"{argument} is empty"): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + **{argument: empty}, + ) + + client.connection.get.assert_not_called() + + @pytest.mark.parametrize( + "argument", ["project_ids", "owner_organization_ids", "user_ids"] + ) + def test_refuses_an_empty_workspace_filter_too(self, client, argument): + with pytest.raises(ValueError, match=f"{argument} is empty"): + client.get_workspace_performance_metric( + "labels_created", "2026-01-01", "2026-01-07", **{argument: []} + ) + with pytest.raises(ValueError, match=f"{argument} is empty"): + client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07", **{argument: []} + ) + + client.connection.get.assert_not_called() + + @pytest.mark.parametrize( + "day, expected", + [ + ("2026-01-31", "2026-01-31"), + (date(2026, 1, 31), "2026-01-31"), + (datetime(2026, 1, 31, 23, 30), "2026-01-31"), + # Reports follow UTC days: 23:30 at UTC-8 is already 1 February. + ( + datetime( + 2026, 1, 31, 23, 30, tzinfo=timezone(timedelta(hours=-8)) + ), + "2026-02-01", + ), + ], + ) + def test_sends_a_date_as_a_utc_day(self, client, project, day, expected): + Project.get_performance_metric(project, "labels_created", day, day) + + _, params = sent(client) + assert params["startDate"] == expected + assert params["endDate"] == expected + + def test_keeps_an_id_from_changing_the_path(self, client, project): + Project.get_performance_metric( + project, "../../labelers", "2026-01-01", "2026-01-07" + ) + + url, _ = sent(client) + assert url == f"{BASE}/projects/project-1/metrics/..%2F..%2Flabelers" + + def test_gives_the_request_a_timeout(self, client, project): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_args.kwargs["timeout"] > 0 + + def test_ignores_fields_added_to_the_response_later(self, client, project): + client.connection.get.return_value = response( + {**SERIES, "asOf": "2026-01-08T09:00:00.000Z"} + ) + + series = Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert series.metric == "labels_created" + + +class TestProjectLabelers: + def test_asks_for_nothing_until_the_rows_are_read(self, client, project): + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + + client.connection.get.assert_not_called() + + def test_returns_the_rows(self, client, project): + client.connection.get.return_value = response( + table([labeler("user-1"), labeler("user-2")], total=2) + ) + + rows = list( + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/labelers" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "deletedLabels": "include", + "perPage": "50", + "page": "1", + } + assert [row.user_id for row in rows] == ["user-1", "user-2"] + first = rows[0] + assert first.email == "user-1@example.com" + assert first.labels_created == 12 + assert first.member_total_time == 3600 + assert first.avg_consensus_agreement == 0.8 + assert first.avg_rating == 4.5 + assert first.is_removed is False + + def test_fetches_page_after_page_until_it_has_every_row( + self, client, project + ): + client.connection.get.side_effect = [ + response(table([labeler("user-1"), labeler("user-2")], total=5)), + response( + table([labeler("user-3"), labeler("user-4")], total=5, page=2) + ), + response(table([labeler("user-5")], total=5, page=3)), + ] + + rows = list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", page_size=2 + ) + ) + + assert [row.user_id for row in rows] == [ + "user-1", + "user-2", + "user-3", + "user-4", + "user-5", + ] + pages = [ + sent(client, call)[1]["page"] + for call in range(client.connection.get.call_count) + ] + assert pages == ["1", "2", "3"] + assert sent(client)[1]["perPage"] == "2" + + def test_stops_if_a_page_comes_back_empty(self, client, project): + # The table shrank while it was being read. + client.connection.get.side_effect = [ + response(table([labeler("user-1")], total=3)), + response(table([], total=3, page=2)), + ] + + rows = list( + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + ) + + assert [row.user_id for row in rows] == ["user-1"] + assert client.connection.get.call_count == 2 + + def test_reads_the_total_and_cache_time_from_the_first_page_once( + self, client, project + ): + client.connection.get.return_value = response( + table( + [labeler("user-1")], + total=1, + cached_at="2026-01-08T09:00:00.000Z", + ) + ) + labelers = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + + assert labelers.total == 1 + assert labelers.cached_at == datetime( + 2026, 1, 8, 9, 0, tzinfo=timezone.utc + ) + assert [row.user_id for row in labelers] == ["user-1"] + assert client.connection.get.call_count == 1 + + def test_has_no_cache_time_for_rows_computed_on_request( + self, client, project + ): + client.connection.get.return_value = response(table([], total=0)) + + labelers = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + + assert labelers.cached_at is None + assert list(labelers) == [] + + def test_sorts_by_an_attribute_of_the_row(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, + "2026-01-01", + "2026-01-07", + sort_by="labels_created", + descending=True, + ) + ) + + _, params = sent(client) + assert params["sort"] == "labelsCreated" + assert params["order"] == "desc" + + @pytest.mark.parametrize( + "sort_by, sent_as", + [ + ("labels_created", "labelsCreated"), + ("avg_time_per_label", "avgTimePerLabel"), + ("rework_percentage", "reworkPercentage"), + # Already in the API's spelling: passed on as it is. + ("labelsCreated", "labelsCreated"), + ("avgTimePerLabel", "avgTimePerLabel"), + ], + ) + def test_spells_the_sort_column_as_the_api_does( + self, client, project, sort_by, sent_as + ): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", sort_by=sort_by + ) + ) + + assert sent(client)[1]["sort"] == sent_as + + def test_reads_the_table_again_on_a_second_pass(self, client, project): + first = [ + response(table([labeler("user-1")], total=2)), + response(table([labeler("user-2")], total=2, page=2)), + ] + # By the second pass a row has been added, in front of the others. + second = [ + response(table([labeler("user-0")], total=3)), + response(table([labeler("user-1")], total=3, page=2)), + response(table([labeler("user-2")], total=3, page=3)), + ] + client.connection.get.side_effect = first + second + + rows = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", page_size=1 + ) + once = [row.user_id for row in rows] + twice = [row.user_id for row in rows] + + assert once == ["user-1", "user-2"] + # Not page 1 as it was, followed by pages 2 and 3 as they are now. + assert twice == ["user-0", "user-1", "user-2"] + assert rows.total == 3 + assert [sent(client, call)[1]["page"] for call in range(5)] == [ + "1", + "2", + "1", + "2", + "3", + ] + + def test_reads_the_total_then_the_rows_with_one_request_for_the_first_page( + self, client, project + ): + client.connection.get.return_value = response( + table([labeler("user-1")], total=1) + ) + + rows = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + assert rows.total == 1 + assert [row.user_id for row in rows] == ["user-1"] + assert rows.total == 1 + + assert client.connection.get.call_count == 1 + + def test_sends_no_order_without_a_sort(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", descending=True + ) + ) + + _, params = sent(client) + assert "sort" not in params + assert "order" not in params + + def test_can_skip_the_cache(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", use_cache=False + ) + ) + + _, params = sent(client) + assert params["useCache"] == "false" + + def test_filters_by_user_group(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, + "2026-01-01", + "2026-01-07", + user_group_ids=["group-a", "group-b"], + ) + ) + + _, params = sent(client) + assert params["userGroupIds"] == "group-a,group-b" + + +class TestProjectReviewers: + def test_returns_the_rows(self, client, project): + client.connection.get.return_value = response( + table( + [ + { + "userId": "user-9", + "projectId": "project-1", + "email": "usr.email.user-9@internal.labelbox.com", + "labelsReviewed": 20, + "labelsReworked": 2, + "memberTotalTime": 1800, + "avgReviewTime": 60, + "avgReworkTime": 30, + "reviewTime": 1200, + "reworkTime": 600, + "reworkPercentage": 0.1, + "approvalPercentage": 0.9, + "isRemoved": True, + } + ], + total=1, + ) + ) + + (reviewer,) = list( + Project.get_reviewer_performance( + project, + "2026-01-01", + "2026-01-07", + sort_by="labels_reviewed", + ) + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/reviewers" + assert params["sort"] == "labelsReviewed" + assert params["order"] == "asc" + assert "useCache" not in params + assert reviewer.labels_reviewed == 20 + assert reviewer.review_time == 1200 + assert reviewer.is_removed is True + # Masked emails come through as the API returns them. + assert reviewer.email == "usr.email.user-9@internal.labelbox.com" + + +class TestProjectReportDownload: + def test_returns_the_link(self, client, project): + client.connection.get.return_value = response( + { + "projectId": "project-1", + "url": "https://files.example/report.csv", + "expiresInSeconds": 86400, + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "generatedAt": "2026-01-08T09:30:00.000Z", + } + ) + + report = Project.get_performance_report_download( + project, "2026-01-01", "2026-01-07", user_group_ids=["group-a"] + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/download" + assert params["userGroupIds"] == "group-a" + assert report.url == "https://files.example/report.csv" + assert report.expires_in_seconds == 86400 + assert report.start_date == date(2026, 1, 1) + + +class TestWorkspace: + def test_returns_a_metric_across_the_workspace(self, client): + client.connection.get.return_value = response( + { + **{k: v for k, v in SERIES.items() if k != "projectId"}, + "organizationId": "org-1", + "projectIds": ["project-1", "project-2"], + } + ) + + series = client.get_workspace_performance_metric( + "labels_created", + date(2026, 1, 1), + date(2026, 1, 7), + project_ids=["project-1", "project-2"], + use_cache=False, + ) + + url, params = sent(client) + assert url == f"{BASE}/workspace/metrics/labels_created" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "interval": "day", + "projectIds": "project-1,project-2", + "deletedLabels": "include", + "useCache": "false", + } + assert series.organization_id == "org-1" + assert series.project_ids == ["project-1", "project-2"] + assert series.project_id is None + + def test_narrows_a_metric_by_member_type_and_owning_organization( + self, client + ): + client.connection.get.return_value = response( + { + **{k: v for k, v in SERIES.items() if k != "projectId"}, + "organizationId": "org-1", + "projectIds": None, + "ownerOrganizationIds": ["org-2"], + "memberType": "external", + "skippedProjectIds": ["project-9"], + } + ) + + series = client.get_workspace_performance_metric( + "labels_created", + "2026-01-01", + "2026-01-07", + member_type=PerformanceMemberType.EXTERNAL, + owner_organization_ids=["org-2"], + ) + + _, params = sent(client) + assert params["memberType"] == "external" + assert params["ownerOrganizationIds"] == "org-2" + assert series.member_type == "external" + assert series.owner_organization_ids == ["org-2"] + assert series.skipped_project_ids == ["project-9"] + + def test_sends_neither_narrowing_unless_asked(self, client): + client.get_workspace_performance_metric( + "labels_created", "2026-01-01", "2026-01-07" + ) + + _, params = sent(client) + assert "memberType" not in params + assert "ownerOrganizationIds" not in params + + def test_returns_a_row_per_labeler_and_project(self, client): + client.connection.get.return_value = response( + workspace_table( + [ + workspace_labeler("user-1", "project-1"), + workspace_labeler( + "user-2", + "project-shared", + memberType="external", + projectOrganizationId="org-2", + projectOrganizationName="Org two", + ), + ], + total=2, + ) + ) + + rows = list( + client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07", user_ids=["user-1", "user-2"] + ) + ) + + url, params = sent(client) + assert url == f"{BASE}/workspace/labelers" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "userIds": "user-1,user-2", + "deletedLabels": "include", + "perPage": "50", + "page": "1", + } + assert [row.project_id for row in rows] == [ + "project-1", + "project-shared", + ] + assert rows[0].project_name == "Project one" + assert rows[0].labels_created == 12 + # Whose project it is, and whose member did the work. + assert rows[0].project_organization_name == "Org one" + assert rows[0].member_type == "internal" + assert rows[1].project_organization_id == "org-2" + assert rows[1].member_type == "external" + + def test_fetches_the_workspace_table_a_page_at_a_time(self, client): + client.connection.get.side_effect = [ + response( + workspace_table( + [ + workspace_labeler("user-1", "project-1"), + workspace_labeler("user-2", "project-1"), + ], + total=3, + cached_at="2026-01-08T09:29:00.000Z", + ) + ), + response( + workspace_table( + [workspace_labeler("user-3", "project-1")], + total=3, + page=2, + ) + ), + ] + + rows = client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07", page_size=2 + ) + + assert rows.total == 3 + assert rows.cached_at == datetime( + 2026, 1, 8, 9, 29, tzinfo=timezone.utc + ) + assert [row.user_id for row in rows] == ["user-1", "user-2", "user-3"] + assert [sent(client, call)[1]["page"] for call in (0, 1)] == ["1", "2"] + assert sent(client)[1]["perPage"] == "2" + + def test_filters_and_sorts_the_workspace_table(self, client): + client.connection.get.return_value = response(workspace_table([], 0)) + + list( + client.get_workspace_labeler_performance( + "2026-01-01", + "2026-01-07", + project_ids=["project-1"], + member_type="external", + owner_organization_ids=["org-2", "org-3"], + sort_by="avg_time_per_label", + descending=True, + use_cache=False, + ) + ) + + _, params = sent(client) + assert params["projectIds"] == "project-1" + assert params["memberType"] == "external" + assert params["ownerOrganizationIds"] == "org-2,org-3" + assert params["sort"] == "avgTimePerLabel" + assert params["order"] == "desc" + assert params["useCache"] == "false" + + def test_reads_a_row_whose_project_can_no_longer_be_found(self, client): + client.connection.get.return_value = response( + workspace_table( + [ + workspace_labeler( + "user-1", + "project-gone", + projectName=None, + projectOrganizationId=None, + projectOrganizationName=None, + isProjectDeleted=True, + ) + ], + 1, + ) + ) + + (row,) = client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07" + ) + + assert row.project_id == "project-gone" + assert row.project_name is None + assert row.project_organization_id is None + assert row.project_organization_name is None + assert row.is_project_deleted is True + assert row.labels_created == 12 + + def test_reads_a_row_whose_member_no_longer_exists(self, client): + client.connection.get.return_value = response( + workspace_table( + [workspace_labeler("user-1", "project-1", memberType=None)], 1 + ) + ) + + (row,) = client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07" + ) + + assert row.member_type is None + + +class TestMemberType: + def test_says_whose_member_each_labeler_and_reviewer_is( + self, client, project + ): + client.connection.get.side_effect = [ + response( + table( + [ + {**labeler("user-1"), "memberType": "internal"}, + {**labeler("user-2"), "memberType": "external"}, + ], + total=2, + ) + ), + response(table([{**REVIEWER, "memberType": "external"}], total=1)), + ] + + labelers = list( + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + ) + reviewers = list( + Project.get_reviewer_performance( + project, "2026-01-01", "2026-01-07" + ) + ) + + assert [row.member_type for row in labelers] == [ + "internal", + "external", + ] + assert reviewers[0].member_type == PerformanceMemberType.EXTERNAL + + def test_reads_a_reply_from_an_api_that_does_not_say(self, client, project): + client.connection.get.return_value = response( + table([labeler("user-1")], total=1) + ) + + (row,) = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + + assert row.member_type is None + + +class TestRetries: + def test_tries_again_after_a_failure_of_the_service(self, client, project): + client.connection.get.side_effect = [ + response({"statusCode": 503, "message": "Unavailable"}, 503), + response({"statusCode": 500, "message": "Oops"}, 500), + response(SERIES), + ] + + series = Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert series.metric == "labels_created" + assert client.connection.get.call_count == 3 + + def test_tries_again_after_a_timeout(self, client, project): + client.connection.get.side_effect = [ + requests.exceptions.ReadTimeout("read timed out"), + response(SERIES), + ] + + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count == 2 + + def test_tries_a_failed_page_again_without_repeating_rows( + self, client, project + ): + client.connection.get.side_effect = [ + response(table([labeler("user-1")], total=2)), + response({"statusCode": 502, "message": "Bad gateway"}, 502), + response(table([labeler("user-2")], total=2, page=2)), + ] + + rows = list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", page_size=1 + ) + ) + + assert [row.user_id for row in rows] == ["user-1", "user-2"] + assert [sent(client, call)[1]["page"] for call in (0, 1, 2)] == [ + "1", + "2", + "2", + ] + + @pytest.mark.parametrize("status", [400, 401, 403, 404, 429]) + def test_does_not_repeat_a_request_that_was_refused( + self, client, project, status + ): + client.connection.get.return_value = response( + {"statusCode": status, "message": "No"}, status + ) + + with pytest.raises(LabelboxError): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count == 1 + + def test_reports_the_failure_itself_once_it_stops_trying( + self, client, project + ): + client.connection.get.return_value = response( + {"statusCode": 503, "message": "Under maintenance"}, 503 + ) + + with pytest.raises(InternalServerError, match="Under maintenance"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count > 1 + + @pytest.mark.parametrize( + "failure", + [ + response({"statusCode": 500, "message": "Oops"}, 500), + requests.exceptions.ReadTimeout("read timed out"), + ], + ) + def test_raises_an_error_whose_causes_do_not_lead_back_to_it( + self, client, project, failure + ): + # A chain that loops makes some traceback printers run forever. + if isinstance(failure, Exception): + client.connection.get.side_effect = failure + else: + client.connection.get.return_value = failure + + with pytest.raises(LabelboxError) as raised: + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + seen = [] + error = raised.value + while error is not None: + assert not any(error is earlier for earlier in seen) + seen.append(error) + error = error.__cause__ or error.__context__ + assert len(seen) < 5 + + def test_has_time_left_to_try_again_after_a_request_that_timed_out( + self, + ): + # A retry starts only if the deadline, counted from the first attempt, + # has not passed after the failed attempt and the pause that follows. + one_attempt = performance._TIMEOUT_SECONDS + longest_first_pause = 2 # seconds, across google-api-core versions + + assert SHIPPED_RETRY.deadline >= one_attempt + longest_first_pause + # ...and it does not start a third attempt after two timeouts. + assert SHIPPED_RETRY.deadline <= 2 * one_attempt + + def test_reports_a_timeout_as_the_sdk_does(self, client, project): + client.connection.get.side_effect = requests.exceptions.ReadTimeout( + "read timed out" + ) + + with pytest.raises(TimeoutError, match="read timed out"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + def test_reports_a_connection_failure_without_repeating_it( + self, client, project + ): + client.connection.get.side_effect = requests.exceptions.ConnectionError( + "connection refused" + ) + + with pytest.raises(NetworkError): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count == 1 + + +class TestErrors: + @pytest.mark.parametrize( + "status, error", + [ + (400, InvalidQueryError), + (401, AuthenticationError), + (403, AuthorizationError), + (404, ResourceNotFoundError), + (429, ApiLimitError), + (500, InternalServerError), + (503, InternalServerError), + (418, LabelboxError), + ], + ) + def test_raises_the_sdk_error_for_the_status( + self, client, project, status, error + ): + client.connection.get.return_value = response( + {"statusCode": status, "message": "Something went wrong"}, status + ) + + with pytest.raises(error, match="Something went wrong"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + def test_names_the_parameters_the_api_refused(self, client, project): + client.connection.get.return_value = response( + { + "statusCode": 400, + "error": "Bad Request", + "message": "Invalid query parameters", + "details": [ + { + "parameter": "endDate", + "message": "must not be before startDate", + }, + { + "parameter": "interval", + "message": "must be one of day, week, month, quarter, year", + }, + ], + }, + 400, + ) + + with pytest.raises(InvalidQueryError) as raised: + Project.get_performance_metric( + project, + "labels_created", + "2026-01-07", + "2026-01-01", + interval="hour", + ) + + message = str(raised.value) + assert "endDate must not be before startDate" in message + assert "interval must be one of day, week" in message + + def test_says_which_project_was_not_found(self, client, project): + client.connection.get.return_value = response( + { + "statusCode": 404, + "message": "Project project-1 was not found.", + "error": "Not Found", + }, + 404, + ) + + with pytest.raises( + ResourceNotFoundError, match="Project project-1 was not found" + ): + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + ) + + def test_reports_a_reply_that_is_not_json(self, client, project): + reply = Mock() + reply.status_code = 502 + reply.json.side_effect = ValueError("not json") + reply.text = "Bad gateway" + client.connection.get.return_value = reply + + with pytest.raises(InternalServerError, match="Bad gateway"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) diff --git a/libs/labelbox/tests/unit/test_embedding_cleanup.py b/libs/labelbox/tests/unit/test_embedding_cleanup.py index 882648d7a..cb179404b 100644 --- a/libs/labelbox/tests/unit/test_embedding_cleanup.py +++ b/libs/labelbox/tests/unit/test_embedding_cleanup.py @@ -2,8 +2,9 @@ from unittest.mock import Mock, call, patch import pytest -from lbox.exceptions import LabelboxError +from lbox.exceptions import LabelboxError, MalformedQueryException +from labelbox import Client from tests.embedding_cleanup import ( EMBEDDING_CAP_ERROR_SNIPPET, EMBEDDING_NAME_PREFIX_V2, @@ -42,8 +43,21 @@ def _v2_name(created_at: int, suffix: str = "0123456789") -> str: return f"{EMBEDDING_NAME_PREFIX_V2}{created_at}-{suffix}" -def _cap_error() -> LabelboxError: - return LabelboxError(f"{EMBEDDING_CAP_ERROR_SNIPPET}: 10") +# The message the API answers with when the organization is at the cap. +CAP_MESSAGE = f"{EMBEDDING_CAP_ERROR_SNIPPET} reached : 10 per organization" + + +def _cap_error() -> MalformedQueryException: + # The API reports the cap with the MALFORMED_REQUEST code, which the SDK + # raises as MalformedQueryException: not a LabelboxError. + return MalformedQueryException(CAP_MESSAGE) + + +def _graphql_response(body: dict) -> Mock: + response = Mock() + response.status_code = 200 + response.json.return_value = body + return response def test_build_and_parse_embedding_name_round_trip(): @@ -96,7 +110,11 @@ def test_select_stale_embeddings_keeps_only_expired_custom_v2_names(): def test_embedding_cap_error_matcher_is_fail_closed(): assert is_embedding_cap_error(_cap_error()) + assert is_embedding_cap_error(LabelboxError(CAP_MESSAGE)) assert not is_embedding_cap_error(LabelboxError("permission denied")) + assert not is_embedding_cap_error( + MalformedQueryException("Name contains invalid characters") + ) def test_workflow_boolean_input_is_parsed_defensively(): @@ -140,6 +158,56 @@ def test_heal_retries_twice_then_creates_and_deletes_only_stale_v2(): assert sleep.call_args_list == [call(4), call(4)] +def test_heal_sweeps_on_the_cap_error_client_create_embedding_raises(): + # Goes through Client.execute, so the heal is checked against the + # exception the SDK raises for the API's answer, not against a stand-in. + now_value = 1_725_000_000 + stale = _embedding( + "stale", + _v2_name(now_value - EMBEDDING_STALE_TTL_SECONDS - 1), + ) + delete = Mock() + client = Client(api_key="api_key") + responses = [ + _graphql_response( + { + "errors": [ + { + "message": CAP_MESSAGE, + "extensions": {"code": "MALFORMED_REQUEST"}, + } + ] + } + ), + _graphql_response( + { + "data": { + "createEmbedding": { + "id": "created", + "name": "custom", + "dims": 8, + "custom": True, + } + } + } + ), + ] + + with patch.object(client.connection, "send", side_effect=responses): + embedding = create_embedding_with_heal( + create_embedding=lambda: client.create_embedding("custom", 8), + list_embeddings=Mock(return_value=[stale]), + delete_embedding=delete, + sleep=Mock(), + now=Mock(return_value=now_value), + retry_delay=Mock(return_value=4), + print_fn=Mock(), + ) + + assert embedding.id == "created" + delete.assert_called_once_with(stale) + + def test_heal_third_cap_error_is_terminal_without_another_sweep_or_sleep(): final_error = _cap_error() create = Mock(side_effect=[_cap_error(), _cap_error(), final_error]) @@ -148,7 +216,7 @@ def test_heal_third_cap_error_is_terminal_without_another_sweep_or_sleep(): sleep = Mock() print_fn = Mock() - with pytest.raises(LabelboxError) as raised: + with pytest.raises(MalformedQueryException) as raised: create_embedding_with_heal( create_embedding=create, list_embeddings=list_embeddings, @@ -170,14 +238,21 @@ def test_heal_third_cap_error_is_terminal_without_another_sweep_or_sleep(): ) -def test_heal_non_cap_error_is_reraised_without_side_effects(): - non_cap_error = LabelboxError("permission denied") +@pytest.mark.parametrize( + "non_cap_error", + [ + LabelboxError("permission denied"), + MalformedQueryException("Name contains invalid characters"), + ], + ids=["LabelboxError", "MalformedQueryException"], +) +def test_heal_non_cap_error_is_reraised_without_side_effects(non_cap_error): create = Mock(side_effect=non_cap_error) list_embeddings = Mock() delete = Mock() sleep = Mock() - with pytest.raises(LabelboxError) as raised: + with pytest.raises(type(non_cap_error)) as raised: create_embedding_with_heal( create_embedding=create, list_embeddings=list_embeddings, @@ -257,3 +332,38 @@ def test_cleanup_real_run_attempts_every_candidate_and_reports_failures(): assert run_cleanup(client, dry_run=False, now=1_725_000_000) == 1 first.delete.assert_called_once_with() second.delete.assert_called_once_with() + + +@pytest.mark.parametrize( + "already_gone", + [ + MalformedQueryException("Embedding not found"), + LabelboxError("Embedding not found"), + ], +) +def test_heal_carries_on_when_a_stale_embedding_is_already_gone(already_gone): + # Jobs for several Python versions hit the cap together and sweep the same + # stale names, so a delete can find its embedding already deleted. The + # delete is a GraphQL mutation, so it can fail with either kind of error. + now_value = 1_725_000_000 + gone = _embedding( + "gone", _v2_name(now_value - EMBEDDING_STALE_TTL_SECONDS - 2) + ) + stale = _embedding( + "stale", _v2_name(now_value - EMBEDDING_STALE_TTL_SECONDS - 1) + ) + created = object() + delete = Mock(side_effect=[already_gone, None]) + + result = create_embedding_with_heal( + create_embedding=Mock(side_effect=[_cap_error(), created]), + list_embeddings=Mock(return_value=[gone, stale]), + delete_embedding=delete, + sleep=Mock(), + now=Mock(return_value=now_value), + retry_delay=Mock(return_value=4), + print_fn=Mock(), + ) + + assert result is created + assert delete.call_args_list == [call(gone), call(stale)]