From 5cb4692886d494ec12626d77283305f03c7a678e Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sat, 10 Oct 2026 02:14:03 -0700 Subject: [PATCH 01/13] Add performance report methods The numbers on a project's Performance page and on the workspace Monitor are now available over REST (/api/v1/performance). This adds the SDK methods that read them: client.get_performance_metrics() client.get_workspace_performance_metric(metric, start_date, end_date) client.get_workspace_labeler_performance(start_date, end_date) project.get_performance_metric(metric, start_date, end_date) project.get_labeler_performance(start_date, end_date) project.get_reviewer_performance(start_date, end_date) project.get_performance_report_download(start_date, end_date) Results are typed models (PerformanceMetricSeries, ProjectLabelerPerformance and so on) with snake_case attributes. The two project tables come back as PerformanceRows, which fetches a page at a time while iterating and exposes the table's total. Dates can be a date, a datetime or a "2026-01-31" string and are sent as UTC days, which is how the reports bucket. ID filters take IDs or SDK objects. API errors are raised as the SDK's own exceptions, and a refused query names the parameters at fault. Project.labeler_performance() is unchanged; it is a different, older report. Co-Authored-By: Claude Opus 5.5 --- docs/labelbox/index.rst | 1 + docs/labelbox/performance.rst | 6 + libs/labelbox/src/labelbox/__init__.py | 13 + libs/labelbox/src/labelbox/client.py | 143 ++++ .../src/labelbox/schema/performance.py | 697 ++++++++++++++++ libs/labelbox/src/labelbox/schema/project.py | 260 ++++++ .../tests/unit/schema/test_performance.py | 748 ++++++++++++++++++ 7 files changed, 1868 insertions(+) create mode 100644 docs/labelbox/performance.rst create mode 100644 libs/labelbox/src/labelbox/schema/performance.py create mode 100644 libs/labelbox/tests/unit/schema/test_performance.py diff --git a/docs/labelbox/index.rst b/docs/labelbox/index.rst index 39e879191..3e27674d0 100644 --- a/docs/labelbox/index.rst +++ b/docs/labelbox/index.rst @@ -41,6 +41,7 @@ Labelbox Python SDK Documentation ontology_kind organization pagination + performance project project-model-config project-sync diff --git a/docs/labelbox/performance.rst b/docs/labelbox/performance.rst new file mode 100644 index 000000000..3e0425ef9 --- /dev/null +++ b/docs/labelbox/performance.rst @@ -0,0 +1,6 @@ +Performance +=============================================================================================== + +.. automodule:: labelbox.schema.performance + :members: PerformanceMetric, PerformanceMetricSeries, PerformanceTimeBucket, PerformanceScoreBucket, PerformanceHandlingTime, ProjectLabelerPerformance, ProjectReviewerPerformance, WorkspaceLabelerPerformance, PerformanceReportDownload, PerformanceRows, PerformanceInterval, PerformanceDeletedLabels + :show-inheritance: diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index 08a58f50e..e3d3cc07d 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -102,6 +102,19 @@ from labelbox.schema.organization import Organization from labelbox.schema.project import Project from labelbox.schema.project_model_config import ProjectModelConfig +from labelbox.schema.performance import ( + PerformanceDeletedLabels, + PerformanceInterval, + PerformanceMetric, + PerformanceMetricSeries, + PerformanceReportDownload, + PerformanceRows, + PerformanceScoreBucket, + PerformanceTimeBucket, + ProjectLabelerPerformance, + ProjectReviewerPerformance, + WorkspaceLabelerPerformance, +) from labelbox.schema.project_overview import ( ProjectOverview, ProjectOverviewDetailed, diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 75755afcc..ae933dc14 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -44,6 +44,7 @@ from labelbox.schema.identifiables import DataRowIds, GlobalKeys from labelbox.schema.label_score import LabelScore from labelbox.schema.labeling_frontend import LabelingFrontend +from labelbox.schema import performance as _performance from labelbox.schema.labeling_service_dashboard import LabelingServiceDashboard from labelbox.schema.media_type import ( MediaType, @@ -2327,6 +2328,148 @@ def get_labeling_service_dashboards( """ return LabelingServiceDashboard.get_all(self, search_query=search_query) + def get_performance_metrics(self) -> List[_performance.PerformanceMetric]: + """Lists the performance metrics that can be queried. + + These are the metrics a project's Performance page draws. Each has a + name to pass to :meth:`Project.get_performance_metric` or + :meth:`get_workspace_performance_metric`, a unit and a definition. + + Returns: + List[PerformanceMetric]: The metrics available to your + organization. + + Example: + >>> for metric in client.get_performance_metrics(): + >>> print(metric.name, metric.unit, metric.description) + """ + return _performance.get_performance_metrics(self) + + def get_workspace_performance_metric( + self, + metric: str, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + interval: Union[ + _performance.PerformanceInterval, str + ] = _performance.PerformanceInterval.DAY, + project_ids: Optional[List[str]] = None, + user_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + use_cache: bool = True, + ) -> _performance.PerformanceMetricSeries: + """Returns one performance metric across the workspace. + + This is the workspace Monitor's data: every project your organization + owns or shares, or only the projects named in ``project_ids``. + + Args: + metric (str): A metric name from :meth:`get_performance_metrics`, + for example ``"labels_created"``. + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + interval (PerformanceInterval or str): The width of each bucket: + day (the default), week, month, quarter or year. + project_ids (Optional[List[str]]): Narrow the report to these + projects. You then need performance access to each of them + instead of the organization-wide permission. + user_ids (Optional[List[str]]): Count only work by these users. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + use_cache (bool): Pass False to recompute instead of using a + result cached within the last few minutes. + + Returns: + PerformanceMetricSeries: The metric as a value per interval. + + Raises: + lbox.exceptions.ResourceNotFoundError: The metric does not exist, + or a project named is not one you can see. + lbox.exceptions.AuthorizationError: You may not view performance + across the organization. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + + Example: + >>> series = client.get_workspace_performance_metric( + >>> "labels_created", "2026-01-01", "2026-01-31", interval="week") + >>> for bucket in series.buckets: + >>> print(bucket.start.date(), bucket.value) + """ + return _performance.get_workspace_metric( + self, + metric, + start_date, + end_date, + interval=interval, + project_ids=project_ids, + user_ids=user_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + use_cache=use_cache, + ) + + def get_workspace_labeler_performance( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + project_ids: Optional[List[str]] = None, + user_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + ) -> List[_performance.WorkspaceLabelerPerformance]: + """Returns per-labeler totals across the workspace. + + This is the workspace Monitor's member table: one row for each + labeler on each project they worked on in the period. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + project_ids (Optional[List[str]]): Narrow the report to these + projects. You then need performance access to each of them + instead of the organization-wide permission. + user_ids (Optional[List[str]]): Return only these users. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + + Returns: + List[WorkspaceLabelerPerformance]: One row per labeler and + project. Emails are masked for labelers you may not identify. + + Raises: + lbox.exceptions.ResourceNotFoundError: A project named is not one + you can see. + lbox.exceptions.AuthorizationError: You may not view performance + across the organization. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + + Example: + >>> rows = client.get_workspace_labeler_performance( + >>> "2026-01-01", "2026-01-31") + >>> for row in rows: + >>> print(row.project_name, row.email, row.labels_created) + """ + return _performance.get_workspace_labelers( + self, + start_date, + end_date, + project_ids=project_ids, + user_ids=user_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + ) + def get_task_by_id(self, task_id: str) -> Union[Task, DataUpsertTask]: """ Fetches a task by ID. diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py new file mode 100644 index 000000000..985e937c2 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -0,0 +1,697 @@ +"""Performance reports. + +The numbers behind a project's Performance page and the workspace Monitor, +read through the Labelbox REST API (``/api/v1/performance``). The methods on +:class:`labelbox.Client` and :class:`labelbox.schema.project.Project` that +return the classes below are thin wrappers over the functions at the bottom +of this module. +""" + +from datetime import date, datetime, timezone +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generic, + Iterable, + Iterator, + List, + Optional, + Type, + TypeVar, + Union, +) + +from urllib.parse import quote + +import requests +from lbox.exceptions import ( + ApiLimitError, + AuthenticationError, + AuthorizationError, + InternalServerError, + InvalidQueryError, + LabelboxError, + ResourceNotFoundError, +) +from pydantic.alias_generators import to_camel + +from labelbox.utils import _CamelCaseMixin + +if TYPE_CHECKING: + from labelbox import Client + +# Reports are computed on request and a wide date range can take a while. +_TIMEOUT_SECONDS = 120 +_DEFAULT_PAGE_SIZE = 50 + +DateLike = Union[date, datetime, str] +"""A calendar day: a ``date``, a ``datetime`` or a string such as ``"2026-01-31"``.""" + + +class PerformanceInterval(str, Enum): + """The width of each bucket of a metric. Buckets follow UTC days.""" + + DAY = "day" + WEEK = "week" + MONTH = "month" + QUARTER = "quarter" + YEAR = "year" + + +class PerformanceDeletedLabels(str, Enum): + """Whether a report counts labels that were later deleted.""" + + INCLUDE = "include" + EXCLUDE = "exclude" + ONLY = "only" + + +class PerformanceMetric(_CamelCaseMixin): + """A metric that can be queried. + + Attributes: + name (str): The name to pass to the metric methods, for example + ``"labels_created"``. + category (str): ``"throughput"``, ``"efficiency"`` or ``"quality"``. + unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. + buckets (str): ``"time"`` for one value per interval, ``"score"`` for + a histogram of agreement scores. + description (str): What the metric measures, as the Performance page + explains it. + """ + + name: str + category: str + unit: str + buckets: str + description: str + + +class PerformanceHandlingTime(_CamelCaseMixin): + """Handling time split by activity, in seconds.""" + + creating: float + reviewing: float + reworking: float + + +class PerformanceTimeBucket(_CamelCaseMixin): + """The value of a metric over one interval. + + Attributes: + start (datetime): Start of the interval, inclusive. + end (datetime): End of the interval, exclusive. + value (float): The metric's value, in the unit of its series. + numerator (Optional[float]): For ratios, the amount divided. + denominator (Optional[float]): For ratios, what it was divided by. + modes (Optional[PerformanceHandlingTime]): The split by activity. + Only average handling time metrics have it. + """ + + start: datetime + end: datetime + value: float + numerator: Optional[float] = None + denominator: Optional[float] = None + modes: Optional[PerformanceHandlingTime] = None + + +class PerformanceScoreBucket(_CamelCaseMixin): + """The number of labels whose agreement score falls in one range. + + Attributes: + range_start (float): Lower bound of the range, inclusive. + range_end (float): Upper bound of the range, exclusive. + value (float): How many labels scored in the range. + """ + + range_start: float + range_end: float + value: float + + +class PerformanceMetricSeries(_CamelCaseMixin): + """One metric over a period. + + Attributes: + metric (str): The metric's name. + unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. + bucket_type (str): ``"time"`` or ``"score"``; says which of the two + bucket classes ``buckets`` holds. + buckets (List[Union[PerformanceTimeBucket, PerformanceScoreBucket]]): + The values. + interval (str): The bucket width that was asked for. + start_date (date): First day of the period. + end_date (date): Last day of the period. + generated_at (datetime): When the response was produced. This is not + how fresh the underlying data is. + project_id (Optional[str]): Set on a project's series. + organization_id (Optional[str]): Set on a workspace series. + project_ids (Optional[List[str]]): The projects a workspace series + was narrowed to, if any. + """ + + metric: str + unit: str + bucket_type: str + buckets: List[Union[PerformanceTimeBucket, PerformanceScoreBucket]] + interval: str + start_date: date + end_date: date + generated_at: datetime + project_id: Optional[str] = None + organization_id: Optional[str] = None + project_ids: Optional[List[str]] = None + + +class ProjectLabelerPerformance(_CamelCaseMixin): + """One labeler's totals on a project over a period. + + Times are in seconds. ``email`` is masked for labelers the caller may not + identify, exactly as on the Performance page. + + Attributes: + user_id (str) + project_id (str) + email (str) + labels_created (int) + labels_skipped (int) + member_total_time (float) + labeling_time_submitted (float) + labeling_time_skipped (float) + reviews_received (int) + review_time_all (float) + rework_time_all (float) + avg_time_per_label (float) + avg_review_time_all (float) + avg_rework_time_all (float) + avg_benchmark_agreement (float) + avg_consensus_agreement (float) + rework_percentage (float) + approval_percentage (float) + avg_rating (Optional[float]): ``None`` when the labeler has no ratings. + is_removed (bool): True when the labeler is no longer on the project. + """ + + user_id: str + project_id: str + email: str + labels_created: int + labels_skipped: int + member_total_time: float + labeling_time_submitted: float + labeling_time_skipped: float + reviews_received: int + review_time_all: float + rework_time_all: float + avg_time_per_label: float + avg_review_time_all: float + avg_rework_time_all: float + avg_benchmark_agreement: float + avg_consensus_agreement: float + rework_percentage: float + approval_percentage: float + avg_rating: Optional[float] = None + is_removed: bool = False + + +class ProjectReviewerPerformance(_CamelCaseMixin): + """One reviewer's totals on a project over a period. + + Times are in seconds. ``email`` is masked for reviewers the caller may + not identify, exactly as on the Performance page. + + Attributes: + user_id (str) + project_id (str) + email (str) + labels_reviewed (int) + labels_reworked (int) + member_total_time (float) + avg_review_time (float) + avg_rework_time (float) + review_time (float) + rework_time (float) + rework_percentage (float) + approval_percentage (float) + is_removed (bool): True when the reviewer is no longer on the project. + """ + + user_id: str + project_id: str + email: str + labels_reviewed: int + labels_reworked: int + member_total_time: float + avg_review_time: float + avg_rework_time: float + review_time: float + rework_time: float + rework_percentage: float + approval_percentage: float + is_removed: bool = False + + +class WorkspaceLabelerPerformance(_CamelCaseMixin): + """One labeler's totals on one project, from the workspace report. + + A labeler who worked on several projects has one of these per project. + Times are in seconds, and ``email`` is masked as on the Monitor. + + Attributes: + user_id (str) + email (str) + project_id (str) + project_name (str) + project_organization_id (str) + project_organization_name (str) + is_project_deleted (bool) + labels_created (int) + labels_skipped (int) + member_total_time (float) + labeling_time_submitted (float) + labeling_time_skipped (float) + reviews_received (int) + review_time_all (float) + rework_time_all (float) + avg_time_per_label (float) + avg_review_time_all (float) + avg_rework_time_all (float) + avg_benchmark_agreement (float) + avg_consensus_agreement (float) + rework_percentage (float) + approval_percentage (float) + is_removed (bool) + """ + + user_id: str + email: str + project_id: str + project_name: str + project_organization_id: str + project_organization_name: str + is_project_deleted: bool + labels_created: int + labels_skipped: int + member_total_time: float + labeling_time_submitted: float + labeling_time_skipped: float + reviews_received: int + review_time_all: float + rework_time_all: float + avg_time_per_label: float + avg_review_time_all: float + avg_rework_time_all: float + avg_benchmark_agreement: float + avg_consensus_agreement: float + rework_percentage: float + approval_percentage: float + is_removed: bool = False + + +class PerformanceReportDownload(_CamelCaseMixin): + """A link to a project's performance report as a file. + + Attributes: + project_id (str) + url (str): Where to download the file from. Anyone with the link can + open it until it expires. + expires_in_seconds (int): How long the link stays valid. + start_date (date) + end_date (date) + generated_at (datetime) + """ + + project_id: str + url: str + expires_in_seconds: int + start_date: date + end_date: date + generated_at: datetime + + +Row = TypeVar("Row", bound=_CamelCaseMixin) + + +class PerformanceRows(Generic[Row]): + """The rows of a performance table. + + Iterating fetches the table a page at a time, so a large table is never + held in one response. Nothing is requested until the rows, ``total`` or + ``cached_at`` are read. + + >>> labelers = project.get_labeler_performance("2026-01-01", "2026-01-31") + >>> labelers.total + 37 + >>> for labeler in labelers: + ... print(labeler.email, labeler.labels_created) + """ + + def __init__( + self, + fetch_page: Callable[[int], Dict[str, Any]], + row_type: Type[Row], + ): + self._fetch_page = fetch_page + self._row_type = row_type + self._first_page: Optional[Dict[str, Any]] = None + + def _page(self, number: int) -> Dict[str, Any]: + if number != 1: + return self._fetch_page(number) + if self._first_page is None: + self._first_page = self._fetch_page(1) + return self._first_page + + @property + def total(self) -> int: + """The number of rows in the whole table.""" + return int(self._page(1)["total"]) + + @property + def cached_at(self) -> Optional[datetime]: + """When the rows were computed, if they were served from a cache.""" + cached_at = self._page(1).get("cachedAt") + return _parse_datetime(cached_at) if cached_at else None + + def __iter__(self) -> Iterator[Row]: + number = 1 + seen = 0 + while True: + page = self._page(number) + rows = page["data"] + for row in rows: + yield self._row_type.model_validate(row) + seen += len(rows) + if not rows or seen >= int(page["total"]): + return + number += 1 + + +def _parse_datetime(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _to_day(value: DateLike) -> str: + """The day as the API takes it. Reports follow UTC days.""" + if isinstance(value, datetime): + if value.tzinfo is not None: + value = value.astimezone(timezone.utc) + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + return value + + +def _to_ids(values: Optional[Iterable[Any]]) -> Optional[str]: + """IDs as one comma-separated parameter. Accepts IDs or SDK objects.""" + if values is None: + return None + if isinstance(values, str): + values = [values] + ids = [getattr(value, "uid", value) for value in values] + return ",".join(ids) if ids else None + + +def _to_query(params: Dict[str, Any]) -> Dict[str, str]: + query: Dict[str, str] = {} + for name, value in params.items(): + if value is None: + continue + if isinstance(value, bool): + query[name] = "true" if value else "false" + elif isinstance(value, Enum): + query[name] = str(value.value) + else: + query[name] = str(value) + return query + + +def _error_message(response: requests.Response) -> str: + try: + body = response.json() + except ValueError: + body = None + if not isinstance(body, dict): + return response.text or f"HTTP {response.status_code}" + + message = body.get("message") or f"HTTP {response.status_code}" + if isinstance(message, list): + message = "; ".join(str(part) for part in message) + details = body.get("details") + if isinstance(details, list) and details: + problems = "; ".join( + f"{detail.get('parameter')} {detail.get('message')}" + for detail in details + if isinstance(detail, dict) + ) + message = f"{message}: {problems}" + return str(message) + + +def _raise_for_response(response: requests.Response) -> None: + message = _error_message(response) + status = response.status_code + if status == 400: + raise InvalidQueryError(message) + if status == 401: + raise AuthenticationError(message) + if status == 403: + raise AuthorizationError(message) + if status == 404: + raise ResourceNotFoundError(message=message) + if status == 429: + raise ApiLimitError(message) + if status >= 500: + raise InternalServerError(message) + raise LabelboxError(message) + + +def _get(client: "Client", path: str, **params: Any) -> Dict[str, Any]: + response = client.connection.get( + f"{client.rest_endpoint}/performance{path}", + params=_to_query(params), + timeout=_TIMEOUT_SECONDS, + ) + if response.status_code != requests.codes.ok: + _raise_for_response(response) + return response.json() + + +def _segment(value: Any) -> str: + """A value as one path segment, so an ID can never alter the path.""" + return quote(str(getattr(value, "uid", value)), safe="") + + +def _period(start_date: DateLike, end_date: DateLike) -> Dict[str, str]: + return {"startDate": _to_day(start_date), "endDate": _to_day(end_date)} + + +def get_performance_metrics(client: "Client") -> List[PerformanceMetric]: + body = _get(client, "/metrics") + return [PerformanceMetric.model_validate(metric) for metric in body["data"]] + + +def get_project_metric( + client: "Client", + project_id: str, + metric: str, + start_date: DateLike, + end_date: DateLike, + interval: Union[PerformanceInterval, str], + user_ids: Optional[Iterable[Any]], + user_group_ids: Optional[Iterable[Any]], + batch_ids: Optional[Iterable[Any]], + deleted_labels: Union[PerformanceDeletedLabels, str], +) -> PerformanceMetricSeries: + body = _get( + client, + f"/projects/{_segment(project_id)}/metrics/{_segment(metric)}", + **_period(start_date, end_date), + interval=interval, + userIds=_to_ids(user_ids), + userGroupIds=_to_ids(user_group_ids), + batchIds=_to_ids(batch_ids), + deletedLabels=deleted_labels, + ) + return PerformanceMetricSeries.model_validate(body) + + +def _project_table( + client: "Client", + path: str, + row_type: Type[Row], + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]], + user_group_ids: Optional[Iterable[Any]], + batch_ids: Optional[Iterable[Any]], + deleted_labels: Union[PerformanceDeletedLabels, str], + sort_by: Optional[str], + descending: bool, + page_size: int, + **extra: Any, +) -> PerformanceRows[Row]: + params = { + **_period(start_date, end_date), + "userIds": _to_ids(user_ids), + "userGroupIds": _to_ids(user_group_ids), + "batchIds": _to_ids(batch_ids), + "deletedLabels": deleted_labels, + # Columns are named as the row's attributes; the API spells them in + # camel case. + "sort": to_camel(sort_by) if sort_by else None, + "order": ("desc" if descending else "asc") if sort_by else None, + "perPage": page_size, + **extra, + } + + def fetch_page(number: int) -> Dict[str, Any]: + return _get(client, path, page=number, **params) + + return PerformanceRows(fetch_page, row_type) + + +def get_project_labelers( + client: "Client", + project_id: str, + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = _DEFAULT_PAGE_SIZE, + use_cache: bool = True, +) -> PerformanceRows[ProjectLabelerPerformance]: + return _project_table( + client, + f"/projects/{_segment(project_id)}/labelers", + ProjectLabelerPerformance, + start_date, + end_date, + user_ids, + user_group_ids, + batch_ids, + deleted_labels, + sort_by, + descending, + page_size, + # Only sent to skip the cache, which is on unless asked otherwise. + useCache=None if use_cache else False, + ) + + +def get_project_reviewers( + client: "Client", + project_id: str, + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = _DEFAULT_PAGE_SIZE, +) -> PerformanceRows[ProjectReviewerPerformance]: + return _project_table( + client, + f"/projects/{_segment(project_id)}/reviewers", + ProjectReviewerPerformance, + start_date, + end_date, + user_ids, + user_group_ids, + batch_ids, + deleted_labels, + sort_by, + descending, + page_size, + ) + + +def get_project_report_download( + client: "Client", + project_id: str, + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, +) -> PerformanceReportDownload: + body = _get( + client, + f"/projects/{_segment(project_id)}/download", + **_period(start_date, end_date), + userIds=_to_ids(user_ids), + userGroupIds=_to_ids(user_group_ids), + batchIds=_to_ids(batch_ids), + deletedLabels=deleted_labels, + ) + return PerformanceReportDownload.model_validate(body) + + +def get_workspace_metric( + client: "Client", + metric: str, + start_date: DateLike, + end_date: DateLike, + interval: Union[PerformanceInterval, str] = PerformanceInterval.DAY, + project_ids: Optional[Iterable[Any]] = None, + user_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, + use_cache: bool = True, +) -> PerformanceMetricSeries: + body = _get( + client, + f"/workspace/metrics/{_segment(metric)}", + **_period(start_date, end_date), + interval=interval, + projectIds=_to_ids(project_ids), + userIds=_to_ids(user_ids), + batchIds=_to_ids(batch_ids), + deletedLabels=deleted_labels, + useCache=None if use_cache else False, + ) + return PerformanceMetricSeries.model_validate(body) + + +def get_workspace_labelers( + client: "Client", + start_date: DateLike, + end_date: DateLike, + project_ids: Optional[Iterable[Any]] = None, + user_ids: Optional[Iterable[Any]] = None, + batch_ids: Optional[Iterable[Any]] = None, + deleted_labels: Union[ + PerformanceDeletedLabels, str + ] = PerformanceDeletedLabels.INCLUDE, +) -> List[WorkspaceLabelerPerformance]: + body = _get( + client, + "/workspace/labelers", + **_period(start_date, end_date), + projectIds=_to_ids(project_ids), + userIds=_to_ids(user_ids), + batchIds=_to_ids(batch_ids), + deletedLabels=deleted_labels, + ) + return [ + WorkspaceLabelerPerformance.model_validate(row) for row in body["data"] + ] diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index bd51e8b20..354aa9f3b 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -53,6 +53,7 @@ LabelingService, LabelingServiceStatus, ) +from labelbox.schema import performance as _performance from labelbox.schema.labeling_service_dashboard import LabelingServiceDashboard from labelbox.schema.media_type import MediaType from labelbox.schema.model_config import ModelConfig @@ -1852,6 +1853,265 @@ def get_overview( return ProjectOverviewDetailed(**overview) + def get_performance_metric( + self, + metric: str, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + interval: Union[ + _performance.PerformanceInterval, str + ] = _performance.PerformanceInterval.DAY, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + ) -> _performance.PerformanceMetricSeries: + """Returns one of this project's performance metrics over a period. + + The values are the ones the chart on the project's Performance page + shows for the same dates and filters. + + Args: + metric (str): A metric name from + :meth:`labelbox.Client.get_performance_metrics`, for example + ``"labels_created"``. + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + interval (PerformanceInterval or str): The width of each bucket: + day (the default), week, month, quarter or year. + user_ids (Optional[List[str]]): Count only work by these users. + user_group_ids (Optional[List[str]]): Count only work by the + current members of these user groups. With ``user_ids`` too, + only users in both are counted. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + + Returns: + PerformanceMetricSeries: The metric as a value per interval, or + per score range for the two distribution metrics. + + Raises: + lbox.exceptions.ResourceNotFoundError: The metric does not exist, + or you cannot view this project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + + Example: + >>> series = project.get_performance_metric( + >>> "labels_created", "2026-01-01", "2026-01-31") + >>> for bucket in series.buckets: + >>> print(bucket.start.date(), bucket.value) + """ + return _performance.get_project_metric( + self.client, + self.uid, + metric, + start_date, + end_date, + interval, + user_ids, + user_group_ids, + batch_ids, + deleted_labels, + ) + + def get_labeler_performance( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = 50, + use_cache: bool = True, + ) -> _performance.PerformanceRows[_performance.ProjectLabelerPerformance]: + """Returns per-labeler totals for this project over a period. + + This is the labeler table on the project's Performance page. It is + not the same report as :meth:`labeler_performance`, which has no + period and fewer figures. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + user_ids (Optional[List[str]]): Return only these users. + user_group_ids (Optional[List[str]]): Return only the current + members of these user groups. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + sort_by (Optional[str]): An attribute of + :class:`ProjectLabelerPerformance` to sort on, for example + ``"labels_created"``. + descending (bool): Sort from high to low. Only used with + ``sort_by``. + page_size (int): Rows fetched per request while iterating, up to + 200. + use_cache (bool): Pass False to recompute instead of using a + result cached within the last few minutes. + + Returns: + PerformanceRows[ProjectLabelerPerformance]: The rows, fetched a + page at a time as you iterate. Emails are masked for labelers you + may not identify. + + Raises: + lbox.exceptions.ResourceNotFoundError: You cannot view this + project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + + Example: + >>> labelers = project.get_labeler_performance( + >>> "2026-01-01", "2026-01-31", + >>> sort_by="labels_created", descending=True) + >>> print(labelers.total) + >>> for labeler in labelers: + >>> print(labeler.email, labeler.labels_created) + """ + return _performance.get_project_labelers( + self.client, + self.uid, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + sort_by=sort_by, + descending=descending, + page_size=page_size, + use_cache=use_cache, + ) + + def get_reviewer_performance( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = 50, + ) -> _performance.PerformanceRows[_performance.ProjectReviewerPerformance]: + """Returns per-reviewer totals for this project over a period. + + This is the reviewer table on the project's Performance page. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + user_ids (Optional[List[str]]): Return only these users. + user_group_ids (Optional[List[str]]): Return only the current + members of these user groups. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + sort_by (Optional[str]): An attribute of + :class:`ProjectReviewerPerformance` to sort on, for example + ``"labels_reviewed"``. + descending (bool): Sort from high to low. Only used with + ``sort_by``. + page_size (int): Rows fetched per request while iterating, up to + 200. + + Returns: + PerformanceRows[ProjectReviewerPerformance]: The rows, fetched a + page at a time as you iterate. Emails are masked for reviewers + you may not identify. + + Raises: + lbox.exceptions.ResourceNotFoundError: You cannot view this + project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + + Example: + >>> for reviewer in project.get_reviewer_performance( + >>> "2026-01-01", "2026-01-31"): + >>> print(reviewer.email, reviewer.labels_reviewed) + """ + return _performance.get_project_reviewers( + self.client, + self.uid, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + sort_by=sort_by, + descending=descending, + page_size=page_size, + ) + + def get_performance_report_download( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + batch_ids: Optional[List[str]] = None, + deleted_labels: Union[ + _performance.PerformanceDeletedLabels, str + ] = _performance.PerformanceDeletedLabels.INCLUDE, + ) -> _performance.PerformanceReportDownload: + """Returns a link to this project's performance report as a file. + + This is the file the Download button on the Performance page gives. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 366 days. + user_ids (Optional[List[str]]): Include only these users. + user_group_ids (Optional[List[str]]): Include only the current + members of these user groups. + batch_ids (Optional[List[str]]): Count only these batches. + deleted_labels (PerformanceDeletedLabels or str): Whether labels + deleted since are counted. Included by default. + + Returns: + PerformanceReportDownload: The link and how long it stays valid. + Anyone with the link can open the file until it expires. + + Raises: + lbox.exceptions.ResourceNotFoundError: You cannot view this + project's performance. + lbox.exceptions.InvalidQueryError: A parameter is not valid. + + Example: + >>> report = project.get_performance_report_download( + >>> "2026-01-01", "2026-01-31") + >>> print(report.url) + """ + return _performance.get_project_report_download( + self.client, + self.uid, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + batch_ids=batch_ids, + deleted_labels=deleted_labels, + ) + def clone(self) -> "Project": """ Clones the current project. diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py new file mode 100644 index 000000000..aefc0ddf4 --- /dev/null +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -0,0 +1,748 @@ +from datetime import date, datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from lbox.exceptions import ( + ApiLimitError, + AuthenticationError, + AuthorizationError, + InternalServerError, + InvalidQueryError, + LabelboxError, + ResourceNotFoundError, +) + +from labelbox import ( + Client, + PerformanceDeletedLabels, + PerformanceInterval, + PerformanceScoreBucket, + PerformanceTimeBucket, + Project, +) + +BASE = "https://api.labelbox.com/api/v1/performance" + +# Responses as the REST API returns them. +SERIES = { + "projectId": "project-1", + "metric": "labels_created", + "unit": "count", + "bucketType": "time", + "buckets": [ + { + "start": "2026-01-01T00:00:00.000Z", + "end": "2026-01-02T00:00:00.000Z", + "value": 4, + "numerator": None, + "denominator": None, + "modes": None, + } + ], + "interval": "day", + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "generatedAt": "2026-01-08T09:30:00.000Z", +} + + +def labeler(user_id): + return { + "userId": user_id, + "projectId": "project-1", + "email": f"{user_id}@example.com", + "labelsCreated": 12, + "labelsSkipped": 1, + "memberTotalTime": 3600, + "labelingTimeSubmitted": 3000, + "labelingTimeSkipped": 100, + "reviewsReceived": 4, + "reviewTimeAll": 300, + "reworkTimeAll": 200, + "avgTimePerLabel": 250, + "avgReviewTimeAll": 75, + "avgReworkTimeAll": 50, + "avgBenchmarkAgreement": 0.9, + "avgConsensusAgreement": 0.8, + "reworkPercentage": 10, + "approvalPercentage": 90, + "avgRating": 4.5, + "isRemoved": False, + } + + +def table(rows, total, page=1, cached_at=None): + return { + "projectId": "project-1", + "data": rows, + "page": page, + "perPage": 50, + "total": total, + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "cachedAt": cached_at, + "generatedAt": "2026-01-08T09:30:00.000Z", + } + + +def response(body, status=200): + reply = Mock() + reply.status_code = status + reply.json.return_value = body + reply.text = str(body) + return reply + + +@pytest.fixture +def client(): + client = Client(api_key="api_key") + client.connection.get = Mock(return_value=response(SERIES)) + return client + + +@pytest.fixture +def project(client): + # The methods only read the project's client and ID. + return SimpleNamespace(client=client, uid="project-1") + + +def sent(client, call=-1): + """The URL and query string of a request the client made.""" + args, kwargs = client.connection.get.call_args_list[call] + return args[0], kwargs["params"] + + +class TestMetricCatalog: + def test_lists_the_metrics(self, client): + client.connection.get.return_value = response( + { + "data": [ + { + "name": "labels_created", + "category": "throughput", + "unit": "count", + "buckets": "time", + "description": "Labels created in the period.", + } + ] + } + ) + + metrics = client.get_performance_metrics() + + assert sent(client) == (f"{BASE}/metrics", {}) + assert [metric.name for metric in metrics] == ["labels_created"] + assert metrics[0].unit == "count" + assert metrics[0].description == "Labels created in the period." + + +class TestProjectMetric: + def test_asks_for_the_metric_over_the_period(self, client, project): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/metrics/labels_created" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "interval": "day", + "deletedLabels": "include", + } + + def test_returns_a_value_per_interval(self, client, project): + series = Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert series.metric == "labels_created" + assert series.unit == "count" + assert series.bucket_type == "time" + assert series.project_id == "project-1" + assert series.start_date == date(2026, 1, 1) + assert series.end_date == date(2026, 1, 7) + assert series.generated_at == datetime( + 2026, 1, 8, 9, 30, tzinfo=timezone.utc + ) + (bucket,) = series.buckets + assert isinstance(bucket, PerformanceTimeBucket) + assert bucket.start == datetime(2026, 1, 1, tzinfo=timezone.utc) + assert bucket.end == datetime(2026, 1, 2, tzinfo=timezone.utc) + assert bucket.value == 4 + assert bucket.modes is None + + def test_returns_score_ranges_for_a_distribution(self, client, project): + client.connection.get.return_value = response( + { + **SERIES, + "metric": "consensus_distribution", + "bucketType": "score", + "buckets": [ + {"rangeStart": 0, "rangeEnd": 10, "value": 3}, + {"rangeStart": 10, "rangeEnd": 20, "value": 9}, + ], + } + ) + + series = Project.get_performance_metric( + project, "consensus_distribution", "2026-01-01", "2026-01-07" + ) + + assert series.bucket_type == "score" + assert all( + isinstance(bucket, PerformanceScoreBucket) + for bucket in series.buckets + ) + assert [ + (b.range_start, b.range_end, b.value) for b in series.buckets + ] == [ + (0, 10, 3), + (10, 20, 9), + ] + + def test_carries_the_ratio_and_the_split_by_activity(self, client, project): + client.connection.get.return_value = response( + { + **SERIES, + "metric": "aht_done_labels", + "unit": "seconds", + "buckets": [ + { + "start": "2026-01-01T00:00:00.000Z", + "end": "2026-01-02T00:00:00.000Z", + "value": 90, + "numerator": 900, + "denominator": 10, + "modes": { + "creating": 60, + "reviewing": 20, + "reworking": 10, + }, + } + ], + } + ) + + (bucket,) = Project.get_performance_metric( + project, "aht_done_labels", "2026-01-01", "2026-01-07" + ).buckets + + assert (bucket.value, bucket.numerator, bucket.denominator) == ( + 90, + 900, + 10, + ) + assert bucket.modes.creating == 60 + assert bucket.modes.reviewing == 20 + assert bucket.modes.reworking == 10 + + def test_sends_the_filters(self, client, project): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + interval=PerformanceInterval.WEEK, + user_ids=["user-1", "user-2"], + user_group_ids=["group-a"], + batch_ids=["batch-1"], + deleted_labels=PerformanceDeletedLabels.EXCLUDE, + ) + + _, params = sent(client) + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "interval": "week", + "userIds": "user-1,user-2", + "userGroupIds": "group-a", + "batchIds": "batch-1", + "deletedLabels": "exclude", + } + + def test_takes_plain_strings_in_place_of_the_enums(self, client, project): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + interval="month", + deleted_labels="only", + ) + + _, params = sent(client) + assert params["interval"] == "month" + assert params["deletedLabels"] == "only" + + def test_accepts_sdk_objects_where_ids_are_expected(self, client, project): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_ids=[SimpleNamespace(uid="user-1"), "user-2"], + batch_ids="batch-1", + ) + + _, params = sent(client) + assert params["userIds"] == "user-1,user-2" + assert params["batchIds"] == "batch-1" + + def test_leaves_an_empty_filter_out(self, client, project): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07", user_ids=[] + ) + + _, params = sent(client) + assert "userIds" not in params + + @pytest.mark.parametrize( + "day, expected", + [ + ("2026-01-31", "2026-01-31"), + (date(2026, 1, 31), "2026-01-31"), + (datetime(2026, 1, 31, 23, 30), "2026-01-31"), + # Reports follow UTC days: 23:30 at UTC-8 is already 1 February. + ( + datetime( + 2026, 1, 31, 23, 30, tzinfo=timezone(timedelta(hours=-8)) + ), + "2026-02-01", + ), + ], + ) + def test_sends_a_date_as_a_utc_day(self, client, project, day, expected): + Project.get_performance_metric(project, "labels_created", day, day) + + _, params = sent(client) + assert params["startDate"] == expected + assert params["endDate"] == expected + + def test_keeps_an_id_from_changing_the_path(self, client, project): + Project.get_performance_metric( + project, "../../labelers", "2026-01-01", "2026-01-07" + ) + + url, _ = sent(client) + assert url == f"{BASE}/projects/project-1/metrics/..%2F..%2Flabelers" + + def test_gives_the_request_a_timeout(self, client, project): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_args.kwargs["timeout"] > 0 + + def test_ignores_fields_added_to_the_response_later(self, client, project): + client.connection.get.return_value = response( + {**SERIES, "asOf": "2026-01-08T09:00:00.000Z"} + ) + + series = Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert series.metric == "labels_created" + + +class TestProjectLabelers: + def test_asks_for_nothing_until_the_rows_are_read(self, client, project): + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + + client.connection.get.assert_not_called() + + def test_returns_the_rows(self, client, project): + client.connection.get.return_value = response( + table([labeler("user-1"), labeler("user-2")], total=2) + ) + + rows = list( + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/labelers" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "deletedLabels": "include", + "perPage": "50", + "page": "1", + } + assert [row.user_id for row in rows] == ["user-1", "user-2"] + first = rows[0] + assert first.email == "user-1@example.com" + assert first.labels_created == 12 + assert first.member_total_time == 3600 + assert first.avg_consensus_agreement == 0.8 + assert first.avg_rating == 4.5 + assert first.is_removed is False + + def test_fetches_page_after_page_until_it_has_every_row( + self, client, project + ): + client.connection.get.side_effect = [ + response(table([labeler("user-1"), labeler("user-2")], total=5)), + response( + table([labeler("user-3"), labeler("user-4")], total=5, page=2) + ), + response(table([labeler("user-5")], total=5, page=3)), + ] + + rows = list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", page_size=2 + ) + ) + + assert [row.user_id for row in rows] == [ + "user-1", + "user-2", + "user-3", + "user-4", + "user-5", + ] + pages = [ + sent(client, call)[1]["page"] + for call in range(client.connection.get.call_count) + ] + assert pages == ["1", "2", "3"] + assert sent(client)[1]["perPage"] == "2" + + def test_stops_if_a_page_comes_back_empty(self, client, project): + # The table shrank while it was being read. + client.connection.get.side_effect = [ + response(table([labeler("user-1")], total=3)), + response(table([], total=3, page=2)), + ] + + rows = list( + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + ) + + assert [row.user_id for row in rows] == ["user-1"] + assert client.connection.get.call_count == 2 + + def test_reads_the_total_and_cache_time_from_the_first_page_once( + self, client, project + ): + client.connection.get.return_value = response( + table( + [labeler("user-1")], + total=1, + cached_at="2026-01-08T09:00:00.000Z", + ) + ) + labelers = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + + assert labelers.total == 1 + assert labelers.cached_at == datetime( + 2026, 1, 8, 9, 0, tzinfo=timezone.utc + ) + assert [row.user_id for row in labelers] == ["user-1"] + assert client.connection.get.call_count == 1 + + def test_has_no_cache_time_for_rows_computed_on_request( + self, client, project + ): + client.connection.get.return_value = response(table([], total=0)) + + labelers = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + + assert labelers.cached_at is None + assert list(labelers) == [] + + def test_sorts_by_an_attribute_of_the_row(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, + "2026-01-01", + "2026-01-07", + sort_by="labels_created", + descending=True, + ) + ) + + _, params = sent(client) + assert params["sort"] == "labelsCreated" + assert params["order"] == "desc" + + def test_sends_no_order_without_a_sort(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", descending=True + ) + ) + + _, params = sent(client) + assert "sort" not in params + assert "order" not in params + + def test_can_skip_the_cache(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", use_cache=False + ) + ) + + _, params = sent(client) + assert params["useCache"] == "false" + + def test_filters_by_user_group(self, client, project): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, + "2026-01-01", + "2026-01-07", + user_group_ids=["group-a", "group-b"], + ) + ) + + _, params = sent(client) + assert params["userGroupIds"] == "group-a,group-b" + + +class TestProjectReviewers: + def test_returns_the_rows(self, client, project): + client.connection.get.return_value = response( + table( + [ + { + "userId": "user-9", + "projectId": "project-1", + "email": "usr.email.user-9@internal.labelbox.com", + "labelsReviewed": 20, + "labelsReworked": 2, + "memberTotalTime": 1800, + "avgReviewTime": 60, + "avgReworkTime": 30, + "reviewTime": 1200, + "reworkTime": 600, + "reworkPercentage": 10, + "approvalPercentage": 90, + "isRemoved": True, + } + ], + total=1, + ) + ) + + (reviewer,) = list( + Project.get_reviewer_performance( + project, + "2026-01-01", + "2026-01-07", + sort_by="labels_reviewed", + ) + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/reviewers" + assert params["sort"] == "labelsReviewed" + assert params["order"] == "asc" + assert "useCache" not in params + assert reviewer.labels_reviewed == 20 + assert reviewer.review_time == 1200 + assert reviewer.is_removed is True + # Masked emails come through as the API returns them. + assert reviewer.email == "usr.email.user-9@internal.labelbox.com" + + +class TestProjectReportDownload: + def test_returns_the_link(self, client, project): + client.connection.get.return_value = response( + { + "projectId": "project-1", + "url": "https://files.example/report.csv", + "expiresInSeconds": 86400, + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "generatedAt": "2026-01-08T09:30:00.000Z", + } + ) + + report = Project.get_performance_report_download( + project, "2026-01-01", "2026-01-07", user_group_ids=["group-a"] + ) + + url, params = sent(client) + assert url == f"{BASE}/projects/project-1/download" + assert params["userGroupIds"] == "group-a" + assert report.url == "https://files.example/report.csv" + assert report.expires_in_seconds == 86400 + assert report.start_date == date(2026, 1, 1) + + +class TestWorkspace: + def test_returns_a_metric_across_the_workspace(self, client): + client.connection.get.return_value = response( + { + **{k: v for k, v in SERIES.items() if k != "projectId"}, + "organizationId": "org-1", + "projectIds": ["project-1", "project-2"], + } + ) + + series = client.get_workspace_performance_metric( + "labels_created", + date(2026, 1, 1), + date(2026, 1, 7), + project_ids=["project-1", "project-2"], + use_cache=False, + ) + + url, params = sent(client) + assert url == f"{BASE}/workspace/metrics/labels_created" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "interval": "day", + "projectIds": "project-1,project-2", + "deletedLabels": "include", + "useCache": "false", + } + assert series.organization_id == "org-1" + assert series.project_ids == ["project-1", "project-2"] + assert series.project_id is None + + def test_returns_a_row_per_labeler_and_project(self, client): + row = { + **{k: v for k, v in labeler("user-1").items() if k != "avgRating"}, + "projectName": "Project one", + "projectOrganizationId": "org-1", + "projectOrganizationName": "Org one", + "isProjectDeleted": False, + } + client.connection.get.return_value = response( + { + "organizationId": "org-1", + "projectIds": None, + "data": [row, {**row, "projectId": "project-2"}], + "total": 2, + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "generatedAt": "2026-01-08T09:30:00.000Z", + } + ) + + rows = client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07", user_ids=["user-1"] + ) + + url, params = sent(client) + assert url == f"{BASE}/workspace/labelers" + assert params == { + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "userIds": "user-1", + "deletedLabels": "include", + } + assert [row.project_id for row in rows] == ["project-1", "project-2"] + assert rows[0].project_name == "Project one" + assert rows[0].project_organization_name == "Org one" + assert rows[0].labels_created == 12 + + +class TestErrors: + @pytest.mark.parametrize( + "status, error", + [ + (400, InvalidQueryError), + (401, AuthenticationError), + (403, AuthorizationError), + (404, ResourceNotFoundError), + (429, ApiLimitError), + (500, InternalServerError), + (503, InternalServerError), + (418, LabelboxError), + ], + ) + def test_raises_the_sdk_error_for_the_status( + self, client, project, status, error + ): + client.connection.get.return_value = response( + {"statusCode": status, "message": "Something went wrong"}, status + ) + + with pytest.raises(error, match="Something went wrong"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + def test_names_the_parameters_the_api_refused(self, client, project): + client.connection.get.return_value = response( + { + "statusCode": 400, + "error": "Bad Request", + "message": "Invalid query parameters", + "details": [ + { + "parameter": "endDate", + "message": "must not be before startDate", + }, + { + "parameter": "interval", + "message": "must be one of day, week, month, quarter, year", + }, + ], + }, + 400, + ) + + with pytest.raises(InvalidQueryError) as raised: + Project.get_performance_metric( + project, + "labels_created", + "2026-01-07", + "2026-01-01", + interval="hour", + ) + + message = str(raised.value) + assert "endDate must not be before startDate" in message + assert "interval must be one of day, week" in message + + def test_says_which_project_was_not_found(self, client, project): + client.connection.get.return_value = response( + { + "statusCode": 404, + "message": "Project project-1 was not found.", + "error": "Not Found", + }, + 404, + ) + + with pytest.raises( + ResourceNotFoundError, match="Project project-1 was not found" + ): + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + ) + + def test_reports_a_reply_that_is_not_json(self, client, project): + reply = Mock() + reply.status_code = 502 + reply.json.side_effect = ValueError("not json") + reply.text = "Bad gateway" + client.connection.get.return_value = reply + + with pytest.raises(InternalServerError, match="Bad gateway"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) From 13ddcf90f76f80bfba3a5ee2efa1c2efb67e2b5b Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sat, 10 Oct 2026 02:17:45 -0700 Subject: [PATCH 02/13] Say that percentages are fractions from 0 to 1 The performance reports return agreement scores and rework and approval rates as fractions (0.9 for 90%). Say so on every class that carries one, and use realistic values in the test rows. Co-Authored-By: Claude Opus 5.5 --- .../labelbox/src/labelbox/schema/performance.py | 17 +++++++++++------ .../tests/unit/schema/test_performance.py | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index 985e937c2..e8edac798 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -76,7 +76,8 @@ class PerformanceMetric(_CamelCaseMixin): name (str): The name to pass to the metric methods, for example ``"labels_created"``. category (str): ``"throughput"``, ``"efficiency"`` or ``"quality"``. - unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. + unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. A + percentage is a fraction from 0 to 1: 0.9 means 90%. buckets (str): ``"time"`` for one value per interval, ``"score"`` for a histogram of agreement scores. description (str): What the metric measures, as the Performance page @@ -138,7 +139,8 @@ class PerformanceMetricSeries(_CamelCaseMixin): Attributes: metric (str): The metric's name. - unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. + unit (str): ``"count"``, ``"seconds"`` or ``"percentage"``. A + percentage is a fraction from 0 to 1: 0.9 means 90%. bucket_type (str): ``"time"`` or ``"score"``; says which of the two bucket classes ``buckets`` holds. buckets (List[Union[PerformanceTimeBucket, PerformanceScoreBucket]]): @@ -170,7 +172,8 @@ class PerformanceMetricSeries(_CamelCaseMixin): class ProjectLabelerPerformance(_CamelCaseMixin): """One labeler's totals on a project over a period. - Times are in seconds. ``email`` is masked for labelers the caller may not + Times are in seconds. Percentages and agreement scores are fractions from + 0 to 1: 0.9 means 90%. ``email`` is masked for labelers the caller may not identify, exactly as on the Performance page. Attributes: @@ -221,8 +224,9 @@ class ProjectLabelerPerformance(_CamelCaseMixin): class ProjectReviewerPerformance(_CamelCaseMixin): """One reviewer's totals on a project over a period. - Times are in seconds. ``email`` is masked for reviewers the caller may - not identify, exactly as on the Performance page. + Times are in seconds. Percentages are fractions from 0 to 1: 0.9 means + 90%. ``email`` is masked for reviewers the caller may not identify, + exactly as on the Performance page. Attributes: user_id (str) @@ -259,7 +263,8 @@ class WorkspaceLabelerPerformance(_CamelCaseMixin): """One labeler's totals on one project, from the workspace report. A labeler who worked on several projects has one of these per project. - Times are in seconds, and ``email`` is masked as on the Monitor. + Times are in seconds. Percentages and agreement scores are fractions from + 0 to 1: 0.9 means 90%. ``email`` is masked as on the Monitor. Attributes: user_id (str) diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index aefc0ddf4..c648d19f3 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -65,8 +65,8 @@ def labeler(user_id): "avgReworkTimeAll": 50, "avgBenchmarkAgreement": 0.9, "avgConsensusAgreement": 0.8, - "reworkPercentage": 10, - "approvalPercentage": 90, + "reworkPercentage": 0.1, + "approvalPercentage": 0.9, "avgRating": 4.5, "isRemoved": False, } @@ -532,8 +532,8 @@ def test_returns_the_rows(self, client, project): "avgReworkTime": 30, "reviewTime": 1200, "reworkTime": 600, - "reworkPercentage": 10, - "approvalPercentage": 90, + "reworkPercentage": 0.1, + "approvalPercentage": 0.9, "isRemoved": True, } ], From bb90dcb80a05e86dc8d3ef06f7edce599ff2ac7f Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sat, 10 Oct 2026 02:22:18 -0700 Subject: [PATCH 03/13] Accept user groups and other objects whose ID is `id` Filters are documented to take IDs or SDK objects, but only objects with a `uid` were understood. UserGroup is a pydantic model that carries its ID as `id`, so passing groups to `user_group_ids` failed while joining the list. Read `uid`, then `id`, and refuse a value that has neither with a message that says what was passed, before any request is made. Co-Authored-By: Claude Opus 5.5 --- .../src/labelbox/schema/performance.py | 22 +++++++++-- .../tests/unit/schema/test_performance.py | 37 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index e8edac798..31d3ffeee 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -23,7 +23,6 @@ TypeVar, Union, ) - from urllib.parse import quote import requests @@ -411,13 +410,30 @@ def _to_day(value: DateLike) -> str: return value +def _to_id(value: Any) -> str: + """The ID of a value that is an ID or an SDK object. + + Most SDK objects carry their ID as ``uid``; the pydantic ones, such as + UserGroup, carry it as ``id``. + """ + if isinstance(value, str): + return value + for attribute in ("uid", "id"): + identifier = getattr(value, attribute, None) + if isinstance(identifier, str) and identifier: + return identifier + raise TypeError( + f"Expected an ID or an object that has one, got {type(value).__name__}" + ) + + def _to_ids(values: Optional[Iterable[Any]]) -> Optional[str]: """IDs as one comma-separated parameter. Accepts IDs or SDK objects.""" if values is None: return None if isinstance(values, str): values = [values] - ids = [getattr(value, "uid", value) for value in values] + ids = [_to_id(value) for value in values] return ",".join(ids) if ids else None @@ -488,7 +504,7 @@ def _get(client: "Client", path: str, **params: Any) -> Dict[str, Any]: def _segment(value: Any) -> str: """A value as one path segment, so an ID can never alter the path.""" - return quote(str(getattr(value, "uid", value)), safe="") + return quote(_to_id(value), safe="") def _period(start_date: DateLike, end_date: DateLike) -> Dict[str, str]: diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index c648d19f3..f05ea6dbe 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -21,6 +21,7 @@ PerformanceTimeBucket, Project, ) +from labelbox.schema.user_group import UserGroup BASE = "https://api.labelbox.com/api/v1/performance" @@ -290,6 +291,42 @@ def test_accepts_sdk_objects_where_ids_are_expected(self, client, project): assert params["userIds"] == "user-1,user-2" assert params["batchIds"] == "batch-1" + def test_accepts_user_group_objects(self, client, project): + # UserGroup is a pydantic model whose ID is `id`, not `uid`. + groups = [ + UserGroup(client=client, id="group-a", name="Vendor A"), + UserGroup(client=client, id="group-b", name="Vendor B"), + ] + + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_group_ids=groups, + ) + + _, params = sent(client) + assert params["userGroupIds"] == "group-a,group-b" + + @pytest.mark.parametrize( + "not_an_id", + [42, None, SimpleNamespace(name="no id"), SimpleNamespace(id="")], + ) + def test_refuses_a_filter_value_that_has_no_id( + self, client, project, not_an_id + ): + with pytest.raises(TypeError, match="Expected an ID"): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_ids=["user-1", not_an_id], + ) + + client.connection.get.assert_not_called() + def test_leaves_an_empty_filter_out(self, client, project): Project.get_performance_metric( project, "labels_created", "2026-01-01", "2026-01-07", user_ids=[] From ca0d0e0addc828d4ced7de317f826ef16e798674 Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sat, 10 Oct 2026 03:00:08 -0700 Subject: [PATCH 04/13] Member type, owning organization, a paged workspace table, and retries Follows the performance REST API: - Labeler and reviewer rows carry `member_type`: whether the member belongs to your organization (`internal`) or to another one, such as a workforce provider (`external`). `PerformanceMemberType` names the two. - `get_workspace_performance_metric` and `get_workspace_labeler_performance` take `member_type` and `owner_organization_ids`, the Monitor's Member Type and "owned by" filters. - `get_workspace_labeler_performance` returns `PerformanceRows`, fetched a page at a time like the project tables, with `sort_by`, `descending`, `page_size` and `use_cache`. It returned the whole table as a list. - A failure of the service (5xx) or a timeout is tried again on the schedule `Client.execute` uses. When it stops trying, the error raised is the failure itself, not a retry error. A timeout is raised as `lbox.exceptions.TimeoutError` and a connection failure as `NetworkError`, as elsewhere in the SDK. A refused request (4xx) is not repeated. Co-Authored-By: Claude Opus 5.5 --- docs/labelbox/performance.rst | 2 +- libs/labelbox/src/labelbox/__init__.py | 1 + libs/labelbox/src/labelbox/client.py | 75 +++- .../src/labelbox/schema/performance.py | 128 ++++-- .../tests/unit/schema/test_performance.py | 367 +++++++++++++++++- 5 files changed, 523 insertions(+), 50 deletions(-) diff --git a/docs/labelbox/performance.rst b/docs/labelbox/performance.rst index 3e0425ef9..b86d5fe98 100644 --- a/docs/labelbox/performance.rst +++ b/docs/labelbox/performance.rst @@ -2,5 +2,5 @@ Performance =============================================================================================== .. automodule:: labelbox.schema.performance - :members: PerformanceMetric, PerformanceMetricSeries, PerformanceTimeBucket, PerformanceScoreBucket, PerformanceHandlingTime, ProjectLabelerPerformance, ProjectReviewerPerformance, WorkspaceLabelerPerformance, PerformanceReportDownload, PerformanceRows, PerformanceInterval, PerformanceDeletedLabels + :members: PerformanceMetric, PerformanceMetricSeries, PerformanceTimeBucket, PerformanceScoreBucket, PerformanceHandlingTime, ProjectLabelerPerformance, ProjectReviewerPerformance, WorkspaceLabelerPerformance, PerformanceReportDownload, PerformanceRows, PerformanceInterval, PerformanceDeletedLabels, PerformanceMemberType :show-inheritance: diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index e3d3cc07d..5a73c5088 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -105,6 +105,7 @@ from labelbox.schema.performance import ( PerformanceDeletedLabels, PerformanceInterval, + PerformanceMemberType, PerformanceMetric, PerformanceMetricSeries, PerformanceReportDownload, diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index ae933dc14..7e14ebdb5 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -2360,11 +2360,16 @@ def get_workspace_performance_metric( _performance.PerformanceDeletedLabels, str ] = _performance.PerformanceDeletedLabels.INCLUDE, use_cache: bool = True, + member_type: Optional[ + Union[_performance.PerformanceMemberType, str] + ] = None, + owner_organization_ids: Optional[List[str]] = None, ) -> _performance.PerformanceMetricSeries: """Returns one performance metric across the workspace. This is the workspace Monitor's data: every project your organization - owns or shares, or only the projects named in ``project_ids``. + owns and every project another organization shares with it, or only + the projects named in ``project_ids``. Args: metric (str): A metric name from :meth:`get_performance_metrics`, @@ -2384,6 +2389,18 @@ def get_workspace_performance_metric( deleted since are counted. Included by default. use_cache (bool): Pass False to recompute instead of using a result cached within the last few minutes. + member_type (Optional[PerformanceMemberType or str]): Count only + work by members of your own organization (``"internal"``) or + only by members of other organizations, such as a workforce + provider's (``"external"``). The Monitor's Member Type + filter. + owner_organization_ids (Optional[List[str]]): Count only projects + owned by these organizations. Use it to separate the projects + you own from the ones shared with you. + + ``member_type`` and ``owner_organization_ids`` report on other + members, so they need permission to view all performance in every + project the report covers. Returns: PerformanceMetricSeries: The metric as a value per interval. @@ -2392,7 +2409,8 @@ def get_workspace_performance_metric( lbox.exceptions.ResourceNotFoundError: The metric does not exist, or a project named is not one you can see. lbox.exceptions.AuthorizationError: You may not view performance - across the organization. + across the organization, or not everyone's in a project the + report covers. lbox.exceptions.InvalidQueryError: A parameter is not valid. Example: @@ -2400,6 +2418,11 @@ def get_workspace_performance_metric( >>> "labels_created", "2026-01-01", "2026-01-31", interval="week") >>> for bucket in series.buckets: >>> print(bucket.start.date(), bucket.value) + >>> + >>> # The same, for the workforce provider's members only. + >>> series = client.get_workspace_performance_metric( + >>> "labels_created", "2026-01-01", "2026-01-31", + >>> member_type="external") """ return _performance.get_workspace_metric( self, @@ -2412,6 +2435,8 @@ def get_workspace_performance_metric( batch_ids=batch_ids, deleted_labels=deleted_labels, use_cache=use_cache, + member_type=member_type, + owner_organization_ids=owner_organization_ids, ) def get_workspace_labeler_performance( @@ -2424,11 +2449,23 @@ def get_workspace_labeler_performance( deleted_labels: Union[ _performance.PerformanceDeletedLabels, str ] = _performance.PerformanceDeletedLabels.INCLUDE, - ) -> List[_performance.WorkspaceLabelerPerformance]: + member_type: Optional[ + Union[_performance.PerformanceMemberType, str] + ] = None, + owner_organization_ids: Optional[List[str]] = None, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = 50, + use_cache: bool = True, + ) -> _performance.PerformanceRows[_performance.WorkspaceLabelerPerformance]: """Returns per-labeler totals across the workspace. This is the workspace Monitor's member table: one row for each - labeler on each project they worked on in the period. + labeler on each project they worked on in the period. It covers the + projects your organization owns and the ones other organizations + share with it. Each row says which organization owns the project + (``project_organization_id``) and whether the labeler is one of your + own members (``member_type``). Args: start_date (date, datetime or str): First day of the period, in @@ -2442,10 +2479,27 @@ def get_workspace_labeler_performance( batch_ids (Optional[List[str]]): Count only these batches. deleted_labels (PerformanceDeletedLabels or str): Whether labels deleted since are counted. Included by default. + member_type (Optional[PerformanceMemberType or str]): Return only + members of your own organization (``"internal"``) or only + members of other organizations, such as a workforce + provider's (``"external"``). + owner_organization_ids (Optional[List[str]]): Return only + projects owned by these organizations. + sort_by (Optional[str]): A column to sort by, named as the row's + attribute, for example ``"labels_created"``. Without it, rows + are ordered by project, then labeler. Sorting by email is not + offered. + descending (bool): Sort from the highest value down. + page_size (int): Rows fetched per request, at most 200. + use_cache (bool): Pass False to read the report again instead of + using rows read within the last few minutes. Leave it on + while reading a large table, so that every page comes from + the same rows. Returns: - List[WorkspaceLabelerPerformance]: One row per labeler and - project. Emails are masked for labelers you may not identify. + PerformanceRows[WorkspaceLabelerPerformance]: One row per labeler + and project, fetched a page at a time as you iterate. Emails are + masked for labelers you may not identify. Raises: lbox.exceptions.ResourceNotFoundError: A project named is not one @@ -2456,7 +2510,8 @@ def get_workspace_labeler_performance( Example: >>> rows = client.get_workspace_labeler_performance( - >>> "2026-01-01", "2026-01-31") + >>> "2026-01-01", "2026-01-31", member_type="external") + >>> print(rows.total) >>> for row in rows: >>> print(row.project_name, row.email, row.labels_created) """ @@ -2468,6 +2523,12 @@ def get_workspace_labeler_performance( user_ids=user_ids, batch_ids=batch_ids, deleted_labels=deleted_labels, + member_type=member_type, + owner_organization_ids=owner_organization_ids, + sort_by=sort_by, + descending=descending, + page_size=page_size, + use_cache=use_cache, ) def get_task_by_id(self, task_id: str) -> Union[Task, DataUpsertTask]: diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index 31d3ffeee..4d20e44a4 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -26,6 +26,8 @@ from urllib.parse import quote import requests +from google.api_core import exceptions as core_exceptions +from google.api_core import retry from lbox.exceptions import ( ApiLimitError, AuthenticationError, @@ -33,7 +35,9 @@ InternalServerError, InvalidQueryError, LabelboxError, + NetworkError, ResourceNotFoundError, + TimeoutError, ) from pydantic.alias_generators import to_camel @@ -68,6 +72,20 @@ class PerformanceDeletedLabels(str, Enum): ONLY = "only" +class PerformanceMemberType(str, Enum): + """Whose account a member's is, seen from your organization. + + A project can be shared with another organization, such as a workforce + provider, whose members then work in it. ``EXTERNAL`` is a member whose + account belongs to another organization, or an Alignerr account; the + Monitor calls these "Workforce members". ``INTERNAL`` is a member of your + own organization. + """ + + INTERNAL = "internal" + EXTERNAL = "external" + + class PerformanceMetric(_CamelCaseMixin): """A metric that can be queried. @@ -153,6 +171,10 @@ class PerformanceMetricSeries(_CamelCaseMixin): organization_id (Optional[str]): Set on a workspace series. project_ids (Optional[List[str]]): The projects a workspace series was narrowed to, if any. + owner_organization_ids (Optional[List[str]]): The owning + organizations a workspace series was narrowed to, if any. + member_type (Optional[str]): The member type a workspace series was + narrowed to, if any: ``"internal"`` or ``"external"``. """ metric: str @@ -166,6 +188,8 @@ class PerformanceMetricSeries(_CamelCaseMixin): project_id: Optional[str] = None organization_id: Optional[str] = None project_ids: Optional[List[str]] = None + owner_organization_ids: Optional[List[str]] = None + member_type: Optional[str] = None class ProjectLabelerPerformance(_CamelCaseMixin): @@ -179,6 +203,10 @@ class ProjectLabelerPerformance(_CamelCaseMixin): user_id (str) project_id (str) email (str) + member_type (Optional[str]): ``"internal"`` for a member of your + organization, ``"external"`` for a member of another one (see + :class:`PerformanceMemberType`). ``None`` if the account no + longer exists. labels_created (int) labels_skipped (int) member_total_time (float) @@ -201,6 +229,7 @@ class ProjectLabelerPerformance(_CamelCaseMixin): user_id: str project_id: str email: str + member_type: Optional[str] = None labels_created: int labels_skipped: int member_total_time: float @@ -231,6 +260,10 @@ class ProjectReviewerPerformance(_CamelCaseMixin): user_id (str) project_id (str) email (str) + member_type (Optional[str]): ``"internal"`` for a member of your + organization, ``"external"`` for a member of another one (see + :class:`PerformanceMemberType`). ``None`` if the account no + longer exists. labels_reviewed (int) labels_reworked (int) member_total_time (float) @@ -246,6 +279,7 @@ class ProjectReviewerPerformance(_CamelCaseMixin): user_id: str project_id: str email: str + member_type: Optional[str] = None labels_reviewed: int labels_reworked: int member_total_time: float @@ -265,12 +299,21 @@ class WorkspaceLabelerPerformance(_CamelCaseMixin): Times are in seconds. Percentages and agreement scores are fractions from 0 to 1: 0.9 means 90%. ``email`` is masked as on the Monitor. + The report covers the projects your organization owns and the ones other + organizations share with it. ``project_organization_id`` says which is + which, and ``member_type`` whether the labeler is one of your own members. + Attributes: user_id (str) email (str) + member_type (Optional[str]): ``"internal"`` for a member of your + organization, ``"external"`` for a member of another one (see + :class:`PerformanceMemberType`). ``None`` if the account no + longer exists. project_id (str) project_name (str) - project_organization_id (str) + project_organization_id (str): The organization that owns the + project. It differs from yours for a project shared with you. project_organization_name (str) is_project_deleted (bool) labels_created (int) @@ -293,6 +336,7 @@ class WorkspaceLabelerPerformance(_CamelCaseMixin): user_id: str email: str + member_type: Optional[str] = None project_id: str project_name: str project_organization_id: str @@ -345,7 +389,8 @@ class PerformanceRows(Generic[Row]): Iterating fetches the table a page at a time, so a large table is never held in one response. Nothing is requested until the rows, ``total`` or - ``cached_at`` are read. + ``cached_at`` are read. The rows come in a fixed order, so no row is + repeated or missed between pages. >>> labelers = project.get_labeler_performance("2026-01-01", "2026-01-31") >>> labelers.total @@ -491,17 +536,39 @@ def _raise_for_response(response: requests.Response) -> None: raise LabelboxError(message) -def _get(client: "Client", path: str, **params: Any) -> Dict[str, Any]: - response = client.connection.get( - f"{client.rest_endpoint}/performance{path}", - params=_to_query(params), - timeout=_TIMEOUT_SECONDS, - ) +def _request(client: "Client", path: str, params: Dict[str, str]) -> Any: + try: + response = client.connection.get( + f"{client.rest_endpoint}/performance{path}", + params=params, + timeout=_TIMEOUT_SECONDS, + ) + except requests.exceptions.Timeout as error: + raise TimeoutError(str(error)) + except requests.exceptions.RequestException as error: + raise NetworkError(error) if response.status_code != requests.codes.ok: _raise_for_response(response) return response.json() +# Reading a report changes nothing, so a failure of the service or a timeout +# is tried again, on the schedule Client.execute retries a query on. +_retry_transient = retry.Retry( + predicate=retry.if_exception_type(InternalServerError, TimeoutError) +) + + +def _get(client: "Client", path: str, **params: Any) -> Dict[str, Any]: + try: + return _retry_transient(_request)(client, path, _to_query(params)) + except core_exceptions.RetryError as error: + # Out of attempts: report what kept failing, not that retrying did. + if isinstance(error.cause, LabelboxError): + raise error.cause from error + raise + + def _segment(value: Any) -> str: """A value as one path segment, so an ID can never alter the path.""" return quote(_to_id(value), safe="") @@ -541,14 +608,13 @@ def get_project_metric( return PerformanceMetricSeries.model_validate(body) -def _project_table( +def _table( client: "Client", path: str, row_type: Type[Row], start_date: DateLike, end_date: DateLike, user_ids: Optional[Iterable[Any]], - user_group_ids: Optional[Iterable[Any]], batch_ids: Optional[Iterable[Any]], deleted_labels: Union[PerformanceDeletedLabels, str], sort_by: Optional[str], @@ -559,7 +625,6 @@ def _project_table( params = { **_period(start_date, end_date), "userIds": _to_ids(user_ids), - "userGroupIds": _to_ids(user_group_ids), "batchIds": _to_ids(batch_ids), "deletedLabels": deleted_labels, # Columns are named as the row's attributes; the API spells them in @@ -592,19 +657,19 @@ def get_project_labelers( page_size: int = _DEFAULT_PAGE_SIZE, use_cache: bool = True, ) -> PerformanceRows[ProjectLabelerPerformance]: - return _project_table( + return _table( client, f"/projects/{_segment(project_id)}/labelers", ProjectLabelerPerformance, start_date, end_date, user_ids, - user_group_ids, batch_ids, deleted_labels, sort_by, descending, page_size, + userGroupIds=_to_ids(user_group_ids), # Only sent to skip the cache, which is on unless asked otherwise. useCache=None if use_cache else False, ) @@ -625,19 +690,19 @@ def get_project_reviewers( descending: bool = False, page_size: int = _DEFAULT_PAGE_SIZE, ) -> PerformanceRows[ProjectReviewerPerformance]: - return _project_table( + return _table( client, f"/projects/{_segment(project_id)}/reviewers", ProjectReviewerPerformance, start_date, end_date, user_ids, - user_group_ids, batch_ids, deleted_labels, sort_by, descending, page_size, + userGroupIds=_to_ids(user_group_ids), ) @@ -678,6 +743,8 @@ def get_workspace_metric( PerformanceDeletedLabels, str ] = PerformanceDeletedLabels.INCLUDE, use_cache: bool = True, + member_type: Optional[Union[PerformanceMemberType, str]] = None, + owner_organization_ids: Optional[Iterable[Any]] = None, ) -> PerformanceMetricSeries: body = _get( client, @@ -688,6 +755,8 @@ def get_workspace_metric( userIds=_to_ids(user_ids), batchIds=_to_ids(batch_ids), deletedLabels=deleted_labels, + memberType=member_type, + ownerOrganizationIds=_to_ids(owner_organization_ids), useCache=None if use_cache else False, ) return PerformanceMetricSeries.model_validate(body) @@ -703,16 +772,27 @@ def get_workspace_labelers( deleted_labels: Union[ PerformanceDeletedLabels, str ] = PerformanceDeletedLabels.INCLUDE, -) -> List[WorkspaceLabelerPerformance]: - body = _get( + member_type: Optional[Union[PerformanceMemberType, str]] = None, + owner_organization_ids: Optional[Iterable[Any]] = None, + sort_by: Optional[str] = None, + descending: bool = False, + page_size: int = _DEFAULT_PAGE_SIZE, + use_cache: bool = True, +) -> PerformanceRows[WorkspaceLabelerPerformance]: + return _table( client, "/workspace/labelers", - **_period(start_date, end_date), + WorkspaceLabelerPerformance, + start_date, + end_date, + user_ids, + batch_ids, + deleted_labels, + sort_by, + descending, + page_size, projectIds=_to_ids(project_ids), - userIds=_to_ids(user_ids), - batchIds=_to_ids(batch_ids), - deletedLabels=deleted_labels, + memberType=member_type, + ownerOrganizationIds=_to_ids(owner_organization_ids), + useCache=None if use_cache else False, ) - return [ - WorkspaceLabelerPerformance.model_validate(row) for row in body["data"] - ] diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index f05ea6dbe..3a399334c 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -3,6 +3,8 @@ from unittest.mock import Mock import pytest +import requests +from google.api_core import retry from lbox.exceptions import ( ApiLimitError, AuthenticationError, @@ -10,17 +12,21 @@ InternalServerError, InvalidQueryError, LabelboxError, + NetworkError, ResourceNotFoundError, + TimeoutError, ) from labelbox import ( Client, PerformanceDeletedLabels, PerformanceInterval, + PerformanceMemberType, PerformanceScoreBucket, PerformanceTimeBucket, Project, ) +from labelbox.schema import performance from labelbox.schema.user_group import UserGroup BASE = "https://api.labelbox.com/api/v1/performance" @@ -87,6 +93,53 @@ def table(rows, total, page=1, cached_at=None): } +REVIEWER = { + "userId": "user-9", + "projectId": "project-1", + "email": "user-9@example.com", + "labelsReviewed": 20, + "labelsReworked": 2, + "memberTotalTime": 1800, + "avgReviewTime": 60, + "avgReworkTime": 30, + "reviewTime": 1200, + "reworkTime": 600, + "reworkPercentage": 0.1, + "approvalPercentage": 0.9, + "isRemoved": False, +} + + +def workspace_labeler(user_id, project_id, **fields): + return { + **{k: v for k, v in labeler(user_id).items() if k != "avgRating"}, + "projectId": project_id, + "memberType": "internal", + "projectName": "Project one", + "projectOrganizationId": "org-1", + "projectOrganizationName": "Org one", + "isProjectDeleted": False, + **fields, + } + + +def workspace_table(rows, total, page=1, cached_at=None): + return { + "organizationId": "org-1", + "projectIds": None, + "ownerOrganizationIds": None, + "memberType": None, + "data": rows, + "page": page, + "perPage": 50, + "total": total, + "cachedAt": cached_at, + "startDate": "2026-01-01", + "endDate": "2026-01-07", + "generatedAt": "2026-01-08T09:30:00.000Z", + } + + def response(body, status=200): reply = Mock() reply.status_code = status @@ -95,6 +148,23 @@ def response(body, status=200): return reply +@pytest.fixture(autouse=True) +def quick_retries(monkeypatch): + # The real schedule waits seconds between attempts, for up to two minutes. + monkeypatch.setattr( + performance, + "_retry_transient", + retry.Retry( + predicate=retry.if_exception_type( + InternalServerError, TimeoutError + ), + initial=0.001, + maximum=0.001, + timeout=0.05, + ), + ) + + @pytest.fixture def client(): client = Client(api_key="api_key") @@ -656,28 +726,63 @@ def test_returns_a_metric_across_the_workspace(self, client): assert series.project_ids == ["project-1", "project-2"] assert series.project_id is None - def test_returns_a_row_per_labeler_and_project(self, client): - row = { - **{k: v for k, v in labeler("user-1").items() if k != "avgRating"}, - "projectName": "Project one", - "projectOrganizationId": "org-1", - "projectOrganizationName": "Org one", - "isProjectDeleted": False, - } + def test_narrows_a_metric_by_member_type_and_owning_organization( + self, client + ): client.connection.get.return_value = response( { + **{k: v for k, v in SERIES.items() if k != "projectId"}, "organizationId": "org-1", "projectIds": None, - "data": [row, {**row, "projectId": "project-2"}], - "total": 2, - "startDate": "2026-01-01", - "endDate": "2026-01-07", - "generatedAt": "2026-01-08T09:30:00.000Z", + "ownerOrganizationIds": ["org-2"], + "memberType": "external", } ) - rows = client.get_workspace_labeler_performance( - "2026-01-01", "2026-01-07", user_ids=["user-1"] + series = client.get_workspace_performance_metric( + "labels_created", + "2026-01-01", + "2026-01-07", + member_type=PerformanceMemberType.EXTERNAL, + owner_organization_ids=["org-2"], + ) + + _, params = sent(client) + assert params["memberType"] == "external" + assert params["ownerOrganizationIds"] == "org-2" + assert series.member_type == "external" + assert series.owner_organization_ids == ["org-2"] + + def test_sends_neither_narrowing_unless_asked(self, client): + client.get_workspace_performance_metric( + "labels_created", "2026-01-01", "2026-01-07" + ) + + _, params = sent(client) + assert "memberType" not in params + assert "ownerOrganizationIds" not in params + + def test_returns_a_row_per_labeler_and_project(self, client): + client.connection.get.return_value = response( + workspace_table( + [ + workspace_labeler("user-1", "project-1"), + workspace_labeler( + "user-2", + "project-shared", + memberType="external", + projectOrganizationId="org-2", + projectOrganizationName="Org two", + ), + ], + total=2, + ) + ) + + rows = list( + client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07", user_ids=["user-1", "user-2"] + ) ) url, params = sent(client) @@ -685,13 +790,239 @@ def test_returns_a_row_per_labeler_and_project(self, client): assert params == { "startDate": "2026-01-01", "endDate": "2026-01-07", - "userIds": "user-1", + "userIds": "user-1,user-2", "deletedLabels": "include", + "perPage": "50", + "page": "1", } - assert [row.project_id for row in rows] == ["project-1", "project-2"] + assert [row.project_id for row in rows] == [ + "project-1", + "project-shared", + ] assert rows[0].project_name == "Project one" - assert rows[0].project_organization_name == "Org one" assert rows[0].labels_created == 12 + # Whose project it is, and whose member did the work. + assert rows[0].project_organization_name == "Org one" + assert rows[0].member_type == "internal" + assert rows[1].project_organization_id == "org-2" + assert rows[1].member_type == "external" + + def test_fetches_the_workspace_table_a_page_at_a_time(self, client): + client.connection.get.side_effect = [ + response( + workspace_table( + [ + workspace_labeler("user-1", "project-1"), + workspace_labeler("user-2", "project-1"), + ], + total=3, + cached_at="2026-01-08T09:29:00.000Z", + ) + ), + response( + workspace_table( + [workspace_labeler("user-3", "project-1")], + total=3, + page=2, + ) + ), + ] + + rows = client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07", page_size=2 + ) + + assert rows.total == 3 + assert rows.cached_at == datetime( + 2026, 1, 8, 9, 29, tzinfo=timezone.utc + ) + assert [row.user_id for row in rows] == ["user-1", "user-2", "user-3"] + assert [sent(client, call)[1]["page"] for call in (0, 1)] == ["1", "2"] + assert sent(client)[1]["perPage"] == "2" + + def test_filters_and_sorts_the_workspace_table(self, client): + client.connection.get.return_value = response(workspace_table([], 0)) + + list( + client.get_workspace_labeler_performance( + "2026-01-01", + "2026-01-07", + project_ids=["project-1"], + member_type="external", + owner_organization_ids=["org-2", "org-3"], + sort_by="avg_time_per_label", + descending=True, + use_cache=False, + ) + ) + + _, params = sent(client) + assert params["projectIds"] == "project-1" + assert params["memberType"] == "external" + assert params["ownerOrganizationIds"] == "org-2,org-3" + assert params["sort"] == "avgTimePerLabel" + assert params["order"] == "desc" + assert params["useCache"] == "false" + + def test_reads_a_row_whose_member_no_longer_exists(self, client): + client.connection.get.return_value = response( + workspace_table( + [workspace_labeler("user-1", "project-1", memberType=None)], 1 + ) + ) + + (row,) = client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07" + ) + + assert row.member_type is None + + +class TestMemberType: + def test_says_whose_member_each_labeler_and_reviewer_is( + self, client, project + ): + client.connection.get.side_effect = [ + response( + table( + [ + {**labeler("user-1"), "memberType": "internal"}, + {**labeler("user-2"), "memberType": "external"}, + ], + total=2, + ) + ), + response(table([{**REVIEWER, "memberType": "external"}], total=1)), + ] + + labelers = list( + Project.get_labeler_performance(project, "2026-01-01", "2026-01-07") + ) + reviewers = list( + Project.get_reviewer_performance( + project, "2026-01-01", "2026-01-07" + ) + ) + + assert [row.member_type for row in labelers] == [ + "internal", + "external", + ] + assert reviewers[0].member_type == PerformanceMemberType.EXTERNAL + + def test_reads_a_reply_from_an_api_that_does_not_say(self, client, project): + client.connection.get.return_value = response( + table([labeler("user-1")], total=1) + ) + + (row,) = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + + assert row.member_type is None + + +class TestRetries: + def test_tries_again_after_a_failure_of_the_service(self, client, project): + client.connection.get.side_effect = [ + response({"statusCode": 503, "message": "Unavailable"}, 503), + response({"statusCode": 500, "message": "Oops"}, 500), + response(SERIES), + ] + + series = Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert series.metric == "labels_created" + assert client.connection.get.call_count == 3 + + def test_tries_again_after_a_timeout(self, client, project): + client.connection.get.side_effect = [ + requests.exceptions.ReadTimeout("read timed out"), + response(SERIES), + ] + + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count == 2 + + def test_tries_a_failed_page_again_without_repeating_rows( + self, client, project + ): + client.connection.get.side_effect = [ + response(table([labeler("user-1")], total=2)), + response({"statusCode": 502, "message": "Bad gateway"}, 502), + response(table([labeler("user-2")], total=2, page=2)), + ] + + rows = list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", page_size=1 + ) + ) + + assert [row.user_id for row in rows] == ["user-1", "user-2"] + assert [sent(client, call)[1]["page"] for call in (0, 1, 2)] == [ + "1", + "2", + "2", + ] + + @pytest.mark.parametrize("status", [400, 401, 403, 404, 429]) + def test_does_not_repeat_a_request_that_was_refused( + self, client, project, status + ): + client.connection.get.return_value = response( + {"statusCode": status, "message": "No"}, status + ) + + with pytest.raises(LabelboxError): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count == 1 + + def test_reports_the_failure_itself_once_it_stops_trying( + self, client, project + ): + client.connection.get.return_value = response( + {"statusCode": 503, "message": "Under maintenance"}, 503 + ) + + with pytest.raises(InternalServerError, match="Under maintenance"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count > 1 + + def test_reports_a_timeout_as_the_sdk_does(self, client, project): + client.connection.get.side_effect = requests.exceptions.ReadTimeout( + "read timed out" + ) + + with pytest.raises(TimeoutError, match="read timed out"): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + def test_reports_a_connection_failure_without_repeating_it( + self, client, project + ): + client.connection.get.side_effect = requests.exceptions.ConnectionError( + "connection refused" + ) + + with pytest.raises(NetworkError): + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + assert client.connection.get.call_count == 1 class TestErrors: From 4efa58b591f9a489c9977937be99dd8ed24ec02e Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sat, 10 Oct 2026 09:53:40 -0700 Subject: [PATCH 05/13] Leave time to retry a request that timed out A request waits up to 120 seconds, and the retry schedule's deadline, which counts from the first attempt, was the library's default of 120 seconds. So by the time a request timed out the deadline had passed, and it was never tried again: only quick failures were. The deadline is now twice the request timeout: one more attempt after a timeout, and no third. Co-Authored-By: Claude Opus 5.5 --- .../labelbox/src/labelbox/schema/performance.py | 11 +++++++++-- .../tests/unit/schema/test_performance.py | 17 ++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index 4d20e44a4..993601644 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -48,6 +48,10 @@ # Reports are computed on request and a wide date range can take a while. _TIMEOUT_SECONDS = 120 +# How long a request is retried for, counted from its first attempt. Twice the +# time one attempt may take: enough to try once more after an attempt that +# timed out, and not to start a third. +_RETRY_DEADLINE_SECONDS = 2 * _TIMEOUT_SECONDS _DEFAULT_PAGE_SIZE = 50 DateLike = Union[date, datetime, str] @@ -553,9 +557,12 @@ def _request(client: "Client", path: str, params: Dict[str, str]) -> Any: # Reading a report changes nothing, so a failure of the service or a timeout -# is tried again, on the schedule Client.execute retries a query on. +# is tried again, with the growing pauses Client.execute retries a query with. +# The deadline is set here because the default (120 seconds) is already spent +# when a request of this length times out, and it would never be retried. _retry_transient = retry.Retry( - predicate=retry.if_exception_type(InternalServerError, TimeoutError) + predicate=retry.if_exception_type(InternalServerError, TimeoutError), + deadline=_RETRY_DEADLINE_SECONDS, ) diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index 3a399334c..0472bd391 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -31,6 +31,9 @@ BASE = "https://api.labelbox.com/api/v1/performance" +# The schedule the SDK ships with. The tests below swap in a quick one. +SHIPPED_RETRY = performance._retry_transient + # Responses as the REST API returns them. SERIES = { "projectId": "project-1", @@ -160,7 +163,7 @@ def quick_retries(monkeypatch): ), initial=0.001, maximum=0.001, - timeout=0.05, + deadline=0.05, ), ) @@ -1000,6 +1003,18 @@ def test_reports_the_failure_itself_once_it_stops_trying( assert client.connection.get.call_count > 1 + def test_has_time_left_to_try_again_after_a_request_that_timed_out( + self, + ): + # A retry starts only if the deadline, counted from the first attempt, + # has not passed after the failed attempt and the pause that follows. + one_attempt = performance._TIMEOUT_SECONDS + longest_first_pause = 2 # seconds, across google-api-core versions + + assert SHIPPED_RETRY.deadline >= one_attempt + longest_first_pause + # ...and it does not start a third attempt after two timeouts. + assert SHIPPED_RETRY.deadline <= 2 * one_attempt + def test_reports_a_timeout_as_the_sdk_does(self, client, project): client.connection.get.side_effect = requests.exceptions.ReadTimeout( "read timed out" From 58e3245b57256ace2cadbaad3c846ac0169c9bbc Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sat, 10 Oct 2026 10:22:30 -0700 Subject: [PATCH 06/13] Fixes from a review of the performance module - When retries run out, the failure is raised without being chained back to the retry error. The retry error already names the failure as its cause, so `raise cause from error` made the two point at each other, and traceback printers that follow the chain (rich's among them) never finish. - An empty list for a filter (`user_ids=[]`) raises `ValueError` instead of leaving the filter off. It is nearly always a list that came out empty, and leaving the filter off reported on everyone. `None` still means no filter. - Iterating `PerformanceRows` a second time reads the table again from its first page. The first page is kept so that reading `total` and then the rows costs one request; a second pass started from that old page and continued with newly fetched ones. - `sort_by` is turned into the API's spelling here instead of by pydantic's `to_camel`, which on older 2.x releases lower-cases a name that is already in camel case. - Project tables no longer document sorting by email; the API does not offer it. Co-Authored-By: Claude Opus 5.5 --- libs/labelbox/src/labelbox/client.py | 4 + .../src/labelbox/schema/performance.py | 88 ++++++++--- libs/labelbox/src/labelbox/schema/project.py | 16 +- .../tests/unit/schema/test_performance.py | 142 +++++++++++++++++- 4 files changed, 219 insertions(+), 31 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 7e14ebdb5..cb6f4949e 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -2412,6 +2412,8 @@ def get_workspace_performance_metric( across the organization, or not everyone's in a project the report covers. lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. Example: >>> series = client.get_workspace_performance_metric( @@ -2507,6 +2509,8 @@ def get_workspace_labeler_performance( lbox.exceptions.AuthorizationError: You may not view performance across the organization. lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. Example: >>> rows = client.get_workspace_labeler_performance( diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index 993601644..dcdfd26d8 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -39,7 +39,6 @@ ResourceNotFoundError, TimeoutError, ) -from pydantic.alias_generators import to_camel from labelbox.utils import _CamelCaseMixin @@ -396,6 +395,9 @@ class PerformanceRows(Generic[Row]): ``cached_at`` are read. The rows come in a fixed order, so no row is repeated or missed between pages. + Iterating a second time reads the table again from its first page, and + ``total`` and ``cached_at`` then describe that reading. + >>> labelers = project.get_labeler_performance("2026-01-01", "2026-01-31") >>> labelers.total 37 @@ -411,6 +413,7 @@ def __init__( self._fetch_page = fetch_page self._row_type = row_type self._first_page: Optional[Dict[str, Any]] = None + self._first_page_iterated = False def _page(self, number: int) -> Dict[str, Any]: if number != 1: @@ -431,6 +434,13 @@ def cached_at(self) -> Optional[datetime]: return _parse_datetime(cached_at) if cached_at else None def __iter__(self) -> Iterator[Row]: + if self._first_page_iterated: + # The first page is kept so that reading ``total`` and then the + # rows costs one request. A later pass must not start from that + # old page and continue with new ones. + self._first_page = None + self._first_page_iterated = True + number = 1 seen = 0 while True: @@ -476,14 +486,33 @@ def _to_id(value: Any) -> str: ) -def _to_ids(values: Optional[Iterable[Any]]) -> Optional[str]: - """IDs as one comma-separated parameter. Accepts IDs or SDK objects.""" +def _to_ids(name: str, values: Optional[Iterable[Any]]) -> Optional[str]: + """IDs as one comma-separated parameter. Accepts IDs or SDK objects. + + ``None`` leaves the filter off. An empty list is refused: it is nearly + always a list that came out empty, and leaving the filter off for it + would report on everyone. + """ if values is None: return None if isinstance(values, str): values = [values] ids = [_to_id(value) for value in values] - return ",".join(ids) if ids else None + if not ids: + raise ValueError( + f"{name} is empty. Pass None to leave this filter off; " + "an empty list would otherwise report on everyone." + ) + return ",".join(ids) + + +def _to_field(attribute: str) -> str: + """A row attribute as the API spells it: labels_created is labelsCreated. + + A name already in the API's spelling is left as it is. + """ + head, *rest = attribute.split("_") + return head + "".join(part[:1].upper() + part[1:] for part in rest) def _to_query(params: Dict[str, Any]) -> Dict[str, str]: @@ -570,10 +599,15 @@ def _get(client: "Client", path: str, **params: Any) -> Dict[str, Any]: try: return _retry_transient(_request)(client, path, _to_query(params)) except core_exceptions.RetryError as error: - # Out of attempts: report what kept failing, not that retrying did. - if isinstance(error.cause, LabelboxError): - raise error.cause from error - raise + if not isinstance(error.cause, LabelboxError): + raise + failure = error.cause + # Out of attempts: report what kept failing, not that retrying did. It is + # raised out here, not inside the handler, because the retry error already + # names the failure as its cause. Chaining the failure back to it would + # make the two point at each other, and some traceback printers follow + # such a chain forever. + raise failure def _segment(value: Any) -> str: @@ -607,9 +641,9 @@ def get_project_metric( f"/projects/{_segment(project_id)}/metrics/{_segment(metric)}", **_period(start_date, end_date), interval=interval, - userIds=_to_ids(user_ids), - userGroupIds=_to_ids(user_group_ids), - batchIds=_to_ids(batch_ids), + userIds=_to_ids("user_ids", user_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), + batchIds=_to_ids("batch_ids", batch_ids), deletedLabels=deleted_labels, ) return PerformanceMetricSeries.model_validate(body) @@ -631,12 +665,12 @@ def _table( ) -> PerformanceRows[Row]: params = { **_period(start_date, end_date), - "userIds": _to_ids(user_ids), - "batchIds": _to_ids(batch_ids), + "userIds": _to_ids("user_ids", user_ids), + "batchIds": _to_ids("batch_ids", batch_ids), "deletedLabels": deleted_labels, # Columns are named as the row's attributes; the API spells them in # camel case. - "sort": to_camel(sort_by) if sort_by else None, + "sort": _to_field(sort_by) if sort_by else None, "order": ("desc" if descending else "asc") if sort_by else None, "perPage": page_size, **extra, @@ -676,7 +710,7 @@ def get_project_labelers( sort_by, descending, page_size, - userGroupIds=_to_ids(user_group_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), # Only sent to skip the cache, which is on unless asked otherwise. useCache=None if use_cache else False, ) @@ -709,7 +743,7 @@ def get_project_reviewers( sort_by, descending, page_size, - userGroupIds=_to_ids(user_group_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), ) @@ -729,9 +763,9 @@ def get_project_report_download( client, f"/projects/{_segment(project_id)}/download", **_period(start_date, end_date), - userIds=_to_ids(user_ids), - userGroupIds=_to_ids(user_group_ids), - batchIds=_to_ids(batch_ids), + userIds=_to_ids("user_ids", user_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), + batchIds=_to_ids("batch_ids", batch_ids), deletedLabels=deleted_labels, ) return PerformanceReportDownload.model_validate(body) @@ -758,12 +792,14 @@ def get_workspace_metric( f"/workspace/metrics/{_segment(metric)}", **_period(start_date, end_date), interval=interval, - projectIds=_to_ids(project_ids), - userIds=_to_ids(user_ids), - batchIds=_to_ids(batch_ids), + projectIds=_to_ids("project_ids", project_ids), + userIds=_to_ids("user_ids", user_ids), + batchIds=_to_ids("batch_ids", batch_ids), deletedLabels=deleted_labels, memberType=member_type, - ownerOrganizationIds=_to_ids(owner_organization_ids), + ownerOrganizationIds=_to_ids( + "owner_organization_ids", owner_organization_ids + ), useCache=None if use_cache else False, ) return PerformanceMetricSeries.model_validate(body) @@ -798,8 +834,10 @@ def get_workspace_labelers( sort_by, descending, page_size, - projectIds=_to_ids(project_ids), + projectIds=_to_ids("project_ids", project_ids), memberType=member_type, - ownerOrganizationIds=_to_ids(owner_organization_ids), + ownerOrganizationIds=_to_ids( + "owner_organization_ids", owner_organization_ids + ), useCache=None if use_cache else False, ) diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index 354aa9f3b..a6ac212f1 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -1899,6 +1899,8 @@ def get_performance_metric( lbox.exceptions.ResourceNotFoundError: The metric does not exist, or you cannot view this project's performance. lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. Example: >>> series = project.get_performance_metric( @@ -1951,9 +1953,9 @@ def get_labeler_performance( batch_ids (Optional[List[str]]): Count only these batches. deleted_labels (PerformanceDeletedLabels or str): Whether labels deleted since are counted. Included by default. - sort_by (Optional[str]): An attribute of + sort_by (Optional[str]): A figure of :class:`ProjectLabelerPerformance` to sort on, for example - ``"labels_created"``. + ``"labels_created"``. Sorting by email is not offered. descending (bool): Sort from high to low. Only used with ``sort_by``. page_size (int): Rows fetched per request while iterating, up to @@ -1970,6 +1972,8 @@ def get_labeler_performance( lbox.exceptions.ResourceNotFoundError: You cannot view this project's performance. lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. Example: >>> labelers = project.get_labeler_performance( @@ -2023,9 +2027,9 @@ def get_reviewer_performance( batch_ids (Optional[List[str]]): Count only these batches. deleted_labels (PerformanceDeletedLabels or str): Whether labels deleted since are counted. Included by default. - sort_by (Optional[str]): An attribute of + sort_by (Optional[str]): A figure of :class:`ProjectReviewerPerformance` to sort on, for example - ``"labels_reviewed"``. + ``"labels_reviewed"``. Sorting by email is not offered. descending (bool): Sort from high to low. Only used with ``sort_by``. page_size (int): Rows fetched per request while iterating, up to @@ -2040,6 +2044,8 @@ def get_reviewer_performance( lbox.exceptions.ResourceNotFoundError: You cannot view this project's performance. lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. Example: >>> for reviewer in project.get_reviewer_performance( @@ -2095,6 +2101,8 @@ def get_performance_report_download( lbox.exceptions.ResourceNotFoundError: You cannot view this project's performance. lbox.exceptions.InvalidQueryError: A parameter is not valid. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. Example: >>> report = project.get_performance_report_download( diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index 0472bd391..0b0abab7d 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -400,14 +400,52 @@ def test_refuses_a_filter_value_that_has_no_id( client.connection.get.assert_not_called() - def test_leaves_an_empty_filter_out(self, client, project): + def test_leaves_a_filter_out_when_it_is_not_given(self, client, project): Project.get_performance_metric( - project, "labels_created", "2026-01-01", "2026-01-07", user_ids=[] + project, + "labels_created", + "2026-01-01", + "2026-01-07", + user_ids=None, ) _, params = sent(client) assert "userIds" not in params + @pytest.mark.parametrize( + "argument", + ["user_ids", "user_group_ids", "batch_ids"], + ) + @pytest.mark.parametrize("empty", [[], (), set(), iter([])]) + def test_refuses_an_empty_filter_instead_of_reporting_on_everyone( + self, client, project, argument, empty + ): + with pytest.raises(ValueError, match=f"{argument} is empty"): + Project.get_performance_metric( + project, + "labels_created", + "2026-01-01", + "2026-01-07", + **{argument: empty}, + ) + + client.connection.get.assert_not_called() + + @pytest.mark.parametrize( + "argument", ["project_ids", "owner_organization_ids", "user_ids"] + ) + def test_refuses_an_empty_workspace_filter_too(self, client, argument): + with pytest.raises(ValueError, match=f"{argument} is empty"): + client.get_workspace_performance_metric( + "labels_created", "2026-01-01", "2026-01-07", **{argument: []} + ) + with pytest.raises(ValueError, match=f"{argument} is empty"): + client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07", **{argument: []} + ) + + client.connection.get.assert_not_called() + @pytest.mark.parametrize( "day, expected", [ @@ -585,6 +623,77 @@ def test_sorts_by_an_attribute_of_the_row(self, client, project): assert params["sort"] == "labelsCreated" assert params["order"] == "desc" + @pytest.mark.parametrize( + "sort_by, sent_as", + [ + ("labels_created", "labelsCreated"), + ("avg_time_per_label", "avgTimePerLabel"), + ("rework_percentage", "reworkPercentage"), + # Already in the API's spelling: passed on as it is. + ("labelsCreated", "labelsCreated"), + ("avgTimePerLabel", "avgTimePerLabel"), + ], + ) + def test_spells_the_sort_column_as_the_api_does( + self, client, project, sort_by, sent_as + ): + client.connection.get.return_value = response(table([], total=0)) + + list( + Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", sort_by=sort_by + ) + ) + + assert sent(client)[1]["sort"] == sent_as + + def test_reads_the_table_again_on_a_second_pass(self, client, project): + first = [ + response(table([labeler("user-1")], total=2)), + response(table([labeler("user-2")], total=2, page=2)), + ] + # By the second pass a row has been added, in front of the others. + second = [ + response(table([labeler("user-0")], total=3)), + response(table([labeler("user-1")], total=3, page=2)), + response(table([labeler("user-2")], total=3, page=3)), + ] + client.connection.get.side_effect = first + second + + rows = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07", page_size=1 + ) + once = [row.user_id for row in rows] + twice = [row.user_id for row in rows] + + assert once == ["user-1", "user-2"] + # Not page 1 as it was, followed by pages 2 and 3 as they are now. + assert twice == ["user-0", "user-1", "user-2"] + assert rows.total == 3 + assert [sent(client, call)[1]["page"] for call in range(5)] == [ + "1", + "2", + "1", + "2", + "3", + ] + + def test_reads_the_total_then_the_rows_with_one_request_for_the_first_page( + self, client, project + ): + client.connection.get.return_value = response( + table([labeler("user-1")], total=1) + ) + + rows = Project.get_labeler_performance( + project, "2026-01-01", "2026-01-07" + ) + assert rows.total == 1 + assert [row.user_id for row in rows] == ["user-1"] + assert rows.total == 1 + + assert client.connection.get.call_count == 1 + def test_sends_no_order_without_a_sort(self, client, project): client.connection.get.return_value = response(table([], total=0)) @@ -1003,6 +1112,35 @@ def test_reports_the_failure_itself_once_it_stops_trying( assert client.connection.get.call_count > 1 + @pytest.mark.parametrize( + "failure", + [ + response({"statusCode": 500, "message": "Oops"}, 500), + requests.exceptions.ReadTimeout("read timed out"), + ], + ) + def test_raises_an_error_whose_causes_do_not_lead_back_to_it( + self, client, project, failure + ): + # A chain that loops makes some traceback printers run forever. + if isinstance(failure, Exception): + client.connection.get.side_effect = failure + else: + client.connection.get.return_value = failure + + with pytest.raises(LabelboxError) as raised: + Project.get_performance_metric( + project, "labels_created", "2026-01-01", "2026-01-07" + ) + + seen = [] + error = raised.value + while error is not None: + assert not any(error is earlier for earlier in seen) + seen.append(error) + error = error.__cause__ or error.__context__ + assert len(seen) < 5 + def test_has_time_left_to_try_again_after_a_request_that_timed_out( self, ): From c6aa3f73b256ebfb51ad29474715979fcf6f9365 Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sun, 11 Oct 2026 19:22:35 -0700 Subject: [PATCH 07/13] Page size is at most 100, and score ranges are fractions Follows the API after it was tested against production: the performance service refuses a page of more than 100 rows, and the score histograms' ranges are now given as fractions from 0 to 1, like the scores themselves. Docstrings only; the default page size (50) is unchanged. Co-Authored-By: Claude Opus 5.5 --- libs/labelbox/src/labelbox/client.py | 2 +- libs/labelbox/src/labelbox/schema/performance.py | 7 +++++-- libs/labelbox/src/labelbox/schema/project.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index cb6f4949e..ea6180a24 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -2492,7 +2492,7 @@ def get_workspace_labeler_performance( are ordered by project, then labeler. Sorting by email is not offered. descending (bool): Sort from the highest value down. - page_size (int): Rows fetched per request, at most 200. + page_size (int): Rows fetched per request, at most 100. use_cache (bool): Pass False to read the report again instead of using rows read within the last few minutes. Leave it on while reading a large table, so that every page comes from diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index dcdfd26d8..24cf2e5cf 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -143,9 +143,12 @@ class PerformanceTimeBucket(_CamelCaseMixin): class PerformanceScoreBucket(_CamelCaseMixin): """The number of labels whose agreement score falls in one range. + There are ten equal ranges, from 0 to 0.1 up to 0.9 to 1: fractions, + like every score these reports return. + Attributes: - range_start (float): Lower bound of the range, inclusive. - range_end (float): Upper bound of the range, exclusive. + range_start (float): Lower bound of the range, from 0 to 0.9. + range_end (float): Upper bound of the range, from 0.1 to 1. value (float): How many labels scored in the range. """ diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index a6ac212f1..83eea92d3 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -1959,7 +1959,7 @@ def get_labeler_performance( descending (bool): Sort from high to low. Only used with ``sort_by``. page_size (int): Rows fetched per request while iterating, up to - 200. + 100. use_cache (bool): Pass False to recompute instead of using a result cached within the last few minutes. @@ -2033,7 +2033,7 @@ def get_reviewer_performance( descending (bool): Sort from high to low. Only used with ``sort_by``. page_size (int): Rows fetched per request while iterating, up to - 200. + 100. Returns: PerformanceRows[ProjectReviewerPerformance]: The rows, fetched a From bce08a40640b7a029c3723459a0e43a1b3a790fa Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sun, 11 Oct 2026 20:49:58 -0700 Subject: [PATCH 08/13] A project that is gone has no name or owner The workspace table keeps the rows of a project that can no longer be found, and the API now returns null for its name and owner where it returned the text "N/A". The three attributes are optional. Also says that an Alignerr account is external to everyone, which testing against production showed. Co-Authored-By: Claude Opus 5.5 --- .../src/labelbox/schema/performance.py | 19 ++++++++----- .../tests/unit/schema/test_performance.py | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index 24cf2e5cf..a336437d8 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -83,6 +83,9 @@ class PerformanceMemberType(str, Enum): account belongs to another organization, or an Alignerr account; the Monitor calls these "Workforce members". ``INTERNAL`` is a member of your own organization. + + An Alignerr account is external to everyone, Alignerr's own organization + included. """ INTERNAL = "internal" @@ -317,10 +320,12 @@ class WorkspaceLabelerPerformance(_CamelCaseMixin): :class:`PerformanceMemberType`). ``None`` if the account no longer exists. project_id (str) - project_name (str) - project_organization_id (str): The organization that owns the - project. It differs from yours for a project shared with you. - project_organization_name (str) + project_name (Optional[str]): ``None`` for a project that can no + longer be found. Its figures are still counted. + project_organization_id (Optional[str]): The organization that owns + the project. It differs from yours for a project shared with you, + and is ``None`` for a project that can no longer be found. + project_organization_name (Optional[str]) is_project_deleted (bool) labels_created (int) labels_skipped (int) @@ -344,9 +349,9 @@ class WorkspaceLabelerPerformance(_CamelCaseMixin): email: str member_type: Optional[str] = None project_id: str - project_name: str - project_organization_id: str - project_organization_name: str + project_name: Optional[str] = None + project_organization_id: Optional[str] = None + project_organization_name: Optional[str] = None is_project_deleted: bool labels_created: int labels_skipped: int diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index 0b0abab7d..8b0af0e29 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -976,6 +976,34 @@ def test_filters_and_sorts_the_workspace_table(self, client): assert params["order"] == "desc" assert params["useCache"] == "false" + def test_reads_a_row_whose_project_can_no_longer_be_found(self, client): + client.connection.get.return_value = response( + workspace_table( + [ + workspace_labeler( + "user-1", + "project-gone", + projectName=None, + projectOrganizationId=None, + projectOrganizationName=None, + isProjectDeleted=True, + ) + ], + 1, + ) + ) + + (row,) = client.get_workspace_labeler_performance( + "2026-01-01", "2026-01-07" + ) + + assert row.project_id == "project-gone" + assert row.project_name is None + assert row.project_organization_id is None + assert row.project_organization_name is None + assert row.is_project_deleted is True + assert row.labels_created == 12 + def test_reads_a_row_whose_member_no_longer_exists(self, client): client.connection.get.return_value = response( workspace_table( From e61d6766e8dfe87c310395c78504b2d66b4dcfb5 Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sun, 11 Oct 2026 21:01:26 -0700 Subject: [PATCH 09/13] Projects left out of a split by owner are listed A workspace metric narrowed by owning organization now says, in `skipped_project_ids`, which of those owners' projects it does not count because the caller may not view everyone's performance in them. Co-Authored-By: Claude Opus 5.5 --- libs/labelbox/src/labelbox/schema/performance.py | 5 +++++ libs/labelbox/tests/unit/schema/test_performance.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index a336437d8..868bef2bb 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -184,6 +184,10 @@ class PerformanceMetricSeries(_CamelCaseMixin): organizations a workspace series was narrowed to, if any. member_type (Optional[str]): The member type a workspace series was narrowed to, if any: ``"internal"`` or ``"external"``. + skipped_project_ids (Optional[List[str]]): On a workspace series + narrowed by owning organization, the projects of those owners that + are not counted because you may not view everyone's performance + in them. Empty when none were left out. """ metric: str @@ -199,6 +203,7 @@ class PerformanceMetricSeries(_CamelCaseMixin): project_ids: Optional[List[str]] = None owner_organization_ids: Optional[List[str]] = None member_type: Optional[str] = None + skipped_project_ids: Optional[List[str]] = None class ProjectLabelerPerformance(_CamelCaseMixin): diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index 8b0af0e29..3075957df 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -848,6 +848,7 @@ def test_narrows_a_metric_by_member_type_and_owning_organization( "projectIds": None, "ownerOrganizationIds": ["org-2"], "memberType": "external", + "skippedProjectIds": ["project-9"], } ) @@ -864,6 +865,7 @@ def test_narrows_a_metric_by_member_type_and_owning_organization( assert params["ownerOrganizationIds"] == "org-2" assert series.member_type == "external" assert series.owner_organization_ids == ["org-2"] + assert series.skipped_project_ids == ["project-9"] def test_sends_neither_narrowing_unless_asked(self, client): client.get_workspace_performance_metric( From 0ceae32554f8c9e7ec020e100f3297d584077d70 Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sun, 11 Oct 2026 22:14:42 -0700 Subject: [PATCH 10/13] Fix the tests that fail against staging on every run The `unit-data` and `integration` jobs have failed on develop, and so on every pull request, for five separate reasons. MEA prediction import is gone from the backend. The prediction import service rejects every import of that type ("9 FAILED_PRECONDITION: MEA prediction import has been deprecated"), so the nine tests that call `ModelRun.add_predictions()` are skipped, with that reason. The tenth test in the file only reads a model run; it passes and stays enabled. The embedding fixture could not heal itself. The staging organization is at the cap of 10 custom embeddings. The fixture sweeps stale ones and retries when it hits the cap, but only inside `except LabelboxError`, and since `Client.create_embedding` became a GraphQL mutation the cap arrives as `MalformedQueryException`, which is not a `LabelboxError`. So the sweep never ran, and every test that needs an embedding errored at setup: three in test_embedding.py, test_export_embeddings_custom and test_mmc_all. The helper now catches both, for the create and for the delete of a stale embedding that another job has already removed. Validation messages are masked. Deployed API environments replace the message of every GraphQL validation error with "GraphQL validation error". `test_semantic_error` and `test_query_complexity_error` expected the detailed text; they now accept the masked message, or the detailed one from an API that does not hide schema details. Nothing else is accepted. Two tests raced the backend. A global key is not free the moment its data row is deleted (the import checks keys against the search index, which drops the row a little later), and three labels are not listed the moment they are created. Both tests now wait for the condition they depend on. Co-Authored-By: Claude Opus 5.5 --- .../test_mea_prediction_import.py | 20 +++ libs/labelbox/tests/embedding_cleanup.py | 16 ++- .../tests/integration/test_client_errors.py | 29 +++- .../tests/integration/test_data_rows.py | 44 ++++++- libs/labelbox/tests/integration/test_label.py | 5 +- .../tests/unit/test_embedding_cleanup.py | 124 +++++++++++++++++- 6 files changed, 218 insertions(+), 20 deletions(-) diff --git a/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py b/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py index 9a286b55e..8281ac5f1 100644 --- a/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py +++ b/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py @@ -25,7 +25,19 @@ """ +# Every test that creates an MEA prediction import is skipped: the backend +# rejects the createModelErrorAnalysisPredictionImport mutation, which is the +# only thing ModelRun.add_predictions() and MEAPredictionImport.create*() call. +# Tests that only read a model run still work and stay enabled. +mea_prediction_import_removed = pytest.mark.skip( + reason="MEA (Model Error Analysis) prediction import is no longer supported " + "by the backend: the prediction import service rejects every import of " + "that type, so ModelRun.add_predictions() fails with '9 FAILED_PRECONDITION: " + "MEA prediction import has been deprecated' (python-monorepo #2515)." +) + +@mea_prediction_import_removed @pytest.mark.order(1) def test_create_from_objects( model_run_with_data_rows, @@ -54,6 +66,7 @@ def test_create_from_objects( ) +@mea_prediction_import_removed def test_create_from_objects_global_key( client, model_run_with_data_rows, @@ -83,6 +96,7 @@ def test_create_from_objects_global_key( ) +@mea_prediction_import_removed def test_create_from_objects_with_confidence( predictions_with_confidence, model_run_with_data_rows, @@ -118,6 +132,7 @@ def test_create_from_objects_with_confidence( ) +@mea_prediction_import_removed def test_create_from_objects_all_project_labels( model_run_with_all_project_labels, object_predictions_for_annotation_import, @@ -196,6 +211,7 @@ def test_model_run_project_labels( ) +@mea_prediction_import_removed def test_create_from_label_objects( model_run_with_data_rows, object_predictions_for_annotation_import, @@ -266,6 +282,7 @@ def test_create_from_label_objects( ) +@mea_prediction_import_removed def test_create_from_local_file( tmp_path, model_run_with_data_rows, @@ -301,6 +318,7 @@ def test_create_from_local_file( ) +@mea_prediction_import_removed def test_predictions_with_custom_metrics( model_run, object_predictions_for_annotation_import, @@ -328,6 +346,7 @@ def test_predictions_with_custom_metrics( ) +@mea_prediction_import_removed def test_get(client, model_run_with_data_rows, annotation_import_test_helpers): name = str(uuid.uuid4()) url = "https://storage.googleapis.com/labelbox-public-bucket/predictions_test_v2.ndjson" @@ -344,6 +363,7 @@ def test_get(client, model_run_with_data_rows, annotation_import_test_helpers): annotation_import.wait_until_done() +@mea_prediction_import_removed @pytest.mark.slow def test_wait_till_done(model_run_predictions, model_run_with_data_rows): name = str(uuid.uuid4()) diff --git a/libs/labelbox/tests/embedding_cleanup.py b/libs/labelbox/tests/embedding_cleanup.py index eed2a0062..ab0da8c72 100644 --- a/libs/labelbox/tests/embedding_cleanup.py +++ b/libs/labelbox/tests/embedding_cleanup.py @@ -5,10 +5,15 @@ from random import uniform from typing import Any, Callable, Iterable, List, Optional -from lbox.exceptions import LabelboxError +from lbox.exceptions import LabelboxError, MalformedQueryException EMBEDDING_NAME_PREFIX_V2 = "sdk-int-ci-v2-" EMBEDDING_CAP_ERROR_SNIPPET = "Max limit of custom embeddings" +# Creating and deleting an embedding are GraphQL mutations, and the API +# reports the cap with the MALFORMED_REQUEST code. The SDK raises that as +# MalformedQueryException, which does not derive from LabelboxError, so +# catching LabelboxError alone lets the cap error through unhealed. +EMBEDDING_CAP_ERROR_TYPES = (LabelboxError, MalformedQueryException) EMBEDDING_STALE_TTL_SECONDS = 12 * 3600 LEGACY_NAME_RE = re.compile(r"^sdk-int-[0-9a-f]{32}$") @@ -40,7 +45,7 @@ def select_stale_embeddings(embeddings: Iterable[Any], now: float) -> List[Any]: return stale_embeddings -def is_embedding_cap_error(error: LabelboxError) -> bool: +def is_embedding_cap_error(error: Exception) -> bool: return EMBEDDING_CAP_ERROR_SNIPPET in str(error) @@ -59,7 +64,7 @@ def create_embedding_with_heal( for attempt in range(1, _MAX_CREATE_ATTEMPTS + 1): try: return create_embedding() - except LabelboxError as error: + except EMBEDDING_CAP_ERROR_TYPES as error: if not is_embedding_cap_error(error): raise @@ -77,8 +82,11 @@ def create_embedding_with_heal( for embedding in stale_embeddings: try: delete_embedding(embedding) - except LabelboxError: + except EMBEDDING_CAP_ERROR_TYPES: # Another worker may have deleted the same stale embedding. + # Several jobs hit the cap at once and sweep the same + # names, and the delete is a GraphQL mutation too, so it + # can fail with either kind of error. pass last_swept_count = len(stale_embeddings) diff --git a/libs/labelbox/tests/integration/test_client_errors.py b/libs/labelbox/tests/integration/test_client_errors.py index b5444eadf..f070387cb 100644 --- a/libs/labelbox/tests/integration/test_client_errors.py +++ b/libs/labelbox/tests/integration/test_client_errors.py @@ -9,6 +9,13 @@ import labelbox.client from labelbox.schema.media_type import MediaType +# Deployed API environments replace the message of every GraphQL validation +# error with this fixed string, so that schema details (field names, "Did you +# mean ..." hints) are not exposed to clients. The error code is unchanged. +# Only an API that does not hide schema details, such as one run locally, +# still returns the detailed message. +MASKED_VALIDATION_ERROR_MESSAGE = "GraphQL validation error" + def test_missing_api_key(): key = os.environ.get(lbox.request_client._LABELBOX_API_KEY, None) @@ -41,7 +48,10 @@ def test_syntax_error(client): def test_semantic_error(client): with pytest.raises(lbox.exceptions.InvalidQueryError) as excinfo: client.execute("query {bbb {id}}", check_naming=False) - assert excinfo.value.message.startswith('Cannot query field "bbb"') + message = excinfo.value.message + assert message == MASKED_VALIDATION_ERROR_MESSAGE or message.startswith( + 'Cannot query field "bbb"' + ) def test_timeout_error(client, project): @@ -59,11 +69,24 @@ def test_timeout_error(client, project): def test_query_complexity_error(client): - with pytest.raises(lbox.exceptions.ValidationFailedError) as excinfo: + # The complexity limit is a validation rule, so where validation messages + # are masked its "Query complexity limit exceeded" message is replaced as + # well. The client then has nothing left to tell it apart from any other + # validation failure and raises InvalidQueryError; ValidationFailedError is + # only reachable against an API that does not hide schema details. + with pytest.raises( + ( + lbox.exceptions.InvalidQueryError, + lbox.exceptions.ValidationFailedError, + ) + ) as excinfo: client.execute( "{projects {datasets {dataRows {labels {id}}}}}", check_naming=False ) - assert excinfo.value.message == "Query complexity limit exceeded" + if isinstance(excinfo.value, lbox.exceptions.ValidationFailedError): + assert excinfo.value.message == "Query complexity limit exceeded" + else: + assert excinfo.value.message == MASKED_VALIDATION_ERROR_MESSAGE def test_resource_not_found_error(client): diff --git a/libs/labelbox/tests/integration/test_data_rows.py b/libs/labelbox/tests/integration/test_data_rows.py index ba3c108d5..2f92c5b48 100644 --- a/libs/labelbox/tests/integration/test_data_rows.py +++ b/libs/labelbox/tests/integration/test_data_rows.py @@ -1,5 +1,6 @@ import json import os +import time import uuid from datetime import datetime, timezone from tempfile import NamedTemporaryFile @@ -1029,9 +1030,41 @@ def _wait_for_task(task, timeout_seconds=600): task.wait_till_done(timeout_seconds=timeout_seconds) if task.status == "IN_PROGRESS": task.refresh() - assert ( - task.status == "COMPLETE" - ), f"Task {task.uid} did not complete within {timeout_seconds}s: {task.status}" + assert task.status == "COMPLETE", ( + f"Task {task.uid} did not complete within {timeout_seconds}s: {task.status}" + ) + + +def _create_data_row_once_global_key_is_released( + dataset, data_row_payload, global_key, timeout_seconds=180 +): + """Create a data row with the global key of a data row deleted just before. + + The key is not free the moment `delete()` returns: the import looks global + keys up in the catalog search index, and the deleted data row leaves that + index a little later. Until it has, the import reports the key as a + duplicate. So repeat the import while that is the only thing it reports, + and fail on any other error or if the key is not released in time. + """ + still_held = f"Duplicate global key: '{global_key}'" + sleep_time = 5 + deadline = time.time() + timeout_seconds + while True: + task = dataset.create_data_rows([data_row_payload]) + _wait_for_task(task) + + failed_data_rows = task.failed_data_rows + if not failed_data_rows: + return task + + messages = [row["message"] for row in failed_data_rows] + assert messages == [still_held], f"Unexpected errors: {messages}" + if time.time() >= deadline: + raise AssertionError( + f"Global key '{global_key}' was still reported as a duplicate " + f"{timeout_seconds}s after its data row was deleted" + ) + time.sleep(sleep_time) def test_data_row_delete_and_create_with_same_global_key( @@ -1065,8 +1098,9 @@ def test_data_row_delete_and_create_with_same_global_key( client.get_data_row(new_data_row_id).delete() # should successfully insert new datarow now - task = dataset.create_data_rows([data_row_payload]) - _wait_for_task(task) + task = _create_data_row_once_global_key_is_released( + dataset, data_row_payload, global_key_1 + ) assert task.result[0]["global_key"] == global_key_1 diff --git a/libs/labelbox/tests/integration/test_label.py b/libs/labelbox/tests/integration/test_label.py index 54e929efc..8de6c5184 100644 --- a/libs/labelbox/tests/integration/test_label.py +++ b/libs/labelbox/tests/integration/test_label.py @@ -41,12 +41,15 @@ def test_label_update(configured_project_with_label): assert label.label == "something else" -def test_label_bulk_deletion(configured_project_with_label): +def test_label_bulk_deletion(configured_project_with_label, label_helpers): project, _, _, _ = configured_project_with_label for _ in range(2): # only run twice, already have one label in the fixture project.create_label() + # A finished label import is not listed by `project.labels()` right away + # (the fixture waits for its own label the same way), so wait for all three + label_helpers.wait_for_labels(project, 3) labels = project.labels() l1 = next(labels) l2 = next(labels) diff --git a/libs/labelbox/tests/unit/test_embedding_cleanup.py b/libs/labelbox/tests/unit/test_embedding_cleanup.py index 882648d7a..cb179404b 100644 --- a/libs/labelbox/tests/unit/test_embedding_cleanup.py +++ b/libs/labelbox/tests/unit/test_embedding_cleanup.py @@ -2,8 +2,9 @@ from unittest.mock import Mock, call, patch import pytest -from lbox.exceptions import LabelboxError +from lbox.exceptions import LabelboxError, MalformedQueryException +from labelbox import Client from tests.embedding_cleanup import ( EMBEDDING_CAP_ERROR_SNIPPET, EMBEDDING_NAME_PREFIX_V2, @@ -42,8 +43,21 @@ def _v2_name(created_at: int, suffix: str = "0123456789") -> str: return f"{EMBEDDING_NAME_PREFIX_V2}{created_at}-{suffix}" -def _cap_error() -> LabelboxError: - return LabelboxError(f"{EMBEDDING_CAP_ERROR_SNIPPET}: 10") +# The message the API answers with when the organization is at the cap. +CAP_MESSAGE = f"{EMBEDDING_CAP_ERROR_SNIPPET} reached : 10 per organization" + + +def _cap_error() -> MalformedQueryException: + # The API reports the cap with the MALFORMED_REQUEST code, which the SDK + # raises as MalformedQueryException: not a LabelboxError. + return MalformedQueryException(CAP_MESSAGE) + + +def _graphql_response(body: dict) -> Mock: + response = Mock() + response.status_code = 200 + response.json.return_value = body + return response def test_build_and_parse_embedding_name_round_trip(): @@ -96,7 +110,11 @@ def test_select_stale_embeddings_keeps_only_expired_custom_v2_names(): def test_embedding_cap_error_matcher_is_fail_closed(): assert is_embedding_cap_error(_cap_error()) + assert is_embedding_cap_error(LabelboxError(CAP_MESSAGE)) assert not is_embedding_cap_error(LabelboxError("permission denied")) + assert not is_embedding_cap_error( + MalformedQueryException("Name contains invalid characters") + ) def test_workflow_boolean_input_is_parsed_defensively(): @@ -140,6 +158,56 @@ def test_heal_retries_twice_then_creates_and_deletes_only_stale_v2(): assert sleep.call_args_list == [call(4), call(4)] +def test_heal_sweeps_on_the_cap_error_client_create_embedding_raises(): + # Goes through Client.execute, so the heal is checked against the + # exception the SDK raises for the API's answer, not against a stand-in. + now_value = 1_725_000_000 + stale = _embedding( + "stale", + _v2_name(now_value - EMBEDDING_STALE_TTL_SECONDS - 1), + ) + delete = Mock() + client = Client(api_key="api_key") + responses = [ + _graphql_response( + { + "errors": [ + { + "message": CAP_MESSAGE, + "extensions": {"code": "MALFORMED_REQUEST"}, + } + ] + } + ), + _graphql_response( + { + "data": { + "createEmbedding": { + "id": "created", + "name": "custom", + "dims": 8, + "custom": True, + } + } + } + ), + ] + + with patch.object(client.connection, "send", side_effect=responses): + embedding = create_embedding_with_heal( + create_embedding=lambda: client.create_embedding("custom", 8), + list_embeddings=Mock(return_value=[stale]), + delete_embedding=delete, + sleep=Mock(), + now=Mock(return_value=now_value), + retry_delay=Mock(return_value=4), + print_fn=Mock(), + ) + + assert embedding.id == "created" + delete.assert_called_once_with(stale) + + def test_heal_third_cap_error_is_terminal_without_another_sweep_or_sleep(): final_error = _cap_error() create = Mock(side_effect=[_cap_error(), _cap_error(), final_error]) @@ -148,7 +216,7 @@ def test_heal_third_cap_error_is_terminal_without_another_sweep_or_sleep(): sleep = Mock() print_fn = Mock() - with pytest.raises(LabelboxError) as raised: + with pytest.raises(MalformedQueryException) as raised: create_embedding_with_heal( create_embedding=create, list_embeddings=list_embeddings, @@ -170,14 +238,21 @@ def test_heal_third_cap_error_is_terminal_without_another_sweep_or_sleep(): ) -def test_heal_non_cap_error_is_reraised_without_side_effects(): - non_cap_error = LabelboxError("permission denied") +@pytest.mark.parametrize( + "non_cap_error", + [ + LabelboxError("permission denied"), + MalformedQueryException("Name contains invalid characters"), + ], + ids=["LabelboxError", "MalformedQueryException"], +) +def test_heal_non_cap_error_is_reraised_without_side_effects(non_cap_error): create = Mock(side_effect=non_cap_error) list_embeddings = Mock() delete = Mock() sleep = Mock() - with pytest.raises(LabelboxError) as raised: + with pytest.raises(type(non_cap_error)) as raised: create_embedding_with_heal( create_embedding=create, list_embeddings=list_embeddings, @@ -257,3 +332,38 @@ def test_cleanup_real_run_attempts_every_candidate_and_reports_failures(): assert run_cleanup(client, dry_run=False, now=1_725_000_000) == 1 first.delete.assert_called_once_with() second.delete.assert_called_once_with() + + +@pytest.mark.parametrize( + "already_gone", + [ + MalformedQueryException("Embedding not found"), + LabelboxError("Embedding not found"), + ], +) +def test_heal_carries_on_when_a_stale_embedding_is_already_gone(already_gone): + # Jobs for several Python versions hit the cap together and sweep the same + # stale names, so a delete can find its embedding already deleted. The + # delete is a GraphQL mutation, so it can fail with either kind of error. + now_value = 1_725_000_000 + gone = _embedding( + "gone", _v2_name(now_value - EMBEDDING_STALE_TTL_SECONDS - 2) + ) + stale = _embedding( + "stale", _v2_name(now_value - EMBEDDING_STALE_TTL_SECONDS - 1) + ) + created = object() + delete = Mock(side_effect=[already_gone, None]) + + result = create_embedding_with_heal( + create_embedding=Mock(side_effect=[_cap_error(), created]), + list_embeddings=Mock(return_value=[gone, stale]), + delete_embedding=delete, + sleep=Mock(), + now=Mock(return_value=now_value), + retry_delay=Mock(return_value=4), + print_fn=Mock(), + ) + + assert result is created + assert delete.call_args_list == [call(gone), call(stale)] From 9c7dd80f8bc79098a22f9a646ad5e4b4152f2c6f Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sun, 11 Oct 2026 22:19:05 -0700 Subject: [PATCH 11/13] Leave an existing assert formatted as the project's formatter wants it A newer formatter than the one CI runs had restyled one statement that this change does not touch. Co-Authored-By: Claude Opus 5.5 --- libs/labelbox/tests/integration/test_data_rows.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/tests/integration/test_data_rows.py b/libs/labelbox/tests/integration/test_data_rows.py index 2f92c5b48..99cc051bb 100644 --- a/libs/labelbox/tests/integration/test_data_rows.py +++ b/libs/labelbox/tests/integration/test_data_rows.py @@ -1030,9 +1030,9 @@ def _wait_for_task(task, timeout_seconds=600): task.wait_till_done(timeout_seconds=timeout_seconds) if task.status == "IN_PROGRESS": task.refresh() - assert task.status == "COMPLETE", ( - f"Task {task.uid} did not complete within {timeout_seconds}s: {task.status}" - ) + assert ( + task.status == "COMPLETE" + ), f"Task {task.uid} did not complete within {timeout_seconds}s: {task.status}" def _create_data_row_once_global_key_is_released( From 259ac04de7a273f2227eaec6e1693ee7b1f79bda Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sun, 11 Oct 2026 22:31:37 -0700 Subject: [PATCH 12/13] Fix two more tests that race other runs of the suite Both failed once in the first run with the other fixes in place. `test_update_project_resource_tags` compared the number of resource tags in the organization before and after creating one (346 > 347). The suite runs for five Python versions at once against one organization, creating and deleting tags, so the count can go either way. It now looks for the two tags it created. `test_model_run_project_labels` read the first label of each data row from a model run export taken straight after `upsert_labels`. The export catches up a little later, and until then a data row comes out with no label for the run (IndexError). It now exports again while any is missing, for up to three minutes. Co-Authored-By: Claude Opus 5.5 --- .../test_mea_prediction_import.py | 63 +++++++++++++++---- .../tests/integration/test_project.py | 10 +-- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py b/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py index 8281ac5f1..96dbe0556 100644 --- a/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py +++ b/libs/labelbox/tests/data/annotation_import/test_mea_prediction_import.py @@ -1,3 +1,4 @@ +import time import uuid from labelbox import parser from labelbox.data.annotation_types.annotation import ObjectAnnotation @@ -162,24 +163,62 @@ def test_create_from_objects_all_project_labels( ) +def _export_model_run_labels( + model_run: ModelRun, expected_data_row_count: int, timeout_seconds=180 +): + """The first label of each data row of a model run, once the export has them. + + `upsert_labels` returns when the labels are attached to the model run, and + the export catches up a little later: until then a data row comes out with + no label for the run. So export again while any is missing, and fail if + they have not all arrived in time. + """ + deadline = time.time() + timeout_seconds + while True: + export_task = model_run.export() + export_task.wait_till_done() + labels_by_data_row = [ + ( + data_row.json["data_row"]["id"], + data_row.json["experiments"][model_run.model_id]["runs"][ + model_run.uid + ]["labels"], + ) + for data_row in export_task.get_buffered_stream() + ] + if len(labels_by_data_row) >= expected_data_row_count and all( + labels for _, labels in labels_by_data_row + ): + return [ + (data_row_id, labels[0]) + for data_row_id, labels in labels_by_data_row + ] + + missing = [ + data_row_id + for data_row_id, labels in labels_by_data_row + if not labels + ] + assert time.time() < deadline, ( + f"After {timeout_seconds}s the export of model run {model_run.uid} " + f"had {len(labels_by_data_row)} of {expected_data_row_count} data " + f"rows, {len(missing)} of them without a label" + ) + time.sleep(5) + + def test_model_run_project_labels( model_run_with_all_project_labels: ModelRun, model_run_predictions ): model_run = model_run_with_all_project_labels - export_task = model_run.export() - export_task.wait_till_done() - stream = export_task.get_buffered_stream() + expected_data_row_count = len( + set(prediction["dataRow"]["id"] for prediction in model_run_predictions) + ) # exports to list of tuples (data_row_id, label) needed to adapt test to export v2 instead of export v1 since data rows ids are not at label level in export v2. - model_run_exported_labels = [ - ( - data_row.json["data_row"]["id"], - data_row.json["experiments"][model_run.model_id]["runs"][ - model_run.uid - ]["labels"][0], - ) - for data_row in stream - ] + model_run_exported_labels = _export_model_run_labels( + model_run, expected_data_row_count + ) labels_indexed_by_name = {} diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index a1bb23d52..dd31ed33e 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -95,18 +95,18 @@ def delete_tag(tag_id: str): assert tagA.uid is not None tags = org.get_resource_tags() - lenA = len(tags) - assert lenA > 0 + assert tagA.uid in {tag.uid for tag in tags} tagB = client.get_organization().create_resource_tag(tagB) assert tagB.text == textB assert "#" + tagB.color == colorB assert tagB.uid is not None + # Look for the two tags themselves, not for a longer list: other runs of + # this suite create and delete tags in the same organization meanwhile, so + # the number of tags can go either way between two readings. tags = client.get_organization().get_resource_tags() - lenB = len(tags) - assert lenB > 0 - assert lenB > lenA + assert {tagA.uid, tagB.uid} <= {tag.uid for tag in tags} project_resource_tag = client.get_project( p1.uid From e983904276a25b8a32442ab90d4005a91aa76af7 Mon Sep 17 00:00:00 2001 From: Manu Sharma Date: Sun, 11 Oct 2026 22:32:37 -0700 Subject: [PATCH 13/13] Say what the server answered when the archived status cannot be read `Client.is_feature_schema_archived` raised "Failed to get the feature schema archived status." for any answer other than 200 or 404, without the status or the body. `test_feature_schema_is_archived` failed that way once against staging and the log could not say why (a rate limit, or a server error). The message now carries both. Co-Authored-By: Claude Opus 5.5 --- libs/labelbox/src/labelbox/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 75755afcc..05e8d26f9 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -1861,7 +1861,8 @@ def is_feature_schema_archived( raise ResourceNotFoundError(Ontology, ontology_id) else: raise LabelboxError( - "Failed to get the feature schema archived status." + "Failed to get the feature schema archived status. " + f"The server answered {response.status_code}: {response.text}" ) def get_model_slice(self, slice_id) -> ModelSlice: