Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/labelbox/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ Labelbox Python SDK Documentation
ontology_kind
organization
pagination
performance
project
project-model-config
project-sync
Expand Down
6 changes: 6 additions & 0 deletions docs/labelbox/performance.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Performance
===============================================================================================

.. automodule:: labelbox.schema.performance
:members: PerformanceMetric, PerformanceMetricSeries, PerformanceTimeBucket, PerformanceScoreBucket, PerformanceHandlingTime, ProjectLabelerPerformance, ProjectReviewerPerformance, WorkspaceLabelerPerformance, PerformanceReportDownload, PerformanceRows, PerformanceInterval, PerformanceDeletedLabels, PerformanceMemberType
:show-inheritance:
14 changes: 14 additions & 0 deletions libs/labelbox/src/labelbox/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,20 @@
from labelbox.schema.organization import Organization
from labelbox.schema.project import Project
from labelbox.schema.project_model_config import ProjectModelConfig
from labelbox.schema.performance import (
PerformanceDeletedLabels,
PerformanceInterval,
PerformanceMemberType,
PerformanceMetric,
PerformanceMetricSeries,
PerformanceReportDownload,
PerformanceRows,
PerformanceScoreBucket,
PerformanceTimeBucket,
ProjectLabelerPerformance,
ProjectReviewerPerformance,
WorkspaceLabelerPerformance,
)
from labelbox.schema.project_overview import (
ProjectOverview,
ProjectOverviewDetailed,
Expand Down
211 changes: 210 additions & 1 deletion libs/labelbox/src/labelbox/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
from labelbox.schema.identifiables import DataRowIds, GlobalKeys
from labelbox.schema.label_score import LabelScore
from labelbox.schema.labeling_frontend import LabelingFrontend
from labelbox.schema import performance as _performance
from labelbox.schema.labeling_service_dashboard import LabelingServiceDashboard
from labelbox.schema.media_type import (
MediaType,
Expand Down Expand Up @@ -1861,7 +1862,8 @@ def is_feature_schema_archived(
raise ResourceNotFoundError(Ontology, ontology_id)
else:
raise LabelboxError(
"Failed to get the feature schema archived status."
"Failed to get the feature schema archived status. "
f"The server answered {response.status_code}: {response.text}"
)

def get_model_slice(self, slice_id) -> ModelSlice:
Expand Down Expand Up @@ -2327,6 +2329,213 @@ def get_labeling_service_dashboards(
"""
return LabelingServiceDashboard.get_all(self, search_query=search_query)

def get_performance_metrics(self) -> List[_performance.PerformanceMetric]:
"""Lists the performance metrics that can be queried.

These are the metrics a project's Performance page draws. Each has a
name to pass to :meth:`Project.get_performance_metric` or
:meth:`get_workspace_performance_metric`, a unit and a definition.

Returns:
List[PerformanceMetric]: The metrics available to your
organization.

Example:
>>> for metric in client.get_performance_metrics():
>>> print(metric.name, metric.unit, metric.description)
"""
return _performance.get_performance_metrics(self)

def get_workspace_performance_metric(
self,
metric: str,
start_date: _performance.DateLike,
end_date: _performance.DateLike,
interval: Union[
_performance.PerformanceInterval, str
] = _performance.PerformanceInterval.DAY,
project_ids: Optional[List[str]] = None,
user_ids: Optional[List[str]] = None,
batch_ids: Optional[List[str]] = None,
deleted_labels: Union[
_performance.PerformanceDeletedLabels, str
] = _performance.PerformanceDeletedLabels.INCLUDE,
use_cache: bool = True,
member_type: Optional[
Union[_performance.PerformanceMemberType, str]
] = None,
owner_organization_ids: Optional[List[str]] = None,
) -> _performance.PerformanceMetricSeries:
"""Returns one performance metric across the workspace.

This is the workspace Monitor's data: every project your organization
owns and every project another organization shares with it, or only
the projects named in ``project_ids``.

Args:
metric (str): A metric name from :meth:`get_performance_metrics`,
for example ``"labels_created"``.
start_date (date, datetime or str): First day of the period, in
UTC. A string is a day such as ``"2026-01-31"``.
end_date (date, datetime or str): Last day of the period. A
request can cover at most 366 days.
interval (PerformanceInterval or str): The width of each bucket:
day (the default), week, month, quarter or year.
project_ids (Optional[List[str]]): Narrow the report to these
projects. You then need performance access to each of them
instead of the organization-wide permission.
user_ids (Optional[List[str]]): Count only work by these users.
batch_ids (Optional[List[str]]): Count only these batches.
deleted_labels (PerformanceDeletedLabels or str): Whether labels
deleted since are counted. Included by default.
use_cache (bool): Pass False to recompute instead of using a
result cached within the last few minutes.
member_type (Optional[PerformanceMemberType or str]): Count only
work by members of your own organization (``"internal"``) or
only by members of other organizations, such as a workforce
provider's (``"external"``). The Monitor's Member Type
filter.
owner_organization_ids (Optional[List[str]]): Count only projects
owned by these organizations. Use it to separate the projects
you own from the ones shared with you.

``member_type`` and ``owner_organization_ids`` report on other
members, so they need permission to view all performance in every
project the report covers.

Returns:
PerformanceMetricSeries: The metric as a value per interval.

Raises:
lbox.exceptions.ResourceNotFoundError: The metric does not exist,
or a project named is not one you can see.
lbox.exceptions.AuthorizationError: You may not view performance
across the organization, or not everyone's in a project the
report covers.
lbox.exceptions.InvalidQueryError: A parameter is not valid.
ValueError: A filter was given as an empty list. Pass ``None``
to leave a filter off; an empty list is not read as one.

Example:
>>> series = client.get_workspace_performance_metric(
>>> "labels_created", "2026-01-01", "2026-01-31", interval="week")
>>> for bucket in series.buckets:
>>> print(bucket.start.date(), bucket.value)
>>>
>>> # The same, for the workforce provider's members only.
>>> series = client.get_workspace_performance_metric(
>>> "labels_created", "2026-01-01", "2026-01-31",
>>> member_type="external")
"""
return _performance.get_workspace_metric(
self,
metric,
start_date,
end_date,
interval=interval,
project_ids=project_ids,
user_ids=user_ids,
batch_ids=batch_ids,
deleted_labels=deleted_labels,
use_cache=use_cache,
member_type=member_type,
owner_organization_ids=owner_organization_ids,
)

def get_workspace_labeler_performance(
self,
start_date: _performance.DateLike,
end_date: _performance.DateLike,
project_ids: Optional[List[str]] = None,
user_ids: Optional[List[str]] = None,
batch_ids: Optional[List[str]] = None,
deleted_labels: Union[
_performance.PerformanceDeletedLabels, str
] = _performance.PerformanceDeletedLabels.INCLUDE,
member_type: Optional[
Union[_performance.PerformanceMemberType, str]
] = None,
owner_organization_ids: Optional[List[str]] = None,
sort_by: Optional[str] = None,
descending: bool = False,
page_size: int = 50,
use_cache: bool = True,
) -> _performance.PerformanceRows[_performance.WorkspaceLabelerPerformance]:
"""Returns per-labeler totals across the workspace.

This is the workspace Monitor's member table: one row for each
labeler on each project they worked on in the period. It covers the
projects your organization owns and the ones other organizations
share with it. Each row says which organization owns the project
(``project_organization_id``) and whether the labeler is one of your
own members (``member_type``).

Args:
start_date (date, datetime or str): First day of the period, in
UTC. A string is a day such as ``"2026-01-31"``.
end_date (date, datetime or str): Last day of the period. A
request can cover at most 366 days.
project_ids (Optional[List[str]]): Narrow the report to these
projects. You then need performance access to each of them
instead of the organization-wide permission.
user_ids (Optional[List[str]]): Return only these users.
batch_ids (Optional[List[str]]): Count only these batches.
deleted_labels (PerformanceDeletedLabels or str): Whether labels
deleted since are counted. Included by default.
member_type (Optional[PerformanceMemberType or str]): Return only
members of your own organization (``"internal"``) or only
members of other organizations, such as a workforce
provider's (``"external"``).
owner_organization_ids (Optional[List[str]]): Return only
projects owned by these organizations.
sort_by (Optional[str]): A column to sort by, named as the row's
attribute, for example ``"labels_created"``. Without it, rows
are ordered by project, then labeler. Sorting by email is not
offered.
descending (bool): Sort from the highest value down.
page_size (int): Rows fetched per request, at most 100.
use_cache (bool): Pass False to read the report again instead of
using rows read within the last few minutes. Leave it on
while reading a large table, so that every page comes from
the same rows.

Returns:
PerformanceRows[WorkspaceLabelerPerformance]: One row per labeler
and project, fetched a page at a time as you iterate. Emails are
masked for labelers you may not identify.

Raises:
lbox.exceptions.ResourceNotFoundError: A project named is not one
you can see.
lbox.exceptions.AuthorizationError: You may not view performance
across the organization.
lbox.exceptions.InvalidQueryError: A parameter is not valid.
ValueError: A filter was given as an empty list. Pass ``None``
to leave a filter off; an empty list is not read as one.

Example:
>>> rows = client.get_workspace_labeler_performance(
>>> "2026-01-01", "2026-01-31", member_type="external")
>>> print(rows.total)
>>> for row in rows:
>>> print(row.project_name, row.email, row.labels_created)
"""
return _performance.get_workspace_labelers(
self,
start_date,
end_date,
project_ids=project_ids,
user_ids=user_ids,
batch_ids=batch_ids,
deleted_labels=deleted_labels,
member_type=member_type,
owner_organization_ids=owner_organization_ids,
sort_by=sort_by,
descending=descending,
page_size=page_size,
use_cache=use_cache,
)

def get_task_by_id(self, task_id: str) -> Union[Task, DataUpsertTask]:
"""
Fetches a task by ID.
Expand Down
Loading
Loading