diff --git a/docs/labelbox/performance.rst b/docs/labelbox/performance.rst index 70c6ff347..730edbaa5 100644 --- a/docs/labelbox/performance.rst +++ b/docs/labelbox/performance.rst @@ -2,5 +2,5 @@ Performance =============================================================================================== .. automodule:: labelbox.schema.performance - :members: PerformanceMetric, PerformanceMetricSeries, PerformanceTimeBucket, PerformanceScoreBucket, PerformanceHandlingTime, ProjectLabelerPerformance, ProjectReviewerPerformance, WorkspaceLabelerPerformance, PerformanceReportDownload, PerformanceRows, PerformanceTableChangedError, PerformanceInterval, PerformanceDeletedLabels, PerformanceMemberType + :members: PerformanceMetric, PerformanceMetricSeries, PerformanceTimeBucket, PerformanceScoreBucket, PerformanceHandlingTime, ProjectLabelerPerformance, ProjectReviewerPerformance, WorkspaceLabelerPerformance, PerformanceReportDownload, SignInSession, SignInEvent, PerformanceRows, PerformanceCursorRows, SignInEventRows, PerformanceTableChangedError, PerformanceInterval, PerformanceDeletedLabels, PerformanceMemberType, SignInSessionStatus, SignInStartKind, SignInEndReason, SignInEventType :show-inheritance: diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index 3e7395549..c9c4c3b71 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -103,6 +103,7 @@ from labelbox.schema.project import Project from labelbox.schema.project_model_config import ProjectModelConfig from labelbox.schema.performance import ( + PerformanceCursorRows, PerformanceDeletedLabels, PerformanceInterval, PerformanceMemberType, @@ -115,6 +116,13 @@ PerformanceTimeBucket, ProjectLabelerPerformance, ProjectReviewerPerformance, + SignInEndReason, + SignInEvent, + SignInEventRows, + SignInEventType, + SignInSession, + SignInSessionStatus, + SignInStartKind, WorkspaceLabelerPerformance, ) from labelbox.schema.project_overview import ( diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index df1989dea..f5f97cfc8 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -2592,6 +2592,200 @@ def get_workspace_labeler_performance( user_group_ids=user_group_ids, ) + def get_sign_in_sessions( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + status: Optional[Union[_performance.SignInSessionStatus, str]] = None, + page_size: int = 100, + ) -> _performance.PerformanceCursorRows[_performance.SignInSession]: + """Returns the sign-in sessions of your organization's members. + + A session is one sign-in in one browser, or a switch into this + workspace from the same person's user in another one. It lasts until + the member signs out, until Labelbox ends it (a new sign-in in the + same browser, a switch to another workspace, the member's removal) + or, when neither happened, until every credential the sign-in issued + has lapsed. That last end is derived when the session is read: + ``end_reason`` is ``"MAX_DURATION_REACHED"``, ``end_derived`` is + True and ``ended_at`` is ``expires_at``. It is the latest the session + could last, not when the member stopped working. Sessions record + sign-ins and how they ended, not activity, so ``duration_seconds`` is + not time worked, whatever the end. + + Only sessions started in your organization are listed, so another + organization's users (a workforce provider's members, Alignerr + accounts) have none, even when they work on your projects or are in + your user groups. A member who has since moved to another + organization keeps the sessions they started here, with ``email`` + None and ``member_deleted`` True. Only sessions that started after + Labelbox began recording them, in October 2026, are known. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. + end_date (date, datetime or str): Last day of the period. A + request can cover at most 31 days. + user_ids (Optional[List[str]]): Return only these users' + sessions. Up to 100. + user_group_ids (Optional[List[str]]): Return only the sessions of + the current members of these user groups of your + organization. Up to 25 groups, with at most 5,000 members + between them. The groups count only if you have the + organization-wide permission to view user groups, and a + restricted API key has it too. Without it no group adds + anyone, and no sessions are returned; no error is raised. A + group of another organization adds no one either, and + sessions started in another organization are never listed, + even for a member of the groups. Who is in the groups is + worked out for each page. + status (Optional[SignInSessionStatus or str]): Return only + sessions still open now (``"open"``) or only sessions that + ended (``"ended"``), by a recorded or a derived end. + page_size (int): Sessions fetched per request, at most 100. + + With both ``user_ids`` and ``user_group_ids``, only users in both are + listed. + + This needs the organization-wide permission to view all performance. + Access to projects does not give it, and without it you may not read + even your own sessions. A restricted API key must include it. + + Returns: + PerformanceCursorRows[SignInSession]: The sessions that started + in the period, oldest first, fetched a page at a time as you + iterate. A session that started before ``start_date`` is not + listed; :meth:`get_sign_in_events` shows what happened in a + period. + + Raises: + lbox.exceptions.AuthorizationError: You may not view all + performance across the organization, or the API key does not + include that permission. + lbox.exceptions.InvalidQueryError: A parameter is not valid, the + period is longer than 31 days, or the user groups have more + than 5,000 members between them. + ValueError: A filter was given as an empty list. Pass ``None`` + to leave a filter off; an empty list is not read as one. + + Example: + >>> for session in client.get_sign_in_sessions( + >>> "2026-11-02", "2026-11-08", status="ended"): + >>> print(session.email, session.started_at, session.end_reason) + >>> + >>> # Only the ends something recorded. + >>> recorded = [ + >>> session for session in client.get_sign_in_sessions( + >>> "2026-11-02", "2026-11-08", status="ended") + >>> if not session.end_derived] + """ + return _performance.get_sign_in_sessions( + self, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + status=status, + page_size=page_size, + ) + + def get_sign_in_events( + self, + start_date: _performance.DateLike, + end_date: _performance.DateLike, + user_ids: Optional[List[str]] = None, + user_group_ids: Optional[List[str]] = None, + types: Optional[List[Union[_performance.SignInEventType, str]]] = None, + include_derived: bool = True, + page_size: int = 100, + ) -> _performance.SignInEventRows: + """Returns the sign-ins and sign-outs of your organization's members. + + A sign-in happens when a session starts and a sign-out when it ends, + by a recorded or a derived end (see :meth:`get_sign_in_sessions`), so + a session that started before the period can sign out in it. Events + are ordered by ``occurred_at``. + + The feed stops at ``complete_through`` on the result: the earlier of + the end of ``end_date`` and five minutes ago. Every event before it + is listed, and a derived sign-out is listed once its time has passed + it. To follow the feed, read it to the end and keep + ``complete_through``. Next time, read from the day an hour before it + and drop the events whose ``event_id`` you have already seen: that + also picks up an event that took unusually long to be recorded. + + Who is listed, and who may ask, is as for + :meth:`get_sign_in_sessions`. + + Args: + start_date (date, datetime or str): First day of the period, in + UTC. A string is a day such as ``"2026-01-31"``. A datetime + with a time zone is read as its UTC day; a naive datetime is + taken to be in UTC already, so its own date is used. + end_date (date, datetime or str): Last day of the period, read as + for ``start_date``. A request can cover at most 31 days. + user_ids (Optional[List[str]]): Return only these users' events. + Up to 100. + user_group_ids (Optional[List[str]]): Return only the events of + the current members of these user groups of your + organization, as for :meth:`get_sign_in_sessions`. Up to 25 + groups, with at most 5,000 members between them. + types (Optional[List[SignInEventType or str]]): The kinds of event + to return: ``"SIGN_IN"``, ``"SIGN_OUT"`` or both. Both when + not given. + include_derived (bool): Pass False to leave out the sign-outs + nothing recorded, which are derived at a session's + ``expires_at``. + page_size (int): Events fetched per request, at most 100. + + Returns: + SignInEventRows: The events in the period, fetched a page at a + time as you iterate, and ``complete_through`` once they have been + read to the end. + + Raises: + lbox.exceptions.AuthorizationError: You may not view all + performance across the organization, or the API key does not + include that permission. + lbox.exceptions.InvalidQueryError: A parameter is not valid, the + period is longer than 31 days, or the user groups have more + than 5,000 members between them. + ValueError: A filter or ``types`` was given as an empty list. + Pass ``None`` to leave it off; an empty list is not read as + that. + + Example: + >>> from datetime import datetime, timedelta, timezone + >>> events = client.get_sign_in_events("2026-11-03", "2026-11-09") + >>> seen = set() + >>> for event in events: + >>> seen.add(event.event_id) + >>> print(event.occurred_at, event.email, event.type) + >>> + >>> # Next time, from an hour before where the feed stopped. + >>> since = events.complete_through - timedelta(hours=1) + >>> events = client.get_sign_in_events( + >>> since, datetime.now(timezone.utc)) + >>> new = [event for event in events if event.event_id not in seen] + >>> + >>> # Recorded sign-outs only. + >>> sign_outs = client.get_sign_in_events( + >>> "2026-11-03", "2026-11-09", types=["SIGN_OUT"], + >>> include_derived=False) + """ + return _performance.get_sign_in_events( + self, + start_date, + end_date, + user_ids=user_ids, + user_group_ids=user_group_ids, + types=types, + include_derived=include_derived, + page_size=page_size, + ) + def get_task_by_id(self, task_id: str) -> Union[Task, DataUpsertTask]: """ Fetches a task by ID. diff --git a/libs/labelbox/src/labelbox/schema/performance.py b/libs/labelbox/src/labelbox/schema/performance.py index 7360729d0..9d8330387 100644 --- a/libs/labelbox/src/labelbox/schema/performance.py +++ b/libs/labelbox/src/labelbox/schema/performance.py @@ -1,7 +1,8 @@ """Performance reports. The numbers behind a project's Performance page and the workspace Monitor, -read through the Labelbox REST API (``/api/v1/performance``). The methods on +and when the organization's members signed in and out, read through the +Labelbox REST API (``/api/v1/performance``). The methods on :class:`labelbox.Client` and :class:`labelbox.schema.project.Project` that return the classes below are thin wrappers over the functions at the bottom of this module. @@ -52,6 +53,8 @@ # timed out, and not to start a third. _RETRY_DEADLINE_SECONDS = 2 * _TIMEOUT_SECONDS _DEFAULT_PAGE_SIZE = 50 +# Sign-in activity is paged at most 100 rows at a time, and by default. +_SIGN_IN_PAGE_SIZE = 100 DateLike = Union[date, datetime, str] """A calendar day: a ``date``, a ``datetime`` or a string such as ``"2026-01-31"``.""" @@ -92,6 +95,73 @@ class PerformanceMemberType(str, Enum): EXTERNAL = "external" +class SignInSessionStatus(str, Enum): + """Whether a sign-in session is still open. + + A session is ``OPEN`` until it ends, by a recorded or a derived end, and + ``ENDED`` after. Used as the ``status`` filter of + :meth:`labelbox.Client.get_sign_in_sessions`, it is sent in lower case, + as the API takes it there: ``open`` or ``ended``. + """ + + OPEN = "OPEN" + ENDED = "ENDED" + + +class SignInStartKind(str, Enum): + """How a sign-in session started. + + ``SIGN_IN``: the member signed in, by any method, including a single + sign-on that completed without asking. ``WORKSPACE_SWITCH``: the member + switched into this workspace from their user in another one. + """ + + SIGN_IN = "SIGN_IN" + WORKSPACE_SWITCH = "WORKSPACE_SWITCH" + + +class SignInEndReason(str, Enum): + """How a sign-in session ended. + + ``SIGNED_OUT``: the member signed out. This includes an idle sign-out by + the label editor when the editor is shown inside another Labelbox page: + the two cannot be told apart. ``REPLACED``: a new sign-in in the same + browser replaced the session; it can be another account's, a sign-up or + an impersonation. ``WORKSPACE_SWITCH``: the member switched + to another workspace. ``ACCOUNT_REMOVED``: the member was removed from + the organization or deactivated. ``REVOKED``: the session was ended + because its account was merged into another. + + ``MAX_DURATION_REACHED``: nothing ended the session before every + credential of its sign-in lapsed, at its ``expires_at``. This end is + derived when the session is read, never recorded. It is when the session + could last no longer, not when the member stopped working. + + Closing the tab or the browser is not recorded, and ends nothing. The + label editor's idle sign-out when the editor is open in its own tab is + not recorded either. + """ + + SIGNED_OUT = "SIGNED_OUT" + REPLACED = "REPLACED" + WORKSPACE_SWITCH = "WORKSPACE_SWITCH" + ACCOUNT_REMOVED = "ACCOUNT_REMOVED" + REVOKED = "REVOKED" + MAX_DURATION_REACHED = "MAX_DURATION_REACHED" + + +class SignInEventType(str, Enum): + """The kind of a sign-in event. + + ``SIGN_IN`` happens when a session starts, at its ``started_at``. + ``SIGN_OUT`` happens when it ends, by a recorded or a derived end, at its + ``ended_at``. + """ + + SIGN_IN = "SIGN_IN" + SIGN_OUT = "SIGN_OUT" + + class PerformanceMetric(_CamelCaseMixin): """A metric that can be queried. @@ -433,6 +503,114 @@ class PerformanceReportDownload(_CamelCaseMixin): generated_at: datetime +class SignInSession(_CamelCaseMixin): + """One sign-in session of a member of your organization. + + A session is one sign-in in one browser, or a switch into this workspace + from the same person's user in another one. It lasts until the member + signs out, until Labelbox ends it (a new sign-in in the same browser, a + switch to another workspace, the member's removal) or, when neither + happened, until every credential the sign-in issued has lapsed. That + last end is derived when the session is read, and is the latest the + session could last: it is not when the member stopped working. + + Attributes: + session_id (str): The session's ID. Its events in + :meth:`labelbox.Client.get_sign_in_events` carry it too. + user_id (str) + email (Optional[str]): ``None`` if the user no longer exists or is + now another organization's. + member_deleted (bool): True if the member was deactivated, no longer + exists, or is now another organization's. A session stays with + the organization it started in. + started_at (datetime): When the member signed in, or switched into + this workspace. + start_kind (str): How the session started: ``"SIGN_IN"`` or + ``"WORKSPACE_SWITCH"`` (see :class:`SignInStartKind`). + status (str): ``"OPEN"`` or ``"ENDED"`` (see + :class:`SignInSessionStatus`). + ended_at (Optional[datetime]): When the session ended, by a recorded + or a derived end. For a derived end it is ``expires_at``. + ``None`` while the session is open. + end_reason (Optional[str]): How the session ended (see + :class:`SignInEndReason`). ``None`` while it is open. + end_derived (Optional[bool]): True when nothing recorded the end. + ``end_reason`` is then ``"MAX_DURATION_REACHED"`` and + ``ended_at`` is ``expires_at``: the moment the session could + last no longer, not when the member stopped working. ``None`` + while the session is open. + expires_at (datetime): The latest the session can end. That is + normally 84 hours and a minute after the sign-in (the sign-in + cookie's lifetime, plus a minute), or 7 days after a switch made + where the organization uses tokens instead of cookies. Opening a + workspace link again, or connecting an application to the + account, sets the cookie again, so while the session is open + ``expires_at`` can still move later. Once it has passed, it does + not change. + duration_seconds (Optional[int]): Whole seconds from start to end. + ``None`` while the session is open. Sessions record sign-ins and + how they ended, not activity, so no duration is time worked, + recorded end or not. For example, a derived end counts all the + time the session could have lasted, a sign-out the next morning + counts the night, and overlapping sessions in two browsers both + count. + """ + + session_id: str + user_id: str + email: Optional[str] = None + member_deleted: bool + started_at: datetime + start_kind: str + status: str + ended_at: Optional[datetime] = None + end_reason: Optional[str] = None + end_derived: Optional[bool] = None + expires_at: datetime + duration_seconds: Optional[int] = None + + +class SignInEvent(_CamelCaseMixin): + """A sign-in or a sign-out of a member of your organization. + + A sign-in happens when a session starts and a sign-out when it ends, by + a recorded or a derived end (see :class:`SignInSession`). + + Attributes: + event_id (str): The same on every read, to drop events already seen. + type (str): ``"SIGN_IN"`` or ``"SIGN_OUT"`` (see + :class:`SignInEventType`). + occurred_at (datetime): When it happened: a sign-in when its session + started, a sign-out when it ended. + session_id (str): The session it starts or ends. + user_id (str) + email (Optional[str]): ``None`` if the user no longer exists or is + now another organization's. + start_kind (Optional[str]): How the session started (see + :class:`SignInStartKind`). ``None`` for a sign-out. + end_reason (Optional[str]): How the session ended (see + :class:`SignInEndReason`). ``None`` for a sign-in. + end_derived (Optional[bool]): True for a sign-out nothing recorded. + ``end_reason`` is then ``"MAX_DURATION_REACHED"`` and the + sign-out is at the session's ``expires_at``: the moment the + session could last no longer, not when the member stopped + working. ``None`` for a sign-in. + session_started_at (datetime): When the session started. For a + sign-in, it is ``occurred_at``. + """ + + event_id: str + type: str + occurred_at: datetime + session_id: str + user_id: str + email: Optional[str] = None + start_kind: Optional[str] = None + end_reason: Optional[str] = None + end_derived: Optional[bool] = None + session_started_at: datetime + + class PerformanceTableChangedError(LabelboxError): """A table changed while its pages were being read. @@ -535,6 +713,96 @@ def __iter__(self) -> Iterator[Row]: number += 1 +class PerformanceCursorRows(Generic[Row]): + """Rows the API pages by cursor: sign-in sessions and sign-in events. + + Iterating fetches the rows a page at a time, each page from where the + one before ended, until the API says there are no more. Nothing is + requested until the rows are read. The rows come in a fixed order. + Unlike :class:`PerformanceRows` there is no ``total``: the API does not + count the rows. + + Each page is a separate request that continues after the last row of + the page before, so the pages never overlap. A row on a later page is as + it was when that page was read: a session that ended meanwhile is listed + as ended, and is left out of a reading of open sessions. Who is in the + groups of a ``user_group_ids`` filter is worked out again for every + page. + + Iterating a second time reads the rows again from the first page. + + >>> for session in client.get_sign_in_sessions("2026-11-02", "2026-11-08"): + ... print(session.email, session.started_at, session.end_reason) + """ + + def __init__( + self, + fetch_page: Callable[[Optional[str]], Dict[str, Any]], + row_type: Type[Row], + ): + self._fetch_page = fetch_page + self._row_type = row_type + # The last page of the latest reading, once it got there. + self._last_page: Optional[Dict[str, Any]] = None + + def __iter__(self) -> Iterator[Row]: + self._last_page = None + cursor: Optional[str] = None + while True: + page = self._fetch_page(cursor) + for row in page["data"]: + yield self._row_type.model_validate(row) + cursor = page.get("nextCursor") + if not cursor: + self._last_page = page + return + + +class SignInEventRows(PerformanceCursorRows[SignInEvent]): + """The sign-ins and sign-outs of :meth:`labelbox.Client.get_sign_in_events`. + + The events are fetched a page at a time as you iterate, as for + :class:`PerformanceCursorRows`, ordered by ``occurred_at``. + + The feed stops at ``complete_through``. Read it to the end and keep + ``complete_through``: every event before it has been listed. To follow + the feed, read it again next time from the day an hour before the + ``complete_through`` you kept, and drop the events whose ``event_id`` + you have already seen. That also picks up an event that took unusually + long to be recorded. + + >>> from datetime import datetime, timedelta, timezone + >>> events = client.get_sign_in_events("2026-11-03", "2026-11-09") + >>> seen = {event.event_id for event in events} + >>> since = events.complete_through - timedelta(hours=1) + >>> # Next time: + >>> for event in client.get_sign_in_events(since, datetime.now(timezone.utc)): + ... if event.event_id not in seen: + ... seen.add(event.event_id) + ... print(event.occurred_at, event.email, event.type) + """ + + def __init__(self, fetch_page: Callable[[Optional[str]], Dict[str, Any]]): + super().__init__(fetch_page, SignInEvent) + + @property + def complete_through(self) -> Optional[datetime]: + """Where the feed stops, as the last page reports it. + + The earlier of the end of ``end_date`` and five minutes before the + last page was read. Every event before it is in the feed, and a + derived sign-out is listed once its time is before it; events at or + after it are not listed yet. + + It is taken from the last page, the one with no next page, so it is + ``None`` until the events have been read to the end. Iterating again + resets it, and it then describes that reading. + """ + if self._last_page is None: + return None + return _parse_datetime(self._last_page["completeThrough"]) + + def _parse_datetime(value: str) -> datetime: return datetime.fromisoformat(value.replace("Z", "+00:00")) @@ -587,6 +855,43 @@ def _to_ids(name: str, values: Optional[Iterable[Any]]) -> Optional[str]: return ",".join(ids) +def _to_status( + status: Optional[Union[SignInSessionStatus, str]], +) -> Optional[str]: + """A session status as the filter takes it: ``open`` or ``ended``. + + A session's own status is in capitals, like SignInSessionStatus, and the + filter is in lower case, so either spelling is accepted. + """ + if status is None: + return None + value = status.value if isinstance(status, Enum) else status + return value.lower() + + +def _to_event_types( + types: Optional[Iterable[Union[SignInEventType, str]]], +) -> Optional[str]: + """Event types as one comma-separated parameter. + + ``None`` lists both kinds. An empty list is refused, as the API refuses + it, before anything is requested. + """ + if types is None: + return None + if isinstance(types, str): + types = [types] + values = [ + str(value.value) if isinstance(value, Enum) else value + for value in types + ] + if not values: + raise ValueError( + "types is empty. Pass None to list both sign-ins and sign-outs." + ) + return ",".join(values) + + def _to_field(attribute: str) -> str: """A row attribute as the API spells it: labels_created is labelsCreated. @@ -934,3 +1239,60 @@ def get_workspace_labelers( userGroupIds=_to_ids("user_group_ids", user_group_ids), useCache=None if use_cache else False, ) + + +def _cursor_pages( + client: "Client", path: str, **params: Any +) -> Callable[[Optional[str]], Dict[str, Any]]: + """Fetches one page of a listing paged by cursor: the first without one.""" + + def fetch_page(cursor: Optional[str]) -> Dict[str, Any]: + # Sent as the page before returned it: a cursor is not to be read. + return _get(client, path, **params, cursor=cursor) + + return fetch_page + + +def get_sign_in_sessions( + client: "Client", + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + status: Optional[Union[SignInSessionStatus, str]] = None, + page_size: int = _SIGN_IN_PAGE_SIZE, +) -> PerformanceCursorRows[SignInSession]: + fetch_page = _cursor_pages( + client, + "/workspace/sign-in-sessions", + **_period(start_date, end_date), + userIds=_to_ids("user_ids", user_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), + status=_to_status(status), + perPage=page_size, + ) + return PerformanceCursorRows(fetch_page, SignInSession) + + +def get_sign_in_events( + client: "Client", + start_date: DateLike, + end_date: DateLike, + user_ids: Optional[Iterable[Any]] = None, + user_group_ids: Optional[Iterable[Any]] = None, + types: Optional[Iterable[Union[SignInEventType, str]]] = None, + include_derived: bool = True, + page_size: int = _SIGN_IN_PAGE_SIZE, +) -> SignInEventRows: + fetch_page = _cursor_pages( + client, + "/workspace/sign-in-events", + **_period(start_date, end_date), + userIds=_to_ids("user_ids", user_ids), + userGroupIds=_to_ids("user_group_ids", user_group_ids), + types=_to_event_types(types), + # Only sent to leave derived sign-outs out; they are in by default. + includeDerived=None if include_derived else False, + perPage=page_size, + ) + return SignInEventRows(fetch_page) diff --git a/libs/labelbox/tests/unit/schema/test_performance.py b/libs/labelbox/tests/unit/schema/test_performance.py index 5182721d5..72ca8792f 100644 --- a/libs/labelbox/tests/unit/schema/test_performance.py +++ b/libs/labelbox/tests/unit/schema/test_performance.py @@ -19,6 +19,7 @@ from labelbox import ( Client, + PerformanceCursorRows, PerformanceDeletedLabels, PerformanceInterval, PerformanceMemberType, @@ -26,6 +27,11 @@ PerformanceTableChangedError, PerformanceTimeBucket, Project, + SignInEndReason, + SignInEventRows, + SignInEventType, + SignInSessionStatus, + SignInStartKind, ) from labelbox.schema import performance from labelbox.schema.user_group import UserGroup @@ -1558,3 +1564,795 @@ def test_reports_a_reply_that_is_not_json(self, client, project): Project.get_performance_metric( project, "labels_created", "2026-01-01", "2026-01-07" ) + + +RULES = { + "cookieLifetimeSeconds": 302400, + "signInTokenLifetimeSeconds": 86400, + "workspaceTokenLifetimeSeconds": 604800, +} + + +def sign_in_session(session_id, **fields): + """A session with a recorded end, as the REST API returns it.""" + return { + "sessionId": session_id, + "userId": "user-1", + "email": "user-1@example.com", + "memberDeleted": False, + "startedAt": "2026-11-03T08:59:41.120Z", + "startKind": "SIGN_IN", + "status": "ENDED", + "endedAt": "2026-11-03T17:31:02.507Z", + "endReason": "SIGNED_OUT", + "endDerived": False, + "expiresAt": "2026-11-06T21:00:41.120Z", + "durationSeconds": 30681, + **fields, + } + + +def sessions_page(rows, next_cursor=None): + return { + "organizationId": "org-1", + "startDate": "2026-11-02", + "endDate": "2026-11-08", + "userIds": None, + "userGroupIds": None, + "status": None, + "rules": RULES, + "data": rows, + "nextCursor": next_cursor, + "perPage": 100, + "generatedAt": "2026-11-09T10:00:00.000Z", + } + + +def sign_in_event(session_id, kind="in", **fields): + """A sign-in, or with kind "out" a recorded sign-out.""" + signing_in = kind == "in" + return { + "eventId": f"{session_id}.{kind}", + "type": "SIGN_IN" if signing_in else "SIGN_OUT", + "occurredAt": "2026-11-03T08:59:41.120Z" + if signing_in + else "2026-11-03T17:31:02.507Z", + "sessionId": session_id, + "userId": "user-1", + "email": "user-1@example.com", + "startKind": "SIGN_IN" if signing_in else None, + "endReason": None if signing_in else "SIGNED_OUT", + "endDerived": None if signing_in else False, + "sessionStartedAt": "2026-11-03T08:59:41.120Z", + **fields, + } + + +def events_page( + rows, next_cursor=None, complete_through="2026-11-04T00:00:00.000Z" +): + return { + "organizationId": "org-1", + "startDate": "2026-11-03", + "endDate": "2026-11-03", + "userIds": None, + "userGroupIds": None, + "types": ["SIGN_IN", "SIGN_OUT"], + "includeDerived": True, + "rules": RULES, + "data": rows, + "nextCursor": next_cursor, + "completeThrough": complete_through, + "perPage": 100, + "generatedAt": "2026-11-09T10:00:00.000Z", + } + + +def cursors_sent(client): + """The cursor each request sent, None for none.""" + return [ + sent(client, call)[1].get("cursor") + for call in range(client.connection.get.call_count) + ] + + +class TestSignInSessions: + def test_asks_for_nothing_until_the_sessions_are_read(self, client): + sessions = client.get_sign_in_sessions("2026-11-02", "2026-11-08") + + client.connection.get.assert_not_called() + assert isinstance(sessions, PerformanceCursorRows) + + def test_returns_the_sessions_that_started_in_the_period(self, client): + client.connection.get.return_value = response( + sessions_page([sign_in_session("session-1")]) + ) + + (session,) = client.get_sign_in_sessions( + date(2026, 11, 2), "2026-11-08" + ) + + url, params = sent(client) + assert url == f"{BASE}/workspace/sign-in-sessions" + assert params == { + "startDate": "2026-11-02", + "endDate": "2026-11-08", + "perPage": "100", + } + assert session.session_id == "session-1" + assert session.user_id == "user-1" + assert session.email == "user-1@example.com" + assert session.member_deleted is False + assert session.started_at == datetime( + 2026, 11, 3, 8, 59, 41, 120000, tzinfo=timezone.utc + ) + assert session.start_kind == SignInStartKind.SIGN_IN + assert session.status == SignInSessionStatus.ENDED + assert session.ended_at == datetime( + 2026, 11, 3, 17, 31, 2, 507000, tzinfo=timezone.utc + ) + assert session.end_reason == SignInEndReason.SIGNED_OUT + assert session.end_derived is False + assert session.expires_at == datetime( + 2026, 11, 6, 21, 0, 41, 120000, tzinfo=timezone.utc + ) + assert session.duration_seconds == 30681 + + def test_sends_the_filters_under_the_names_the_api_gives_them(self, client): + client.connection.get.return_value = response(sessions_page([])) + + list( + client.get_sign_in_sessions( + "2026-11-02", + "2026-11-08", + user_ids=["user-1", SimpleNamespace(uid="user-2")], + # UserGroup is a pydantic model whose ID is `id`, not `uid`. + user_group_ids=[ + UserGroup(client=client, id="group-a", name="Vendor A"), + "group-b", + ], + status=SignInSessionStatus.OPEN, + page_size=25, + ) + ) + + _, params = sent(client) + assert params == { + "startDate": "2026-11-02", + "endDate": "2026-11-08", + "userIds": "user-1,user-2", + "userGroupIds": "group-a,group-b", + "status": "open", + "perPage": "25", + } + + @pytest.mark.parametrize( + "status, sent_as", + [ + (SignInSessionStatus.OPEN, "open"), + (SignInSessionStatus.ENDED, "ended"), + ("open", "open"), + ("ended", "ended"), + # As a session's own status spells it. + ("OPEN", "open"), + ("ENDED", "ended"), + ], + ) + def test_sends_the_status_in_lower_case_as_the_api_takes_it( + self, client, status, sent_as + ): + client.connection.get.return_value = response(sessions_page([])) + + list( + client.get_sign_in_sessions( + "2026-11-02", "2026-11-08", status=status + ) + ) + + assert sent(client)[1]["status"] == sent_as + + def test_follows_the_cursor_until_there_is_no_next_page(self, client): + client.connection.get.side_effect = [ + response( + sessions_page( + [ + sign_in_session("session-1"), + sign_in_session("session-2"), + ], + next_cursor="cursor-after-2", + ) + ), + response( + sessions_page( + [ + sign_in_session("session-3"), + sign_in_session("session-4"), + ], + next_cursor="cursor-after-4", + ) + ), + response(sessions_page([sign_in_session("session-5")])), + ] + + sessions = list( + client.get_sign_in_sessions( + "2026-11-02", "2026-11-08", status="ended", page_size=2 + ) + ) + + assert [session.session_id for session in sessions] == [ + "session-1", + "session-2", + "session-3", + "session-4", + "session-5", + ] + # Each page continues from the cursor the page before returned, as + # it was returned, and asks for the same rows otherwise. + assert cursors_sent(client) == [ + None, + "cursor-after-2", + "cursor-after-4", + ] + for call in range(3): + _, params = sent(client, call) + assert params["status"] == "ended" + assert params["perPage"] == "2" + assert params["startDate"] == "2026-11-02" + + def test_reads_an_empty_period_with_one_request(self, client): + client.connection.get.return_value = response(sessions_page([])) + + assert ( + list(client.get_sign_in_sessions("2026-11-02", "2026-11-08")) == [] + ) + assert client.connection.get.call_count == 1 + + def test_reads_the_sessions_again_from_the_first_page_on_a_second_pass( + self, client + ): + first = [ + response( + sessions_page( + [sign_in_session("session-1")], next_cursor="cursor-1" + ) + ), + response(sessions_page([sign_in_session("session-2")])), + ] + # By the second pass another session has started. + second = [ + response( + sessions_page( + [sign_in_session("session-1")], next_cursor="cursor-1" + ) + ), + response( + sessions_page( + [sign_in_session("session-2")], next_cursor="cursor-2" + ) + ), + response(sessions_page([sign_in_session("session-3")])), + ] + client.connection.get.side_effect = first + second + + sessions = client.get_sign_in_sessions( + "2026-11-02", "2026-11-08", page_size=1 + ) + once = [session.session_id for session in sessions] + twice = [session.session_id for session in sessions] + + assert once == ["session-1", "session-2"] + assert twice == ["session-1", "session-2", "session-3"] + assert cursors_sent(client) == [ + None, + "cursor-1", + None, + "cursor-1", + "cursor-2", + ] + + def test_reads_a_derived_end(self, client): + client.connection.get.return_value = response( + sessions_page( + [ + sign_in_session( + "session-1", + startedAt="2026-11-04T09:02:10.004Z", + endedAt="2026-11-07T21:03:10.004Z", + endReason="MAX_DURATION_REACHED", + endDerived=True, + expiresAt="2026-11-07T21:03:10.004Z", + durationSeconds=302460, + ) + ] + ) + ) + + (session,) = client.get_sign_in_sessions("2026-11-02", "2026-11-08") + + assert session.status == SignInSessionStatus.ENDED + assert session.end_reason == SignInEndReason.MAX_DURATION_REACHED + assert session.end_derived is True + # The latest the session could last, not when the member left. + assert session.ended_at == session.expires_at + assert session.duration_seconds == 302460 + + def test_reads_an_open_session(self, client): + client.connection.get.return_value = response( + sessions_page( + [ + sign_in_session( + "session-1", + startKind="WORKSPACE_SWITCH", + status="OPEN", + endedAt=None, + endReason=None, + endDerived=None, + durationSeconds=None, + ) + ] + ) + ) + + (session,) = client.get_sign_in_sessions( + "2026-11-02", "2026-11-08", status="open" + ) + + assert session.status == SignInSessionStatus.OPEN + assert session.start_kind == SignInStartKind.WORKSPACE_SWITCH + assert session.ended_at is None + assert session.end_reason is None + assert session.end_derived is None + assert session.duration_seconds is None + assert session.expires_at == datetime( + 2026, 11, 6, 21, 0, 41, 120000, tzinfo=timezone.utc + ) + + def test_reads_a_member_whose_account_is_now_another_organizations( + self, client + ): + client.connection.get.return_value = response( + sessions_page( + [sign_in_session("session-1", email=None, memberDeleted=True)] + ) + ) + + (session,) = client.get_sign_in_sessions("2026-11-02", "2026-11-08") + + assert session.email is None + assert session.member_deleted is True + assert session.user_id == "user-1" + + def test_ignores_fields_added_to_the_response_later(self, client): + client.connection.get.return_value = response( + { + **sessions_page( + [sign_in_session("session-1", deviceKind="browser")] + ), + "asOf": "2026-11-09T10:00:00.000Z", + } + ) + + (session,) = client.get_sign_in_sessions("2026-11-02", "2026-11-08") + + assert session.session_id == "session-1" + assert not hasattr(session, "device_kind") + + def test_tries_a_failed_page_again_with_the_same_cursor(self, client): + client.connection.get.side_effect = [ + response( + sessions_page( + [sign_in_session("session-1")], next_cursor="cursor-1" + ) + ), + response({"statusCode": 502, "message": "Bad gateway"}, 502), + response(sessions_page([sign_in_session("session-2")])), + ] + + sessions = list(client.get_sign_in_sessions("2026-11-02", "2026-11-08")) + + assert [session.session_id for session in sessions] == [ + "session-1", + "session-2", + ] + assert cursors_sent(client) == [None, "cursor-1", "cursor-1"] + + +class TestSignInEvents: + def test_asks_for_nothing_until_the_events_are_read(self, client): + events = client.get_sign_in_events("2026-11-03", "2026-11-03") + + client.connection.get.assert_not_called() + assert isinstance(events, SignInEventRows) + assert events.complete_through is None + + def test_returns_the_sign_ins_and_sign_outs(self, client): + client.connection.get.return_value = response( + events_page( + [ + sign_in_event("session-1", "in"), + sign_in_event("session-1", "out"), + ] + ) + ) + + sign_in, sign_out = client.get_sign_in_events( + "2026-11-03", "2026-11-03" + ) + + url, params = sent(client) + assert url == f"{BASE}/workspace/sign-in-events" + # Both types and the derived sign-outs are the API's default. + assert params == { + "startDate": "2026-11-03", + "endDate": "2026-11-03", + "perPage": "100", + } + assert sign_in.event_id == "session-1.in" + assert sign_in.type == SignInEventType.SIGN_IN + assert sign_in.occurred_at == datetime( + 2026, 11, 3, 8, 59, 41, 120000, tzinfo=timezone.utc + ) + assert sign_in.session_id == "session-1" + assert sign_in.user_id == "user-1" + assert sign_in.email == "user-1@example.com" + assert sign_in.start_kind == SignInStartKind.SIGN_IN + assert sign_in.end_reason is None + assert sign_in.end_derived is None + assert sign_in.session_started_at == sign_in.occurred_at + assert sign_out.event_id == "session-1.out" + assert sign_out.type == SignInEventType.SIGN_OUT + assert sign_out.start_kind is None + assert sign_out.end_reason == SignInEndReason.SIGNED_OUT + assert sign_out.end_derived is False + assert sign_out.session_started_at == sign_in.occurred_at + + def test_sends_the_filters_under_the_names_the_api_gives_them(self, client): + client.connection.get.return_value = response(events_page([])) + + list( + client.get_sign_in_events( + "2026-11-03", + "2026-11-09", + user_ids=["user-1"], + user_group_ids=[ + UserGroup(client=client, id="group-a", name="Vendor A") + ], + types=[SignInEventType.SIGN_OUT], + include_derived=False, + page_size=10, + ) + ) + + _, params = sent(client) + assert params == { + "startDate": "2026-11-03", + "endDate": "2026-11-09", + "userIds": "user-1", + "userGroupIds": "group-a", + "types": "SIGN_OUT", + "includeDerived": "false", + "perPage": "10", + } + + @pytest.mark.parametrize( + "types, sent_as", + [ + ([SignInEventType.SIGN_IN], "SIGN_IN"), + (["SIGN_OUT"], "SIGN_OUT"), + ( + [SignInEventType.SIGN_IN, "SIGN_OUT"], + "SIGN_IN,SIGN_OUT", + ), + # One type on its own, as the ID filters take one ID. + (SignInEventType.SIGN_OUT, "SIGN_OUT"), + ("SIGN_IN", "SIGN_IN"), + ], + ) + def test_takes_the_types_as_enums_or_strings(self, client, types, sent_as): + client.connection.get.return_value = response(events_page([])) + + list(client.get_sign_in_events("2026-11-03", "2026-11-03", types=types)) + + assert sent(client)[1]["types"] == sent_as + + def test_sends_include_derived_only_to_leave_them_out(self, client): + client.connection.get.return_value = response(events_page([])) + + list( + client.get_sign_in_events( + "2026-11-03", "2026-11-03", include_derived=True + ) + ) + + assert "includeDerived" not in sent(client)[1] + + @pytest.mark.parametrize("empty", [[], (), iter([])]) + def test_refuses_empty_types_before_asking(self, client, empty): + with pytest.raises(ValueError, match="types is empty"): + client.get_sign_in_events("2026-11-03", "2026-11-03", types=empty) + + client.connection.get.assert_not_called() + + def test_follows_the_cursor_and_says_where_the_last_page_stops( + self, client + ): + client.connection.get.side_effect = [ + response( + events_page( + [sign_in_event("session-1", "in")], + next_cursor="cursor-1", + complete_through="2026-11-09T09:54:00.000Z", + ) + ), + response( + events_page( + [sign_in_event("session-1", "out")], + next_cursor="cursor-2", + complete_through="2026-11-09T09:54:30.000Z", + ) + ), + # Each page is read a little later, so stops a little later. + response( + events_page( + [sign_in_event("session-2", "in")], + complete_through="2026-11-09T09:55:00.000Z", + ) + ), + ] + + events = client.get_sign_in_events( + "2026-11-03", "2026-11-09", page_size=1 + ) + read = [] + for event in events: + # Not until the last page has been read. + assert events.complete_through is None + read.append(event.event_id) + + assert read == ["session-1.in", "session-1.out", "session-2.in"] + assert cursors_sent(client) == [None, "cursor-1", "cursor-2"] + assert events.complete_through == datetime( + 2026, 11, 9, 9, 55, tzinfo=timezone.utc + ) + + def test_says_nothing_of_where_the_feed_stops_if_it_was_not_read_to_the_end( + self, client + ): + client.connection.get.return_value = response( + events_page( + [sign_in_event("session-1", "in")], next_cursor="cursor-1" + ) + ) + + events = client.get_sign_in_events("2026-11-03", "2026-11-03") + next(iter(events)) + + assert events.complete_through is None + assert client.connection.get.call_count == 1 + + def test_reads_the_feed_again_from_the_first_page_on_a_second_pass( + self, client + ): + client.connection.get.side_effect = [ + response( + events_page( + [sign_in_event("session-1", "in")], + next_cursor="cursor-1", + complete_through="2026-11-09T09:55:00.000Z", + ) + ), + response( + events_page( + [sign_in_event("session-1", "out")], + complete_through="2026-11-09T09:55:00.000Z", + ) + ), + # By the second pass the feed has moved on by an event. + response( + events_page( + [ + sign_in_event("session-1", "in"), + sign_in_event("session-1", "out"), + sign_in_event("session-2", "in"), + ], + complete_through="2026-11-09T10:05:00.000Z", + ) + ), + ] + + events = client.get_sign_in_events("2026-11-03", "2026-11-09") + once = [event.event_id for event in events] + first_stop = events.complete_through + twice = [] + for event in events: + # The first reading's stop does not describe this one. + assert events.complete_through is None + twice.append(event.event_id) + + assert once == ["session-1.in", "session-1.out"] + assert twice == ["session-1.in", "session-1.out", "session-2.in"] + assert cursors_sent(client) == [None, "cursor-1", None] + assert first_stop == datetime(2026, 11, 9, 9, 55, tzinfo=timezone.utc) + assert events.complete_through == datetime( + 2026, 11, 9, 10, 5, tzinfo=timezone.utc + ) + + def test_reads_a_derived_sign_out(self, client): + client.connection.get.return_value = response( + events_page( + [ + sign_in_event( + "session-1", + "out", + occurredAt="2026-11-07T21:03:10.004Z", + endReason="MAX_DURATION_REACHED", + endDerived=True, + sessionStartedAt="2026-11-04T09:02:10.004Z", + ) + ] + ) + ) + + (sign_out,) = client.get_sign_in_events("2026-11-07", "2026-11-07") + + assert sign_out.type == SignInEventType.SIGN_OUT + assert sign_out.end_reason == SignInEndReason.MAX_DURATION_REACHED + assert sign_out.end_derived is True + assert sign_out.start_kind is None + # The session started before the period and signs out in it. + assert sign_out.session_started_at == datetime( + 2026, 11, 4, 9, 2, 10, 4000, tzinfo=timezone.utc + ) + + def test_reads_a_member_whose_account_is_now_another_organizations( + self, client + ): + client.connection.get.return_value = response( + events_page([sign_in_event("session-1", "in", email=None)]) + ) + + (event,) = client.get_sign_in_events("2026-11-03", "2026-11-03") + + assert event.email is None + assert event.user_id == "user-1" + + def test_ignores_fields_added_to_the_response_later(self, client): + client.connection.get.return_value = response( + { + **events_page( + [sign_in_event("session-1", "in", deviceKind="browser")] + ), + "asOf": "2026-11-09T10:00:00.000Z", + } + ) + + events = client.get_sign_in_events("2026-11-03", "2026-11-03") + (event,) = events + + assert event.event_id == "session-1.in" + assert not hasattr(event, "device_kind") + assert events.complete_through == datetime( + 2026, 11, 4, tzinfo=timezone.utc + ) + + +class TestSignInFilters: + @pytest.mark.parametrize("argument", ["user_ids", "user_group_ids"]) + @pytest.mark.parametrize("empty", [[], (), set(), iter([])]) + def test_refuses_an_empty_filter_instead_of_listing_everyone( + self, client, argument, empty + ): + with pytest.raises(ValueError, match=f"{argument} is empty"): + client.get_sign_in_sessions( + "2026-11-02", "2026-11-08", **{argument: empty} + ) + with pytest.raises(ValueError, match=f"{argument} is empty"): + client.get_sign_in_events( + "2026-11-02", "2026-11-08", **{argument: []} + ) + + client.connection.get.assert_not_called() + + @pytest.mark.parametrize( + "day, expected", + [ + (date(2026, 11, 2), "2026-11-02"), + # The day an hour before a completeThrough, read as a UTC day. + ( + datetime(2026, 11, 3, 0, 30, tzinfo=timezone.utc) + - timedelta(hours=1), + "2026-11-02", + ), + ( + datetime( + 2026, 11, 2, 20, 0, tzinfo=timezone(timedelta(hours=-8)) + ), + "2026-11-03", + ), + # A naive datetime is taken to be in UTC already: its own date. + (datetime(2026, 11, 2, 20, 0), "2026-11-02"), + ], + ) + def test_sends_a_date_as_a_utc_day(self, client, day, expected): + client.connection.get.side_effect = [ + response(sessions_page([])), + response(events_page([])), + ] + + list(client.get_sign_in_sessions(day, day)) + list(client.get_sign_in_events(day, day)) + + for call in (0, 1): + _, params = sent(client, call) + assert params["startDate"] == expected + assert params["endDate"] == expected + + @pytest.mark.parametrize( + "status, error", + [ + (400, InvalidQueryError), + (401, AuthenticationError), + (403, AuthorizationError), + (429, ApiLimitError), + (500, InternalServerError), + ], + ) + def test_raises_the_sdk_error_for_the_status(self, client, status, error): + client.connection.get.return_value = response( + {"statusCode": status, "message": "Something went wrong"}, status + ) + + with pytest.raises(error, match="Something went wrong"): + list(client.get_sign_in_sessions("2026-11-02", "2026-11-08")) + with pytest.raises(error, match="Something went wrong"): + list(client.get_sign_in_events("2026-11-02", "2026-11-08")) + + def test_says_when_the_api_key_may_not_read_sign_ins(self, client): + client.connection.get.return_value = response( + { + "statusCode": 403, + "message": "This API key does not include the permission " + "to view all performance.", + "error": "Forbidden", + }, + 403, + ) + + with pytest.raises( + AuthorizationError, match="permission to view all performance" + ): + list(client.get_sign_in_events("2026-11-02", "2026-11-08")) + # A refusal is not tried again. + assert client.connection.get.call_count == 1 + + def test_names_the_parameters_the_api_refused(self, client): + client.connection.get.return_value = response( + { + "statusCode": 400, + "error": "Bad Request", + "message": "Invalid query parameters", + "details": [ + { + "parameter": "endDate", + "message": "must be within 31 days of startDate", + }, + { + "parameter": "types", + "message": "must be one or both of SIGN_IN, SIGN_OUT", + }, + ], + }, + 400, + ) + + with pytest.raises(InvalidQueryError) as raised: + list( + client.get_sign_in_events( + "2026-10-01", "2026-11-30", types=["sign_in"] + ) + ) + + message = str(raised.value) + assert "endDate must be within 31 days of startDate" in message + assert "types must be one or both of SIGN_IN, SIGN_OUT" in message