apify · Pijukatel · Jun 4, 2025 · Jun 2, 2025 · Jun 2, 2025 · Jun 3, 2025
diff --git a/src/crawlee/_autoscaling/_types.py b/src/crawlee/_autoscaling/_types.py
@@ -137,6 +137,9 @@ class ClientSnapshot:
     error_count: int
     """The number of errors (HTTP 429) that occurred."""
 
+    new_error_count: int
+    """The number of new errors (HTTP 429) that occurred since the last snapshot."""
+
     max_error_count: int
     """The maximum number of errors that is considered acceptable."""
 
@@ -146,7 +149,7 @@ class ClientSnapshot:
     @property
     def is_overloaded(self) -> bool:
         """Indicate whether the client is considered as overloaded."""
-        return self.error_count > self.max_error_count
+        return self.new_error_count > self.max_error_count
 
 
 Snapshot = Union[MemorySnapshot, CpuSnapshot, EventLoopSnapshot, ClientSnapshot]
diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py
@@ -310,7 +310,12 @@ def _snapshot_client(self) -> None:
         rate_limit_errors: dict[int, int] = client.get_rate_limit_errors()
 
         error_count = rate_limit_errors.get(self._CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT, 0)
-        snapshot = ClientSnapshot(error_count=error_count, max_error_count=self._max_client_errors)
+        previous_error_count = self._client_snapshots[-1].error_count if self._client_snapshots else 0
+        snapshot = ClientSnapshot(
+            error_count=error_count,
+            new_error_count=error_count - previous_error_count,
+            max_error_count=self._max_client_errors,
+        )
 
         snapshots = cast('list[Snapshot]', self._client_snapshots)
         self._prune_snapshots(snapshots, snapshot.created_at)

diff --git a/tests/unit/_autoscaling/test_snapshotter.py b/tests/unit/_autoscaling/test_snapshotter.py
@@ -9,7 +9,7 @@
 
 from crawlee import service_locator
 from crawlee._autoscaling import Snapshotter
-from crawlee._autoscaling._types import CpuSnapshot, EventLoopSnapshot, Snapshot
+from crawlee._autoscaling._types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, Snapshot
 from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.system import CpuInfo, MemoryInfo
 from crawlee.configuration import Configuration
@@ -68,6 +68,13 @@ def test_snapshot_client(snapshotter: Snapshotter) -> None:
     assert len(snapshotter._client_snapshots) == 1
 
 
+def test_snapshot_client_overloaded() -> None:
+    assert not ClientSnapshot(error_count=1, new_error_count=1, max_error_count=2).is_overloaded
+    assert not ClientSnapshot(error_count=2, new_error_count=1, max_error_count=2).is_overloaded
+    assert not ClientSnapshot(error_count=4, new_error_count=2, max_error_count=2).is_overloaded
+    assert ClientSnapshot(error_count=7, new_error_count=3, max_error_count=2).is_overloaded
+
+
 async def test_get_cpu_sample(snapshotter: Snapshotter) -> None:
     now = datetime.now(timezone.utc)
     cpu_snapshots = Snapshotter._get_sorted_list_by_created_at(

diff --git a/tests/unit/_autoscaling/test_system_status.py b/tests/unit/_autoscaling/test_system_status.py
@@ -153,10 +153,10 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None:
     # Add client snapshots
     system_status._snapshotter._client_snapshots = Snapshotter._get_sorted_list_by_created_at(
         [
-            ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=3)),
-            ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=2)),
-            ClientSnapshot(error_count=2, max_error_count=2, created_at=now - timedelta(minutes=1)),
-            ClientSnapshot(error_count=0, max_error_count=2, created_at=now),
+            ClientSnapshot(error_count=1, new_error_count=1, max_error_count=2, created_at=now - timedelta(minutes=3)),
+            ClientSnapshot(error_count=2, new_error_count=1, max_error_count=2, created_at=now - timedelta(minutes=2)),
+            ClientSnapshot(error_count=4, new_error_count=2, max_error_count=2, created_at=now - timedelta(minutes=1)),
+            ClientSnapshot(error_count=4, new_error_count=0, max_error_count=2, created_at=now),
         ]
     )
 
@@ -181,3 +181,26 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None:
         created_at=historical_system_info.created_at,
     )
     assert historical_system_info.is_system_idle is False
+
+
+@pytest.mark.parametrize(('client_overload_threshold', 'is_overloaded'), [(0.66, True), (0.67, False)])
+def test_client_overloaded(
+    *, snapshotter: Snapshotter, now: datetime, client_overload_threshold: float, is_overloaded: bool
+) -> None:
+    system_status = SystemStatus(
+        snapshotter,
+        max_snapshot_age=timedelta(minutes=1),
+        client_overload_threshold=client_overload_threshold,
+    )
+
+    system_status._snapshotter._client_snapshots = Snapshotter._get_sorted_list_by_created_at(
+        [
+            ClientSnapshot(error_count=1, new_error_count=1, max_error_count=0, created_at=now - timedelta(minutes=3)),
+            ClientSnapshot(error_count=2, new_error_count=1, max_error_count=0, created_at=now - timedelta(minutes=2)),
+            ClientSnapshot(error_count=3, new_error_count=1, max_error_count=0, created_at=now - timedelta(minutes=1)),
+            ClientSnapshot(error_count=3, new_error_count=0, max_error_count=0, created_at=now),
+        ]
+    )
+
+    # Ratio of overloaded snapshots is 2/3 (2 minutes out of 3)
+    assert system_status._is_client_overloaded().is_overloaded == is_overloaded