fix: Do not add a request that is already in progress to MemoryRequestQueueClient (#1384)

Mantisus · web-flow · commit 3af326c9dfa8 · 2025-09-03T10:00:28.000+02:00
### Description - Do not add a request that is already in progress for `MemoryRequestQueueClient`. ### Issues - Closes: #1383
diff --git a/src/crawlee/storage_clients/_memory/_request_queue_client.py b/src/crawlee/storage_clients/_memory/_request_queue_client.py
@@ -137,6 +137,7 @@ async def add_batch_of_requests(
 
             was_already_present = existing_request is not None
             was_already_handled = was_already_present and existing_request and existing_request.handled_at is not None
+            is_in_progress = request.unique_key in self._in_progress_requests
 
             # If the request is already in the queue and handled, don't add it again.
             if was_already_handled:
@@ -149,6 +150,17 @@ async def add_batch_of_requests(
                 )
                 continue
 
+            # If the request is already in progress, don't add it again.
+            if is_in_progress:
+                processed_requests.append(
+                    ProcessedRequest(
+                        unique_key=request.unique_key,
+                        was_already_present=True,
+                        was_already_handled=False,
+                    )
+                )
+                continue
+
             # If the request is already in the queue but not handled, update it.
             if was_already_present and existing_request:
                 # Update the existing request with any new data and
