@@ -815,95 +815,3 @@ def _broadcast_worker_state(payload):
815815 "worker" : payload ,
816816 },
817817 )
818-
819-
820- @app .task (queue = "site-worker" , soft_time_limit = 120 )
821- def refresh_compute_worker_health ():
822- celery_app = app
823- r = get_redis_connection ("default" )
824- known_queue_names = known_compute_queue_names ()
825- broker_sources = []
826- broker_sources .append (("default" , celery_app .conf .broker_url , celery_app ))
827-
828- private_queues = (
829- Queue .objects .filter (competitions__isnull = False )
830- .exclude (name__isnull = True )
831- .exclude (name = "" )
832- .distinct ()
833- )
834- for queue in private_queues :
835- if not queue .broker_url :
836- continue
837- parsed = urllib .parse .urlparse (queue .broker_url )
838- vhost = parsed .path
839- broker_url = urllib .parse .urljoin (celery_app .conf .broker_url , vhost )
840- broker_sources .append ((queue .name , broker_url , app_for_vhost (vhost )))
841-
842- inspected_brokers = set ()
843- for source_name , broker_url , broker_app in broker_sources :
844- if broker_url in inspected_brokers :
845- continue
846- inspected_brokers .add (broker_url )
847-
848- try :
849- # timeout=5 : 4 appels × 5s × N brokers
850- inspector = broker_app .control .inspect (timeout = 5 )
851- if inspector is None :
852- logger .warning (
853- "Celery inspect returned None for broker=%s" , source_name
854- )
855- continue
856- stats = inspector .stats () or {}
857- active = inspector .active () or {}
858- reserved = inspector .reserved () or {}
859- active_queues = inspector .active_queues () or {}
860- except Exception :
861- logger .exception (
862- "Unable to inspect Celery workers for broker %s" , source_name
863- )
864- continue
865-
866- for worker_name in stats .keys ():
867- queues = active_queues .get (worker_name , []) or []
868- queue_names = extract_queue_names (queues )
869- if not is_compute_worker (worker_name , queue_names , known_queue_names ):
870- continue
871-
872- running_jobs = len (active .get (worker_name , [])) + len (
873- reserved .get (worker_name , [])
874- )
875- status = "busy" if running_jobs > 0 else "available"
876- payload = {
877- "hostname" : worker_name ,
878- "status" : status ,
879- "running_jobs" : running_jobs ,
880- "timestamp" : now ().timestamp (),
881- "queue_source" : source_name ,
882- "queue_names" : sorted (queue_names ),
883- }
884- heartbeat_key = f"worker:{ source_name } :{ worker_name } :heartbeat"
885- r .set (heartbeat_key , json .dumps (payload ), ex = WORKER_HEARTBEAT_TTL )
886- r .hset (
887- WORKERS_REGISTRY_KEY ,
888- f"{ source_name } :{ worker_name } " ,
889- json .dumps (
890- {
891- "hostname" : worker_name ,
892- "status" : status ,
893- "running_jobs" : running_jobs ,
894- "last_seen" : payload ["timestamp" ],
895- "queue_source" : source_name ,
896- "queue_names" : sorted (queue_names ),
897- }
898- ),
899- )
900- _broadcast_worker_state (payload )
901- # Logs about CW health HERE
902- # logger.info(
903- # "[WORKER-HEALTH] source=%s worker=%s status=%s jobs=%d queues=%s",
904- # source_name,
905- # worker_name,
906- # status,
907- # running_jobs,
908- # sorted(queue_names),
909- # )
0 commit comments