
    hm                     t   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlZd dlZd dlZd dlZ G d d          Zd Zd Zd Z G d	 d
ej                  Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          ZddZdS )    N)
namedtuple)update_wrapperc                       e Zd Zi Zi Zed             Zd Zed             Zed             Z		 	 	 ddZ
d Zdd	Zd
 Z	 	 ddZdS )CmfDeferredJobWrapperc                 $    | j          d| j         S )N.)
__module____name__funcs    ./cmf/cmf_deferred_job.pygen_task_namez#CmfDeferredJobWrapper.gen_task_name   s    
 /33DM333    c                 0    | j         j         d| j         dS )Nz(name=))	__class__r
   nameselfs    r   __repr__zCmfDeferredJobWrapper.__repr__    s     .)======r   c                     | dk    rd} | S )Nz	@minutelyz* * * * * H )schedules    r   adapt_schedulez$CmfDeferredJobWrapper.adapt_schedule#   s    {""(r   c                     	 t           j                             | d           d S # t           j        $ r}t          d|            |d }~ww xY w)N   1hash_idzinvalid schedule: )croniterexpandCroniterBadCronError
ValueError)r   es     r   validate_schedulez'CmfDeferredJobWrapper.validate_schedule)   si    	E$$Xt$<<<<<, 	E 	E 	E<(<<==1D	Es   !% AAANFc                 h   t          | |           || _        |                     |          | _        |	p| j        | _        || _        || _        || _        || _        || _	        || _
        || _        |
| _        | j        r|st          d| j         d          |st          d| j         d          	 |                     |                     | j                             n2# t          $ r%}t          d| j         d| j                   |d }~ww xY w| | j        | j        <   | | j        | j        <   d S )Nz$CmfDeferredJobWrapper: periodic job(z ) must be with option only_once.z!) must be with option system_job.zinvalid job configuration(z): )r   r   r   r   descriptionpriority
system_job	only_onceonly_once_argssoft_time_limit	countdownr   show_bg_progressbarr"   r$   r   periodicregistry)r   r   r'   r)   r*   r+   r(   r,   r   r&   r-   r#   s               r   __init__zCmfDeferredJobWrapper.__init__0   sf    	tT"""	&&t,,	&3$) $",." #6 = 
	, u !s	!s!s!sttt v !t	!t!t!tuuud&&t':':4='I'IJJJJ d d d ![di![![DM![![\\bccd'+DM$)$#'di   s   4-C" "
D, DDc                      | j         |i |S Nr   r   argskwargss      r   __call__zCmfDeferredJobWrapper.__call__M   s    ty$)&)))r   c                 ,    |g }|i } | j         |i |S r2   r   r3   s      r   applyzCmfDeferredJobWrapper.applyP   s-    <D>Fty$)&)))r   c                 0    |                      ||          S )N)r4   r5   )apply_asyncr3   s      r   delayzCmfDeferredJobWrapper.delayW   s    T&999r   c	                    ddl m}	m}
 dd l}|g }i || j        }|| j        }|| j        }|| j        }|| j        }|		                    | j
        | j        ||          }|| j        |_        |s|
j        j        |_        || j        }|r)t!                      t#          j        |          z   |_        | j        rt)          | j
                  }d}| j        ]|r-t-          d| j
         d|           f |t)          |          z  }r+|t)          fdt/                    D                       z  }n"| j        D ]}|t)          |                   z  }|r>|d	t1          j        |                                                                          z   z  }||_        |	j	                            |d
dg          }|r|
                    d| j
         d           |j        r|j        r|j        |j        k     r{|
                    d| j
         d|j                    |j        |_        |j        j         !                                5  |"                                 d d d            n# 1 swxY w Y   d S || j        | j        |j        j#        | j        ||j        j#        ||
j$        |
j%        |
j&        |dd|_'        |j        j         !                                5  |"                                 d d d            n# 1 swxY w Y   |r,|
j        (                    t)          |j                             t)          |j                  S )Nr   modelsg)r   r)   r&   r-   seconds zJCmfDeferredJobWrapper.apply_async: !!! Warning! Use args + only_once! job=z, args=c                 "    i | ]}||         S r   r   ).0kr5   s     r   
<dictcomp>z5CmfDeferredJobWrapper.apply_async.<locals>.<dictcomp>   s    $J$J$JaQq	$J$J$Jr   __open*)only_once_keystatusfieldszCmfDeferredJobWrapper(z): already queuedz): reschedule existent job )r)   r*   	person_idr(   r+   r'   r,   
admin_modecomponent_idsession_tab_idr-   )r4   r5   options))cmf.includer>   r?   
cmf.fieldsr+   r(   r'   r&   r-   CmfDeferredJobr   r)   current_personidrM   r,   cmf_nowdatetime	timedeltaplan_start_datetimestrr*   printsortedhashlibmd5encode	hexdigestrJ   getdebugutilcmfutildisable_aclsavevalueacl_admin_moderO   rP   params_jsonappend)r   r4   r5   r'   r+   r(   r,   r&   r-   r>   r?   cmfjobrJ   args_strarg_keyexistent_jobs     `              r   r:   z!CmfDeferredJobWrapper.apply_asyncZ   sc    	*)))))))<D>F""2OJ}H*K&"&":##dn+cv $ x x=CL 	0,/CM I 	X&-ii(2DY2W2W2W&WC#> %		NNMH"*
  *O6:iO OHLO OP P Q Q D		)H L$J$J$J$J6&>>$J$J$J K KKH  $2 5 5GF7O 4 44HH SHOO4E4E(F(F(P(P(R(R!RR -C "044=Y_ilhm4nnL MMMMNNN3 ,!$!8,<?<SVbVv<v<vGGlTYll[g[jllmmm7:7NL4)5577 , ,$))+++, , , , , , , , , , , , , , , !^"&"5 ]0"o#2L.&. !"#"2': 
 
$ X))++ 	 	HHJJJ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  	6!((SV55536{{s$   I22I69I6&LLL)	NFNNFNNNF)NN)NNNNNNNN)r
   r	   __qualname__r/   r.   staticmethodr   r   r   r$   r0   r6   r8   r;   r:   r   r   r   r   r      s        HH4 4 \4> > >   \
 E E \E HL_c %( ( ( (:* * ** * * *: : : sw:>h h h h h hr   r   c                      fd}d}| r/t          |           dk    r	| d         }nt          d|  d          |r ||          S |S )za
    job decorator:
        return object like celery task with __call__, delay, apply_async
    c                     t          | fi S r2   )r   )func_r5   s    r   wrapperz!cmf_deferred_job.<locals>.wrapper   s    $U55f555r   N   r   z+cmf_deferred_job takes exactly 1 argument (z given)len	TypeError)r4   r5   rv   r   s    `  r   cmf_deferred_jobrz      s{    
6 6 6 6 6 D Xt99>>7DDV$VVVWWW wt}}r   c                  P    dd l } | j        j                                        j        S )Nr   )rS   rL   CmfDateTimenowrh   )rl   s    r   rW   rW      s(    :!%%''--r   c                    t          j                    }| d         }t          j        | d         | d         |d|           |                                }|                                 |dd         dk    r
|dd         }|S )zfrom logging Formatter class   r   rw   N
)ioStringIO	tracebackprint_exceptiongetvalueclose)eisiotbss       r   format_exceptionr      sv    
+--C	AB beRUBc:::AIIKKKv~~crcFHr   c                   <     e Zd Zed             Zd fd	Zd Z xZS )JobWorkerGreenletc                  ^    t           j                             t           j        j                  S r2   )rX   r}   timezoneutcr   r   r   r}   zJobWorkerGreenlet.now   s     $$X%6%:;;;r   Nc                 V    || _         t                                                       d S r2   )	worker_idsuperr0   )r   r   r   s     r   r0   zJobWorkerGreenlet.__init__   s&    "r   c                 0    | j         j         d| j         dS )N(r   )r   r
   r   r   s    r   r   zJobWorkerGreenlet.__repr__   s     .)==DN====r   r2   )r
   r	   rq   rr   r}   r0   r   __classcell__r   s   @r   r   r      se        < < \<     > > > > > > >r   r   c                   V     e Zd ZdZddef fdZd Zd Zd ZddZ	d	 Z
d
 Zd Z xZS )QueueProcessorzJobQueueProcessor:get:lockNr'   c                 H    || _          t                      j        di | d S )Nr   )r'   r   r0   )r   r'   r5   r   s      r   r0   zQueueProcessor.__init__   s-     ""6"""""r   c                    ddl m}m}m} dd l}|j                                        5  	 |j        j        	                    | j
        dd          5  d }| j        
dd| j        g}g dd	d| j        gg}|r|                    |           |j                            |
          D ]8}|                    d           t#          j        dd| d| j         d           9g ddg dddt'                      ggg}	|r|	                    |           |j                            g dg dg dgddg          }
|
r'|	                    dg dddd |
D             gg           |j                            |	dgd|          }|rbd|_        |j                                         | j        |_        |                    d            |                                 	 d d d            ng	 d d d            n# 1 swxY w Y    |             | j        st7          j        d!           n't7          j        t;          d!| j                             	 d d d            n# 1 swxY w Y   |S )"Nr   )r>   socketio
cmf_commitT
   timeoutblocking_timeoutr'   =rK   r   in_progressr   filterz+Job in_progress but queue processor is not.z%sz!!! Not running job , priority=z, mark as dead!)rK   r   rH   OR)rZ   r   NrZ   z<=)r)   r   T)rJ   z!=Nz--rJ   )r   rL   )rJ   r   NNOT INc                     g | ]	}|j         
S r   )rJ   )rD   job_s     r   
<listcomp>z0QueueProcessor._job_get_next.<locals>.<listcomp>.  s    8n8n8nPT9K8n8n8nr   cmf_created_at)r   order_by
for_updaterL   r   z	job startr   )rR   r>   r   r   cmf.appappcmf_contextrd   re   CmfLock_GET_LOCK_KEYr'   r   rk   rT   liston_errorloggingerrorrW   slistrb   rK   start_datetimeset_now	error_logrg   timesleepmax)r   rL   r>   r   r   rl   priority_filterstuck_filter	stuck_job
job_filteronly_once_in_progressnext_jobs               r   _job_get_nextzQueueProcessor._job_get_next  s   <<<<<<<<<<W  "" :	2 :	292!))$*<b[])^^ 1 1"&=,'13&FO 322 #t~6  # 9 ''888!'!6!;!;<!;!P!P u uI&&'TUUUM$(sy(s(sUYUb(s(s(stttt ,++:::.gii@
 # 7%%o666(.(=(C(C4446S6S6SUsUsUst /2 )D )4 )4% ) %% 888,h8n8nXm8n8n8no   "044%1A0Bt! 5 # #  &3HO+33555)-H&&&|444MMOOOc1 1 1 1 1 1 1VW1 1 1 1 1 1 1 1 1 1 1 1 1 1 1d JLLL= 2
1 
3q$-00111s921:	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2z s=   )IFG/I#I/G3	3I6G3	7AIIIc                     ddl m} dd l}|j                                        5  |                    |           d d d            d S # 1 swxY w Y   d S )Nr   r   )rR   r   r   r   r   
on_success)r   rm   resultr   rl   s        r   _job_set_successzQueueProcessor._job_set_successF  s    ((((((W  "" 	# 	#NN6"""	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	#s   AAAc                     ddl m} dd l}|j                                        5  d}t          j                    }|d         rt          |          }|                    |           d d d            d S # 1 swxY w Y   d S )Nr   r   zError?)	rR   r   r   r   r   sysexc_infor   r   )r   rm   r   r   rl   
error_textr   s          r   _job_set_failzQueueProcessor._job_set_failO  s    ((((((W  "" 	% 	%!J|~~H{ 8-h77
LL$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%s   AA33A7:A7c                 \    ddl mm dd lfd}t	          j        ||          S )Nr   r=   c                  ^   t           j        j                 } j                                        5  j                                       j        r#j        j        j	        k    rj        
                    j                            j        j        j        j                             j                                         j        d                             dd          rj                                         j        d                             dd          rj        d         d         _        j        d                             dd          rj        d         d         _         | j        d         i j        d         cd d d            S # 1 swxY w Y   d S )	NrL   rQ   rN   FrO   rP   r4   r5   )r   r/   r   r   r   rT   set_current_deferred_jobrM   system_personrV   set_current_person	CmfPersonrb   APPcurrent_person_fieldsCmfAccessListsetup_contextrj   activate_admin_moderO   rP   )job_funcrl   r?   rm   r>   s    r   runnerz'QueueProcessor._job_run.<locals>.runner`  s   ,5ch?H$$&& W W%>>sCCC= 
XS]ao6H%H%HG..(,,S]37;Cd,eeg g g(66888y155lEJJ C,@@BBBy155neLL T),)CN)Sy1556FNN X+.?9+EFV+W(x!8VCOH<UVVW W W W W W W W W W W W W W W W W Ws   E#F""F&)F&)rR   r>   r?   r   geventwith_timeout)r   rm   r+   r   rl   r?   r>   s    `  @@@r   _job_runzQueueProcessor._job_run\  sl    ))))))))	W 	W 	W 	W 	W 	W 	W 	W$ "?F;;;r   c           	         |                      dg          }|sd S |j        d                             d          }	 |                     ||          }|                     ||           d S # t
          t          t          j        f$ rP}t          j
        d|j         d|j         d|            |                     |t          |                      d }~wt          t          j        f$ rS}t          j
        d|j         d|j         d	           |                     |t          |                     Y d }~d S d }~ww xY w)
NrI   r   rQ   r+   )r+   zjob_runner(z, z) interrupt by ) error)r   rj   rb   r   r   
SystemExitKeyboardInterruptr   GreenletExitr   	exceptionrV   r   r   r[   	ExceptionTimeout)r   rm   r+   r   r#   s        r   
_run_cyclezQueueProcessor._run_cyclet  se      .. 	F/)4889JKK	,]]3]HHF !!#v.....-v/BC 	 	 	RCFRRchRRqRRSSSsCFF+++6>* 	, 	, 	,GCFGGchGGGHHHsCFF+++++++++	,s%   -A, ,D?ACD?,AD::D?c                    	 	 |                                   t          j        d           nW# t          t          t
          j        f$ r  t          $ r. t          j	        |  d           t          j        d           Y nw xY w)zmain() for GreenletTrw   z cycle error.   )
r   r   r   r   r   r   r   r   r   r   r   s    r   _runzQueueProcessor._run  s    		!!!
1 163FG      !T"8"8"8999
2		s   (, AB ?B c                 @    | j         j         d| j         d| j         dS )Nr   r   r   )r   r
   r   r'   r   s    r   r   zQueueProcessor.__repr__  s*    .)WWDNWWt}WWWWr   r2   )r
   r	   rq   r   intr0   r   r   r   r   r   r   r   r   r   s   @r   r   r      s        0M# # # # # # # #A A AF# # #	% 	% 	%< < < <0, , ,,  X X X X X X Xr   r   c                       e Zd ZdZd ZdS )QueueCleanerzJobQueueCleaner:lockc           
      `   dd l }ddlm}m} 	 	 |j        j                            | j        dd          5  |j        	                                5  t                      }|t          j        d          z
  }|t          j        d	          z
  }|j                            d
ddg dgdd|ggg ddd|ggg           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   n-# |$ r Y n&t          $ r t!          j        |  d           Y nw xY wt%          j        d           ")Nr   CmfGetLockErrorr>   T,  r      )days   )hoursr   rK   IN)failcanceldeadend_datetime<)rK   r   successr    error)r   rR   r   r>   rd   re   r   	_LOCK_KEYr   r   rW   rX   rY   rT   bulk_deleter   r   r   r   r   )r   rl   r   r>   r}   max_end_datetimemax_success_end_datetimes          r   r   zQueueCleaner._run  s   77777777	3X%--dnc\]-^^ 
 
`c`g`s`s`u`u 
 
!))C'*X-?Q-G-G-G'G$/2X5Gb5Q5Q5Q/Q,)55 &.H.H.HIN\_aqKrs777.#Og9hi  6   
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 #    3 3 3!T///222223JsOOO%	s_   'C- C!A.C
>C!
C	C!C	C!C- !C%%C- (C%)C- -D4 DDNr
   r	   rq   r  r   r   r   r   r   r     s(        &I    r   r   c                       e Zd ZdZd ZdS )QueueWatcherzJobQueueWatcher:lockc           
         dd l }ddlm}m} |j        j        j        j        }	 	 |j        j	        
                    | j        dd          5  |j                                        5  |                    d          }|r't          j                    t          |          z
  dk     r,t          j        d           	 d d d            d d d            |                    d	          }t%          |          D ]}|                    d
|                                           \|                    d	|           |                    |           t-          d|                                 dt.          j                   t3                      }|t5          j        d          z
  }	|j                            dgddd |D             gg ddd|	gg          D ]}
|
                    d           	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   n-# |$ r Y n&t<          $ r t?          j         |  d           Y nw xY wt          j        d           =)Nr   r   Tr   r   redis_idr   r   deferred_job_worker:workersdeferred_job_worker:heartbeat:zQueueWatcher: worker z seems dead!filer@   rI   r   r   c                 6    g | ]}|                                 S r   )decode)rD   w_ids     r   r   z%QueueWatcher._run.<locals>.<listcomp>  s     8^8^8^48^8^8^r   r   r   r   )rL   r   zJob Worker is deadr  <   )!r   rR   r   r>   r   r   REDIS_DBredisrd   re   r   r  r   rb   r   floatr   smembersr   r  sremremover\   r   stderrrW   rX   rY   rT   r   r   r   r   )r   rl   r   r>   redis_dbr  
db_workersr   r}   max_start_daterm   s              r   r   zQueueWatcher._run  sU   77777777"%'+"6"< 	3X%--dnc\]-^^ ; ;`c`g`s`s`u`u ; ;'||J77H# !ty{{U8__'Dr'I'I
2 ; ; ; ; ; ; ; ; ; ; ; ; ; "*!2!23Q!R!RJ%)*%5%5 m m	#<<(]IYIYI[I[(](]^^f$MM*H)TTT&--i888!"Z):J:J:L:L"Z"Z"Zadakllll "))C%(8+=c+J+J+J%JN%499$'5!,h8^8^S]8^8^8^ _ > > >!13 G$  :     ; ; %9::::;#; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;2 #    3 3 3!T///222223JrNNNA 	ss   'H H&AG;:HH DG;/H;G?	?HG?	HH HH HH I% IINr  r   r   r   r	  r	    s(        &I& & & & &r   r	  c                   @     e Zd ZdZ edd          Z fdZd Z xZS )JobScheduleru   
    - защита от параллельного запуска
    - "размазывание нагрузки" во времени.
    PeriodicDatazjob croniterc                 H     t                      j        |i | g | _        d S r2   )r   r0   r.   )r   r4   r5   r   s      r   r0   zJobScheduler.__init__  s*    $)&)))r   c           
      `   dd l }ddlm} |j        j        j        j        }d}t          j                            t          j	        j
                                                  j        }t          j                                        D ]e}|j        }|j        rE|j        |j        v r7|j        |j                 }t'          |  d|j         d|j         d|            n"t'          |  d|j         d|j         d           |st'          |  d|j         d           t                              |          }t                              |           t-          j        ||j        	          }||_        |                                 |r"t3          ||                                          n|                                }| j                            |                     ||                     g	 |r|t;          j                    z
  }	nt'          d           d}	|	dk    rt'          d|	dd           d}	t;          j        |	           	 t;          j                    }
d}| j        D ]\  }}|                                }|
|k    r|                                 |                    d|j         d| | j         dd
          rv	 |j        !                                5  |"                                 d d d            n# 1 swxY w Y   n/# tF          $ r" tI          j%        |  d|j         d           Y nw xY w|r"t3          ||                                          n|                                }n;# tF          $ r. tI          j%        |  d           t;          j        d           Y nw xY w)Nr   )configz: config: job=z, app="z ", config.JOB_SCHEDULE_OVERRIDE="z: config: job z
 disabled!r   Tz1JobScheduler: warning next_ts == 0, nothing to dor   z)JobScheduler: warning too low sleep time z.3fr      zJobScheduler:lock::)exnxz schedule job(r   r  )&r   rR   r#  r   r   r  r  rX   r}   r   r   
astimezonetzinfor   r.   valuesr   JOB_SCHEDULE_OVERRIDEr   r\   r   r$   r   APP_FQDNget_nextminget_currentrk   r   r   r   setr   r   r:   r   r   r   )r   rl   r#  r  next_tslocal_tzinform   r   	cron_itersleep_tsnow_tscurr_tss               r   r   zJobScheduler._run  s5   &&&&&&"%'+"6"<(,,X->-BCCNNPPW(188:: 	D 	DC|H+ OF<X0X0X!7AvvSXvvclvvltvvwwwwMMSXMMclMMMNNN AASXAAABBB,;;HEEH!33H=== )(FOLLLI+I   ?Fcc'9#8#8#:#:;;;ILaLaLcLcGM  !2!23	!B!BCCCC	 "TY[[0JKKK1}}Q(QQQQRRRJx   &*m l lNC'3355G((!**,,,#<<(QSX(Q(Q(Q(QSWSafhmq<rr \\%(W%8%8%:%: !6 !6$'OO$5$5$5!6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6#, \ \ \ ' 1T2Z2Z2Z2Z2Z [ [ [ [ [\GNkc'9+@+@+B+BCCCT]TiTiTkTkGGl    !T///222
27	s[   	A9M3 L	K=1L	=L	L	L	L	M3 	)L52M3 4L55=M3 35N+*N+)	r
   r	   rq   __doc__r   r   r0   r   r   r   s   @r   r  r    sd          :nn==L    8 8 8 8 8 8 8r   r  c                       e Zd ZdZdZd ZdS )RedisCleaneruD   Раз в сутки сбрасываем редис БД. В 02-50.zRedisCleaner:lockc                    dd l }ddlm}m} |j        j        j        j        }	 t          j	        d           	 t          j
                                        }|j        dk    s|j        dk     rK|j        j                            | j        dd          5  |                    d          }|r't          j                    t'          |          z
  d	k     r	 d d d            t)          d
t*          j                   |                                 d d d            n# 1 swxY w Y   nB# t          j        j        $ r Y n,|$ r Y n&t4          $ r t7          j        |  d           Y nw xY wH)Nr   )r   	CMF_CACHETr   r   2   r   r  i`T  z%RedisCleaner: run CMF_CACHE.flushdb()r  r  )r   rR   r   r<  r   r   r  r  r   r   rX   r}   hourminuterd   re   r   r  rb   r  r\   r   r  flushdb
exceptionsLockNotOwnedErrorr   r   r   )r   rl   r   r<  r  r}   r  s          r   r   zRedisCleaner._run.  s   ::::::::"%'+"6"<	3JsOOO3'++--8q==CJOOX%--dnc\]-^^ ( ('||J77H# !ty{{U8__'Dx'O'O ( ( ( ( ( ( ( BTTTT%%'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( #5   "    3 3 3!T///222223%	3sT   4D( .'D( A DD( !/DD( D  D( #D $D( (E'>E' E'&E'N)r
   r	   rq   r8  r  r   r   r   r   r:  r:  *  s.        NN#I3 3 3 3 3r   r:  Fc                    ddl }t          j                     dt          j                     }|j        j        j        j        }| sd|j        j	        _
        g }| r%|                    t          d|                     n6t          d          D ]&}|                    t          ||                     '|                    t          |                     |                    t          |                     |                    t!          |                     |                    t#          |                     |D ]}|                                 	 	 	 |                    d| d	d
           |                    d|           n.# t*          $ r!}t-          j        d|            Y d}~nd}~ww xY wt1          j        |d          5 }|D ]'}t5          d| dt6          j                   t:          	 ddd           n# 1 swxY w Y   # t:          t<          t0          j        f$ r)}t5          d| dt6          j                   Y d}~nd}~ww xY w	 t1          j         |d           n0# t0          j!        $ r t5          dt6          j                   Y nw xY w|D ]}|r t5          d| dt6          j                   $|"                                r t5          d| dt6          j                   X	 |#                                 n# t:          t<          t0          j        t*          f$ r t-          j        d| d           Y w xY wdS )u!  
    Функция - обработчик очередей задач.
    На каждый приоритет(очередь) запускаем отдельный поток - обработчик.
    Сервисный поток для очистки очереди от устаревших задач.
    Сервисный поток для контроля погибших задач(чей воркер погиб).
    Сервисный поток Beat, для планирования периодических задач.
    r   Nr&  T)r'   r      )r   r  -   )r'  r  z$deferred_job_worker: redis error!!! r   )r   z%deferred_job_worker(): system thread(z) exited!!! Stop work.r  zdeferred_job_worker(): stop by z. Try kill all threads...z(deferred_job_worker(): killall timeout!.zdeferred_job_worker(): Thread z is not stopped!!!z stops normally!z error.)$r   platformnodeuuiduuid1r   r   r  r  r<  job_daemon_moderk   r   ranger   r	  r  r:  startr1  saddr   r   r   r   iwaitr\   r   r  r   r   r   killallr   
successfulrb   )	single_queuerl   r   r  	greenletsr'   glr#   gl_iters	            r   deferred_job_workerrV  J  s?    NNN =??33TZ\\33I!gk28H 1,0)I UKKKLLLLb		 	U 	UH^XSSSTTTT\I666777\I666777\I666777\I666777  




_	%NIiII3SUVVV<iHHHH  N N N!"L"L"LMMMMMMMMN i444 %! % %B\"\\\cfcmnnnn$$%% % % % % % % % % % % % % % %	% )6+>? _ _ _LLLLSVS]^^^^^^^^^_Ly"-----> L L L9
KKKKKKL  L L 	I2IIIPSPZ[[[[==?? 	G2GGGcjYYYY	LFFHHHH-v/BIN 	L 	L 	LJrJJJKKKKK	LL Ls   H 1F H 
F;F61H 6F;;H +H H HH HH I4II!I8 8*J%$J%L:MM)F)rX   r^   r   randomr   r   r   r   rG  rI  collectionsr   	functoolsr   r   r   r  cmf.util.cmfutilrl   r   rz   rW   r   Greenletr   r   r   r	  r  r:  rV  r   r   r   <module>r\     sT     				       



    " " " " " " $ $ $ $ $ $       m m m m m m m m`  *. . .  
> 
> 
> 
> 
> 
> 
> 
>\X \X \X \X \X& \X \X \X~    $   :) ) ) ) )$ ) ) )XC C C C C$ C C CL3 3 3 3 3$ 3 3 3@CL CL CL CL CL CLr   