
    h                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ d dlZd dlZd dlZd dlZ G d d          Zd Zd Zd Z G d	 d
ej                  Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          ZddZ dS )    N)
namedtuple)update_wrapperc                       e Zd Zi Zi Zed             Zd Zed             Zed             Z		 	 	 ddZ
d Zdd	Zd
 Z	 	 	 ddZdS )CmfDeferredJobWrapperc                 $    | j          d| j         S )N.)
__module____name__funcs    ./cmf/cmf_deferred_job.pygen_task_namez#CmfDeferredJobWrapper.gen_task_name   s    
 /33DM333    c                 0    | j         j         d| j         dS )Nz(name=))	__class__r
   nameselfs    r   __repr__zCmfDeferredJobWrapper.__repr__!   s     .)======r   c                     | dk    rd} | S )Nz	@minutelyz* * * * * H )schedules    r   adapt_schedulez$CmfDeferredJobWrapper.adapt_schedule$   s    {""(r   c                     	 t           j                             | d           d S # t           j        $ r}t          d|            |d }~ww xY w)N   1hash_idzinvalid schedule: )croniterexpandCroniterBadCronError
ValueError)r   es     r   validate_schedulez'CmfDeferredJobWrapper.validate_schedule*   si    	E$$Xt$<<<<<, 	E 	E 	E<(<<==1D	Es   !% AAANFc                 h   t          | |           || _        |                     |          | _        |	p| j        | _        || _        || _        || _        || _        || _	        || _
        || _        |
| _        | j        r|st          d| j         d          |st          d| j         d          	 |                     |                     | j                             n2# t          $ r%}t          d| j         d| j                   |d }~ww xY w| | j        | j        <   | | j        | j        <   d S )Nz$CmfDeferredJobWrapper: periodic job(z ) must be with option only_once.z!) must be with option system_job.zinvalid job configuration(z): )r   r   r   r   descriptionpriority
system_job	only_onceonly_once_argssoft_time_limit	countdownr   show_bg_progressbarr"   r$   r   periodicregistry)r   r   r'   r)   r*   r+   r(   r,   r   r&   r-   r#   s               r   __init__zCmfDeferredJobWrapper.__init__1   sf    	tT"""	&&t,,	&3$) $",." #6 = 
	, u !s	!s!s!sttt v !t	!t!t!tuuud&&t':':4='I'IJJJJ d d d ![di![![DM![![\\bccd'+DM$)$#'di   s   4-C" "
D, DDc                      | j         |i |S Nr   r   argskwargss      r   __call__zCmfDeferredJobWrapper.__call__N   s    ty$)&)))r   c                 ,    |g }|i } | j         |i |S r2   r   r3   s      r   applyzCmfDeferredJobWrapper.applyQ   s-    <D>Fty$)&)))r   c                 0    |                      ||          S )N)r4   r5   )apply_asyncr3   s      r   delayzCmfDeferredJobWrapper.delayX   s    T&999r   c                    ddl m}m} dd l}|g }i || j        }|| j        }|| j        }|| j        }|| j        }|	                    | j
        | j        ||          }|| j        |_        |	s|j        j        }	|s|	|_        || j        }|r)t!                      t#          j        |          z   |_        | j        rt)          | j
                  }d}| j        ]|r-t-          d| j
         d|           f |t)          |          z  }r+|t)          fdt/                    D                       z  }n"| j        D ]}|t)          |                   z  }|r>|d	t1          j        |                                                                          z   z  }||_        |j	                            |d
dg          }|r|                    d| j
         d           |j        r|j        r|j        |j        k     r{|                    d| j
         d|j                    |j        |_        |j        j         !                                5  |"                                 d d d            n# 1 swxY w Y   d S |
|j#        }
||j$        }||j%        }|| j        | j        |j        j&        | j        ||j        j&        ||
|||dd|_'        |j        j         !                                5  |"                                 d d d            n# 1 swxY w Y   |rU|j        (                    t)          |j                             ddl m)}  |ddt)          |j                  i|	g           t)          |j                  S )Nr   modelsg)r   r)   r&   r-   seconds zJCmfDeferredJobWrapper.apply_async: !!! Warning! Use args + only_once! job=z, args=c                 "    i | ]}||         S r   r   ).0kr5   s     r   
<dictcomp>z5CmfDeferredJobWrapper.apply_async.<locals>.<dictcomp>   s    $J$J$JaQq	$J$J$Jr   __open*)only_once_keystatusfieldszCmfDeferredJobWrapper(z): already queuedz): reschedule existent job )r)   r*   	person_idr(   r+   r'   r,   
admin_modecomponent_idsession_tab_idr-   )r4   r5   options)cmf_emit_eventz deferred-job-show_bg_progressbarjob_id)event_persons)*cmf.includer>   r?   
cmf.fieldsr+   r(   r'   r&   r-   CmfDeferredJobr   r)   current_personidrM   r,   cmf_nowdatetime	timedeltaplan_start_datetimestrr*   printsortedhashlibmd5encode	hexdigestrJ   getdebugutilcmfutildisable_aclsaveacl_admin_moderO   rP   valueparams_jsonappendrR   )r   r4   r5   r'   r+   r(   r,   r&   r-   g_current_user_idg_acl_admin_modeg_component_idg_session_tab_idr>   r?   cmfjobrJ   args_strarg_keyexistent_jobrR   s     `                   r   r:   z!CmfDeferredJobWrapper.apply_async[   s    	*)))))))<D>F""2OJ}H*K&"&":##dn+cv $ x x=CL  	4 ! 0 3 	.-CM I 	X&-ii(2DY2W2W2W&WC#> &		NNMH"*
  *O6:iO OHLO OP P Q Q D		)H L$J$J$J$J6&>>$J$J$J K KKH  $2 5 5GF7O 4 44HH SHOO4E4E(F(F(P(P(R(R!RR -C "044=Y_ilhm4nnL 	MMMMNNN3 ,!$!8,<?<SVbVv<v<vGGlTYll[g[jllmmm7:7NL4)5577 , ,$))+++, , , , , , , , , , , , , , ,  # /!^N# / !^"&"5 ]0"o#2L.&. ."2': 
 
$ X))++ 	 	HHJJJ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  	|!((SV555222222N>3sv;;@Whygz{{{{36{{s$   I66I:=I:6LLL)	NFNNFNNNF)NN)NNNNNNNNNNNN)r
   r	   __qualname__r/   r.   staticmethodr   r   r   r$   r0   r6   r8   r;   r:   r   r   r   r   r      s        HH4 4 \4> > >   \
 E E \E HL_c %( ( ( (:* * ** * * *: : : sw:>imu u u u u ur   r   c                      fd}d}| r/t          |           dk    r	| d         }nt          d|  d          |r ||          S |S )za
    job decorator:
        return object like celery task with __call__, delay, apply_async
    c                     t          | fi S r2   )r   )func_r5   s    r   wrapperz!cmf_deferred_job.<locals>.wrapper   s    $U55f555r   N   r   z+cmf_deferred_job takes exactly 1 argument (z given)len	TypeError)r4   r5   r}   r   s    `  r   cmf_deferred_jobr      s{    
6 6 6 6 6 D Xt99>>7DDV$VVVWWW wt}}r   c                  P    dd l } | j        j                                        j        S )Nr   )rV   rL   CmfDateTimenowrl   )rs   s    r   rZ   rZ      s(    :!%%''--r   c                    t          j                    }| d         }t          j        | d         | d         |d|           |                                }|                                 |dd         dk    r
|dd         }|S )zfrom logging Formatter class   r   r~   N
)ioStringIO	tracebackprint_exceptiongetvalueclose)eisiotbss       r   format_exceptionr      sv    
+--C	AB beRUBc:::AIIKKKv~~crcFHr   c                   <     e Zd Zed             Zd fd	Zd Z xZS )JobWorkerGreenletc                  ^    t           j                             t           j        j                  S r2   )r[   r   timezoneutcr   r   r   r   zJobWorkerGreenlet.now   s     $$X%6%:;;;r   Nc                 V    || _         t                                                       d S r2   )	worker_idsuperr0   )r   r   r   s     r   r0   zJobWorkerGreenlet.__init__  s&    "r   c                 0    | j         j         d| j         dS )N(r   )r   r
   r   r   s    r   r   zJobWorkerGreenlet.__repr__  s     .)==DN====r   r2   )r
   r	   rx   ry   r   r0   r   __classcell__r   s   @r   r   r      se        < < \<     > > > > > > >r   r   c                   V     e Zd ZdZddef fdZd Zd Zd ZddZ	d	 Z
d
 Zd Z xZS )QueueProcessorzJobQueueProcessor:get:lockNr'   c                 H    || _          t                      j        di | d S )Nr   )r'   r   r0   )r   r'   r5   r   s      r   r0   zQueueProcessor.__init__  s-     ""6"""""r   c                    ddl m}m}m} dd l}|j                                        5  	 |j        j        	                    | j
        dd          5  d }| j        
dd| j        g}g dd	d| j        gg}|r|                    |           |j                            |
          D ]8}|                    d           t#          j        dd| d| j         d           9g ddg dddt'                      ggg}	|r|	                    |           |j                            g dg dg dgddg          }
|
r'|	                    dg dddd |
D             gg           |j                            |	dgd|          }|rbd|_        |j                                         | j        |_        |                    d            |                                 	 d d d            ng	 d d d            n# 1 swxY w Y    |             | j        st7          j        d!           n't7          j        t;          d!| j                             	 d d d            n# 1 swxY w Y   |S )"Nr   )r>   socketio
cmf_commitT
   timeoutblocking_timeoutr'   =rK   r   in_progressr   filterz+Job in_progress but queue processor is not.z%sz!!! Not running job , priority=z, mark as dead!)rK   r   rH   OR)r]   r   Nr]   z<=)r)   r   T)rJ   z!=Nz--rJ   )r   rL   )rJ   r   NNOT INc                     g | ]	}|j         
S r   )rJ   )rD   job_s     r   
<listcomp>z0QueueProcessor._job_get_next.<locals>.<listcomp><  s    8n8n8nPT9K8n8n8nr   cmf_created_at)r   order_by
for_updaterL   r   z	job startr   )rU   r>   r   r   cmf.appappcmf_contextrg   rh   CmfLock_GET_LOCK_KEYr'   r   rn   rW   liston_errorloggingerrorrZ   slistre   rK   start_datetimeset_now	error_logrj   timesleepmax)r   rL   r>   r   r   rs   priority_filterstuck_filter	stuck_job
job_filteronly_once_in_progressnext_jobs               r   _job_get_nextzQueueProcessor._job_get_next  s   <<<<<<<<<<W  "" :	2 :	292!))$*<b[])^^ 1 1"&=,'13&FO 322 #t~6  # 9 ''888!'!6!;!;<!;!P!P u uI&&'TUUUM$(sy(s(sUYUb(s(s(stttt ,++:::.gii@
 # 7%%o666(.(=(C(C4446S6S6SUsUsUst /2 )D )4 )4% ) %% 888,h8n8nXm8n8n8no   "044%1A0Bt! 5 # #  &3HO+33555)-H&&&|444MMOOOc1 1 1 1 1 1 1VW1 1 1 1 1 1 1 1 1 1 1 1 1 1 1d JLLL= 2
1 
3q$-00111s921:	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2z s=   )IFG/I#I/G3	3I6G3	7AIIIc                     ddl m} dd l}|j                                        5  |                    |           d d d            d S # 1 swxY w Y   d S )Nr   r   )rU   r   r   r   r   
on_success)r   rt   resultr   rs   s        r   _job_set_successzQueueProcessor._job_set_successT  s    ((((((W  "" 	# 	#NN6"""	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	#s   AAAc                     ddl m} dd l}|j                                        5  d}t          j                    }|d         rt          |          }|                    |           d d d            d S # 1 swxY w Y   d S )Nr   r   zError?)	rU   r   r   r   r   sysexc_infor   r   )r   rt   r   r   rs   
error_textr   s          r   _job_set_failzQueueProcessor._job_set_fail]  s    ((((((W  "" 	% 	%!J|~~H{ 8-h77
LL$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%s   AA33A7:A7c                 \    ddl mm dd lfd}t	          j        ||          S )Nr   r=   c                  ^   t           j        j                 } j                                        5  j                                       j        r#j        j        j	        k    rj        
                    j                            j        j        j        j                             j                                         j        d                             dd          rj                                         j        d                             dd          rj        d         d         _        j        d                             dd          rj        d         d         _         | j        d         i j        d         cd d d            S # 1 swxY w Y   d S )	NrL   rQ   rN   FrO   rP   r4   r5   )r   r/   r   r   r   rW   set_current_deferred_jobrM   system_personrY   set_current_person	CmfPersonre   APPcurrent_person_fieldsCmfAccessListsetup_contextrm   activate_admin_moderO   rP   )job_funcrs   r?   rt   r>   s    r   runnerz'QueueProcessor._job_run.<locals>.runnern  s   ,5ch?H$$&& W W%>>sCCC= 
XS]ao6H%H%HG..(,,S]37;Cd,eeg g g(66888y155lEJJ C,@@BBBy155neLL T),)CN)Sy1556FNN X+.?9+EFV+W(x!8VCOH<UVVW W W W W W W W W W W W W W W W W Ws   E#F""F&)F&)rU   r>   r?   r   geventwith_timeout)r   rt   r+   r   rs   r?   r>   s    `  @@@r   _job_runzQueueProcessor._job_runj  sl    ))))))))	W 	W 	W 	W 	W 	W 	W 	W$ "?F;;;r   c           	         |                      dg          }|sd S |j        d                             d          }	 |                     ||          }|                     ||           d S # t
          t          t          j        f$ rP}t          j
        d|j         d|j         d|            |                     |t          |                      d }~wt          t          j        f$ rS}t          j
        d|j         d|j         d	           |                     |t          |                     Y d }~d S d }~ww xY w)
NrI   r   rQ   r+   )r+   zjob_runner(z, z) interrupt by ) error)r   rm   re   r   r   
SystemExitKeyboardInterruptr   GreenletExitr   	exceptionrY   r   r   r^   	ExceptionTimeout)r   rt   r+   r   r#   s        r   
_run_cyclezQueueProcessor._run_cycle  se      .. 	F/)4889JKK	,]]3]HHF !!#v.....-v/BC 	 	 	RCFRRchRRqRRSSSsCFF+++6>* 	, 	, 	,GCFGGchGGGHHHsCFF+++++++++	,s%   -A, ,D?ACD?,AD::D?c                    	 	 |                                   t          j        d           nW# t          t          t
          j        f$ r  t          $ r. t          j	        |  d           t          j        d           Y nw xY w)zmain() for GreenletTr~   z cycle error.   )
r   r   r   r   r   r   r   r   r   r   r   s    r   _runzQueueProcessor._run  s    		!!!
1 163FG      !T"8"8"8999
2		s   (, AB ?B c                 @    | j         j         d| j         d| j         dS )Nr   r   r   )r   r
   r   r'   r   s    r   r   zQueueProcessor.__repr__  s*    .)WWDNWWt}WWWWr   r2   )r
   r	   rx   r   intr0   r   r   r   r   r   r   r   r   r   s   @r   r   r   
  s        0M# # # # # # # #A A AF# # #	% 	% 	%< < < <0, , ,,  X X X X X X Xr   r   c                       e Zd ZdZd ZdS )QueueCleanerzJobQueueCleaner:lockc           
      `   dd l }ddlm}m} 	 	 |j        j                            | j        dd          5  |j        	                                5  t                      }|t          j        d          z
  }|t          j        d	          z
  }|j                            d
ddg dgdd|ggg ddd|ggg           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   n-# |$ r Y n&t          $ r t!          j        |  d           Y nw xY wt%          j        d           ")Nr   CmfGetLockErrorr>   T,  r      )days   )hoursr   rK   IN)failcanceldeadend_datetime<)rK   r   successr    error)r   rU   r   r>   rg   rh   r   	_LOCK_KEYr   r   rZ   r[   r\   rW   bulk_deleter   r   r   r   r   )r   rs   r   r>   r   max_end_datetimemax_success_end_datetimes          r   r   zQueueCleaner._run  s   77777777	3X%--dnc\]-^^ 
 
`c`g`s`s`u`u 
 
!))C'*X-?Q-G-G-G'G$/2X5Gb5Q5Q5Q/Q,)55 &.H.H.HIN\_aqKrs777.#Og9hi  6   
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 #    3 3 3!T///222223JsOOO%	s_   'C- C!A.C
>C!
C	C!C	C!C- !C%%C- (C%)C- -D4 DDNr
   r	   rx   r
  r   r   r   r   r   r     s(        &I    r   r   c                       e Zd ZdZd ZdS )QueueWatcherzJobQueueWatcher:lockc           
         dd l }ddlm}m} |j        j        j        j        }	 	 |j        j	        
                    | j        dd          5  |j                                        5  |                    d          }|r't          j                    t          |          z
  dk     r,t          j        d           	 d d d            d d d            |                    d	          }t%          |          D ]}|                    d
|                                           \|                    d	|           |                    |           t-          d|                                 dt.          j                   t3                      }|t5          j        d          z
  }	|j                            dgddd |D             gg ddd|	gg          D ]}
|
                    d           	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   n-# |$ r Y n&t<          $ r t?          j         |  d           Y nw xY wt          j        d           =)Nr   r   Tr   r   redis_idr   r   deferred_job_worker:workersdeferred_job_worker:heartbeat:zQueueWatcher: worker z seems dead!filer@   rI   r   r   c                 6    g | ]}|                                 S r   )decode)rD   w_ids     r   r   z%QueueWatcher._run.<locals>.<listcomp>  s     8^8^8^48^8^8^r   r   r   r  )rL   r   zJob Worker is deadr	  <   )!r   rU   r   r>   r   r   REDIS_DBredisrg   rh   r   r
  r   re   r   floatr   smembersr   r  sremremover_   r   stderrrZ   r[   r\   rW   r   r   r   r   )r   rs   r   r>   redis_dbr  
db_workersr   r   max_start_datert   s              r   r   zQueueWatcher._run  sU   77777777"%'+"6"< 	3X%--dnc\]-^^ ; ;`c`g`s`s`u`u ; ;'||J77H# !ty{{U8__'Dr'I'I
2 ; ; ; ; ; ; ; ; ; ; ; ; ; "*!2!23Q!R!RJ%)*%5%5 m m	#<<(]IYIYI[I[(](]^^f$MM*H)TTT&--i888!"Z):J:J:L:L"Z"Z"Zadakllll "))C%(8+=c+J+J+J%JN%499$'5!,h8^8^S]8^8^8^ _ > > >!13 G$  :     ; ; %9::::;#; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;2 #    3 3 3!T///222223JrNNNA 	ss   'H H&AG;:HH DG;/H;G?	?HG?	HH HH HH I% IINr  r   r   r   r  r    s(        &I& & & & &r   r  c                   @     e Zd ZdZ edd          Z fdZd Z xZS )JobScheduleru   
    - защита от параллельного запуска
    - "размазывание нагрузки" во времени.
    PeriodicDatazjob croniterc                 H     t                      j        |i | g | _        d S r2   )r   r0   r.   )r   r4   r5   r   s      r   r0   zJobScheduler.__init__  s*    $)&)))r   c           
      `   dd l }ddlm} |j        j        j        j        }d}t          j                            t          j	        j
                                                  j        }t          j                                        D ]e}|j        }|j        rE|j        |j        v r7|j        |j                 }t'          |  d|j         d|j         d|            n"t'          |  d|j         d|j         d           |st'          |  d|j         d           t                              |          }t                              |           t-          j        ||j        	          }||_        |                                 |r"t3          ||                                          n|                                }| j                            |                     ||                     g	 |r|t;          j                    z
  }	nt'          d           d}	|	dk    rt'          d|	dd           d}	t;          j        |	           	 t;          j                    }
d}| j        D ]\  }}|                                }|
|k    r|                                 |                    d|j         d| | j         dd
          rv	 |j        !                                5  |"                                 d d d            n# 1 swxY w Y   n/# tF          $ r" tI          j%        |  d|j         d           Y nw xY w|r"t3          ||                                          n|                                }n;# tF          $ r. tI          j%        |  d           t;          j        d           Y nw xY w)Nr   configz: config: job=z, app="z ", config.JOB_SCHEDULE_OVERRIDE="z: config: job z
 disabled!r   Tz1JobScheduler: warning next_ts == 0, nothing to dor   z)JobScheduler: warning too low sleep time z.3fr      zJobScheduler:lock::)exnxz schedule job(r   r	  )&r   rU   r+  r   r   r  r  r[   r   r   r   
astimezonetzinfor   r.   valuesr   JOB_SCHEDULE_OVERRIDEr   r_   r   r$   r   APP_FQDNget_nextminget_currentrn   r'  r   r   setr   r   r:   r   r   r   )r   rs   r+  r"  next_tslocal_tzinfort   r   	cron_itersleep_tsnow_tscurr_tss               r   r   zJobScheduler._run  s5   &&&&&&"%'+"6"<(,,X->-BCCNNPPW(188:: 	D 	DC|H+ OF<X0X0X!7AvvSXvvclvvltvvwwwwMMSXMMclMMMNNN AASXAAABBB,;;HEEH!33H=== )(FOLLLI+I   ?Fcc'9#8#8#:#:;;;ILaLaLcLcGM  !2!23	!B!BCCCC	 "TY[[0JKKK1}}Q(QQQQRRRJx   &*m l lNC'3355G((!**,,,#<<(QSX(Q(Q(Q(QSWSafhmq<rr \\%(W%8%8%:%: !6 !6$'OO$5$5$5!6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6#, \ \ \ ' 1T2Z2Z2Z2Z2Z [ [ [ [ [\GNkc'9+@+@+B+BCCCT]TiTiTkTkGGl    !T///222
27	s[   	A9M3 L	K=1L	=L	L	L	L	M3 	)L52M3 4L55=M3 35N+*N+)	r
   r	   rx   __doc__r   r'  r0   r   r   r   s   @r   r&  r&    sd          :nn==L    8 8 8 8 8 8 8r   r&  c                       e Zd ZdZdZd ZdS )RedisCleaneruD   Раз в сутки сбрасываем редис БД. В 02-50.zRedisCleaner:lockc                    dd l }ddlm}m} |j        j        j        j        }	 t          j	        d           	 t          j
                                        }|j        dk    s|j        dk     rK|j        j                            | j        dd          5  |                    d          }|r't          j                    t'          |          z
  d	k     r	 d d d            t)          d
t*          j                   |                                 d d d            n# 1 swxY w Y   nB# t          j        j        $ r Y n,|$ r Y n&t4          $ r t7          j        |  d           Y nw xY wH)Nr   )r   	CMF_CACHETr   r   2   r   r  i`T  z%RedisCleaner: run CMF_CACHE.flushdb()r  r	  )r   rU   r   rD  r   r   r  r  r   r   r[   r   hourminuterg   rh   r   r
  re   r  r_   r   r!  flushdb
exceptionsLockNotOwnedErrorr   r   r   )r   rs   r   rD  r"  r   r  s          r   r   zRedisCleaner._run<  s   ::::::::"%'+"6"<	3JsOOO3'++--8q==CJOOX%--dnc\]-^^ ( ('||J77H# !ty{{U8__'Dx'O'O ( ( ( ( ( ( ( BTTTT%%'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( #5   "    3 3 3!T///222223%	3sT   4D( .'D( A DD( !/DD( D  D( #D $D( (E'>E' E'&E'N)r
   r	   rx   r@  r
  r   r   r   r   rB  rB  8  s.        NN#I3 3 3 3 3r   rB  c                   f     e Zd Z fdZd ZdefdZddedefdZ	defdZ
dded
efdZd Z xZS )RedisMonitorc                     || _         |j        | _        || _        |                                  t	          j        d          | _         t                      j        di | d S )Nzredis-monitorr   )	r"  redis_instance_nameredis_settings_manager_RedisMonitor__init_redisr   	getLoggerloggerr   r0   )r   r"  rO  r5   r   s       r   r0   zRedisMonitor.__init__Y  sd     #+#? &<#'88""6"""""r   c                 &   ddl m} t          |d          rm|j        | j                 d         dk    rQ|j        | j                                                 }|                    d           t          j        di || _        d S t          d          )Nr   r*  cache_settingstyper  uJ   В конфиге не содержится конфигурации Redisr   )
rU   r+  hasattrrT  rN  copypopr  Redisr"   )r   r+  cfgs      r   __init_rediszRedisMonitor.__init_redisa  s    &&&&&& 6+,, 	k1FtG_1`ag1hls1s1s'(@AFFHHCGGFOOO++s++DJJJijjjr   returnc                 2   ddl m} t          |j                                                  }t          |          dk    r|d         S 	 |                    | j                  }||dz   t          |          z           }n# t          $ r |d         }Y nw xY w|S )u!  
        Возвращает следующую redis конфигурацию из config.cache_settings.
        В случае, если текущий инстанс последний в списке, то начинает обходить
        список заново
        r   r*  r~   )	rU   r+  r   rT  keysr   indexrN  r"   )r   r+  r^  idxnext_instances        r   _get_new_instancezRedisMonitor._get_new_instancem  s     	'&&&&&F)..0011t99>>7N	$**T566C #'SYY!67MM 	$ 	$ 	$ GMMM	$ s   	5A? ?BB   intervalc                     	 |                                  | _        | j                            d| j                   |                                  |                                 }|rdS t          j        |           z)NTu=   Получена новая конфигурация Redis %s)rb  rN  rR  warningrP  check_redis_connectionr   r   )r   rd  
is_successs      r   _check_new_instancez RedisMonitor._check_new_instance  s}    	#'+'='='?'?D$K _aeayzzz4466J tL"""	#r   c                 |   | j         j        j        }|                    d          p|                    d          }	 | j                                          dS # t           j        $ r%}| j                            d|           Y d }~n5d }~wt          $ r%}| j                            d|           Y d }~nd }~ww xY wdS )NhostpathTuA   Ошибка подключения к Redis по адресу %su=   Системная ошибка с Redis по адресу %sF)	r  connection_poolconnection_kwargsre   pingConnectionErrorrR  rf  r   )r   rn  current_hostr#   s       r   rg  z#RedisMonitor.check_redis_connection  s     J6H(,,V44U8I8M8Mf8U8U	oJOO4$ 	s 	s 	sK ceqrrrrrrrr 	o 	o 	oK _amnnnnnnnn	o us#   A B9'BB9B44B9r-  max_failure_countc                    d}	 |                                  }|rd}n|dz  }| j                            d|           ||k    ry| j                            d|           |                                 }|rH| j                            d| j                   | j                            | j        d           t          t          j	        |           )Nr   Tr~   u.   Redis недоступен (%d попытка)u3   Redis недоступен %d раза подрядuA   Записываем новую конфигурацию Redis %s)
with_flush)
rg  rR  rf  ri  rN  rO  
save_to_dbr   r   r   )r   rd  rr  failure_countrh  is_new_successs         r   monitor_rediszRedisMonitor.monitor_redis  s    	#4466J ) !"##$TVcddd $555K''(]_lmmm%)%=%=%?%?N% )++,oqu  rJ  K  K  K3>>t?Wdh>iii((L"""#	#r   c                 .    |                                   d S r2   )rx  r   s    r   r   zRedisMonitor._run  s    r   )rc  )rc  r-  )r
   r	   rx   r0   rP  r^   rb  r   boolri  rg  rx  r   r   r   s   @r   rL  rL  X  s        # # # # #
k 
k 
k3    (	# 	#C 	# 	# 	# 	# 	#    # #c ## # # # #*      r   rL  Fc                 x   ddl }ddlm} |j        j        }t          j                     dt          j                     }|j	        r5t          |j        j        j        ||          }|                                 |                                 |j        j        j        j        }| sd|j        j        _        g }| r%|                    t'          d|                     n6t)          d          D ]&}|                    t'          ||                     '|                    t+          |	                     |                    t-          |	                     |                    t/          |	                     |                    t1          |	                     |D ]}	|	                                 	 	 	 |                    d
| dd           |                    d|           n.# t6          $ r!}
t9          j        d|
            Y d}
~
nd}
~
ww xY wt=          j        |d          5 }|D ]'}	tA          d|	 dtB          j"                   tF          	 ddd           n# 1 swxY w Y   # tF          tH          t<          j%        f$ r)}
tA          d|
 dtB          j"                   Y d}
~
nd}
~
ww xY w	 t=          j&        |d           n0# t<          j'        $ r tA          dtB          j"                   Y nw xY w|D ]}	|	r tA          d|	 dtB          j"                   $|	(                                r tA          d|	 dtB          j"                   X	 |	)                                 n# tF          tH          t<          j%        t6          f$ r t9          j        d|	 d           Y w xY wdS )u!  
    Функция - обработчик очередей задач.
    На каждый приоритет(очередь) запускаем отдельный поток - обработчик.
    Сервисный поток для очистки очереди от устаревших задач.
    Сервисный поток для контроля погибших задач(чей воркер погиб).
    Сервисный поток Beat, для планирования периодических задач.
    r   Nr*  r.  )r"  rO  r   T)r'   r      )r   r  -   )r/  r  z$deferred_job_worker: redis error!!! r   )r   z%deferred_job_worker(): system thread(z) exited!!! Stop work.r  zdeferred_job_worker(): stop by z. Try kill all threads...z(deferred_job_worker(): killall timeout!.zdeferred_job_worker(): Thread z is not stopped!!!z stops normally!z error.)*r   rU   r+  r   REDIS_SETTINGS_MANAGERplatformnodeuuiduuid1CACHE_REDIS_FAILOWERrL  r   r  start
init_redisr  rD  job_daemon_modern   r   ranger   r  r&  rB  r9  saddr   r   r   r   iwaitr_   r   r!  r   r   r   killallr   
successfulre   )single_queuers   r+  rO  r   redis_monitorr"  	greenletsr'   glr#   gl_iters               r   deferred_job_workerr    s    NNN"""""" W;=??33TZ\\33I" $W[)#9
 
 

 	%%'''!gk28H 1,0)I UKKKLLLLb		 	U 	UH^XSSSTTTT\I666777\I666777\I666777\I666777  




_	%NIiII3SUVVV<iHHHH  N N N!"L"L"LMMMMMMMMN i444 %! % %B\"\\\cfcmnnnn$$%% % % % % % % % % % % % % % %	% )6+>? _ _ _LLLLSVS]^^^^^^^^^_Ly"-----> L L L9
KKKKKKL  L L 	I2IIIPSPZ[[[[==?? 	G2GGGcjYYYY	LFFHHHH-v/BIN 	L 	L 	LJrJJJKKKKK	LL Ls   >I:  1G2 1I: 2
H<HI: HI: 6+I."I: .I22I: 5I26I: :J?J::J?K *LL%M:::N76N7)F)!r[   ra   r   ossignalr   r   r   r   r  r  collectionsr   	functoolsr   r   r   r  cmf.util.cmfutilrs   r   r   rZ   r   Greenletr   r   r   r  r&  rB  rL  r  r   r   r   <module>r     s     				 				       



    " " " " " " $ $ $ $ $ $       z z z z z z z zz  *. . .  
> 
> 
> 
> 
> 
> 
> 
>\X \X \X \X \X& \X \X \X~    $   :) ) ) ) )$ ) ) )XC C C C C$ C C CL3 3 3 3 3$ 3 3 3@W W W W W$ W W WtOL OL OL OL OL OLr   