
    +ir                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlZ G d d          Zd Zd Zd Z G d	 d
ej                  Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          Z ddZ!dS )    N)
namedtuple)update_wrapperc                       e Zd Zi Zi Zed             Zd Zed             Zed             Z		 	 	 ddZ
d Zdd	Zd
 Z	 	 	 ddZdS )CmfDeferredJobWrapperc                 $    | j          d| j         S )N.)
__module____name__funcs    ./cmf/cmf_deferred_job.pygen_task_namez#CmfDeferredJobWrapper.gen_task_name   s    
 /33DM333    c                 0    | j         j         d| j         dS )Nz(name=))	__class__r
   nameselfs    r   __repr__zCmfDeferredJobWrapper.__repr__"   s     .)======r   c                     | dk    rd} | S )Nz	@minutelyz* * * * * H )schedules    r   adapt_schedulez$CmfDeferredJobWrapper.adapt_schedule%   s    {""(r   c                     	 t           j                             | d           d S # t           j        $ r}t          d|            |d }~ww xY w)N   1hash_idzinvalid schedule: )croniterexpandCroniterBadCronError
ValueError)r   es     r   validate_schedulez'CmfDeferredJobWrapper.validate_schedule+   si    	E$$Xt$<<<<<, 	E 	E 	E<(<<==1D	Es   !% AAANFc                 h   t          | |           || _        |                     |          | _        |	p| j        | _        || _        || _        || _        || _        || _	        || _
        || _        |
| _        | j        r|st          d| j         d          |st          d| j         d          	 |                     |                     | j                             n2# t          $ r%}t          d| j         d| j                   |d }~ww xY w| | j        | j        <   | | j        | j        <   d S )Nz$CmfDeferredJobWrapper: periodic job(z ) must be with option only_once.z!) must be with option system_job.zinvalid job configuration(z): )r   r   r   r   descriptionpriority
system_job	only_onceonly_once_argssoft_time_limit	countdownr   show_bg_progressbarr"   r$   r   periodicregistry)r   r   r'   r)   r*   r+   r(   r,   r   r&   r-   r#   s               r   __init__zCmfDeferredJobWrapper.__init__2   sf    	tT"""	&&t,,	&3$) $",." #6 = 
	, u !s	!s!s!sttt v !t	!t!t!tuuud&&t':':4='I'IJJJJ d d d ![di![![DM![![\\bccd'+DM$)$#'di   s   4-C" "
D, DDc                      | j         |i |S Nr   r   argskwargss      r   __call__zCmfDeferredJobWrapper.__call__O   s    ty$)&)))r   c                 ,    |g }|i } | j         |i |S r2   r   r3   s      r   applyzCmfDeferredJobWrapper.applyR   s-    <D>Fty$)&)))r   c                 0    |                      ||          S )N)r4   r5   )apply_asyncr3   s      r   delayzCmfDeferredJobWrapper.delayY   s    T&999r   c                 r   ddl m}m} dd l}|g }i || j        }|| j        }|| j        }|| j        }|| j        }|s| j	        |j
        v rd}|                    | j	        | j        ||          }|| j        |_        |	s|j        j        }	|s|	|_        || j        }|r)t#                      t%          j        |          z   |_        | j        rVt+          | j	                  }d}| j        ]|r-t/          d| j	         d|           f |t+          |          z  }r+|t+          fd	t1                    D                       z  }n"| j        D ]}|t+          |                   z  }|r>|d
t3          j        |                                                                          z   z  }||_        |j                            |ddg          }|rR|j        j         !                    |j        |j	        dd           |"                    d| j	         d           d}|j        rJ|j        r|j        |j        k     r3|"                    d| j	         d|j                    |j        |_        d}|j        s`|r^d|_        d}|j        #                    t+          |j                             ddl m$}  |ddt+          |j                  i|	g           |rJ|j        j%        &                                5  |'                                 d d d            n# 1 swxY w Y   d S |
|j(        }
||j)        }||j*        }|| j        | j        |j        j+        | j        ||j        j+        ||
|||dd|_,        |j        j%        &                                5  |'                                 d d d            n# 1 swxY w Y   |rU|j        #                    t+          |j                             ddl m$}  |ddt+          |j                  i|	g           t+          |j                  S )Nr   modelsgT)r   r)   r&   r-   seconds zJCmfDeferredJobWrapper.apply_async: !!! Warning! Use args + only_once! job=z, args=c                 "    i | ]}||         S r   r   ).0kr5   s     r   
<dictcomp>z5CmfDeferredJobWrapper.apply_async.<locals>.<dictcomp>   s    $J$J$JaQq	$J$J$Jr   __open*)only_once_keystatusfieldsskip_sched_only_once   zCmfDeferredJobWrapper(z): already queuedFz): reschedule existent job )cmf_emit_eventz deferred-job-show_bg_progressbarjob_id)event_persons)r)   r*   	person_idr(   r+   r'   r,   
admin_modecomponent_idsession_tab_idr-   )r4   r5   options)-cmf.includer>   r?   
cmf.fieldsr+   r(   r'   r&   r-   r   deferred_force_show_progressbarCmfDeferredJobr)   current_personidrR   r,   cmf_nowdatetime	timedeltaplan_start_datetimestrr*   printsortedhashlibmd5encode	hexdigestrJ   getutilmetricswrite_deferred_job_counterdebugappendrO   cmfutildisable_aclsaveacl_admin_moderT   rU   valueparams_json)r   r4   r5   r'   r+   r(   r,   r&   r-   g_current_user_idg_acl_admin_modeg_component_idg_session_tab_idr>   r?   cmfjobrJ   args_strarg_keyexistent_job	need_saverO   s     `                    r   r:   z!CmfDeferredJobWrapper.apply_async\   s    	*)))))))<D>F""2OJ}H*K&"&":" 	'tyA4U'U'U"&##dn+cv $ x x=CL  	4 ! 0 3 	.-CM I 	X&-ii(2DY2W2W2W&WC#> 1		NNMH"*
  *O6:iO OHLO OP P Q Q D		)H L$J$J$J$J6&>>$J$J$J K KKH  $2 5 5GF7O 4 44HH SHOO4E4E(F(F(P(P(R(R!RR -C "044=Y_ilhm4nnL  ;;CL#(TjlmnnnMMMMNNN!	3 %!$!8%<?<SVbVv<v<vGGlTYll[g[jllmmm7:7NL4 $I#7 M<O M7;L4 $I)00\_1E1EFFF::::::"N#FSVWcWfSgSgHh  zK  yL  M  M  M  M ,)5577 , ,$))+++, , , , , , , , , , , , , , ,  # /!^N# / !^"&"5 ]0"o#2L.&. ."2': 
 
$ X))++ 	 	HHJJJ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  	|!((SV555222222N>3sv;;@Whygz{{{{36{{s$   L""L&)L&"OO
O)	NFNNFNNNF)NN)NNNNNNNNNNNN)r
   r	   __qualname__r/   r.   staticmethodr   r   r   r$   r0   r6   r8   r;   r:   r   r   r   r   r      s        HH4 4 \4> > >   \
 E E \E HL_c %( ( ( (:* * ** * * *: : : sw:>imB B B B B Br   r   c                      fd}d}| r/t          |           dk    r	| d         }nt          d|  d          |r ||          S |S )za
    job decorator:
        return object like celery task with __call__, delay, apply_async
    c                     t          | fi S r2   )r   )func_r5   s    r   wrapperz!cmf_deferred_job.<locals>.wrapper   s    $U55f555r   NrN   r   z+cmf_deferred_job takes exactly 1 argument (z given)len	TypeError)r4   r5   r   r   s    `  r   cmf_deferred_jobr      s{    
6 6 6 6 6 D Xt99>>7DDV$VVVWWW wt}}r   c                  P    dd l } | j        j                                        j        S )Nr   )rX   rL   CmfDateTimenowrr   )rx   s    r   r]   r]      s(    :!%%''--r   c                    t          j                    }| d         }t          j        | d         | d         |d|           |                                }|                                 |dd         dk    r
|dd         }|S )zfrom logging Formatter class   r   rN   N
)ioStringIO	tracebackprint_exceptiongetvalueclose)eisiotbss       r   format_exceptionr      sv    
+--C	AB beRUBc:::AIIKKKv~~crcFHr   c                   <     e Zd Zed             Zd fd	Zd Z xZS )JobWorkerGreenletc                  ^    t           j                             t           j        j                  S r2   )r^   r   timezoneutcr   r   r   r   zJobWorkerGreenlet.now  s     $$X%6%:;;;r   Nc                 V    || _         t                                                       d S r2   )	worker_idsuperr0   )r   r   r   s     r   r0   zJobWorkerGreenlet.__init__  s&    "r   c                 0    | j         j         d| j         dS )N(r   )r   r
   r   r   s    r   r   zJobWorkerGreenlet.__repr__  s     .)==DN====r   r2   )r
   r	   r~   r   r   r0   r   __classcell__r   s   @r   r   r     se        < < \<     > > > > > > >r   r   c                   V     e Zd ZdZddef fdZd Zd Zd ZddZ	d	 Z
d
 Zd Z xZS )QueueProcessorzJobQueueProcessor:get:lockNr'   c                 H    || _          t                      j        di | d S )Nr   )r'   r   r0   )r   r'   r5   r   s      r   r0   zQueueProcessor.__init__  s-     ""6"""""r   c                    ddl m}m}m} dd l}|j                                        5  	 |j        j        	                    | j
        dd          5  d }| j        
dd| j        g}g dd	d| j        gg}|r|                    |           |j                            |
          D ]8}|                    d           t#          j        dd| d| j         d           9g ddg dddt'                      ggg}	|r|	                    |           |j                            g dg dg dgddg          }
|
r'|	                    dg dddd |
D             gg           |j                            |	dgd|          }|rbd|_        |j                                         | j        |_        |                    d            |                                 	 d d d            ng	 d d d            n# 1 swxY w Y    |             | j        st7          j        d!           n't7          j        t;          d!| j                             	 d d d            n# 1 swxY w Y   |S )"Nr   )r>   socketio
cmf_commitT
   timeoutblocking_timeoutr'   =rK   r   in_progressr   filterz+Job in_progress but queue processor is not.z%sz!!! Not running job , priority=z, mark as dead!)rK   r   rH   OR)r`   r   Nr`   z<=)r)   r   T)rJ   z!=Nz--rJ   )r   rL   )rJ   r   NNOT INc                     g | ]	}|j         
S r   )rJ   )rD   job_s     r   
<listcomp>z0QueueProcessor._job_get_next.<locals>.<listcomp>J  s    8n8n8nPT9K8n8n8nr   cmf_created_at)r   order_by
for_updaterL   r   z	job startr   )rW   r>   r   r   cmf.appappcmf_contextri   rn   CmfLock_GET_LOCK_KEYr'   r   rm   rZ   liston_errorloggingerrorr]   slistrh   rK   start_datetimeset_now	error_logrp   timesleepmax)r   rL   r>   r   r   rx   priority_filterstuck_filter	stuck_job
job_filteronly_once_in_progressnext_jobs               r   _job_get_nextzQueueProcessor._job_get_next  s   <<<<<<<<<<W  "" :	2 :	292!))$*<b[])^^ 1 1"&=,'13&FO 322 #t~6  # 9 ''888!'!6!;!;<!;!P!P u uI&&'TUUUM$(sy(s(sUYUb(s(s(stttt ,++:::.gii@
 # 7%%o666(.(=(C(C4446S6S6SUsUsUst /2 )D )4 )4% ) %% 888,h8n8nXm8n8n8no   "044%1A0Bt! 5 # #  &3HO+33555)-H&&&|444MMOOOc1 1 1 1 1 1 1VW1 1 1 1 1 1 1 1 1 1 1 1 1 1 1d JLLL= 2
1 
3q$-00111s921:	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2 :	2z s=   )IFG/I#I/G3	3I6G3	7AIIIc           
         ddl m} dd l}|j                                        5  |j        j                            |j        |j	        dd           |j        j        
                    |j        |j	        dt          t          j                                        |j        z
                                  dz                       |                    |           d d d            d S # 1 swxY w Y   d S )Nr   )r   successrN   duration  )rW   r   r   r   r   ri   rj   rk   r'   r   write_deferred_job_timingsintr^   r   r   total_seconds
on_success)r   ry   resultr   rx   s        r   _job_set_successzQueueProcessor._job_set_successb  s9   ((((((W  "" 	# 	#H77chPY[\]]]H77chPZ8;X=N=R=R=T=TWZWi=i<x<x<z<z  ~B  =B  9C  9CD D DNN6"""		# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	#s   B4C%%C),C)c           
      2   dd l }|j                                        5  |j        j                            |j        |j        dd           |j        j                            |j        |j        dt          t          j
                                        |j        z
                                  dz                       d}t          j                    }|d         rt!          |          }|                    ||           d d d            d S # 1 swxY w Y   d S )Nr   r   rN   r   r   zError?)r   r   r   ri   rj   rk   r'   r   r   r   r^   r   r   r   sysexc_infor   r   )r   ry   r   rx   
error_textr   s         r   _job_set_failzQueueProcessor._job_set_failn  sT   W  "" 		/ 		/H77chPWYZ[[[H77chPZ8;X=N=R=R=T=TWZWi=i<x<x<z<z  ~B  =B  9C  9CD D D "J|~~H{ 8-h77
LLX...		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/ 		/s   C!DDDc                 \    ddl mm dd lfd}t	          j        ||          S )Nr   r=   c            
      "   t           j        j                 } j                                        5  j        j                            j        j        dd           j	        dk    r,j        j                            j        j        dd           j
        }|sj        }t          j                                        |z
                                  }j        j                            j        j        dt!          |dz                       j                                       j        r#j        j        j        k    rj                            j                            j        j        j        j                             j                                         j        d                             d	d
          rj                                         j        d                             dd
          rj        d         d         _        j        d                             dd
          rj        d         d         _          | j        d         i j        d         cd d d            S # 1 swxY w Y   d S )NrunrN   r   	run_retrystart_latencyr   rL   rV   rS   FrT   rU   r4   r5   )!r   r/   r   r   r   ri   rj   rk   r'   retry_countr`   r   r^   r   r   r   r   rZ   set_current_deferred_jobrR   system_personr\   set_current_person	CmfPersonrh   APPcurrent_person_fieldsCmfAccessListsetup_contextrs   activate_admin_moderT   rU   )job_funcrun_latency_planrun_latency_secrx   r?   ry   r>   s      r   runnerz'QueueProcessor._job_run.<locals>.runner  s   ,5ch?H$$&& W W ;;CL#(TY[\]]]?Q&&H$??chXcefggg#&#: ' :'*'9$#+#4#8#8#:#:=M#M"\"\"^"^ ;;CL#(Tc<?RV@V<W<WY Y Y %>>sCCC= 
XS]ao6H%H%HG..(,,S]37;Cd,eeg g g(66888y155lEJJ C,@@BBBy155neLL T),)CN)Sy1556FNN X+.?9+EFV+W(x!8VCOH<UVV5W W W W W W W W W W W W W W W W W Ws   IJJJ)rW   r>   r?   r   geventwith_timeout)r   ry   r+   r   rx   r?   r>   s    `  @@@r   _job_runzQueueProcessor._job_run~  sl    ))))))))	W 	W 	W 	W 	W 	W 	W 	W< "?F;;;r   c           	         |                      dg          }|s.t          j        j                            | j        ddd           d S |j        d                             d          }	 |                     ||          }| 	                    ||           d S # t          t          t          j        f$ rP}t          j        d|j         d	|j         d
|            |                     |t'          |                      d }~wt(          t          j        f$ rS}t          j        d|j         d	|j         d           |                     |t'          |                     Y d }~d S d }~ww xY w)NrI   r   idlerN   rV   r+   )r+   zjob_runner(z, z) interrupt by ) error)r   rx   ri   rj   rk   r'   rs   rh   r   r   
SystemExitKeyboardInterruptr   GreenletExitr   	exceptionr\   r   r   ra   	ExceptionTimeout)r   ry   r+   r   r#   s        r   
_run_cyclezQueueProcessor._run_cycle  s      .. 	H77vvWXYYYF/)4889JKK	/]]3]HHF !!#v..... -v/BC 	 	 	RCFRRchRRqRRSSSsCFF+++6>* 	, 	, 	,GCFGGchGGGHHHsCFF+++++++++	,s&   )B E+4AC??E+AE&&E+c                    	 	 |                                   t          j        d           nW# t          t          t
          j        f$ r  t          $ r. t          j	        |  d           t          j        d           Y nw xY w)zmain() for GreenletTrN   z cycle error.   )
r  r   r   r   r   r   r   r  r   r   r   s    r   _runzQueueProcessor._run  s    		!!!
1 163FG      !T"8"8"8999
2		s   (, AB ?B c                 @    | j         j         d| j         d| j         dS )Nr   r   r   )r   r
   r   r'   r   s    r   r   zQueueProcessor.__repr__  s*    .)WWDNWWt}WWWWr   r2   )r
   r	   r~   r   r   r0   r   r   r   r   r  r  r   r   r   s   @r   r   r     s        0M# # # # # # # #A A AF# # #/ / / "< "< "< "<H/ / /,  X X X X X X Xr   r   c                       e Zd ZdZd ZdS )QueueCleanerzJobQueueCleaner:lockc           
      `   dd l }ddlm}m} 	 	 |j        j                            | j        dd          5  |j        	                                5  t                      }|t          j        d          z
  }|t          j        d	          z
  }|j                            d
ddg dgdd|ggg ddd|ggg           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   n-# |$ r Y n&t          $ r t!          j        |  d           Y nw xY wt%          j        d           ")Nr   CmfGetLockErrorr>   T,  r      )days   )hoursr   rK   IN)failcanceldeadend_datetime<)rK   r   r   r    error)r   rW   r  r>   ri   rn   r   	_LOCK_KEYr   r   r]   r^   r_   rZ   bulk_deleter  r   r   r   r   )r   rx   r  r>   r   max_end_datetimemax_success_end_datetimes          r   r  zQueueCleaner._run  s   77777777	3X%--dnc\]-^^ 
 
`c`g`s`s`u`u 
 
!))C'*X-?Q-G-G-G'G$/2X5Gb5Q5Q5Q/Q,)55 &.H.H.HIN\_aqKrs777.#Og9hi  6   
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 #    3 3 3!T///222223JsOOO%	s_   'C- C!A.C
>C!
C	C!C	C!C- !C%%C- (C%)C- -D4 DDNr
   r	   r~   r  r  r   r   r   r	  r	    s(        &I    r   r	  c                       e Zd ZdZd ZdS )QueueWatcherzJobQueueWatcher:lockc           
         dd l }ddlm}m} |j        j        j        j        }	 	 |j        j	        
                    | j        dd          5  |j                                        5  |                    d          }|r't          j                    t          |          z
  dk     r,t          j        d           	 d d d            d d d            |                    d	          }t%          |          D ]}|                    d
|                                           \|                    d	|           |                    |           t-          d|                                 dt.          j                   t3                      }|t5          j        d          z
  }	|j                            dgddd |D             gg ddd|	gg          D ]}
|
                    d           	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   n-# |$ r Y n&t<          $ r t?          j         |  d           Y nw xY wt          j        d           =)Nr   r  Tr  r   redis_idr  r   deferred_job_worker:workersdeferred_job_worker:heartbeat:zQueueWatcher: worker z seems dead!filer@   rI   r   r   c                 6    g | ]}|                                 S r   )decode)rD   w_ids     r   r   z%QueueWatcher._run.<locals>.<listcomp>  s     8^8^8^48^8^8^r   r   r   r  )rL   r   zJob Worker is deadr  <   )!r   rW   r  r>   r   r   REDIS_DBredisri   rn   r   r  r   rh   r   floatr   smembersr   r'  sremremoverb   r   stderrr]   r^   r_   rZ   r   r  r   r   )r   rx   r  r>   redis_dbr!  
db_workersr   r   max_start_datery   s              r   r  zQueueWatcher._run  sU   77777777"%'+"6"< 	3X%--dnc\]-^^ ; ;`c`g`s`s`u`u ; ;'||J77H# !ty{{U8__'Dr'I'I
2 ; ; ; ; ; ; ; ; ; ; ; ; ; "*!2!23Q!R!RJ%)*%5%5 m m	#<<(]IYIYI[I[(](]^^f$MM*H)TTT&--i888!"Z):J:J:L:L"Z"Z"Zadakllll "))C%(8+=c+J+J+J%JN%499$'5!,h8^8^S]8^8^8^ _ > > >!13 G$  :     ; ; %9::::;#; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;2 #    3 3 3!T///222223JrNNNA 	ss   'H H&AG;:HH DG;/H;G?	?HG?	HH HH HH I% IINr  r   r   r   r  r    s(        &I& & & & &r   r  c                   @     e Zd ZdZ edd          Z fdZd Z xZS )JobScheduleru   
    - защита от параллельного запуска
    - "размазывание нагрузки" во времени.
    PeriodicDatazjob croniterc                 H     t                      j        |i | g | _        d S r2   )r   r0   r.   )r   r4   r5   r   s      r   r0   zJobScheduler.__init__  s*    $)&)))r   c           
      `   dd l }ddlm} |j        j        j        j        }d}t          j                            t          j	        j
                                                  j        }t          j                                        D ]e}|j        }|j        rE|j        |j        v r7|j        |j                 }t'          |  d|j         d|j         d|            n"t'          |  d|j         d|j         d           |st'          |  d|j         d           t                              |          }t                              |           t-          j        ||j        	          }||_        |                                 |r"t3          ||                                          n|                                }| j                            |                     ||                     g	 |r|t;          j                    z
  }	nt'          d           d}	|	dk    rt'          d|	dd           d}	t;          j        |	           	 t;          j                    }
d}| j        D ]\  }}|                                }|
|k    r|                                 |                    d|j         d| | j         dd
          rv	 |j        !                                5  |"                                 d d d            n# 1 swxY w Y   n/# tF          $ r" tI          j%        |  d|j         d           Y nw xY w|r"t3          ||                                          n|                                }n;# tF          $ r. tI          j%        |  d           t;          j        d           Y nw xY w)Nr   configz: config: job=z, app="z ", config.JOB_SCHEDULE_OVERRIDE="z: config: job z
 disabled!r   Tz1JobScheduler: warning next_ts == 0, nothing to dor   z)JobScheduler: warning too low sleep time z.3fr      zJobScheduler:lock::)exnxz schedule job(r   r  )&r   rW   r:  r   r   r*  r+  r^   r   r   r   
astimezonetzinfor   r.   valuesr   JOB_SCHEDULE_OVERRIDEr   rb   r   r$   r   APP_FQDNget_nextminget_currentrm   r6  r   r   setr   r   r:   r  r   r   )r   rx   r:  r1  next_tslocal_tzinfory   r   	cron_itersleep_tsnow_tscurr_tss               r   r  zJobScheduler._run  s5   &&&&&&"%'+"6"<(,,X->-BCCNNPPW(188:: 	D 	DC|H+ OF<X0X0X!7AvvSXvvclvvltvvwwwwMMSXMMclMMMNNN AASXAAABBB,;;HEEH!33H=== )(FOLLLI+I   ?Fcc'9#8#8#:#:;;;ILaLaLcLcGM  !2!23	!B!BCCCC	 "TY[[0JKKK1}}Q(QQQQRRRJx   &*m l lNC'3355G((!**,,,#<<(QSX(Q(Q(Q(QSWSafhmq<rr \\%(W%8%8%:%: !6 !6$'OO$5$5$5!6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6 !6#, \ \ \ ' 1T2Z2Z2Z2Z2Z [ [ [ [ [\GNkc'9+@+@+B+BCCCT]TiTiTkTkGGl    !T///222
27	s[   	A9M3 L	K=1L	=L	L	L	L	M3 	)L52M3 4L55=M3 35N+*N+)	r
   r	   r~   __doc__r   r6  r0   r  r   r   s   @r   r5  r5    sd          :nn==L    8 8 8 8 8 8 8r   r5  c                       e Zd ZdZdZd ZdS )RedisCleaneruD   Раз в сутки сбрасываем редис БД. В 02-50.zRedisCleaner:lockc                    dd l }ddlm}m} |j        j        j        j        }	 t          j	        d           	 t          j
                                        }|j        dk    s|j        dk     rK|j        j                            | j        dd          5  |                    d          }|r't          j                    t'          |          z
  d	k     r	 d d d            t)          d
t*          j                   |                                 d d d            n# 1 swxY w Y   nB# t          j        j        $ r Y n,|$ r Y n&t4          $ r t7          j        |  d           Y nw xY wH)Nr   )r  	CMF_CACHETr  r   2   r   r!  i`T  z%RedisCleaner: run CMF_CACHE.flushdb()r$  r  )r   rW   r  rS  r   r   r*  r+  r   r   r^   r   hourminuteri   rn   r   r  rh   r,  rb   r   r0  flushdb
exceptionsLockNotOwnedErrorr  r   r   )r   rx   r  rS  r1  r   r!  s          r   r  zRedisCleaner._run\  s   ::::::::"%'+"6"<	3JsOOO3'++--8q==CJOOX%--dnc\]-^^ ( ('||J77H# !ty{{U8__'Dx'O'O ( ( ( ( ( ( ( BTTTT%%'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( #5   "    3 3 3!T///222223%	3sT   4D( .'D( A DD( !/DD( D  D( #D $D( (E'>E' E'&E'N)r
   r	   r~   rO  r  r  r   r   r   rQ  rQ  X  s.        NN#I3 3 3 3 3r   rQ  c                   f     e Zd Z fdZd ZdefdZddedefdZ	defdZ
dded
efdZd Z xZS )RedisMonitorc                     || _         |j        | _        || _        |                                  t	          j        d          | _         t                      j        di | d S )Nzredis-monitorr   )	r1  redis_instance_nameredis_settings_manager_RedisMonitor__init_redisr   	getLoggerloggerr   r0   )r   r1  r^  r5   r   s       r   r0   zRedisMonitor.__init__y  sd     #+#? &<#'88""6"""""r   c                 &   ddl m} t          |d          rm|j        | j                 d         dk    rQ|j        | j                                                 }|                    d           t          j        di || _        d S t          d          )Nr   r9  cache_settingstyper+  uJ   В конфиге не содержится конфигурации Redisr   )
rW   r:  hasattrrc  r]  copypopr+  Redisr"   )r   r:  cfgs      r   __init_rediszRedisMonitor.__init_redis  s    &&&&&& 6+,, 	k1FtG_1`ag1hls1s1s'(@AFFHHCGGFOOO++s++DJJJijjjr   returnc                 2   ddl m} t          |j                                                  }t          |          dk    r|d         S 	 |                    | j                  }||dz   t          |          z           }n# t          $ r |d         }Y nw xY w|S )u!  
        Возвращает следующую redis конфигурацию из config.cache_settings.
        В случае, если текущий инстанс последний в списке, то начинает обходить
        список заново
        r   r9  rN   )	rW   r:  r   rc  keysr   indexr]  r"   )r   r:  rm  idxnext_instances        r   _get_new_instancezRedisMonitor._get_new_instance  s     	'&&&&&F)..0011t99>>7N	$**T566C #'SYY!67MM 	$ 	$ 	$ GMMM	$ s   	5A? ?BB   intervalc                     	 |                                  | _        | j                            d| j                   |                                  |                                 }|rdS t          j        |           z)NTu=   Получена новая конфигурация Redis %s)rq  r]  ra  warningr_  check_redis_connectionr   r   )r   rs  
is_successs      r   _check_new_instancez RedisMonitor._check_new_instance  s}    	#'+'='='?'?D$K _aeayzzz4466J tL"""	#r   c                 |   | j         j        j        }|                    d          p|                    d          }	 | j                                          dS # t           j        $ r%}| j                            d|           Y d }~n5d }~wt          $ r%}| j                            d|           Y d }~nd }~ww xY wdS )NhostpathTuA   Ошибка подключения к Redis по адресу %su=   Системная ошибка с Redis по адресу %sF)	r+  connection_poolconnection_kwargsrh   pingConnectionErrorra  ru  r  )r   r}  current_hostr#   s       r   rv  z#RedisMonitor.check_redis_connection  s     J6H(,,V44U8I8M8Mf8U8U	oJOO4$ 	s 	s 	sK ceqrrrrrrrr 	o 	o 	oK _amnnnnnnnn	o us#   A B9'BB9B44B9r<  max_failure_countc                    d}	 |                                  }|rd}n|dz  }| j                            d|           ||k    ry| j                            d|           |                                 }|rH| j                            d| j                   | j                            | j        d           t          t          j	        |           )Nr   TrN   u.   Redis недоступен (%d попытка)u3   Redis недоступен %d раза подрядuA   Записываем новую конфигурацию Redis %s)
with_flush)
rv  ra  ru  rx  r]  r^  
save_to_dbr   r   r   )r   rs  r  failure_countrw  is_new_successs         r   monitor_rediszRedisMonitor.monitor_redis  s    	#4466J ) !"##$TVcddd $555K''(]_lmmm%)%=%=%?%?N% )++,oqu  rJ  K  K  K3>>t?Wdh>iii((L"""#	#r   c                 .    |                                   d S r2   )r  r   s    r   r  zRedisMonitor._run  s    r   )rr  )rr  r<  )r
   r	   r~   r0   r_  ra   rq  r   boolrx  rv  r  r  r   r   s   @r   r[  r[  x  s        # # # # #
k 
k 
k3    (	# 	#C 	# 	# 	# 	# 	#    # #c ## # # # #*      r   r[  Fc                 x   ddl }ddlm} |j        j        }t          j                     dt          j                     }|j	        r5t          |j        j        j        ||          }|                                 |                                 |j        j        j        j        }| sd|j        j        _        g }| r%|                    t'          d|                     n6t)          d          D ]&}|                    t'          ||                     '|                    t+          |	                     |                    t-          |	                     |                    t/          |	                     |                    t1          |	                     |D ]}	|	                                 	 	 	 |                    d
| dd           |                    d|           n.# t6          $ r!}
t9          j        d|
            Y d}
~
nd}
~
ww xY wt=          j        |d          5 }|D ]'}	tA          d|	 dtB          j"                   tF          	 ddd           n# 1 swxY w Y   # tF          tH          t<          j%        f$ r)}
tA          d|
 dtB          j"                   Y d}
~
nd}
~
ww xY w	 t=          j&        |d           n0# t<          j'        $ r tA          dtB          j"                   Y nw xY w|D ]}	|	r tA          d|	 dtB          j"                   $|	(                                r tA          d|	 dtB          j"                   X	 |	)                                 n# tF          tH          t<          j%        t6          f$ r t9          j        d|	 d           Y w xY wdS )u!  
    Функция - обработчик очередей задач.
    На каждый приоритет(очередь) запускаем отдельный поток - обработчик.
    Сервисный поток для очистки очереди от устаревших задач.
    Сервисный поток для контроля погибших задач(чей воркер погиб).
    Сервисный поток Beat, для планирования периодических задач.
    r   Nr9  r=  )r1  r^  r   T)r'   r      )r   r#  -   )r>  r"  z$deferred_job_worker: redis error!!! r   )r   z%deferred_job_worker(): system thread(z) exited!!! Stop work.r$  zdeferred_job_worker(): stop by z. Try kill all threads...z(deferred_job_worker(): killall timeout!.zdeferred_job_worker(): Thread z is not stopped!!!z stops normally!z error.)*r   rW   r:  r   REDIS_SETTINGS_MANAGERplatformnodeuuiduuid1CACHE_REDIS_FAILOWERr[  r   r*  start
init_redisr+  rS  job_daemon_moderm   r   ranger	  r  r5  rQ  rH  saddr  r   r   r   iwaitrb   r   r0  r   r   r   killallr  
successfulrh   )single_queuerx   r:  r^  r   redis_monitorr1  	greenletsr'   glr#   gl_iters               r   deferred_job_workerr    s    NNN"""""" W;=??33TZ\\33I" $W[)#9
 
 

 	%%'''!gk28H 1,0)I UKKKLLLLb		 	U 	UH^XSSSTTTT\I666777\I666777\I666777\I666777  




_	%NIiII3SUVVV<iHHHH  N N N!"L"L"LMMMMMMMMN i444 %! % %B\"\\\cfcmnnnn$$%% % % % % % % % % % % % % % %	% )6+>? _ _ _LLLLSVS]^^^^^^^^^_Ly"-----> L L L9
KKKKKKL  L L 	I2IIIPSPZ[[[[==?? 	G2GGGcjYYYY	LFFHHHH-v/BIN 	L 	L 	LJrJJJKKKKK	LL Ls   >I:  1G2 1I: 2
H<HI: HI: 6+I."I: .I22I: 5I26I: :J?J::J?K *LL%M:::N76N7)F)"r^   rd   r   ossignalr   r   r   r   r  r  collectionsr   	functoolsr   r   r   r+  cmf.util.cmfutilrx   cmf.util.metricsr   r   r]   r   Greenletr   r   r	  r  r5  rQ  r[  r  r   r   r   <module>r     s     				 				       



    " " " " " " $ $ $ $ $ $           G G G G G G G GT  *. . .  
> 
> 
> 
> 
> 
> 
> 
>nX nX nX nX nX& nX nX nXb    $   :) ) ) ) )$ ) ) )XC C C C C$ C C CL3 3 3 3 3$ 3 3 3@W W W W W$ W W WtOL OL OL OL OL OLr   