U
    Įwh                     @   sJ   d dl Z d dlZd dlZd dlZd dlT d dlmZ G dd dejZdS )    N)*)cmf_backbone_instancec                       s   e Zd Zejjdg ZejG dd dZe	e
dddddd Zd	d
 Zdd ZedZed ddZdd Zdd Z fddZed! fdd	Z fddZd"ddZd#ddZ  ZS )$CmfBackboneInstance
force_syncc                   @   s   e Zd ZU dZeed< dZeed< dZeed< dZ	eed< dZ
eed< dZeed< dZeed< dZeed	< dZeed
< dZeed< dZeed< dZeed< dd ZdS )zCmfBackboneInstance.PeerNbackbone_instancedispositiontypeurltoken
ssl_verifyid
project_idproject_codeproject_nameis_src_peertask_filter_ubqlc                 C   s  | j r| jdkr| jp"t| j j| _| jp6t| j j| _| jpNt| j j	 | _| j
d krjt| j j| _
| jpzt| j j| _| jpt| j j| _| jpt| j j| _| jpt| j j| _| jd kr| j jdk| _| jpt| j j| _n
| jdkr| jpt| j j| _| jpt| j j| _| jp8t| j j	 | _| j
d krVt| j j| _
| jpht| j j| _| jp~t| j j| _| jpt| j j| _| jpt| j j | _| jd kr| j jdk| _| jpt| j j!| _n| jrt"d| jd S )Na)abbib)bar   zInvalid peer disposition)#r   r   r   strpeer_a_typer	   
peer_a_urlr
   peer_a_tokenZdecryptr   boolpeer_a_ssl_verifyr   	peer_a_idr   peer_a_project_idr   peer_a_project_coder   peer_a_project_namer   	directionr   peer_a_task_filter_ubqlpeer_b_type
peer_b_urlpeer_b_tokenpeer_b_ssl_verify	peer_b_idpeer_b_project_idpeer_b_project_codepeer_b_project_namepeer_b_task_filter_ubql
ValueErrorself r/   2./modules/backbone/models/cmf_backbone_instance.py__post_init__   s:    


z&CmfBackboneInstance.Peer.__post_init__)__name__
__module____qualname__r   object__annotations__r   r   r   r	   r
   r   r   r   r   r   r   r   r   r1   r/   r/   r/   r0   Peer   s   
r7   Tz	@minutely)Z	only_onceZ
system_jobZschedulec                  C   s  t jjdddgdddddd	d
dddddddddddgdD ]} | jpJ| jj}| jr`| jj|k r`q:t }z|   d | _	W nV t
tfk
r    Y n> tk
r } z td|  t  t|| _	W 5 d }~X Y nX tt | | _| j  |   t  q:d S )Nstatus=runningr   r   r   r   r   r   r    r#   r$   r%   r'   r(   r)   r*   r!   sync_periodlast_sync_datetime)filterfieldszSync %s error)modelsr   listr;   defaultr<   ZagetimesyncZlast_sync_error
SystemExitKeyboardInterrupt	Exceptionlogging	exceptionZrollback_purge_eventr   intZlast_sync_durationZset_nowsaveZcommit_with_event)r   r;   Z
sync_starter/   r/   r0   sync_all=   sJ                 

zCmfBackboneInstance.sync_allc                 C   s   d | _ |   td d S )Nuu   Синхронизация запущена, она может выполняться длительное время)r<   rJ   	cmf_alertr-   r/   r/   r0   r   Y   s    zCmfBackboneInstance.force_syncc                 C   s   t jrtd|  d dS td|  d |   | jd| d}| jd| d}| jdkrn| j| ||d	 nX| jd
kr| j| ||d	 n<| jdkr| j| ||d	 | j| ||d	 nt	d|  dtd|  d dS )u   
        Можно тоже сделать джобом, тогда будет распараллеливание между инстансами.
        zCmfBackboneInstance.sync(z/):Skip sync due config.MODULE_BACKBONE_DISABLEDNz): sync startr   r   r   r   r   )r   src_peerdst_peerr   r   z): sync direction is not set.z): sync end)
configZMODULE_BACKBONE_DISABLEDrG   warninginfosetup_peersr7   r!   _sync_onewayr,   )r.   Zpeer_aZpeer_br/   r/   r0   rC   ^   s     	


zCmfBackboneInstance.syncz<img(?:[\s][^>]*)?[\s]data-attach-id="(?P<data_attach_id>CmfAttachment:[0-9a-f-]+)"(?:[\s][^>]*)?[\s]src="(?P<src>[^"]+)"(?:[\s][^>]*)?>Nc                    s  fdd}fdd}fdd}fdd}fd	d
}dPfdd	

fdd	fdd	fddfdd 
fdd}	t  i }
tjjdddgd|  }rg i }|D ]Jd |
kr2d |
 k r2tdj dj d   qd! tjjjd"d#d"d$d%d&gd'stjjjd"d#d"d$d%d&gd(}|rtjj|j	|j
|j|j|j|jd)	d* d+  r<d,d* krd* d, jksd-d* krd* d- jkrd* d,_d* d-_jdd. n d/krtd0j dj  d1 |}|std2j dj d   qd  |d < d* }td3j dj d  | ||d |
< stjjd* d,d* d-d4|d  < |	  qrg }zd5d6 | D }W n& tk
rr   d7d8 D }Y nX D ]}||d   }|d9}td:jj dj | |d  |d;}|| |d<kr|d=}|rx||d=< qx|d}|rx||d   |d>}|d-}j	|ksFj|ksFj|krH|_	|_|_td?jj dj j
 d@j dAj	 d@j dB   |d! dCkrH|d* d+ d1krH|d* dD }tjj|dEgdF}||j
}|d* dG dH }|dI rH|dI |dI< dJ|d! |d/dH|idKdL}tjjdMj|ggdN |d* d+ dOkrx|d! dCkrx|j
jj	j qx|| qdJS )Qu  
        TODO: синхронизация должна быть толерантна к ошибкам, что можем - исправляем, что не можем - игнорируем.
            но все отклонения от основной логики должны быть с подробностями в журнале/логах.
        while src_log_list = SrcPeer.log_list():
            dst_log_list = _sync_log_map(src_log_list)
            apply_list = DstPeer.log_apply(dst_log_list)
            SrcPeer.log_done(apply_list)

        ?:
        - удаление Dst
        - смена проекта в src/dst?
        - что делать с ошибками? Пропускать все последующие изменения по объектам с ошибками?
            Что делать со ссылками на объекты, которые не синхронизируются?
        c                      s(   t jjdjgtddgt ddS )NZlog_listr      )sliceskip_ids)peerargskwargs)r?   CmfBackbonePeer	call_peerr   dictr@   r/   )rX   rO   r/   r0   get_change_list   s      z9CmfBackboneInstance._sync_oneway.<locals>.get_change_listc                      s   t jjdj gdS )N	log_applyrY   rZ   r?   r\   r]   r   r/   )dst_change_listrP   r/   r0   apply_change_list   s
     z;CmfBackboneInstance._sync_oneway.<locals>.apply_change_listc                    s   t jjd  j| gdS )Nget_objra   rb   )obj_idrO   r/   r0   re      s
     z1CmfBackboneInstance._sync_oneway.<locals>.get_objc                    s   t jjd  j| gdS )Nlog_mark_donera   rb   )log_mark_listrg   r/   r0   rh      s
     z7CmfBackboneInstance._sync_oneway.<locals>.log_mark_donec                    s*   t jj| |d}t jj||| d d S )N)rY   )r?   r\   Zpeer_get_attachment_contentZpeer_put_attachment_content)Zsrc_attachment_idsrc_content_pathZdst_attachment_iddst_content_pathZcontent)rP   rO   r/   r0   transfer_attachment_content   s         zECmfBackboneInstance._sync_oneway.<locals>.transfer_attachment_contentTc                    sr   t jj j| dgd}|o"|jj}|sV jdkrVt jj j| dgd}|oT|jj}|rn|sntd j| |S )N
dst_obj_id)r   rO   
src_obj_idr>   r   rn   )r   rP   rm   r>   zMapping not found)r?   CmfBackboneObjMappinggetr   rm   jsonr!   CmfError)Zsrc_idrequiredmappingdst_id)r   rP   rO   r/   r0   map_id   s$        z0CmfBackboneInstance._sync_oneway.<locals>.map_idc                    s4   | s| S d| kr0 | d dd}|s(d S || d< | S )Nr   F)rs   r/   )	obj_valueru   )rv   r/   r0   map_field_obj_value   s    z=CmfBackboneInstance._sync_oneway.<locals>.map_field_obj_valuec                    s$    fdd}t j|| S )Nc              	      s  t jj dj| d ddgd}|rhtd j dj d | d |j t|j}t|j}n2t	d	 j dj d | d  d
}d
}| d d | 
d| 
    | | d | d| 
  | 
d| 
    | | d | d| 
  d   S )NCmfAttachmentZdata_attach_idrm   rk   r   	obj_modelrO   rn   r>   zTCmfBackboneInstance._sync_oneway(%s, %s): skip img patch %s for obj %s: map %s -> %s -> r   z\CmfBackboneInstance._sync_oneway(%s, %s): skip img patch %s for obj %s: map for %s not found r   src)r?   ro   rp   r   rG   rS   rm   r   rk   rR   startend)matchZattach_mappingru   Zdst_path)r   rP   itemrn   rO   r/   r0   
img_mapper   s>           
   rzWCmfBackboneInstance._sync_oneway.<locals>.map_text_inline_reference.<locals>.img_mapper)resub_ATTACHMENT_IMG_RE)Z
text_valuer   )r   clsrP   r   rn   rO   r/   r0   map_text_inline_reference   s    zCCmfBackboneInstance._sync_oneway.<locals>.map_text_inline_referencec                    s   t | }|d }|dkrD|dkrdkr|d r|d |d< nd|dkr^ |d |d< nJ|dkrdD ]<}||krjg }|| D ]} |}|r|| q|||< qj|S )Nr   valuetext)CmfTaskZ
CmfCommentr5   Zobject_list)r   ZappendedZremoved)copydeepcopyappend)Zsrc_changes
field_nameZdst_changesZ	data_typeZ	list_nameZdst_listrw   Z	dst_value)rx   r   r{   r/   r0   map_field_changes   s"    

z;CmfBackboneInstance._sync_oneway.<locals>.map_field_changesc                    sP    D ]F}|j dkr(|j| kr(|jj  S |j dkr|j| kr|jj  S qqd S )Nr   r   )r!   peer_a_fieldpeer_b_fieldr   )r   Zfm)fields_mappingsr/   r0   map_custom_field  s    z:CmfBackboneInstance._sync_oneway.<locals>.map_custom_fieldc                     s   i } t  | d}t d o$jj|d}d d}|rN|}||d< d di  D ].\}}|dr|}|sqb||| |< qb|S )uU   Модифицируем изменения для применения на dst peeractionr>   r   r   r{   rf   obj_changesr   	parent_idr>   Zcf_)r^   rm   rq   rp   items
startswith)Z
obj_fieldsr   resZsrc_parent_idZdst_parent_ifr   Zfield_changes)r   r   r   r   rv   obj_mappingr{   r/   r0   map_item  s&    


z2CmfBackboneInstance._sync_oneway.<locals>.map_itemr!   r   r   )r   r>   rf   ZdatetimezVCmfBackboneInstance._sync_oneway(%s, %s): skip record %s for obj %s, due full obj syncr|   r   r{   src_obj_coderm   dst_obj_coderj   rk   rz   )r   r{   rP   rm   r>   )	r   r{   rO   rn   rm   r   r   rj   rk   r   r   codeZcontent_path)Z	only_dataupdatezZCmfBackboneInstance._sync_oneway(%s, %s): mapping for %s not found. Request full obj sync.createz`CmfBackboneInstance._sync_oneway(%s, %s): get_obj return none, skip changes(%s) for new task(%s)zSCmfBackboneInstance._sync_oneway(%s, %s): replace changes %s with full obj data: %s)r   r{   rO   rn   r   rj   c                 S   s   i | ]}|d  |qS )r   r/   ).0
apply_datar/   r/   r0   
<dictcomp>  s      z4CmfBackboneInstance._sync_oneway.<locals>.<dictcomp>c                 S   s   g | ]}|d  ddqS )r   errorr   r8   r/   )r   
dst_changer/   r/   r0   
<listcomp>  s     z4CmfBackboneInstance._sync_oneway.<locals>.<listcomp>r8   z:CmfBackboneInstance._sync_one_way(%s, %s): apply result %sr   success
error_textZobj_codez<CmfBackboneInstance._sync_one_way(%s, %s): update mapping %sz (z) -> )ry   r   rn   )rm   r>   r>   r   r   Nr   r   r`   ra   delete)T)setr?   ZCmfBackboneFieldMappingr@   rG   rS   r   ro   rp   rm   rn   r   r   rk   rj   rJ   rR   debugr   rr   r   r\   r]   r   )r   r   rO   rP   r_   rd   re   rh   rl   r   Zfull_sync_datetimesZsrc_change_listZobj_mappingsZreverse_mappingZfull_obj_dataZ
_orig_itemri   Zapply_result_dictr   r   r8   Zlog_markr   rm   r   rk   Zdst_parent_idZparent_obj_mappingZ
src_parentZ
text_fieldZparent_obj_changesr/   )r   r   r   rc   rP   r   r   r   r   rx   rv   r   r   r{   rX   rn   rO   r0   rU      s    
%
         	          


        

 





 " 

 
   z CmfBackboneInstance._sync_onewayc                 C   s2   | j jr| j s.| jr | jr | js(d| _ nd| _ d S )Nstoppedr:   )r8   
is_changedr   r'   r!   r-   r/   r/   r0   _calc_status  s    z CmfBackboneInstance._calc_statusc                 C   s   | j jrL| j rLztjjt| j jd W n  tk
rJ   t	ddd Y nX | j
jr| j
rztjjt| j
jd W n  tk
r   t	ddd Y nX d S )N)r=   u>   Невалидный BQL-фильтр для проекта A!Tabortu>   Невалидный BQL-фильтр для проекта B!)r"   r   r?   r   countrq   loadsr   rF   rM   r+   r-   r/   r/   r0   _calc_filters  s    z!CmfBackboneInstance._calc_filtersc                    s   t   ddddg S )Nr!   r8   r   r'   )supersave_preload_fieldsr-   	__class__r/   r0   r     s       z'CmfBackboneInstance.save_preload_fieldsc                    s(   |p|}|p|}t  jf ||d|S )N)r   r(   )r   r   )r   Zpeer_a_projectZpeer_b_projectr   r(   r[   r   r/   r0   r     s    zCmfBackboneInstance.createc                    s:  | j jr| j jr| js,| jjr8| jjr8| jr8tddd | jrzd| _t	j
| jj| _| jp^d| _d| _d| _| jdd |   |   t jf |}| js6| jjr| jdkrt	j
j| jdd}nt	j
j| jdd}| j|_|  nL| jjr6| jd	krt	j
j| jdd}nt	j
j| jdd}| j|_|  |S )
Nu   Нельзя изменять проект в запущенной синхронизации, нужно создать новую.Tr   localr   F	full_sync)backbone_instance_idr   r   )r   r   oldr   r(   r'   rM   Zis_newr   r?   r\   Zget_peer_typer$   r   r#   r!   r   r&   rT   r   r   r   rJ   r"   rp   r   Zsrc_task_filter_bqlr+   )r.   r[   r   rY   r   r/   r0   rJ     sJ    


zCmfBackboneInstance.saveFc                 C   s   | j || d}tjjd|t| jj| jj| jjt	j
|j|j|jdd}|rh|jrhtjjd||d gd | j}|jdkr|d | _|d	 | _|d
 | _n6|jdkr|d | _|d	 | _|d
 | _ntd|j| jr|s|   d S )NrN   Z
setup_peer)r   Zbackbone_instance_codeZbackbone_instance_nameZbackbone_instance_domainZpeer_project_idr   Zsrc_task_filter_ubql)rY   r[   r   r   ra   r   r   r   r   zpeer_disposition must be a or b)r7   r?   r\   r]   r^   r   rq   r   namerQ   ZAPP_FQDNr   r   r   r   r   r   r   r    r'   r)   r*   r,   rJ   )r.   peer_dispositionr   Z	peer_datarY   Zwas_changedr/   r/   r0   _setup_peer.  s@        	
  






zCmfBackboneInstance._setup_peerc                 C   sd   |  ddddddddd	d
dddddddddg |sR| jd|d | jd|d n| j||d dS )uU   Создаём пиры, если их нет, и обновляем кеш поля.r!   r   r   r   r   r   r   r   r    r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r   r   r   N)Zload_fieldsr   )r.   r   r   r/   r/   r0   rT   Q  s2                  zCmfBackboneInstance.setup_peers)NNN)NNNN)F)NF)r2   r3   r4   r   r   Zapi_methodsdataclassesZ	dataclassr7   staticmethodZcmf_deferred_jobrL   r   rC   r   compiler   classmethodrU   r   r   r   r   rJ   r   rT   __classcell__r/   r/   r   r0   r   
   s2   -  `-
#r   )Zastr   r   rB   Zcmf.includeZmodules.backbone.fieldsr   r   r/   r/   r/   r0   <module>   s   