U
    d                    @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m	Z	 d dl
mZmZmZmZ d dlZd dlmZ d dlT d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlZejG dd dZejG dd dZ G dd dZ!dS )    N)cmf_context)Path)urlparse)datetime)DictUnionListOptional)commit_all_ds)*)CmfActiveEntity)ConfluenceApi)CmfPluginImportMixin)BeautifulSoup)ApiNotFoundErrorc                   @   sv   e Zd ZU dZejedZee	e
jf ed< ejedZee	eee	e	f  f ed< ejedZee	e
jf ed< dS )	TotalDatauJ    Данные используемые в процессе импорта default_factoryuserslink_documentsresult_documentsN)__name__
__module____qualname____doc__dataclassesfielddictr   r   strmodels	CmfPerson__annotations__r   r   r   CmfDocument r#   r#   "./modules/confluence/api_import.pyr      s   
(r   c                   @   s@   e Zd ZU dZejed< eed< ej	e
dZeeef ed< dS )DocDatauR    Данные используемые при создании документа Zdocumentdoc_idr   link_attachmentsN)r   r   r   r   r   r"   r!   r   r   r   r   r'   r   r#   r#   r#   r$   r%      s   

r%   c                   @   s  e Zd ZdZeejdddZdd Ze	ddd	Z
eeed
ddZe	dddZdd Zdd ZedddZedddZeeedddZdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Z ed/d0d1Z!d2d3 Z"d4d5 Z#d6d7 Z$ed8d9d:Z%d;d< Z&eed=d>d?Z'd@dA Z(edBdC Z)dDdE Z*dFdG Z+edHdIdJZ,dKdL Z-dMdN Z.dOdP Z/dQdR Z0dSdT Z1dUdV Z2eedWdXdYZ3dZd[ Z4d\d] Z5e6dd^d_Z7d`da Z8doedHdcddZ9eddedfZ:dgdh Z;dpeeedjdkdlZ<dmdn Z=diS )qConfluenceAPIImportu;   Класс загрузки объектов через API)
confluence
cmf_importc                 C   s   || _ || _t | _tjjdd}|j | _	d| _
d| _td}| jjj|_|| jjj |   d| _| j jjr~d| _d| j_d| j_t | _t | _d S )Nzsystem:defaultcoder   zhistory,body.export_view.webresource,body.storage,body.editor,body.view,body.editor2,body.styled_view,body.atlas_doc_format,body.dynamic,children.page,ancestors,body.plain,body.anonymous_export_view,body.wiki,history.ownedBy,history.lastUpdatedzatlassian.rest_clientZuserKeyZ	accountId)confluence_apir*   r   Z
total_datar   ZCmfSchemeWfgetZdefault_document_workflowloaddoc_workflow
has_errorsexpandloggingZ	getLoggerloggerZhandlersZsetLevellevel_prepare_multiprocessinguser_keyconncloudimported_object_countjson_object_countr   treeset_selected_objs)selfr)   r*   Z	scheme_wfr4   r#   r#   r$   __init__)   s$    

zConfluenceAPIImport.__init__c                 C   s(   dd l }dd l}|| _ || _tj| _d S )Nr   )	threadingqueueconfigZIMPORT_THREADSthreading_max_forks)r?   rA   rB   r#   r#   r$   r6   C   s    z,ConfluenceAPIImport._prepare_multiprocessing)returnc                 C   s2   dt tjjt tjjdddg ddd dgS )NZcurrent_user)emailname)rG   fieldsselectedObjects)rG   valueimport_type)r   gZcurrent_personrF   rG   r?   r#   r#   r$   calc_import_settingsJ   s    
z(ConfluenceAPIImport.calc_import_settings)settingsrE   c                 C   s   dd | D S )uO   Преобразуем список с настройками в словарьc                 S   s   i | ]}|d  |qS rG   r#   ).0sr#   r#   r$   
<dictcomp>S   s      z;ConfluenceAPIImport._normalize_settings.<locals>.<dictcomp>r#   )rO   r#   r#   r$   _normalize_settingsP   s    z'ConfluenceAPIImport._normalize_settingsc                 C   s   g S )uo    Для импорта из Confluence предварительный просмотр не требуется r#   rM   r#   r#   r$   previewU   s    zConfluenceAPIImport.previewc                 C   s  g g d}d }|  |d D ]}|d dkr|d d d D ]*}d}| || j }|r>|d | q>|d d	 d D ]}d}| jjjrd
| jjjj	 d
|d  d
}	nd
| jjjj	 d
|d  d
}	ddd|	 dg}
t
jj|
d}|rz|d | qz|d dkr|d d d D ]8}|dkr,d}| || j }|r|d | q|d d	 d D ]}|dkrvd}| jjjrd
| jjjj	 d
|d  d
}	nd
| jjjj	 d
|d  d
}	ddd|	 dg}
t
jj|
d}|rd|d | qdq|jr|r||_|jr"|jdd |j  dd|jgddd gg}
t
jj|
dD ]}|  qP|d rt
j|j| jd|d djdd |d rt
j|j| jd|d djdd t  d S )N)readwritepathZ	operationrV   restrictionsuserresultsZprivategroup::idrG   ext_idLIKE%filterupdatereadonlyrW   TZsave_importparent==r*   !=)rg   r*   Zaccess_levelZsubjects)_get_restrictions_get_personr7   appendr-   r8   r9   r*   pluginsource_hashr   CmfPersonGroupr.   import_originalperm_policy
is_changedsaveZperm_aclr/   ZCmfAccessRulelistdeletecommit_with_event)r?   docdoc_objZpermsrq   restrictionrZ   person
group_infor_   _filterr\   Zruler#   r#   r$   _process_restrictionsY   sx    

 

 

 
 z)ConfluenceAPIImport._process_restrictionsc           	      C   s<  | ddd  d }t|j d}t|dkrb|d dkrb|d dkrb|d	 d
krb|d }|S t|d	kr8|d dkr8|\}}}}z |dd}| jj||}W nN tk
r } z0| j	j
d| d| d| d|  W Y d S d }~X Y nX |s| j	j
d| d|  d S | j	j
t| |d }|S )N/r         wiki   Zspaces   Zpages   Zdisplay+ uC   Ошибка получения документа по pretty url : uo   Не смогли получить документ по pretty url (возможно не тот домен): r^   )splitr   rX   lenreplacer-   r8   Zget_page_by_title	Exceptionr*   r4   warninginfor   )	r?   urlr,   Zsplitted_url_spacetitleZfind_confl_docer#   r#   r$   _get_pageid_by_url   s6    

z&ConfluenceAPIImport._get_pageid_by_urlrP   c              	   C   s   | j  | d}| j  | d}| r8d S t|d6}| D ](}| j  jd7  _|t|d  qJW 5 Q R X t	
|| | j jdd t  d S )N.jsonz.json.dirtya+r   
TZ	only_data)r*   get_download_pathjoinpathexistsopenr;   rW   jsondumpsshutilmovers   rv   )r?   Zapi_funcrG   	file_pathZfile_path_tmpfrowr#   r#   r$   _simple_dump   s    
 z ConfluenceAPIImport._simple_dumpc              	   c   sF   | j  | d}t|d}|D ]}t|V  q&W 5 Q R X d S )Nr   r+)r*   r   r   r   r   loads)r?   rG   r   r   r   r#   r#   r$   _simple_get   s    zConfluenceAPIImport._simple_get)textattachments_dirobjc                 C   s  |sd S | j jd t|ddd D ]}d}|drBd}|j| }zPd|kr\W q*|dr~| j jd	|  W q*|d
st	j
| jj|}|| jjs| j jd|  W q*tj|dd dd }t|\}}	|s| j jd|  W q*tj|}||}
| j jjj d|d  }tjj||| j jjjd}|sxtj| j ||| j jjjd}|
|_
||_d|_|  |j   W q* t!k
r } z*| j j"  | j jd| d|  W 5 d }~X Y q*X q*d S )Nu/   Обработка ссылок в текстеlxmlc                 S   s   |  dp|  dS Nhrefsrchas_attrtagr#   r#   r$   <lambda>       z1ConfluenceAPIImport._dump_links.<locals>.<lambda>r   r   z/attachments/zmailto:u   Это почта: httpuH   Это внешняя ссылка, оставляем как есть: r~   r   ?r   u   Это не файл: r]   r^   r   
obj_ext_idrn   r*   r   r   rn   Fu0   Не удалось скачать ссылку r   )#r*   r4   r   r   find_allr   attrs
startswithr   osrX   joinr-   r   urllibparseunquoter   	mimetypes
guess_typer   r"   _get_filenamer   rm   rn   CmfImportDownloadr.   rG   
downloadedrs   dpcommitr   rollback)r?   r   r   r   r   keyr   	file_name	mime_typeencodingr   r_   download_jobr   r#   r#   r$   _dump_links   sZ    



 


zConfluenceAPIImport._dump_linksc              	   C   s   | j  d| d}| s| jj|dd}|sD| jj|dd}|sX| jj|dd}|rt|d}t	
|| W 5 Q R X n t|d}t	|}W 5 Q R X |S )Nr   r   
operationsr2   w+r)r*   r   r   r   r-   Zget_user_infoZget_user_info_by_keyZget_user_info_by_usernamer   r   dumpr/   )r?   user_idZ	user_file	user_infor   r#   r#   r$   _get_user_info   s    z"ConfluenceAPIImport._get_user_infoc                 C   s@   |   D ]2}|d|kr"|  S || j|kr|  S qd S )NZusername)
_get_usersr.   r7   )r?   r   r   r#   r#   r$   _get_user_info_from_dump	  s
    z,ConfluenceAPIImport._get_user_info_from_dumpc                 C   s0  t dd || _d}| jjd| d|  | j|d ||D ]}zzl| j rlW W  q| jjd|d  d	 |	t
|d
 }	|	 rW W `qF| jj|d
 | jd}|d d | jr| |d d | j  |d d}
|
r|
d dkr| |
| j  | |d d d | j  |	t
|d
 d }|jdd |		d}|	d}dd | jj|d
 dD }t|}t|dd}t|| W 5 Q R X |rtj| jjd|d
  }| jjjrtj| jjd|d
  }d}|	|}| jjjj d |d
  }tj j||| jjjjd!}|sftj | j||| jjjjd"}||_||_!d|_"|#  |j  | $|d# d$ d% || |	d&}| rt%| | jjd' t|d(}| j d)}| j&|d
 |D ]}|'t(|d*  | j j)d+7  _)| $|d# d, d% || |d drd| |d d | j  |d dr| |d d | j  |d dr| |d d d | j  qW 5 Q R X |	d-}| rt%| | jjd. t|d(Z}| j*|d
 D ]B}|'t(|d*  |d/ d0 d1 D ]}| || j  q6qW 5 Q R X t+,||	 |jdd |		d2}t
|	|d3< t|d"}t|| | j j)d+7  _)W 5 Q R X W nD   |d+7 }| jj-d4|d  d5|d
  d6 | jj.  Y nX W 5 | jj  X qF|/| W 5 Q R X d S )7NFZinit_views_and_dsr   u   Документы с u    до r   u   Документ 'r   'r^   r   history	createdByownedBytype	anonymouslastUpdatedby.dirtyTexist_okattachmentsattachments.jsonc                 S   s   g | ]}|qS r#   r#   rQ   r   r#   r#   r$   
<listcomp>.  s     z;ConfluenceAPIImport._dump_document_part.<locals>.<listcomp>)page_idr   )modez+pages/downloadallattachments.action?pageId=z%wiki/download/all_attachments?pageId=all_attachments.zipr]   r   r   bodyviewrJ   zcomments.jsonu#   Дампим комментарииr   z2,extensions.inlineProperties,extensions.resolutionr   r   Zexport_viewzrestrictions.jsonu   Дампим праваrY   rZ   r[   	info.jsonrX   u5   Не удалось скачать документ   ())0r   r-   r*   r4   r   get_all_pages_from_spacer   r   	is_cancelr   r   r   Zget_page_by_idr2   r.   r7   r   mkdirZget_all_attachmentboolr   r   r   r   rX   r   r   r8   r9   rm   rn   r   r   rG   r   rs   r   removeZget_page_commentsrW   r   r;   Z get_all_restrictions_for_contentr   r   	exceptionr   put)r?   startmaxobjs_dirr   obj_apiobjs_that_are_doneerrorsrw   doc_dirownerZdoc_dir_tmpr   Z	info_fileattach_infoZ
has_attachr   r   r   Zattach_file_pathr_   r   Zcomments_file_namer2   commentZrestriction_file_namery   rZ   Z	info_pathr#   r#   r$   _dump_document_part  s    










*


"
$z'ConfluenceAPIImport._dump_document_partc           
      C   s  t |d  d}|jdd | j }d}| j|d }| jjd|  t	|| j
 d }|sjd	}d}g }|dkr| jj| jd
| || || | ||t| jj| jj| jj| jj| jj| jjjd|dd}	||	 |	  | jjd|	  |d7 }||8 }qr|D ]}	|	  ~	q| j rZd| j_| j  t  | jjd | jjd | s|| 7 }qh| jjd|  |S )NrX   
/documentsTr   r   r   u:   Дампим документы пространства: r   r   r  r   logintokenpassword
verify_sslr9   )r   r   r   r   r   r   targetrG   kwargsu   Форкаем proc=import-canceling   Импорт прерванuU   Закончили параллельное скачивание документовu   Ошибок: ) r   r   rB   Queuer-   Zget_count_pages_from_spacer*   r4   r   intrD   rA   Threadr  r   r   r  r  r  r	  r8   r9   rl   r   r   r   statusrs   rv   r   emptyr.   )
r?   r   r   r   r   cntstepiZprocessprocr#   r#   r$   _dump_documentsv  sV    

  



z#ConfluenceAPIImport._dump_documentsc              
   C   sL   | j  d}t| d| dd}t|}|W  5 Q R  S Q R X d S )Nprojectsr~   
/info.jsonr   )r*   r   r   r   r   r/   )r?   project_keyproject_dirr   projectr#   r#   r$   _get_project  s    
z ConfluenceAPIImport._get_projectc              
   c   sb   | j  d}t|D ]B}|dr*qt| d| dd}t|}W 5 Q R X |V  qd S )Nr  r   r~   r  r   )	r*   r   r   r   listdirendswithr   r   r/   )r?   r  r  r   r  r#   r#   r$   _get_projects  s    
z!ConfluenceAPIImport._get_projectsc           	   	      s  t td fdd jjd }|d d d }|d sDd	S d
d |d D }|sʈ D ]f}d|d< g |d< |d D ]}|d |dg d q~|d D ]}|d |dg d q|| qbt }|D ]}z|d r|d sN|d d }|d D ]}|d |dg d q
|d D ]}|d |dg d q0|d D ]p}|d rV|	|d  |
dstj d|d d |d<  ||d D ]}|	| qqVW q   jjd| d Y qX q|_d	S )uJ   Прогружаем все выбранные документы
        )doc_infor   c                 3   s   | d sd S | d s t|d| d }|sX jd7  _jjd|   d S |d d d D ]}| d |d d	g d
 qh| d D ]&}|d r ||E d H  |d V  qd S )N	isCheckedchildren	documentsr^   r   u'   Документ не загружен pager[   Tr^   r#  r$  )_get_documentr   r   r1   r*   r4   errorrl   )r"  r   Zdoc_dumpchildget_childrenr?   r#   r$   r,    s    z=ConfluenceAPIImport._load_selected_objs.<locals>.get_childrenimport_settingsrI   rJ   r   r#  Nc                 S   s   g | ]}|d  r|qS r#  r#   rQ   setting_objr#   r#   r$   r     s      z;ConfluenceAPIImport._load_selected_objs.<locals>.<listcomp>r$  T
root_pagesr'  archived_pagesr   r   r^   rX   r  uc   Не удалось вычислить выбранные документы для проекта u4   , скорее всего он не сдамплен)r   r   rT   r*   json_settingsr!  rl   r=   r  addr.   r   r   r4   r   r>   )	r?   rO   rootselected_spacesr  r&   resultr   r"  r#   r+  r$   _load_selected_objs  sF    



&z'ConfluenceAPIImport._load_selected_objsc              
   C   s  |  | jjd }|d d d }|d s.dS dd |d	 D | _|d rpt|d	 dkrpd
d | j D | _| j d}|j	dd | jD ]}z| j
 rd| j_| j  W  d S | j jd7  _|t|d d }|d}| s8| jj|d d }| jjd|d   |d dkrDtd| t||d d  d|d< tj|d dd g |d< g |d< | j|d D ]}|d |d  q| jj|d dddD ]}	|d |	d  q|d stdt|d | t||d< t|d}
t||
 W 5 Q R X t  n t|d }
t |
}W 5 Q R X |  j!| "|7  _!W q   t#  |  j!d7  _!| jj$d!|  Y qX qd S )"Nr-  rI   rJ   r   r#  Tc                 S   s   g | ]}|d  r|qS r.  r#   r/  r#   r#   r$   r     s      z4ConfluenceAPIImport._dump_spaces.<locals>.<listcomp>r$  c                 S   s   g | ]}||d  dqS )r^   )r   r^   r#   r   r#   r#   r$   r     s     r  r   r  r   r   r   r   u&   Дампим пространство rG   r   ZpersonaluY   Миграция персональных пространств в разработке r   rX   r1  r2  r^   Zarchived)r  uJ   Пустой проект, возможно нет прав доступаr   r   u1   Ошибка дампа пространства )%rT   r*   r3  r6  r   r-   Zget_all_spacesr   r   r   r   r  rs   r;   r   r   r8   Z	get_spacer4   r   CmfErrorr   makedirsZget_root_pages_from_spacerl   r   r   r   r   r   r   rv   r/   r1   r  rollback_purge_eventr   )r?   rO   r5  Zprojects_dirr0  r  	data_filer   rw   Zarchived_docr   r#   r#   r$   _dump_spaces  sZ    



 
z ConfluenceAPIImport._dump_spacesc                 C   s  | j  d}| rd S |jdd | j D ]J}| jjjrPt	|d }nt	|d }||}|jdd |d}| j  j
d7  _
t|d}t|| W 5 Q R X |d	}| j  d	}|jdd |jdd | j|d D ]}	||	| j d
 }
| j  j
d7  _
t|
d}t|	| W 5 Q R X ||	| j d
 }
|
 rZqt|
d}t|	| W 5 Q R X qq2d S )NgroupsTr   r^   rG   r   r   r   r   r   )r*   r   r   r   r   r-   Zget_all_groupsr8   r9   r   r;   r   r   r   Zget_group_membersr7   )r?   
groups_dirr\   r_   Z	group_dirr<  r   	users_dirZall_users_dirrZ   Zuser_data_filer#   r#   r$   _dump_groups  s8    




z ConfluenceAPIImport._dump_groupsc                 C   sl   | j  }t|\}}}|| d }| j jd| d | j jd | j jd |   |   d S )Nd   u   Свободно места: ra   u$   Скачаем данные из APIu'   Дампим пользователей)r*   r   r   
disk_usager4   r   rA  r=  )r?   Zdownload_pathtotalusedfreeZfree_percentr#   r#   r$   download_data5  s    
z!ConfluenceAPIImport.download_data)r   c                 C   s  dd l }dd l}|| _d}tdd | j r6q| }|dkrJq| jjd|d   t	dD ]N}z| jj
|d dd	d
}	|	  t|d d"}
|	jddD ]}|
| qW 5 Q R X ||d \}}|r|dkr||d }W 5 Q R X | jjd|d   tjj|d d}d	|_|  t  W 5 Q R  W  q(W 5 Q R X W qj tk
r } z:t  |dkr|d7 }| jjd|d  d|	  W 5 d }~X Y qjX qjq(|| W 5 Q R X d S )Nr   Fr   DONEu    Пытаемся скачать r      x   T)timeoutstreamrX   zwb+i    )Z
chunk_sizezapplication/zipu   Сохранили в r^   r^   r   r   u@   Не удалось скачать файл с 3 попыток r   )zipfiler   r-   r   r*   r   r.   r4   r   rangeZconfluence_requestZraise_for_statusr   Ziter_contentrW   r   ZipFiler   r   r   rs   rv   r   r;  r   r   )r?   r   download_queue
done_queuerN  r   r   r   r  resr   chunkZmimetypeencodemyzipZdownload_objr   r#   r#   r$   _download_file_worker@  sD    


4z)ConfluenceAPIImport._download_file_workerc           	      C   s  | j jd | j }| j }| j dg dd| j jjjgdddgg}tj	j
|dsl| j jd d S g }t| jD ]v}| jj| jd	| ||t| jj| jj| jj| jj| jj| jjjd
dd}|| |  | j jd|  qztj	j|ddgdD ]}||j|j|jd q|D ]}|d q*|  sZ|D ]}|!  qJ| j jd | j " rd| j _#| j $  t%  | j j&d d}|  s||' 7 }q| j jd|  |  j(|7  _(d S )Nu=   Скачиваем все найденные вложенияzplugin.plugin.source_hashrn   rh   r   Frb   u   Вложений нетrW  r  )rQ  rR  r   r
  u    Запустили задачу r   rX   rc   rH   )r^   r   rX   rH  u%   Закончили скачиватьr  r  r   uC   Закончили скачивание файлов, ошибок ))r*   r4   r   rB   r  load_fieldsrm   rn   r   r   countrO  rD   rA   r  rW  r   r-   r   r  r  r  r	  r8   r9   rl   r   slistr   r^   r   rX   r  r   r   r  rs   rv   r   r.   r1   )	r?   rQ  rR  r|   Zprocsr  r  r   r1   r#   r#   r$   download_filesg  sR    

  



z"ConfluenceAPIImport.download_filesc              	   c   s:   | d}t |d}|D ]}t|V  qW 5 Q R X d S )Nz/comments.jsonr   r   r   r   r?   rX   Zcomments_file_pathr   r   r#   r#   r$   _get_comments  s    
z!ConfluenceAPIImport._get_commentsc              	   c   s:   | d}t |d}|D ]}t|V  qW 5 Q R X d S )Nz/restrictions.jsonr   r]  r^  r#   r#   r$   rj     s    
z%ConfluenceAPIImport._get_restrictions)r   c                    s6    d D ]"}d fdd	  |d qd S )NrX   c              	      sL  z| d dkr*j jd|   W d S j jjj d| d  }tjj|dgd}|rf|jsfW d S |s~tj|j d}n|	d	g | d
 dr
| d
 d j |_n2| d
 drވ
| d
 d j |_ntj|_| d
 d |_| d
 dr 
| d
 d j |_n|j|_| d
 drj
| d
 d d j |_| d
 d d |_n|j|_|j|_| d d d |_|jj|_| |_||_| d d dkrd|_| d d d dkrd|_|jdd | d D ]} || q|W S    j jdd  d|    jd 7  _Y nX d S )!Nr  ZdraftuJ   Не обрабатываем черновики комментариев r]   r^   rp   r_   rH   )rg   r_   r*   **r   r   r   createdDater   r   whenr   r   rJ   
extensionslocationinlineTZ
resolutionZresolvedclosedrf   r$  u7   Ошибка обработки комментария r   r   r   )r*   r4   r   rm   rn   r   
CmfCommentr.   rp   rY  rk   r7   	cmf_ownerrL   anonymous_usercmf_created_at
cmf_authorcmf_modified_bycmf_modified_at_process_mentionsr   _process_linksrJ   import_raw_jsontree_parentrf  r  rs   r   r1   )r  rr  r_   Zcmf_commentr*  creater   rg   r?   r#   r$   rt    sP    
z9ConfluenceAPIImport._process_obj_comments.<locals>.create)r  )N)r_  )r?   rg   r   r  r#   rs  r$   _process_obj_comments  s    1z)ConfluenceAPIImport._process_obj_commentsc                 c   s4   | d}t |D ]}|| d| dV  qd S )Nz/attachmentsr~   )r   r   )r   r  )r?   Zobj_pathZattachments_pathr   r#   r#   r$   _get_attachments  s    
z$ConfluenceAPIImport._get_attachments)r   c              	   C   sJ   t t|d}t|}W 5 Q R X |D ]}|d | kr,|  S q,d S )Nr   r   )r   r   r   r   r/   )r   r   r   Zattach_info_listattachr#   r#   r$   _get_confluence_attach_info  s
    z/ConfluenceAPIImport._get_confluence_attach_infoc                 C   sT  ddl m} dd l}t|d  d}d}| rP||
}| D ]}z|dr^W qJ| ||d }	t	j
|}
t	jj|
|dddd	gd
}|st	j| j|d|
d}|jr||T}|jdd |j| dd |jdd |	r|j|	d d kstt  W 5 Q R X W qJ   | jjd|  |d7 }Y qJX qJW 5 Q R X |S )Nr   )RDiskrX   z /attachments/all_attachments.zipr~   rp   r   url_preview_imgurl_preview)rG   rg   rH   T)r*   rg   rp   rG   rf   )Zsmart_backuprd  ZfileSizeu:   Не удалось обработать вложение r   )Zrdisk.rdiskry  rN  r   r   rP  namelistr   rx  r   r"   r   CmfAttachmentr.   r*   rp   r   rs   upload_filerV   st_sizeAssertionErrorrv   r4   r   )r?   rg   r   ry  rN  Zall_attach_zip_pathr   rV  Zzip_attach_namer  Zattach_name
attachmentZattach_fr#   r#   r$   _process_obj_attachments  sB    



z,ConfluenceAPIImport._process_obj_attachmentsc              
   C   sH   |  |d}| sd S t|d}t|}|W  5 Q R  S Q R X d S )Nr   r   )r   r   r   r   r/   )r   r&   Zdoc_filer   rS  r#   r#   r$   r(    s    
z!ConfluenceAPIImport._get_documentc                 c   s:   t | d}t|D ]}d|kr&q| ||V  qd S )Nr  r   )r   r   r  r(  )r?   r  r   r&   r#   r#   r$   _get_documents  s
    z"ConfluenceAPIImport._get_documentsc                 C   s\  d}t ddB | }|dkr&qD| j r4qDz| jjjj d|d  }tjj|d}|s|tj| jd||d	 d
}|jr|j	s|
dg |d d }	| jjd|	  |	d dkrtj}
n| |	| j }
|
s| jjd|d	  d|	  |d d}|rL| jjd|  |d dkr:tj}n| || j }n|
}|d d d }| jjd|  |d dkrtj}n| || j }|d d d }||d	  ||
| j| jjt|d d|d d |d d d |||d}|j	r||_|d jd kr||d!< t|d"d#r6d|d$< |D ]}t||||  q:|jrl|jdd% t  |jdd% | jjjj d&| }tjj|d}|stj|| jd'}||_|d( d)rt j!"| j#j$|d( d) }n| jjjj%}|j& d*| d+| d,|_'d-|_(|jdd% n| jjd.|  || )||7 }| *|| |jr| j#j+j,rz0t-.|d d/ d }t/| 0|||d0 }W n2   | jj1d1|d	   |  j2d27  _2Y nX | 3|}| 4||}| 5|||_'|jr|jdd% | 6|| t  W q   | jj1d3|  |d27 }t7  Y qX q|8| W 5 Q R X d S )4Nr   Fr   rH  r]   r^   r_   Tr   )r*   rp   r_   rG   ra  r   r   "   Определяем автора r   r   u1   Не найден автор документа: r   r   (   Определяем владельца r   r   =   Определяем последнего редактора r   r   rJ   r  )status_typerb  rc  )rg   rG   ri  rl  Zworkflowr  rk  rn  r_   rm  rq  ZCLOSEDrr  r   tableZfull_screenrf   z
::comment:)r_   r*   Z_linksZwebuiu4    импортирован из Confluence: <a href="z" target="_blank">z</a>r   uQ   Документ изменен пользователем, пропускаем atlas_doc_formatcontentC   Не удалось конвертировать разметку: r   u   Ошибка создания )9r   r.   r*   r   rm   rn   r   r"   rp   is_newrY  r4   r   rL   rj  rk   r7   r   r0   Zget_default_statusr   Zget_status_typerr  r  r   r   setattrrr   rs   rv   rh  rg   r   rX   r   r-   r   ext_urlverbose_namer   	log_levelr  ru  r8   r9   r   r   r   _convert_atlas_docr   r1   ro  _process_inline_commentsrp  r}   r;  r   )r?   r  archive_folder	obj_queuerR  r   rw   r_   rx   authorrl  r   ri  modifiedrm  r   Zdoc_dictr   Zcomment_ext_idr  r   	atlas_rawr#   r#   r$   _process_document_fork  s    
  


z*ConfluenceAPIImport._process_document_fork)r   c              
   C   s   t |d}tjjdd|gdddgddd ggddd	gd
D ]}zr|jd d d r|jd|jd d d idD ]<}|jd |jd< |j|jd< |jd |j	 d|jd< qnW q8   | j
jd| d|j  |  jd7  _Y q8X q8t|S )Nhtml.parserrg   rh   rf  Trq  ri   r   r  rX  rd  ZinlinePropertiesZ	markerRefzdata-refr   zdata-ref_origclasscmf_converteduO   Не удалось конвертировать inline комментарий r   r   )r   r   rh  r[  rq  r   r   r^   rl   r  r*   r4   r   r   r1   r   )r?   r   r   doc_soupr  r   r#   r#   r$   r    s(    


z,ConfluenceAPIImport._process_inline_commentsc              	   C   s  g }| j  }| j  }tjjd|d}| jjd t| j	D ]N}| j
j| jd| ||||dd}|| |  | jjd|  q@t | _| |d D ]j}	|	d	 |d
 d krq|	d d d rdd |	d d d D | j|	d	 < |	d	 | jkrq||	 q|D ]}|d q| jjd |D ]}|  ~q:| j rd| j_| j  t  | jjd d S d}
| s|
| 7 }
q| jjd|
  |  j|
7  _d S )Narchive)Zsys_typerr  uD   Грузим документы в несколько потоковr  )r  rR  r  r  r
  u.   Параллельный обработчик rX   r^   r1  r   r$  r&  r[   c                 S   s   g | ]}|d  qS rM  r#   r   r#   r#   r$   r     s     z:ConfluenceAPIImport._process_documents.<locals>.<listcomp>rH  u0   Ждем обработки документовr  r  uI   Закончили обработку документов, ошибок )rB   r  r   	CmfFolderr.   r*   r4   r   rO  rD   rA   r  r  rl   r   r   r<   r  r>   r   r   r   r  rs   rv   r   r  r1   )r?   r   r  Zobj_processr  rR  r  r  r  rw   r1   r#   r#   r$   _process_documents  sT    


$

z&ConfluenceAPIImport._process_documentsc                 C   s4  | j  D ]"\}}z| j r(W  q0| jjjj d| }tjj|dgd}|D ]~}| jjjj d| }tjj|ddgd}|s| jj	
d| d|  qT|jrT|j|krT||_|js|j|_|jdd	 qT|r|jsd|_|jdd	 t  W q
   t  | jj	d
|  |  jd7  _Y q
X q
d S )Nr]   tree_node_is_branchr`  rr  rp   u-   Не найден потомок по ext_id=z tree_parent_ext_id=Trf   uI   Не получилось привязать ноду tree_parent_ext_id=r   )r<   itemsr*   r   rm   rn   r   r"   r.   r4   r)  rp   rr  rg   rs   r  rv   r;  r   r1   )r?   Ztree_parent_ext_idr$  rr  r_   r*  r#   r#   r$   _process_documents_tree  s2    


z+ConfluenceAPIImport._process_documents_treec                 C   s  |  | jjd }|d d d dkr<dtjjddd}n`|d d d d	krfdtjjd
dd}n6|d d d dkrdddtjjd
dd}ntd|   |  D ]<}z | j	 rW  q|d dd | j
D krW q| jjd|d   |d |d< |d |d< | jjjj d|d  |d< tjjdd|d gddgd}d}|r|jj|d jks|jr|j|d kr|d  d|d  d|d< | jjd|j d|j d |d  d!|j d"|d  
 n|js|d |_|jd#d$ tjj|d | d%}|s8d#}tj| j|d |d d&}t|d' d(}|d)rl| ||d) d }n(| jjd*|  | ||d+ d, }|d- d. }| jjd/|  |d0 d1krtj|d2< n| || j |d2< |d- d3}	|	r:| jjd4|	  |	d0 d1kr$tj|d5< n| |	| j |d5< n|d2 |d5< |d- d6 d7 }
| jjd8|
  |
d0 d1krtj|d9< n| |
| j |d9< |d- d: |d;< |d- d6 d< |d=< | || | || |j s|j!r|D ]}t"||||  q|d> d? d }| j#j$j%rz0t&'|d> d@ d }t(| )|||dA }W n2   | jj*dB|dC   |  j+dD7  _+Y nX | ,|}| -||}| .|||_/||_0|  | 1|| || _2|rtj3jdEdFd|jgdddGggdH}|d d d dkr"tj4j|dIdJ}|r|5  nv|d d d d	ksN|d d d dkrtj4j|dIdJ}tj4j|dKdJ}|r||5  |r|5  |r|5  t6  | 7|| | 8  W q   t9  | jj*dL|  |  j+dD7  _+Y qX qd S )MNr-  rK   rJ   r^   r  zproject.agile:defaultr+   )project_type
logic_typeZproject_wikizproject.wiki:defaultr   sectionzCmfWiki:Class)r  	parent_idtree_parent_idr  u   Неизвестный типc                 S   s   g | ]}|d  qS rM  r#   r   r#   r#   r$   r     s     z7ConfluenceAPIImport._process_spaces.<locals>.<listcomp>u0   Импортируем пространство r   rG   task_code_prefixr]   r_   rh   r  rX  Fr   r   u   Проект с именем u    уже существует! z != z id=u   Переименуем в Trf   r`  )r*   r_   rG   rX   r%  homepageuC   У пространсва нет домашней страницы r1  r   r   r   r  r   r   rl  r   r  ri  r   r   r  rm  rb  rk  rc  rn  r   r   r  r  r  r   r   ZANDr  u   Документыrb   u   Документ1)rr  rG   u   Документ2u5   Ошибка импорта пространства ):rT   r*   r3  r   ZCmfLogicTyper.   r9  r8  r!  r   r6  r4   r   rm   rn   
CmfProjectr  r^   r_   r   rG   rs   keysr   r   r(  rL   rj  rk   r7   r  ru  rp   r  r  r-   r8   r9   r   r   r   r  r   r1   ro  r  rp  r   rq  r}   r  r  r"   ru   rv   r  r  r;  )r?   rO   Zproject_fieldsr   r  Zproject_is_newr   r  r  r   r  r   r   r  Zrm_doc_folderZrm_doc1Zrm_doc2r#   r#   r$   _process_spaces  s    
<



z#ConfluenceAPIImport._process_spacesc              
   c   sN   | j  d}t|D ].}t||dd}t|V  W 5 Q R X qd S )Nr>  r   r   r*   r   r   r   r  r   r   r/   )r?   r?  Zgroup_idr   r#   r#   r$   _get_groupsi  s    zConfluenceAPIImport._get_groupsc              
   c   sL   | j  d}t|D ],}t||d}t|V  W 5 Q R X qd S Nr   r   r  )r?   r@  r   r   r#   r#   r$   r   o  s    zConfluenceAPIImport._get_usersc              
   c   sF   | d}t|D ],}t| |d}t|V  W 5 Q R X qd S r  )r   r   r  r   r   r/   )r?   Z
group_pathZ	user_pathr   r   r#   r#   r$   _get_group_membersu  s    
z&ConfluenceAPIImport._get_group_members)r   rE   c              	   C   sb  | d}| jjd|  t|d t| dd d}|s| jjd|  tjj	j
ttj d }t|d d |  d	tj }t|d ||t| dd d
}d| jjjj d|| j  d}d}tjj dd|gdgd}|s|tjj ddd| dgdgd}|s|tjjdd|d gd}t|dkrn|  jd7  _| jjd|d  dt|  |r||d }|sd}tj||| jd}|js|jr|D ]}	t||	||	  q|dddg |jr||jkr |j | |_n||_tj }
tj }|
|j kr,|j !|
 ||j krD|j !| d|_"| j j#d7  _#||fS )NrF   u0   Импортируем пользователя ZdisplayNamer   T)rG   does_not_worku8   Не указан Email для пользователя r   @)rG   rF   r,   r  r]   Fr  rh   r   rX  r_   r`   ra   rG   rb   u+   Пользователей с именем z > r   )r_   r  r*   rg_member_of
user_local)$r.   r*   r4   r   r   r   warnZcmfrH   ZCmfEmailZ
max_lengthr   rC   
ORG_DOMAINZcmfutilZtranslit_striprm   rn   r7   r   r    rt   r1   r)  rp   r  r  rY  r_   ro   Zconfluence_group
user_groupr  rl   r  r:   )r?   r   rF   Z	user_dictZmax_lenr_   
is_createdrz   Zpersonsr   Z
conf_groupr  r#   r#   r$   _create_person{  sX    
"  "

z"ConfluenceAPIImport._create_personc                 C   s   t  }|  D ]L}| |\}}|jr8|jdd t  |r|jjt	j
s||jj q|r| jjdt| d|  ttjj|gd d S )NTrf   u,   Отправляем приглашения u    пользователям: )args)r=   r   r  rr   rs   rv   rF   rJ   r   rC   r  r4  r*   r4   r   r   Zcmf_deferred_taskr   r    Zregister_persons)r?   Zsend_invitesr   rz   r  r#   r#   r$   _process_persons  s    z$ConfluenceAPIImport._process_personsc                 C   s.  |   D ]}d}| jjjr d}d| jjjj d||  d}tjj	ddd| dgdgd}|s~tjj	dd	|d gdgd}|stj|| j|d d
|d}n|j
r|d |_||jkr|j | |_|jd
d | | j d|| D ]6}| |\}}|j| |jr|jd
d t  qqd S )NrG   r^   r]   r_   r`   ra   r   rX  rh   T)r_   r*   rG   rp   rq  rf   r>  )r  r-   r8   r9   r*   rm   rn   r   ro   r.   rp   rG   r_   rs   r  r   r   r  r  rl   rr   rv   )r?   r{   r   r_   r\   r   rz   r   r#   r#   r$   _process_groups  s0    
  

 z#ConfluenceAPIImport._process_groupsc                 C   sN   |    |   |   |   |   |   | jjd| j	  | j	 S )u   
        Запуск импорта

        :return bool: Истина в случае успешного импорта
        u#   Ошибок обнаружено: )
rG  r\  r  r  r  _process_cross_linksr*   r4   r   r1   rM   r#   r#   r$   process_import  s    z"ConfluenceAPIImport.process_importc              	   C   s   d}|j dtdids.|jdks.| dr|jrzt|ddjf |j|jdd	 |j|j|j	d
d}|
|j n@t|ddjf |j|jdd	 dd|j	d
d}|
|j |S )uS   
        Из документов MS создаем наши превью
        au  
        <div contenteditable="false" class="app-tinymce-card-preview app-tinymce-is-preview">
              <a class="app-tinymce-href-preview app-tinymce-is-preview" title="{fileName}"
                rel="{urlPreview}"
                download="{url}" cmf_converted=True>
                <span class="app-tinymce-img-preview-block app-tinymce-is-preview">
                <span class="app-tinymce-img-preview-size app-tinymce-is-preview">
                  <img class="app-tinymce-img-preview app-tinymce-is-preview"
                  src="{urlPreviewImg}" alt="attachment" cmf_converted=True>
                </span>
                </span>
                <span class="app-tinymce-file-preview-block app-tinymce-is-preview">
                      <span class="app-tinymce-file-name-block app-tinymce-is-preview">
                        <span class="app-tinymce-file-name-view app-tinymce-is-preview">{fileName}</span>
                        <span class="app-tinymce-file-name-view app-tinymce-is-preview">{fileName2}</span>
                      </span>
                      <span class="app-tinymce-file-create-view app-tinymce-is-preview display-contents"></span>
                      <svg class="remove-file-icon" style="width:24px;height:24px" viewBox="0 0 24 24">
                          <path style="pointer-events: none;" fill="#e8e8e8" d="M19,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3M19,19H5V5H19V19M17,8.4L13.4,12L17,15.6L15.6,17L12,13.4L8.4,17L7,15.6L10.6,12L7,8.4L8.4,7L12,10.6L15.6,7L17,8.4Z" />
                      </svg>
                </span>
              </a>
            </div>
            styleZ
backgroundr  Zimgr    Ni)ZfileNameZ	fileName2ZurlPreviewImgZ
urlPreviewr   r  z/static/file-outline.png)findrecompilerG   r{  r   r   formatrz  r   replace_with
currentTag)r?   r   rw  TAG_TMPLrR   r#   r#   r$   _create_preview  s,    *z#ConfluenceAPIImport._create_previewTc                 C   sh  dd l }|s|S t|d}|dd D ]4}|jdrP| jjd|  q(|dr|jd 	dr| jjd	|  q(| jjd
|  |dr|jd }n
|jd }d|jddkrP|jd 
dd 
dd }| |}	|	r4|jd |jd< |	j|jd< d|jd< | jjd|	j  q(| jjd|jd   q(tdt|}
|
r|
d}tjj| jjjj d| d}|r(d|jd< |jd |jd< |j|jd< q(tj|
dd 
dd }tj|}|jdkr:|jdr:|jd |j
dd kr:t| j dt|jj d|jd dd}|! sz|  j"d7  _"| jj#d|  q(n|$|d }||% kr|&|T}tj'j||d!}|stj'||| jd"}|j(dd# |)|*  |j(dd# W 5 Q R X n4|  j"d7  _"| jj#d$| d%|  W 5 Q R  q(W 5 Q R X tj'j+|d&d'd(g|d)}|r|dr:|jd |jd*< |j,|jd< d|jd< |d+r|jd+ |jd,< |jd+= t-.|jd 
dd }|r|d r|d 	d-rd-|_/d.|jd.< d/|jd0< d/|jd1< n,|r.|d r:|d 	d2s:| 0|| |dr|jd |jd< |j,|jd< d|jd< t-.|jd 
dd \}}|r|	d-rd-|_/d.|jd.< d/|jd0< d/|jd1< |1d3}|jd |jd< ||jd4< |2| n| 0|| | jjd|j,  n6|jd5d6krJ| jj#d7|  |  j"d7  _"q(| jjd8|  q(t|S )9Nr   r  c                 S   s   |  dp|  dS r   r   r   r#   r#   r$   r     r   z4ConfluenceAPIImport._process_links.<locals>.<lambda>r  u   Уже обработали r   #u   Это якорь u&   Обрабатываем ссылку r   z/people/r  r~   r   r   	href_origTu   Заменили на u?   Пользователь не найден для ссылки z+href="/pages/viewpage.action\?pageId=(\d+)"r   r]   r  r"   z!data-linked-resource-container-idr  r%  r   r   uA   Документ с файлом не найден в дампе r   )rG   rg   )rG   rg   r*   rf   u	   Файл u    не найден в r   r{  rz  )rG   rH   rg   Zsrc_origZsrcsetZsrcset_origZvideoZcontrols100%widthheightZimagesourcer   zdata-linked-resource-typer  u%   Не найдено вложение u'   Не обработали ссылку )3rN  r   r   r   r.   r*   r4   r   r   r   r   rk   r   r   r  searchr   r\   r   r"   rm   rn   r   r   r   r   
class_namer_   r   r   r   rg   r  r   r1   r)  rP  r|  r   r}  rs   r~  rV   sgetr   r   r   rG   r  new_tagrl   )r?   r   rg   Zmake_previewrN  Z	text_soupr   r   Z
account_idrz   matchr   Zrel_docrG   r   attach_pathrV  r   Z
attach_objr   rU  r  r#   r#   r$   rp    s    





 
 



 

 





z"ConfluenceAPIImport._process_linksc                 C   sf  | j jd t|d}|jdtdidD ],}z|jdrT| 	|jd }nv|jdrt
jj|jddd	d
}| |}| 	|| j }|jdr|jd |jd< |jd |jd< ntd|jd |jd< |j|jd< d|jd< |jj|jd< |jj|jd< W q. tk
rZ } z,| j jd| d|  |  jd7  _W 5 d}~X Y q.X q.t|S )u]   
        Упоминания пользователей
        confluence-userlink
        uH   Конвертируем упоминания пользователейr  r  zconfluence-userlinkr  zdata-account-idzdata-usernamezutf-8r   )r   r   zdata-linked-resource-idzdata-linked-resource-id_origzdata-username_origu]   Не найдены аттрибуты для определения пользователяr   r  Tr  uZ   Не удалось преобразовать ссылку на пользователя r   r   N)r*   r4   r   r   r   r  r  r   r.   rk   r   r   Zunquote_plusr   r7   r   r   r^   rJ   r  r)  r1   r   )r?   r   r  r   rz   Z	user_namer   r   r#   r#   r$   ro    s.    


"z%ConfluenceAPIImport._process_mentionsc                 C   sj   d| j jjj d| d}ddd| dg}tjj|d}|sf| j jd|  |  jd7  _t	j
}|S )Nr]   r_   r`   ra   rb   u.   Не найден пользователь id=r   )r*   rm   rn   r   r    r.   r4   r)  r1   rL   rj  )r?   r   r_   r|   rz   r#   r#   r$   rk     s    zConfluenceAPIImport._get_personN)r   r  rE   c           (      C   sL	  dd l }|s$| jjd t|d}t|}d}t|D 	]}| j rN 	qH|d7 }|d dkr4|jddd	d
dd}	dd |j	ddidD 
 }
g }|
jD ]}|dkrq|| qt|d D ]N\}}|d d }|jd| dd	d| ddd}|| j|_|	| q|
|	 tt|d}n|d dkrld }
|d dr|jd|d d id}
|
s| jjd|d d  d|d d   q8nNd|d d kr|d d d d r|jd!|d d d d  d" id}
|d d d#krd$}|jd|d d id}
|
rj|d d d% d& d" }|d d d% d'}|rjt|dd&j|d" |d(}|
|j tt|d}	q$|d d d)kr2d*}|jd|d d id}
|
rj|d d d% d& d" }|rjt|dd&jd&d+d |
jD |d,d}|
|j tt|d}	q$|d d d-r,|d d d% d. d" }tj|}tjj||d/}d0}|s(|d d d% d1 d" }t| j d2t|jj d3|d4d5}|! s|  j"d7  _"| jj#d6|  q8n|$|d7}||% kr|&|>}tj|}tj||| jd8}|j'd9d: |(|)  W 5 Q R X t*+| d;}|% D ]v}|,|rv|&|T}tj|}tjj||d/}|stj||| jd8}|j'd9d: |(|)  W 5 Q R X qvW 5 Q R X |s(|  j"d7  _"| jj#d<| d=|  q8tj|}tjjd>d?|gd'd@| dAggdBgdC}dD}dD}|d d d% dr|d d d% d d" }|d d d% dEr|d d d% dE d" }|j-j.|j/j.||dF} |r|j0| dG< nd&| dG< t|dd&jf | d}|
|j tt|d}	q$|d d dHkr|j/j.}!|d d d% dIrtjj|d d d% dI d" dJ}"|"stj1j|d d d% dI d" dJ}"|"r|"j/j.}!n| jjdK|d   q8|jd|!dd9dLd}#|dM}$|#|$ |
|# tt|d}	q$|d d dNkr|jddNd	dOd}%|jdPddQid}&|&|d d d% dQ d"  |%|& |jdRdSddTdUd}'|'|d d d% dV d"  |%|' |
|% tt|d}n|d d dWk	r$tj|d d d% d' d" }tjj|dJ}|sN| jj2dX|d d  dY| t3j4d9d9dZ |  j"d7  _"|
|5  tt|d}n|d d[k	r$|d d d\k	r$|d d d% d' d" }d]}|jd|d d id}
|
	rt|dd&j|d&d^d |
jD d_d}|
|j tt|d}n| jjd`|  |dr8| 6t|||d |}q8|S )aNr   uC   Конвертация текста из atlassian разметкиr  r   r   ZlayoutSectionZdivZfalsetruezmceNonEditable selected)contenteditablezdata-layout-sectionr  r  c                 S   s,   g | ]$}d |j d krd|j d kr|qS )zfixed-widthr  Zsingler  r   r#   r#   r$   r     s     z:ConfluenceAPIImport._convert_atlas_doc.<locals>.<listcomp>r  ZcolumnLayoutr   r  r   r  ra   zflex-basis: )zdata-column-widthzdata-layout-columnr  	extensionZlocalIdzdata-local-idu&   Не удалось найти тег u    для ZextensionKeyZmacroMetadataZ
parametersZmacroIdzdata-macro-idrJ   zexcerpt-includeuG  
                    <div class="macros-panel macros-include-block" contenteditable="false" data-include-block-name="{name}" data-localid="{page_name}" data-mce-contenteditable="false">
                    <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false">
                    <span aria-label="Макрос Include. " role="img"> <svg height="14" id="Outline" viewBox="0 0 24 24" width="14" xmlns="http://www.w3.org/2000/svg"><path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span> <span class="extension-title">Макрос Include. "Вставить выборку"</span> <span class="macros-actions"> <span class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17="" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;" viewBox="0 0 24 24" data-mce-style="width: 14px; height: 14px; color: #444444;"><path _ngcontent-aiw-c17="" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z" fill="#444"></path></svg> </span> <span class="macros-delete"> <svg class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;" viewBox="0 0 18 18" data-mce-style="width: 14px; height: 14px; color: #222222;"><path class="ng-star-inserted" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path></svg> </span> </span>
                    </div>
                    <div class="ak-renderer-wrapper" data-fabric-macro="null" data-macro-body="" data-macro-parameters="[object Object]">
                    </div>
                    </div>
                    </div>ZmacroParamsr  rG   )rG   	page_nameZincludeu  
                    <div class="macros-panel macros-include-page" contenteditable="false" data-macros="include-page"
                         data-param-pageid="{page_name}" data-mce-contenteditable="false"
                         data-rendered="true">
                        <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false">
                            <span aria-label="Макрос Include. " role="img"> <svg xmlns="http://www.w3.org/2000/svg" id="Outline"
                                                                                 viewBox="0 0 24 24" width="14" height="14"><path
                                d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span>
                            <span class="extension-title">Макрос Include. "Вставить страницу"</span> <span class="macros-actions"> <span
                            class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17=""
                                                                                                                     viewBox="0 0 24 24"
                                                                                                                     class="ng-star-inserted"
                                                                                                                     style="width: 14px; height: 14px; color: #444444;"
                                                                                                                     data-mce-style="width: 14px; height: 14px; color: #444444;"><path
                            _ngcontent-aiw-c17="" fill="#444"
                            d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path></svg> </span> <span
                            class="macros-delete"> <svg viewBox="0 0 18 18" class="ng-star-inserted"
                                                        style="width: 14px; height: 14px; color: #222222;"
                                                        data-mce-style="width: 14px; height: 14px; color: #222222;"><path fill="#222"
                                                                                                                          d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z"
                                                                                                                          stroke="none"
                                                                                                                          stroke-width="1"
                                                                                                                          class="ng-star-inserted"></path><path
                            fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path><path
                            fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path></svg> </span> </span>
                        </div>
                        <div class="ak-renderer-wrapper">{content}</div>
                    </div>
                    c                 S   s   g | ]}t |qS r#   r   )rQ   valr#   r#   r$   r     s     )r  r  ZdrawioZdiagramName)rg   rG   u  
                    <div data-drawio-title="{name}" data-drawio-type="diagram" data-attach-id="{attach_id}" class="drawio-content" contenteditable="false">
                        <div class="header">Диаграмма: "{name}"</div>
                        <div class="content">
                            <img cmf_converted=True src="{img_src}" alt="" data-mce-src="{img_src}" width="{width}" height="{height}">
                        </div>
                    </div>
                    ZpageIdr  r%  r   r   uI   Документ с диаграммой не найден в дампе r   )rg   rG   r*   Trf   z.*\.png$u   В документе u$    не найден файл с xml: rg   rh   r`   z%.pngr   rX  r  r  )rG   Z	attach_idr  r  Zimg_srcr$  r&  rP   uE   Не удалось обработать тег оглавления )zdata-children-docsr  r  ulzpaste-code-macro)r  r  pr   Zprezlanguage-markup1)r  r  zdata-mce-selectedZ__bodyContentZviewxlsu)   Ошибка обработки тега u'     вложение не найдено )r   exc_infoZbodiedExtensionZexcerptu  
                    <div class="macros-panel macros-excerpt-block" contenteditable="false" data-excerpt-name="{name}" data-mce-contenteditable="false">
                    <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false"><span aria-label="Макрос Include. 'Создать выборку'" role="img"> <svg height="14" id="Outline" viewBox="0 0 24 24" width="14" xmlns="http://www.w3.org/2000/svg"><path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span> <span class="extension-title">Макрос Include. 'Создать выборку' <span class="excerpt-title">| name = {name}</span></span> <span class="macros-actions"> <span class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17="" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;" viewBox="0 0 24 24" data-mce-style="width: 14px; height: 14px; color: #444444;"><path _ngcontent-aiw-c17="" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z" fill="#444"></path></svg> </span> <span class="macros-delete"> <svg class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;" viewBox="0 0 18 18" data-mce-style="width: 14px; height: 14px; color: #222222;"><path class="ng-star-inserted" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path></svg> </span> </span></div>
                    <div class="ak-renderer-wrapper" contenteditable="false" data-fabric-macro="null" data-macro-body="" data-macro-parameters="{name}" data-mce-contenteditable="true">
                    {content}
                    </div>
                    </div>
                    c                 S   s   g | ]}t |qS r#   r  r   r#   r#   r$   r     s     )rG   r  u   Не нашли тег для )7rN  r*   r4   r   r   r   reversedr   r  r   popcontentsrl   	enumerater  r   r.   r  r   r   r  r  r   r   r   r"   r   r}  r   r   r   rg   r  r   r1   r)  rP  r|  r   rs   r~  rV   r  r  r  rG   rJ   r^   r   r  logr3   ZERRORZcreate_preview_tagr  )(r?   r   r   r  souprN  r  r  elemZtag_layout_sectionr   r  ccolr  Ztag_layout_colr  r  Zexcerpt_namerR   rG   r   rw  r   r  rV  r   ZregexpZname_zipZfile_name_zipZ
attach_imgr  datar  rg   Zdata_children_docsr  Z
code_blockr   rJ   r#   r#   r$   r    sx   






**
"

&"









   .
z&ConfluenceAPIImport._convert_atlas_docc              	      s:  t t d fdd}t t d fdd}dddgd	d
dgg}dD ]}tt| }|j|d} jjd|j d|  d}d}|dkr@ j rd j_	 j
   jjd  dS |j|ddddg|| || | gd}	|d7 }||8 }|	D ]8}
zd}t|
drR||
jj|
_||
jj|
_|
jr|d7 }|

  |
  t  n>||
jj|
_||
jj|
_|
jr|d7 }|
j
dd t  |
jD ]F}|jdkrq||jj|_|jr|d7 }|j
dd t  q jjd|
 d|  W q   t   jjd|
    jd7  _Y qX qqq@dS )uU   
        Заменим ссылки на задачи и документы
        )r   rE   c           	         s   | s| S t | d}|dD ]}|jdr0q|jd}|sV jjd|  q|dsv jjjj	j
 | }| jjjj	j
rd }d }d|jd kr2td|jd D ]<} jjjj d	|d
d   }tjj|dgd}|r qq|s jjd|jd   q|s jjd|  qnd|jd krtd|jd }|st jjd|jd   qtjjddd|dd
d   dgdgd}|D ].}|jd |dd
d  kr|} q֐q|s jjd|dd
d    qnq|tj| qt|S )Nr  ar  r   u!   Ссылка без адреса r   z/wiki/z\/\d+r]   r   r`  u:   Не удалось найти ид документа в u:   Не удалось найти документ по ид z/browse/z	\/\w+-\d+u6   Не удалось найти код задачи в rq  r`   z%"key":"r   "%rX  r   u:   Не удалось найти задачу по коду )r   r   r   r.   r*   r4   r   r   rm   r  rJ   r  findallrn   r   r"   r  CmfTaskrt   r\   rq  r  ZCmfPluginCsvZcreate_tag_linkr   )	r   r  r   r   r   r_   Zext_codeobjsr   rM   r#   r$   check_links  sZ    

$
"z=ConfluenceAPIImport._process_cross_links.<locals>.check_linksc                    s  | s| S t | d}|jddidD ]}|jdr4q"|jd}|sZ jjd|  q"|jd} jjjj d	| }t	j
j|d
}|st	jjdd jjjj dgddd| dggd}|rt|}|jj|d< t||d< d|d< |d= q" jjd|  q"t|S )uz   
            Заменяет ссылку на корневую страницу в макросе Page Tree
            r  zdata-macrosz	page-treer  r  zdata-macro-parametersu3   Макрос Page Tree без параметров Zroot_page_originalr]   r  r_   r`   z::%rq  z%"homepage":{"id":"r  rb   r5  r  ue   В макросе Page Tree не удалось найти корневую страницу по ID )r   r   r   r.   r*   r4   r   rm   rn   r   r"   r  r   r   r^   rJ   r   r   )r   r  r   Zdata_macro_parametersZroot_page_idZroot_page_ext_idZpagetree_rootZmacro_paramsrM   r#   r$   check_pagetree_root  s:    

zEConfluenceAPIImport._process_cross_links.<locals>.check_pagetree_rootr*   ri   Nrp   rh   T)r"   r  r  r  rb   u)   Конвертируем ссылки в u=    на локальные задачи и документы: rB  r   r  u'   Отменили конвертациюr   zcomments.log_levelzcomments.text
text_draft)rc   rH   slicer   r   r   u   Проверили u!    и конвертировали u>   Не удалось конвертировать ссылки )r   varsr   rZ  r*   r4   r   r  r   r  rs   rt   hasattrr   rJ   r  rr   Z
do_approverv   Zcommentsr  r;  r   r1   )r?   r  r  r|   Z	mode_nameZmodelr  r  r  r  r   rS  r  r#   rM   r$   r    sd    1*





z(ConfluenceAPIImport._process_cross_links)T)N)>r   r   r   r   r   r   Z	CmfImportr@   r6   rt   rN   staticmethodr   r   rT   rU   r}   r   r   r   r   r   r   r   r   r  r  r  r!  r8  r=  rA  rG  rW  r\  r_  rj   ru  rv  rx  r  r(  r  r  r  r  r  r  r  r   r  r  r  r  r   r  r  rp  ro  rk   r  r  r#   r#   r#   r$   r(   &   sl   >!.f)	71'&5!
s, 0/v mr(   )"r   r  r   Zcmf.appr   pathlibr   Zurllib.parser   r   typingr   r   r   r	   r   Zcmf.data_providers.baser
   Zcmf.includeZcommon.models.cmf_active_entityr   Zmodules.confluence.apir   Z%common.models.cmf_plugin_import_mixinr   Zbs4r   Zatlassian.errorsr   r   Z	dataclassr   r%   r(   r#   r#   r#   r$   <module>   s*   	