U
    #d                    @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m	Z	 d dl
mZmZmZmZ d dlZd dlmZ d dlT d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlZejG dd dZejG dd dZ G dd dZ!dS )    N)cmf_context)Path)urlparse)datetime)DictUnionListOptional)commit_all_ds)*)CmfActiveEntity)ConfluenceApi)CmfPluginImportMixin)BeautifulSoup)ApiNotFoundErrorc                   @   sv   e Zd ZU dZejedZee	e
jf ed< ejedZee	eee	e	f  f ed< ejedZee	e
jf ed< dS )	TotalDatauJ    Данные используемые в процессе импорта default_factoryuserslink_documentsresult_documentsN)__name__
__module____qualname____doc__dataclassesfielddictr   r   strmodels	CmfPerson__annotations__r   r   r   CmfDocument r#   r#   "./modules/confluence/api_import.pyr      s   
(r   c                   @   s@   e Zd ZU dZejed< eed< ej	e
dZeeef ed< dS )DocDatauR    Данные используемые при создании документа documentdoc_idr   link_attachmentsN)r   r   r   r   r   r"   r!   r   r   r   r   r(   r   r#   r#   r#   r$   r%      s   

r%   c                   @   s  e Zd ZdZeejdddZdd Ze	ddd	Z
eeed
ddZe	dddZdd ZdjddZdd Zdd ZedddZedddZeeedddZd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zed,d-d.Zd/d0 Z d1d2 Z!d3d4 Z"ed5d6d7Z#d8d9 Z$eed:d;d<Z%d=d> Z&ed?d@ Z'dAdB Z(dCdD Z)edEdFdGZ*dHdI Z+dJdK Z,dLdM Z-dNdO Z.dPdQ Z/dRdS Z0e1ddTdUZ2dVdW Z3dkedEdYdZZ4edd[d\Z5dlee6ej7ej8f e9d]d^d_Z:d`da Z;dmee9e<dbdcddZ=dneeededfdgZ>dhdi Z?dS )oConfluenceAPIImportu;   Класс загрузки объектов через API)
confluence
cmf_importc                 C   s   || _ || _t | _tjjdd}|j | _	d| _
d| _td}| jjj|_|| jjj |   d| _| j jjr~d| _d| j_d| j_t | _d S )Nzsystem:defaultcoder   zhistory,body.export_view.webresource,body.storage,body.editor,body.view,body.editor2,body.styled_view,body.atlas_doc_format,body.dynamic,children.page,ancestors,body.plain,body.anonymous_export_view,body.wiki,history.ownedBy,history.lastUpdatedzatlassian.rest_clientkey	accountId)confluence_apir+   r   
total_datar   ZCmfSchemeWfgetZdefault_document_workflowloaddoc_workflow
has_errorsexpandZloggingZ	getLoggerloggerZhandlersZsetLevellevel_prepare_multiprocessinguser_keyconncloudimported_object_countjson_object_countr   tree)selfr*   r+   Z	scheme_wfr7   r#   r#   r$   __init__)   s"    

zConfluenceAPIImport.__init__c                 C   s(   dd l }dd l}|| _ || _tj| _d S )Nr   )	threadingqueueconfigZIMPORT_THREADSthreading_max_forks)r@   rB   rC   r#   r#   r$   r9   A   s    z,ConfluenceAPIImport._prepare_multiprocessing)returnc                 C   s2   dt tjjt tjjdddg ddd dgS )NZcurrent_user)emailname)rH   fieldsselectedObjects)rH   valueimport_type)r   gZcurrent_personrG   rH   r@   r#   r#   r$   calc_import_settingsH   s    
z(ConfluenceAPIImport.calc_import_settings)settingsrF   c                 C   s   dd | D S )uO   Преобразуем список с настройками в словарьc                 S   s   i | ]}|d  |qS rH   r#   ).0sr#   r#   r$   
<dictcomp>Q   s      z;ConfluenceAPIImport._normalize_settings.<locals>.<dictcomp>r#   )rP   r#   r#   r$   _normalize_settingsN   s    z'ConfluenceAPIImport._normalize_settingsc                 C   s   g S )uo    Для импорта из Confluence предварительный просмотр не требуется r#   rN   r#   r#   r$   previewS   s    zConfluenceAPIImport.previewc                 C   s  g g d}d }|  |d D ]>}|d dkr|d d d D ]*}d}| || j }|r>|d | q>|d d	 d D ]8}d}d
d|d  g}	tjj|	d}
|
rz|d |
 qz|d dkr|d d d D ]2}|dkrd}| || j }|r|d | q|d d	 d D ]F}|dkr&d}d
d|d  g}	tjj|	d}
|
r|d |
 qq|jr|r||_|j	r|
  |j  dd|jgddd gg}	tjj|	dD ]}|  q|d rtj|j| jd|d d
  |d rtj|j| jd|d d
  t  d S )N)readwritepathZ	operationrW   ZrestrictionsuserresultsZprivategroupext_id==idfilterupdatereadonlyrX   parentr+   !=)rd   r+   Zaccess_levelZsubjects)_get_restrictions_get_personr:   appendr   CmfPersonGroupr2   import_originalperm_policy
is_changedsave_importZperm_aclr3   ZCmfAccessRulelistdeleter+   commit_with_event)r@   docdoc_objZpermsrk   restrictionrZ   person
group_info_filterr\   Zruler#   r#   r$   _process_restrictionsW   sd    



 

 
z)ConfluenceAPIImport._process_restrictionsNc                 C   sZ   |s
| j }|D ]F}|d |d kr.|d   S |dr| ||d }|s|  S qdS )u   
        Проверяем выбор в настройках, если обьекта нет в настройках, значит по умолчанию выбран
        r_   	isCheckedchildrenT)selected_spacesr2   	_is_check)r@   objpagesZsoresr#   r#   r$   r{      s    

zConfluenceAPIImport._is_checkc           	      C   s<  | ddd  d }t|j d}t|dkrb|d dkrb|d dkrb|d	 d
krb|d }|S t|d	kr8|d dkr8|\}}}}z |dd}| jj||}W nN tk
r } z0| j	j
d| d| d| d|  W Y d S d }~X Y nX |s| j	j
d| d|  d S | j	j
t| |d }|S )N/r         wiki   Zspaces   r}      Zdisplay+ uC   Ошибка получения документа по pretty url : uo   Не смогли получить документ по pretty url (возможно не тот домен): r_   )splitr   rY   lenreplacer0   r;   Zget_page_by_title	Exceptionr+   r7   warninginfor   )	r@   urlr-   Zsplitted_url_spacetitleZfind_confl_docer#   r#   r$   _get_pageid_by_url   s6    

z&ConfluenceAPIImport._get_pageid_by_urlc           
      C   s  |j jrdS d}z~t|j jd}|jddidD ]
}|jdrT| jj	d q2|jd }|d	kr| jj	d
|jd   | 
|jd }tjjdd|gdgd}|stjjdd|gdgd}|stjjdd|gdgd}|r|jd |jd< |j|jd< d|jd< |jj|jd< q2| jjd| d|jd  d q2|jdddD ]\}| jj	t| |jdr| jj	d qN|d }|sqNd|kr| jj	d |dd }n
| 
|}|sԐqN|dd }tjjdd|gdgd}|stjjdd|gdgd}|s6tjjdd|gdgd}|r| jj	d|j  |jd |jd< |j|jd< d|jd< |jj|jd< n"| jjd| d|jd  d qN|jdd idD ]t}tjjd!d"d|jd# gd$d|jd# ggd%}|r|jj|jd#< d|jd< n"|  jd&7  _| jjd'|  qt|tjrjt||_|jr|jdd( |  nt||_ |jr|jdd( t  W nD tk
r }	 z$| jjd)| d*| d+|	  W 5 d}	~	X Y nX dS ),u   
        Проходимся по всем ссылкам документа и пытаемся преобразовать в наши ссылки на импортированные документы
        Nhtml.parserzdata-linked-resource-typepageattrscmf_convertedu   Уже обработалиdata-linked-resource-idZnulluM   В ссылке не указан ид, пытаемся найти по url=hrefr]   r^   cmf_versionra   rI   	href_origTzdata-linked-resource-versionu-   Не найден документ с ext_id z (url: uC   ),возможно он не участвовал в импортеa)r   pageIdu   Ищем по pageIdzpageId=r   ?r   u   Меняем на ссылку classzmacros-include-blockORrH   zdata-localidr_   r`   r   u>   Не удалось конвертировать макрос Z	only_datau+   Ошибка привязки ссылок u    на документы в r   )textZis_nullr   rK   find_allr   r2   r+   r7   r   r   r   r"   
CmfProject	CmfFolderr   r   r   r   r   r_   r5   error
isinstance
text_draftrl   save
do_approverp   r   	exception)
r@   objectlinksoupr]   Zref_docr   tagZcmf_docr   r#   r#   r$   _convert_links   s    

$


&



z"ConfluenceAPIImport._convert_linksrQ   c              	   C   s   | j  | d}| j  | d}| r8d S t|d6}| D ](}| j  jd7  _|t|d  qJW 5 Q R X t	
|| | j jdd t  d S )N.jsonz.json.dirtya+r   
Tr   )r+   get_download_pathjoinpathexistsopenr>   rX   jsondumpsshutilmover   rp   )r@   Zapi_funcrH   	file_pathZfile_path_tmpfrowr#   r#   r$   _simple_dump  s    
 z ConfluenceAPIImport._simple_dumpc              	   c   sF   | j  | d}t|d}|D ]}t|V  q&W 5 Q R X d S )Nr   r+)r+   r   r   r   r   loads)r@   rH   r   r   r   r#   r#   r$   _simple_get)  s    zConfluenceAPIImport._simple_get)r   attachments_dirr|   c                 C   s  |sd S | j jd t|ddd D ]}d}|drBd}|j| }z<d|kr\W q*|dr~| j jd	|  W q*|d
s| j	j
 | }|| j	j
s| j jd|  W q*tj|dd dd }t|\}}	|s
| j jd|  W q*tj|}||}
tjj||d | j jjjd}|sdtj| j ||d | j jjjd}|
|_||_d|_|  |j  W q* tk
r } z*| j j   | j jd| d|  W 5 d }~X Y q*X q*d S )Nu/   Обработка ссылок в текстеlxmlc                 S   s   |  dp|  dS Nr   srchas_attrr   r#   r#   r$   <lambda>4      z1ConfluenceAPIImport._dump_links.<locals>.<lambda>r   r   z/attachments/zmailto:u   Это почта: httpuH   Это внешняя ссылка, оставляем как есть: r   r   r   r   u   Это не файл: r_   r   
obj_ext_idsource_hashr+   r   r   r   Fu0   Не удалось скачать ссылку r   )!r+   r7   r   r   r   r   r   
startswithr   r0   r   urllibparseunquoter   	mimetypes
guess_typer   r"   _get_filenamer   CmfImportDownloadr2   pluginr   rY   rH   
downloadedr   dpcommitr   Zrollback)r@   r   r   r|   r   r.   r   	file_name	mime_typeencodingr   download_jobr   r#   r#   r$   _dump_links/  sX    



 


zConfluenceAPIImport._dump_linksc                 C   s  t dd || _d}| jjd| d|  | j|d ||D ]}z|z<| j rlW W f q| jjd|d  d	 |	t
|d
 }	|	 rW W $qF| |s| jjd|d  d|d
  d W W qF| jj|d
 | jd}|	t
|d
 d }
|
jdd |		d}|
	d}dd | jj|d
 dD }t|}t|dd}t|| W 5 Q R X |r| jj d|d
  }d}|	|}tjj||d
 | jjjjd}|stj| j||d
 | jjjjd}||_||_d|_|  | |d d d || |
	d }| r6t !| | jjd! t|d"j}| j d#}| j"|d
 |D ]D}|#t$|d$  | j j%d%7  _%| |d d& d || qnW 5 Q R X |
	d'}| rt !| | jjd( t|d"2}| j&|d
 D ]}|#t$|d$  qW 5 Q R X t'(|
|	 |jdd |		d)}t
|	|d*< t|d"}t|| | j j%d%7  _%W 5 Q R X W n8   |d%7 }| jj)d+|d  d|d
  d, Y nX W 5 | jj  X qF|*| W 5 Q R X d S )-NFZinit_views_and_dsr   u   Документы с u    до r.   u   Документ 'r   'r_   u   Пропускаем  (u0   ) т.к. не выбран для импорта)r6   .dirtyTexist_okattachmentsattachments.jsonc                 S   s   g | ]}|qS r#   r#   rR   r|   r#   r#   r$   
<listcomp>v  s     z;ConfluenceAPIImport._dump_document_part.<locals>.<listcomp>page_idw+)modez&/wiki/download/all_attachments?pageId=all_attachments.zipr   r   bodyviewrK   zcomments.jsonu#   Дампим комментарииr   2,extensions.inlineProperties,extensions.resolutionr   r   export_viewzrestrictions.jsonu   Дампим права	info.jsonrY   u5   Не удалось скачать документ  ))+r   r0   r+   r7   r   Zget_all_pages_from_spacer   r   	is_cancelr   r   r   r{   Zget_page_by_idr6   mkdirget_all_attachmentboolr   r   dumpr   r   r   r2   r   r   rY   rH   r   r   r   osremoveget_page_commentsrX   r   r>   Z get_all_restrictions_for_contentr   r   r   put)r@   startmaxobjs_dirr   obj_apiobjs_that_are_doneerrorsrq   doc_dirZdoc_dir_tmpr   Z	info_fileattach_infoZ
has_attachr   r   r   Zattach_file_pathr   Zcomments_file_namer6   commentZrestriction_file_namers   Z	info_pathr#   r#   r$   _dump_document_part\  s    


$








(


"
.z'ConfluenceAPIImport._dump_document_partc           
      C   s  t |d  d}|jdd | j }d}| j|d }| jjd|  t	|| j
 d }|sjd	}d}g }|dkr| jj| jd
| || || | ||t| jj| jj| jj| jj| jj| jjjd|dd}	||	 |	  | jjd|	  |d7 }||8 }qr|D ]}	|	  q| j rXd| j_| j  t  | jjd | jjd | s|| 7 }qf| jjd|  |S )NrY   
/documentsTr   r   r.   u:   Дампим документы пространства: r   r   r  r   logintokenpassword
verify_sslr<   )r  r  r  r   r  r  targetrH   kwargsu   Форкаем proc=import-canceling   Импорт прерванuU   Закончили параллельное скачивание документовu   Ошибок: ) r   r   rC   Queuer0   Zget_count_pages_from_spacer+   r7   r   intrE   rB   Threadr  r   r   r  r  r  r  r;   r<   rh   r  joinr   statusr   rp   r   emptyr2   )
r@   r   r  r  r	  cntstepiZprocessprocr#   r#   r$   _dump_documents  sT    

  



z#ConfluenceAPIImport._dump_documentsc              
   c   s|   | j  d}t|D ]\}|dr*qt| d| dd}t|}W 5 Q R X |d dd | j	D krpq|V  qd S )	Nprojectsr   r   z
/info.jsonr   r_   c                 S   s   g | ]}|d  qS r_   r#   r   r#   r#   r$   r     s     z5ConfluenceAPIImport._get_projects.<locals>.<listcomp>)
r+   r   r   r   listdirendswithr   r   r3   rz   )r@   project_dirZproject_keyr   projectr#   r#   r$   _get_projects  s    
z!ConfluenceAPIImport._get_projectsc           
   
   C   s`  |  | jjd }|d d d }|d s.dS dd |d	 D | _|d rpt|d	 dkrpd
d | j D | _| j d}|j	dd | jD ]}z| j
 rd| j_| j  W  d S | j jd7  _|t|d d }|d}| s| jj|d d }| jjd|d   |d dkrDtd| t||d d  d|d< tj|d dd g |d< | j|d D ]}|d |d  qt|d | t||d< t|d}	t||	 W 5 Q R X t  n t|d}	t|	}W 5 Q R X |  j | !|7  _ W q   t"  |  j d7  _ | jj#d|  Y qX qd S )Nimport_settingsrJ   rK   r   rx   Tc                 S   s   g | ]}|d  r|qS )rx   r#   )rR   setting_objr#   r#   r$   r     s      z4ConfluenceAPIImport._dump_spaces.<locals>.<listcomp>ry   c                 S   s   g | ]}||d  dqS )r_   )r|   r_   r#   r   r#   r#   r$   r     s     r$  r   r  r   r|   r.   r   u&   Дампим пространство rH   typeZpersonaluY   Миграция персональных пространств в разработке r   rY   
root_pagesr_   r   ru1   Ошибка дампа пространства )$rU   r+   json_settingsrz   r   r0   Zget_all_spacesr   r   r   r   r  r   r>   r   r   r;   Z	get_spacer7   r   CmfErrorr   makedirsZget_root_pages_from_spacerh   r   r   r   r   r   rp   r3   r5   r#  rollback_purge_eventr   )
r@   rP   rootZprojects_dirr,  r(  	data_filer   rq   r   r#   r#   r$   _dump_spaces  sP    



 z ConfluenceAPIImport._dump_spacesc           
      C   s  | j  d}| rd S |jdd | j D ]}t|d }||}|jdd |d}| j  jd7  _t	|d}t
|| W 5 Q R X |d}|jdd | j|d	 D ]J}||| j d
 }	| j  jd7  _t	|	d}t
|| W 5 Q R X qq2d S )NgroupsTr   r_   r   r   r   r   rH   r   )r+   r   r   r   r   r0   Zget_all_groupsr   r>   r   r   r   Zget_group_membersr:   )
r@   
groups_dirr\   r]   Z	group_dirr5  r   Z	users_dirrZ   Zuser_data_filer#   r#   r$   _dump_groups  s&    


z ConfluenceAPIImport._dump_groupsc                 C   sl   | j  }t|\}}}|| d }| j jd| d | j jd | j jd |   |   d S )Nd   u   Свободно места: %u$   Скачаем данные из APIu'   Дампим пользователей)r+   r   r   
disk_usager7   r   r9  r6  )r@   Zdownload_pathtotalusedfreeZfree_percentr#   r#   r$   download_data&  s    
z!ConfluenceAPIImport.download_data)r  c                 C   s  dd l }dd l}|| _d}tdd | j r6q| }|dkrJq| jjd|d   t	dD ]F}| jj
|d dd	d
$}	z|	  t|d d"}
|	jddD ]}|
| qW 5 Q R X ||d \}}|r|dkr||d }W 5 Q R X | jjd|d   tjj|d d}d	|_|  t  W W 5 Q R   q(W nP tk
r } z0t  |d7 }| jjd|d  d|	  W 5 d }~X Y nX W 5 Q R X qjq(|| W 5 Q R X d S )Nr   Fr   DONE    Пытаемся скачать r      r   TtimeoutstreamrY   wb+    Z
chunk_sizezapplication/zipu   Сохранили в r_   r%  r   u,   Не удалось скачать файл r   )zipfiler   r0   r   r+   r   r2   r7   r   rangeconfluence_requestraise_for_statusr   iter_contentrX   r   ZipFiler   r   r   r   rp   r   r3  r   r  )r@   r  download_queue
done_queuerJ  r   r	  r   r!  r~   r   chunkZmimetypeencodemyzipZdownload_objr   r#   r#   r$   _download_file_worker1  sB    

>z)ConfluenceAPIImport._download_file_workerc           	      C   s  | j jd | j }| j }| j dg dd| j jjjgdddgg}tj	j
|dsl| j jd d S g }t| jD ]v}| jj| jd	| ||t| jj| jj| jj| jj| jj| jjjd
dd}|| |  | j jd|  qztj	j|ddgdD ]}||j|j|jd q|D ]}|d q*|  sZ|D ]}|!  qJ| j jd | j " rd| j _#| j $  t%  | j j&d d}|  s||' 7 }q| j jd|  |  j(|7  _(d S )Nu=   Скачиваем все найденные вложенияzplugin.plugin.source_hashr   r^   r   Fr`   u   Вложений нетrU  r  )rP  rQ  r  r  u    Запустили задачу r   rY   r   )r_   r   rY   rA  u%   Закончили скачиватьr  r  r   uC   Закончили скачивание файлов, ошибок ))r+   r7   r   rC   r  load_fieldsr   r   r   r   countrK  rE   rB   r  rU  r   r0   r   r  r  r  r  r;   r<   rh   r  slistr  r_   r   rY   r  r  r   r  r   rp   r   r2   r5   )	r@   rP  rQ  rv   Zprocsr!  r"  r   r5   r#   r#   r$   download_filesW  sR    

  



z"ConfluenceAPIImport.download_filesc              	   c   s:   | d}t |d}|D ]}t|V  qW 5 Q R X d S )Nz/comments.jsonr   r   r   r   r@   rY   Zcomments_file_pathr   r   r#   r#   r$   _get_comments}  s    
z!ConfluenceAPIImport._get_commentsc              	   c   s:   | d}t |d}|D ]}t|V  qW 5 Q R X d S )Nz/restrictions.jsonr   rZ  r[  r#   r#   r$   rf     s    
z%ConfluenceAPIImport._get_restrictions)r|   c                    s6    d D ]"}d fdd	  |d qd S )NrY   c              	      s  z| d dkr*j jd|   W d S tjj| d dgd}|rP|jsPW d S |sltj| d j d}n|dg | d	 d
 j	 |_
| d	 d |_| d	 drʈ| d	 d j	 |_n|j
|_| d	 dr| d	 d d j	 |_| d	 d d |_n|j
|_|j|_| d d d |_|jj|_| |_||_| d d dkrrd|_| d d d dkrd|_|  | d D ]} || q|W S    j jdd  d|    jd7  _Y nX d S )Nr  draftuJ   Не обрабатываем черновики комментариев r_   rj   r]   rI   )rd   r]   r+   **historyownedBycreatedDate	createdBylastUpdatedbywhenr   r   rK   
extensionslocationinlineT
resolutionresolvedclosedry   u7   Ошибка обработки комментария r   r   r   )r+   r7   r   r   
CmfCommentr2   rj   rV  rg   r:   	cmf_ownercmf_created_at
cmf_authorcmf_modified_bycmf_modified_at_process_mentionsr   _process_linksrK   import_raw_jsontree_parentri  r  rm   r   r5   )r  rv  Zcmf_commentchildcreater|   rd   r@   r#   r$   ry    sF    
z9ConfluenceAPIImport._process_obj_comments.<locals>.create)r  )N)r\  )r@   rd   r|   r  r#   rx  r$   _process_obj_comments  s    *z)ConfluenceAPIImport._process_obj_commentsc                 c   s4   | d}t |D ]}|| d| dV  qd S )Nz/attachmentsr   )r   r   )r   r&  )r@   Zobj_pathZattachments_pathr   r#   r#   r$   _get_attachments  s    
z$ConfluenceAPIImport._get_attachments)r
  c              	   C   sJ   t t|d}t|}W 5 Q R X |D ]}|d | kr,|  S q,d S )Nr   r   )r   r   r   r   r3   )r   r
  r   Zattach_info_listattachr#   r#   r$   _get_confluence_attach_info  s
    z/ConfluenceAPIImport._get_confluence_attach_infoc                 C   sF  ddl m} dd l}t|d  d}d}| rB||}| D ]}z|dr\W qH| ||d }	t	j
|}
t	jj|
|dddd	gd
}|st	j| j|d|
d}|jr
||H}|  |j| dd |  |	r|j|	d d kstt  W 5 Q R X W qH   | jjd|  |d7 }Y qHX qHW 5 Q R X |S )Nr   )RDiskrY   z /attachments/all_attachments.zipr   rj   r   url_preview_imgurl_preview)rH   rd   rI   T)r+   rd   rj   rH   )Zsmart_backuprg  ZfileSizeu:   Не удалось обработать вложение r   )Zrdisk.rdiskr~  rJ  r   r   rO  namelistr'  r}  r   r"   r   CmfAttachmentr2   r+   rj   r   rm   upload_filerW   st_sizeAssertionErrorrp   r7   r   )r@   rd   r|   r~  rJ  Zall_attach_zip_pathr	  rT  Zzip_attach_namer  Zattach_name
attachmentZattach_fr#   r#   r$   _process_obj_attachments  sB    



z,ConfluenceAPIImport._process_obj_attachmentsc              
   C   s8   t | |dd}t|}|W  5 Q R  S Q R X d S )Nr   r/  )r   r   r   r3   )r
  r'   r   r~   r#   r#   r$   _get_document  s    
z!ConfluenceAPIImport._get_documentc                 c   s:   t | d}t|D ]}d|kr&q| ||V  qd S )Nr  r   )r   r   r&  r  )r@   r(  r
  r'   r#   r#   r$   _get_documents  s
    z"ConfluenceAPIImport._get_documentsc                 C   s4  d}t dd | }|dkr&q| j r4qzt|d }tjj|d}|sntj| jd||d d	}|jr|js|	d
g |d d }	| jj
d|	  |	d dkrtj}
n| |	| j }
|
s| jj
d|d  d|	  |d d}|r>| jj
d|  |d dkr,tj}n| || j }n|
}|d d d }| jj
d|  |d dkr|tj}n| || j }|d d d }||d  ||
| j| jjt|d d|d d |d d d |||d}|jr||_|d jdkr||d < t|d!d"r(d|d#< |D ]}t||||  q,|jrZ|  t  |  d$|d  }tjj|d}|stj|| jd%}||_|d& d'r| jj  |d& d'  }n| jj!j!j"}|j# d(| d)| d*|_$d+|_%|  n| jj
d,|  || &||7 }| '|| |jr| jj(j)rz0t*+|d d- d }t| ,|||d. }W n2   | jj
-d/|d   |  j.d07  _.Y nX | /|}| 0||}| 1|||_$|jr|  | 2|| t  W q   | jj
-d1|  |d07 }t3  Y qX q|4| W 5 Q R X d S )2Nr   Fr   rA  r_   r]   Tr   )r+   rj   r]   rH   r_  r`  rc  "   Определяем автора r-  	anonymousu1   Не найден автор документа: r   ra  (   Определяем владельца rd  re  =   Определяем последнего редактора r   r   rK   r  status_typerb  rf  )rd   rH   rn  rp  workflowr  ro  rr  r]   rq  ru  ZCLOSEDrv  r   tablefull_screencomment:r]   r+   _linkswebui4    импортирован из Confluence: <a href="" target="_blank"></a>r   uQ   Документ изменен пользователем, пропускаем atlas_doc_formatcontentC   Не удалось конвертировать разметку: r   u   Ошибка создания )5r   r2   r+   r   r   r   r"   rj   is_newrV  r7   r   rM   anonymous_userrg   r:   r   r4   get_default_statusr   get_status_typerv  r  r   r   setattrrl   rm   rp   rm  rd   r0   r   r   ext_urlverbose_namer   	log_levelr  rz  r;   r<   r   r   _convert_atlas_docr   r5   rs  _process_inline_commentsrt  rw   r3  r  )r@   r)  archive_folder	obj_queuerQ  r	  rq   r]   rr   authorrp  ownerrn  modifiedrq  r   doc_dictr.   comment_ext_idr  r   	atlas_rawr#   r#   r$   _process_document_fork  s    
  



z*ConfluenceAPIImport._process_document_fork)r   c              
   C   s   t |d}tjjdd|gdddgddd ggddd	gd
D ]}zr|jd d d r|jd|jd d d idD ]<}|jd |jd< |j|jd< |jd |j	 d|jd< qnW q8   | j
jd| d|j  |  jd7  _Y q8X q8t|S )Nr   rd   r^   ri  Tru  re   r   r  r   rg  inlineProperties	markerRefdata-refr   data-ref_origr   r   uO   Не удалось конвертировать inline комментарий r   r   )r   r   rm  rX  ru  r   r   r_   rh   r  r+   r7   r   r   r5   r   )r@   r   r|   doc_soupr  r   r#   r#   r$   r  e  s(    


z,ConfluenceAPIImport._process_inline_commentsc              	   C   s  g }| j  }| j  }tjjd|d}| jjd t| j	D ]N}| j
j| jd| ||||dd}|| |  | jjd|  q@t | _| |d D ]`}	|	d	 |d
 krq|	d d d rdd |	d d d D | j|	d	 < | |	sq||	 q|D ]}|d q| jjd |D ]}|  q0| j rvd| j_| j  t  | jjd d S d}
| s|
| 7 }
qz| jjd|
  |  j|
7  _d S )Narchive)Zsys_typerv  uD   Грузим документы в несколько потоковr  )r  rQ  r)  r  r  u.   Параллельный обработчик rY   r_   r.  ry   r   r[   c                 S   s   g | ]}|d  qS r%  r#   r   r#   r#   r$   r     s     z:ConfluenceAPIImport._process_documents.<locals>.<listcomp>rA  u0   Ждем обработки документовr  r  r   uI   Закончили обработку документов, ошибок )rC   r  r   r   r2   r+   r7   r   rK  rE   rB   r  r  rh   r  r   r?   r  r{   r  r  r   r  r   rp   r   r  r5   )r@   r   r)  Zobj_processr  rQ  r  r!  r"  rq   r5   r#   r#   r$   _process_documentsy  sR    


$


z&ConfluenceAPIImport._process_documentsc                 C   s   | j D ]}z| j rW  qtjj|dgd}| j | D ]f}tjj|dddgd}|sr| jjd| d|  q8|jr8|j	|kr8||_	|j	s|j
|_	|  q8|r|jsd|_|  t  W q   t  | jjd	|  |  jd
7  _Y qX qd S )Ntree_node_is_branchr^  rv  rj   rd   u-   Не найден потомок по ext_id=z tree_parent_ext_id=TuI   Не получилось привязать ноду tree_parent_ext_id=r   )r?   r+   r   r   r"   r2   r7   r   rj   rv  rd   rm   r  rp   r3  r   r5   )r@   Ztree_parent_ext_idrv  r]   rw  r#   r#   r$   _process_documents_tree  s.    




z+ConfluenceAPIImport._process_documents_treec                 C   s  |  | jjd }|d d d dkr<dtjjddd}n`|d d d d	krfdtjjd
dd}n6|d d d dkrdddtjjd
dd}ntd|  D ]}z| j rW  q| jj	
d|d   |d |d< |d |d< t|d |d< tjjdd|d gddgd}d}|r|jj|d jksT|jr|j|d kr|d  d|d  d|d< | jj	d|j d|j d|d  d|j d|d  
 n|js|d |_|  tjj|d | d }|sd!}tj| j|d |d d"}t|d# d$}| ||d% d }|d& d' }| jj	
d(|  |d) d*krbtj|d+< n| || j |d+< |d& d,}	|	r| jj	
d-|	  |	d) d*krtj|d.< n| |	| j |d.< n|d+ |d.< |d& d/ d0 }
| jj	
d1|
  |
d) d*krtj|d2< n| |
| j |d2< |d& d3 |d4< |d& d/ d5 |d6< | || | || |jsx|jrR|D ]}t||||  q||d7 d8 d }| j j!j"rz0t#$|d7 d9 d }t| %|||d: }W n2   | jj	&d;|d<   |  j'd=7  _'Y nX | (|}| )||}| *|||_+||_,|-  | .|| || _/|r,tj0jd>d?d|jgddd@ggdA}|d d d dkrtj1j|dBdC}|r,|2  nv|d d d d	ks|d d d dkr,tj1j|dBdC}tj1j|dDdC}|r|2  |r|2  |r,|2  t3  | 4|| | 5  W q   t6  | jj	&dE|  |  j'd=7  _'Y qX qd S )FNr+  rL   rK   r_   r)  zproject.agile:defaultr,   )project_type
logic_typeZproject_wikizproject.wiki:defaultr   sectionzCmfWiki:Class)r  	parent_idtree_parent_idr  u   Неизвестный типu0   Импортируем пространство r.   rH   task_code_prefixr]   r^   r  r   Fr   r   u   Проект с именем u    уже существует! z != z id=u   Переименуем в r^  T)r+   r]   rH   rY   	documentshomepager`  rc  r  r-  r  rp  ra  r  rn  rd  re  r  rq  rb  ro  rf  rr  r   r   r  r  r  r   r   ZANDr  u   Документыr`   u   Документ1)rv  rH   u   Документ2u5   Ошибка импорта пространства )7rU   r+   r0  r   ZCmfLogicTyper2   r1  r*  r   r7   r   r   r   r  r_   r]   r   rH   rm   keysr   r   r  rM   r  rg   r:   r  rz  rj   r  r  r0   r;   r<   r   r   r  r   r5   rs  r  rt  r   ru  r   rw   r)  r   r"   ro   rp   r  r  r3  )r@   rP   Zproject_fieldsr   r)  Zproject_is_newr
  r  r  r  r  r.   r   r  Zrm_doc_folderZrm_doc1Zrm_doc2r#   r#   r$   _process_spaces  s    
<



z#ConfluenceAPIImport._process_spacesc              
   c   sN   | j  d}t|D ].}t||dd}t|V  W 5 Q R X qd S )Nr7  r   r/  )r+   r   r   r   r&  r   r   r3   )r@   r8  Zgroup_idr   r#   r#   r$   _get_groupsF  s    zConfluenceAPIImport._get_groupsc              
   c   sF   | d}t|D ],}t| |d}t|V  W 5 Q R X qd S )Nr   r/  )r   r   r&  r   r   r3   )r@   Z
group_pathZ	user_pathZuser_idr   r#   r#   r$   _get_group_membersL  s    
z&ConfluenceAPIImport._get_group_membersc              	   C   s  t  }|  D ]8}d|d  d}tjjddd| dgdgd}|sdtjjdd	|d gdgd}|stj|| j|d d
|d}n|jr|d |_||jkr|j | |_|	  | 
| j d|d D ]r}|d}| jjd|  |s@| jjd|  dttj }t|d d |  dtj }t|d ||t|d  d}d|d  d}d}	tjjdd	|d gdgd}
|
stjjddd| dgdgd}
|
stjjdd	|d gd}t|dkr|  jd7  _| jjd|d  dt|  |r|d }
|
s>d
}	tj||d | jd}
|
jsN|
jrr|D ]}t|
|||  qR|
	  |
dddg |
jr||
jkr|
j |d  d|
_n||
_tj }tj }||
j kr|
j !| ||
j kr|
j !| |
j !| d
|
_"|
	  |	r0|#tjs0|$| t%  | j j&d7  _&qq|r| jjd t| d!|  t'tjj(|gd" d S )#Nz::r_   r]   LIKEr;  r   r   rH   r^   T)r]   r+   rH   rj   ru  r7  rG   u0   Импортируем пользователя u8   Не указан Email для пользователя @   ZdisplayName@Z
operations)rH   rG   r-   Zdoes_not_workr/   Fr  r`   r   u+   Пользователей с именем z > r   )r]   r  r+   rg_member_of
user_localu,   Отправляем приглашения u    пользователям: )args))setr  r   ri   r2   r+   rj   rH   r]   rm   r  r   r   r7   r   warnr   rD   Z
ORG_DOMAINZcmfutilZtranslit_stripr   r   r    rn   r5   r   r  r  rV  Zconfluence_group
user_groupr  rh   r  r'  addrp   r=   Zcmf_deferred_taskZregister_persons)r@   Zsend_invitesru   r]   r\   Z	user_inforG   Zmax_lenZ	user_dictZ
is_createdrt   Zpersonsr.   Z
conf_groupr  r#   r#   r$   _process_groupsR  s      

"
" "


z#ConfluenceAPIImport._process_groupsc                 C   sF   |    |   |   |   |   | jjd| j  | j S )u   
        Запуск импорта

        :return bool: Истина в случае успешного импорта
        u#   Ошибок обнаружено: )	r@  rY  r  r  _process_cross_linksr+   r7   r   r5   rN   r#   r#   r$   process_import  s    z"ConfluenceAPIImport.process_importc              	   C   s   d}|j dtdids$|jdkr|jrpt|ddjf |j|jdd	 |j|j|j	d
d}|
|j n@t|ddjf |j|jdd	 dd|j	d
d}|
|j |S )uS   
        Из документов MS создаем наши превью
        au  
        <div contenteditable="false" class="app-tinymce-card-preview app-tinymce-is-preview">
              <a class="app-tinymce-href-preview app-tinymce-is-preview" title="{fileName}"
                rel="{urlPreview}"
                download="{url}" cmf_converted=True>
                <span class="app-tinymce-img-preview-block app-tinymce-is-preview">
                <span class="app-tinymce-img-preview-size app-tinymce-is-preview">
                  <img class="app-tinymce-img-preview app-tinymce-is-preview"
                  src="{urlPreviewImg}" alt="attachment" cmf_converted=True>
                </span>
                </span>
                <span class="app-tinymce-file-preview-block app-tinymce-is-preview">
                      <span class="app-tinymce-file-name-block app-tinymce-is-preview">
                        <span class="app-tinymce-file-name-view app-tinymce-is-preview">{fileName}</span>
                        <span class="app-tinymce-file-name-view app-tinymce-is-preview">{fileName2}</span>
                      </span>
                      <span class="app-tinymce-file-create-view app-tinymce-is-preview display-contents"></span>
                      <svg class="remove-file-icon" style="width:24px;height:24px" viewBox="0 0 24 24">
                          <path style="pointer-events: none;" fill="#e8e8e8" d="M19,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3M19,19H5V5H19V19M17,8.4L13.4,12L17,15.6L15.6,17L12,13.4L8.4,17L7,15.6L10.6,12L7,8.4L8.4,7L12,10.6L15.6,7L17,8.4Z" />
                      </svg>
                </span>
              </a>
            </div>
            styleZ
backgroundr   Zimgr    Ni)ZfileNameZ	fileName2ZurlPreviewImgZ
urlPreviewr   r   z/static/file-outline.png)findrecompilerH   r  r   r   formatr  r   replace_with
currentTag)r@   r   r|  TAG_TMPLrS   r#   r#   r$   _create_preview  s,     z#ConfluenceAPIImport._create_previewTc                 C   s  |s|S t |}|dd D ]}|jdrF| jjd|  q|drv|jd drv| jjd|  q| jjd|  |dr|jd }n
|jd	 }t	j
|d
d dd }tj|}tjj|dddg|d}|r|d	r|jd	 |jd< |j|jd	< d|jd< t|jd	 dd }	|	r|	d r|	d drd|_d|jd< d|jd< d|jd< n,|	r|	d r|	d ds| || |drt|jd |jd< |j|jd< d|jd< t|jd dd \}	}
|	rh|	drhd|_d|jd< d|jd< d|jd< |d}|jd |jd	< |	|jd< || n| || | jjd|j  qd|jddkr|jd d
d dd }| |}|r|jd |jd< |j|jd< d|jd< | jjd|j  qt|S ) Nc                 S   s   |  dp|  dS r   r   r   r#   r#   r$   r     r   z4ConfluenceAPIImport._process_links.<locals>.<lambda>r   u   Уже обработали r   #u   Это якорь u&   Обрабатываем ссылку r   r   r   r   r   r   r  r  )rH   rI   rd   Zsrc_origTZvideoZcontrols100%widthheightZimager   sourcer-  u   Заменили на z/people/r  )r   r   r   r2   r+   r7   r   r   r   r   r   r   r   r   r"   r   r  sgetr   r   r   rH   r  new_tagrh   rg   r   r   )r@   r   rd   make_previewZ	text_soupr   r   r   
attach_objr   rS  r  Z
account_idrt   r#   r#   r$   rt    sv    

 
 

 







z"ConfluenceAPIImport._process_linksc                 C   s&  | j jd t|d}|jdtdidD ]}z|jdrR| 	|jd }nF|jdr| 	|jd }|jd |jd	< |jd |jd
< nt
d|jd |jd< |j|jd< d|jd< |jj|jd< |jj|jd< W q. t
k
r } z| j jd| d|  W 5 d}~X Y q.X q.t|S )u]   
        Упоминания пользователей
        confluence-userlink
        uH   Конвертируем упоминания пользователейr   r   zconfluence-userlinkr   zdata-account-idzdata-usernamer   zdata-linked-resource-id_origzdata-username_origu]   Не найдены аттрибуты для определения пользователяr   r   Tr   uZ   Не удалось преобразовать ссылку на пользователя r   N)r+   r7   r   r   r   r  r  r   r2   rg   r   r   r_   rK   r  r   r   )r@   r   r  r   rt   r   r#   r#   r$   rs    s&    

.z%ConfluenceAPIImport._process_mentions)rq   rd   rF   c                 C   sD  |d d }| j jd|  d|kr8| |d }n*|d dkrLtj}n| |d }t| | j jd|  | jjj	r| |d d	 d
 d }n| |d d	 d
 d }|d d d }|r|}n||d  ||| j
| j
jt|d d|d d |d d	 d |d || jd}	|j s<d|_|  | j jd tj| j tj|	dd|d gd\}}
| jjj	rz*| j jd t|d d d |_W n6 tk
r   | j jd|d d d   Y nX n
d|i|_|| jj|d < t||d }d}d}|dd d!d"g | j jd# d$}| jj|d d%D ]}zh|d&7 }| j jd'|d   |d }tj| j tj||d(d)d|gd*d|d ggd\}}|j s| jjj!|d+ d,  }| j jd-|  | jj"|d.dd/x}|#  |$  t%|j&d0$}|j'd1d2D ]}|(| qW 5 Q R X |j)*  tj+j)j,- }|.|j/}|0  W 5 Q R X | j jd3|j1 d4|  |d5g |j!j2|j3|d < |j45| W nD tk
r   |  j6d&7  _6| j j7d6| d7| d8|  Y nX qD| j jd9|  |j8r@| j jd: t9|d;}| :t;||}| <|}| =||}|d<d g |j>d=kr||_?|@d>rd|_A|  |B  n||_C|  d?|d  }tjDjE|d@}|stjD|| j dA}||_F|d+ EdBr| jj! |d+ dB  }n| j jGjGjH}|jI dC| dD| dE|_CdF|_J|  |S )Gu`  
        Создание документа

        :param doc_id: Идентификатор документа.
        :param parent: Родительский объект.
        :param all_docs: Все документы полученные из пространства.
        :param total_data: Данные для импорта.
        r`  rc  r  r/   r-  r  userKeyr  rd  re  r   r   rK   r   r  r  rb  rf  r_   )rv  rH   rn  rp  r  r  ro  rr  r]   rq  rd   Tu+   Создаем документ в базеr]   r^   Z
filter_obju'   Грузим atlassian разметкуr  u2   Ошибка получение raw данных: r  Nzattachments.urlr   zattachments.url_previewzattachments.url_preview_imgu:   Запрашиваем вложения документаr   r   r   u=   Обрабатываем вложение документа )rH   rd   rd   rH   r  ZdownloadrB  r   rD  rG  rH  rI  u3   Скачали вложение документа r   r   u=   Ошибка загрузки вложения download_link=z, attachment=z
, content=u   Загрузили u2   Обработка текста документаr   r   r"   r  r  r  r  r  r  r  r  r   )Kr+   r7   r   rg   rM   r  printr0   r;   r<   r4   r  r   r  r)  r  r3   r   process_any_table_fieldsr   r"   r   r   ru  r   r   r1   r   r%   rV  r   r  Zfile_existsr   rL  rM  Z
upload_dirr   full_path_filerN  rX   r   r   ZCmfRFileZdata_driverZget_rdZ	get_rfileZ
_file_namer  r&   rK   r(   r   rh   r5   r   rj   r   r  r   rs  rt  
class_namer   r   r  r   r   rm  r2   rd   r   r  r  r  )r@   rq   rd   Zfirst_root_docr  Zdoc_userrq  r  rr   r  Z
doc_is_newdoc_dataZdownload_linkr~   r!  r  r   r  r  r   rR  ZrdiskZrfiler  r   r  r  r   r#   r#   r$   _create_document.  s    


(

	
*


z$ConfluenceAPIImport._create_documentc                 C   s    ddd| dg}t jj|dS )Nr]   r  z%::z::%r`   )r   r    r2   )r@   r]   rv   r#   r#   r$   rg     s    zConfluenceAPIImport._get_person)comment_parentr  rF   c              	   C   sL  |s| j jd|jj  d}zt|jjjd}| j d}| j	
|d |D ]}d}|d7 }z| j  r|W  W dS |dd	kr| j jd
|  W qR|d d d }	|	dd}	|d }
|
d }tj}|dr| |d }n0|dr
| |d }n| j jd|  |j||||
d |
d |d d}|d d dkrZd|d< |rh||d< tj| j tj|dd|d gd\}}|jr|d d d d krd!|_| |	|_| |jj|jd"|_||_|  |jr\|d d# d$ r\|jd%|d d# d$ id&D ]D}|jd% |jd'< |j j|jd%< |jd( !t"|j d|jd)< qW n8 t#k
r   |  j$d7  _$| j j%d*|  Y nX || &|||7 }t'|jtj(rt"||j_)|j  |j*  qRt"||j_|j  qR|dkr| j jd+|  W n2   | j j%d,|d   |  j$d7  _$Y nX |S )-u;   
        Создание комментариев
        uK   **** Импортируем комментарии по документу r   r   r   r_   Nr   r  r]  uD   Не импортируем черновик комментария r   r   rK   spanpr`  rc  r  r/   uO   Не возможно определить автора комментария rb  )rd   rp  rn  rq  ro  rr  r]   rg  rh  ri  Trv  r]   r^   r  rj  rk  rl  Fr  r  r  r   r  r   r   u3   Ошибка импорта комментария u    **** Импортировано u@   Ошибка импорта комментариев: parent_id=)+r+   r7   r   r&   rH   r   r   rK   r6   r0   r  r   r2   r  r   rM   r  rg   r   r   r  r   rm  rj   r  rs  rt  ru  r   ri  r   r   r_   rh   r   r   r5   r   _process_commentsr   r"   r   r   )r@   r  r  rv  r  r  r6   r  Zcomment_objr  r`  r  Zcomment_userr  newr   r#   r#   r$   r    s    


	



z%ConfluenceAPIImport._process_comments)r   r  rF   c           '      C   s  dd l }|s$| jjd t|d}t|}d}t|D ]V}| j rN q|d7 }|d dkr4|jddd	d
dd}	dd |j	ddidD 
 }
g }|
jD ]}|dkrq|| qt|d D ]N\}}|d d }|jd| dd	d| ddd}|| j|_|	| q|
|	 tt|d}n:|d dkrd }
|d dr|jd|d d id}
|
s| jjd|d d  d|d d   q8nNd|d d kr|d d d d r|jd!|d d d d  d" id}
|d d d#krd$}|jd|d d id}
|
r|d d d% d& d" }|d d d% d'}|rt|dd&j|d" |d(}|
|j tt|d}qn|d d d)kr0d*}|jd|d d id}
|
r|d d d% d& d" }|rt|dd&jd&d+d |
jD |d,}|
|j tt|d}qn|d d d-r|d d d% d. d" }tjj||d/}d0}|s|d d d% d1 d" }t| j d2t|jjd3|d4d5}| s|  j d7  _ | jj!d6|  q8n|"|d7}||# kr|$|.}tj||| jd8}|%  |&|'  W 5 Q R X t()| d9}|# D ]f}|*|rV|$|D}tjj||d/}|stj||| jd8}|%  |&|'  W 5 Q R X qVW 5 Q R X |s|  j d7  _ | jj!d:| d;|  q8t$|j+d7}|' }W 5 Q R X tjjd<d=|gd'd>| d?ggd@gdA}dB}dB}|d d d% drx|d d d% d d" }|d d d% dCr|d d d% dC d" }|j,j-t.j/0|||dD}|r|j1|dE< nd&|dE< t|dd&jf |d}|
|j tt|d}qn|d d dFkr|j2j-} |d d d% dGrtj3j|d d d% dG d" dH}!|!stj4j|d d d% dG d" dH}!|!r|!j2j-} n| jjdI|d   q8|jd| ddJdKd}"|dL}#|"|# |
|" tt|d}n|d d dMkrn|jddMd	dNd}$|jdOddPid}%|%|d d d% dP d"  |$|% |jdQdRddSdTd}&|&|d d d% dU d"  |$|& |
|$ tt|d}n|d dVkrn|d d dWkrn|d d d% d' d" }dX}|jd|d d id}
|
rZt|dd&j|d&dYd |
jD dZd}|
|j tt|d}n| jjd[|  |dr8| 5t|||d |}q8|S )\Nr   uC   Конвертация текста из atlassian разметкиr   r   r-  ZlayoutSectionZdivZfalsetruezmceNonEditable selected)contenteditablezdata-layout-sectionr   r   c                 S   s,   g | ]$}d |j d krd|j d kr|qS )zfixed-widthr   Zsingler   r   r#   r#   r$   r   "  s     z:ConfluenceAPIImport._convert_atlas_doc.<locals>.<listcomp>r   ZcolumnLayoutr   r  r   r  r;  zflex-basis: )zdata-column-widthzdata-layout-columnr  	extensionZlocalIdzdata-local-idu&   Не удалось найти тег u    для ZextensionKeyZmacroMetadataZ
parametersZmacroIdzdata-macro-idrK   zexcerpt-includeuG  
                    <div class="macros-panel macros-include-block" contenteditable="false" data-include-block-name="{name}" data-localid="{page_name}" data-mce-contenteditable="false">
                    <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false">
                    <span aria-label="Макрос Include. " role="img"> <svg height="14" id="Outline" viewBox="0 0 24 24" width="14" xmlns="http://www.w3.org/2000/svg"><path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span> <span class="extension-title">Макрос Include. "Вставить выборку"</span> <span class="macros-actions"> <span class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17="" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;" viewBox="0 0 24 24" data-mce-style="width: 14px; height: 14px; color: #444444;"><path _ngcontent-aiw-c17="" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z" fill="#444"></path></svg> </span> <span class="macros-delete"> <svg class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;" viewBox="0 0 18 18" data-mce-style="width: 14px; height: 14px; color: #222222;"><path class="ng-star-inserted" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path></svg> </span> </span>
                    </div>
                    <div class="ak-renderer-wrapper" data-fabric-macro="null" data-macro-body="" data-macro-parameters="[object Object]">
                    </div>
                    </div>
                    </div>ZmacroParamsr  rH   )rH   	page_nameZincludeu  
                    <div class="macros-panel macros-include-page" contenteditable="false" data-macros="include-page"
                         data-param-pageid="{page_name}" data-mce-contenteditable="false"
                         data-rendered="true">
                        <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false">
                            <span aria-label="Макрос Include. " role="img"> <svg xmlns="http://www.w3.org/2000/svg" id="Outline"
                                                                                 viewBox="0 0 24 24" width="14" height="14"><path
                                d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span>
                            <span class="extension-title">Макрос Include. "Вставить страницу"</span> <span class="macros-actions"> <span
                            class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17=""
                                                                                                                     viewBox="0 0 24 24"
                                                                                                                     class="ng-star-inserted"
                                                                                                                     style="width: 14px; height: 14px; color: #444444;"
                                                                                                                     data-mce-style="width: 14px; height: 14px; color: #444444;"><path
                            _ngcontent-aiw-c17="" fill="#444"
                            d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path></svg> </span> <span
                            class="macros-delete"> <svg viewBox="0 0 18 18" class="ng-star-inserted"
                                                        style="width: 14px; height: 14px; color: #222222;"
                                                        data-mce-style="width: 14px; height: 14px; color: #222222;"><path fill="#222"
                                                                                                                          d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z"
                                                                                                                          stroke="none"
                                                                                                                          stroke-width="1"
                                                                                                                          class="ng-star-inserted"></path><path
                            fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path><path
                            fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path></svg> </span> </span>
                        </div>
                        <div class="ak-renderer-wrapper">{content}</div>
                    </div>
                    c                 S   s   g | ]}t |qS r#   r   )rR   valr#   r#   r$   r   r  s     )r  r  ZdrawioZdiagramName)rd   rH   u  
                    <div data-drawio-title="{name}" data-drawio-type="diagram" data-drawio-xml="{encoded_xml}" class="drawio-content" contenteditable="false">
                        <div class="header">Диаграмма: "{name}"</div>
                        <div class="content">
                            <img cmf_converted=True src="{img_src}" alt="" data-mce-src="{img_src}" width="{width}" height="{height}">
                        </div>
                    </div>
                    r   r$  r  r   r   uI   Документ с диаграммой не найден в дампе r/  )rd   rH   r+   z.*\.png$u   В документе u$    не найден файл с xml: rd   r^   r  z%.pngr   r   r  r  )rH   Zencoded_xmlr  r  Zimg_srcry   r   rQ   uE   Не удалось обработать тег оглавления T)zdata-children-docsr  r   ulzpaste-code-macro)r   r   r  r   Zprezlanguage-markup1)r   r  zdata-mce-selectedZ__bodyContentZbodiedExtensionZexcerptu  
                    <div class="macros-panel macros-excerpt-block" contenteditable="false" data-excerpt-name="{name}" data-mce-contenteditable="false">
                    <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false"><span aria-label="Макрос Include. 'Создать выборку'" role="img"> <svg height="14" id="Outline" viewBox="0 0 24 24" width="14" xmlns="http://www.w3.org/2000/svg"><path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span> <span class="extension-title">Макрос Include. 'Создать выборку' <span class="excerpt-title">| name = {name}</span></span> <span class="macros-actions"> <span class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17="" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;" viewBox="0 0 24 24" data-mce-style="width: 14px; height: 14px; color: #444444;"><path _ngcontent-aiw-c17="" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z" fill="#444"></path></svg> </span> <span class="macros-delete"> <svg class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;" viewBox="0 0 18 18" data-mce-style="width: 14px; height: 14px; color: #222222;"><path class="ng-star-inserted" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path></svg> </span> </span></div>
                    <div class="ak-renderer-wrapper" contenteditable="false" data-fabric-macro="null" data-macro-body="" data-macro-parameters="{name}" data-mce-contenteditable="true">
                    {content}
                    </div>
                    </div>
                    c                 S   s   g | ]}t |qS r#   r  r   r#   r#   r$   r     s     )rH   r  u   Не нашли тег для )6rJ  r+   r7   r   r   r   reversedr   r  r   popcontentsrh   	enumerater  r   r2   r  r   r   r  r  r  r'  r   r  r   r   r   rd   r  r   r5   r   rO  r  r   rm   r  rW   r  r  matchr  rH   rK   r   r   Zquoter   r_   r"   r   r  )'r@   r   r|   r  r   rJ  r  r!  elemZtag_layout_sectionr   r  ccolr  Ztag_layout_colr  r  Zexcerpt_namerS   rH   r|  r   Zattach_pathrT  r   ZregexpZname_zipZ
attach_imgZxmlr  datar  rd   Zdata_children_docsr  Z
code_blockr   rK   r#   r#   r$   r    sT   






**
"

&
"









.
z&ConfluenceAPIImport._convert_atlas_docc              	      s
  t t d fdd}dddgddd	gg}d
D ]}tt| }|j|d} jjd|j d|  d}d}|dkr, j rd j_	 j
   jjd  dS |j|ddddg|| || | gd}|d7 }||8 }|D ]}	zd}
t|	dr0||	jj|	_|	jr`|
d7 }
|	
  |	  t  n0||	jj|	_|	jr`|
d7 }
|	j
d	d t  |	jD ]F}|jdkrzqf||jj|_|jrf|
d7 }
|j
d	d t  qf jjd|	 d|
  W q   t   jjd|	    jd7  _Y qX qqnq,dS )uU   
        Заменим ссылки на задачи и документы
        )r   rF   c           
         s@  | s| S t | d}|dD ]}|jdr0q|jd}|sV jjd|  q|dsv jjjj	j
 | }| jjjj	j
rd }d }d|jd kr"td|jd D ]&}tjj|d	d  dgd
}|r qq|s jjd|jd   q|s jjd|d	d    qnd|jd krtd|jd }|sd jjd|jd   qtjjddd|dd	d   dgdgd}|D ].}|jd |dd	d  kr|} qƐq|s jjd|dd	d    qnq|d}	||	jd< |j|	jd< |	|jj
  d|	d< ||	 qt|S )Nr   r   r   r   u!   Ссылка без адреса r   z/wiki/z\/\d+r   r^  u:   Не удалось найти ид документа в u:   Не удалось найти документ по ид z/browse/z	\/\w+-\d+u6   Не удалось найти код задачи в ru  r  z%"key":"r   z"%r   r.   u:   Не удалось найти задачу по коду r   T)r   r   r   r2   r+   r7   r   r   r   r  rK   r  findallr   r"   searchCmfTaskrn   r\   ru  r  r   rh   r-   r  r   )
r   r   r   r   r|   r]   Zext_codeobjsr   r  rN   r#   r$   check_links  sb    

$
"

z=ConfluenceAPIImport._process_cross_links.<locals>.check_linksr+   re   Nrj   r^   T)r"   r  r   r   r`   u)   Конвертируем ссылки в u=    на локальные задачи и документы: r:  r   r  u'   Отменили конвертациюr   zcomments.log_levelzcomments.textr   )ra   rI   slicer   r   r   u   Проверили u!    и конвертировали u>   Не удалось конвертировать ссылки )r   varsr   rW  r+   r7   r   r  r   r  r   rn   hasattrr   rK   r   rl   r   rp   Zcommentsr  r3  r   r5   )r@   r  rv   Z	mode_nameZmodelr  r   r!  r  r|   r~   r  r#   rN   r$   r    s^    5





z(ConfluenceAPIImport._process_cross_links)N)T)N)N)N)@r   r   r   r   r   r   Z	CmfImportrA   r9   rn   rO   staticmethodr   r   rU   rV   rw   r{   r   r   r   r   r   r   r   r  r#  r*  r6  r9  r@  rU  rY  r\  rf   rz  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r   r  r  rt  rs  r   r   r"   r%   r  rg   r  r  r  r  r#   r#   r#   r$   r)   &   sn   3
!b-U(,&&.!
s+ 	E/@  Q `r)   )"r   r  r   Zcmf.appr   pathlibr   Zurllib.parser   r   typingr   r   r   r	   r   Zcmf.data_providers.baser
   Zcmf.includeZcommon.models.cmf_active_entityr   Zmodules.confluence.apir   Z%common.models.cmf_plugin_import_mixinr   Zbs4r   Zatlassian.errorsr   r   Z	dataclassr   r%   r)   r#   r#   r#   r$   <module>   s*   	