B
    f'                @   s
  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZmZmZ d dlZd dlmZ d dlT d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlZej G dd dZ!ej G dd dZ"G dd dZ#dS )    N)cmf_context)Path)urlparse)datetime)DictUnionListOptional)commit_all_ds)*)CmfActiveEntity)ConfluenceApi)CmfPluginImportMixin)BeautifulSoup)ApiNotFoundErrorc               @   sv   e Zd ZU dZejedZee	e
jf ed< ejedZee	eee	e	f  f ed< ejedZee	e
jf ed< dS )	TotalDatauJ    Данные используемые в процессе импорта )default_factoryuserslink_documentsresult_documentsN)__name__
__module____qualname____doc__dataclassesfielddictr   r   strmodels	CmfPerson__annotations__r   r   r   CmfDocument r"   r"   "./modules/confluence/api_import.pyr      s   
(r   c               @   s@   e Zd ZU dZejed< eed< ej	e
dZeeef ed< dS )DocDatauR    Данные используемые при создании документа documentdoc_id)r   link_attachmentsN)r   r   r   r   r   r!   r    r   r   r   r   r'   r   r"   r"   r"   r#   r$       s   

r$   c               @   sL  e Zd ZdZeejdddZdd Ze	ddd	Z
eeed
ddZe	dddZdd Zdd ZdedddZedddZdd ZeeedddZdd Zdd!d"Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Z dd2d3Z!dd4d5Z"d6d7 Z#d8d9 Z$d:d; Z%d<d= Z&d>d? Z'd@dA Z(dBdC Z)edDdEdFZ*dGdH Z+dIdJ Z,dKdL Z-edMdNdOZ.dPdQ Z/eedRdSdTZ0dUdV Z1ddWdXZ2dYdZ Z3ed[d\d]Z4ed^d_ Z5d`da Z6ddddeZ7dfdg Z8e9j:fdhdiZ;djdk Z<dldm Z=dndo Z>dpdq Z?drds Z@ddtduZAdvdw ZBdxdy ZCdzd{ ZDd|d} ZEedd~dZFdd ZGdddZHdd ZIdd ZJedddZKdd ZLdd ZMdd ZNdd ZOdd ZPdd ZQdd ZRedddZSdd ZTdd ZUdeedddZVdd ZWdd ZXeYdddZZdd Z[deedddZ\edddZ]dd Z^dddZ_deedddZ`dd Zad S )ConfluenceAPIImportu;   Класс загрузки объектов через API)
confluence
cmf_importc             C   s   || _ || _t | _tjjdd}|j | _	d| _
d| _td}| jjj|_|| jjj |   d| _| j jjr~d| _d| j_d| j_t | _t | _d| _tj| _d S )	Nzsystem:default)coder   zancestors,history,history.lastUpdated,history.ownedBy,body,body.editor,body.atlas_doc_format,body.view,body.styled_view,body.wiki,body.plain,body.dynamic,body.storage,body.editor2,body.export_view.webresourcezatlassian.rest_clientZuserKey	accountIdF) confluence_apir*   r   Z
total_datar   ZCmfSchemeWfgetZdefault_document_workflowloaddoc_workflow
has_errorsexpandloggingZ	getLoggerloggerZhandlersZsetLevellevel_prepare_multiprocessinguser_keyconncloudimported_object_countjson_object_countr   treeset_selected_objsdump_page_versionsconfig$IMPORT_CONFLUENCE_DUMP_LAST_PAGE_VERdump_last_page_ver)selfr)   r*   Z	scheme_wfr4   r"   r"   r#   __init__+   s(    

zConfluenceAPIImport.__init__c             C   s   dd l }dd l}dd l}dd l}|| _ || _tj| _tj| _|	 d | _
| jjd| j
d  ddd tjr~ttj| j
| _
| jjdtj dd | j}| jjdt|d	  d
dd tt|d | j
| _
| j
stdd S )Nr      z
cpu_count=u%   , надо не менее 2 ядерT)	anonymouszconfig.IMPORT_PROCESS=zavailable_memory=i   @u-   Гб, по 2 Гигабайта на ядроl        u   Ресурсов системы недостаточно для импорта, обратитесь в техподдержку для дополнительной информации.)	threadingqueuemultiprocessingpsutilr@   ZIMPORT_THREADSZthreading_max_forksZIMPORT_DOWNLOAD_THREADSdownload_threading_max_forks	cpu_countmax_processesr*   logZIMPORT_PROCESSminZvirtual_memoryZ	availableint	Exception)rC   rG   rH   rI   rJ   Zavailable_memoryr"   r"   r#   r6   H   s&     
z,ConfluenceAPIImport._prepare_multiprocessing)returnc             C   s2   dt tjjt tjjdddg ddd dgS )NZcurrent_user)emailname)rT   fieldsselectedObjects)rT   valueimport_type)r   gZcurrent_personrS   rT   )rC   r"   r"   r#   calc_import_settingsh   s    z(ConfluenceAPIImport.calc_import_settings)settingsrR   c             C   s   dd | D S )uO   Преобразуем список с настройками в словарьc             S   s   i | ]}||d  qS )rT   r"   ).0sr"   r"   r#   
<dictcomp>q   s    z;ConfluenceAPIImport._normalize_settings.<locals>.<dictcomp>r"   )r[   r"   r"   r#   _normalize_settingsn   s    z'ConfluenceAPIImport._normalize_settingsc             C   s   g S )uo    Для импорта из Confluence предварительный просмотр не требуется r"   )rC   r"   r"   r#   previews   s    zConfluenceAPIImport.previewc             C   s  g g d}d }x |  |d D ]}|d dkrx>|d d d D ]*}d}| || j }|rF|d | qFW x|d d	 d D ]}d}| jjjrd
| jjjj	 d
|d  d
}	nd
| jjjj	 d
|d  d
}	ddd|	 dg}
t
jj|
d}|r|d | qW |d dkr xL|d d d D ]8}|dkr<d}| || j }|r*|d | q*W x|d d	 d D ]}|dkrd}| jjjrd
| jjjj	 d
|d  d
}	nd
| jjjj	 d
|d  d
}	ddd|	 dg}
t
jj|
d}|rx|d | qxW q W |jr|r||_|jrD|jdd |j  |j  dd|jgddd gg}
t
jj|
d t
jj  |d rt
j|j| jd|d djdd |d rt
j|j| jd|d djdd t
jj  d S )N)readwritepath	operationra   restrictionsuserresultsZprivategroupz::idrT   ext_idLIKE%)filterupdatereadonlyrb   T)save_importparentz==r*   z!=)rq   r*   Zaccess_levelsubjects)_get_restrictions_get_personr7   appendr-   r8   r9   r*   pluginsource_hashr   CmfPersonGroupr.   import_originalperm_policy
is_changedsavedpcommitZperm_aclr/   ZCmfAccessRuleZbulk_delete)rC   docdoc_objZpermsrz   restrictionrf   person
group_inforj   _filterrh   r"   r"   r#   _process_restrictionsw   sj    

 

 



z)ConfluenceAPIImport._process_restrictionsc       	      C   s6  | ddd  d }t|j d}t|dkrb|d dkrb|d dkrb|d	 d
krb|d }|S t|d	kr2|d dkr2|\}}}}y |dd}| jj||}W nH tk
r } z*| j	j
d| d| d| d|  d S d }~X Y nX |s| j	j
d| d|  d S | j	j
t| |d }|S )N/r      rE   wiki   spaces   pages   Zdisplay+ uC   Ошибка получения документа по pretty url z: uo   Не смогли получить документ по pretty url (возможно не тот домен): ri   )splitr   rc   lenreplacer-   r8   Zget_page_by_titlerQ   r*   r4   warninginfor   )	rC   urlr+   Zsplitted_url_spacetitleZfind_confl_docer"   r"   r#   _get_pageid_by_url   s.    z&ConfluenceAPIImport._get_pageid_by_urlF)rT   c          	   C   s   | j  | d}| j  | d}| r<|s<d S t|d:}x2| D ](}| j  jd7  _|t|d  qPW W d Q R X t	
|| | j jdd | j j  d S )Nz.jsonz.json.dirtyza+rE   
T)	only_data)r*   get_download_pathjoinpathexistsopenr;   rb   jsondumpsshutilmover|   r}   r~   )rC   Zapi_funcrT   Z	overwrite	file_pathZfile_path_tmpfrowr"   r"   r#   _simple_dump   s    "z ConfluenceAPIImport._simple_dumpc          	   c   sR   | j  | d}| rNt|d }x|D ]}t|V  q0W W d Q R X d S )Nz.jsonzr+)r*   r   r   r   r   r   loads)rC   rT   r   r   r   r"   r"   r#   _simple_get   s    
zConfluenceAPIImport._simple_getc          
   c   sN   | j  d}x8t|D ]*}t||}t|V  W d Q R X qW d S )Nr   )r*   r   r   oslistdirr   r   r/   )rC   Zdir_pathfpr   r"   r"   r#   _list_users_from_dump   s    z)ConfluenceAPIImport._list_users_from_dump)textattachments_dirobjc             C   s  |sd S | j jd xt|ddd D ]}d}|drFd}|j| }yHd|kr^w.|dr~| j jd	|  w.|d
st	j
| jj|}|| jjs| j jd|  w.tj|dd dd }t|\}}	|s| j jd|  w.tj|}||}
| j jjj d|d  }tjj||| j jjjd}|sttj| j ||| j jjjd}|
|_
||_d|_|  |j   W q. t!k
r } z*| j j"  | j jd| d|  W d d }~X Y q.X q.W d S )Nu/   Обработка ссылок в текстеlxmlc             S   s   |  dp|  dS )Nhrefsrc)has_attr)tagr"   r"   r#   <lambda>       z1ConfluenceAPIImport._dump_links.<locals>.<lambda>r   r   z/attachments/zmailto:u   Это почта: httpuH   Это внешняя ссылка, оставляем как есть: r   r   ?r   u   Это не файл: z::ri   )r   
obj_ext_idrw   )r*   r   r   rw   Fu0   Не удалось скачать ссылку z: )#r*   r4   r   r   find_allr   attrs
startswithr   r   rc   joinr-   r   urllibparseunquoter   	mimetypes
guess_typer   r!   _get_filenamer   rv   rw   CmfImportDownloadr.   rT   
downloadedr|   r}   r~   rQ   rollback)rC   r   r   r   r   keyr   	file_name	mime_typeencodingr   rj   download_jobr   r"   r"   r#   _dump_links   sV    



 
zConfluenceAPIImport._dump_linksc                s   y
|  |S    d}| jj||d  s:| jj||d  sN| jj||d  r| jjjrp| jj d d}n| jj d d}  fdd|	 D  |
d}t|d	}|t d
  W d Q R X | jjd| d Y nX d S )Nz%operations,status,email,personalSpace)r2   r,   )usernamer   c             3   s*   | ]"\}}| ks|d kr||fV  qdS ) Nr"   )r\   kv)	user_infor"   r#   	<genexpr>/  s    z5ConfluenceAPIImport._get_user_info.<locals>.<genexpr>z
users.jsonza+r   u4   Информация о пользователе ID u    добавлена в дамп)_get_user_info_from_dumpr-   Zget_user_infoZget_user_info_by_keyZget_user_info_by_usernamer8   r9   Zget_user_detailrn   itemsr   r   rb   r   r   r*   r4   r   )rC   user_idZ	dump_pathr2   Zuser_detail
users_pathr   r"   )r   r#   _get_user_info  s$    


z"ConfluenceAPIImport._get_user_infoNc             C   sj   xV|  dD ]H}|| j|kr$|S |dd |kr<|S |dd |kr|S qW td| d S )Nr   rT   r   r   uQ   Не нашли информацию по пользователю в дампе )r   r.   r7   lowerrQ   )rC   r   r   r   r"   r"   r#   r   8  s    z,ConfluenceAPIImport._get_user_info_from_dumpc          
   C   s  | j dkrdn
d| j  }| jjd| d|  |d}g }|d}| rtt|d}t|}W d Q R X t	
 }	y^|d	}
t|
d}t|}W d Q R X |d
}| rt| t|d}d}| jjjr|d7 }x| jj||d d || j dD ]}t|tr>| jjd| d| dddd q
|tj|ddd  | j jd7  _| jd | |d d d d || |dr
|d d  d!kr
|d |kr
||d  q
W W d Q R X W nD tk
r  } z$| jjd"| d| d#ddd W d d }~X Y n0X d|d$< t|
d%}tj||dd W d Q R X t	
 |	 }| jjd| d&|d'd( t|d%}tj||d)dd* W d Q R X d S )+Nr   u   Все версии.u   Последние: u)   Дамп версий документа z. attachmentsz
users.jsonrz	info.jsonzversions.jsonza+zNcontent.version,content.body.view,content.body.editor,content.body.export_viewz,content.body.atlas_doc_formatversionnumber)current_versionr2   Zlast_versionsu6   Не удалось сдампить документ r   zERR-0038r!   dump)
error_codeobj_type
error_typeF)ensure_asciir   rE   contentbodyexport_viewrW   bytyperF   uE   Не удалось сдампить версии документа zERR-0025r?   zw+u    выполнен за z.3fu    сек.r   )indentr   )rB   r*   r4   r   r   r   r   r   r/   time	monotonicr   remover-   r8   r9   Zget_page_versions
isinstancerQ   	log_errorrb   r   r;   inc_statr   r.   ru   r   )rC   r&   doc_pathdoc_log_msgnum_ver_msgr   	doc_users
users_filer   t1	info_filedoc_dataZversions_file_namer2   Zpage_versionexct2Zusers_fr"   r"   r#   _dump_versionsB  sl    






"z"ConfluenceAPIImport._dump_versionsc          
      s   fdd} fdd} fdd}t dd | _d	}x| }|d kr\|| P  jjd
|   d|d  d|d  d}	z$y j rP |d dkrȈ jj	d|	 dt
jd w@xtdD ]}
yb||d |d |	 ||d |d |	 ||d |d |	 |d r4 |d |d |	 P W q   |
dkrL  jj	d|	 d|
 t
jd Y qX qW W n:    jj   jjd|	 dddd |d 7 }Y nX W d  jj  X || q@W W d Q R X d S )!Nc          	      sZ    j jd|  dd  j| D }|d}t|d}t|| W d Q R X d S )NuI   Дамп списка дочерних страниц документа c             S   s   g | ]}|qS r"   r"   )r\   r   r"   r"   r#   
<listcomp>  s    z_ConfluenceAPIImport._dump_documents_expand_worker.<locals>.dump_child_pages.<locals>.<listcomp>z
pages.jsonzw+)	r*   r4   r   r-   get_childrenr   r   r   r   )r&   r   r   children
pages_filer   )rC   r"   r#   dump_child_pages  s
    
zKConfluenceAPIImport._dump_documents_expand_worker.<locals>.dump_child_pagesc       	   	      s  g }| d}| r6t|d}t|}W d Q R X | d}| rRt|  jj	d|  t|dr}xj j
| D ]Z}|t|d  x@|d d d	 D ],} jj	d
| d ||kr|| qW qW W d Q R X t|d}tj||ddd W d Q R X d S )Nz
users.jsonr   zrestrictions.jsonu%   Дамп прав документа za+r   re   rf   rg   zrestriction )zw+r   F)r   r   )r   r   r   r   r/   r   r   r*   r4   r   r-   Z get_all_restrictions_for_contentrb   r   ru   r   )	r&   r   r   r   r   r   Zrestriction_filer   rf   )rC   r"   r#   dump_restrictions  s$    


zLConfluenceAPIImport._dump_documents_expand_worker.<locals>.dump_restrictionsc          	      st   | d}| rt|  jjd|  t|d0}x( j	| D ]}|
t|d  qJW W d Q R X d S )Nzlabels.jsonu'   Дамп тегов документа za+r   )r   r   r   r   r*   r4   r   r   r-   Zget_all_page_labelsrb   r   r   )r&   r   r   Zlables_filer   label)rC   r"   r#   	dump_tags  s    

zDConfluenceAPIImport._dump_documents_expand_worker.<locals>.dump_tagsF)init_views_and_dsr   uG   Осталось обработать примерно страниц: "r   z" (ri   r  statustrashedu   Документ uW    был удален. Пропускаем дамп расширенных данных)r5   r   rc   r?      uO   Повтор дампа расширенных данных документа u   , попытка u\   Не удалось сдампить расширенные данные документа zERR-0025r!   r   )r   r   r   rE   )r   r-   r.   putr*   r4   r   Zqsize	is_cancelrN   r3   WARNINGranger   r}   r   r   r~   )rC   obj_api
page_queueobjs_that_are_doner  r  r  errors	page_datapage_log_msgir"   )rC   r#   _dump_documents_expand_worker  sV    	





z1ConfluenceAPIImport._dump_documents_expand_workerc             C   s  d}| j  }| j  }d}x| D ]}| r&t|jds&|d7 }|d}t|d}	t	
|	}
W d Q R X |
d |
d t|
d |
d	 |
d
p| jd}|| q&W |d  | jjd|  g }xt|D ]z}| jj| jd|d  t| jj| jj| jj| jj| jj| jjjd||dd}|| |   | jjd|  qW x|D ]}|!  ~qbW |j "  x|# s|| 7 }qW | jjd|  |S )Nr   )z.dirtyz.trashedrE   z	info.jsonr   ri   r   rc   r  r?   )ri   r   rc   r  r?   uC   Дамп расширенных данных документов: z_dump_documents_expand_worker )r   logintokenpassword
verify_sslr9   )r  r  r  )targetrT   kwargsu   Форк proc=uR   Ошибок дампа расширенных данных документов: )$rH   Queueiterdiris_dirr   rT   endswithr   r   r   r/   r   r.   r?   r  r*   r4   r   r  rG   Threadr  r   r-   r   r  r  r  r  r8   r9   ru   startr   clearempty)rC   objs_dirmax_workersr  r  r  
page_countdoc_dirr   r   Z	info_datadataprocessr  procr"   r"   r#   _dump_documents_expand  sT    







z*ConfluenceAPIImport._dump_documents_expandc                s   fdd}t ddv |_d}xX| }|d krF|| P x4|D ]*}zyԈjjj|j ddd}	|	d	 d
krwNd|	d  d|	d  d}
j rP jj	
d|
  xZtdD ]N}y||	 P W q   |dkr jjd|
 d|d  dtjd Y qX qW j jd7  _W n:   jj  jjd|
 dddd |d7 }Y nX W d jj  X qNW q*W || W d Q R X d S )Nc          	      s<  d| d  d| d  d}  t| d }  | d  d}|jdd g }| d	 d
 |krr|| d	 d
  | d	 d d |kr|| d	 d d  | d	 d}|r|d dkr| d	 d |kr|| d	 d  | d}| d}dd jj| d dD }t|}	t|dd}
t	
||
 W d Q R X |	r
tjjjd| d  }jjjrtjjjd| d  }d}| |}jjjj d| d  }tjj||jjjjd}|stjj||jjjjd}||_||_d|_|  |j  | d d d  ||  | d!}| rBt| jjd"|  t|d#"}
j  d$}xjj!| d |d%D ]}|
"t	#|d&  j j$d'7  _$j%d( |d d) d  ||  |d	 dr
|d	 d |kr
||d	 d  |d	 d
r>|d	 d
 |kr>||d	 d
  |d	 dr|d	 d d |kr||d	 d d  qW W d Q R X d| d*< | d+}t|d}
t	j
||
d,dd- W d Q R X t&'|| |jdd t|| d.< | d/}t|d}
t	j
| |
dd0 W d Q R X j(d1| d2 j%d3 d S )4Nr  r   z" (ri   r  z.dirtyT)exist_okhistory	createdBylastUpdatedr   ownedByr   rF   r   zattachments.jsonc             S   s   g | ]}|qS r"   r"   )r\   r   r"   r"   r#   r   7  s    zUConfluenceAPIImport._dump_documents_main_worker.<locals>.dump_doc.<locals>.<listcomp>)page_idzw+)modez+pages/downloadallattachments.action?pageId=z%wiki/download/all_attachments?pageId=zall_attachments.zipz::)r   r   rw   )r*   r   r   rw   Fr   viewrW   zcomments.jsonu5   Дамп комментариев документа za+z@,extensions.inlineProperties,extensions.resolution,children.page)r2   r   rE   commentr   r?   z
users.jsonr   )r   r   rc   z	info.json)r   u   Документ u     успешно сдампленr%   ))r   r   mkdirru   r.   r-   get_all_attachmentboolr   r   r   r   rc   r   r   r8   r9   r*   rv   rw   r   r   rT   r   r|   r}   r~   r   r   r   r4   r   r2   get_page_commentsrb   r   r;   r   r   r   rN   )r   r   r(  Zdoc_dir_tmpr   ownerr   r   attach_infoZ
has_attachr   r   r   Zattach_file_pathrj   r   Zcomments_file_namer2   r5  r   Z	info_path)r%  rC   r"   r#   dump_doc"  s    






&

zAConfluenceAPIImport._dump_documents_main_worker.<locals>.dump_docF)r  r   z,versionany)r2   r  r  )currentr	  r  r   z" (ri   r  u   Дамп документа r   r
  u+   Повтор дампа документа u   , попытка rE   u    из 3)r5   u6   Не удалось сдампить документ zERR-0025r!   r   )r   r   r   )r   r-   r.   r  r8   Zget_page_by_idr2   r*   r  r4   r   r  rN   r3   r  r;   r}   r   r   r~   )rC   r%  r  bulk_page_queuer  r<  r  Zpage_idsr2  r  r  r  r"   )r%  rC   r#   _dump_documents_main_worker!  sP    `


z/ConfluenceAPIImport._dump_documents_main_workerc       	      C   s   d}| j  }g }xt|D ]|}| jj| jd|d  |t| jj| jj	| jj
| jj| jj| jjjd||dd}|| |  | jjd|  qW x|D ]}|  ~qW |j   x| s|| 7 }qW | jjd|  |S )	Nr   z_dump_documents_main_worker rE   )r   r  r  r  r  r9   )r%  r  r?  r  )r  rT   r  u   Форк proc=u.   Ошибок дампа документов: )rH   r  r  rG   r!  r@  r   r-   r   r  r  r  r  r8   r9   ru   r"  r*   r4   r   r   r#  r$  r.   )	rC   r%  r?  r&  r  r  r*  r  r+  r"   r"   r#   _dump_documents_main  s6    




z(ConfluenceAPIImport._dump_documents_mainc             C   s  d}|d }| j d }d}| j }| jjd| d d}g }	t }
xj| D ]^}|	 rTt
|jdsrqT|d7 }|jdd }|	t
| t|	|krT||	 g }	qTW |	r||	 |d  t |
 }| jjd	|d
d | jjd| d|  t }
|| |||7 }t |
 }| jjd| d|d
d t }
|| ||7 }t |
 }| jjd|d
d | j r| jjd |S )Nr   r   rE   2   uU   Добавление страниц в очередь дампа пачками по u    страницz.dirty.uY   Добавление страниц в очередь дампа выполнено за z.3fu    сек.uU   Дамп основных данных документов пространства z: u    выполнен за uC   Дамп расширенных данных выполнен за u   Импорт прерван)rK   rH   r  r*   r4   r   r   r   r  r  r   rT   r   r   ru   r   r  rA  r,  r  r   )rC   r   r%  r  	space_keyr&  Zmax_bulkr?  r'  Z	temp_listr   r(  r2  r   r"   r"   r#   _dump_documents  sF    




z#ConfluenceAPIImport._dump_documentsc             C   sz  i }| j |d }y| j jjr@dddddddd	d
ddddd}x|D ]}|d d |d d f}||}|svqJ|||g g dd}|dr*|d dr|d d d d d }	| |	| j  |d 	|	 |d dr*|d 	|d d d d d |d d d d d d |d rJd |d!< qJW nx|D ]}
x|
d" D ]}||d# |d# g g dd}|d$ r| |d$ | j  |d 	|d$  |d% r|d 	d|d% i |d$ sT|d% sTd |d!< qTW qFW W n"   | jj
d&d'd(d)d*  Y nX t|d+  d,}| r<t| t|d-*}d.d/ | D |d0}t|| W d Q R X d S )1Nr   	EDITSPACE
REMOVEPAGEZEDITBLOGZ
REMOVEBLOGCOMMENTREMOVECOMMENTCREATEATTACHMENTREMOVEATTACHMENT	VIEWSPACEREMOVEOWNCONTENTSETPAGEPERMISSIONSSETSPACEPERMISSIONSZEXPORTSPACE))pagecreate)rP  delete)blogpostrQ  )rS  rR  )r5  rQ  )r5  rR  )
attachmentrQ  )rT  rR  )r   ra   )r   rR  )r   Zrestrict_content)r   Z
administer)r   Zexportrd   Z
targetTypeF)
permissionr   groupsrF   rr   rf   rg   r   r,   r   rh   rV  ri   rT   )ri   rT   ZanonymousAccessTrF   ZspacePermissionsr   ZuserNameZ	groupNameu>   Не удалось сдампить права доступаzERR-0056
CmfProjectr   )r   r   rc   z/permissions.jsonza+c             S   s   g | ]}|qS r"   r"   )r\   r   r"   r"   r#   r   i  s    z9ConfluenceAPIImport._dump_permissions.<locals>.<listcomp>)	convertedZoriginal)r-   Zget_space_permissionsr8   r9   r.   
setdefaultr   r*   r   ru   r   r   r   r   r   r   valuesr   r   )rC   r   Zpermissions_dictpermissionsZmapping_typerU  rd   Ztype_permrf   Zpermission_typepermission_filer   Zpermissions_datar"   r"   r#   _dump_permissions  s    








z%ConfluenceAPIImport._dump_permissions   c             C   sB  | d}|jdd d}	|	 d}
|	 d}tdd d	d	d	d	d
}|| _x| j | j}|  j|7  _W dQ R X | jjj||||
dd}|sP x|D ]}|d dkrq|d  d7  < t|d }|d }y|d dkrh|d d d r*dd | jj	|d|ddD }||d d d< |d d d rhdd | jj
|d|	dD }||d d d< d}| | d}| rt|d }| }W dQ R X tj|ddd!}t|d" }||krw|dkr|d#  d7  < n|d$  d7  < t|d%}|| W dQ R X | |}| | d&}| | d'}|d dkr| jjd(| d)| d* | rt|| n| rt|| n<| rt|| n$| rt|| n|jdd W q   | jjd+| d)| d,d-d.d/d0 |d1  d7  < Y qX qW qTW || W dQ R X dS )2u  
        Получает минимальные данные страницы с информацией о последнем изменении
        и зависимыми объектами(комментарии, вложения).
        Вычисляет хеш данных и сравнивает его с хешем предыдущей проверки,
        если хеши отличаются, то помечает каталог документа .dirty для повторного дампа
        rP  T)r-  zancestors,history.lastUpdatedz%,children.comment,children.attachmentz2,extensions.inlineProperties,extensions.resolutionF)r  r   )r  allnewupdatedNr=  )r"  limitr2   r  r  )r>  r	  r`  rE   ri   r   r	  r   r5  rg   c             S   s   g | ]}|qS r"   r"   )r\   r5  r"   r"   r#   r     s    z:ConfluenceAPIImport._check_hash_worker.<locals>.<listcomp>r_  )rc  r2   r   rT  c             S   s   g | ]}|qS r"   r"   )r\   rT  r"   r"   r#   r     s    )rc  r2   z.hashr   )r   Z	sort_keyszutf-8ra  rb  zw+z.dirtyz.trashedu   Документ "z" (uQ   ) удален. Он не будет сдамплен и импортированuB   Не удалось проверить хеш документа "r  zERR-0025r!   r   )r   r   r   r  )r   r6  r   r-   _lock_startr8   get_all_pages_from_spacer   r9  r7  r   r   ra   r   r   hashlibZmd5encodeZ	hexdigestrb   r*   r4   r   r   r   r   r  )rC   rD  r%  hash_dirr  hash_stats_queuerc  	dump_dataZcontent_hash_dirr2   Zpage_expandZcomment_expand
hash_statsr"  r   rP  r2  Z
page_titlecommentsr   Zcurrent_hashZ	hash_filer   Zcontent_jsonZnew_hashr(  Zdoc_tmp_dirZdoc_trashed_dirr"   r"   r#   _check_hash_workern  s    










z&ConfluenceAPIImport._check_hash_workerc             C   s  | j  }d| _| j | _| jd }|d }t|d  d}|jdd t	
 }g }	xt|D ]~}
| jj| jd|
d  |||t| jj| jj| jj| jj| jj| jjjd	|d
d}|	| |  | jjd|  qbW x|	D ]}|  ~qW t	
 | }d}d}d}d}xH| s`| }||d 7 }||d 7 }||d 7 }||d 7 }qW | jjd|dd| d| d| d	 | jjd|  |S )Nr   rE   r   rc   z/hashT)r-  z_check_hash_worker )r   r  r  r  r  r9   )rD  r%  ri  r  rj  )r  rT   r  u   Форк proc=r  r`  ra  rb  uC   Проверка хеша контента выполнена за z.3fu8    сек. (всего/новых/обновленных: r   r  u(   Ошибок хеша контента: )rH   r  re  rG   ZLockrd  rK   r   r6  r   r   r  r!  rn  r   r-   r   r  r  r  r  r8   r9   ru   r"  r*   r4   r   r   r$  r.   )rC   r   r%  rk  rj  r&  rD  ri  r   r*  r  r+  r   r  Zobj_allZobj_newZobj_updatedrl  r"   r"   r#   _check_hash  sV    



*zConfluenceAPIImport._check_hashc          
      sD   j d g }t }|d}| rZt|d"}t|} fdd|D }W dQ R X t }xt	|D ]x}||d}	|	 rlt|	J}xBt|D ]4}
|
d dkrq|
|
 j  |
|kr||
 qW W dQ R X |	  qlW t|d	}tjt||d
dd W dQ R X ||}x |D ]} | j   q$W dS )us    Собирает пользователей из файла users.json в каталогах документов u=   Сбор пользователей из документовz
users.jsonr   c                s   h | ]}t | j qS r"   )r   r7   )r\   u)rC   r"   r#   	<setcomp>6  s    z5ConfluenceAPIImport._collect_users.<locals>.<setcomp>Nr   rF   zw+r   F)r   r   )r*   rN   r=   r   r   r   r   r/   r   r   addr7   ru   unlinkr   list
differencer   r   )rC   project_dirr%  Zcurrent_usersZcurrent_user_idsr   r   Znew_user_idsr&   r   rf   Z
diff_usersr   r"   )rC   r#   _collect_users+  s2    


 

z"ConfluenceAPIImport._collect_usersc          	   C   s@   | j  d}t| d| dd}t|}|S Q R X d S )Nprojectsr   z
/info.jsonzr+)r*   r   r   r   r   r/   )rC   project_keyrv  r   projectr"   r"   r#   _get_projectN  s    
z ConfluenceAPIImport._get_projectc          
   c   s   | j  d}xvt|D ]h}|dr,q|drR| j jd| d|  qt| d| dd}t	
|}W d Q R X |V  qW d S )Nrx  z.dirtyz.metau(   Пропускается каталог r   z
/info.jsonzr+)r*   r   r   r   r   r   r4   r   r   r   r/   )rC   rv  ry  r   rz  r"   r"   r#   _get_projectsT  s    

z!ConfluenceAPIImport._get_projectsc       	   	      s  t td fdd jjd }|d d d }|d sDd	S d
d |d D }|sxz D ]n}d|d< g |d< x&|d D ]}|d |dg d qW x&|d D ]}|d |dg d qW || qdW t }x4|D ]*}y |d r|d sh|d d }x(|d D ]}|d |dg d qW x(|d D ]}|d |dg d qHW x|d D ]t}|d rr|	|d  |
dstj d|d d |d< x" ||d D ]}|	| qW qrW W q   jjd| d Y qX qW |_d	S )uJ   Прогружаем все выбранные документы
        )doc_infor(  c             3   s   | d sd S | d s t|d| d }|sPjjd|  dddd	 d S x2|d d
 d D ]}| d |d dg d qbW x2| d D ]&}|d r ||E d H  |d V  qW d S )N	isCheckedr   	documentsri   u'   Документ не загружен zERR-0015r!   r   )r   r   r   rP  rg   T)ri   r~  r   )_get_documentr   r   r*   r   ru   )r}  r(  Zdoc_dumpchild)r   rC   r"   r#   r   c  s    z=ConfluenceAPIImport._load_selected_objs.<locals>.get_childrenimport_settingsrV   rW   r   r~  Nc             S   s   g | ]}|d  r|qS )r~  r"   )r\   setting_objr"   r"   r#   r   w  s    z;ConfluenceAPIImport._load_selected_objs.<locals>.<listcomp>r   T
root_pages)ri   r~  r   archived_pagesr   r   ri   rc   rx  uc   Не удалось вычислить выбранные документы для проекта u4   , скорее всего он не сдамплен)r   r   r_   r*   json_settingsr|  ru   r=   r{  rr  r.   r   r   r4   r   r>   )	rC   r[   rootselected_spacesrz  r&   resultr   r}  r"   )r   rC   r#   _load_selected_objs`  sF    


& z'ConfluenceAPIImport._load_selected_objsc          
   C   s  |  | jjd }|d d d }|d s.dS dd |d	 D | _|d rpt|d	 dkrpd
d | j D | _| j d}|j	dd x*| jD ]}t
|d d }| jj|}y| j rd S | j jd7  _||}|d}d }	d }
| r6t|d$}t|}|d}	|d}
W d Q R X d }tjd k	r~|
d ksr|
dkr~tjdksr|
tjk r~d| _tj}|p|
pd| _| jjd|d   | st
|| d|d< tj|d dd g |d< x,| j|d D ]}|d |d  qW |d s$tdg |d< x2| jj|d dddD ]}|d |d  qDW | szt !|d | t
||d< t|d$}|	|d< | j|d< t"|| W d Q R X t#  | jjd  | $| t%|d  d!}|j	dd | jjd"|  t
t&' ( ) }| j*|||	d#}|s^||d< t|d}t"|| W d Q R X |  j+| ,||7  _+| -|| W q   | jj./  | jj0d$| d%d&d'd( Y qX qW d S ))Nr  rV   rW   r   r~  Tc             S   s   g | ]}|d  r|qS )r~  r"   )r\   r  r"   r"   r#   r     s    z4ConfluenceAPIImport._dump_spaces.<locals>.<listcomp>r   c             S   s   g | ]}||d  dqS )ri   )r   ri   r"   )r\   r   r"   r"   r#   r     s    rx  )r-  r   r   rE   z	info.jsonr   Z	dump_daterB   r   u&   Дампим пространство rT   z.dirtyrc   r  ri   uJ   Пустой проект, возможно нет прав доступаr  Zarchived)r  zw+u=   Дампим права доступа (Разрешения)z
/documentsuD   Проверка хеша контента пространства )rk  u1   Ошибка дампа пространства zERR-0026rW  r   )r   r   r   )1r_   r*   r  r  r   r-   Zget_all_spacesr   r   r6  r   r8   Z	get_spacer  r;   r   r   r   r/   r.   r@   rA   r?   rB   r4   r   r   makedirsZget_root_pages_from_spaceru   CmfErrorrf  r   r   r   commit_with_eventr^  r   r   ZnowZ
astimezoneZdatero  r1   rE  rw  r}   r   r   )rC   r[   r  Zprojects_dirr  rD  r   rv  r   Zlast_dump_dateZprev_dump_last_page_verr   Zcurrent_dumpZnew_dump_last_page_verr   Zarchived_docr%  Znew_dump_dateZhash_errorsr"   r"   r#   _dump_spaces  s    












z ConfluenceAPIImport._dump_spacesc             C   s  | j  d}| rd S |jdd xr| j D ]b}| jjjrTt	|d }nt	|d 
dd
dd}||}|jdd |d	}| j  jd
7  _t|d}t|| W d Q R X |d}| j  d}|jdd |jdd x| j|d D ]}	||	| j d }
| j  jd
7  _t|
d}t|	| W d Q R X ||	| j d }
|
 rrqt|
d}t|	| W d Q R X qW q6W d S )NrV  T)r-  ri   rT   r   r   r   z	info.jsonrE   zw+r   z.json)r*   r   r   r   r6  r-   Zget_all_groupsr8   r9   r   r   r;   r   r   r   Zget_group_membersr7   )rC   
groups_dirrh   rj   Z	group_dirZ	data_filer   Z	users_dirZall_users_dirrf   Zuser_data_filer"   r"   r#   _dump_groups  s8    




z ConfluenceAPIImport._dump_groupsc             C   s   | j  }t|\}}}|| d }| j jd| d | j jd | j jjddr|| j jd | 	| j
jd n| j jd	tjd
 |   d}x| dD ]}|d7 }qW | j jd|d d S )Nd   u   Свободно места: rl   u$   Скачаем данные из API
load_usersTu'   Дампим пользователейr   uA   Не дампим пользователей из-за опций)r5   r   rE   rf   )rW   )r*   r   r   Z
disk_usager4   r   r  rW   r.   r   r-   Zget_all_usersrN   r3   r  r  r   r   )rC   Zdownload_pathZtotalZusedZfreeZfree_percentZuser_cntr   r"   r"   r#   download_data"  s    
z!ConfluenceAPIImport.download_data)r  c             C   s  dd l }dd l}|| _d}tdd x| j r8P | }|dkrJP | jjd|d   d }xnt	dD ]`}	y| jj
|d tjdd	}|  t|d
 d&}
x|jddD ]}|
| qW W d Q R X ||d
 \}}|r|dkr||d
 }W d Q R X | jjd|d
   tjj|d d}d|_|  |j  | jd P W d Q R X W qr tk
r } zD| jj  |	dkr| jjd|d  d| ddd|d d W d d }~X Y qrX qrW q,W || W d Q R X d S )Nr   F)r  DONEu    Пытаемся скачать r   r
  T)Ztimeoutstreamrc   zwb+i    )Z
chunk_sizezapplication/zipu   Сохранили в ri   )ri   rT  r   u@   Не удалось скачать файл с 3 попыток z: zERR-0027r!   r   )r   r   r   Zext_href)zipfiler   r-   r   r*   r  r.   r4   r   r  Zconfluence_requestr@   ZIMPORT_DOWNLOAD_TIMEOUTZraise_for_statusr   Ziter_contentrb   r   ZipFiler   r   r   r|   r}   r~   r   rQ   r   r   r  )rC   r  download_queue
done_queuer  r   r  r   resr  r   chunkZmimetyperh  myzipZdownload_objr   r"   r"   r#   _download_file_worker7  sL    



,z)ConfluenceAPIImport._download_file_workerc       	      C   s  | j jd | j }| j }d}| j dg dd| j jjjgdddgg}tj	j
|dsp| j jd	 |S g }xt| jD ]v}| jj| jd
| ||t| jj| jj| jj| jj| jj| jjjddd}|| |  | j jd|  qW x6tj	j|ddgdD ]}||j|j|jd qW x|D ]}|d q8W |  snx|D ]}|!  q\W | j jd | j " rt#  | j j$d x|  s||% 7 }qW | j jd|  |S )Nu=   Скачиваем все найденные вложенияr   zplugin.plugin.source_hashrw   z==r   F)rm   u   Вложений нетr  )r   r  r  r  r  r9   )r  r  r  )r  rT   r  u    Запустили задачу r   rc   )rm   rU   )ri   r   rc   r  u%   Закончили скачиватьu   Импорт прерванuC   Закончили скачивание файлов, ошибок )&r*   r4   r   rH   r  load_fieldsrv   rw   r   r   countr  rK   rG   r!  r  r   r-   r   r  r  r  r  r8   r9   ru   r"  slistr  ri   r   rc   r$  r   r  r  r   r.   )	rC   r  r  r1   r   Zprocsr  r+  r   r"   r"   r#   download_filesa  sD    





z"ConfluenceAPIImport.download_filesc          	   c   s>   | d}t |d }x|D ]}t|V  qW W d Q R X d S )Nz/comments.jsonzr+)r   r   r   )rC   rc   comments_file_pathr   r   r"   r"   r#   _get_comments  s    

z!ConfluenceAPIImport._get_commentsc          	   c   s>   | d}t |d }x|D ]}t|V  qW W d Q R X d S )Nz/restrictions.jsonzr+)r   r   r   )rC   rc   r  r   r   r"   r"   r#   rs     s    

z%ConfluenceAPIImport._get_restrictions)r   c                s:   x4 d D ]"}d fdd	  |d qW d S )Nrc   c          
      s  zΐy| d dkr,j jd|   d S j jjj d| d  }tjj|dgd}|rf|jsfd S |s~tj|j d}n|	d	g | d
 dr| d
 d d dkrȈ
| d
 d j |_nJ| d
 dr
| d
 d d dkr

| d
 d j |_ntj|_| d
 d |_| d
 drb| d
 d d dkrb
| d
 d j |_n|j|_| d
 dr| d
 d d d dkrƈ
| d
 d d j |_| d
 d d |_n|j|_|j|_|| d d d | d d d |_|jj| d d d |_| |_||_| d d dkrBd|_| d d d dkr^d|_|jdd x| d D ]} || qtW |j  |S    |j  j jd d!  d"|  d#d$ Y nX W d j  d%d& X d S )'Nr  ZdraftuJ   Не обрабатываем черновики комментариев z::ri   ry   )rj   rU   )rq   rj   r*   z**r.  r1  r   rF   r/  createdDater0  r   whenr   r4  rW   editor
extensionslocationinlineTZ
resolutionZresolvedclosed)rp   r   u7   Ошибка обработки комментария r   z: zERR-0028)r   r   r5  	processed)!r*   r4   r   rv   rw   r   
CmfCommentr.   ry   r  rt   r7   	cmf_ownerrY   anonymous_usercmf_created_at
cmf_authorcmf_modified_bycmf_modified_at_process_mentionsr   _process_linksrW   import_raw_jsontree_parentr  r  r|   r}   r~   r   r   r   )r5  r  rj   Zcmf_commentr  )rQ  r   rq   rC   r"   r#   rQ    sf    


z9ConfluenceAPIImport._process_obj_comments.<locals>.create)r5  )N)r  )rC   rq   r   r5  r"   )rQ  r   rq   rC   r#   _process_obj_comments  s    >z)ConfluenceAPIImport._process_obj_commentsc             c   s8   | d}x(t |D ]}|| d| dV  qW d S )Nz/attachmentsr   )r   r   )r   r   )rC   Zobj_pathZattachments_pathr   r"   r"   r#   _get_attachments  s    
z$ConfluenceAPIImport._get_attachments)r(  c          	   C   sJ   t t|d}t|}W d Q R X x|D ]}|d | kr.|S q.W d S )Nzattachments.jsonr   )r   r   r   r   r/   )r   r(  r   Zattach_info_listattachr"   r"   r#   _get_confluence_attach_info  s
    
z/ConfluenceAPIImport._get_confluence_attach_infoc             C   s  ddl m} dd l}t|d  d}d}| r||<}x2| D ]$}y|drbwN| ||d }	t	j
|}
t	jj|
|dddd	gd
}|st	j| j|d|
d}|jr<||t}|jdd |j| dd |jdd |	r(|j|	d d ks(td|j d|	d d  d|j  W d Q R X W qN   |j  | jjd| d| d|d Y qNX qNW W d Q R X | jdd |S )Nr   )RDiskrc   z /attachments/all_attachments.zipr   ry   r   url_preview_imgurl_preview)rT   rq   rU   T)r*   rq   ry   rT   )rp   )Zsmart_backupr  ZfileSizeu   Текущий размер z != u    из метыuD   Не удалось обработать вложение file_name=z all_attach_zip_path=zERR-0029)r   r   rT  r  )Zrdisk.rdiskr  r  r   r   r  namelistr   r  r   r!   r   CmfAttachmentr.   r*   ry   r   r|   upload_filera   st_sizeAssertionErrorr}   r~   r   r   r   )rC   rq   r   r  r  Zall_attach_zip_pathr  r  Zzip_attach_namer;  Zattach_namerT  Zattach_fr"   r"   r#   _process_obj_attachments  sB    


4
z,ConfluenceAPIImport._process_obj_attachmentsc       	      C   s  | d}|d d }| jjjryH| jjd t|d}t	|d d }t
| j||d ||d}W nZ tk
r   d	| d
 d}| dr|d|d d  d7 }| jj|d|d Y nX | |||d d }| |||d d }| ||}| |||d d }| |||d d }| j|||d d |d}| |||d d }| |||d d }| |||d d }| |||d d }| |||d d }| j|||d d |d}| |||d d }| |||d d }|S )Nr   r4  rW   uC   Конвертация текста из atlassian разметкиzhtml.parserZatlas_doc_formatr   )soupsubobjuT   Не удалось конвертировать разметку страницы "r   r  r   u    (версия r   r  zERR-0031)r   r   r  )r  )r.   r-   r8   r9   r*   r4   r   r   r   r   r   _convert_atlas_docrQ   r   r  r  _process_inline_comments_convert_plantuml_convert_drawio_convert_msoviewer_convert_childpages_convert_excerpt_convert_excerpt_include_convert_include_convert_content_by_label_convert_details_convert_details_summary_convert_anchor)	rC   r   r   r  Z	body_datar   r  	atlas_rawZexception_textr"   r"   r#   _convert_doc_text  s8    



z%ConfluenceAPIImport._convert_doc_textc          	   c   sJ   t | d}| rFt|d }x|D ]}t|V  q(W W d Q R X d S )Nz/versions.jsonzr+)r   r   r   r   r   )rC   rc   Zversions_file_pathr   r   r"   r"   r#   _get_versions&  s    
z!ConfluenceAPIImport._get_versions)r   c       
         s    fdd}x$t jjddgdD ]}|  q&W t jj  d }xBt d D ],\}}|tj	krnP ||d}|rZ|j
rZ|}qZW t jdd	d
d}	|	jd	d |	j  |_jrԈjd	d j  d S )Nc          
      s  d }z yj jd d  d d  d| d   j jjj d d  d| d  }tjj|d	gd
}|r||js|d S |stj|j d}| d|_	| d|_
| dr| d d dkr| d j }||_||_||_ntj|_tj|_tj|_| d|_| d|_| d|_t|jjdd d |_|j|_jjdd|_| di }t|ddk|_j||d|_ d|_!| |_"|j#dd |j$%  W nP t&k
r   j j'd d  d d  d|  dd  j(d7  _(Y nX W d j )d d! X |S )"Nu7   Обрабатываем версию страницы r   z (ri   z): version r   z::r   z**)rj   rU   )rq   rj   r*   r  r   r   rF   messagez%Y%m%d%H%M%S%fCLOSED)status_typer   r  r>  )r  T)rp   u@   Ошибка обработки истории страницы z): zERR-0033)r   rE   r   r  )*r*   r4   r   rv   rw   r   CmfDocumentHistoryr.   ry   r  r  rt   r7   r  r  r  rY   r  reasonofficial_numberZofficial_daterP   rW   ZstrftimeZcmf_versionZdoc_versionworkflowget_default_statusr  r8  cur_workflowr  r   Zhas_publishedr  r|   r}   r~   rQ   r   r1   r   )version_dataZcmf_document_historyrj   r   Zversion_content)r   r   rC   r"   r#   create_version/  sX    $$


"z=ConfluenceAPIImport._process_versions.<locals>.create_versionrq   =)rm   rc   )r  FTr   )rq   r  Zcmf_deletedr   )rp   )r   r  rt  rR  r}   r~   	enumerater  r@   $IMPORT_CONFLUENCE_PROC_LAST_PAGE_VERr  r|   cur_workflow_versionr{   )
rC   r   r   r  Zdhr   version_countr   Zdocument_historyZfake_historyr"   )r   r   rC   r#   _process_versions.  s$    D



z%ConfluenceAPIImport._process_versionsc          
   C   s   |  |d}| sd S t|d}t|}g |d< |  |d}| rt|d(}x |D ]}|d t| q\W W d Q R X |  |d}| rt|d&}	t|	}
d|
t|
di|d< W d Q R X |S Q R X d S )	Nz	info.jsonr   labelszlabels.jsonz
pages.jsonrP  )rg   sizer   )r   r   r   r   r/   ru   r   r   )r(  r&   Zdoc_filer   r  Zlabels_filef2Z	label_strr   Zf3r   r"   r"   r#   r    s"    

"
 z!ConfluenceAPIImport._get_documentc             C   s$   t |d }tdd |}|S )Nr  c             S   s   |   o| jd S )N)z.dirtyz.trashed)r  rT   r   )	path_objsr"   r"   r#   r     r   z3ConfluenceAPIImport._get_doc_dirs.<locals>.<lambda>)r   r   r  rm   )rC   
space_pathr  Zdoc_dirsr"   r"   r#   _get_doc_dirs  s    z!ConfluenceAPIImport._get_doc_dirsr     c             c   s^   t | d}d}xF| |D ]8}|d7 }||k r4q||| d krFP | ||jV  qW d S )Nz
/documentsr   rE   )r   r  r  rT   )rC   rv  offsetrc  r(  enum_dirr&   r"   r"   r#   _get_documents  s    z"ConfluenceAPIImport._get_documentsc             C   s   t dd | |D S )Nc             s   s   | ]
}d V  qdS )rE   Nr"   )r\   r   r"   r"   r#   r     s    z=ConfluenceAPIImport._count_space_documents.<locals>.<genexpr>)sumr  )rC   r  r"   r"   r#   _count_space_documents  s    z*ConfluenceAPIImport._count_space_documentsc       	      C   s   i }d}d}d}xn|  |D ]`}|d7 }||kr8|}d}||d ||  d7  < tdd | t|D }||d 7 }qW dd | D }|S )Nr   r   rE   c             s   s   | ]
}d V  qdS )rE   Nr"   )r\   r   r"   r"   r#   r     s    zCConfluenceAPIImport._get_offsets_for_execute_doc.<locals>.<genexpr>c             S   s   g | ]\}}||fqS r"   r"   )r\   r  rc  r"   r"   r#   r     s    zDConfluenceAPIImport._get_offsets_for_execute_doc.<locals>.<listcomp>)r  rY  r  r  r   r   )	rC   r  rc  offsetsZcurrent_offsetZ	obj_countr  r(  r  r"   r"   r#   _get_offsets_for_execute_doc  s    z0ConfluenceAPIImport._get_offsets_for_execute_docc       
      C   sv  d}t dd\ t F x2| }|dkr4P | j r@P yd|d  d|d  d	}tjd
krld}ndtj }| jj	d| d|  | jj
j
j d|d  }tjj|dddgd}|std|jr| || n| jj	d|  W q" tk
rN }	 z6tjj  | jjd| d|	 ddd |d7 }W d d }	~	X Y q"X q"W || W d Q R X W d Q R X d S )Nr   F)r  r  r  r   z" (ri   r  r   u   Все версии.u   Последние: u3   Обработка версий документа z. z::r  r  rj   )rj   rU   u4   Документ не найден в системеuQ   Документ изменен пользователем, пропускаем u@   Ошибка обработки версий документа zERR-0084r!   )r   r   rE   )r   cmfutildisable_aclr.   r*   r  r@   r  r4   r   rv   rw   r   r!   rQ   ry   r  r}   r   r   r  )
rC   	obj_queuer  r  r   r   r   rj   r   r   r"   r"   r#   _process_doc_versions_fork  s>    

z.ConfluenceAPIImport._process_doc_versions_forkc             C   sZ  d}t dd@ t * x| }|dkr4P | j r@P zy| jjjj d|d  }| jd|  t	j
j|d}|st	j
| jd	||d
 d}|jr4|js|ddg |d d }	| jjd|	  |	d dkrtj}
n| |	| j }
|
s"| jjd|d
  d|	  |d d}|rr| jjd|  |d dkr`tj}n| || j }n|
}|d d d }| jjd|  |d dkrtj}n| || j }|d d d }||d
  ||
| j| jjt|d d|d d |d d d  |||d	d!}|jr.| j|_|d jd"krF||d#< x|D ]}t||||  qLW |jr|jd	d$ |j   | jjjj d%| }t	j!j|d}|st	j!|| jd&}||_"|d' d(rt#j$%| j&j'|d' d( }n| jjjj(}|j) d)| d*| d+|_*d,|_+|jd	d$ |j   n| jjd-|  || ,||7 }| -|| |jr g }x||d. D ]p}d| jjjj d|d  d}t	j.jd/d0d1| d1gd2}|r|/| n| jj0d3| d4| d5|d6 qzW ||_1| 2|||_*|jr |jd	d$ |j   | 3|| W n t4k
r   t	j
j5  | jj0d7|d
  d8d9d:d; | j6  |d<7 }Y n t7k
r   t	j
j5  | jj0d7|d
  d=| d>d? d9d:d; | j6  |d<7 }Y nF   t	j
j5  | jj0d7|d
  d9d:d; | j6  |d<7 }Y nX W d | j8d@dA X q"W |9| W d Q R X W d Q R X d S )BNr   F)r  r  z::ri   zprocess )rj   Tr   )r*   ry   rj   rT   z**tagsr.  r/  u5   Определяем автора документа r   rF   u1   Не найден автор документа: z: r1  u:   Определяем владельца документаr0  r   u=   Определяем последнего редактора r   r4  rW   r  )r  r  r  )rq   rT   r  r  r  r  r  r  rj   r  r  Zfull_screenr  r  )rp   z
::comment:)rj   r*   Z_linksZwebuiu4    импортирован из Confluence: <a href="z" target="_blank">z</a>r   uQ   Документ изменен пользователем, пропускаем r  rj   rk   rl   )rm   u   Не найден тег z ext_id=zERR-0017)r   r   u   Ошибка создания uS   : попробуйте очистить дамп и повторить импортzERR-0018r!   )r   r   rE   u^   : попробуйте отменить импорты связанные с проектом z, uW   чтобы он удалился из системы и повторите импортr%   r  ):r   r  r  r.   r*   r  rv   rw   rN   r   r!   ry   is_newr  r4   r   rY   r  rt   r7   r   r0   r  r   Zget_status_typeroot_parentr  r  setattrr{   r|   r}   r~   r  rq   r   rc   r   r-   r   ext_urlverbose_namer   	log_levelr  r  CmfTagru   r   r  r  r   FileNotFoundErrorr   
log_detailAttributeErrorr   r  )rC   rz  Zarchive_folderr  r  r  r   rj   r   authorr  r:  r  modifiedr  r   Zdoc_dictr   Zcomment_ext_idr5  r   r  Ztag_rawZ
tag_ext_idr   r"   r"   r#   _process_document_fork  s    










z*ConfluenceAPIImport._process_document_forkc             C   s  d}yt |d}x|jdddidD ]}td|jd }tj|d	}tj	j
||d
}	|	s| jjd| d| d|d q(tj	j
dd|gdd| dggdgd}
d}td|jd }|r|d	}d}td|jd }|r|d	}|	jj|	jj||d}|
r|
j|d< nd|d< t |ddjf |d}t |d}xF|jddidD ]2}|jdd|jd   idr\||j P q\W dd!d" |jjD }q(W |S    | jjd#| d|d Y nX d S )$Nu  
        <div data-drawio-title="{name}" data-drawio-type="diagram" data-attach-id="{attach_id}" class="drawio-content" contenteditable="false">
            <div class="header">Диаграмма: "{name}"</div>
            <div class="content">
                <img cmf_converted=True src="{img_src}" alt="" data-mce-src="{img_src}" width="{width}" height="{height}">
            </div>
        </div>
        r   imgzdata-macro-namedrawio)r   zdiagramName=([^|]+)zdata-macro-parametersrE   )rT   rq   u#   Диаграмма с именем u@    не найдена во вложениях документа zERR-0019)r   r   rq   z==rT   rk   z%.pngr   )rm   rU   z100%zdiagramWidth=([^|]+)zdiagramHeight=([^|]+))rT   	attach_idwidthheightimg_srcr   r   zhtml.parserri   zdrawio-macro-content-zdata-macro-idc             S   s   g | ]}t |qS r"   )r   )r\   xr"   r"   r#   r     s    z7ConfluenceAPIImport._convert_drawio.<locals>.<listcomp>uU   Неудалось конвертировать диаграмму из текста )r   r   researchr   r   r!   r   rh   r  r.   r*   r   rT   rW   ri   r   r   formatfindreplace_with
currentTagr   r   r   )rC   r   r   raw_dataTAG_TMPLZsoup_rawr   matchr   r  
attach_imgr  r  r)  r]   r  Z
drawio_tagr"   r"   r#   r  }  sN    




z#ConfluenceAPIImport._convert_drawioc          	   C   s   d}|s|S t |d}t |d}x|jddidD ]}ytj|j}W n*   d}| jjd|j d|d	 Y nX t |jd
|j|dd}	x$|jddidD ]}
|
	|	 P qW t t
|d}q2W ddd |jjD S )Nu  
        <div class="macros-panel plantuml" contenteditable="false" data-macros-parameters="{params}"
            style="display: inline-flex; flex-direction: column; position: relative; width: 100%;"
            data-rendered="true" data-macros="plantuml">
            <div data-layout="default" class="extension-container" contenteditable="false">
                <span role="img" aria-label="Макрос PlantUML">
                    <svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewBox="0 0 24 24" width="16" height="16">
                        <path d="M22,15h2v7h-7v-2h5v-5Zm2-10v6.731l-2-1.976v-.755H7v11h4.738c.033,.036,.057,.078,.092,.113l1.887,1.887H0V5c0-1.654,1.346-3,3-3H21c1.654,0,3,1.346,3,3ZM2,5v2h3v-3H3c-.551,0-1,.448-1,1Zm3,4H2v11h3V9Zm17-2v-2c0-.552-.448-1-1-1H7v3h15Zm-9.715,9.292c-.38,.391-.38,1.025,0,1.416l2.715,2.792v-2.5h2c1.654,0,3-1.346,3-3v-2.023h2.455l-2.756-2.711c-.386-.386-1.012-.386-1.398,0l-2.756,2.711h2.455v2.023c0,.552-.448,1-1,1h-2v-2.5l-2.715,2.792Z"></path>
                    </svg>
                </span>
                <span class="extension-title">
                    <span class="excerpt-title">PlantUML</span>
                </span>
                <span class="macros-actions">
                    <span class="macros-refresh">
                        <svg viewBox="0 0 24 24" style="width: 14px; height: 14px; color: rgb(32, 32, 32);"><path fill="#202020" d="M12,2a10.032,10.032,0,0,1,7.122,3H16a1,1,0,0,0-1,1h0a1,1,0,0,0,1,1h4.143A1.858,1.858,0,0,0,22,5.143V1a1,1,0,0,0-1-1h0a1,1,0,0,0-1,1V3.078A11.981,11.981,0,0,0,.05,10.9a1.007,1.007,0,0,0,1,1.1h0a.982.982,0,0,0,.989-.878A10.014,10.014,0,0,1,12,2Z"></path><path fill="#202020" d="M22.951,12a.982.982,0,0,0-.989.878A9.986,9.986,0,0,1,4.878,19H8a1,1,0,0,0,1-1H9a1,1,0,0,0-1-1H3.857A1.856,1.856,0,0,0,2,18.857V23a1,1,0,0,0,1,1H3a1,1,0,0,0,1-1V20.922A11.981,11.981,0,0,0,23.95,13.1a1.007,1.007,0,0,0-1-1.1Z"></path></svg>
                    </span>
                    <span class="macros-edit" style="margin-right: 7px;">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span class="macros-delete">
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </div>
            <div data-fabric-macro="null" data-macro-body="" class="ak-renderer-wrapper" contenteditable="true">
                <textarea class="puml-code">{uml}</textarea>
            </div>
            <div class="macros-preview" contenteditable="true">
                {svg}
            </div>
        </div>
        r   zdata-macro-nameZplantuml)r   erroru.   Не удалось получить svg из zERR-0002)r   r   z{})paramsZumlsvgzhtml.parserr   c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r     s    z9ConfluenceAPIImport._convert_plantuml.<locals>.<listcomp>)r   r   r   r!   Zconvert_uml2svgr   r*   r   r  r  r   r   r   r   )rC   r   r   r  ZTMPr  	text_soupr   r  new_tagold_tagr"   r"   r#   r    s$    '


z%ConfluenceAPIImport._convert_plantumlc             C   s  d}| dg |j}t|j}| }|  g }xFt|D ]:\}	}
|	| d|	d  d|	d  |
|d}|| q>W ddd	d
dgdddddddgddd}dt|j||d dt|d}|	|
| t|jtj|dd|||d dd}|jS )u8    Создает тег макроса Word или Excel u  
            <div
                class="macros-panel mso-viewer"
                contenteditable="false"
                data-id
                data-macros="msoviewer"
                data-macros-parameters='{params}'
                data-rendered="1"
            >
                <div class="extension-container" contenteditable="false" data-id data-layout="default">
                    <span aria-label="Макрос {plugin_title}" data-id role="img">
                        <svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewbox="0 0 24 24" width="16" height="16">
                            <path d="M22,15h2v7h-7v-2h5v-5Zm2-10v6.731l-2-1.976v-.755H7v11h4.738c.033,.036,.057,.078,.092,.113l1.887,1.887H0V5c0-1.654,1.346-3,3-3H21c1.654,0,3,1.346,3,3ZM2,5v2h3v-3H3c-.551,0-1,.448-1,1Zm3,4H2v11h3V9Zm17-2v-2c0-.552-.448-1-1-1H7v3h15Zm-9.715,9.292c-.38,.391-.38,1.025,0,1.416l2.715,2.792v-2.5h2c1.654,0,3-1.346,3-3v-2.023h2.455l-2.756-2.711c-.386-.386-1.012-.386-1.398,0l-2.756,2.711h2.455v2.023c0,.552-.448,1-1,1h-2v-2.5l-2.715,2.792Z"></path>
                        </svg>
                    </span>
                    <span class="extension-title" data-id>
                        <span class="excerpt-title" data-id>{plugin_title}</span>
                    </span>
                    <span class="macros-actions" data-id>
                        <span class="macros-edit" data-id>
                            <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                                <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                            </svg>
                        </span>
                        <span class="macros-delete" data-id>
                            <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                                <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            </svg>
                        </span>
                    </span>
                </div>
                <div class="macros-preview" data-id>
                    <div class="mso-preview-wrapper" data-id>
                        <img class="mso-preview-image" data-id id="mso-img_{attach_id}_1" src="{url_preview_img}"/>
                    </div>
                </div>
            </div>
        r  r   rE   u   Лист)numri   r   r   ZimgUrlzOffice WordZwordz
.doc,.docxz.docz.docx)pluginTitleZ
pluginTypeZacceptTypesZacceptTypesListZ	pageScalezOffice ExcelZexcelz
.xlsx,.xlsz.xlsxz.xlsr   )viewdocviewxlsr   r   )verparentIdr   ZcurrentPageRefZslicePageNumZ
pagesCountF)r   r   )r  r
  r  Zplugin_titlezhtml.parser)r  ri   r   r  Zget_fullviewsortr  ru   r   rn   r.   r   r  r   r   r  )rC   r   r  Zplugin_typeTMPLr
  r  Zfullview_pagesr   r  Zfullview_pagerP  Zparam_typesr  r  r"   r"   r#   _create_msoviewer_macro_tag  sJ    )

z/ConfluenceAPIImport._create_msoviewer_macro_tagc             C   s<  |s|S t |d}t |d}ddg}x|D ]}x|jd|idD ]}	|	d}
tdd |
d	D }
tj|
d
 }tjj||d}|sd| d| d|j	 d|j
d  d	}t|tjr|d|j 7 }| jj|d|p|d q@|jd|id}|r|| ||| t t|d}q@W q*W ddd |jjD S )u7    Конвертирует макросы Word и Excel r   r!  r"  zdata-macro-name)r   zdata-macro-parametersc             s   s   | ]}| d V  qdS )r  N)r   )r\   paramr"   r"   r#   r   X  s    z9ConfluenceAPIImport._convert_msoviewer.<locals>.<genexpr>|rT   )rq   rT   u*   Ошибка обработки тега "u)   ". Не найдено вложение "u   " документа "z" (ri   r  u    для версии v.zERR-0012)r   r   r   c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r   m  s    z:ConfluenceAPIImport._convert_msoviewer.<locals>.<listcomp>)r   r   r.   r   r   r   r!   r   r  rT   r  r   r  r  r*   r   r  r  r'  r   r   r   r   )rC   r   r   r  r  r  r  Zmacro_types
macro_typer   original_paramsr   r  
error_textr  r"   r"   r#   r  K  s,    



&z&ConfluenceAPIImport._convert_msoviewerc       	      C   s   |s|S t |d}t |d}xn|jddidD ]Z}|d}|s\| jjd| d|  q.|jddd	d}|rz||d
< t t|d}q.W ddd |j	j
D S )u  
        Добавляет параметры настройки макроса из тега в editor разметке в тег во view разметке
        для последующей конвертации макроса при save_import
        r   zdata-macro-namer   )r   zdata-macro-parametersu-   В макросе "Children" объекта "u-   " отсутствуют параметры: F)zdata-macro-namezoriginal-data-macro-parameterszoriginal-data-macro-parametersr   c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r     s    z;ConfluenceAPIImport._convert_childpages.<locals>.<listcomp>)r   r   r.   r*   r4   r   r  r   r   r   r   )	rC   r   r   r  r  r  r   data_macro_parametersZview_tagr"   r"   r#   r  o  s    


z'ConfluenceAPIImport._convert_childpagesc             C   s  d}|s|S t |d}t |d}x|jddidD ]}|d}|d}	tdd	 |	d
D }	|	dd}
|jddid}t |j|
ddd	 |jD dd}|jd|id}|s|jddid}|r|	|j
 t t|d}q2W ddd |jjD S )uH   
        Конвертирует макрос "Выборка"
        u~  
        <div
            class="macros-panel macros-excerpt-block"
            contenteditable="false"
            data-id
            data-macros="exceprt"
            data-param-name="{name}"
        >
            <div class="extension-container" contenteditable="false" data-id data-layout="default">
                <span aria-label="Макрос Include. 'Создать выборку'" data-id role="img">
                    <svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewbox="0 0 24 24" width="14" height="14">
                        <path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path>
                    </svg>
                </span>
                <span class="extension-title" data-id>Макрос Include. 'Создать выборку' <span class="excerpt-title" data-id>| name = {name}</span></span>
                <span class="macros-actions" data-id>
                    <span class="macros-edit" data-id style="margin-right: 7px;">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span class="macros-delete" data-id>
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </div>
            <div class="ak-renderer-wrapper" contenteditable="true" data-fabric-macro="null" data-id data-macro-body="">{content}</div>
        </div>
        r   zdata-macro-nameexcerpt)r   zdata-macro-idzdata-macro-parametersc             s   s   | ]}| d V  qdS )r  N)r   )r\   r(  r"   r"   r#   r     s    z7ConfluenceAPIImport._convert_excerpt.<locals>.<genexpr>r)  rT   r   classzwysiwyg-macro-bodyc             s   s   | ]}t |V  qd S )N)r   )r\   r  r"   r"   r#   r     s    )rT   r   zhtml.parserc             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r     s    z8ConfluenceAPIImport._convert_excerpt.<locals>.<listcomp>)r   r   r.   r   r   r  r  r   r   r  r  r   r   )rC   r   r   r  r&  raw_soupr  r   macro_idmacro_parametersexcerpt_nameZ
macro_bodyr  r  r"   r"   r#   r    s.    $



z$ConfluenceAPIImport._convert_excerptc             C   sF  d}|s|S t |d}t |d}x
|jddidD ]}|d}|d}	d|	krf|	dd	\}
}n0t|tjr|j d
 }
n|j	j d
 }
|	}|d}|rt
dd |dD }|dd}nd}t |j|||
dd}|jd|id}|s
|jddid}|r||j t t|d}q4W ddd |jjD S )u~   
        Конвертирует макрос "Включить выборку"("Вставить фрагмент")
        u  
        <div
            class="macros-panel macros-include-block"
            contenteditable="false"
            data-id
            data-macros="include-excerpt"
            data-param-excerpt="{excerpt_name}"
            data-param-pageid=""
            data-param-pagename="{page_name}"
            data-param-projectkey="{space_key}"
        >
            <div class="extension-container" contenteditable="false" data-id data-layout="default">
                <span aria-label="Макрос Include. " data-id role="img">
                    <svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewbox="0 0 24 24" width="14" height="14">
                        <path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path>
                    </svg>
                </span>
                <span class="extension-title" data-id>Макрос Include. "Вставить выборку" <span class="excerpt-title" data-id>| Имя = {excerpt_name}</span></span>
                <span class="macros-actions" data-id>
                    <span class="macros-edit" data-id style="margin-right: 7px;">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span class="macros-delete" data-id>
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </div>
            <div class="ak-renderer-wrapper" data-id></div>
        </div>
        r   zdata-macro-namezexcerpt-include)r   zdata-macro-idzdata-macro-default-parameter:rE   r   zdata-macro-parametersc             s   s   | ]}| d V  qdS )r  N)r   )r\   r(  r"   r"   r#   r   	  s    z?ConfluenceAPIImport._convert_excerpt_include.<locals>.<genexpr>r)  rT   r   )r3  	page_namerD  zhtml.parserc             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r   #	  s    z@ConfluenceAPIImport._convert_excerpt_include.<locals>.<listcomp>)r   r   r.   r   r   r   rW  r  r/   rq   r   r  r  r  r  r   r   r   r   )rC   r   r   r  r&  r0  r  r   r1  macro_default_parameterrD  r5  r2  r3  r  r  r"   r"   r#   r    s@    '




z,ConfluenceAPIImport._convert_excerpt_includec             C   s  d}|s|S t |d}t |d}x|jddidD ]}|d}|d}	d|	krd|	dd	\}
}n0t|tjr|j d
 }
n|j	j d
 }
|	}t |j
||
dd}|jd|id}|s|jddid}|r||j t t|d}q2W ddd |jjD S )u[   
        Конвертирует макрос "Включить страницу"
        uM  
        <div
            class="macros-panel macros-include-page"
            contenteditable="false"
            data-macros="include-page"
            data-param-pageid=""
            data-param-page-label="{page_label}"
            data-param-projectkey="{space_key}"
            data-rendered="true"
        >
            <div data-layout="default" class="extension-container" contenteditable="false">
                <span role="img" aria-label="Макрос Include. ">
                    <svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewbox="0 0 24 24" width="14" height="14">
                        <path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path>
                    </svg>
                </span>
                <span class="extension-title">Макрос Include. "Вставить страницу" <span class="excerpt-title">| Имя = {page_label}</span></span>
                <span class="macros-actions">
                    <span class="macros-edit" style="margin-right: 7px;">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span class="macros-delete">
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </div>
            <div class="ak-renderer-wrapper"></div>
        </div>
        r   zdata-macro-nameinclude)r   zdata-macro-idzdata-macro-default-parameterr4  rE   r   )
page_labelrD  zhtml.parserr   c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r   p	  s    z8ConfluenceAPIImport._convert_include.<locals>.<listcomp>)r   r   r.   r   r   r   rW  r  r/   rq   r  r  r  r  r   r   r   r   )rC   r   r   r  r&  r0  r  r   r1  r6  rD  r8  r  r  r"   r"   r#   r  %	  s4    &



z$ConfluenceAPIImport._convert_includec             C   s  ddl m} t }t }t| d}y̐x|D ]}|dkrt|}t|}	|dkrh|	dg}
n"|dkrdd	 |	d
dD }
t }t }x|
D  ]}|dddd}||ddd}y`t	j
jdddd| dgdd|gddd| dggddgd}|s$t	j
|d}|jdd W n< tk
rb } ztd| d|  wW d d }~X Y nX |jj|kr||jj ||jj|jj|jjdd qW |r|dd |g ||d< q2|d!kr2t|}t|}	|dkr|	dg}n$|dkrd"d	 |	d
dD }t }t }x|D ]}|d#krB|rB|}n2t	jjd$d%| d&gd'd%d(| dggddgd}|std)| d* q(|jj|kr(||jj ||jj|jj|jjd+d q(W |r2|d,d |g ||d,< q2W W n tk
r
   Y nX ||fS )-Nr   )translitr   r  z\=r  inc             S   s   g | ]}| d qS )r  )strip)r\   r  r"   r"   r#   r   	  s    z5ConfluenceAPIImport._bql_from_cql.<locals>.<listcomp>z(),r   #r   ruT)language_codereversedORaliasILIKEz"%rT   r+   )rm   rU   )rT   )rp   u*   Не удалось создать тег z: r  )rT   r+   ri   Z	classNamer  INr   c             S   s   g | ]}| d qS )r  )r;  )r\   r]   r"   r"   r#   r   	  s    zcurrentSpace()rj   rk   z::%r  z%"key":"u.   Не удалось найти проект «   »rW  rq   )transliterater9  rt  r   iterr   nextr;  r   r   r  r.   r|   rQ   r3   r   ri   rW   ru   rT   r+   rW  StopIteration)cqlrw   rq   r9  Zbql_listbql_dictZ	cql_partsZcql_partoprZ  r  Ztags_idsZ	tags_listr  Zlabel_aliasr   r   r   Z
parent_idsZparent_listr   rz  r"   r"   r#   _bql_from_cqlr	  s    




z!ConfluenceAPIImport._bql_from_cqlc             C   s  d}|s|S t |d}t |d}xz|jddidD ]d}|d}t|d}	y|	d}
|
d	d
d}tdd |
D }| || j	j
j
j\}}ddd|dddkt|dd|dddk|dddkd||dg d
}W n*   | j	jd| d|	 d|d Y nX t |jt|dd d!d"d#d$d%d&d'd(}|jd|id}|sz|jddid}|r||j t t|d}q4W d"d)d |jjD S )*u`   
        Конвертирует макрос "Содержимое по меткам"
        u  
        <div
            class="macros-panel content-by-label"
            contenteditable="false"
            data-id
            data-macros="contentbylabel"
            data-macros-parameters="{parameters}"
        >
            <div class="extension-container" contenteditable="false" data-id data-layout="default">
                <span aria-label="Макрос Содержимое по тегам" data-id role="img"><svg xmlns="http://www.w3.org/2000/svg" viewbox="0 0 24 24" width="14" height="14">
                    <path d="M20.457,4.555,12.486.126a1,1,0,0,0-.972,0L3.543,4.555A3,3,0,0,0,2,7.177V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V7.177A3,3,0,0,0,20.457,4.555ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V7.177A1,1,0,0,1,4.515,6.3L12,2.144,19.486,6.3A1,1,0,0,1,20,7.177Z"></path><circle cx="12" cy="7" r="1.5"></circle></svg>
                </span>
                <span class="extension-title" data-id><span class="excerpt-title" data-id>Содержимое по тегам</span></span>
                <span class="macros-actions" data-id>
                    <span class="macros-action" data-id data-macros-action="macros-edit">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span class="macros-action macros-delete" data-id data-macros-action="macros-delete">
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path><path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path><path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </div>
            <div class="macros-preview" data-id></div>
        </div>
        r   zdata-macro-nameZcontentbylabel)r   zdata-macro-idzdata-macro-parametersr)  r   rJ  r  c             S   s   g | ]}| d qS )r  )r   )r\   pr"   r"   r#   r   	  s    zAConfluenceAPIImport._convert_content_by_label.<locals>.<listcomp>r   defaultreverseF)TtrueTruemax   
showLabelsT	showSpacenoner  )
r#  modelr%  rP  rS  rU  rV  excerptTypebqlr  ui   Не удалось конвертировать макрос «Содержимое по меткам» u    с параметрами: zERR-0034)r   r   'z&quot;r   r   z:Truez:truez:Falsez:false)
parameterszhtml.parserc             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r   
  s    )r   r   r.   r   r   poplstripr   rM  r*   rv   rw   rP   r   r  r   r  r  r  r   r   r   )rC   r   r   r  r&  r0  r  r   r1  r2  all_parametersrJ  r  rZ  rK  r\  r  r  r"   r"   r#   r  	  sT     




z-ConfluenceAPIImport._convert_content_by_labelc             C   s  |s|S t jj}d|  d|  d|  d|  d|  d|  d|  d|  d	|  d
}t|d}t|d}i }	xj|jddidD ]T}
|
d}t|
d}t }y,|r|dkr|d}tdd |D }W n*   | j	j
d| d| d|d Y nX |d|dddkd}| }tj|dddd }d!d"d |
jD }t|j||rf|nd!|d#d$}|jd|id}|s|jddid}|r||j tt|d}t|t jrqi }yt|dd%}|std&|d'd( jd)d*idsbd+d |d'd( jd)d,idD }d-d |d'd. jd)d*idD }tt||}ntd/d |d'D }t|t js||	|< W q tk
r } z,t|t js| j	jd| d0|  W d1d1}~X Y qX qW |	ryj|j  |jjd1kr
i |_|	|jd< |j  |jjd1kr2i |_|	|jd< |jd2d3 |j !  W nR tk
r } z2|j "  | j	j
d| d4|	 d5| d6|d W d1d1}~X Y nX d!d7d |j#j$D S )8u[   
        Конвертирует макрос "Свойства страницы"
        aC  
        <div
            class="macros-panel details"
            contenteditable="false"
            data-macros="details"
            data-macros-parameters="{parameters}"
            data-id="{macro_data_id}"
        >
            <div class="extension-container" contenteditable="false" data-layout="default" data-id="ui   ">
                <span aria-label="Макрос Свойства страницы" role="img" data-id="a  ">
                    <svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewbox="0 0 24 24" width="16" height="16">
                        <path d="M22,15h2v7h-7v-2h5v-5Zm2-10v6.731l-2-1.976v-.755H7v11h4.738c.033,.036,.057,.078,.092,.113l1.887,1.887H0V5c0-1.654,1.346-3,3-3H21c1.654,0,3,1.346,3,3ZM2,5v2h3v-3H3c-.551,0-1,.448-1,1Zm3,4H2v11h3V9Zm17-2v-2c0-.552-.448-1-1-1H7v3h15Zm-9.715,9.292c-.38,.391-.38,1.025,0,1.416l2.715,2.792v-2.5h2c1.654,0,3-1.346,3-3v-2.023h2.455l-2.756-2.711c-.386-.386-1.012-.386-1.398,0l-2.756,2.711h2.455v2.023c0,.552-.448,1-1,1h-2v-2.5l-2.715,2.792Z"></path>
                    </svg>
                </span>
                <span class="extension-title" data-id="z<">
                    <span class="excerpt-title" data-id="uy   ">Свойства страницы</span>
                </span>
                <span class="macros-actions" data-id="zU">
                    <span class="macros-edit" style="margin-right: 7px;" data-id="a  ">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span class="macros-delete" data-id="aJ  ">
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </div>
            <div class="ak-renderer-wrapper" contenteditable="true" data-fabric-macro="null" data-macro-body="" data-id="za">
                {content}
            </div>
            <div class="macros-preview" data-id="z "></div>
        </div>
        r   zdata-macro-namedetails)r   zdata-macro-idzdata-macro-parametersNoner)  c             S   s   g | ]}| d qS )r  )r   )r\   rN  r"   r"   r#   r   P
  s    z8ConfluenceAPIImport._convert_details.<locals>.<listcomp>ud   Не удалось конвертировать макрос «Свойства страницы» u    с параметрами: zERR-0035)r   r   ri   hiddenF)TrQ  rR  )ri   rb  )r   r  z&quot;r   c             S   s   g | ]}t |qS r"   )r   )r\   cr"   r"   r#   r   \
  s    )macro_data_idr\  r   zhtml.parsertableu5   Отсутствует таблица свойств.trr   r/  confluenceTdc             S   s   g | ]}d |j d  qS )zkey_details r   )contents)r\   thr"   r"   r#   r   y
  s    confluenceThc             S   s   g | ]}t |jd  qS )r   )r   rh  )r\   Ztdr"   r"   r#   r   |
  s   rE   c             S   s@   g | ]8}d |j ddidjd  t|j ddidjd fqS )zkey_details r/  rj  )r   r   rg  )r  rh  r   )r\   rf  r"   r"   r#   r   
  s   z: NT)rp   uE   : не удалось сохранить значения macro_db «u   »: zERR-0037c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r   
  s    )%r   r!   get_data_idr   r   r.   r   r   r   r*   r   r   r   r   r   rh  r  r  r  r  r   rW  rQ   zipr  r4   r   Z	macros_dbr/   rW   Zmacros_db_draftr|   r}   r~   r   r   r   )rC   r   r   r  r  data_idr&  r0  r  r`  r   r1  r2  r  r_  r\  rd  prmZcontents_strr  r  Z
propertiesZcontents_tableZ	first_rowZ
second_rowr   r"   r"   r#   r  
  s    &N



0




z$ConfluenceAPIImport._convert_detailsc             C   s  t jj}d|  d|  d|  d|  d|  d|  d|  d|  d	|  d
|  d}|sf|S t|d}t|d}x|jddidD ]l}|d}	t|d}
y|
d}|d	d	d}t
dd |D }| || jjjj|j\}}|dddkdd|dddkdd|dddkt|ddd|d |d!|d"||d#}W n*   | jjd$| d%|
 d&|d' Y nX tj|dd(d)d*}t|j|d+d,}|jd|	id}|s|jddid}|r||j tt|d}qW d-d.d |jjD S )/um   
        Конвертирует макрос "Отчёт по свойствам страницы"
        a6  
        <div
            class="macros-panel details-summary"
            contenteditable="false"
            data-macros="detailssummary"
            data-macros-parameters="{parameters}"
            style="display: inline-flex; flex-direction: column; position: relative; width: 100%;"
            data-id="zp"
        >
            <div class="extension-container" contenteditable="false" data-layout="default" data-id="u{   ">
                <span aria-label="Макрос Отчёт по свойствам страницы" role="img" data-id="a  ">
                    <svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewbox="0 0 24 24" width="16" height="16">
                        <path d="M22,15h2v7h-7v-2h5v-5Zm2-10v6.731l-2-1.976v-.755H7v11h4.738c.033,.036,.057,.078,.092,.113l1.887,1.887H0V5c0-1.654,1.346-3,3-3H21c1.654,0,3,1.346,3,3ZM2,5v2h3v-3H3c-.551,0-1,.448-1,1Zm3,4H2v11h3V9Zm17-2v-2c0-.552-.448-1-1-1H7v3h15Zm-9.715,9.292c-.38,.391-.38,1.025,0,1.416l2.715,2.792v-2.5h2c1.654,0,3-1.346,3-3v-2.023h2.455l-2.756-2.711c-.386-.386-1.012-.386-1.398,0l-2.756,2.711h2.455v2.023c0,.552-.448,1-1,1h-2v-2.5l-2.715,2.792Z"></path>
                    </svg>
                </span>
                <span class="extension-title" data-id="z;>
                    <span class="excerpt-title" data-id="u   ">Отчёт по свойствам страницы</span>
                </span>
                <span class="macros-actions" data-id="zU">
                    <span class="macros-edit" style="margin-right: 7px;" data-id="a  ">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span class="macros-delete" data-id="aJ  ">
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </div>
            <div class="ak-renderer-wrapper" contenteditable="true" data-fabric-macro="null" data-macro-body="" data-id="z:"></div>
            <div class="macros-preview" data-id="z "></div>
        </div>
        r   zdata-macro-nameZdetailssummary)r   zdata-macro-idzdata-macro-parametersr)  r   rJ  r  c             S   s   g | ]}| d qS )r  )r   )r\   rN  r"   r"   r#   r   
  s    z@ConfluenceAPIImport._convert_details_summary.<locals>.<listcomp>showCommentsCountF)TrQ  rR  showLikesCountreverseSortpageSize   NsortByfirstcolumnZheadings)ro  ZshowCreatorZshowLastModifiedrp  ZshowPageLabelsZshowUnresolvedCommentsCountrq  rr  ri   rt  ru  headingZsettingsParamsrZ  uv   Не удалось конвертировать макрос «Отчёт по свойствам страницы» u    с параметрами: zERR-0036)r   r   )r   r  z&quot;)r\  zhtml.parserr   c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r     s    )r   r!   rk  r   r   r.   r   r   r]  r^  r   rM  r*   rv   rw   rq   rP   r   r   r   r   r  r  r  r  r   r   r   )rC   r   r   r  rm  r&  r0  r  r   r1  r2  r_  rJ  r  rZ  rK  r\  rn  r  r  r"   r"   r#   r  
  sX    %V



z,ConfluenceAPIImport._convert_details_summaryc             C   s  |s|S t jj}d|  d|  d|  d|  d|  d|  d|  d|  d	|  d
}t|d}t|d}x|jddidD ]x}|d}	t|d}
t|jtj	
|
dd}|jd|	id}|s|jddid}|r||j tt|d}qW ddd |jjD S )uD   
        Конвертирует макрос "Якорь"
        a:  
        <span
            class="macros-panel anchor-macros"
            contenteditable="false"
            style="display: inline-flex; flex-direction: column; position: relative;"
            data-param-name="{param_name}"
            data-rendered="true"
            data-macros="anchor"
            data-id="z'"
        >
            <span data-id="zl" data-layout="default" class="extension-container" contenteditable="false">
                <span data-id="ur  " role="img" aria-label="Макрос Якорь">
                    <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Capa_1" x="0px" y="0px" viewbox="0 0 204.851 204.851" style="enable-background: new 0 0 204.851 204.851;" xml:space="preserve" width="16" height="16">
                        <path xmlns="http://www.w3.org/2000/svg" style="fill: #010002;" d="M139.518,128.595l16.834,16.336c0,0-20.644,29.877-42.725,30.473 c0.479,0,0.117-84.092,0.039-104.472c14.694-4.797,25.402-18.182,25.402-34.117c0-20.009-16.697-36.218-37.273-36.218 c-20.615,0-37.312,16.209-37.312,36.208c0,15.671,10.376,28.929,24.748,33.961l0.098,104.277 c-26.643-1.837-42.061-27.474-42.061-27.474l17.997-17.41L0,120.505l9.887,63.301l17.362-16.795 c15.036,12.105,32.017,37.244,72.876,37.244c51.332-1.309,63.184-28.939,76.344-39.804l18.993,18.514l9.389-63.907 L139.518,128.595z M82.558,36.208c0-10.298,8.608-18.661,19.218-18.661s19.257,8.363,19.257,18.661 c0,10.327-8.647,18.681-19.257,18.681S82.558,46.535,82.558,36.208z"></path>
                    </svg>
                </span>
                <span data-id="u^   " class="extension-title">Якорь
                    <span class="excerpt-title" data-id="z>">| {param_name}</span></span>
                <span data-id="z=" class="macros-actions">
                    <span data-id="u  " class="anchor-copy" role="img" title="Скопировать ссылку" area-label="Скопировать ссылку">
                        <svg xmlns="http://www.w3.org/2000/svg" viewbox="0 0 24 24" width="14" height="14">
                            <path d="M22,4.145,17.986,0H10A3,3,0,0,0,7,3V4H5A3,3,0,0,0,2,7V24H18V19h4ZM16,22H4V7A1,1,0,0,1,5,6H7V19h9ZM9,17V3a1,1,0,0,1,1-1h6V6h4V17Z"></path>
                        </svg>
                    </span>
                    <span data-id="a  " class="macros-edit" style="margin-right: 7px;">
                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                        </svg>
                    </span>
                    <span data-id="a   " class="macros-delete">
                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                        </svg>
                    </span>
                </span>
            </span>
        </span>
        r   zdata-macro-nameZanchor)r   zdata-macro-idzdata-macro-default-parameter)Z
param_namezhtml.parserr   c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r   I  s    z7ConfluenceAPIImport._convert_anchor.<locals>.<listcomp>)r   r!   rk  r   r   r.   r   r  r   r   quoter  r  r  r   r   r   )rC   r   r   r  rm  r&  r0  r  r   r1  Zmacro_param_namer  r  r"   r"   r#   r    s(    (N


z#ConfluenceAPIImport._convert_anchor)r   c          	   C   s   t |d}xtjjdd|gdddgddd ggddd	gd
D ]}yv|jd d d rx^|jd|jd d d idD ]<}|jd |jd< |j|jd< |jd |j	 d|jd< qrW W q:   | j
jd| d|j d|d Y q:X q:W t|S )Nzhtml.parserrq   z==r  Tr  z!=r   r  )rm   rU   r  ZinlinePropertiesZ	markerRefzdata-ref)r   zdata-ref_origr/  cmf_converteduO   Не удалось конвертировать inline комментарий z: zERR-0020)r   r   )r   r   r  r  r  r   r   ri   ru   r  r*   r   r   r   )rC   r   r   Zdoc_soupr5  r   r"   r"   r#   r  K  s"    

z,ConfluenceAPIImport._process_inline_commentsc             C   s   dd l }|jdddd| jjj d| jj d|j d| d	| d
gddtddtddd}| jd|j d| d| d | j  |S )Nr   z/usr/bin/python3z	manage.pyshellz,plugin = models.CmfPluginConfluence.get(id="z+");retcode = plugin.process_document_fork("z", "z", z, z);Tz"/var/log/eva-import-subprocess.logza+z&/var/log/eva-import-subprocess.err.log)	close_fdsstart_new_sessionstdoutstderru.   Запустили process_document_fork (pid=z offset=z limit=r  )	
subprocessPopenr*   rv   ri   r   rN   pidr  )rC   rz  r  rc  r~  r+  r"   r"   r#   _execute_doc_  s    0"
z ConfluenceAPIImport._execute_docc                s  t  _|d}|s t d d}i }xrj|d |d dD ]R}d }|d rb|d d d }|d d	 }|d
kr~|d }|t|d||d < qBW dd  fdd  ||d d jjdj	  
|d }g }	x>tj	D ]0}
|sP |d\}}|||}|	| qW x|rtd xjt|	D ]^}| }|d k	rD j|7  _|	| |szP |d\}}|||}|	| qDW q*W x2t|	D ]&}|  |jr j|j7  _qW jjd d S )Nhomepage)ri   rc   )rc  Z	ancestorsr   ri   r  positionrW  r   )rq   r  c             S   s   dd t d| D S )Nc             S   s$   g | ]}|  rt|n| qS r"   )isdigitrP   r   )r\   r   r"   r"   r#   r     s    zKConfluenceAPIImport._process_documents.<locals>.natsort.<locals>.<listcomp>z(\d+))r  r   )r]   r"   r"   r#   natsort  s    z7ConfluenceAPIImport._process_documents.<locals>.natsortc                sv   fdd   D }|j fddd |}g }x.|D ]&}|d }|||f  ||}q:W |rr|j< |S )Nc                s    g | ]\}}|d   kr|qS )rq   r"   )r\   r  r   )rq   r"   r#   r     s    zNConfluenceAPIImport._process_documents.<locals>.build_tree.<locals>.<listcomp>c                s    |  d S )Nr  r"   )r  )r)  r  r"   r#   r     r   zLConfluenceAPIImport._process_documents.<locals>.build_tree.<locals>.<lambda>)r   rB  )r   r%  ru   r<   )r)  rq   ordernor   Zcurrent_ordernoZchildren_in_orderr  Znext_orderno)
build_treer  rC   )r)  rq   r#   r    s    

z:ConfluenceAPIImport._process_documents.<locals>.build_treei uJ   Грузим документы в несколько процессов: r   rE   u:   Закончили обработку документов)r   r<   r.   r  r  r   r*   r4   r   rM   r  r  r]  r  ru   r   sleeprt  pollr1   r   communicate
returncode)rC   r   rz  r  r   r   rq   r  r  r*  r   r  rc  r+  r  new_procr"   )r  r  rC   r#   _process_documentsr  sT    

"


z&ConfluenceAPIImport._process_documentsc          	   C   sX  | j jd xB| j D ]2\}}y| j  r4P | j jjj d| }tj	j
|dgd}x|D ]\}}| j jjj d| }tj	j
|ddddgd}|s| j jd	| d
| d|d qb|jrb|j|kr||_|js|j|_||_|jdd qbW |r|jsd|_|jdd t  W q   | j j  | j jd| ddd Y qX qW d S )Nu6   Построение дерева документовz::tree_node_is_branch)rj   rU   rq   r  ry   r  u-   Не найден потомок по ext_id=z tree_parent_ext_id=zERR-0003)r   r   T)rp   uI   Не получилось привязать ноду tree_parent_ext_id=zERR-0024r!   )r   r   )r*   r4   r   r<   r   r  rv   rw   r   r!   r.   r   ry   r  rq   r  r|   r  r  r}   r   )rC   Ztree_parent_ext_idr   r  rj   r  r  r"   r"   r#   _process_documents_tree  s:    



z+ConfluenceAPIImport._process_documents_treec             C   s   dd l }|jdddd| jjj d| jj d|j d| d	| d
gddtddtddd}| jd|j d| d| d | j  |S )Nr   z/usr/bin/python3z	manage.pyry  z,plugin = models.CmfPluginConfluence.get(id="z/");retcode = plugin.process_doc_versions_fork("z", "z", z, z);Tz"/var/log/eva-import-subprocess.logza+z&/var/log/eva-import-subprocess.err.log)rz  r{  r|  r}  u2   Запустили process_doc_versions_fork (pid=z offset=z limit=r  )	r~  r  r*   rv   ri   r   rN   r  r  )rC   rz  r  rc  r~  r+  r"   r"   r#   _execute_doc_versions  s    0"
z)ConfluenceAPIImport._execute_doc_versionsc             C   s$  | j jd| j  | |d }g }x>t| jD ]0}|s>P |d\}}| |||}|| q4W xx|rt	
d xdt|D ]X}| }	|	d k	r|  j|	7  _|| |sP |d\}}| |||}
||
 qW qjW x.t|D ]"}|  |jr|  j|j7  _qW | j jd d S )Nue   Обрабатываем версии документов в несколько процессов: rc   r   rE   uP   Закончили обработку всех версий документов)r*   r4   r   rM   r  r  r]  r  ru   r   r  rt  r  r1   r   r  r  )rC   r   rz  r  r*  r   r  rc  r+  r  r  r"   r"   r#   _process_documents_versions  s4    

z/ConfluenceAPIImport._process_documents_versionsc             C   s:  dddddgdgddd	gd
gdgdgdgdgg dgd
}t |d  d}t|d}t|}W d Q R X | jjjj d|d  }tjj	|d}|stj|d| jd}||_
|d  d|d  |_|jdd dddd d!d"d#d$d%d&d'd(d)d*d+g}|d,d-g }	xdtjj||	d.D ]P}
|
jrg |
_x|D ]}t|
|d/ q*W |
jjd0krTd|
_|
jdd qW x|d1 D ]}y|	|d2 d }|d kr| jjd3|  wrxH|D ]>}tjj	|dd4}|s| jjd5| d6| d7d8d9 |  jd:7  _qtjj	||d;d-gd<}|sN| jjd=| d6| d7|d> |  jd:7  _q|jsZqx\|d? D ]P}| |}|r|j| n,| jjd@| dA| dB|d> |  jd:7  _qdW x|dC D ]}|d }| jjjrd| jjjj d|d  d}nd| jjjj d|d  d}tjj	dDdEdF| dFgdDgdG}|sZtjj	ddH|gdDgdG}|r|j sp||_ n||j kr|j  | |_ |jdd |j| n,| jjdI| dA| dJ|d> |  jd:7  _qW |dK rd|_!|jdd qW W n2   | jjdL| dMdNd9 |  jd:7  _Y nX qrW |S )ONzPPP-DOC-CREATEzPPP-DOC-EDITzPPP-DOC-PUBLISHzPPP-DOC-TREEMOVEzPPP-OBJ-ORDERzPPP-DOC-DELETEzPPP-COM-ADDzPPP-COM-EDIT-OWNzPPP-COM-DELETE-OWNzPPP-COM-DELETE-ALLzPPP-ATT-CREATEzPPP-ATT-DELETE-ALLzPPP-PR-BROWSEzPPP-ATT-DELETE-OWNzPPP-PR-ADMIN)
rF  rG  rH  rI  rJ  rK  rL  rM  rN  rO  rc   z/permissions.jsonr   z::ri   )rj   T)rj   ry   r*   r   u?   : Схема проектных прав доступа для rT   )rp   Zaccess_local_userZaccess_ownerZaccess_owner_assistantaccess_authorZaccess_spectatorsZaccess_executorsZaccess_responsibleZaccess_list_ownerZaccess_project_owner_assistantZaccess_project_spectatorsZaccess_project_executorsZaccess_sdesk_clientaccess_anonymousZaccess_guestZaccess_sharelink_anonymousproject_permissionaccess_members)rq   rU   F)zPPP-DOC-DELETEzPPP-OBJ-DELETErX  rU  u8   Не найдено соответствие права )r+   Zinclude_deletedu    Не найдено право "u	   " для zERR-0059CmfProjectPermPermission)r   rE   r   )r  rq   rU   u2   В схеме не найдено правило ")r   r   u)   Не нашли пользователя u    для прав zERR-0060rV  rj   rk   rl   )rm   rU   rC  u   Не нашли группу zERR-0061rF   u>   Не удалось импортировать правило zERR-0063CmfProjectPermSchemeRule)"r   r   r   r/   r*   rv   rw   r   ZCmfProjectPermSchemer.   r  rT   r|   r  rt  ry   r  r  r  r+   r  r4   r   r  r   r1   rt   ru   r-   r8   r9   rx   rj   r  )rC   r   Zmapping_permr]  r   Z	perm_listZperm_scheme_ext_idZperm_schemeZrule_bool_fieldsZrule_fieldsZruler   Z	perm_dictr[  rU  r\  Zscheme_rulerf   r   rh   Z
group_namerj   Z	group_objr"   r"   r#   _process_permissions  s    




 


z(ConfluenceAPIImport._process_permissionsc             C   sd  |  | jjd }|d d d dkr<dtjjddd}n`|d d d d	krfdtjjd
dd}n6|d d d dkrdddtjjd
dd}ntd|   x|  D ]}yn| j	 rP |d dd | j
D krw| jjd|d   |d |d< |d dkr,|d  d|d< d|d< | j|d |d< | jjjj d|d  }||d< d}|d d d dkr"tjjdd|d gd dd!|d gdd!|d gggddgd"}|r| jjd#|d  d$|j d%|j d& ||jkrd'}|j d| |_|jd'd( |j  ntjjdd!|d gddgd"}|r|jj|d jksr|jr|j|d kr|d  d)|d  d&|d< | jjd*|j d+|j d,|d  d-|j d.|d  
 n(|js|d |_|jd'd( |j  d}tjj|d | d/}|s:d'}tjjdd0d1| d1g| d"}|sbd}d'}tj| j|d |d d2}t|d3 d4}d'}	d }
|d5r|d5 d }
| ||
}n`| jjd6|  d}	xF|d7 D ]:}y| ||}P W n    | jd8| d9 Y nX qW |s"| jjd:| d;|d< w|d= d> }| jjd?|  |d d@kr\tj |dA< n| !|| j" |dA< |d= dB}|r| jjdC|  |d d@krtj |dD< n| !|| j" |dD< n|dA |dD< |d= dE dF }| jjdG|  |d d@krtj |dH< n| !|| j" |dH< |d= dI |dJ< |d= dE dK |dL< |j#sZ|j$r|s| %||_&x|D ]}t'||||  qrW |  |j  |jj(dkr<tj)jdMd|jgdddNggdO}|j#r4| jjjj d|
 }||_|dA |_*|dD |_+|dH |_,|dJ |_-|dL |_.||_/|jd'd( || _0n(|j#sL|j$r^||_/|jd'd( || _0|	r| 1| j0| | 2| j0| | j0j#s| j0j$r| 3|| j0| j0_4| j0jd'd( | j0j  | 5|| j0 || _6|rtj)jdPdMd|jgdddNggdO}|d d d dkr8tj)j|dQdR}|r|7  nv|d d d d	ksd|d d d dkrtj)j|dQdR}tj)j|dSdR}|r|7  |r|7  |r|7  t8  | 9|| | :|| | ;  t<j=rt<j=}t<j=dTkrdU}| jjdV| dW | >|| n| jjdX W q   | jj?  | jjdY|d  dZd[d\ Y qX qW d S )]Nr  rX   rW   ri   rz  zproject.agile:default)r+   )project_type
logic_typeZproject_wikizproject.wiki:defaultr   sectionzCmfWiki:Class)r  	parent_idtree_parent_idr  u   Неизвестный типc             S   s   g | ]}|d  qS )ri   r"   )r\   r   r"   r"   r#   r     s    z7ConfluenceAPIImport._process_spaces.<locals>.<listcomp>u0   Импортируем пространство r   rT   r   Zpersonalu(    (Личное пространство)u%   Личное пространствоdescriptionZtask_code_prefixz::rj   Fr  z==rA  rC  )rm   rU   u$   Импортируем проект u<    в раздел "Документы" Agile проекта "z" (r  T)rp   z (u   Проект с именем u    уже существует! z != z id=u   Переименуем в )rj   rU   rk   rl   )r*   rj   rT   rc   r  r  uC   У пространсва нет домашней страницы r  uW   Не удалось получить данные корневой страницы id=u   , возможно ее нет.uD   Не найдена домашняя страница в дампе zERR-0021)r   r   r.  r/  uD   Определяем автора домашней страницы rF   r  r1  uJ   Определяем владельца домашней страницы r  r0  r   u=   Определяем последнего редактора r  r  r  r  r  r  u   Документы)rm   ZANDu   Документ1)r  rT   u   Документ2r   u   ВсеuY   У документов будет обработано последних версий: u   . Для изменения, поменяйте значение параметра IMPORT_CONFLUENCE_PROC_LAST_PAGE_VER в config.u   Обработка версий документов отключена параметром IMPORT_CONFLUENCE_PROC_LAST_PAGE_VER в config.u5   Ошибка импорта пространства zERR-0032rW  )r   r   )@r_   r*   r  r   ZCmfLogicTyper.   r  r  r|  r  r  r4   r   Zget_correct_keyrv   rw   rW  rT   rj   r|   r}   r~   r  ri   r   keysr   r   r  rN   r   rY   r  rt   r7   ry   r  r  Zproject_perm_schemer  r+   r!   r  r  r  r  r  r  r  r  r  r  r   r   rz  rR  r  _process_project_tagsr  r  r@   r  r  r   )rC   r[   Zproject_fieldsr   rj   Zproject_is_newrz  Zproject_mergedr(  Zhomepage_originalZhomepage_idr  Z	root_pager  r:  r  r   Z
doc_folderZhomepage_ext_idZrm_doc_folderZrm_doc1Zrm_doc2r   r"   r"   r#   _process_spaces  sV   
"<

 










z#ConfluenceAPIImport._process_spaces)r   c          	   C   s  ddl m} t }x\| j|d | |d dD ]<}| j rBd S x(|d D ]}|d |krL|||d < qLW q0W xr| D ]d}| j rd S |d dd	d
d}||ddd}yd| jj	j	j
 d|d  d}	tjjddd|	 dgdgd}
|
s<dddd| dgdd|gddd| dgg}tjj|dgd}
|
sVtj|| jd|	d}
|
jsf|	|
_n|	|
jkr|
j |	 |
_||
_||
_|
jr|
jdd |
j  W q|   | jj  | jjd| d|d w|Y q|X q|W d S )Nr   )r9  rc   )rc  r  ri   rT   r   r   r=  r   r>  T)r?  r@  z::rj   rk   rl   r   )rm   rU   rA  rB  rC  r  z"%)rT   r*   ry   rj   )rp   u*   Не удалось создать тег zERR-0022)r   r   )rF  r9  r   r  r  r*   r  rZ  r   rv   rw   r   r  r.   rj   rT   r  r{   r|   r}   r~   r   r   )rC   r   rz  r9  r  r   r   Ztag_nameZ
name_aliasrj   Ztag_objr   r"   r"   r#   r    sL    "

 
z)ConfluenceAPIImport._process_project_tagsc          
   c   sR   | j  d}x<t|D ].}t||dd}t|V  W d Q R X qW d S )NrV  z	info.jsonr   )r*   r   r   r   r   r   r   r/   )rC   r  Zgroup_idr   r"   r"   r#   _get_groups  s    zConfluenceAPIImport._get_groupsc          
   c   sJ   | d}x:t|D ],}t| |d}t|V  W d Q R X qW d S )Nr   r   )r   r   r   r   r   r/   )rC   Z
group_pathZ	user_pathr   r   r"   r"   r#   _get_group_members  s    
z&ConfluenceAPIImport._get_group_membersT)r   rR   c          	   C   sX  | d}| jjd|  t|d t| dd d}|s| jjd|  tjj	j
ttj d }t|d d |  d	tj }t|d ||t| dd d
}d| jjjj d|| j  d}d}|d d}	t|	dkr|	d nd|d< t|	dkr|	d n|	d |d< ddg}
tjj dd|d g|
d}|stjj ddd| dgdgd}|stjjdd|d g|
d}t|dkr| jjd|d  dt| ddd |  jd7  _g }|sftjjdd |d gd!}|sft|	dkrdd |	d g}nHt|	dkrXd"dd |	d gdd |	d ggdd |	d gdd |	d ggg}tjj|d!}t|dkr|  jd7  _| jjd#|d  dt| ddd |r|d }d}|sp|rhd}d}|d }xhtjj dd| g|
dr>| jd$| d%tj |d d	\}}| d&| d	| }|d7 }qW | jd'|  tj||| jdd(}nd |fS |r@|js|jrx|D ]}t||||  qW | ddd)g |j!r||j!kr|j! | |_!n||_!tj"# }tj"$ }||j%kr|j%&| ||j%kr&|j%&| d|_'|j(r@|j)dd* | j j*d7  _*||fS )+NrS   u2   Обрабатываем пользователя ZdisplayNameZ
operationsT)rT   does_not_worku8   Не указан Email для пользователя rE   @)rT   rS   r+   r  z::Fr   r   r   Z
first_name	last_namer   rg_member_ofr  rC  )rm   rU   rj   rk   rl   u+   Пользователей с почтой z > zERR-0004rf   )r   r   rT   z==)rm   rA  u+   Пользователей с именем uQ   Обнаружена учетная запись с дублирующим email u   , добавляем цифруr   u1   Создадим пользователя ext_id=)rj   r  r*   ry   
user_local)rp   )+r.   r*   r4   r   r   r8  warnZcmfrU   ZCmfEmailZ
max_lengthr   r@   
ORG_DOMAINr  Ztranslit_striprv   rw   r7   r   r   r   rt  r   r1   r   rN   r3   r  ry   r  r  r  rj   rx   Zconfluence_group
user_groupr  ru   r  r{   r|   r:   )rC   r   rQ  rn   rS   Z	user_dictZmax_lenrj   
is_createdZ	full_namerU   r   Zpersonsr   nr  Zprefix_emaildomainr   Z
conf_groupr  r"   r"   r#   _process_person  s    
" " 





z#ConfluenceAPIImport._process_personc       	   	   C   s  | j jjdds(| j jdtjd d S t }x| dD ]}| j 	dd | j 
 r^d S | |\}}|jr|jdd |j  |r|jjtjs||jj x|d	g D ]}d
}| jjjrd}d| j jjj d||  d}tjjddd| dgdgd}|s.tjjd
d|d
 gdgd}|sPtj|| j |d
 d|d}n|jrb|d
 |_||jkr~|j | |_|jdd |j  |j | qW |jr<|jdd |j  q<W |r| j jjddr| j j!"dt#| d|  t$tj%j&t'|gd d S )Nr  TuA   Не грузим пользователей из-за опций)r5   r   rf   r  )rp   rV  rT   ri   z::rj   rk   rl   r   )rm   rU   z==)rj   r*   rT   ry   r  send_invitesu,   Отправляем приглашения u    пользователям: )args)(r*   r  rW   r.   rN   r3   r  r=   r   r   r  r  r{   r|   r}   r~   rS   r   r@   r  rr  r-   r8   r9   rv   rw   r   rx   ry   rT   rj   r  ru   r4   r   r   Zschedule_deferred_jobr   Zregister_personsrt  )	rC   r  r   r   r  r   r   rj   rh   r"   r"   r#   _process_persons<  sL    


 

z$ConfluenceAPIImport._process_personsc             C   sV  xN|   D ]@}d}| jjjr$d}d| jjjj d||  d}tjj	ddd| dgdgd}|stjj	dd	|d gdgd}|stj|| j|d d
|d}n|j
r|d |_||jkr|j | |_|jd
d |j  xj| | j d|| D ]J}| |d\}}|sq |j| |jr |jd
d |j  q W qW d S )NrT   ri   z::rj   rk   rl   r   )rm   rU   z==T)rj   r*   rT   ry   r  )rp   rV  F)r  r-   r8   r9   r*   rv   rw   r   rx   r.   ry   rT   rj   r|   r}   r~   r  r   r   r  r  ru   r{   )rC   r   r   rj   rh   r   r   r   r"   r"   r#   _process_groupse  s2    
 


"z#ConfluenceAPIImport._process_groupsc             C   s   xNt dD ]B}|dkr6| jjd| j d| tjd d| _|   | js
P q
W xHt dD ]<}d}|dkr| jjd| d| tjd |  }|sZP qZW |  j|7  _|   | 	  | 
  | j S )u   
        Запуск импорта

        :return bool: Истина в случае успешного импорта
        r
  r   uB   При дампе данных обнаружены ошибки (u0   ), попыка повторного дампа )r5   uF   При дампе вложений обнаружены ошибки ()r  r*   rN   r1   r3   r  r  r  r  r  _process_cross_links)rC   r  r1   r"   r"   r#   process_import  s&     z"ConfluenceAPIImport.process_importc             C   s<   |j dtdids*|jdks*| dr8||  |S )uS   
        Из документов MS создаем наши превью
        styleZ
background)r   r  )r  r  compilerT   r  Zcreate_preview_tag)rC   r   r  r"   r"   r#   _create_preview  s    *z#ConfluenceAPIImport._create_preview)r   r  c       !      C   s  dd l }|s|S t|d}t|d}xx|dd D ]d}zFy |jdrf| jjd|  w6| jjd|  |dr|jd }n
|jd	 }|drN|jd 	d
rN|j
|d}	|	r|	ds| jjd|jd   w6tj|	d d
}
|j d|
 }|jd |jd< ||jd< d|jd< | jjd|  w6d|jddkr|jd dd dd }| |}|r|jd |jd< |j|jd< d|jd< | jjd|j  n| jjd|jd   w6td|}|r|d}|stj|d}|r| |}|rr|jd |jd< |j|jd< d|jd< | jjd|j  n| jjd|jd   w6|j
|d}	|	r|	dr|	ddkr|	d|d< |	d|d< w6td|}|rV|d}tjj| jjjj d| d }|rDd|jd< |jd |jd< |j|jd< nd!|d< ||d< w6|}tj|dd dd }tj|}|jd"}|jd#krV|rV||jdd krVt| jj ! d$ }t| jj ! d% }t"| j# $d&|d'|d(d)}|% sf| jjjj d| }tjj|d*gd+}|rft"| j# $d&t|j&j d$ d'|d(d)}|}|% s| jj'd,| d-| d.| d/| d0	d1|d2 w6n|(|d3}||) kr,|*|^}tj+j||d4}|stj+||| jd5}|j,dd6 |-|.  |j,dd6 |j/0  W d Q R X n | jj'd7| d8| d9|d2 w6W d Q R X tj+j|d:d;d<g|d=}|r4|d	rX|jd	 |jd>< |j1j2|jd	< d|jd< |d?r|jd? |jd@< |jd?= t34|jd	 dd }|r,|d r,|d 	dAr,dA|_5dB|jdB< dC|jdD< dC|jdE< n,|rL|d rX|d 	dFsX| 6|| |dr|jd |jd< |j1j2|jd< d|jd< t34|jd dd \}}|r|	dArdA|_5dB|jdB< dC|jdD< dC|jdE< |7dG} |jd | jd	< || jdH< |8|  n| 6|| | jjd|j1  n,|jddkr`| jj'dI| dJ|d2 w6W n   | jj'dKdL|d2 Y nX W d | jjdM|  X q6W t|S )NNr   zhtml.parserc             S   s   |  dp|  dS )Nr   r   )r   )r   r"   r"   r#   r     r   z4ConfluenceAPIImport._process_links.<locals>.<lambda>rx  u   Уже обработали u&   Обрабатываем ссылку r   r   r=  )r   zdata-anchoru1   Не найден якорь для ссылки z##	href_origTu   Заменили на z/people/r   r   r   r   u?   Пользователь не найден для ссылки z/display/~([^/]+)(?:/)?$zdata-usernamerE   zdata-linked-resource-typerT  zdata-linked-resource-idz$/pages/viewpage.action\?pageId=(\d+)z::)rj   rP  z!data-linked-resource-container-idr!   r   rT   rx  r  r   zall_attachments.zipzparent.import_raw_json)rj   rU   u   Документ ID u    с вложением (u<   ) не найден в дампе пространства "z" (u~   ). Возможно он находится в пространстве, которое еще не импортированоzERR-0005)r   r   r   )rT   rq   )rT   rq   r*   )rp   u	   Файл u    не найден в zERR-0006r   r  r  )rT   rU   rq   Zsrc_origZsrcsetZsrcset_origZvideoZcontrolsz100%r  r  Zimagesourcer   u%   Не найдено вложение zERR-0007u,   Ошибка обработки ссылкиzERR-0008u'   Не обработали ссылку )9r  r   r   r   r.   r*   r4   r   r   r   r  r   r   r   rw  r^  r   r   rt   r  r  r   rh   r   r!   rv   rw   r   
class_namerj   r   r  r  r/   r   r   r   r   rq   r   r  r  r   r  r|   r  ra   r}   r~   r   rW   r   r   rT   r  r  ru   )!rC   rq   r   r  Zmake_previewr  r  r   r   Zraw_tagZanchor_nameZanchor_linkZ
account_idr   r  Z	user_namer2  Zrel_docZattach_parentrT   r   Zresource_container_idrD  Z
space_nameattach_pathrj   Zdoc_with_attachr  r   Z
attach_objr   rh  r  r"   r"   r#   r    s>   











 
 


 

 






z"ConfluenceAPIImport._process_linksc             C   s  |j dkrd}nd}| jjd t|d}t|d}x|jddidD ]}y|d	rh| |d	 }ntd
t|j	t
|jt
|jt
|jt
|jdd}|jd|d d}	|	r|	|j tt
|d}W qH tk
r }
 z| jjd| d|d W dd}
~
X Y qHX qHW t
|S )u]   
        Упоминания пользователей
        confluence-userlink
        r  z
                <a class="external" href="{person_href}" rel="noopener" target="_blank" data-macros="mentions" data-mention-type="person" data-object-id="{person_id}">@{person_name}</a>
            u	  
                <span class="macros-panel mentions-macros" contenteditable="false" data-id style="display: inline-flex; flex-direction: column; position: relative;">
                    <span class="extension-container" contenteditable="false" data-id data-layout="default">
                        <span class="extension-title" data-id>
                            @Упоминания <span class="excerpt-title" data-id>| {person_login}</span>
                        </span>
                        <span class="macros-actions" data-id>
                            <span class="macros-delete" data-id>
                                <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                                    <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                    <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                    <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                </svg>
                            </span>
                        </span>
                    </span>
                    <span class="ak-renderer-wrapper" data-id>
                        <a class="external" data-id data-macros="mentions" data-mention-type="person" data-object-id="{person_id}" rel="noopener">
                            <span class="lds-ring" data-id>
                                <span data-id></span>
                                <span data-id></span>
                                <span data-id></span>
                                <span data-id></span>
                            </span>
                        </a>
                    </span>
                </span>
            uH   Конвертируем упоминания пользователейzhtml.parserzdata-linked-resource-typeZuserinfo)r   Zuserkeyu]   Не найдены аттрибуты для определения пользователя)Z	person_idZperson_loginZperson_hrefZperson_namezuser-mentionr   )class_r   uZ   Не удалось преобразовать ссылку на пользователя zERR-0009)r   r   N)r  r*   r4   r   r   r   r.   rt   rQ   r  r   ri   r  r   rT   r  r  r  r   )rC   rq   r   r  r&  r  r   r   r  r  r   r"   r"   r#   r  i  s0    



 z%ConfluenceAPIImport._process_mentionsc             C   s   |dkrd S y|  |}W n&   | jjd| dtjd tjS | |dd\}}|s| jjj	
drtjj
| jjd d}| jjd| d| tjd n | jjd| d	d
dd tj}|S )Nunknownu6   Не найден пользователь user_id = "u9   ", в дампе, указываем системного)r5   FZdefault_user_id)ri   uA   ", ставим пользователя по умолчанию u*   ", указываем системногоzERR-0023r   )r   r   )r   r*   rN   r3   ZERRORrY   Zsystem_userr  r  rW   r.   r   r   r  r   )rC   r   rf   r   r   r"   r"   r#   rt     s      
zConfluenceAPIImport._get_personc       	   	   C   s   |d d d d d }t j|}t jj||d}|sd|d d  d	| d
|j d|jd  d	}t|t jr|d|j	 7 }| j
j|d|p|d || ||| dS )u_    Конвертирует макрос Word или Excel через atlas_doc разметку r   r\  macroParamsrT   rW   )rq   rT   u*   Ошибка обработки тега "extensionKeyu)   ". Не найдено вложение "u   " документа "z" (ri   r  u    для версии v.zERR-0012)r   r   N)r   r!   r   r  r.   rT   r  r   r  r  r*   r   r  r'  )	rC   r   elemr   r*  r  r   r  r,  r"   r"   r#   _convert_msoviewer_in_atlas_doc  s    .z3ConfluenceAPIImport._convert_msoviewer_in_atlas_doc)r  rR   c       1      C   s
  dd l }t|}d}
xt|D 
]}| j r2P |d7 }|d dkr |jddddd	d
}	dd |jddid
D  }
g }x"|
jD ]}|dkrq|	| qW x^t
|d D ]N\}}|d d }|jd| ddd| ddd
}|| j|_|		| qW |
|	 tt|d}	n|d dk	rd }
|d dr|jd|d d id
}
|
s| jjd|d d  d|d d   q nNd|d d kr|d d d dr|jd |d d d d d! id
}
|d d d"krd#}|jd|d d id
}
|
	r|d d d$ d% d! }d&|krD|d&d\}}n2t|tjrb|j d' }n|jj d' }|}|d d d$ d(}|	rt|dd%j|d! ||d)}|
|j tt|d}
q|d d d*krd+}|jd|d d id
}
|
	r|d d d$ d% d! }|	rd&|kr>|d&d\}}n2t|tjr\|j d' }n|jj d' }|}t|dd%j||d,d}|
|j tt|d}
q|d d d-r|d d d$ d. d! }tj|}tjj||d/}d0}|s|d d d$ d1 d! }t | j! "d2t| j#j d' d3|d4d5}|$ sj| jj%d6| d7|d8 q n|&|d9}||' krh|(|>}tj|}tj||| jd:}|j)d;d< |*|+  W d Q R X t,-| d=}x|' D ]v}|.|r|(|T}tj|} tjj|| d/}!|!sJtj|| | jd:}!|!j)d;d< |!*|+  W d Q R X qW W d Q R X |s| jj%d>| d?| d@|d8 q tj|}tjjdAdB|gd(dC| dDggdEgdF}!dG}dG}"|d d d$ dr|d d d$ d d! }|d d d$ dHr4|d d d$ dH d! }"|j/j0|j1j0||"dI}#|!r\|!j2|#dJ< nd%|#dJ< t|dd%jf |#d}|
|j tt|d}
q|d d dKkrdL}$t|j1}%t|j/}&dM|%|%dNd%dOdMdP}'|d d d$ }(|(rb|(dQrt34|(dQ d! 5 |'dR< |(dSr|(dS d! }tjj|j6|d/})|)sTtjj|dT})|)rtt|)j1|'dU< t|)j/}&n| jjdV|d   |(dWr|(dW d! |'dX< |(dYr|(dY d! |'dZ< |(d[r|(d[ d! |'d\< |(d]rdOddd^}*|*|(d] d! |'d_< |(d`r<d(dadbdc}+|+|(d` d! |'dd< |(derbt34|(de d! 5 |'df< dg7dhdi |(8 D },t|$j|%|&t3j9|'dMdj|,dkdd%d}-|
|-j tt|d}
q|d d dlk	rz|jddlddmd
}.|jdnddoid
}/|/	|d d d$ do d!  |.	|/ |jdpdqddrdsd
}0|0	|d d d$ dt d!  |.	|0 |
|. tt|d}nj|d d duk	r| j:|||
du|dv tt|d}n4|d d dwk
r| j:|||
dw|dv tt|d}n|d dxk
r|d d dyk
r|d d d$ d(
r|d d d$ d( d! }dz}|jd|d d id
}
|

rt|dd%j|d%7d{d |
jD d|d}|
|j tt|d}n| jjd}|  |dr | j;||d ||dv}q W |S )~Nr   rE   r   ZlayoutSectionZdivZfalserQ  zmceNonEditable selected)contenteditablezdata-layout-sectionr/  )r   c             S   s,   g | ]$}d |j d krd|j d kr|qS )zfixed-widthr/  Zsingle)r   )r\   r   r"   r"   r#   r     s    z:ConfluenceAPIImport._convert_atlas_doc.<locals>.<listcomp>r/  ZcolumnLayoutr   r   r   r  rl   zflex-basis: )zdata-column-widthzdata-layout-columnr  zhtml.parser	extensionZlocalIdzdata-local-idu&   Не удалось найти тег u    для r  ZmacroMetadatar\  ZmacroIdzdata-macro-idrW   zexcerpt-includeuU  
                    <div
                        class="macros-panel macros-include-block"
                        contenteditable="false"
                        data-id
                        data-macros="include-excerpt"
                        data-param-excerpt="{excerpt_name}"
                        data-param-pageid=""
                        data-param-pagename="{page_name}"
                        data-param-projectkey="{space_key}"
                    >
                        <div class="extension-container" contenteditable="false" data-id data-layout="default">
                            <span aria-label="Макрос Include. " data-id role="img">
                                <svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewbox="0 0 24 24" width="14" height="14">
                                    <path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path>
                                </svg>
                            </span>
                            <span class="extension-title" data-id>Макрос Include. "Вставить выборку" <span class="excerpt-title" data-id>| Имя = {excerpt_name}</span></span>
                            <span class="macros-actions" data-id>
                                <span class="macros-edit" data-id style="margin-right: 7px;">
                                    <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                                        <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                                    </svg>
                                </span>
                                <span class="macros-delete" data-id>
                                    <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                                        <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                        <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                        <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                    </svg>
                                </span>
                            </span>
                        </div>
                        <div class="ak-renderer-wrapper" data-id></div>
                    </div>
                    r  r   r4  r   rT   )r3  r5  rD  r7  u  
                    <div
                        class="macros-panel macros-include-page"
                        contenteditable="false"
                        data-macros="include-page"
                        data-param-pageid=""
                        data-param-page-label="{page_label}"
                        data-param-projectkey="{space_key}"
                        data-rendered="true"
                    >
                        <div data-layout="default" class="extension-container" contenteditable="false">
                            <span role="img" aria-label="Макрос Include. ">
                                <svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewbox="0 0 24 24" width="14" height="14">
                                    <path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path>
                                </svg>
                            </span>
                            <span class="extension-title">Макрос Include. "Вставить страницу" <span class="excerpt-title">| Имя = {page_label}</span></span>
                            <span class="macros-actions">
                                <span class="macros-edit" style="margin-right: 7px;">
                                    <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                                        <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                                    </svg>
                                </span>
                                <span class="macros-delete">
                                    <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                                        <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                        <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                        <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                    </svg>
                                </span>
                            </span>
                        </div>
                        <div class="ak-renderer-wrapper"></div>
                    </div>
                    )r8  rD  r	  ZdiagramName)rq   rT   u  
                    <div data-drawio-title="{name}" data-drawio-type="diagram" data-attach-id="{attach_id}" class="drawio-content" contenteditable="false">
                        <div class="header">Диаграмма: "{name}"</div>
                        <div class="content">
                            <img cmf_converted=True src="{img_src}" alt="" data-mce-src="{img_src}" width="{width}" height="{height}">
                        </div>
                    </div>
                    ZpageIdrx  r  r   zall_attachments.zipuI   Документ с диаграммой не найден в дампе zERR-0010)r   r   r   )rq   rT   r*   T)rp   z.*\.png$u   В документе u$    не найден файл с xml: zERR-0011rq   z==rk   z%.pngr   )rm   rU   z100%r  )rT   r
  r  r  r  r   u  
                        <div
                            class="macros-panel"
                            contenteditable="false"
                            data-children-docs="{doc_id}"
                            data-id
                            data-macro-parameters='{params}'
                            data-macros="children-docs"
                            original-data-macro-parameters="{original_params}"
                        >
                            <div class="extension-container" data-id data-layout="default">
                                <span aria-label="Макрос " data-id role="img">
                                    <svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewbox="0 0 24 24" width="20" height="20">
                                        <path d="M22.485,10.975,12,17.267,1.515,10.975A1,1,0,1,0,.486,12.69l11,6.6a1,1,0,0,0,1.03,0l11-6.6a1,1,0,1,0-1.029-1.715Z"></path>
                                        <path d="M22.485,15.543,12,21.834,1.515,15.543A1,1,0,1,0,.486,17.258l11,6.6a1,1,0,0,0,1.03,0l11-6.6a1,1,0,1,0-1.029-1.715Z"></path>
                                        <path d="M12,14.773a2.976,2.976,0,0,1-1.531-.425L.485,8.357a1,1,0,0,1,0-1.714L10.469.652a2.973,2.973,0,0,1,3.062,0l9.984,5.991a1,1,0,0,1,0,1.714l-9.984,5.991A2.976,2.976,0,0,1,12,14.773ZM2.944,7.5,11.5,12.633a.974.974,0,0,0,1,0L21.056,7.5,12.5,2.367a.974.974,0,0,0-1,0h0Z"></path>
                                    </svg>
                                </span>
                                <span class="extension-title" data-id>Макрос "Отображение дочерних"</span>
                                <span class="extension-object-name" data-id>| {doc_name}</span>
                                <span class="macros-actions" data-id>
                                    <span class="macros-edit" data-id style="margin-right: 7px;">
                                        <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                                            <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                                        </svg>
                                    </span>
                                    <span class="macros-delete" data-id>
                                        <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                                            <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                            <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                            <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                        </svg>
                                    </span>
                                </span>
                            </div>
                            <div class="ak-renderer-wrapper" data-fabric-macro="null" data-id data-macro-body=""></div>
                        </div>
                    FZ50rW  )isShowChildrenr$  
documentIdchildrenAmountr5   includeViewTypeisOrderByDeskr`  r  rP  )rT   r  uE   Не удалось обработать тег оглавления firstr  depthr5   r  Z
headerTyperY  )rW  Zsimplezrich contentr  r%  r  r  )r   Zcreationr  ZsortTyperP  r  r)  c             s   s$   | ]\}}| d |d  V  qdS )r  rW   Nr"   )r\   r   r   r"   r"   r#   r     s    z9ConfluenceAPIImport._convert_atlas_doc.<locals>.<genexpr>)r   )r&   doc_namer  r+  zpaste-code-macro)r/  rx  rN  r   Zprezlanguage-markup1)r/  r  zdata-mce-selectedZ__bodyContentr!  )r  r"  ZbodiedExtensionr.  u  
                    <div
                        class="macros-panel macros-excerpt-block"
                        contenteditable="false"
                        data-id
                        data-macros="exceprt"
                        data-param-name="{name}"
                    >
                        <div class="extension-container" contenteditable="false" data-id data-layout="default">
                            <span aria-label="Макрос Include. 'Создать выборку'" data-id role="img">
                                <svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewbox="0 0 24 24" width="14" height="14">
                                    <path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path>
                                </svg>
                            </span>
                            <span class="extension-title" data-id>Макрос Include. 'Создать выборку' <span class="excerpt-title" data-id>| name = {name}</span></span>
                            <span class="macros-actions" data-id>
                                <span class="macros-edit" data-id style="margin-right: 7px;">
                                    <svg _ngcontent-aiw-c17="" viewbox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;">
                                        <path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path>
                                    </svg>
                                </span>
                                <span class="macros-delete" data-id>
                                    <svg viewbox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;">
                                        <path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                        <path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                        <path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path>
                                    </svg>
                                </span>
                            </span>
                        </div>
                        <div class="ak-renderer-wrapper" contenteditable="true" data-fabric-macro="null" data-id data-macro-body="">{content}</div>
                    </div>
                    c             S   s   g | ]}t |qS r"   )r   )r\   r   r"   r"   r#   r   M  s    )rT   r   u   Не нашли тег для )<r  r   r@  r*   r  r  r   r]  rh  ru   r  r  r   r   r.   r  r4   r   r   r   r   rW  r  r/   rq   r   r  r  r   r!   r   r  r   r   r   r  r   r   r  r  r   r|   r  ra   r  r  r  rT   rW   ri   r   r   r   r   r  r   r   r   r  r  )1rC   r   r  r  r  r  cntr  r  Ztag_layout_sectionr   rh  rc  colr  Ztag_layout_colr  macro_paramsrD  r5  r3  r]   r8  rT   r   r  r2  r  r  r   ZregexpZname_zipZfile_name_zipr  r  r)  r&  r  r  r  r+  Zparent_pageZview_type_mapZsort_type_mapZoriginal_params_strZnew_soupZ
code_blockr   rW   r"   r"   r#   r    s   

** $

#


(&





*!.
z&ConfluenceAPIImport._convert_atlas_docc                s  t t d fdd}t t d fdd}t t d fdd}dd	d
gdddgg}xjdD ]`}tt| }|j|d} jjd|j d|  d}d}	x|dkr j rĈ jjd d
S |j	|ddddg|	| |	| | gd}
|	d7 }	||8 }x|
D ]}yld}t
|drx||jj|_||jj|_||jj|_|jr|d7 }|jdd |  |j  nP||jj|_||jj|_||jj|_|jr|d7 }|jdd |j  x|jD ]}|jdkrqy8||jj|_|jr|d7 }|jdd |j  W n8   |j   jjd| d|j d |d! P Y nX qW  jjd"| d#|  W n6   |j   jjd| d|j d$|d! Y nX qW qW qXW d
S )%uU   
        Заменим ссылки на задачи и документы
        )r   rR   c                sH  | s| S t | d}|js| S x|jddidD ]}d }|d}|sT jjd q.tjj	ddd	| d
gdgd}x |D ]}|j
d |krz|}P qzW |s jjd| d q.|tj| q.W xd|dD ]T}|jdrq|jd}|s jjd|  q|ds4 jjjjj | }| jjjjjrd }d }|d}	|d}
|	dkr@|
r@ jjjj d|
 }tjj|dgd}|stjjddd jjjj dgddd|
 d
ggd}|s jjd|  q|jt|jkst|tjrd|jd< |jd |jd < |j|jd< q֐n|	d!kr|
rtjjddd"|
 gdgd}|s jjd#|
  qd|jd< |jd |jd < |j|jd< q֐ndd$|jd kr^xTtd%|jd D ]>} jjjj d|d&d   }tjj|dgd}|rP qW |s> jjd'|jd   q|s jjd|  q֐nd(|jd kr0td)|jd }|s jjd*|jd   qtjj	ddd	|d+d&d   d
gdgd}x2|D ]*}|j
d |d+d&d  kr|}P qW |s jjd,|d+d&d    qnd-|jd krtd.|}|sRq|d&}|d/d0d1}tjj	ddd jjjj dgd2d|ggdd3gd}x*|D ]"}t|j j
d |kr|}P qW |s jjd4|d+   qd|jd< |jd |jd < |j|jd< qnq|tj| qW d5!d6d7 |jj"D S )8Nr   zdata-macro-nameZjira)r   zdata-jira-keyuV   Не удалось найти код задачи в атрибуте "data-jira-key"r  rk   z%"key":"z"%)rm   rU   r   u;   Не удалось найти задачу по коду "r  arx  r   u!   Ссылка без адреса r   zdata-linked-resource-typezdata-linked-resource-idrP  z::)rj   rU   rj   rl   z::%z%"homepage":{"id":")rm   u:   Не удалось найти документ по ид Tr  r   z%::uB   Не удалось найти пространство по ид z/wiki/z\/\d+rE   u:   Не удалось найти ид документа в z/browse/z	\/\w+-\d+u6   Не удалось найти код задачи в r   u:   Не удалось найти задачу по коду z	/display/z'/display/([~]?[A-Za-z0-9]+)+(?:/)?(.*)?r   r   r   rT   zparent.import_raw_jsonuB   Не удалось найти документ по ссылке r   c             S   s   g | ]}t |qS r"   )r   )r\   r  r"   r"   r#   r     s    zQConfluenceAPIImport._process_cross_links.<locals>.check_links.<locals>.<listcomp>)#r   r   r   r.   r*   r4   r   r   CmfTaskrt  r  r  ZCmfPluginCsvZcreate_tag_linkr   r   rv   r  rW   rw   r!   rW  stringr   rT   r   r   r  findallr  rh   r   rq   r   r   )r   r  r   r   Ztask_keyobjsr   r   rj   Zresource_typeZresource_idZext_coder  rD  r5  Zdocsr   )rC   r"   r#   check_links[  s    





 


$

"



z=ConfluenceAPIImport._process_cross_links.<locals>.check_linksc                s.  | s| S t | d}x|jddidD ]}|jdr8q&|jd}|s^ jjd|  q&|jd} jjjj d	| }t	j
j|d
}|st	jjddd jjjj dgddd| dggd}|rt|}|jj|d< |jj|d< t||d< d|d< |d= q& jjd|  q&W t|S )uz   
            Заменяет ссылку на корневую страницу в макросе Page Tree
            zhtml.parserzdata-macrosz	page-tree)r   rx  zdata-macro-parametersu3   Макрос Page Tree без параметров Zroot_page_originalz::)rj   rj   rk   rl   z::%r  z%"homepage":{"id":"z"%)rm   r  Z
rootLocalerQ  ue   В макросе Page Tree не удалось найти корневую страницу по ID )r   r   r   r.   r*   r4   r   rv   rw   r   r!   rW  r   r   ri   rW   rT   r   r   )r   r  r   r-  Zroot_page_idZroot_page_ext_idZpagetree_rootr  )rC   r"   r#   check_pagetree_root  s6    

zEConfluenceAPIImport._process_cross_links.<locals>.check_pagetree_rootc                s   | s| S t | d}x|jdddgidD ]}|d}|dpH|d}tjjd	d
 jjjj dgdd
d| dggd}|s jj	d| d q(tj
j||d}|rt|j|d< q( jj	d| d q(W t|S )u\   
            Добавляет ID документа в макрос "Include"
            zhtml.parserzdata-macroszinclude-excerptzinclude-page)r   zdata-param-projectkeyzdata-param-pagenamezdata-param-page-labelrj   rk   z::%r  z%"key":"z"%)rm   u^   В макросе "Include" не удалось найти пространство по key "uA   ". Возможно оно еще не импортировано)rT   rq   zdata-param-pageidu]   В макросе "Include" не удалось найти страницу по имени "u@   ".Возможно она еще не импортирована)r   r   r.   r   rW  r*   rv   rw   r4   r   r!   r   ri   )r   r  r   Z
projectkeyr5  rz  Zexcerpt_page)rC   r"   r#   check_include  s&    

z?ConfluenceAPIImport._process_cross_links.<locals>.check_includer*   z!=Nry   z==T)r!   r  rW  Z	CmfFolder)rm   u)   Конвертируем ссылки в u=    на локальные задачи и документы: r  r   u'   Отменили конвертациюr   zcomments.log_levelzcomments.text
text_draft)rm   rU   slicerE   )rp   )r   r   u>   Не удалось конвертировать ссылки z: zERR-0013)r   r   u   Проверили u!    и конвертировали zERR-0014)r   varsr   r  r*   r4   r   r  r  rt  hasattrr   rW   r  r{   r|   Z
do_approver}   r~   rm  r   r   r   )rC   r  r  r  r   Z	mode_namerX  r  stepr  r  r   r  r5  r"   )rC   r#   r  W  sv     +$



z(ConfluenceAPIImport._process_cross_links)F)N)r_  N)N)N)r   r  )N)N)N)TT)T)N)NN)br   r   r   r   r   r   Z	CmfImportrD   r6   rt  rZ   staticmethodr   r   r_   r`   r   r   r   r   r   r   r   r   r   r   r   r  r,  r@  rA  rE  r^  rn  ro  rw  r{  r|  r  r  r  r  r  r  r  rs   r  r  r  r  r  r  r  r  r  r  r  r@   ZIMPORT_OBJ_CNTr  r  r  r  r  r'  r  r  r  r  r  rM  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r8  r  r  r  r  rt   r  r  r  r"   r"   r"   r#   r(   (   s    >!.

KZ: %5_
{
B#7l*%B#
"\
+ 2;a
$GVMPT
 ^FQ(  v*X) DB
   	r(   )$r   r  r   r   rg  Zcmf.appr   Zpathlibr   Zurllib.parser   r   typingr   r   r   r	   r   Zcmf.data_providers.baser
   Zcmf.includeZcommon.models.cmf_active_entityr   Zmodules.confluence.apir   Z%common.models.cmf_plugin_import_mixinr   Zbs4r   Zatlassian.errorsr   r   Z	dataclassr   r$   r(   r"   r"   r"   r#   <module>   s*   
