U
    =e!                    @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m	Z	 d dl
mZmZmZmZ d dlZd dlmZ d dlT d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlZejG dd dZejG dd dZ G dd dZ!dS )    N)cmf_context)Path)urlparse)datetime)DictUnionListOptional)commit_all_ds)*)CmfActiveEntity)ConfluenceApi)CmfPluginImportMixin)BeautifulSoup)ApiNotFoundErrorc                   @   sv   e Zd ZU dZejedZee	e
jf ed< ejedZee	eee	e	f  f ed< ejedZee	e
jf ed< dS )	TotalDatauJ    Данные используемые в процессе импорта default_factoryuserslink_documentsresult_documentsN)__name__
__module____qualname____doc__dataclassesfielddictr   r   strmodels	CmfPerson__annotations__r   r   r   CmfDocument r#   r#   "./modules/confluence/api_import.pyr      s   
(r   c                   @   s@   e Zd ZU dZejed< eed< ej	e
dZeeef ed< dS )DocDatauR    Данные используемые при создании документа Zdocumentdoc_idr   link_attachmentsN)r   r   r   r   r   r"   r!   r   r   r   r   r'   r   r#   r#   r#   r$   r%      s   

r%   c                   @   s$  e Zd ZdZeejdddZdd Ze	ddd	Z
eeed
ddZe	dddZdd Zdd ZdsedddZedddZdd ZeeedddZdd Zdtd!d"Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Z d1d2 Z!ed3d4d5Z"d6d7 Z#d8d9 Z$d:d; Z%ed<d=d>Z&d?d@ Z'eedAdBdCZ(dDdE Z)edFdG Z*dHdI Z+dJdK Z,edLdMdNZ-dOdP Z.dQdR Z/dSdT Z0edUdVdWZ1dXdY Z2dZd[ Z3dueed]d^d_Z4d`da Z5dbdc Z6e7ddddeZ8dfdg Z9dvedLdhdiZ:eddjdkZ;dldm Z<dweeedndodpZ=dqdr Z>d S )xConfluenceAPIImportu;   Класс загрузки объектов через API)
confluence
cmf_importc                 C   s   || _ || _t | _tjjdd}|j | _	d| _
d| _td}| jjj|_|| jjj |   d| _| j jjr~d| _d| j_d| j_t | _t | _d S )Nzsystem:defaultcoder   zhistory,body.export_view.webresource,body.storage,body.editor,body.view,body.editor2,body.styled_view,body.atlas_doc_format,body.dynamic,children.page,ancestors,body.plain,body.anonymous_export_view,body.wiki,history.ownedBy,history.lastUpdatedzatlassian.rest_clientZuserKeyZ	accountId)confluence_apir*   r   Z
total_datar   ZCmfSchemeWfgetZdefault_document_workflowloaddoc_workflow
has_errorsexpandloggingZ	getLoggerloggerZhandlersZsetLevellevel_prepare_multiprocessinguser_keyconncloudimported_object_countjson_object_countr   treeset_selected_objs)selfr)   r*   Z	scheme_wfr4   r#   r#   r$   __init__)   s$    

zConfluenceAPIImport.__init__c                 C   s0   dd l }dd l}|| _ || _tj| _tj| _d S )Nr   )	threadingqueueconfigZIMPORT_THREADSthreading_max_forksZIMPORT_DOWNLOAD_THREADSdownload_threading_max_forks)r?   rA   rB   r#   r#   r$   r6   C   s
    z,ConfluenceAPIImport._prepare_multiprocessing)returnc                 C   s2   dt tjjt tjjdddg ddd dgS )NZcurrent_user)emailname)rH   fieldsselectedObjects)rH   valueimport_type)r   gZcurrent_personrG   rH   r?   r#   r#   r$   calc_import_settingsK   s    
z(ConfluenceAPIImport.calc_import_settings)settingsrF   c                 C   s   dd | D S )uO   Преобразуем список с настройками в словарьc                 S   s   i | ]}|d  |qS rH   r#   ).0sr#   r#   r$   
<dictcomp>T   s      z;ConfluenceAPIImport._normalize_settings.<locals>.<dictcomp>r#   )rP   r#   r#   r$   _normalize_settingsQ   s    z'ConfluenceAPIImport._normalize_settingsc                 C   s   g S )uo    Для импорта из Confluence предварительный просмотр не требуется r#   rN   r#   r#   r$   previewV   s    zConfluenceAPIImport.previewc                 C   s  g g d}d }|  |d D ]}|d dkr|d d d D ]*}d}| || j }|r>|d | q>|d d	 d D ]}d}| jjjrd
| jjjj	 d
|d  d
}	nd
| jjjj	 d
|d  d
}	ddd|	 dg}
t
jj|
d}|rz|d | qz|d dkr|d d d D ]8}|dkr,d}| || j }|r|d | q|d d	 d D ]}|dkrvd}| jjjrd
| jjjj	 d
|d  d
}	nd
| jjjj	 d
|d  d
}	ddd|	 dg}
t
jj|
d}|rd|d | qdq|jr|r||_|jr,|jdd |j  |j  dd|jgddd gg}
t
jj|
dD ]}|jdd |j  qZ|d rt
j|j| jd|d djdd |d rt
j|j| jd|d djdd t
jj  d S )N)readwritepathZ	operationrW   restrictionsuserresultsZprivategroup::idrH   ext_idLIKE%filterupdatereadonlyrX   TZsave_importparent==r*   !=)ZTEXKOM_db_delete)rh   r*   Zaccess_levelZsubjects)_get_restrictions_get_personr7   appendr-   r8   r9   r*   pluginsource_hashr   CmfPersonGroupr.   import_originalperm_policy
is_changedsavedpcommitZperm_aclr/   ZCmfAccessRulelistdelete)r?   docdoc_objZpermsrr   restrictionr[   person
group_infor`   _filterr]   Zruler#   r#   r$   _process_restrictionsZ   s|    

 

 


 
 z)ConfluenceAPIImport._process_restrictionsc           	      C   s<  | ddd  d }t|j d}t|dkrb|d dkrb|d dkrb|d	 d
krb|d }|S t|d	kr8|d dkr8|\}}}}z |dd}| jj||}W nN tk
r } z0| j	j
d| d| d| d|  W Y d S d }~X Y nX |s| j	j
d| d|  d S | j	j
t| |d }|S )N/r         wiki   Zspaces   Zpages   Zdisplay+ uC   Ошибка получения документа по pretty url : uo   Не смогли получить документ по pretty url (возможно не тот домен): r_   )splitr   rY   lenreplacer-   r8   Zget_page_by_title	Exceptionr*   r4   warninginfor   )	r?   urlr,   Zsplitted_url_spacetitleZfind_confl_docer#   r#   r$   _get_pageid_by_url   s6    

z&ConfluenceAPIImport._get_pageid_by_urlFrQ   c              	   C   s   | j  | d}| j  | d}| r<|s<d S t|d6}| D ](}| j  jd7  _|t|d  qNW 5 Q R X t	
|| | j jdd | j j  d S )N.jsonz.json.dirtya+r   
TZ	only_data)r*   get_download_pathjoinpathexistsopenr;   rX   jsondumpsshutilmovert   ru   rv   )r?   Zapi_funcrH   Z	overwrite	file_pathZfile_path_tmpfrowr#   r#   r$   _simple_dump   s    
 z ConfluenceAPIImport._simple_dumpc              	   c   sF   | j  | d}t|d}|D ]}t|V  q&W 5 Q R X d S )Nr   r+)r*   r   r   r   r   loads)r?   rH   r   r   r   r#   r#   r$   _simple_get   s    zConfluenceAPIImport._simple_getc              
   c   sJ   | j  d}t|D ]*}t||}t|V  W 5 Q R X qd S )Nr   r*   r   r   oslistdirr   r   r/   )r?   Zdir_pathfpr   r#   r#   r$   _list_users_from_dump   s    z)ConfluenceAPIImport._list_users_from_dump)textattachments_dirobjc                 C   s  |sd S | j jd t|ddd D ]}d}|drBd}|j| }zPd|kr\W q*|dr~| j jd	|  W q*|d
st	j
| jj|}|| jjs| j jd|  W q*tj|dd dd }t|\}}	|s| j jd|  W q*tj|}||}
| j jjj d|d  }tjj||| j jjjd}|sxtj| j ||| j jjjd}|
|_
||_d|_|  |j   W q* t!k
r } z*| j j"  | j jd| d|  W 5 d }~X Y q*X q*d S )Nu/   Обработка ссылок в текстеlxmlc                 S   s   |  dp|  dS Nhrefsrchas_attrtagr#   r#   r$   <lambda>       z1ConfluenceAPIImport._dump_links.<locals>.<lambda>r   r   z/attachments/zmailto:u   Это почта: httpuH   Это внешняя ссылка, оставляем как есть: r   r   ?r   u   Это не файл: r^   r_   r   
obj_ext_idro   r*   r   r   ro   Fu0   Не удалось скачать ссылку r   )#r*   r4   r   r   find_allr   attrs
startswithr   r   rY   joinr-   r   urllibparseunquoter   	mimetypes
guess_typer   r"   _get_filenamer   rn   ro   CmfImportDownloadr.   rH   
downloadedrt   ru   rv   r   rollback)r?   r   r   r   r   keyr   	file_name	mime_typeencodingr   r`   download_jobr   r#   r#   r$   _dump_links   sZ    



 


zConfluenceAPIImport._dump_linksc                 C   s   z|  |W S    | jj|dd}|s8| jj|dd}|sL| jj|dd}|r|d}| sjt }n t|d}t	
|}W 5 Q R X || t|d}t	j||dd W 5 Q R X Y nX d S )N
operationsr2   
users.jsonrw+r   )indent)_get_user_info_from_dumpr-   Zget_user_infoZget_user_info_by_keyZget_user_info_by_usernamer   r   rw   r   r   r/   rm   dump)r?   user_idZ	dump_path	user_info
users_pathr   r   r#   r#   r$   _get_user_info  s"    

z"ConfluenceAPIImport._get_user_infoNc                 C   sr   |  dD ]T}|dd |kr*|  S |dd |krF|  S || j|kr
|  S q
td| d S )Nr   rH    ZusernameuQ   Не нашли информацию по пользователю в дампе )r   r.   lowerr7   r   )r?   r   r   r   r#   r#   r$   r     s    
z,ConfluenceAPIImport._get_user_info_from_dumpc                 C   s  t dd || _d}| jjd| d|  | j|d ||D ]f}zPz| j rlW W : q| jjd|d  d	 |	t
|d
 }	|	 rW W qF| jj|d
 | jd}|	t
|d
 d }
|
jdd |d d | jr| |d d | j |
 |d d}|rF|d dkrF| || j |
 |d d d | jr~| |d d d | j |
 |		d}|
	d}dd | jj|d
 dD }t|}t|dd}t|| W 5 Q R X |rtj| jjd|d
  }| jjjr$tj| jjd|d
  }d}|	|}| jjjj d |d
  }tj j||| jjjjd!}|stj | j||| jjjjd"}||_||_!d|_"|#  |j  | $|d# d$ d% || |
	d&}| rt%| | jjd' t|d(}| j d)}| j&|d
 |D ]}|'t(|d*  | j j)d+7  _)| $|d# d, d% || |d dr| |d d | j |
 |d dr| |d d | j |
 |d dr| |d d d | j |
 qW 5 Q R X |
	d-}| rt%| | jjd. t|d(\}| j*|d
 D ]D}|'t(|d*  |d/ d0 d1 D ]}| || j |
 q^q6W 5 Q R X |
	d2}| rt%| | jjd3 t|d(2}| j+|d
 D ]}|'t(|d*  qW 5 Q R X t,-|
|	 |jdd |		d4}t
|	|d5< t|d"}t|| | j j)d+7  _)W 5 Q R X W nD   |d+7 }| jj.d6|d  d7|d
  d8 | jj/  Y nX W 5 | jj  X qF|0| W 5 Q R X d S )9NFZinit_views_and_dsr   u   Документы с u    до r   u   Документ 'r   'r_   r   .dirtyTexist_okhistory	createdByownedBytype	anonymouslastUpdatedbyattachmentsattachments.jsonc                 S   s   g | ]}|qS r#   r#   rR   r   r#   r#   r$   
<listcomp>?  s     z;ConfluenceAPIImport._dump_document_part.<locals>.<listcomp>)page_idr   )modez+pages/downloadallattachments.action?pageId=z%wiki/download/all_attachments?pageId=all_attachments.zipr^   r   r   bodyviewrK   zcomments.jsonu#   Дампим комментарииr   z2,extensions.inlineProperties,extensions.resolutionr   r   Zexport_viewzrestrictions.jsonu   Дампим праваrZ   r[   r\   labels.jsonu   Дампим теги	info.jsonrY   u5   Не удалось скачать документ   ())1r   r-   r*   r4   r   get_all_pages_from_spaceru   rv   	is_cancelr   r   r   Zget_page_by_idr2   mkdirr.   r7   r   Zget_all_attachmentboolr   r   r   r   rY   r   r   r8   r9   rn   ro   r   r   rH   r   rt   r   removeZget_page_commentsrX   r   r;   Z get_all_restrictions_for_contentZget_all_page_labelsr   r   	exceptionr   put)r?   startmaxobjs_dirr   obj_apiobjs_that_are_doneerrorsry   doc_dirZdoc_dir_tmpownerr   Z	info_fileattach_infoZ
has_attachr   r   r   Zattach_file_pathr`   r   Zcomments_file_namer2   commentZrestriction_file_namer{   r[   Zlables_file_nameZlabelZ	info_pathr#   r#   r$   _dump_document_part  s    










,


$


"
$z'ConfluenceAPIImport._dump_document_partc                    s"  t |d  d}|jdd | j }d}| j|d }| jjd|  t	|| j
 d }|sjd	}d}g }|dkr| jj| jd
| || || | ||t| jj| jj| jj| jj| jj| jjjd|dd}	||	 |	  | jjd|	  |d7 }||8 }qr|D ]}	|	  ~	q| j rB| jjd | jjd | sj|| 7 }qP| jjd|  | jd dd | dD  t |D ]^}
|!|
d}|" rt#|,}t$%|D ]}| krֈ | qW 5 Q R X |&  q| ' fdddd |S )NrY   
/documentsTr   r   r   u:   Дампим документы пространства: r   r   r  r   logintokenpassword
verify_sslr9   )r  r  r  r   r  r  targetrH   kwargsu   Форкаем proc=   Импорт прерванuU   Закончили параллельное скачивание документовu   Ошибок: uC   Соберем пользователей из документовc                 S   s   g | ]}|qS r#   r#   )rR   ur#   r#   r$   r     s     z7ConfluenceAPIImport._dump_documents.<locals>.<listcomp>r   r   c                      s    S )Nr#   r#   r   r#   r$   r     r   z5ConfluenceAPIImport._dump_documents.<locals>.<lambda>)(r   r   rB   Queuer-   Zget_count_pages_from_spacer*   r4   r   intrE   rA   Threadr  r   r   r  r  r  r  r8   r9   rm   r  r   r   r   emptyr.   logr   r   r   r   r   r   r   r/   unlinkr   )r?   r   r  r  r  cntstepiZprocessprocr&   r   r   r  r#   r  r$   _dump_documents  sf    

  





z#ConfluenceAPIImport._dump_documentsc              
   C   sL   | j  d}t| d| dd}t|}|W  5 Q R  S Q R X d S )Nprojectsr   
/info.jsonr   )r*   r   r   r   r   r/   )r?   project_keyproject_dirr   projectr#   r#   r$   _get_project  s    
z ConfluenceAPIImport._get_projectc              
   c   sb   | j  d}t|D ]B}|dr*qt| d| dd}t|}W 5 Q R X |V  qd S )Nr$  r   r   r%  r   )	r*   r   r   r   r   endswithr   r   r/   )r?   r'  r&  r   r(  r#   r#   r$   _get_projects  s    
z!ConfluenceAPIImport._get_projectsc           	   	      s  t td fdd jjd }|d d d }|d sDd	S d
d |d D }|sʈ D ]f}d|d< g |d< |d D ]}|d |dg d q~|d D ]}|d |dg d q|| qbt }|D ]}z|d r|d sN|d d }|d D ]}|d |dg d q
|d D ]}|d |dg d q0|d D ]p}|d rV|	|d  |
dstj d|d d |d<  ||d D ]}|	| qqVW q   jjd| d Y qX q|_d	S )uJ   Прогружаем все выбранные документы
        )doc_infor  c                 3   s   | d sd S | d s t|d| d }|sX jd7  _jjd|   d S |d d d D ]}| d |d d	g d
 qh| d D ]&}|d r ||E d H  |d V  qd S )N	isCheckedchildren	documentsr_   r   u'   Документ не загружен pager\   Tr_   r-  r.  )_get_documentr   r   r1   r*   r4   errorrm   )r,  r  Zdoc_dumpchildget_childrenr?   r#   r$   r6    s    z=ConfluenceAPIImport._load_selected_objs.<locals>.get_childrenimport_settingsrJ   rK   r   r-  Nc                 S   s   g | ]}|d  r|qS r-  r#   rR   setting_objr#   r#   r$   r     s      z;ConfluenceAPIImport._load_selected_objs.<locals>.<listcomp>r.  T
root_pagesr1  archived_pagesr   r   r_   rY   r$  uc   Не удалось вычислить выбранные документы для проекта u4   , скорее всего он не сдамплен)r   r   rU   r*   json_settingsr+  rm   r=   r)  addr.   r   r   r4   r   r>   )	r?   rP   rootselected_spacesr(  r&   resultr   r,  r#   r5  r$   _load_selected_objs  sF    



&z'ConfluenceAPIImport._load_selected_objsc              
   C   s  |  | jjd }|d d d }|d s.dS dd |d	 D | _|d rpt|d	 dkrpd
d | j D | _| j d}|j	dd | jD ]}z| j
 rW  d S | j jd7  _|t|d d }|d}| s&| jj|d d }| jjd|d   |d dkr2td| t||d d  d|d< tj|d dd g |d< g |d< | j|d D ]}|d |d  q| jj|d dddD ]}	|d |	d  q|d stdt|d | t||d< t|d}
t||
 W 5 Q R X t  n t|d}
t|
}W 5 Q R X |  j|  |7  _W q   t!  |  jd7  _| jj"d |  Y qX qd S )!Nr7  rJ   rK   r   r-  Tc                 S   s   g | ]}|d  r|qS r8  r#   r9  r#   r#   r$   r     s      z4ConfluenceAPIImport._dump_spaces.<locals>.<listcomp>r.  c                 S   s   g | ]}||d  dqS )r_   )r   r_   r#   r   r#   r#   r$   r     s     r$  r   r   r   r   r   u&   Дампим пространство rH   r   ZpersonaluY   Миграция персональных пространств в разработке r   rY   r;  r<  r_   Zarchived)statusuJ   Пустой проект, возможно нет прав доступаr   r   u1   Ошибка дампа пространства )#rU   r*   r=  r@  r   r-   Zget_all_spacesr   r   r   r   r;   r   r   r8   Z	get_spacer4   r   CmfErrorr   makedirsZget_root_pages_from_spacerm   r   r   r   r   r   r   commit_with_eventr/   r1   r#  rollback_purge_eventr   )r?   rP   r?  Zprojects_dirr:  r'  	data_filer   ry   Zarchived_docr   r#   r#   r$   _dump_spaces  sV    


 
z ConfluenceAPIImport._dump_spacesc                 C   s  | j  d}| rd S |jdd | j D ]^}| jjjrPt	|d }nt	|d 
dd
dd}||}|jdd |d	}| j  jd
7  _t|d}t|| W 5 Q R X |d}| j  d}|jdd |jdd | j|d D ]}	||	| j d }
| j  jd
7  _t|
d}t|	| W 5 Q R X ||	| j d }
|
 rlq t|
d}t|	| W 5 Q R X q q2d S )NgroupsTr   r_   rH   r   r   r   r   r   r   r   r   )r*   r   r   r   r   r-   Zget_all_groupsr8   r9   r   r   r;   r   r   r   Zget_group_membersr7   )r?   
groups_dirr]   r`   Z	group_dirrH  r   Z	users_dirZall_users_dirr[   Zuser_data_filer#   r#   r$   _dump_groups5  s8    




z ConfluenceAPIImport._dump_groupsc                 C   st   | j  }t|\}}}|| d }| j jd| d | j jd | j jd | | jjd | 	  d S )Nd   u   Свободно места: rb   u$   Скачаем данные из APIu'   Дампим пользователейr   )
r*   r   r   
disk_usager4   r   r   r-   Zget_all_usersrI  )r?   Zdownload_pathtotalusedfreeZfree_percentr#   r#   r$   download_dataT  s    
z!ConfluenceAPIImport.download_data)r  c                 C   s  dd l }dd l}|| _d}tdd | j r6q| }|dkrJq| jjd|d   d }t	dD ]T}	z| jj
|d tjdd	}|  t|d
 d"}
|jddD ]}|
| qW 5 Q R X ||d
 \}}|r
|dkr
||d
 }W 5 Q R X | jjd|d
   tjj|d d}d|_|  |j  W 5 Q R  W  q(W 5 Q R X W qn tk
r } z:t  |	dkr|d7 }| jjd|d  d|  W 5 d }~X Y qnX qnq(|| W 5 Q R X d S )Nr   Fr   DONEu    Пытаемся скачать r      T)timeoutstreamrY   zwb+i    )Z
chunk_sizezapplication/zipu   Сохранили в r_   r_   r   r   u@   Не удалось скачать файл с 3 попыток r   )zipfiler   r-   r   r*   r   r.   r4   r   rangeZconfluence_requestrC   ZIMPORT_DOWNLOAD_TIMEOUTZraise_for_statusr   Ziter_contentrX   r   ZipFiler   r   r   rt   ru   rv   r   rG  r   r  )r?   r  download_queue
done_queuerX  r   r  r   resr!  r   chunkZmimetypeencodemyzipZdownload_objr   r#   r#   r$   _download_file_worker`  sN    
 


4z)ConfluenceAPIImport._download_file_workerc           	      C   s  | j jd | j }| j }d}| j dg dd| j jjjgdddgg}tj	j
|dsp| j jd	 |S g }t| jD ]v}| jj| jd
| ||t| jj| jj| jj| jj| jj| jjjddd}|| |  | j jd|  q~tj	j|ddgdD ]}||j|j|jd q
|D ]}|d q.|  s^|D ]}|!  qN| j jd | j " rt#  | j j$d |  s||% 7 }q| j jd|  |S )Nu=   Скачиваем все найденные вложенияr   zplugin.plugin.source_hashro   ri   r   Frc   u   Вложений нетra  r  )r[  r\  r  r  u    Запустили задачу r   rY   rd   rI   )r_   r   rY   rS  u%   Закончили скачиватьr  uC   Закончили скачивание файлов, ошибок )&r*   r4   r   rB   r  load_fieldsrn   ro   r   r   countrY  rE   rA   r  ra  r   r-   r   r  r  r  r  r8   r9   rm   r  slistr  r_   r   rY   r  r   r   rF  r   r.   )	r?   r[  r\  r1   r~   Zprocsr!  r"  r   r#   r#   r$   download_files  sN    

  


z"ConfluenceAPIImport.download_filesc              	   c   s:   | d}t |d}|D ]}t|V  qW 5 Q R X d S )Nz/comments.jsonr   r   r   r   r?   rY   Zcomments_file_pathr   r   r#   r#   r$   _get_comments  s    
z!ConfluenceAPIImport._get_commentsc              	   c   s:   | d}t |d}|D ]}t|V  qW 5 Q R X d S )Nz/restrictions.jsonr   rg  rh  r#   r#   r$   rk     s    
z%ConfluenceAPIImport._get_restrictions)r   c                    s6    d D ]"}d fdd	  |d qd S )NrY   c              	      sV  z| d dkr*j jd|   W d S j jjj d| d  }tjj|dgd}|rf|jsfW d S |s~tj|j d}n|	d	g | d
 dr
| d
 d j |_n2| d
 drވ
| d
 d j |_ntj|_| d
 d |_| d
 dr 
| d
 d j |_n|j|_| d
 drj
| d
 d d j |_| d
 d d |_n|j|_|j|_| d d d |_|jj|_| |_||_| d d dkrd|_| d d d dkrd|_|jdd | d D ]} || q|j  |W S    j jdd  d|    jd 7  _Y nX d S )!NrC  ZdraftuJ   Не обрабатываем черновики комментариев r^   r_   rq   r`   rI   )rh   r`   r*   **r   r   r   createdDater   r   whenr   r   rK   
extensionslocationinlineTZ
resolutionZresolvedclosedrg   r.  u7   Ошибка обработки комментария r   r   r   ) r*   r4   r   rn   ro   r   
CmfCommentr.   rq   rc  rl   r7   	cmf_ownerrM   anonymous_usercmf_created_at
cmf_authorcmf_modified_bycmf_modified_at_process_mentionsr   _process_linksrK   import_raw_jsontree_parentrp  rC  rt   ru   rv   r   r1   )r  r|  r`   Zcmf_commentr4  creater   rh   r?   r#   r$   r~    sR    

z9ConfluenceAPIImport._process_obj_comments.<locals>.create)r  )N)ri  )r?   rh   r   r  r#   r}  r$   _process_obj_comments  s    2z)ConfluenceAPIImport._process_obj_commentsc                 c   s4   | d}t |D ]}|| d| dV  qd S )Nz/attachmentsr   )r   r   )r   r   )r?   Zobj_pathZattachments_pathr   r#   r#   r$   _get_attachments  s    
z$ConfluenceAPIImport._get_attachments)r  c              	   C   sJ   t t|d}t|}W 5 Q R X |D ]}|d | kr,|  S q,d S )Nr   r   )r   r   r   r   r/   )r   r  r   Zattach_info_listattachr#   r#   r$   _get_confluence_attach_info  s
    z/ConfluenceAPIImport._get_confluence_attach_infoc                 C   sX  ddl m} dd l}t|d  d}d}| rT||}| D ]}z|dr^W qJ| ||d }	t	j
|}
t	jj|
|dddd	gd
}|st	j| j|d|
d}|jr||X}|jdd |j| dd |jdd |	r|j|	d d kst|j  W 5 Q R X W qJ   | jjd|  |d7 }Y qJX qJW 5 Q R X |S )Nr   )RDiskrY   z /attachments/all_attachments.zipr   rq   r   url_preview_imgurl_preview)rH   rh   rI   T)r*   rh   rq   rH   rg   )Zsmart_backuprn  ZfileSizeu:   Не удалось обработать вложение r   )Zrdisk.rdiskr  rX  r   r   rZ  namelistr*  r  r   r"   r   CmfAttachmentr.   r*   rq   r   rt   upload_filerW   st_sizeAssertionErrorru   rv   r4   r   )r?   rh   r   r  rX  Zall_attach_zip_pathr  r`  Zzip_attach_namer
  Zattach_name
attachmentZattach_fr#   r#   r$   _process_obj_attachments  sB    



z,ConfluenceAPIImport._process_obj_attachmentsc              
   C   s   |  |d}| sd S t|dl}t|}g |d< |  |d}| r~t|d$}|D ]}|d t| qZW 5 Q R X |W  5 Q R  S Q R X d S )Nr   r   labelsr   )r   r   r   r   r/   rm   r   )r  r&   Zdoc_filer   r]  Zlabels_filef2Z	label_strr#   r#   r$   r2    s    
 z!ConfluenceAPIImport._get_documentc                 c   s:   t | d}t|D ]}d|kr&q| ||V  qd S )Nr  r   )r   r   r   r2  )r?   r'  r  r&   r#   r#   r$   _get_documents-  s
    z"ConfluenceAPIImport._get_documentsc                 C   s  d}t dd  | }|dkr&q| j r4qz| jjjj d|d  }| jd|  tjj|d}|stj| jd	||d
 d}|j	r6|j
s|ddg |d d }	| jjd|	  |	d dkrtj}
n| |	| j }
|
s| jjd|d
  d|	  |d d}|rb| jjd|  |d dkrPtj}n| || j }n|
}|d d d }| jjd|  |d dkrtj}n| || j }|d d d }||d
  ||
| j| jjt|d d|d d |d d d  |||d!}|j
r||_|d jd"kr2||d#< t|d$d%rLd	|d&< |D ]}t||||  qP|jr|jd	d' |j  | jjjj d(| }tj j|d}|stj || jd)}||_!|d* d+rt"j#$| j%j&|d* d+ }n| jjjj'}|j( d,| d-| d.|_)d/|_*|jd	d' |j  n| jjd0|  || +||7 }| ,|| |j	rg }|d1 D ]~}d| jjjj d|d  d}tj-jd2d3d4| d4gd5}|r|.| n,| jjd6| d7| t/j0d8 |  j1d97  _1qz||_2| j%j3j4rpz0t56|d d: d }t7| 8|||d; }W n2   | jj9d<|d
   |  j1d97  _1Y nX | :|}| ;||}| <|||_)|jr|jd	d' |j  | =|| W q   | jjd=| t/j0d	d> | j>  |d97 }t?  Y qX q|@| W 5 Q R X d S )?Nr   Fr   rS  r^   r_   zprocess r`   Tr   )r*   rq   r`   rH   rk  tagsr   r   u5   Определяем автора документа r   r   u1   Не найден автор документа: r   r   u:   Определяем владельца документаr   r   =   Определяем последнего редактора r   r   rK   rC  )status_typerl  rm  )rh   rH   rs  rv  ZworkflowrC  ru  rx  r`   rw  r{  ZCLOSEDr|  r   tableZfull_screenrg   z
::comment:)r`   r*   Z_linksZwebuiu4    импортирован из Confluence: <a href="z" target="_blank">z</a>r   uQ   Документ изменен пользователем, пропускаем r  r`   ra   rb   rc   u   Не найден тег z ext_id=r5   r   atlas_doc_formatcontentC   Не удалось конвертировать разметку: u   Ошибка создания r5   exc_info)Ar   r.   r*   r   rn   ro   r  r   r"   rq   is_newrc  r4   r   rM   rt  rl   r7   r   r0   Zget_default_statusr   Zget_status_typer|  r  r   r   setattrrs   rt   ru   rv   rr  rh   r   rY   r   r-   r   ext_urlverbose_namer   	log_levelr  r  CmfTagrm   r3   ERRORr1   r  r8   r9   r   r   r   _convert_atlas_docr   ry  _process_inline_commentsrz  r   Z
log_detailrG  r  )r?   r(  archive_folder	obj_queuer\  r  ry   r`   rz   authorrv  r	  rs  modifiedrw  r   Zdoc_dictr   Zcomment_ext_idr  r   r  Ztag_rawZ
tag_ext_idr   	atlas_rawr#   r#   r$   _process_document_fork4  s    
  




z*ConfluenceAPIImport._process_document_fork)r   c              
   C   s   t |d}tjjdd|gdddgddd ggddd	gd
D ]}zr|jd d d r|jd|jd d d idD ]<}|jd |jd< |j|jd< |jd |j	 d|jd< qnW q8   | j
jd| d|j  |  jd7  _Y q8X q8t|S )Nhtml.parserrh   ri   rp  Tr{  rj   r   rC  rb  rn  ZinlinePropertiesZ	markerRefzdata-refr   zdata-ref_origclasscmf_converteduO   Не удалось конвертировать inline комментарий r   r   )r   r   rr  re  r{  r   r   r_   rm   rC  r*   r4   r   r   r1   r   )r?   r   r   doc_soupr  r   r#   r#   r$   r    s(    


z,ConfluenceAPIImport._process_inline_commentsc              	   C   s  g }| j  }| j  }tjjd|d}| jjd t| j	D ]N}| j
j| jd| ||||dd}|| |  | jjd|  q@t | _|d}	|	st|d	 d
 d}	| |d D ]h}
|
d |	d krq|
d d d rdd |
d d d D | j|
d < |
d | jkr$q||
 q|D ]}|d q4| jjd |D ]}|  ~qX| j rt  | jjd d S d
}| s|| 7 }q| jjd|  |  j|7  _d S )Narchive)Zsys_typer|  uD   Грузим документы в несколько потоковr  )r  r\  r(  r  r  u.   Параллельный обработчик homepager;  r   rW  rY   r_   r.  r0  r\   c                 S   s   g | ]}|d  qS rW  r#   r   r#   r#   r$   r     s     z:ConfluenceAPIImport._process_documents.<locals>.<listcomp>rS  u0   Ждем обработки документовr  uI   Закончили обработку документов, ошибок )rB   r  r   	CmfFolderr.   r*   r4   r   rY  rD   rA   r  r  rm   r  r   r<   r  r>   r  r   r   rF  r   r  r1   )r?   r   r(  Zobj_processr  r\  r  r!  r"  r  ry   r1   r#   r#   r$   _process_documents  sV    



$
z&ConfluenceAPIImport._process_documentsc                 C   s4  | j  D ]"\}}z| j r(W  q0| jjjj d| }tjj|dgd}|D ]~}| jjjj d| }tjj|ddgd}|s| jj	
d| d|  qT|jrT|j|krT||_|js|j|_|jdd	 qT|r|jsd|_|jdd	 t  W q
   t  | jj	d
|  |  jd7  _Y q
X q
d S )Nr^   tree_node_is_branchrj  r|  rq   u-   Не найден потомок по ext_id=z tree_parent_ext_id=Trg   uI   Не получилось привязать ноду tree_parent_ext_id=r   )r<   itemsr*   r   rn   ro   r   r"   r.   r4   r3  rq   r|  rh   rt   r  rF  rG  r   r1   )r?   Ztree_parent_ext_idr.  r|  r`   r4  r#   r#   r$   _process_documents_tree  s2    


z+ConfluenceAPIImport._process_documents_treec                 C   s4  |  | jjd }|d d d dkr<dtjjddd}n`|d d d d	krfdtjjd
dd}n6|d d d dkrdddtjjd
dd}ntd|   |  D ]}zD| j	 rW  q0|d dd | j
D krW q| jjd|d   |d |d< |d |d< | jjjj d|d  |d< tjjdd|d gddgd}d}|r|jj|d jks|jr|j|d kr|d  d|d  d|d< | jjd|j d|j d |d  d!|j d"|d  
 n(|js|d |_|jd#d$ |j  tjj|d | d%}|sBd#}tj| j|d |d d&}t|d' d(}|d)rv| ||d) d }n(| jjd*|  | ||d+ d, }|s| jjd-| tjd#d. W q|d/ d0 }| jjd1|  |d2 d3krtj |d4< n| !|| j" |d4< |d/ d5}	|	rh| jjd6|	  |	d2 d3krRtj |d7< n| !|	| j" |d7< n|d4 |d7< |d/ d8 d9 }
| jjd:|
  |
d2 d3krtj |d;< n| !|
| j" |d;< |d/ d< |d=< |d/ d8 d> |d?< | #|| | $|| |j%s|j&r|D ]}t'||||  q|d@ dA d }| j(j)j*rz0t+,|d@ dB d }t-| .|||dC }W n2   | jj/dD|dE   |  j0dF7  _0Y nX | 1|}| 2||}| 3|||_4||_5|  |j  | 6|| || _7|rtj8jdGdHd|jgdddIggdJ}|d d d dkrZtj9j|dKdL}|r|:  nv|d d d d	ks|d d d dkrtj9j|dKdL}tj9j|dMdL}|r|:  |r|:  |r|:  t;  | <|| | =|| | >  W q   t?  | jj/dN|  |  j0dF7  _0Y qX qd S )ONr7  rL   rK   r_   r(  zproject.agile:defaultr+   )project_type
logic_typeZproject_wikizproject.wiki:defaultr   sectionzCmfWiki:Class)r  	parent_idtree_parent_idr  u   Неизвестный типc                 S   s   g | ]}|d  qS rW  r#   r   r#   r#   r$   r   )  s     z7ConfluenceAPIImport._process_spaces.<locals>.<listcomp>u0   Импортируем пространство r   rH   task_code_prefixr^   r`   ri   r  rb  Fr   r   u   Проект с именем u    уже существует! z != z id=u   Переименуем в Trg   rj  )r*   r`   rH   rY   r/  r  uC   У пространсва нет домашней страницы r;  r   uD   Не найдена домашняя страница в дампе r  r   r   uD   Определяем автора домашней страницы r   r   rv  r   uJ   Определяем владельца домашней страницы rs  r   r   r  rw  rl  ru  rm  rx  r   r   r  r  r  r   r   ZANDr  u   Документыrc   u   Документ1)r|  rH   u   Документ2u5   Ошибка импорта пространства )@rU   r*   r=  r   ZCmfLogicTyper.   rD  rB  r+  r   r@  r4   r   rn   ro   
CmfProjectr  r_   r`   r   rH   rt   ru   rv   keysr   r   r2  r  r3   r  rM   rt  rl   r7   r  r  rq   r  r  r-   r8   r9   r   r   r   r  r   r1   ry  r  rz  r   r{  r   r(  r  r"   rx   rF  _process_project_tagsr  r  rG  )r?   rP   Zproject_fieldsr   r(  Zproject_is_newr  r  r  r	  r  r   r   r  Zrm_doc_folderZrm_doc1Zrm_doc2r#   r#   r$   _process_spaces  s    
<





z#ConfluenceAPIImport._process_spaces)r   c              
   C   s  ddl m} t }| |d D ]:}| j r4 d S |d D ]}|d |kr<|||d < q<q | D ]j}| j rz d S |d ddd	d
}||ddd}zd| jjjj	 d|d  d}	t
jjddd|	 dgdgd}
|
s&dddd| dgdd|gddd| dgg}t
jj|dgd}
|
s@t
j|| jd|	d}
|	|
jkr\|
j |	 |
_||
_||
_|
jr||
jdd |
j  W qd   | jjd| tjddd |  jd7  _| jj  Y qdY qdX qdd S )Nr   )translitrY   r  r_   rH   r   r   #r   ruT)Zlanguage_codereversedr^   r`   ra   rb   r   rb  ORaliasZILIKE""%)rH   r*   rq   r`   rg   u*   Не удалось создать тег r   r  r   )Ztransliterater  r   r  r*   r   valuesr   rn   ro   r   r  r.   r`   rH   r{  rs   rt   ru   rv   r  r3   r  r1   r   )r?   r   r(  r  r  ry   r   Ztag_nameZ
name_aliasr`   Ztag_objr~   r#   r#   r$   r    sJ    

 z)ConfluenceAPIImport._process_project_tagsc              
   c   sN   | j  d}t|D ].}t||dd}t|V  W 5 Q R X qd S )NrJ  r   r   r   )r?   rK  Zgroup_idr   r#   r#   r$   _get_groups  s    zConfluenceAPIImport._get_groupsc              
   c   sF   | d}t|D ],}t| |d}t|V  W 5 Q R X qd S )Nr   r   )r   r   r   r   r   r/   )r?   Z
group_pathZ	user_pathr   r   r#   r#   r$   _get_group_members  s    
z&ConfluenceAPIImport._get_group_membersT)r   rF   c              	   C   sJ  | d}| jjd|  t|d t| dd d}|s| jjd|  tjj	j
ttj d }t|d d |  d	tj }t|d ||t| dd d
}d| jjjj d|| j  d}d}|d d}t|dkr|d nd|d< t|dkr|d n|d |d< tjj dd|gdgd}	|	sBtjj ddd| dgdgd}	|	sBtjjdd|d gd}
|
st|dkrdd|d g}nHt|dkrddd|d gdd|d ggdd|d gdd|d ggg}tjj|d}
t|
dkr4|  jd7  _| jjd|d  dt|
  |
rB|
d }	|	s|rxd}| jd|  tj||| jd}	nd |fS |	js|	jr|D ]}t|	|||  q|	dd d!g |	jr||	jkr|	j | |	_n||	_tj  }tj! }||	j"kr|	j"#| ||	j"kr,|	j"#| d|	_$| j j%d7  _%|	|fS )"NrG   u2   Обрабатываем пользователя ZdisplayNamer   T)rH   does_not_worku8   Не указан Email для пользователя r   @)rH   rG   r,   r  r^   Fr   r   r   Z
first_name	last_namer  ri   r   rb  r`   ra   rb   rH   rc   r  u+   Пользователей с именем z > u1   Создадим пользователя ext_id=)r`   r  r*   rg_member_of
user_local)&r.   r*   r4   r   r   r   warnZcmfrI   ZCmfEmailZ
max_lengthr   rC   
ORG_DOMAINZcmfutilZtranslit_striprn   ro   r7   r   r   r    rw   r1   r3  r  rq   r  r  rc  r`   rp   Zconfluence_group
user_groupr  rm   r  r:   )r?   r   r~  rG   Z	user_dictZmax_lenr`   
is_createdZ	full_namer|   Zpersonsr~   r   Z
conf_groupr  r#   r#   r$   _process_person  sv    
" " "

z#ConfluenceAPIImport._process_personc           	   	   C   s  | j jjddsd S t }| dD ]r}| j  r< d S | |\}}|jrf|j	dd |j
  |r|jjtjs||jj |dg D ]}d}| jjjrd}d| j jjj d||  d}tjjd	d
d| dgdgd}|s
tjjdd|d gdgd}|s,tj|| j |d d|d}n|jr>|d |_||jkrZ|j | |_|j	dd |j
  |j| q|jr&|j	dd |j
  q&|r| j jjddr| j jdt| d|  t tj!j"|gd d S )NZ
load_usersTr   rg   rJ  rH   r_   r^   r`   ra   rb   r   rb  ri   r`   r*   rH   rq   r{  send_invitesu,   Отправляем приглашения u    пользователям: )args)#r*   r=  rK   r.   r=   r   r   r  rs   rt   ru   rv   rG   r*  rC   r  r>  r-   r8   r9   rn   ro   r   rp   rq   rH   r`   r  rm   r4   r   r   Zcmf_deferred_taskr    Zregister_persons)	r?   r  r   r|   r  r}   r   r`   r]   r#   r#   r$   _process_persons  sL    


  

z$ConfluenceAPIImport._process_personsc                 C   sF  |   D ]6}d}| jjjr d}d| jjjj d||  d}tjj	ddd| dgdgd}|s~tjj	dd	|d gdgd}|stj|| j|d d
|d}n|j
r|d |_||jkr|j | |_|jd
d |j  | | j d|| D ]D}| |d\}}|sq|j| |jr|jd
d |j  qqd S )NrH   r_   r^   r`   ra   rb   r   rb  ri   Tr  rg   rJ  F)r  r-   r8   r9   r*   rn   ro   r   rp   r.   rq   rH   r`   rt   ru   rv   r  r   r   r  r  rm   rs   )r?   r}   r   r`   r]   r   r|   r   r#   r#   r$   _process_groupsA  s6    
  


 z#ConfluenceAPIImport._process_groupsc                 C   s   t dD ]}|   | js q qt dD ]}|  }|s( q>q(|  j|7  _|   |   |   | jj	d| j  | j S )u   
        Запуск импорта

        :return bool: Истина в случае успешного импорта
        rT  u#   Ошибок обнаружено: )
rY  rR  r1   rf  r  r  _process_cross_linksr*   r4   r   )r?   r   r1   r#   r#   r$   process_import^  s    z"ConfluenceAPIImport.process_importc                 C   s<   |j dtdids*|jdks*| dr8||  |S )uS   
        Из документов MS создаем наши превью
        styleZ
backgroundr  Zimg)findrecompilerH   replace_withcreate_preview_tag)r?   r   r  r#   r#   r$   _create_previewt  s    *z#ConfluenceAPIImport._create_previewc                 C   s~  dd l }|s|S t|d}|dd D ]J}|jdrP| jjd|  q(|dr|jd 	dr| jjd	|  q(| jjd
|  |dr|jd }n
|jd }d|jddkrP|jd 
dd 
dd }| |}	|	r4|jd |jd< |	j|jd< d|jd< | jjd|	j  q(| jjd|jd   q(tdt|}
|
r|
d}tjj| jjjj d| d}|r(d|jd< |jd |jd< |j|jd< q(tj|
dd 
dd }tj|}|jdkrL|jdrL|jd |j
dd krLt| j dt|jj ! d d|jd dd}|" s|  j#d7  _#| jj$d |  q(n|%|d!}||& kr|'|^}tj(j||d"}|stj(||| jd#}|j)dd$ |*|+  |j)dd$ |j,-  W 5 Q R X n4|  j#d7  _#| jj$d%| d&|  W 5 Q R  q(W 5 Q R X tj(j|d'd(d)g|d*}|r*|drN|jd |jd+< |j.j/|jd< d|jd< |d,r|jd, |jd-< |jd,= t01|jd 
dd }|r"|d r"|d 	d.r"d.|_2d/|jd/< d0|jd1< d0|jd2< n,|rB|d rN|d 	d3sN| 3|| |dr|jd |jd< |j.j/|jd< d|jd< t01|jd 
dd \}}|r|	d.rd.|_2d/|jd/< d0|jd1< d0|jd2< |4d4}|jd |jd< ||jd5< |5| n| 3|| | jjd|j.  n6|jd6d7kr`| jj$d8|  |  j#d7  _#q(| jjd9|  q(t|S ):Nr   r  c                 S   s   |  dp|  dS r   r   r   r#   r#   r$   r     r   z4ConfluenceAPIImport._process_links.<locals>.<lambda>r  u   Уже обработали r   r  u   Это якорь u&   Обрабатываем ссылку r   z/people/r   r   r   r   	href_origTu   Заменили на u?   Пользователь не найден для ссылки z+href="/pages/viewpage.action\?pageId=(\d+)"r   r^   r  r"   z!data-linked-resource-container-idr$  r   r/  r   r   uA   Документ с файлом не найден в дампе r   )rH   rh   )rH   rh   r*   rg   u	   Файл u    не найден в r   r  r  )rH   rI   rh   Zsrc_origZsrcsetZsrcset_origZvideoZcontrols100%widthheightZimagesourcer   zdata-linked-resource-typer  u%   Не найдено вложение u'   Не обработали ссылку )6rX  r   r   r   r.   r*   r4   r   r   r   r   rl   r   r   r  searchr   r]   r   r"   rn   ro   r   r   r   r   
class_namer`   r   r   r   rh   r{  r/   r   r1   r3  rZ  r  r   r  rt   r  rW   ru   rv   r   rK   r   r   rH   r  new_tagrm   )r?   r   rh   Zmake_previewrX  Z	text_soupr   r   Z
account_idr|   matchr   Zrel_docrH   r   attach_pathr`  r   Z
attach_objr   r_  r  r#   r#   r$   rz  |  s    





 
 


 

 





z"ConfluenceAPIImport._process_linksc                 C   sf  | j jd t|d}|jdtdidD ],}z|jdrT| 	|jd }nv|jdrt
jj|jddd	d
}| |}| 	|| j }|jdr|jd |jd< |jd |jd< ntd|jd |jd< |j|jd< d|jd< |jj|jd< |jj|jd< W q. tk
rZ } z,| j jd| d|  |  jd7  _W 5 d}~X Y q.X q.t|S )u]   
        Упоминания пользователей
        confluence-userlink
        uH   Конвертируем упоминания пользователейr  r  zconfluence-userlinkr  zdata-account-idzdata-usernamezutf-8r   )r   r  zdata-linked-resource-idzdata-linked-resource-id_origzdata-username_origu]   Не найдены аттрибуты для определения пользователяr   r  Tr  uZ   Не удалось преобразовать ссылку на пользователя r   r   N)r*   r4   r   r   r   r  r  r   r.   rl   r   r   Zunquote_plusr   r7   r   r   r_   rK   r  r3  r1   r   )r?   r   r  r   r|   Z	user_namer   r   r#   r#   r$   ry    s.    


"z%ConfluenceAPIImport._process_mentionsc                 C   s   |dkrd S |  |}| |d\}}|s| jjjdrptjj| jjd d}| jjd| d| t	j
d n*| jjd| dt	jd	d
 |  jd7  _|S )NunknownFZdefault_user_idrW  u6   Не найден пользователь user_id = "uA   ", ставим пользователя по умолчанию r  r  T)r5   r   r   )r   r  r*   r=  rK   r.   r   r    r  r3   ZWARNINGr  r1   )r?   r   r[   r|   r   r#   r#   r$   rl     s    
 zConfluenceAPIImport._get_person)r   r  rF   c           )      C   s~	  dd l }|s$| jjd t|d}t|}d}t|D 	]>}| j rN 	qz|d7 }|d dkr4|jddd	d
dd}	dd |j	ddidD 
 }
g }|
jD ]}|dkrq|| qt|d D ]N\}}|d d }|jd| dd	d| ddd}|| j|_|	| q|
|	 tt|d}n"|d dkrd }
|d dr|jd|d d id}
|
s| jjd|d d  d|d d   q8nNd|d d kr|d d d d r|jd!|d d d d  d" id}
|d d d#krd$}|jd|d d id}
|
r|d d d% d& d" }|d d d% d'}|rt|dd&j|d" |d(}|
|j tt|d}	qV|d d d)kr2d*}|jd|d d id}
|
r|d d d% d& d" }|rt|dd&jd&d+d |
jD |d,d}|
|j tt|d}	qV|d d d-r,|d d d% d. d" }tj|}tjj||d/}d0}|s(|d d d% d1 d" }t| j d2t|jj d3|d4d5}|! s|  j"d7  _"| jj#d6|  q8n|$|d7}||% kr|&|>}tj|}tj||| jd8}|j'd9d: |(|)  W 5 Q R X t*+| d;}|% D ]v}|,|rv|&|T}tj|}tjj||d/}|stj||| jd8}|j'd9d: |(|)  W 5 Q R X qvW 5 Q R X |s(|  j"d7  _"| jj#d<| d=|  q8tj|}tjjd>d?|gd'd@| dAggdBgdC}dD}dD}|d d d% dr|d d d% d d" }|d d d% dEr|d d d% dE d" }|j-j.|j/j.||dF} |r|j0| dG< nd&| dG< t|dd&jf | d}|
|j tt|d}	qV|d d dHkr2dI}!|j/j.}"|d d d% dJrtjj|d d d% dJ d" dK}#|#stj1j|d d d% dJ d" dK}#|#r|#j/j.}"n| jjdL|d   q8t23d9|"|"dMd&dNdOdPdQdR}$t|!j|"|$dSdd&d}%|
|%j tt|d}	qV|d d dTkr|jddTd	dUd}&|jdVddWid}'|'|d d d% dW d"  |&|' |jdXdYddZd[d}(|(|d d d% d\ d"  |&|( |
|& tt|d}n|d d d]k	rVtj|d d d% d' d" }tjj|dK}|sh| jj4d^|d d  d_| t5j6d9d9d` |  j"d7  _"|
|7  tt|d}n|d dak	rV|d d dbk	rV|d d d% d'	rV|d d d% d' d" }dc}|jd|d d id}
|
	rBt|dd&j|d&ddd |
jD ded}|
|j tt|d}n| jjdf|  |dr8| 8t|||d |}q8|S )gNr   uC   Конвертация текста из atlassian разметкиr  r   r   ZlayoutSectionZdivZfalsetruezmceNonEditable selected)contenteditablezdata-layout-sectionr  r  c                 S   s,   g | ]$}d |j d krd|j d kr|qS )zfixed-widthr  Zsingler  r   r#   r#   r$   r   1  s     z:ConfluenceAPIImport._convert_atlas_doc.<locals>.<listcomp>r  ZcolumnLayoutr   r  r   r  rb   zflex-basis: )zdata-column-widthzdata-layout-columnr  	extensionZlocalIdzdata-local-idu&   Не удалось найти тег u    для ZextensionKeyZmacroMetadataZ
parametersZmacroIdzdata-macro-idrK   zexcerpt-includeuG  
                    <div class="macros-panel macros-include-block" contenteditable="false" data-include-block-name="{name}" data-localid="{page_name}" data-mce-contenteditable="false">
                    <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false">
                    <span aria-label="Макрос Include. " role="img"> <svg height="14" id="Outline" viewBox="0 0 24 24" width="14" xmlns="http://www.w3.org/2000/svg"><path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span> <span class="extension-title">Макрос Include. "Вставить выборку"</span> <span class="macros-actions"> <span class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17="" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;" viewBox="0 0 24 24" data-mce-style="width: 14px; height: 14px; color: #444444;"><path _ngcontent-aiw-c17="" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z" fill="#444"></path></svg> </span> <span class="macros-delete"> <svg class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;" viewBox="0 0 18 18" data-mce-style="width: 14px; height: 14px; color: #222222;"><path class="ng-star-inserted" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path></svg> </span> </span>
                    </div>
                    <div class="ak-renderer-wrapper" data-fabric-macro="null" data-macro-body="" data-macro-parameters="[object Object]">
                    </div>
                    </div>
                    </div>ZmacroParamsr   rH   )rH   	page_nameZincludeu  
                    <div class="macros-panel macros-include-page" contenteditable="false" data-macros="include-page"
                         data-param-pageid="{page_name}" data-mce-contenteditable="false"
                         data-rendered="true">
                        <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false">
                            <span aria-label="Макрос Include. " role="img"> <svg xmlns="http://www.w3.org/2000/svg" id="Outline"
                                                                                 viewBox="0 0 24 24" width="14" height="14"><path
                                d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span>
                            <span class="extension-title">Макрос Include. "Вставить страницу"</span> <span class="macros-actions"> <span
                            class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17=""
                                                                                                                     viewBox="0 0 24 24"
                                                                                                                     class="ng-star-inserted"
                                                                                                                     style="width: 14px; height: 14px; color: #444444;"
                                                                                                                     data-mce-style="width: 14px; height: 14px; color: #444444;"><path
                            _ngcontent-aiw-c17="" fill="#444"
                            d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path></svg> </span> <span
                            class="macros-delete"> <svg viewBox="0 0 18 18" class="ng-star-inserted"
                                                        style="width: 14px; height: 14px; color: #222222;"
                                                        data-mce-style="width: 14px; height: 14px; color: #222222;"><path fill="#222"
                                                                                                                          d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z"
                                                                                                                          stroke="none"
                                                                                                                          stroke-width="1"
                                                                                                                          class="ng-star-inserted"></path><path
                            fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path><path
                            fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path></svg> </span> </span>
                        </div>
                        <div class="ak-renderer-wrapper">{content}</div>
                    </div>
                    c                 S   s   g | ]}t |qS r#   r   )rR   valr#   r#   r$   r     s     )r  r  ZdrawioZdiagramName)rh   rH   u  
                    <div data-drawio-title="{name}" data-drawio-type="diagram" data-attach-id="{attach_id}" class="drawio-content" contenteditable="false">
                        <div class="header">Диаграмма: "{name}"</div>
                        <div class="content">
                            <img cmf_converted=True src="{img_src}" alt="" data-mce-src="{img_src}" width="{width}" height="{height}">
                        </div>
                    </div>
                    ZpageIdr$  r/  r   r   uI   Документ с диаграммой не найден в дампе r   )rh   rH   r*   Trg   z.*\.png$u   В документе u$    не найден файл с xml: rh   ri   ra   z%.pngr   rb  r  r  )rH   Z	attach_idr  r  Zimg_srcr.  u
  <div class="macros-panel" contenteditable="false" data-children-docs="{doc_id}" data-macro-parameters="{params}" data-macros="children-docs" data-mce-contenteditable="false" data-rendered="true">
                            <div class="extension-container" data-layout="default"><span aria-label="Макрос " role="img"><svg xmlns="http://www.w3.org/2000/svg" id="Outline" viewBox="0 0 24 24" width="20" height="20"> <path d="M22.485,10.975,12,17.267,1.515,10.975A1,1,0,1,0,.486,12.69l11,6.6a1,1,0,0,0,1.03,0l11-6.6a1,1,0,1,0-1.029-1.715Z"></path> <path d="M22.485,15.543,12,21.834,1.515,15.543A1,1,0,1,0,.486,17.258l11,6.6a1,1,0,0,0,1.03,0l11-6.6a1,1,0,1,0-1.029-1.715Z"></path> <path d="M12,14.773a2.976,2.976,0,0,1-1.531-.425L.485,8.357a1,1,0,0,1,0-1.714L10.469.652a2.973,2.973,0,0,1,3.062,0l9.984,5.991a1,1,0,0,1,0,1.714l-9.984,5.991A2.976,2.976,0,0,1,12,14.773ZM2.944,7.5,11.5,12.633a.974.974,0,0,0,1,0L21.056,7.5,12.5,2.367a.974.974,0,0,0-1,0h0Z"></path> </svg></span> <span class="extension-title">Макрос "Отображение дочерних"</span> <span class="macros-actions"> <span class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17="" viewBox="0 0 24 24" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;" data-mce-style="width: 14px; height: 14px; color: #444444;"><path _ngcontent-aiw-c17="" fill="#444" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"></path></svg> </span> <span class="macros-delete"> <svg viewBox="0 0 18 18" class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;" data-mce-style="width: 14px; height: 14px; color: #222222;"><path fill="#222" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path><path fill="#222" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted" ></path><path fill="#222" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" stroke="none" stroke-width="1" class="ng-star-inserted"></path></svg> </span> </span></div>
                            <ul></ul><p></p></div>r0  rQ   uE   Не удалось обработать тег оглавления Z50ZnoneF)ZisShowChildrenZparentIdZ
documentIdZchildrenAmountr5   ZincludeViewTypeZisOrderByDeskr  z&quot;)r&   paramszpaste-code-macro)r  r  pr   Zprezlanguage-markup1)r  r  zdata-mce-selectedZ__bodyContentZviewxlsu)   Ошибка обработки тега u'     вложение не найдено r  ZbodiedExtensionZexcerptu  
                    <div class="macros-panel macros-excerpt-block" contenteditable="false" data-excerpt-name="{name}" data-mce-contenteditable="false">
                    <div class="extension-container" contenteditable="false" data-layout="default" data-mce-contenteditable="false"><span aria-label="Макрос Include. 'Создать выборку'" role="img"> <svg height="14" id="Outline" viewBox="0 0 24 24" width="14" xmlns="http://www.w3.org/2000/svg"><path d="M19.949,5.536,16.465,2.05A6.958,6.958,0,0,0,11.515,0H7A5.006,5.006,0,0,0,2,5V19a5.006,5.006,0,0,0,5,5H17a5.006,5.006,0,0,0,5-5V10.485A6.951,6.951,0,0,0,19.949,5.536ZM18.535,6.95A4.983,4.983,0,0,1,19.316,8H15a1,1,0,0,1-1-1V2.684a5.01,5.01,0,0,1,1.051.78ZM20,19a3,3,0,0,1-3,3H7a3,3,0,0,1-3-3V5A3,3,0,0,1,7,2h4.515c.164,0,.323.032.485.047V7a3,3,0,0,0,3,3h4.953c.015.162.047.32.047.485Z"></path></svg> </span> <span class="extension-title">Макрос Include. 'Создать выборку' <span class="excerpt-title">| name = {name}</span></span> <span class="macros-actions"> <span class="macros-edit" style="margin-right: 7px;" data-mce-style="margin-right: 7px;"> <svg _ngcontent-aiw-c17="" class="ng-star-inserted" style="width: 14px; height: 14px; color: #444444;" viewBox="0 0 24 24" data-mce-style="width: 14px; height: 14px; color: #444444;"><path _ngcontent-aiw-c17="" d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z" fill="#444"></path></svg> </span> <span class="macros-delete"> <svg class="ng-star-inserted" style="width: 14px; height: 14px; color: #222222;" viewBox="0 0 18 18" data-mce-style="width: 14px; height: 14px; color: #222222;"><path class="ng-star-inserted" d="M16.5 3H12.75V1.5C12.75 1.10218 12.592 0.720644 12.3107 0.43934C12.0294 0.158035 11.6478 0 11.25 0L6.75 0C6.35218 0 5.97064 0.158035 5.68934 0.43934C5.40804 0.720644 5.25 1.10218 5.25 1.5V3H1.5V4.5H3V15.75C3 16.3467 3.23705 16.919 3.65901 17.341C4.08097 17.7629 4.65326 18 5.25 18H12.75C13.3467 18 13.919 17.7629 14.341 17.341C14.7629 16.919 15 16.3467 15 15.75V4.5H16.5V3ZM6.75 1.5H11.25V3H6.75V1.5ZM13.5 15.75C13.5 15.9489 13.421 16.1397 13.2803 16.2803C13.1397 16.421 12.9489 16.5 12.75 16.5H5.25C5.05109 16.5 4.86032 16.421 4.71967 16.2803C4.57902 16.1397 4.5 15.9489 4.5 15.75V4.5H13.5V15.75Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M8.25 7.5H6.75V13.5H8.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path><path class="ng-star-inserted" d="M11.25 7.5H9.75V13.5H11.25V7.5Z" fill="#222" stroke="none" stroke-width="1"></path></svg> </span> </span></div>
                    <div class="ak-renderer-wrapper" contenteditable="false" data-fabric-macro="null" data-macro-body="" data-macro-parameters="{name}" data-mce-contenteditable="true">
                    {content}
                    </div>
                    </div>
                    c                 S   s   g | ]}t |qS r#   r  r   r#   r#   r$   r     s     )rH   r  u   Не нашли тег для )9rX  r*   r4   r   r   r   r  r   r  r   popcontentsrm   	enumerater  r   r.   r  r   r   formatZ
currentTagr   r*  r   r"   r   r  r   r   r   rh   r  r   r1   r3  rZ  r  r   rt   r  rW   r  r  r  rH   rK   r_   r   r  r   r   r  r3   r  r  r  ))r?   r   r   r  souprX  r  r!  elemZtag_layout_sectionr   r  ccolr  Ztag_layout_colZTAG_TMPLr  Zexcerpt_namerS   rH   r   r  r   r  r`  r   ZregexpZname_zipZfile_name_zipZ
attach_imgr  dataZTMPLr  rh   r  Znew_soupZ
code_blockr   rK   r#   r#   r$   r     sr   






**
"

&"
"




   *.
z&ConfluenceAPIImport._convert_atlas_docc              	      s:  t t d fdd}t t d fdd}dddgd	d
dgg}dD ]}tt| }|j|d} jjd|j d|  d}d}|dkr@ j r jjd  dS |j	|ddddg|| || | gd}	|d7 }||8 }|	D ]J}
zd}t
|
drJ||
jj|
_||
jj|
_|
jr|d7 }|
jdd |
  |
j  nB||
jj|
_||
jj|
_|
jr|d7 }|
jdd |
j  |
jD ]J}|jdkrq||jj|_|jr|d7 }|jdd |
j  q jjd|
 d|  W q   t   jjd|
    jd7  _Y qX qqq@dS )uU   
        Заменим ссылки на задачи и документы
        )r   rF   c           	         s0  | s| S t | d}|dD ]}|jdr0q|jd}|sV jjd|  q|dsv jjjj	j
 | }| jjjj	j
rd }d }d|jd kr2td|jd D ]<} jjjj d	|d
d   }tjj|dgd}|r qq|s jjd|jd   q|s jjd|  qnd|jd krtd|jd }|st jjd|jd   qtjjddd|dd
d   dgdgd}|D ].}|jd |dd
d  kr|} q֐q|s jjd|dd
d    qnq|tj| qddd |jjD S )Nr   ar  r   u!   Ссылка без адреса r   z/wiki/z\/\d+r^   r   rj  u:   Не удалось найти ид документа в u:   Не удалось найти документ по ид z/browse/z	\/\w+-\d+u6   Не удалось найти код задачи в r{  ra   z%"key":"r   r  rb  r   u:   Не удалось найти задачу по коду r   c                 S   s   g | ]}t |qS r#   r  )rR   xr#   r#   r$   r   @  s     zQConfluenceAPIImport._process_cross_links.<locals>.check_links.<locals>.<listcomp>)r   r   r   r.   r*   r4   r   r   rn   r  rK   r  findallro   r   r"   r  CmfTaskrw   r]   r{  r  ZCmfPluginCsvZcreate_tag_linkr   r   r.  )	r   r  r   r   r   r`   Zext_codeobjsr   rN   r#   r$   check_links  sZ    

$
"z=ConfluenceAPIImport._process_cross_links.<locals>.check_linksc                    s&  | s| S t | d}|jddidD ]}|jdr4q"|jd}|sZ jjd|  q"|jd} jjjj d	| }t	j
j|d
}|st	jjdd jjjj dgddd| dggd}|rt|}|jj|d< |jj|d< t||d< d|d< |d= q" jjd|  q"t|S )uz   
            Заменяет ссылку на корневую страницу в макросе Page Tree
            r  zdata-macrosz	page-treer  r  zdata-macro-parametersu3   Макрос Page Tree без параметров Zroot_page_originalr^   r  r`   ra   z::%r{  z%"homepage":{"id":"r  rc   r?  Z
rootLocaler  ue   В макросе Page Tree не удалось найти корневую страницу по ID )r   r   r   r.   r*   r4   r   rn   ro   r   r"   r  r   r   r_   rK   rH   r   r   )r   r  r   Zdata_macro_parametersZroot_page_idZroot_page_ext_idZpagetree_rootZmacro_paramsrN   r#   r$   check_pagetree_rootB  s<    

zEConfluenceAPIImport._process_cross_links.<locals>.check_pagetree_rootr*   rj   Nrq   ri   T)r"   r  r  r  rc   u)   Конвертируем ссылки в u=    на локальные задачи и документы: rM  r   u'   Отменили конвертациюr   zcomments.log_levelzcomments.text
text_draft)rd   rI   slicer   rg   r   r   u   Проверили u!    и конвертировали u>   Не удалось конвертировать ссылки )r   varsr   rd  r*   r4   r   r  r   rw   hasattrr   rK   r  rs   rt   Z
do_approveru   rv   Zcommentsr  rG  r   r1   )r?   r  r  r~   Z	mode_nameZmodelr  r   r!  r  r   r]  r  r#   rN   r$   r    s`    1+




z(ConfluenceAPIImport._process_cross_links)F)N)T)T)N)?r   r   r   r   r   r   Z	CmfImportr@   r6   rw   rO   staticmethodr   r   rU   rV   r   r   r   r   r   r   r   r   r   r   r  r#  r)  r+  rB  rI  rL  rR  ra  rf  ri  rk   r  r  r  r  r2  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  rz  ry  rl   r  r  r#   r#   r#   r$   r(   &   sn   ?!.

p1	7/)%6!
~- 'C'w nr(   )"r   r  r   Zcmf.appr   pathlibr   Zurllib.parser   r   typingr   r   r   r	   r   Zcmf.data_providers.baser
   Zcmf.includeZcommon.models.cmf_active_entityr   Zmodules.confluence.apir   Z%common.models.cmf_plugin_import_mixinr   Zbs4r   Zatlassian.errorsr   r   Z	dataclassr   r%   r(   r#   r#   r#   r$   <module>   s*   	