U
    h``                    @   s   d dl Zd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZ d dlmZ d dlT d d	lmZ d d
lmZ dd ZG dd dejZdS )    N)permutations)Path)Queue)LockThread)	unidecodecmf_context)*)cmf_plugin_kaiten)KaitenClientc                    s    fdd}|S )Nc                    s    fdd}|S )Nc                    s   zL| j jd d dd dd | j  r2W dS  | f||}t  |W S  tk
r   t  |  jd7  _| j d  Y dS X d S )Nz===== T)Z	anonymousr      u   Ошибка )
cmf_importlog	is_cancel
cmf_commit	Exceptioncmf_rollback
has_errors	log_error)objargskwargsres)funcname ,./modules/kaiten/models/cmf_plugin_kaiten.pywrapper   s     
z3catch_exception.<locals>.decorator.<locals>.wrapperr   )r   r   r   )r   r   	decorator   s    z"catch_exception.<locals>.decoratorr   )r   r!   r   r    r   catch_exception   s    r"   c                       s  e Zd ZdZejjd Zejjdg Z fddZe	dd Z
edd	 Zed
d Zedd ZeedddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zed!d"d#Zd$d% Zdd(d)Zd*d+ Zdd-d.Zd/d0 Zed1ed!d2d3Z d4d5 Z!d6d7 Z"ed8d9d: Z#ed;d<d= Z$ed>d?d@ Z%ddBdCZ&dDdE Z'dFdG Z(dHdI Z)dJdK Z*dLdM Z+dNdO Z,dPdQ Z-ddSdTZ.edUdVdW Z/dXdY Z0dZd[ Z1d\d] Z2d^d_ Z3ed`dadb Z4edcddde Z5edfdgdh Z6edidjdk Z7edldmdn Z8edodpdq Z9edrdsdt Z:edudvdw Z;edxdydz Z<d{d| Z=d}d~ Z>dd Z?dd Z@dd ZAeddd ZBdd ZCdd ZD  ZES )CmfPluginKaitenN)clientZkaiten_fieldsget_all_projectsc                    s0   t  j|| i | _i | _i | _g | _d| _d S )Nr   )super__init__task_relationsZepicssubtasksselected_projectsr   )selfr   r   	__class__r   r   r'   3   s    zCmfPluginKaiten.__init__c                 C   sJ   | j sD| ddddg tt| jj| jj | jjj	t
 d| _ | j S )u5    Клиент для работы с Kaiten REST API pluginzplugin.ext_urlzplugin.ext_tokenzplugin.verify_ssl)urltoken
verify_sslZsession)_CmfPluginKaiten__clientload_fieldsr   strr.   Zext_urlZ	ext_tokenZdecryptr1   valuecmfutilZrequests_session)r+   r   r   r   r$   <   s    

zCmfPluginKaiten.clientc                 C   sR   t | } |  } tdd| } tdd| } tdd| } tdd| } | dS )	N&z and z\+r   z[^\w\s-] z[\s_-]+-)r   lowerresubstrip)textr   r   r   _slugifyN   s    zCmfPluginKaiten._slugifyc                  C   s   g } | S )Nr   )r   r   r   r   calc_models_settingsX   s    z$CmfPluginKaiten.calc_models_settingsc                 C   s   |sdS |  |}|S )u`   
        Метод получает значение параметра объекта
        N)get)r   attrr5   r   r   r   
_get_value]   s    
zCmfPluginKaiten._get_value)
field_namec                 C   s&   |sd S | j |gd}|r"|d S d S )N)Zfields_namer   )Zimport_shop_fields)modelrD   r   r   r   r   _get_field_nameg   s    zCmfPluginKaiten._get_field_namec                 C   s0   ddddddddddd	dd
ddg}|S )uO    Сопоставление настроек для модели models.CmfTaskr   titlemodel_fieldrB   r>   descriptioncmf_created_atcreatedcmf_modified_atupdateddeadlinedue_dater   )r+   Zissue_settingsr   r   r   _get_issue_import_settingsn   s    z*CmfPluginKaiten._get_issue_import_settingsc                 C   s    dddddddddg}|S )uQ    Сопоставление настроек для модели models.CmfPersonr   	full_namerH   emaildoes_not_workZ	activatedr   )r+   Zuser_settingsr   r   r   _get_user_import_settingsz   s
    z)CmfPluginKaiten._get_user_import_settingsc                 C   s   |dkr| S |dkrZ|d kr"|S d| j  d|d  d}tjjddd| dgd}|S |d	kr|d krn|S | j  d|d
 d  d|d  }tjjdd|gdgd}|S |dkr|d kr|S | |d S |S )NrT   
logic_type::idext_idLIKE%filterstatusproject=status_typer]   fieldsresponsible)source_hashmodelsCmfLogicTyperA   	CmfStatus_get_person)r+   r   r5   rD   logic_type_ext_idrV   status_ext_idr^   r   r   r   _normalize_value   s0    "z CmfPluginKaiten._normalize_valuec                 C   sR   i }|D ]D}|d sq|  ||d }| |||d d }|||d d < q|S )uC    Преобразование настроек для модели rI   rB   rD   )rC   rl   )r+   r   settingsZnormalized_objsettingr5   r   r   r   _normalize_obj   s    
zCmfPluginKaiten._normalize_objc                    s
  g  |_ j dg  fdd}d }d }j  d}| r D ]$}|d D ]} qjq`|rN qtqNdD ]} qq~|sdd jj	d	d
D }|r|d }|st
d|sʈj }|st
d||jtj ||jtj  dg d  S )Nplugin.plugin.*c                    s`   | }|D ]>} | |d |d< ||d |d< d|d< |j|d< q
 |j|d d S )NrB   r5   rI   r8   rG   icon)r   rc   )rC   rF   rq   appendverbose_name)r   Zsettings_funcrE   Zobj_settingsrn   resultr+   r   r   _get_settings   s    z<CmfPluginKaiten.tmplt_import_settings.<locals>._get_settingsprojectsrX   usersc                 S   s   g | ]}|qS r   r   ).0_r   r   r   
<listcomp>   s     z9CmfPluginKaiten.tmplt_import_settings.<locals>.<listcomp>r   limitr   uT   Не найдено ни одной задачи, импорт невозможен!ub   Не найдено ни одного пользователя, импорт невозможен!selectedObjects)r   r5   )r   r3   get_download_pathjoinpathexists_get_projects_get_project_tasks_simple_getr$   	get_cardsr   Zget_current_userrQ   rf   CmfTaskrU   	CmfPersonrr   )r+   r   rv   taskuserprojects_dirr_   tasksr   rt   r   tmplt_import_settings   s:    	
z%CmfPluginKaiten.tmplt_import_settingsc              
   O   s  |  dg | jj| _g }z| j}| D ]}t|d }|d }| |}tj	|}	|d rj|d7 }d| j d| d}
tjj
ddd	|
 d	gd
dgd}|r|d7 }||	||d d}|| q,W 5 Q R X W n0 tk
r } zt|  W 5 d }~X Y nX |S )Nzplugin.source_hashrX   rG   Zarchivedu    (архивный)rW   rY   rZ   r[   T--r]   include_deletedrc   u    [импортирован])rX   keyr   original_name)r3   r.   re   r$   Z
get_spacesr4   r?   rf   
CmfProjectZget_prefix_from_namesgetrr   r   gdebug)r+   r   r   rw   r$   space
project_idproject_nameZproject_codeproject_keyproject_ext_idr_   project_dataexcr   r   r   r%      s<    


z CmfPluginKaiten.get_all_projectsc                 C   s   t |}| j| jj d| }|d k	r4t|S | dD ]}t |d|kr\|  S |d|krr|  S |dd | kr|  S |dd | kr|  S |dd | kr>|  S q>t	d	| d S )
N:user:rx   rX   uidrS   r8   usernamerR   uH   В дампе не найдены данные пользователя )
r4   redis_dbrA   r   rX   pickleloadsr   r:   r   )r+   user_id	user_datar   r   r   _get_user_data_from_dump  s     

z(CmfPluginKaiten._get_user_data_from_dumpc                 C   sz  t |}z| |W S  tk
rt   z|d}| rt|d}|D ]}t|}t |d |kr|  W  5 Q R  W  Y S |d|kr|  W  5 Q R  W  Y S |dd	 |	 krL|  W  5 Q R  W  Y S qLW 5 Q R X | j
|}|r4t|d}|tj|dd	d
  W 5 Q R X |W  Y S W n8 tk
rn } z| jjd|  W 5 d }~X Y nX Y nX d S )N
users.jsonrrX   r   rS   r8   a+Fensure_ascii
uS   Не удалось получить данные пользователя по ID )r4   r   r   r   r   openjsonr   rA   r:   r$   Zget_userwritedumpsr   loggerwarning)r+   r   Zdump_dir
users_filefrowr   r   r   r   r   _get_user_data  s2    

&"zCmfPluginKaiten._get_user_datar    c              	   c   sN   | j  | d}| rJt|d}|D ]}t|V  q.W 5 Q R X d S )N.jsonr+)r   r   r   r   r   r   r   )r+   r   	file_pathr   r   r   r   r   r   :  s    zCmfPluginKaiten._simple_getc              
   c   sf   | j  d}| rbt|D ]>}|dr2q"t| d| dd}t	|V  W 5 Q R X q"d S )Nrw   .dirtyz.meta/
/info.jsonr   )
r   r   r   r   oslistdirendswithr   r   load)r+   project_dirr   r   r   r   r   r   B  s    
zCmfPluginKaiten._get_projectsr     c           	   
   c   s   | j  d|d}| s:| j jd| d|  d S d}t| D ]r}| sXqJ|j	}|
drjqJ||k r||d7 }qJ||| kr qt|dd	}t|V  W 5 Q R X |d7 }qJd S )
Nrw   r   u   Каталог u    отсутствует. Возможно он был удален или еще не создан. Необходимо повторно запустить импорт проекта r   r   r   	info.jsonr   )r   r   r   r   r   r   r   iterdiris_dirr   r   r   r   r   )	r+   r   startr}   	tasks_dircounttask_dirtask_idr   r   r   r   r   L  s*    
z"CmfPluginKaiten._get_project_tasksc                 C   s   z|  |}W n,   | jjd| dddd tj Y S X | |dd\}}|s| jjjdrt	j
j| jjd d}| jjd| d	| tjd
 n.| jjd| dddd tj}|  jd7  _|S )Nu6   Не найден пользователь user_id = "u9   ", в дампе, указываем системногоzERR-0045ZCmfUserobj_typeFZdefault_user_idrX   uA   ", ставим пользователя по умолчанию )levelub   ", указываем системного, в дампе, указываем системногоr   )r   r   r   r   system_user_process_personjson_settingsr5   rA   rf   r   r   loggingZWARNINGr   )r+   r   r   personrz   r   r   r   ri   e  s2    

zCmfPluginKaiten._get_personFc                 C   sj   d}| j  d|d}| rft| D ]8}| s:q,|jdrHq,|r\|d s\q,|d7 }q,|S )Nr   rw   r   r   z.processingr   )	r   r   r   r   r   r   r   r   r   )r+   r   Zonly_processingr   r   r   r   r   r   _count_project_tasks  s    
z$CmfPluginKaiten._count_project_tasksc                 C   s,  | j d g }t }| j  d}| rt|d}t|dD ]\}}| }|sZqDz*t	
|}|t|d  || W qD tk
r }	 z"| j d| d|	 d|  W 5 d	}	~	X Y qDX qDW 5 Q R X t|d }
| j|
| |
d
D ]}t|d d}| sqt|}t|dD ]\}}| }|sBq(zPt	
|}t|d }||kr| j| j j d| t| || W nB tk
r }	 z"| j d| d|	 d|  W 5 d	}	~	X Y nX q(W 5 Q R X |  qt|d*}|D ]}|t	j|ddd  qW 5 Q R X d	S )us    Собирает пользователей из файла users.json в каталогах документов u9   Сбор пользователей из объектовr   r   r   rX   u   Ошибка в строке : . Nr|   r   r   w+Fr   r   )r   r   setr   r   r   r   	enumerater=   r   r   addr4   rr   r   r   r   r   r   r   rX   r   r   unlinkr   )r+   r   Zcurrent_usersZcurrent_user_idsZall_users_filer   Zline_numberliner   r   r   r   Ztask_users_filer   r   r   r   _collect_users  s`    
"




$
zCmfPluginKaiten._collect_usersu   Дампc              	   C   s   | j  | d}| r(t| | j  | d}t|d:}| D ],}| j  jd7  _|t	j
|ddd  qPW 5 Q R X t|| | j jdd	 d S )
Nr   z.json.dirtyr   r   Fr   r   TZ	only_data)r   r   r   r   r   remover   json_object_countr   r   r   shutilmovesave)r+   Zapi_funcr   r   Zfile_tmp_pathr   r   r   r   r   _simple_dump  s    

$zCmfPluginKaiten._simple_dumpc           
      C   s   t |d }|d }|d }||}| j d| d| }tjj|| jdddgd}	|	sntj|| j| jd}	||	_||	_||	_	|	j
s| sd	|	_||	_|	  t  d S )
NrX   r   r/   rW   hrefpath)
obj_ext_idre   rc   )r   re   r   F)r4   r   re   rf   CmfImportDownloadrA   r   r   r   r   
is_changedr   
downloadedimport_raw_jsonr   r   )
r+   r   
attachmentattachments_dirattachment_id	file_nameZdownload_urlattachment_fileattachment_ext_iddownload_jobr   r   r   _dump_task_attachment  s0    
z%CmfPluginKaiten._dump_task_attachmentc                 C   sJ   t |d }|d}|jdd |dg D ]}| |d || q.d S )Nr   attachmentsTexist_okfilesrX   )r   r   mkdirrA   r   )r+   r   r   r   r   r   r   r   _dump_task_attachments  s
    
z&CmfPluginKaiten._dump_task_attachmentsu.   Дамп комментариев задачиc              	   C   s   t |d }|d}| r(t| | j|d }t|dN}|D ]B}|drf| 	|d | |
tj|ddd  | jd	 qHW 5 Q R X d S )
Nr   comments.jsonrX   r   	author_idFr   r   comment)r   r   r   r   r   r$   Zget_card_commentsr   rA   r   r   r   r   r   inc_stat)r+   r   r   comments_fileZcommentsr   r   r   r   r   _dump_task_comments  s    


z#CmfPluginKaiten._dump_task_commentsu/   Дамп журнала работ задачиc              	   C   s   t |d }|d}| r(t| | j|d }t|dB}|D ]6}|drf| 	|d | |
tj|ddd  qHW 5 Q R X d S )	Nr   timelogs.jsonrX   r   r   Fr   r   )r   r   r   r   r   r$   Zget_card_time_logsr   rA   r   r   r   r   )r+   r   r   time_logs_fileZ	time_logsr   time_logr   r   r   _dump_task_time_logs  s    


z$CmfPluginKaiten._dump_task_time_logsu)   Дамп чек-листов задачиc              	   C   s   t |d }|d}| r(t| t|dF}|dg D ]2}| j|d |d }|	t
j|ddd  q@W 5 Q R X d S )	Nr   checklists.jsonr   Z
checklistsrX   Fr   r   )r   r   r   r   r   r   rA   r$   Zget_card_checklistr   r   r   )r+   r   r   checklists_filer   	checklistchecklist_datar   r   r   _dump_task_checklists  s    

z%CmfPluginKaiten._dump_task_checklistsd   c                 C   s
  t dd d}|d }|d }| j | j}|  j|7  _W 5 Q R X zd| jj|||d}|shW q|D ]<}	t|	d }
|	d }| jjd| d|
 d	 |	|
}|	|
d
 }|
 rt|| |jdd t||	d< | |	 | |	 | |	 |	d r| |	d | |	dg D ]}| |d | q*t|| t||	d< t|	dddd}tj|	|ddd W 5 Q R X |d7 }| jd | |	 qlW q" tk
r } z*| jjd| d| d| dddd W 5 d }~X Y q"X q"W 5 Q R X d S )NFZinit_views_and_dsr   rX   rG   )Zspace_idoffsetr}   u   Задача '' ()r   Tr   r   owner_idmembersr   wzutf-8)encoding   )indentr   r   r   uB   Не удалось получить задачи проекта 'z). zERR-0025r   dump)
error_coder   
error_type)r	   _lock_offsetr$   r   r4   r   r   infor   r   r   r   r   r   r  r  r   rA   r   r   r  r   r   r   r   )r+   r   r   r}   
task_countr   r   r  r   r   r   Z	task_namer   Ztask_tmp_dirmemberr   r   r   r   r   _dump_task_worker   s\    





z!CmfPluginKaiten._dump_task_workerc                 C   s   d}t |d d}|jdd d| _t | _g }t| jD ]J}t| j	d|d  ||dd	}|
| |  | jjd
|  q>|D ]}|  q| | |S )Nr   r   r   Tr   Zdump_tasks_r   )r   r   targetr   r   $   Запущен обработчик )r   r   r   r  r   r  rangedownload_threading_max_forksr   r  rr   r   r   r   r  joinr   )r+   r   errorsr   threadsithreadr   r   r   _dump_tasks[  s*    


zCmfPluginKaiten._dump_tasksc              	   C   sd   | d}| rt| | j|}t|d(}|D ]}|tj	|ddd  q8W 5 Q R X d S )Nr   r   Fr   r   )
r   r   r   r   r$   Zget_space_usersr   r   r   r   )r+   r   r   r   Z
users_datar   r   r   r   r   _dump_project_usersy  s    

z#CmfPluginKaiten._dump_project_usersc                 C   sJ  | j  d}|jdd t| j}td| }| jD ]}|d }|d }|d }z| j  rlW  d S | j  jd7  _| j j	
d	| d
| d ||}|| d}	| rt||	 n|	jdd | j|}
t|
d |
d< ||
d< t|	|
d< | ||	 |
dr6| |
d | j  |
d< dtjd< | j  j|7  _| j jdd | j j  dtjd< t|	| t||
d< |
d  d}t|d}tj|
|dd W 5 Q R X d}|| j|
d7 }|  j|7  _W q8 tk
r6 } z2|  jd7  _| j j d| d| dddd W 5 d }~X Y q8X q8| j !d  d S )!Nrw   Tr   P   rX   r   r   r   u   Дамп проекта 'r  r  r   r   Z
author_uidauthorr8   NO_CACHEr   1r   r   Fr   r   )r   u2   Не удалось получить проект r   zERR-0058r   r  )r   r  u*   Дамп проектов завершен)"r   r   r   r   lenr*   intr   r   r   r  r   r   r   r$   Z	get_spacer4   r(  rA   r   r   environprogressr   ZdpZcommitr   r   r  r'  r   r   r   r   )r+   r   Zcntstepr_   r   r   r   r   Zproject_tmp_dirr   Z	data_filer   Zdump_task_errorsr   r   r   r   _dump_projects  sd    




zCmfPluginKaiten._dump_projectsc                 C   s   t d}| jjj|_|| jjj | j }t	|\}}}|| d }| jj
d| d | jj
d | jj
d | | jjd | jj
d | | jjd	 d
tjd< d| j_| j  t  dtjd< | jj
d |   dS )u^   
        Загружает все данные из API, кроме вложений
        z'modules.kaiten.kaiten_client.connectionr	  u   Свободно места: r[   u5   Скачивание данных через API ... u#   Дамп пользователейrx   u   Дамп типов задачissue_typesr8   r+     r,  u,   Дамп выбранных проектовN)r   Z	getLoggerr   r   ZhandlersZsetLevelr   r   r   
disk_usager  r   r$   Z	get_usersZget_card_typesr   r/  r0  r   r   r2  )r+   r   Zdownload_pathtotalusedfreeZfree_percentr   r   r   download_data  s&    




zCmfPluginKaiten.download_datac                 C   s  d}t dd | j r q| }|dkr4qtjj|ddddd	gd
}t|j}t|j}t|j	}t|j
d}	t|	dkrd|	d  dnd}
d| d}|jr|jd }|d }| d|dd}| jj|
 d| d| d zX| jj||tjd | jj|
 d| d| d d|_|  t  | jd W q tk
r } z>t  | jj|
 d| d| d| dd d!|d" |d7 }W 5 d }~X Y qX q|| W 5 Q R X d S )#Nr   Fr
  DONEr   r   r   r   r   rX   rc   rW   r   u   [Задача ID: z] r8   'sizei    (.3fz MB)u   Скачивание z: ')timeoutu	   Файл u    сохранен в 'Tr   u"   Ошибка скачивания z'. zERR-0027r   r  )r  r   r  Zext_href)r	   r   r   rA   rf   r   r4   r   r   r   r   splitr-  r   r   r  r$   Zdownload_fileconfigZIMPORT_DOWNLOAD_TIMEOUTr   r   r   r   r   r   r   put)r+   download_queue
done_queuer#  Zdownload_job_idZdownload_objr   Zfile_urlr   Zext_id_splitZtask_id_msg	file_sizeZfile_size_mbr   r   r   r   _download_file_worker  sf    




 
 z%CmfPluginKaiten._download_file_workerc                 C   s  t  }t  }d}dd| jgdddgg}| j d}d}g }| jD ]r}t|d }	||	}
t|
dsxt|
d }
d	d
|
 dgg}tj	j
|| d}|rD||7 }|||  qD|s| jjd |S | jjd|  g }t| jD ]J}t| jd|d  ||dd}|| |  | jjd|  q|D ],}tj	j|dgdD ]}||j qVq@t  |D ]}|d qx| s|D ]}|  q| j r| jjd | s|| 7 }q| jjd|  |S )Nr   re   ==r   Frw   rX   r   r   rZ   r[   r\   u3   Нет вложений для скачиванияu'   Скачивание вложений: Z_download_file_worker_r   )rD  rE  r  r  r   rb   r:  u   Импорт прерванuD   Скачивание файлов завершено. Ошибок: )r   re   r   r   r   r*   r4   r   rf   r   r   rr   r   r  r   r!  r   rG  r   ZslistrC  rX   r   emptyr"  r   r   rA   )r+   rD  rE  r   Zmain_filterZprojects_pathZattachment_countfiltersr   r   Zproject_pathZpath_filterZdownload_countr$  r%  r&  filter_r   r   r   r   download_files"  sd    





zCmfPluginKaiten.download_filesc                 C   s:   | j d d d }|d s"tddd |d D | _d S )	Nr~   r5   r   	isCheckedu8   Не выбраны проекты для импортаc                 S   s$   g | ]}|d  r|d r|d  qS )r   rM  r   )ry   sr   r   r   r{   e  s    z:CmfPluginKaiten._set_selected_projects.<locals>.<listcomp>Zchildren)rm   r   r*   )r+   Zroot_selectedr   r   r   _set_selected_projects`  s    z&CmfPluginKaiten._set_selected_projectsTc              	   C   s  |  || jtjj d }d| j d|d  d}|d dd}d}|D ]}t||krJt|}qJttjj	j
tjjj
tjjj
}	||	krt|dkr|d |d	< d
|d< d
|d< n`t|dkr|d |d	< |d |d< d
|d< n2t|dkr|d |d	< |d |d< |d |d< ddg}
tjjddd| dg|
d}|sdddd| j dgddd gg}tjjdd|d g|g|
d}|stjjdd|d g|g|
d}t|dkrg }|sdd|d g|g}tjj||
d}|sdg}t|t|D ]}t|dkrd	d|d g}nnt|dkrL|dd|d gd	d|d gg n<t|dkr|d	d|d gdd|d gdd|d gg q||g}tjj||
d}|r|d }d}|s:|r2d}d}|d }tjjdd| g|
dr |d d\}}| d| d| }|d7 }qtj||dd }nd |fS |rtd!t|j}d"d# |D }t|dkrz|d |d< |jr|D ]}t||||  q|jr||jkr|j | |_n||_d|_|jr|jdd$ ||fS )%Nrc   rW   rX   r   r   r  r   r   	last_namer8   
first_namesecond_name   r
   Zrg_member_ofrY   rZ   r[   rb   ORzNOT LIKE%::::%rH  loginILIKErS   FT@+)rY   rW  import_originalz[,;\s]+c                 S   s   g | ]}|  r|  qS r   )r=   )ry   rS   r   r   r   r{     s      z3CmfPluginKaiten._process_person.<locals>.<listcomp>Zsave_import)ro   rm   rf   r   rs   re   rA  r-  minrQ  
max_lengthrP  rR  rA   listr   rr   r:   r;   r4   rW  r[  setattrrY   Z
user_localr   r   )r+   r   ZcreateupdateZ	user_dictrY   rR   Zcurrent_length_namer   Zmax_length_namerc   r   Znot_source_hash_filterZpersonsZ_filterZ
name_combo
is_creatednrW  Zprefix_emaildomainZemailsr   r   r   r   r   j  s    




zCmfPluginKaiten._process_personu-   Обработка пользователейc              	   C   s   t  }| dD ]}z^| j r(W  d S | |\}}| j jd7  _|jrp|jsp|jdsp|rp|	|jj
 W q   |  jd7  _| jjd| ddd Y qX qd S )Nrx   r   z.evateam.ruu@   Не удалось загрузить пользователя zERR-0038r   r   )r   r   r   r   r   imported_object_countrS   rT   r   r   r5   r   r   )r+   Znew_user_emailsr   r   rb  r   r   r   _process_users  s.    	

zCmfPluginKaiten._process_usersc                 C   s   dddddddddd	d
dg}|D ]v}| j jd|d  d | j d| d|d  }tjj|d}|stj|d || j d}|d |_|jdd q&t	  d S )Nr      ОчередьOPENrX   r   typer     В работеIN_PROGRESSrS     ГотовоCLOSEDu#   Обработка статуса 'r   r<  rW   rX   rY   )r   rY   r   rj  Tr\  )
r   r   r  re   rf   CmfStatusCoderA   ra   r   r   )r+   r   states_datastateZstate_ext_idstatus_coder   r   r   _process_statuses  s"    



z!CmfPluginKaiten._process_statusesc           	      C   s  dddddddddddddddddddddddddd	ddd	ddd	dd
ddd
ddd
ddd}|  dD ]}| jjd|d  d | j d|d  }tjjddd| dgd}|rq||d }|r>tjjdd|d gdgd}|jr"d| |j |_nd| d|_|j	dd qdd|d 
 g}tjj|dgd}|r|jrd| |j |_nd| d|_|j	dd q| jjd|d  d d}tjjdd|gd gd}tj|d |d!t|jd| d| jd"}|j	dd# qt  d$S )%u#    Маппинг Type в logic_type    Подзадачаztask.sub:default)r   codeZ	UserStoryztask.userstory:defaultz
Task Agileztask.agile:defaultBugztask.bug:defaultEpicztask.epic:default)ru  ZSubtaskzSub-tasku   ИсторияZStoryu   Задачаu   ЗаданиеZTasku   Ошибкаu   Багrw  u   Эпикu   Вехаrx  r3  u*   Обработка типа задачи 'r   r<  rW   rX   rY   rZ   rU  rV  r\   rv  rH  rb   Tr   rX  u   Тип 'u.   ' не найден и будет созданui_colorr   )r   templatecmf_model_namery  rY   r   r\  N)r   r   r   r  re   rf   rg   rA   rY   r   r=   r4   ry  r   )	r+   Z	name2codeZ
issue_typerj   rV   Zlogic_type_dictrK  Ztemplate_codeZlogic_type_templater   r   r   _process_issue_types3  s~    
z$CmfPluginKaiten._process_issue_typesc           	      C   s   | j  d|d  d|d  }tjj|d}|j d|d  }tjjdd|gd}|s~tjjdd|gdd|gddd ggd}|sd	d|d	 g}tjj|dd|gddd ggd}|stj||| jd
}|d	 |_||_d|_	||_|j
dd |S )NrW   rX   ro  rY   rH  r\   rs  workflowr   )rs  r}  r   Tr\  )re   rf   rp  rA   rY   rh   r   r   rs  Zallow_empty_transitionr   )	r+   Zstatus_datar}  r   rY   rs  rk   r^   Zfilter_namer   r   r   _create_status  s:    
zCmfPluginKaiten._create_statusc              
   C   s  | j  d|d  }tjj|dgd}|sbtjjdd}tj|d  d||| jd	}|jd
d z>ddddddddddddg}|D ]}| ||| qW n: tk
r   | jj	d|d  d|d  d Y d S X tj
jdd|gddd ggdgd}|D ]B}tj
jdd|jgdd|gddd ggd s4q|jd
d
d! q| j  d"|d  }	tjj|	d#}
|
stjjd$d}tj|d  d%||	| jd	}
||
_||
_||
_|
jd
d tjj|
d&D ]}|jd
d' qd(||
| jd)}tjf |}|jd
d |
S )*NrW   rX   rY   rY   rc   zdefault.system:defaultrv  r   u3   : Simple бизнес-процесс проекта)r   rz  rY   r   Tr\  r   rg  rh  ri  r  rk  rl  rS  rm  rn  u   Проект Z	shortNamer   r   u-    не имеет статусов задач.r}  rH  ra   rb   !=r\   )TEXKOM_db_deleteforcez
::SIMPLE::ro  zsoftdev:defaultu@   : Simple-схема бизнес-процесса проекта)	scheme_wfr  r   )r{  Ztarget_workflowr  r   )re   rf   ZCmfWorkflowrA   r   r   r~  r   r   r   rh   r_  ra   deleteZCmfSchemeWfZdefault_epic_workflowZdefault_task_workflowZdefault_subtask_workflowZCmfSchemeWfRule)r+   r   Zworkflow_ext_idr}  Zworkflow_templaterq  rr  Zworkflow_statusesr^   Zscheme_ext_idschemeZscheme_templateZruleZ	rule_dictr   r   r   _create_scheme_wf_for_project  s|    



z-CmfPluginKaiten._create_scheme_wf_for_projectu0   Обработка вложений задачиc              
   C   s|  t |d d}|dg D ]V}|d r.qzt|d }| j d| }||}| sdW q|d }tjj|d}	|	stjj||d	}	|	stj|||| jd
d}	|	j	d
d |	j
r2||	_|d|	_|dr| |d |	_ntj|	_t|d0}
|	j|
 d
d d
|	_
|	j	d
d t  W 5 Q R X | jdd W q   t  | jjd| d| dd|d Y qX qd S Nr   r   r   Z
comment_idrX   rW   r   ro  )r   parentT)r   r  rY   r   r[  r\  rL   r   rb)Zsmart_backupr   	processedu;   Не удалось обработать вложение 'r  r  zERR-0029)r   )r   r   rA   r4   re   r   rf   CmfAttachmentr   r   r[  rY   rK   ri   
cmf_authorr   anonymous_userr   upload_filereadr   r   r   r   )r+   task_obj	task_datar   attachment_datar   r   r   attachment_namer   r   r   r   r   _process_task_attachments  sT    

z)CmfPluginKaiten._process_task_attachmentsu8   Обработка наблюдателей задачиc                 C   s^   g }| dg D ]B}| |d }|s(q||jks||jks||jkrHq|| q||_d S )Nr  rX   )rA   ri   rd   r  	cmf_ownerrr   
spectators)r+   r  r  r  Zwatcherr   r   r   r   _process_task_watchers&  s    z&CmfPluginKaiten._process_task_watchersu*   Обработка тегов задачиc           
      C   s   ddl m} g }|dg D ]}|d dddd}||d	d
d}zbdddd| dgdd|gddd| dgg}tjj|d}|stj|d}|  || W q tk
r }	 zW 5 d }	~	X Y qX q||_	d S )Nr   )translittagsr   r   rz   #r8   ruT)Zlanguage_codereversedrT  aliasrX  "z"%r\   r    )
Ztransliterater  rA   replacerf   ZCmfTagr   rr   r   r  )
r+   r  r  r  r  Zlabelr  rK  tagr   r   r   r   _process_task_tags3  s&    z"CmfPluginKaiten._process_task_tagsu9   Обработка журнала работ задачиc           	      C   s
  t |d d}t|d}|D ]}t|}| j d|d  ||ddd|d	 |d
 d|| jd	}|dr| |d|d< |d |d< |dr| |d|d< t	j
j|d d}|s"t	j
f |}|jjtj|jjd |_|jdd q"W 5 Q R X d S )Nr   r   r   rW   rX   r   r8   closedZfor_date
time_spentr   )	rY   r  r>   r^   
start_dater  Zremaining_estimater   r   r   r  r  Z
updater_idZcmf_modified_byrY   ro  )ZminutesTr\  )r   r   r   r   r   re   rA   r   ri   rf   ZCmfTimeTrackerHistoryr  r5   dtZ	timedeltar  Zend_dater   )	r+   r  r  r  r   r   r  Ztimetracker_historyhistoryr   r   r   _process_task_worklogK  s0    



z%CmfPluginKaiten._process_task_worklogu:   Обработка вложений комментарияc              
   C   s  t |d d}g }|dg D ]j}|d |d kr:q"zt|d }| j d| }||}	|	 spW q"|d }
tjj|d}|stjj|
|d	}|stj|
||| jd
d}|j	d
d |j
r8||_|d|_|dr| |d |_ntj|_t|	d*}|j| d
d d
|_
|j	d
d W 5 Q R X |t|  | jdd W q"   t  | jjd|
 d| dd|d Y q"X q"|S r  )r   r   rA   r4   re   r   rf   r  r   r   r[  rY   rK   ri   r  r   r  r   r  r  rr   Zcreate_preview_tagr   r   r   )r+   Zcomment_objcomment_datar  r   r   r  r   r   r   r  r   r   r   r   r   _process_comment_attachmentsj  sX    

z,CmfPluginKaiten._process_comment_attachmentsu8   Обработка комментариев задачиc           
   	   C   s:  t |d d}t|d}|D ]}t|}| j d|d  }tjj|d}|sptj||| j	dd}n|
d	g |jsq$|d
r| |d
 |_|j|_|d |_|d |_||_||_|jdd | |||}	|	r|d d d|	 |_n
|d |_|jr|jdd | j	dd q$W 5 Q R X d S )Nr   r   r   rW   rX   ro  Tr  rY   r   r[  z**r   rN   rL   r\  r>   z<p></p>r8   r   r  )r   r   r   r   r   re   rf   Z
CmfCommentrA   r   r3   r[  ri   r  r  rM   rK   r  r   r   r  r"  r>   r   r   )
r+   r  r  r   r   r   r  Zcomment_ext_idr   r   r   r   r   _process_task_comments  s>    





z&CmfPluginKaiten._process_task_commentsu3   Обработка чек-листов задачиc                 C   s  t |d d}t|d}|D ]}t|}| j d|d  }tjjdd|gdd	d gd
d|ggddgd}|r|j	r|j
r W 5 Q R  d S |stj||| jdd}|jdd tjjdd|gddddgdddgggdd}	|	r W 5 Q R  d S tjj|dD ]}
|
jdd qd}d}t|d dd d}|D ]T}|d7 }|d rX|d7 }tj|||d |d stdndd|d }
|
jdd q:|d! |_||_||_||_|jdd q$W 5 Q R X d S )"Nr   r  r   rW   rX   r  r`   r   r  rY   Tcmf_deletedr   r  r\  rT  r[  F)r]   r   r  r  r   itemsc                 S   s   | d S )NZ
sort_orderr   )xr   r   r   <lambda>      z:CmfPluginKaiten._process_task_checklists.<locals>.<lambda>)r   r   checkedr>   rh  r:  )Z
cache_taskr  r   r^   r[  r   r   )r   r   r   r   r   re   rf   ZCmfChecklistrA   r[  r  r   r   ZCmfChecklistItemr   r_  r  sortedr   Zchecklist_items_countZchecklist_items_done_countr   )r+   r  r  r  r   r   r  rY   r  Zmodified_itemitemZitems_countZitems_done_countZ
items_dataZ	item_datar   r   r   _process_task_checklists  sr    

	

z(CmfPluginKaiten._process_task_checklistsuI   Обработка ссылок на веб-страницы задачиc           
      C   s   | j jd}td| d}|dg D ]}|d }||}|rHq,| j d|d  }tj	j|d}	|	stj	||| j
d	d
}	|	jr,|d |	_|d |	_||	_|	jd	d q,d S )Nr   ^/(\d*)$external_linksr/   rW   rX   ro  Tr  r\  )r$   r/   rstripr;   compilerA   matchre   rf   ZCmfLinkr   r[  r   r   r   )
r+   r  r  app_urlpatternexternal_linkr/   r  Zexternal_link_ext_idlinkr   r   r   _process_task_remotelinks  s*    


z)CmfPluginKaiten._process_task_remotelinksu   Обработка задачиc                 C   s  d}t |d }| j d| }d| j d|d d  d}tjjddd| dgd}| j d|d	 d
 d  d|d  }tjjdd|gdgd}	d }
|dg D ]"}|d dkr| |d }
 qq| jd||||	|d |d |d | |d |
d}tjj	j
}t|d |krPd|d  d|dd |d< |d d | |d< n|d |d< |d|d< |d r|d j|d< | jd | d!|  tjj|d"d# |D d$}|stj|d | jd|d%}t|d&r|js|jr| D ]\}}t||| q|jr"|jdd' t  | || | || | || | || | || | || ||_|jr|jdd' | || t  | jd(d) |S )*Nr   rX   rW   rj  rY   rZ   r[   r\   Z	path_datar   rr  r`   ra   rb   r  r  TrP   rN   rL   r  )activityZapprovedr  rY   rV   r^   rO   rM   rK   r  rd   rG   z<p><strong>z</strong></p><p></p>rJ   r8   r>   r   r^   Zcache_status_typeu+   Попытка создать задачу r   c                 S   s   g | ]}|qS r   r   )ry   rD   r   r   r   r{   X  s     z1CmfPluginKaiten._process_task.<locals>.<listcomp>r  )r   r   r[  rY   r[  r\  r   r  ) r4   re   rf   rg   rA   rh   ri   Zdefault_activityr   r   r^  r-  ra   r   r   hasattrr[  is_newr  r`  r   r   r   r  r  r  r  r  r  r   r  r   )r+   r  project_objr#  r   task_ext_idrj   rV   rk   r^   rd   r  Znormalized_taskZmax_task_name_lengthr  rD   r5   r   r   r   _process_task  s    &

zCmfPluginKaiten._process_taskc              	   C   sX   d}t dd6 | j rq@| }|dkr.q@|| ||7 }qW 5 Q R X || d S )Nr   Fr
  r:  )r	   r   r   rA   r  rC  )r+   
task_queuetask_queue_errorsr_   r#  r   r   r   r   _process_task_thread  s    
z$CmfPluginKaiten._process_task_threadc              
   C   s"  ddl m} d}t  t  tjj| _t	j
j|ddgd| _| jjjj| _| | jjd | _| j d| }t	jjdd	d
| dgdgd}|std| | jjd| j  t }	t }
g }t }t| jD ]H}t| jd| |	|
|dd}|| |  | jjd|  qd}| |||D ]*}| j  rV ql|	!| |d7 }q@|D ]}|	!d qp| jjd |D ]}|"  qt | }| jjd d|| dd d}|
# s||
 7 }q| jjd|  ||7 }W 5 Q R X W 5 Q R X |S )Nr   r   r
   rp   r;  import_settingsrW   rY   rZ   rU  rV  r   rb   u)   Не найден проект с ext_id uD   Обработка задач в несколько потоков: Z_process_task_thread_)r  r  r_   r  u=   Запущен параллельный обработчик r   r:  u3   Ожидание обработки задач ... z-----u&    скорость обработки: r?  u    задач в секундуu@   Обработка задач завершена, ошибок: )$cmf.appr	   r6   Zdisable_aclZdisable_notifyAPPREDIS_DBredisr   rf   Z	CmfImportrA   r   r.   re   _normalize_settingsr   rm   r   ZCmfErrorr   r  Zthreading_max_forksr   timeperf_counterr   r   r  rr   r   r   r   rC  r"  rI  )r+   Zcmf_import_idr   skipr}   r	   r#  r   r_   r  r  Ztask_threadst1r%  r&  r  r   t2r   r   r   r   process_task_fork  sp    
	


z!CmfPluginKaiten.process_task_forkc                 C   s   dd l }|jdddd| jjj d| jj d|d  d	| d
| dgddtddtddd}| jd|j d| d| d | j  |S )Nr   z/usr/bin/python3z	manage.pyshellz(plugin = models.CmfPluginKaiten.get(id='z'');retcode = plugin.process_task_fork('z', 'rX   z', z, z);Tz"/var/log/eva-import-subprocess.logr   z&/var/log/eva-import-subprocess.err.log)Z	close_fdsZstart_new_sessionstdoutstderru"   Запущен процесс PID z (start=z, limit=r  )	
subprocessPopenr   r.   rX   r   r   pidZ
log_detail)r+   r   r   r}   r  procr   r   r   _execute_task  s      0"
zCmfPluginKaiten._execute_taskc                    sz  d|d  d|d  d}|  t|d }|sH| jj| d d S |  }| jj| d| d|   fd	d
 d} ||}g }t|D ]0}| |||}	||7 }||	 ||kr qq||k r2t	
d t|D ]L}	|	 }
|
d k	r|  j|
7  _||	 | |||}||7 }|| qqt|D ]&}	|	  |	jr:|  j|	j7  _q:| jj| d d S )Nu   Проект 'rG   r  rX   r  uO   . Нет новых/обновленных задач для обработкиuJ   . Обработка задач в несколько процессов: r   c                    s.   t | | }|tjkr& | |d S |S d S )Nr   )mathZceilrB  ZIMPORT_OBJ_CNT)r  max_processesZ
chunk_sizecalc_obj_limitr   r   r     s    
z6CmfPluginKaiten._process_tasks.<locals>.calc_obj_limitr   r   u2   . Закончили обработку задач)r   r4   r   r   r  r  r   r  rr   r  sleepr_  Zpollr   r   Zcommunicate
returncode)r+   r   Zproject_msgZtask_cntr  r   r}   Z	processesrz   r  r  Znew_procr   r  r   _process_tasks  sD    




zCmfPluginKaiten._process_tasksc                 C   s  | j jd|d   tjjdd|d gddgd}|r|jj|d ks^|jr|d |jkr|d  d|d  d	|d< | j j	d
|j
 d|d   n|js|d |_|  d|d< d|d< d|d< d|d< tjjddd|d  dg| d}|stj|d |d | j dd}|j}|jrr|D ]$}|dkr@q.t||||  q.|ddshd|_|  n| j j	d| d |rtjj|dD ]}|jdd qt  | |}||_|jrd|_d|_d|_d|_d|_|jdd t  | | t  |S )Nu"   Обработка проекта rG   r   rH  rV   rY   rb   r>  r  u   Проект с именем u8    уже существует! Переименуем в TZtask_allow_multiple_sprintsFZsl_only_owner_approveZsl_deny_no_approveZsl_task_only_owner_closerZ   r[   )r   rY   r   r[  Z	is_publicZprivateu"   Настройки проекта u    были изменены пользователем. Изменения из импорта не будут примененыr  r  r\  )r   r   r  rf   r   rA   rV   rX   rY   r   r   r   keysr  r[  r`  Zperm_policyr   r_  r  r   r  r  Z
show_listsZ	show_blogZ
show_pfeedZ	show_diskZ	show_chatr  )r+   project_structr   r  r  r   r   r  r   r   r   _process_project%  s    





z CmfPluginKaiten._process_projectu#   Обработка проектовc           
   
   C   s@  t jjddjj}t jjdd}dd | jD }|  D ] }t|d }|d }z| j	
 rfW  d S ||krrW q8d| j d| d}| | |d	r| |d	 d }ntj}||d
< d||jj||d |||d}	| |	| t  W q8 tk
r8   t  |  jd7  _| j	jd| d| dddd Y q8X q8d S )NZsoftdevr  zproject.agile:defaultc                 S   s   g | ]}t |d  qS r   )r4   ry   pr   r   r   r{     s     z5CmfPluginKaiten._process_projects.<locals>.<listcomp>rX   rG   rW   r*  rY   r_   r   )Zproject_typer   rV   r  Ztask_code_prefixr  rY   r   r   u=   Не удалось импортировать проект 'r  r  zERR-0104r   r   )rf   ZCmfActivityrA   rX   r5   rg   r*   r   r4   r   r   re   rt  ri   r   r   r  r   r   r   r   r   )
r+   r  rV   Zselected_projects_idsr   r   r   r   r  r  r   r   r   _process_projects  sH    



z!CmfPluginKaiten._process_projectsc                 C   s  | j jd}td| d}|  D ]}t|d }| j|| |dD ]p}| j	
 rf  d S t|d }| j d| }t|dg D ]\}}	|dkr| j d|	d  }
| j|
g }|| || j|
< qt|	d }| d	| }d
||d| j d| |dd| j|< q|dg D ]}|d }||}|sLq,|d}| d| }|| jkrtq,| d| }|| jkrq,d||d| j d| |dd| j|< q,qNq(| j	dt| j  tjjd
d}| jD ]}
tjj|
d}|s | j	jd|
  q| j|
 D ]H}tjj|dgd}|s^| j	jd|  q*||_|jdd q*qt  | j	dt| j   | j D ]^\}}z|d }tjj|d}tjj|d d d}|s| j	jd|d d  d|d d  d  W qtjj|d! d d}|sb| j	jd"|d! d  d|d! d  d  W qtjj|d}|stj|d}||_ ||_!||_"|jdd W n^   | j	jd#|d  d$|d d  d|d d  d%|d! d  d|d! d  d& Y nX qt  d S )'Nr   r  r  rX   r|   rW   parentsr   z::additional_parent::zsystem.additional_parent)rY   rX   )rv  outwardIssueinwardIssuer  r/   r   :zsystem.linku   Подзадачи: r  ro  uq   Не удалось связать задачи, возможно задача не попала в импорт parent_taskr  Tr\  u   Связи задач: rv  r  rY   u=   Не удалось найти входящую задачу r>  u   ). Возможно задача не попала в импорт или находится в проекте, который еще не импортирован.r  u?   Не удалось найти исходящую задачу u.   Не удалось создать связь z. in_link: z), out_link: r  )#r$   r/   r  r;   r  r   r4   r   r   r   r   re   r   rA   r)   rr   r(   r  groupr   r-  rf   ZCmfRelationTyper   r   r   r  r   r   r  r  ZCmfRelationOptionin_linkout_linkrelation_type)r+   r  r  r_   r   r  r   r  r%  r  Zparent_ext_idZ	sub_tasksZ	parent_idZrelation_ext_idr  r/   r  Zboth_issue_idr  r  Zchild_ext_idZ
child_taskZtask_relationZrelation_coder  r  Zrelation_optionr   r   r   _process_relations  s    





  Fz"CmfPluginKaiten._process_relationsc                 C   s>  d| _ tjj| _|| _d| j_d| j_d| j_| j	  t
  | jj| _| jjjj| _| | jjd | _|   |   |   | j  |   |   |   |   d| j_| j	  t
  | jd dd | jD }|  D ]6}|d }||krq| jjd	|d
  d| d q| jd| jj  | j  S )Nr   u   Импорт из Kaitenr  r	  z2--------------------------------------------------c                 S   s   g | ]}|d  qS r   r   r  r   r   r   r{   N  s     z2CmfPluginKaiten.process_import.<locals>.<listcomp>rX   u'   Импортирован проект 'rG   r  r  u-   Импортировано объектов: ) r   r  r  r  r   r   r   re  r0  r   r   Zget_max_processesr  r.   re   r  r   rm   rO  r9  rL  r$   closerf  r|  r  r  r   r*   r   r   r  )r+   r   Zselected_project_idsr_   r   r   r   r   process_import0  s>    




"zCmfPluginKaiten.process_import)r   r   )F)r	  )TT)F__name__
__module____qualname__r2   r   r#   Zui_meta_skipZapi_methodsr'   propertyr$   staticmethodr?   r@   rC   r4   rF   rQ   rU   rl   ro   r   r%   r   r   r   r   r   ri   r   r   r"   r   r   r   r   r  r  r  r'  r(  r2  r9  rG  rL  rO  r   rf  rt  r|  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  __classcell__r   r   r,   r   r#   )   s   	

	

		 2&


8



;
C>>

 
$P%O
.



0
#
D

bK6\
0~r#   )Zdatetimer  r  r   r;   r   r  	itertoolsr   pathlibr   Zqueuer   	threadingr   r   Ztext_unidecoder   r  r	   Zcmf.includeZmodules.kaiten.fieldsr   Zmodules.kaiten.kaiten_clientr   r"   r#   r   r   r   r   <module>   s    