U
    [Se.                    @   s  d dl mZ d dlT d dlZd dlZd dlmZ d dlZd dlm	Z	 d dl
Z
d dlmZmZmZmZmZmZmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ d dlm Z  eee!j" ee!j# ee!j$ f Z%ej&G dd dZ'dd Z(G dd de Z)dS )    )cmf_context)*N)Path)TemporaryDirectory)DictOptionalListCallableUnionTypeIterator)BeautifulSoup)cached_property)commit_all_dsinit_ds)	BaseModel)CmfPluginImportMixin)JiraApi)JiraBaseImportc                   @   s:   e Zd ZU dZeed< eed< eed< dd Zdd Zd	S )
EpicDatauD    Данные используемые при создании epic namekeyproject_keyc                 C   s
   t | jS N)hashr   self r   ./modules/jira/api_import.py__hash__   s    zEpicData.__hash__c                 C   s   | j |j kS r   )r   )r   otherr   r   r   __eq__"   s    zEpicData.__eq__N)__name__
__module____qualname____doc__str__annotations__r   r!   r   r   r   r   r      s   
r   c                    s    fdd}|S )Nc                    s    fdd}|S )Nc                    s   zD| j d  | j  r$W dS  | f||}| j j  |W S  tk
r   | j j  |  jd7  _| j jd tj	dd Y dS X d S )Nz================== r      u   Ошибка Texc_info)

cmf_importlog	is_canceldpcommit	Exceptionrollback
has_errorsloggingERROR)objargskwargsres)funcr   r   r   wrapper(   s    
z3catch_exception.<locals>.decorator.<locals>.wrapperr   )r9   r:   r   )r9   r   	decorator'   s    z"catch_exception.<locals>.decoratorr   )r   r<   r   r;   r   catch_exception&   s    r=   c                       s  e Zd ZU dZeed< eeef ed< e	dd fddZ
eeeef dd	d
Zdd Zdd Zeeeeee  dddZdd Zeeeef  dddZedd Zedd Zdd ZedddZee ddd Zee dd!d"Zee ed#d$d%Zdd(d)Zed*d+d, Zed-d.d/ Z ed0eed1d2d3Z!d4d5 Z"ed6dee#d7d8d9Z$dd:d;Z%ed<d=d> Z&ed?d@dA Z'e(j)ddBdCZ*dDdE Z+dFdG Z,dHdI Z-edJdKdL Z.eedMdNdOZ/dPdQ Z0dRdS Z1edTdUdV Z2edTdWdX Z3edYdZd[ Z4ed\d]d^ Z5d_d` Z6dadb Z7e8dcdddeZ9edfedgdhdiZ:edjedgdkdlZ;edmdndo Z<dpdq Z=edrddsdtZ>edudvdw Z?edxdydz Z@ed{ed|d}d~ZAeddd ZBeddd ZCeddd ZDeddd ZEeddd ZFedddZGedddZHdd ZIdd ZJdd ZKdd ZLdd ZMdd ZNdd ZOdd ZPdd ZQdd ZRdddZSedgddZTedgddZUedgddZVedgddZWedgddZXdd ZYdd ZZedee8edddZ[dd Z\dd Z]eedddÄZ^ddń Z_ddǄ Z`ddɄ Zadd˄ Zbdd̈́ Zcddτ Zdddф Zeddӄ ZfddՄ Zgddׄ Zhed؜ddڄZidd܄ Zjddބ Zk  ZlS )JiraAPIImportu;   Класс загрузки объектов через APIr2   _JiraAPIImport__jira_fieldsN)jirareturnc                    s   t  j|| || _d| _| jjjr*d| _tjjdd| _	i | _
i | _i | _i | _g | _g | _d| _|   td}| jjj|_d | _d S )Nr   	accountIdsoftdevcoder   zatlassian.rest_client)super__init__r@   user_keyconncloudmodelsCmfActivitygetdefault_activityjira_global_settingsZepicsissue_relationssubtasksboardsselected_projectsr2   _prepare_multiprocessingr3   Z	getLoggerr+   loggerZhandlersr?   )r   r@   r6   r7   rU   	__class__r   r   rG   @   s"    

zJiraAPIImport.__init__)rA   c              	   C   s|   | j r| j S | j d}| rFt|}t|| _ W 5 Q R X n0| j	 | _ t|d}t
| j | W 5 Q R X | j S )Nfields.jsonw+)r?   r+   get_download_pathjoinpathexistsopenjsonloadr@   Z
get_fieldsdump)r   	file_pathfr   r   r   jira_fieldsU   s    
zJiraAPIImport.jira_fieldsc              	   C   sl   dd l }|| _| j d}| j dt  }t|d}t	| j| W 5 Q R X |
|| d S )Nr   rX   zfields.json.rY   )shutilr?   r+   rZ   r[   osgetpidr]   r^   r`   move)r   fieldsrd   ra   file_path_tmprb   r   r   r   update_jira_fieldsd   s    z JiraAPIImport.update_jira_fieldsc                 C   s(   | j  D ]}|d |kr
|  S q
d S )Nr   )rc   values)r   r   fieldr   r   r   _get_field_by_namem   s    z JiraAPIImport._get_field_by_name)model
field_namerA   c                 C   s&   |sd S | j |gd}|r"|d S d S )N)Zfields_namer   )import_shop_fields)rn   ro   r8   r   r   r   _get_field_namer   s    zJiraAPIImport._get_field_namec                 C   s   dd l }dd l}dd l}dd l}|| _ || _tj| _tj| _|	 d | _
| jjd| j
d  ddd tjr~ttj| j
| _
| jjdtj dd | j}| jjdt|d	  d
dd tt|d | j
| _
| j
stdd S )Nr   r(   z
cpu_count=u%   , надо не менее 2 ядерT	anonymouszconfig.IMPORT_PROCESS=zavailable_memory=i   @u-   Гб, по 2 Гигабайта на ядроl        u   Ресурсов системы недостаточно для импорта, обратитесь в техподдержку для дополнительной информации.)	threadingqueuemultiprocessingpsutilconfigZIMPORT_THREADSZthreading_max_forksZIMPORT_DOWNLOAD_THREADSdownload_threading_max_forks	cpu_countmax_processesr+   r,   ZIMPORT_PROCESSminZvirtual_memoryZ	availableintr0   )r   rt   ru   rv   rw   Zavailable_memoryr   r   r   rT   z   s     
 z&JiraAPIImport._prepare_multiprocessingc              	   C   sn   g }ddddddddd	}d
ddd}| j  D ]8\}}d| ||d }|||d||d q0|S )uO    Сопоставление настроек для модели models.CmfTaskr   textprioritystatusZ
alarm_dateZdeadlinecmf_modified_atcmf_created_at)Zsummarydescriptionr   r   Zcustomfield_10015Zduedateupdatedcreatedz.namez.watchCountz.votes)r   ZwatchesZvoteszfields. N)model_field	json_path	ext_field)rc   itemsrM   append)r   Zissue_settingsZdefault_local_attrs_mappingZadditional_json_pathZcustom_field_keyZcustom_fieldr   r   r   r   _get_issue_import_settings   s,    

z(JiraAPIImport._get_issue_import_settingsc                  C   s    dddddddddg} | S )Nr   displayNamer   r   emailemailAddressdoes_not_workactiver   )Zuser_settingsr   r   r   _get_user_import_settings   s
    z'JiraAPIImport._get_user_import_settingsc                  C   s    dddddddddg} | S )Nr   r   r   
cmf_authorauthorr~   renderedBodyr   )Zcomment_settingsr   r   r   _get_comment_import_settings   s
    z*JiraAPIImport._get_comment_import_settingsc                 C   sX   dddddddd}||kr$|| S | d	}t|d
kr>dS | j|d
 i ddS )N   ИмяZEmailu   Активныйu   Созданоu
   Автор
   Текстu   Ключ)r   r   r   r   r   bodyr   .r(   r   r   )splitlenrc   rM   )r   Zjsonpathmappingr   r   r    _get_setting_title_from_jsonpath   s    	
z.JiraAPIImport._get_setting_title_from_jsonpathTc           	      C   sj  |sd S | dD ](}t| ts& d S | |} | s d S q|rvt| trvdddg}|D ]}|| krX| |   S qXdS |st| tri }d| kr| d |d< d| kr| d |d< d| kr| d |d< d| kr| S |S |rft| trfg }| D ]l}d	|kr||d	 d  qd
|kr.||d
 d  qd|krH||d  q|r|t| qd|S | S )Nr   r   r   r   r   r   r   valueinwardIssueoutwardIssue,)r   
isinstancedictrM   listr   r&   join)	r5   r   to_strro   Z
str_fieldsZ	str_fieldZnew_objresultZrelated_objr   r   r   _get_file_val   sL    






zJiraAPIImport._get_file_valc                    sN  g  t ttt f tg tt f ttd fdd}jdg j 	d}d}|
 r D ]$}|d D ]} qqz|rh qqhdD ]} qq|sd	d
 jjddD d }dd
 jdD d }|std||jtj |std||jtj ddddddg}||jtj  dg d  S )u-    Инициализация настроек )objssettings_funcrn   rA   c                    sz   | D ]p}| }|D ]H} ||d |d< ||d |d< |d |d< |j|d< q |j|d |  S d S )Nr   Zfile_valr   titleicon)r   rh   )r   rq   r   r   r   verbose_name)r   r   rn   r5   Zobj_settingssettingr   r   r   r   _get_settings  s    z:JiraAPIImport.tmplt_import_settings.<locals>._get_settingszplugin.plugin.*projectsNidusersc                 S   s   g | ]}|qS r   r   .0r5   r   r   r   
<listcomp>!  s     z7JiraAPIImport.tmplt_import_settings.<locals>.<listcomp>r(   limitr   c                 S   s   g | ]}|qS r   r   r   r   r   r   r   "  s     uS   Не найдено ни одной задачи, импорт невозможенua   Не найдено ни одного пользователя, импорт невозможенz2022-05-22T15:22:22.593+0300r   u!   Электронная почта)r   r   r   )r   r   r   selectedObjects)r   r   )r
   r   r   r   r	   	TypeModelr+   load_fieldsrZ   r[   r\   _get_projectsget_project_tasks_simple_getr@   
get_issues	get_usersr0   r   rK   CmfTaskr   	CmfPersonr   
CmfCommentr   )r   r   Zdump_dirtaskprojectuserZcomment_exampler   r   r   tmplt_import_settings  sF      z#JiraAPIImport.tmplt_import_settingsc                 C   s   |  | jjd }ddg g gdg}| jjddD ]}t|tjj d D ]f\}}|d s\qJ| 	||d	 }| 
|d d
 |}|d d |d d  |d d d | qJq2|S )Nimport_settingsu   Задачиzcalendar-today)r   r   titlesrows   r   rh   r   r   ro   r   r   captionr   )Z_normalize_settingsr+   json_settingsr@   r   	enumeraterK   r   r   r   _normalize_jira_valuer   )r   settingsr   issueZnumr   r   r   r   r   preview7  s    zJiraAPIImport.preview)r   rA   c                 C   s\   i }|D ]N}|d sq| j ||d dd}| |d d ||d}|||d d < q|S )uC    Преобразование настроек для модели r   r   F)r   ro   r   )r   r   rM   )r   r5   r   Znormalized_objr   r   r   r   r   _normalize_objF  s    zJiraAPIImport._normalize_objsprintkanbanc           
      C   s   dd|j gg}|dr*|dddg n|dd|g tjj|d}| jjjj d|d  }tj	| jtj
|d	 ||||d
|d|ddd|gd\}}	|	r|d	 |_|jdd |S )NZtree_parent_id==completeDatesys_typearchivefilter::r   r   	startDateendDate)r   parenttree_parent	list_typeplan_start_dateplan_end_dateext_idr   Z
filter_objTZsave_import)r   rM   r   rK   	CmfFolderr+   pluginsource_hashr   process_any_table_fieldsCmfListr   save)
r   r   r   Zparent_sys_typer   _filterr   r   Zlist_objnewr   r   r   _process_listR  s*    


zJiraAPIImport._process_listu8   Обработка наблюдателей задачиc                 C   s   |  |D ]j}| jjjr(| |d }n| |d }|sHtd| ||jks
||jks
||jkrhq
|j	
| q
t  dS )u;   Добавляем наблюдателей в задачуrB   r   uI   _process_watchers: Не найден наблюдатель задачи N)_get_issue_watchersr@   rI   rJ   _get_personCmfErrorresponsibler   	cmf_owner
spectatorsr   commit_with_event)r   	issue_objr   watcherpersonr   r   r   _process_watchersk  s    
zJiraAPIImport._process_watchersu#   Обработка вложенийc                 C   s  ddl m} dd l}t|d  d}d}| r||}| D ]}z|drbW qJ|d d D ]}	|	d |krn qqntj	
|}
tjj|
|d	d
ddgd}|stj| j|d|
d}|jrb|	r| |	d | j |_|	d |_||d}|jdd || |jdd |	rN|j|	d ksNtd|j d|	d  d|j  W 5 Q R X W qJ tk
r   | jjd| dtjddd |j  |d7 }Y qJ   | jjd| tjddd | jj  |d7 }Y qJX qJW 5 Q R X | |D ]}ztj	
|d }
tjj|
|d}|s<tj| j|d|
d}|jr|jdd |  t !|d |j" tj#jj$% }|&|j'}|(  |j  W n<   | jjd| tjddd | jj  |d7 }Y nX q|S )Nr   )RDisk
issue_pathz /attachments/all_attachments.zip/rh   
attachmentfilenameimport_originalurlZurl_preview_imgZurl_preview)r   r   rh   T)r+   r   r  r   r   r   r   sizeu   Текущий размер z != u    из метыu7   Обранужена проблема вложения uI   , размер файла не совпадает с метой из JIRArs   r*   r(   :   Не удалось обработать вложение 	file_name)r   r   ra   ))Zrdisk.rdiskr   zipfiler   r\   ZipFileZnamelistendswithrK   CmfDocument_get_filenameCmfAttachmentrM   r+   r  r   rH   r   r   r]   r   Zupload_stream_filest_sizeAssertionErrorr.   r/   r,   r3   r4   r1   _get_issue_attachmentsZ
upload_dirrd   copyZfull_path_fileZCmfRFileZdata_driverZget_rdZ	get_rfileZ
_file_namemake_preview)r   r   
jira_issuer   r  Zall_attach_zip_patherrorsZmyzipZzip_attach_nameattach_infoZattach_namer   Zattach_fattachment_infoZrdiskZrfiler   r   r   _process_attachments{  s    





,
 z"JiraAPIImport._process_attachmentsuB   Обработка упоминаний пользователей)r~   rA   c                 C   sx  ddl m}m} |sdS t|d}|jddidD ]:}z|jdrX| |jd }n||jd	r| |jd	 }|jd
 |jd< |jd	 |jd< n>||jd }||j}| 	|d d 
 }	| |	| j }|jd |jd< |j|jd< d|jd< |jj|jd
< |jj|jd	< W q2 tk
rl }
 z4|  jd7  _| jjd| d|
 tjddd W 5 d}
~
X Y q2X q2t|S )u]   
        Упоминания пользователей
        confluence-userlink
        r   )urlparseparse_qsNhtml.parserclass
user-hoverattrsdata-account-iddata-usernamezdata-linked-resource-idzdata-linked-resource-id_origzdata-username_orighrefr   	href_origTcmf_convertedr(   u)   Ошибка обработки тега : r  )Zurllib.parser  r  r   find_allr  rM   r   query_get_user_info_from_dumplowerrH   r!  r   r   loginr0   r2   r+   r,   r3   r4   r&   )r   r~   r  r  doc_souptagr   Z
parsed_urlr&  	user_infoer   r   r   _process_mentions  s2    


6zJiraAPIImport._process_mentionsc                 C   sr   |  dD ]T}|dd |kr*|  S |dd |krF|  S || j|kr
|  S q
td| d S )Nr   r   r   usernameuQ   Не нашли информацию по пользователю в дампе )r   rM   r(  rH   r0   )r   user_idr,  r   r   r   r'    s    
z&JiraAPIImport._get_user_info_from_dumpu#   Конвертация ссылок)r~   r   c                    s  |s|S t |}|dd D ]x}|jdrF| jjd|  q|drv|jd drv| jjd|  q| jjd|  |dr|jd }n
|jd	 }t	j
|d
d dd }tdt	j
|dd }	|	r(|	d  |d d }
tt fdd|
d }|r(|d }tj|}tjj|dg|d}|r|d	r|jd	 |jd< |j|jd	< d|jd< t|jd	 }|r|d r|d drd|_d|jd< d|jd< d|jd< |dr|jd |jd< |j|jd< d|jd< | jjd|j  qd|jdd kr|jd d
d dd }| |}|r|jd |jd< |j|jd< d|jd< | jjd|j  qt|S )!Nc                 S   s   |  dp|  dS Nr!  srchas_attrr+  r   r   r   <lambda>      z.JiraAPIImport._process_links.<locals>.<lambda>r#  u   Уже обработали r!  #u   Это якорь u&   Обрабатываем ссылку r2  r   ?r   (/attachment(?:/content)?/(\d+)(?:/)?(.*)r(   rh   r   c                    s   |  d kS Nr   rM   itemZ	attach_idr   r   r6     r7  r  r  )r   rh   r   Zsrc_origTZvideoZcontrolsz100%widthZheightr"  u   Заменили на z/people/r   )r   r%  r  rM   r+   rU   infor4  
startswithurllibparseunquoter   researchnextr   rK   r  r  r  sgetr  	mimetypes
guess_typer   r   r!  r&   )r   r~   r   r  r  Z	text_soupr+  r  r  	url_matchattachmentsattachZ
attach_obj	mime_type
account_idr   r   r@  r   _process_links  sd    

 
 





zJiraAPIImport._process_linksc                 C   s  | j jd|  | jjjrB|d dkrB| j jd|d   | || jt	j
j d }d| j jjj d|| j  d}|d d}t|d	kr|d
 nd|d< t|d	kr|d	 n|d
 |d< ddg}t	j
jdd|d g|d}|st	j
jddd| dg|d}|st	j
jdd|d g|d}	t|	d	krt| j jd|d  dt|	 tjdd |  jd	7  _g }	|	sdd|d g}
t	j
j|
|d}	|	st|d	krdd|d
 g}
nHt|d	krddd|d
 gdd|d	 ggdd|d	 gdd|d
 ggg}
t	j
j|
|d}	t|	d	krx|  jd	7  _| j jd|d  dt|	 tjdd | j jdt|	 tjddd |	r|	d
 }n| j d |d  tj d!}|sF|r>d}d	}|d }t	j
jdd| g|dr(| j d"| d#tj |d d$\}}| d%| d$| }|d	7 }qt	j
||| j dd&}nd |fS |r|jrp|D ]}t||||  qX|jr||jkr|j | |_n||_t	j }t	j }|j !| |j !| d|_"|d' d( D ]}| jjjrd|d)  d}nd|d  d}d| j jjj | }t	jjddd| dgdgd}|sft	jjdd|d gdgd}|st	j|| j d|d |d*}|jr|d |_#|js||_n||jkr|j | |_|j$dd+ |j !| q|j%r|j$dd+ ||fS ),Nu0   Импортируем пользователя ZaccountTypeZ	atlassianu   Неизвестный тип rh   r   r    r(   r   r   Z
first_name	last_namer   rg_member_ofr)  ILIKEr   r   rh   r   LIKE%u+   Пользователей с почтой z > Tr)   r   ORu+   Пользователей с именем uB   Пользователей с одинаковым именем > r  uM   Не нашли пользователя среди существующих FuQ   Обнаружена учетная запись с дублирующим email u   , добавляем цифру@+)r   r)  r+   r  groupsr   groupId)r   r+   r  r   import_raw_jsonr   )&r+   rU   rB  r@   rI   rJ   warningr   r   rK   r   r   r   r   rH   r   r   rM   r   r,   r3   r4   r2   INFOr(  WARNINGr  setattrr   CmfPersonGroup
jira_group
user_grouprU  r   Z
user_localr   r   
is_changed)r   r   ZcreateupdateZ	user_dictr   r   rh   r   Zpersonsr   
is_creatednr)  Zprefix_emaildomainr   re  rf  Z
group_infogroupr   r   r   _process_person"  s      (( 
 

   
zJiraAPIImport._process_personu-   Обработка пользователейc              
   C   s$  | j jjdds(| j jdtjd d S t }| dD ]}z^| j 	 rPW  d S | 
|\}}| j  jd7  _|jr|js|jds|r||jj W q8   |  jd7  _| j jd| tjdd	 | j jd|| j  tjddd
 Y q8X q8|r | j jjddr ttjj|gd d S )NZ
load_usersTuA   Не грузим пользователей из-за опцийlevelr   r(   z.evateam.ruu@   Не удалось загрузить пользователя r)   r  Zsend_invites)r6   )r+   r   r   rM   r,   r3   rb  setr   r-   rm  imported_object_countr   r   r
  addr2   r4   rH   Zcmf_deferred_taskrK   r   Zregister_persons)r   Znew_user_emailsr   r   ri  r   r   r   _process_users  s(    
*zJiraAPIImport._process_usersu#   Обработка статусовc                 C   s   |  dD ]}| j r dS | jjjr2|d }n|d d }| jjjj d|d  }tj	j
|d}|sdd|d gd	ddgg}tj	j
|d
}|stj	|d | jd}||_t||_|d |_|jdd | jj  | j jd7  _q
dS )uc   
        Грузим статусы из Jira и сопостовляем с нашими
        statusesNstatusCategoryr   r   r   r   r   r   r   )r   r+   Tr   r(   )r   r+   r-   r@   rI   rJ   r   r   rK   CmfStatusCoderM   r   r   get_status_typestatus_typer   r   r.   r/   rq  )r   r   ry  r   Zcmf_status_coder   r   r   r   _process_statuses  s*    




zJiraAPIImport._process_statusesc                 C   s  t  }d}d}| |d D ]}| jjjj d|d  d|d  }d|d  d|d  }tjjdd	|gd
}|d dkrd}	n|d dkrd}	nd}	|stjj|	d}
tj||
| jd}||_|j	dd |s|d dkr|}|s|d dkr|}d| jjjj d|d  d}|||< | j
|d dD ]6}|d r.|d |d d d kr.| || q.tjjdd|gdddggdgdD ]@}tjjdd|jgdd|gdddggd
sq|jdd qq| jjjj d|d  }d|d  }tjj|d}|s(tjjd d}
tj||
| jd}||_||_||_|j	dd tjj|d!D ]}|  qT| D ]b\}}| j r qtjjdd"d#| d#gd
}||d$|| jd%}tjf |}|j	dd ql|S )&u)  
        У простого проекта воркфлоу привязан к issue type 1 к 1
        Поэтому надо получит все типы привязанные к проекту, на каждый тип создать WF и замапить через схему
        Nr   
::SIMPLE::r   (   Simple-схема для проекта r       и типа r   =r   hierarchyLevelr(   task.epic:defaultr   task.agile:defaultdefault.system:defaultrD   r   templater+   Tr   r9  
project_idZusagesZ
issueTypesworkflowr   ry  rW  !=forcerv  softdev:default	scheme_wfrX  rY  r   
logic_typeZtarget_workflowcmf_model_namer  r+   )r   _get_issue_types_for_projectr+   r   r   rK   CmfWorkflowrM   r   r   _get_statuses_for_project_create_status	CmfStatusr   ry  deleteCmfSchemeWfdefault_task_workflowZ!default_subtask_workflow_workflowCmfSchemeWfRuler   r-   CmfLogicType)r   Zjira_project_info	issue_mapZdefault_task_wfZdefault_subtask_wf
issue_typeworkflow_ext_idworkflow_namer  rE   r  issue_type_ext_idjira_statusr   Zscheme_ext_idZscheme_nameschemerulelogic_type_ext_idr  	rule_dictr   r   r   #_create_schemewf_for_simple_project  sx    $$
 
 z1JiraAPIImport._create_schemewf_for_simple_projectc                 C   s0  | j jjj d|j d|d  }| j jjj d|d  }tjj|d}t|d }|srtj||d | j d}||_	|j
dd tjjd	d
|gd}|stjjdd|gdd|ggd}|stjjdd
|d gdd
|ggd}|stj|| j |d}||_|d |_|d |_||_||_	|j
|j d |S )Nr   r   rv  ru  r   )r   r   r+   Tr   r   r   r   r  r~  status_code)r  r+   r  r   Z	only_data)r+   r   r   r   rK   rw  rM   r   rx  ry  r   r  r   r   r~   r  is_new)r   Zjira_status_datar  status_ext_idr   r  ry  r   r   r   r   r     s6    "

zJiraAPIImport._create_statusc                 C   s<  | j jjj d|d d  }tjj|d}|d d }|sztjjdd}tj|d ||d	d
|| j |d}|jdd t }| j|d dD ],}d|d kr| j jjj d|d  d|d  }d|d  d|d  }	tj	jdd|gd}
d| j jjj d|d  d}|
sxtj
jddd| dgd}tj	j|jdd  dd}|sftj	jdd}tj	||| j d}
|	|
_|
jdd |
||< |d D ]"}|d d  |d< | ||
 qqtjj|d!D ]}|  q| D ]P\}}
tj
jddd| dgd}||
d"|| j d#}tjf |}|jdd q|S )$Nr   jira_workflow_schemer   rv  rB  r  rD   r   r   r   )r   r   r~   r_  r+   r  Tr   r  	issuetyper   r{  r|  r}  r   r~  r   rX  rY  :r   z:defaultr  r  rt  ru  r   r  r   r  )r+   r   r   rK   r  rM   r   r   r  r  r  rE   r   r   r  r  r   r  r   )r   project_infor   Z	wf_schemeZwf_scheme_infor  r  r  r  r  r  r  r  r  r  r  r  r   r   r   _process_wf_scheme'  sZ    

$  z JiraAPIImport._process_wf_schemec              	   C   sV   d}t dd4 | j rq>| }|dkr.q>|| |7 }qW 5 Q R X || d S )Nr   FZinit_views_and_dsDONE)r   r+   r-   rM   _process_issueput)r   Zissue_queueZissue_queue_errorsr  r   r   r   r   _process_issue_threadU  s    
z#JiraAPIImport._process_issue_threadu   Обработка теговc           
      C   s,  ddl m} t }| j|d | |d dD ]4}| j rB d S |d dr.||d d }q.|D ]}|	dd	d	d
}||ddd}z\dddd| dgdd|gddd| dgg}t
jj|d}|st
j|d}|jdd W qh tk
r$ }	 ztd| d|	  W 5 d }	~	X Y qhX qhd S )Nr   )translitr   r   rh   labelsrS  _r8  r   ruT)Zlanguage_codereversedrZ  aliasrV  "z"%r   r   r;   r   u*   Не удалось создать тег r$  )Ztransliterater  rp  r   _count_project_tasksr+   r-   rM   unionreplacerK   ZCmfTagr   r0   r3   r`  )
r   r  r  tagsr   Ztag_nameZ
name_aliasr   Ztag_objr-  r   r   r   _process_tagsa  s,     
zJiraAPIImport._process_tags)project_structr  c                 C   s  t jjdd|d gddgd}|r|jj|d ksF|jr|j|d kr|d  d|d  d|d< | jjd|j	 d	|d   n|js|d |_|
  d
|d< t jj| jjjjd|d< t jj|d | d}|st j| j|d |d d
d}|jr,|D ]}t||||  q|dr,d|_|j}|
  |rbt jj|dD ]}|jd
d qN| |d D ]T}| jjjj d|d  }t jj|d}	|	st jjddd gdd|d ggd}	|	st j|d | jd
d}	||	_|d |	_	|d|	_|	j
d
d t jj||	d}
|
s2t j||	| jd}
g |
_|
j
d
d |dg D ]d}|d dkr| jjjr| |d  d! }nJz| |d" }W n6   | jjd#|	 d$|d"  t j!d
d
d% d }Y nX |r|
j"| n| jjd&|  n|d d'kr| jjjr(d|d( d)  d}nd|d  d}d| jjjj | }t j#jdd*d+| d+gd}|r~|
j"| n,| jjd,| d-t j!d
d
d% |  j$d.7  _$nt%d/qP|
j
d
d qp| &  | '| | (| | &  | j)|d0 | *||_+t,|d1  d2}t-|}t./|}W 5 Q R X |rd| jjjj d|d  }t j0j|d|_1|j
d
d | 2| | j3 rd S t4  | 5| | &  | 6| | j j7d.7  _7| &  |S )3Nr   r   r  r   rW  z ()u   Проект с именем u8    уже существует! Переименуем в TZtask_allow_multiple_sprints)Zexit_idui_form_schemer   rh   )r+   r   r   r  	isPrivateprivate)r   )ZTEXKOM_db_deleter   r   rv  r   )r   r+   r  r   r   )r   project_role)r   r  r+   actorstypeatlassian-user-role-actorr,  rB   r   uX   Не удалось найти и добавить пользователя в роль u    проекта:: r  uR   Не удалось привязать пользователя к проекту zatlassian-group-role-actorZ
actorGroupr^  rX  rY  u<   Не найдена группа пользователей u#    для проектной ролиr(   u&   Неизвестный тип ролиr  project_dir/issue_security_scheme.json)8rK   
CmfProjectrM   r  r   r   r+   rU   r`  r   r   CmfUiFormSchemer   r   keysr  rc  Zperm_policyr  r   r   r  _get_project_rolesCmfProjectRoler~   ZCmfProjectRoleAssignmembersr@   rI   rJ   r   r,   r3   r4   r   rd  r2   r0   _calc_progress_process_versions_process_componentsr  _process_permissionsZproject_perm_schemer   r]   r^   r_   CmfSecurityLevelSchemesecurity_level_scheme_process_tasksr-   r   _process_boards_process_sprintsrq  )r   r  r  project_objr   r  r   	role_infor   roleZrole_assignactorr   Z	sub_group	dump_filerb   r  r   r   r   _process_projectz  s    "
 "
  






zJiraAPIImport._process_projectc                 C   s   dd l }|jdddd| jjj d| jj d|d  d	| d
tj dgddtddtddd}| jd|j	  | j
  |S )Nr   /usr/bin/python3	manage.pyshellz&plugin = models.CmfPluginJira.get(id="z");plugin.process_issue_fork("", "r   ", , );T"/var/log/eva-import-subprocess.loga+&/var/log/eva-import-subprocess.err.logZ	close_fdsZstart_new_sessionstdoutstderr   Запустили )
subprocessPopenr+   r   r   rx   IMPORT_OBJ_CNTr]   r,   pid
log_detail)r   r  doner  procr   r   r   _execute_task  s    
2  
zJiraAPIImport._execute_taskc           
      C   s  dd l }| jjd| j  d}| |d }g }t| jD ]0}| ||}|tj	7 }|
| ||kr> qpq>||k r|d t|D ]L}| }|d k	r|  j|7  _|| | ||}	|tj	7 }|
|	 qqpt|D ]"}|  |jr|  j|j7  _q| jjd d S )Nr   uD   Грузим задачи в несколько процессов: r   r(   u0   Закончили обработку задач)timer+   rU   rB  r{   r  ranger  rx   r  r   sleepr   pollr2   removecommunicate
returncode)
r   r  r  r  Ztask_cntprocessr  r  r  new_procr   r   r   r    s4    




zJiraAPIImport._process_tasksu#   Обработка проектовc              
   C   s  t jjddjj}|  D ]}zv|d dd | jD kr@W qd }| j r\t	  W  d S |di }|r~| 
|| j }n
ddd	}|d
 dkrt jjdd}nL|d
 dkrt jjdd}n0|d
 dkrt jjdd}ntd|d
  d| jjjj d|d  |d< |d |jj|d ||d ||d |d}|d }|rZ| |}	|	|d< n0| jjjj d|d d  }
t jj|
d|d< | ||}W q tk
r   t  |  jd7  _| jjd| tjddd  Y qX qd S )!NrC   rD   r   c                 S   s   g | ]}|d  qS r   r   r   r   r   r   r   "  s     z3JiraAPIImport._process_projects.<locals>.<listcomp>leaddefault_import_jira@evateam.comdefault_import_jirar   r   projectTypeKeybusinessproject.base:defaultsoftwareproject.agile:defaultservice_deskproject.servicedesk:defaultu$   ERROR! Проекты с типом 3    временно не поддерживаютсяr   r   r   r   r   r   r  r~   activityZtask_code_prefixr   r   r_  
simplifiedr  r  rv  r(   <   Не удалось импортировать проект Tr  )rK   rL   rM   r   r   r   rS   r+   r-   r   r   rH   r  r0   r   r   r  r  r  Zrollback_purge_eventr2   r,   r3   r4   )r   r  r  r  project_leadr   r  r  Zproject_simplifiedr  r   r   r   r   _process_projects  sZ    


zJiraAPIImport._process_projectsc              
   C   s  d }t jjddjj}|  D ]v}z(| j r:W  d S |d dd | jD krVW q| jj	j	j
 d|d  |d< |di }|r| || j }n
d	d
d}|d dkrt jjdd}nL|d dkrt jjdd}n0|d dkrt jjdd}ntd|d  d|d |jj|d ||d ||d |d}| ||d< | ||}W q tk
r   t  |  jd7  _| jjd| tjddd Y qX qd S )NrC   rD   r   c                 S   s   g | ]}|d  qS r  r   r   r   r   r   r   c  s     z7JiraAPIImport._process_projects_box.<locals>.<listcomp>r   r   r  r  r  r  r  r  r  r	  r
  r  r  u$   ERROR! Поректы с типом r  r   r   r   r  r  r(   r  Tr  )rK   rL   rM   r   r   r   r+   r-   rS   r   r   r   rH   r  r0   r  r  r   r2   r,   r3   r4   )r   r  r  r  r  r   r  r  r   r   r   _process_projects_box[  sN    
z#JiraAPIImport._process_projects_boxu   Обработка досокc           
      C   s$  |  |d D ]}z| j r*W  dS tjj|d d}| jjjj d|d  |d< tjj|d dddd	gd
}|stj|d | jd}|d |_	|d |_
||_||_|jdd |j  |jr|jD ]}|  q|j  |d d d D ]}ztj|d || jdd}|jdd |d D ]P}| jjjj d|d  }tjj|d}	|	sntd| d|j|	 q,|jdd |j  W q tk
r   |j  | jjd| tjddd Y qX q| j jd7  _W q   | jj  | jjd| tjddd Y qX qdS )z
        https://bcrm.carbonsoft.ru/desk/favorites/Document/DOC-007367#spec-006348-b
        https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-rest-agile-1-0-board-boardid-quickfilter-get
        r   Nr   rv  r   kanban_board_columnsr_  r   r   r  r   r+   r   rB  Tr   ZcolumnConfigcolumns)r   r   r+   r  rt  u   Не найден статус u    для колонкиuD   Не удалось создать колонку для доски r  r(   u:   Не удалось импортировать доску )_get_project_boardsr+   r-   rK   r  rM   r   r   ZCmfKanbanBoardr   r_  r   r   r   r.   r/   r  r  r  ZCmfKanbanBoardColumnrw  r  Zmapped_status_codesr   r1   r,   r3   r4   rq  )
r   r  boardr   Z	board_objcolZ	board_colr   r  Zstatus_globalr   r   r   r    sX    
 





 
$zJiraAPIImport._process_boardsu#   Обработка спринтовc                 C   s  t jj|d d}t jjdd|gdddggd}t jjdd|gdddggd	gd
}t jjdd}| |d D ]}| j r d S | jjjj	 d|d  }t j
j|dddddddddg	d}|st j
|d| j|d}|jr||_|d |_|d|_|d|_|jr|jdd |d dkrZ||_|jsJd|_|jdd |jd|_n"||_|d dkr||jd |_|jr|jdd t  | jjd!|j d"t|d#   |d# D ]0}	| jjjj	 d|	 }
t jj|
d$dd%gd}|r|d&r| jjjj	 d'|j }t jj|d(d)d*gd}|sJt j|| jd+}d,|_d-|j d.|j d/|_ ||_!|jr|j|j" d0 |j#dkrt|j$d1kr|j%&| n|j%&| n4|  j'd17  _'| jj(d2|	 d3|d  t)j*ddd4 q|jdd t  | j j+d17  _+qnd S )5Nr   rv  r   r   r   r   r   r   tree_node_is_branchrW  zlist.agile_sprint:defaultrD   r   r   r   r  r  r  r   r   r   r  T)r   r  r+   r  r   r   r   stateclosedCLOSEDr   ZIN_PROGRESSu0   Добавляем задачи в спринт r$  issueslistscache_status_typer   z::comment-sprint:	log_levelr~   r   r     uB   Задача была в архивном спринте <a href="" target="_blank"></a>r  r(   u-   Неизвестная задача issue_id=u    в спринте r  ),rK   r  rM   r   r  _get_all_sprintsr+   r-   r   r   r   r  r  r   r   r   rg  r   r   r  r  Zget_default_statusr   r   rU   rB  r   r   r   r   r"  r!  r~   r   r  r!  r   r  r   r2   r,   r3   r4   rq  )r   r  r   Zsprint_folderarchive_folderZsprint_logic_typer   r   Z
sprint_objZissue_idZissue_ext_idr   comment_ext_idcommentr   r   r   r    sz    "
     
$*zJiraAPIImport._process_sprintsc                 C   s$   |sd}|sd}| d| d| S )Nu   Не указаноr$  u    ➔ r   )r   r   	old_valueZ	new_valuer   r   r   _simple_html_diff  s
    zJiraAPIImport._simple_html_diffc                 C   s   |dkrd S z|  |}W n,   | jjd| dtjd tj Y S X | |dd\}}|s| jjj	
drtjj
| jjd d}| jjd| d| tjd n$| jjd| d	 |  jd
7  _|S )Nunknownu6   Не найден пользователь user_id = "u9   ", в дампе, указываем системногоrn  FZdefault_user_idr  uA   ", ставим пользователя по умолчанию r  r(   )r'  r+   r,   r3   r4   gZsystem_userrm  r   r   rM   rK   r   rb  rU   errorr2   )r   r0  r   r   r  r   r   r   r     s     zJiraAPIImport._get_person)	dump_pathc                 C   s   z|  |W S  tk
r   z|d}| rt|dB}|D ]6}t|}||| j kr@|  W  5 Q R  W  Y S q@W 5 Q R X | j	|}|rt|d}|
t|d  W 5 Q R X |W  Y S W n$   | jjd| tjd Y nX Y nX d S )N
users.jsonrr  
uM   Неудалось сдампить пользователя по ключу rn  )r'  r0   r[   r\   r]   r^   loadsrH   r@   Zget_user_infowritedumpsr+   r,   r3   rb  )r   r0  r/  Z
users_pathrb   rowr5   r,  r   r   r   _get_user_info  s$    

&zJiraAPIImport._get_user_infou,   Обработка журнала работ)r   c                 C   s*  dd l }| jjd | |D ]}t }d|d< | j|d< | |d | j |d< |d |d< | |d	 | j |d
< ||d< |d |d< |d d |d< |dd|d< | jj	j	j
 d|d  |d< ||d< d|d< tjj|d d}|s tjf |}|jj|j|jjd |_|jdd q d S )Nr   u*   Загружаем журнал работr  r   r+   r   r   r   updateAuthorcmf_modified_byr   started
start_dateZtimeSpentSeconds<   
time_spentr)  r   r~   r   r   r   r_  Zremaining_estimaterv  )ZminutesTr   )datetimer+   rU   rB  _get_issue_worklogsr   r   rH   rM   r   r   rK   ZCmfTimeTrackerHistoryr;  r   Z	timedeltar=  Zend_dater   )r   r   r   r>  worklogZtimetracker_historyhistoryr   r   r   _process_issue_worklog&  s.    


z$JiraAPIImport._process_issue_worklogu8   Обработка комментариев задачиc           	      C   s  d}d}|  |D ]v}|d7 }| jjjj d|d  }tjj|d}|rn|jsn| jjd| t	j
dd q|stj||| jdd	}| |d
 | j |_|j|_| |d | j |_|d |_|d |_| |d |_| |jj|||_||_||_|dr~|d7 }|d d dkrNtjj|d d d}|j| d|_n0| jjd|d  t	j
ddd |  jd7  _q|jdd q| jd| d|  d S )Nr   r(   r   r   rv  uM   Комментарий уже есть, и правился в системеTrr   )r   r   r+   r  r   r8  r   r   r   Z
visibilityr  r  r   r;   uL   Ошибка обработки приватного комментария r  r   u   Обработано u7    комментариев из ни приватных )_get_issue_commentsr+   r   r   rK   r   rM   r  r,   r3   r4   r   rH   r   r   r9  r   r   r.  r~   rR  r   r_  r   r  r   r   r  r2   r   )	r   r   r   cntZprivate_cntr)  r   Zcmf_commentr  r   r   r   _process_issue_comments?  s>    


 z%JiraAPIImport._process_issue_commentsu0   Обработка плагинов задачиc           	      C   s   |j jp
d}| |D ]}|d dkr|d rt|d}|jdddid	}|d d
D ]N}|sbqX|drx|d}n|jdddid	}||dd   || qX|| t||j _q|j	r|j
dd d S )Nr   r   z&com.railsware.SmartChecklist.checklistr   r  Zulr  ztox-checklistr  r2  -liztox-checklist--checkedr(   Tr   )r~   r   _get_issue_propertiesr   Znew_tagr   rC  r   r&   rg  r   )	r   r   r   r~   Zissue_propertyZsoupZ	checklistvalrG  r   r   r   _process_propertiesd  s"    


z!JiraAPIImport._process_propertiesc              	   C   sb  dd | j tjj d D }|d d D ]2}| jjjj d|d  }d }|drl| |d | j	 }n| jj
d	|  |d
 |d|d}|d D ]h}|d }	||d }
|
r|
d rd}	| |	|d |d }d|kr||d< q|d  d| |d< qtjj|ddddddgd}|s8tj| jd|d}|D ]}t||||  q<|  q(d S )Nc                 S   s   i | ]}|d  |qS )r   r   )r   rl   r   r   r   
<dictcomp>{  s      z2JiraAPIImport._process_history.<locals>.<dictcomp>rh   	changelog	historiesz	::historyr   r   u   Аудит без автора r   r(   )r   r   r"  r   r   rl   r   r   Z
fromStringZtoStringr~   z<br>r+   r   r   r"  r   r  T)r+   r  r   )r   rK   r   r   r+   r   r   rM   r   rH   rU   r`  r+  r   rc  r   )r   r   r   Zjira_map_fieldsrA  r   r   r)  r?  r   rl   ZdiffZhistory_commentr   r   r   r   _process_historyz  s.    

zJiraAPIImport._process_historyu   Обработка задачиc                 C   s  d}| j jjj d|d d d  }tjj|d}|sRtd|d d d  | || jtj	j
 d }tjjd	d
| j jjj d|d d d  gdgd|d< |d j|d< | j|d< d|d< | j jjj d|d  |d	< |d |d< |d drt| j jjj d|d dd  }tjj|d|d< |d st| j jd|d d tjddd |  jd7  _| d}|r|d |d |d< |d dr4| |d d| j|d< |d s4|  jd7  _| j jd|d d tjdd | j jd|d d| j tjddd |d dr| |d d| j|d < |d  s|  jd7  _| j jd!|d d tjdd | j jd!|d d| j tjddd |d d"rh| |d d"| j|d#< |d# sh|  jd7  _| j jd$|d d" tjdd | j jd$|d d"| j tjddd ||d%< |d& r|d' d( |d&< ||d%< |pg }d| j jjj d|d d) d  d}	tjjd	d
d*|	 d*gd+|d,< | j jd-|  tj| j tj	|d	d.|d	 gd/\}
}|| |
|7 }| |
jj|
_| |
jj|
||
_|d d0 d1 rl|  |
| | j!d2r|d d3 d1 r| "|
| |D ]}|
j#$| q|d d4g |d d5g  D ]l}| j jjj d6|d  }tj%j|d}|s.| j jd7| tjddd |  jd7  _n|
j&$| q|d d8g D ]l}| j jjj d|d  }tj'j|d}|s| j jd9| tjddd |  jd7  _n|
j($| qN||
_)|
j*r|
j+dd: t,  | -|
| |d d;g }|r.zt.| j |
| W n   | j j/d< Y nX | 0||
 | 1||
 | j jjj d=|d  }tj2j|d}|stj2|| j d>}|
|_3| j4j5 d?|d  }d@| dA| dB|_dC|_6|j+dd: |j78  |S )Du   
        Общая логика обработки которая потом разделяется на задачи и эпики
        r   r   rh   r   r   rv  u$   Не проекта с ключем r   r   rX  %::r   ry  rW  r!  r  TZapprovedrE   ZsecurityZperm_security_levelu:   Не найден уровень безопасности r  r(   zStory PointsZagile_story_pointsreporterr   u*   Не найден автор задачи r)   creatorr   u?   Не найден ответственный по задаче assigneer   u6   Не найден исполнитель задачи r   r~   renderedFieldsr   r  rY  r   r  u2   Пытаемся создать задачу из r   r   r)  totaltimeTrackingEnabledr@  ZfixVersionsZversions
::version:u!   Не найдена версия 
componentsu%   Не найден компонент r   r  u/   Не удалось привязать тегиz
::comment:r  zbrowse/u<   Задача импортирована из Jira: <a href="r$  r%  r#  )9r+   r   r   rK   r  rM   r   r   r   r   r   r  ry  rN   CmfSecurityLevelr,   r3   r4   r2   rm   r   rH   r  rU   rB  r   r   r  r.  r~   r   rR  rE  rO   rB  r   r   r   Zfix_versionsCmfComponentrW  r_  rg  r   r   r   Zprocess_tagsZexcepionrJ  rN  r   r   r@   r  r"  r.   r/   )r   r   sprintsr  Zproject_ext_idr   Znormalized_issuer   Zstory_p_fieldr  r   r  r   versionZversion_obj	componentZcomp_objr  r(  r)  r!  r   r   r   r    s    ",

$
&
 
$. 
$. 
$.

& 
 $
zJiraAPIImport._process_issueu&   Обработка всех задачc                 C   s2   | j  }|D ]}| j r" d S | | qd S r   )r@   r   r+   r-   r  )r   r  r   r   r   r   _process_issues  s
    

zJiraAPIImport._process_issuesu3   Обработка связей всех задачc                 C   s  |   D ]b}| j|d | |d dD ]>}| j rB  dS |d d}| jjjj d|d  }|r| jjjj d|d  }| j|g }|	| || j|< |d dg }|D ]}| jjjj d|d  }	d|kr$d	|i|d
< | jjjj d|d d  |d d	< || j
|	< qd
|krd	|i|d< | jjjj d|d
 d  |d
 d	< || j
|	< qq*q| jdt| j  | jD ]z}| j r dS tjj|d}
| j| D ]H}tjj|dgd}|s| jjd|  q|
|_|jdd qq| jj  ddddd}| jjdt| j
   | j
 D ]\}}| j rf dS zB| jjjj d|d d  }||d d }|rtjj|d}ntjj|d}|stjjdd|d d gd}|s(tj|| j|d d d}|d d |_|d d |_|jdd | jjjj d|d  }tjj|d}|sftj|| jd }tjj|d
 d	 d|_tjj|d d	 d|_||_|jdd W nN   |  jd!7  _| jjd"|d  d#|d
  d$|d  t j!ddd% Y nX qJ| jj  dS )&uD   
        Постобработка связей задач
        r   r   Nrh   r   r   Z
issuelinksr   r   r   u   Подзадачи: rv  parent_taskr  uq   Не удалось связать задачи, возможно задача не попала в импорт Tr   zsystem.finish:finishzsystem.duplicatezsystem.linkzsystem.clone)ZBlocksZ	DuplicateZRelatesZClonersu   Связи: r  r   rD   r   r   )r   r+   r   ZinwardZoutwardr  r(   u.   Не удалось создать связь z out=z in=r  )"r   r   r  r+   r-   rM   r   r   rQ   r   rP   r,   r   rK   r   rU   r`  r^  r   r.   r/   rB  r  r   ZCmfRelationTypeZin_type_nameZout_type_nameZCmfRelationOptionZin_linkZout_linkrelation_typer2   r3   r4   )r   r   r   Zissue_parentr   Zparent_ext_idZ	sub_tasksZissue_linksZ
issue_linkZlink_ext_idr^  Zchild_ext_idZ
child_taskZ	map_namesZrelation_ext_idZissue_relationZrelation_coder_  Z
rel_ext_idZrelation_optionr   r   r   _process_relations"  s    "



&&


&  z JiraAPIImport._process_relationsu)   Обработка компонентовr  c              
   C   s  t jj|d ddgd}t jj|dddgd}|s@td	| | |d
 D ]}zp|jsld|_|  | j	 r~W  dS | jj
j
j d|d
  }t jj|ddddddgd}|st j|| jdd}||_|d |_||_|dr| |d | j |_|d dkr|j|_n.|d dkr2|j|_n|d dkrH|j|_|jrd|jrd|jdd |ddr|js|  d|j_|jjr|dd|j_|jjjdd | j jd7  _W qN   |  jd7  _| jjd| tj ddd Y qNX qNdS ) uG  https://docs.atlassian.com/software/jira/docs/api/REST/1000.824.0/#api/2/project-getProjectComponents
        Пока грузим как списки, после доработки будут отдельные обекты

        Args:
            project_info (dict): проект из джиры как есть
        r   r   r   r  rW  Tr  )r   r   Zinclude_systemrh   uV   Данный тип проекта не поддерживает компоненты r   Nr   descr_documentdefault_ownerdefault_responsibler  r   r   r   r+   r  r   r  ZassigneeTypeZPROJECT_DEFAULTZPROJECT_LEADZCOMPONENT_LEADr   r   r   r(   u3   Ошибка загрузки компонента r  )!rK   r  rM   r   r  _get_project_componentsr  r   r+   r-   r   r   rY  r   r   r   r   rH   rb  r   rc  r   r  rg  ra  Zcreate_descr_documentZ
text_draftr   rq  r2   r,   r3   r4   )r   r  r  r   r\  r   Zcomponents_listr   r   r   r  x  sV    
  



z!JiraAPIImport._process_componentsu   Обработка версийc              
   C   s  t jj|d dgd}t jjd|d}t jjd|d}| j|d dD ]X}z| j rbW  d S | jjjj d	|d  }t j	j|d
}|st j	|| jdd}t j
jdd|_||_|d |_|dd|_|jj|_|jr|jr|jdd |d r||_n||_|jr:|d s$|d r0|d n
|d |jrV|jrV|jdd | j jd7  _W qF   |  jd7  _| jjd| tjddd Y qFX qFd S )Nr   z"scheme_wf.default_release_workflowr  release)r   r   r   r   r  rV  rv  Trd  zlist.release:defaultrD   r   r   r   r   ZarchivedZreleasedr  ZOPENr(   u+   Ошибка загрузки версии r  )rK   r  rM   r   _get_project_versionsr+   r-   r   r   r   r  r  r   r   r~   r  Zdefault_release_workflowr  r  rg  r   r   Zset_default_statusrq  r2   r,   r3   r4   )r   r  r  Zrelease_folderr'  r[  r   Zversion_listr   r   r   r    s>    




zJiraAPIImport._process_versionsu   Обработка аудитаc                 C   s   dS )u)   
        Грузим аудит
        Nr   r   r   r   r   _process_audit  s    zJiraAPIImport._process_auditu2   Обработка логических типовc                 C   sr  ddddddddddddddddddddddd	ddd	ddd	dd
ddd
ddd}|  dD ]}| jjd|  | j r dS | jjjj d|d  }tjj	ddd| dgd}|rqp|	|d }|r>tjj	dd|d gdgd}|j
r"d| |j
 |_
nd| d|_
|jdd qp| jjd|d  d tjj	dd|d  gdgd}|r|j
rd| |j
 |_
nd| d|_
|jdd qp| jjd  d}| jjjr|d! d"krd}n|d! d#kr
d}n|d$ r
d}tjj	dd|gd}td| d|d  |d% d&|d'}t| jtj| t  | j jd#7  _qpdS )(u$   Маппим issueType в logic_type   Подзадачаztask.sub:default)r   rE   Z	UserStoryztask.userstory:defaultz
Task Agiler  Bugztask.bug:defaultEpicr  )zSub-taskZSubtaskri  ZStoryu   ИсторияZTasku   Задачаrj  u   Багu   Ошибкаu   Эпикrk  issue_typesu-   Обрабатываем тип задачи Nr   r   r   rX  rO  ::%r   r   rE   r   rW  Tr  u   Ищем по имени "r  rV  u*   Не нашли, создаем новыйr  r9  r(   Zsubtaskr   r   )r   r   r~   r  r  )r   r+   rU   rB  r-   r   r   rK   r  rM   r   r   stripr@   rI   rJ   r   r   r   r   rq  )r   Z	name2coder  r  r  Zlogic_type_dictZtemplate_coder  r   r   r   _process_issue_type  sz    


z!JiraAPIImport._process_issue_typeu2   Обработка бизнес-процессовc                    s  i }  dD ]}|||d < q  dD ]P} j r@ dS  jj|d d    jjjj d|d d  d|d d  }tjj	|dstjj	d	d
}tj|| jd|d _
|d d _jdd |d D ]} ||d   qtjjddgdddggdgdD ]@}tjj	dd|jgddgdddggdsPq|jdd q|d D ] } j r  dS  jjjj dj d|d  }tjj	|d}	|	stj| jd}	|d |	_|d |	_
 fdd|d D }
tjjdd|
gd|	_t|	jt|
ks*td jjjj dj d|d  }tjj	|d|	_|	jdd qh j jd 7  _q*dS )!u   Импортируем workflowrt  r   	workflowsNr   r   ZentityIdrv  r  rD   r  r   Tr   r  r   r   ry  rW  r  r   r  Ztransitions)r   r  )r   r  r+   c                    s*   g | ]"} j jjj d j d | qS )r   )r+   r   r   r   )r   str   r  r   r   r   T  s     z3JiraAPIImport._process_workflow.<locals>.<listcomp>fromINuN   DEV: Fatal Вероятно, сломали создание статусов!tor(   )r   r+   r-   rU   rB  r   r   rK   r  rM   r~   r   r   r  r  r   ry  r  r   ZCmfTransZstatus_fromr   r  Z	status_torq  )r   Zjira_statusesr  Zjira_workflowZworkflow_idZtemplate_workflowr   Zjira_transitionZtransition_ext_idZ
transitionZstatus_from_ext_id_listZstatus_to_ext_idr   rr  r   _process_workflow$  sT    
,

 
"

"zJiraAPIImport._process_workflowu    Обработка схем БПc              
   C   s  |  dD ]}| j r  d S | jjjj d|d  }tjjdd| dgd}|sjtd|d  |d	 |d
d| jjjj d|d  ddi||d}t	
| jtj|dd|d g\}}|stjj|dD ]}|  q|d  D ]\}}	| j r  d S | jjjj d| }tjjddd| dgd}
tjjdd| jjjj d|	 dgd}|
|d|| jd}tjf |}|jdd q| j jd7  _q
d S )Nworkflow_schemer   ZdefaultWorkflowr   rX  rm  r   u+   Не импортировался workflow r   r   r   r   rE   r  )r   r~   r   r  r  Zdefault_subtask_workflowr   r  ZissueTypeMappingsrO  r   r  Tr   r(   )r   r+   r-   r   r   rK   r  rM   r  r   r   r  r  r   r  r   r  r   rq  )r   rw  Zdefault_task_workflow_ext_idr  Zscheme_dictZworkflow_scheme_objr  r  r  r  r  r  r  r   r   r   _process_workflow_scheme\  sP    

 
(z&JiraAPIImport._process_workflow_schemer;   c              	   C   s   | j  | d}| j  | d}| r8d S t|d6}| D ](}| j  jd7  _|t|d  qJW 5 Q R X t	
|| | j jdd t  d S )N.jsonz.json.dirtyr  r(   r2  Tr  )r+   rZ   r[   r\   r]   json_object_countr4  r^   r5  rd   rg   r   r   )r   Zapi_funcr   ra   ri   rb   r6  r   r   r   _simple_dump  s    
 zJiraAPIImport._simple_dumpc              	   c   sN   | j  | d}| rJt|d}|D ]}t|V  q.W 5 Q R X d S )Nry  r+)r+   rZ   r[   r\   r]   r^   r3  )r   r   ra   rb   r6  r   r   r   r     s    zJiraAPIImport._simple_getc              
   c   sf   | j  d}| rbt|D ]>}|dr2q"t| d| dd}t	|V  W 5 Q R X q"d S )Nr   .dirtyr   
/info.jsonr|  )
r+   rZ   r[   r\   re   listdirr
  r]   r^   r_   )r   r  r  rb   r   r   r   r     s    
zJiraAPIImport._get_projectsc                 C   s&   |   D ]}||d kr|  S qd S r<  )Zget_projects)r   r  r  r   r   r   get_project  s    zJiraAPIImport.get_projectc              	   c   sH   | j  d|}t| dd}|D ]}t|V  q(W 5 Q R X d S )Nr   /issue_types.jsonr|  r+   rZ   r[   r]   r^   r3  r   r  r  rb   r6  r   r   r   r    s    z*JiraAPIImport._get_issue_types_for_projectc              	   c   sH   | j  d|}t| dd}|D ]}t|V  q(W 5 Q R X d S )Nr   /statuses.jsonr|  r  r  r   r   r   r    s    z'JiraAPIImport._get_statuses_for_projectc              	   c   sH   | j  d|}t| dd}|D ]}t|V  q(W 5 Q R X d S )Nr   /roles.jsonr|  r  r  r   r   r   r    s    z JiraAPIImport._get_project_rolesc              
   c   sV   | j  d|d}t|D ]2}t| d| dd}t|V  W 5 Q R X qd S )Nr   rR   r   r~  r|  r+   rZ   r[   re   r  r]   r^   r_   )r   r  	board_dirZboard_idrb   r   r   r   r    s    z!JiraAPIImport._get_project_boardsc              
   c   sV   | j  d|d}t|D ]2}t| d| dd}t|V  W 5 Q R X qd S )Nr   rZ  r   r~  r|  r  )r   r  sprints_dirZ	sprint_idrb   r   r   r   r&    s    zJiraAPIImport._get_all_sprintsc              	   c   sH   | j  d|}t| dd}|D ]}t|V  q(W 5 Q R X d S )Nr   /versions.jsonr|  r  r  r   r   r   rg    s    z#JiraAPIImport._get_project_versionsc              	   c   sH   | j  d|}t| dd}|D ]}t|V  q(W 5 Q R X d S )Nr   /components.jsonr|  r  r  r   r   r   re    s    z%JiraAPIImport._get_project_componentsc                 C   s0   | j  d|d}tdd t|D }|S )Nr   tasksc                 s   s   | ]}| d  V  qdS )r}  N)r
  )r   entryr   r   r   	<genexpr>  s     z5JiraAPIImport._count_project_tasks.<locals>.<genexpr>)r+   rZ   r[   sumre   r  )r   r  	tasks_dircountr   r   r   r    s    z"JiraAPIImport._count_project_tasksr     c              
   c   s   | j  d|d}tt|D ]\\}}||k r4q"|dr@q"||| krP qt| d| dd}t	|V  W 5 Q R X q"d S )Nr   r  r}  r   r~  r|  )
r+   rZ   r[   r   re   r  r
  r]   r^   r_   )r   r  startr   r  iZtask_idrb   r   r   r   r     s    
zJiraAPIImport.get_project_tasksc                 c   sB   |d  d}t |D ]$}|dkr&q|| d| dV  qd S )Nr   z/attachmentsall_attachments.zipr   )r  ra   )re   r  )r   r   Zattachments_pathr  r   r   r   r    s
    z$JiraAPIImport._get_issue_attachmentsc              	   c   s>   |d  d}t |d}|D ]}t|V  qW 5 Q R X d S )Nr   z/comments.jsonr|  )r]   r^   r3  )r   r   comments_file_pathrb   r6  r   r   r   rC    s    z!JiraAPIImport._get_issue_commentsc              	   C   s6   |d  d}t |d}t|}W 5 Q R X |p4g S )Nr   z/worklog.jsonr|  r]   r^   r_   )r   r   r  rb   r8   r   r   r   r?    s    z!JiraAPIImport._get_issue_worklogsc              
   C   s:   |d  d}t |d}t|W  5 Q R  S Q R X d S )Nr   z/properties.jsonr|  r  )r   r   Zproperties_file_pathrb   r   r   r   rH    s    z#JiraAPIImport._get_issue_propertiesc              
   C   s:   |d  d}t |d}t|W  5 Q R  S Q R X d S )Nr   z/watchers.jsonr|  r  )r   r   Zwatchers_file_pathrb   r   r   r   r     s    z!JiraAPIImport._get_issue_watchersc              
   C   sD   | j  d}| r@t|d}t|W  5 Q R  S Q R X i S )Nglobal_settings.jsonr|  )r+   rZ   r[   r\   r]   r^   r_   )r   global_settings_file_pathrb   r   r   r   _get_jira_global_settings  s
    z'JiraAPIImport._get_jira_global_settingsc              	   C   s@   | j  }| j d}t|d}t|| W 5 Q R X d S )Nr  rY   )r@   Zget_jira_global_settingsr+   rZ   r[   r]   r^   r`   )r   Zglobal_settings_datar  rb   r   r   r   _dump_jira_global_settings  s    
z(JiraAPIImport._dump_jira_global_settingsu/   Обработка ссылок в тексте)r~   attachments_dirr   c                    s  |sdS t |ddd D ]*}d}|dr4d}|j| }z|drd| jjd|  W q|d	stj	
| jj|}|| jjs| jjd
|  W qt|dd \}}|s| jjd|  W qtdtj	|}	|	r\|	d  |	d dd}
 |
d kr\|d d }tt fdd|d }|r\| jjd|  W qtj	|dd dd }tj|}||}tjj||d | jjjjd}|stj| j||d | jjjjd}||_||_d|_|   |j!"  W q t#k
rF } z*| jj!$  | jjd| d|  W 5 d }~X Y qX qt |d}|jddid D ]P}|jd!r| %|jd! |d"  n$|jd#rd| %|jd# |d"  qddS )$Nr   Zlxmlc                 S   s   |  dp|  dS r1  r3  r5  r   r   r   r6    r7  z+JiraAPIImport._dump_links.<locals>.<lambda>r2  r!  zmailto:u   Это почта: ZhttpuH   Это внешняя ссылка, оставляем как есть: r:  u   Это не файл: r;  r(   r#  r  rh   r   c                    s   |  d kS r<  r=  r>  r@  r   r   r6  (  r7  uE   Этот файл уже обработан во вложениях: r   r9  r   r!  
obj_ext_idr   r+   r!  r  r   Fu0   Не удалось скачать ссылку r$  r  r  r  r  r  r   r   )&r   r%  r4  r  rC  r+   rU   r`  rD  rE  Zurljoinr@   r  rK  rL  r   rG  rH  rF  rI  r   rK   r  r  r[   CmfImportDownloadrM   r   r   pathr   
downloadedr   r.   r/   r0   r1   r7  )r   r~   r  r   r+  r   r  rP  encodingrM  Zattach_filenamerN  rO  r  ra   download_jobr-  r*  r   r@  r   _dump_links
  st    



 


.
zJiraAPIImport._dump_linksc                 C   s   | j jjrdS | j |d }| jD ]j}| j| d s8q$|d |sHq$|d | d}|r$|D ]*}|| j| d krb| j| d | qbq$| | j dS )u   На коробке только через мету задачи можно получить варианты выбора полей

        Args:
            issue (dict): задача
        Nr   customrh   ZallowedValuesoptions)r@   rI   rJ   Zget_issue_metarc   rM   r   rj   )r   r   metaro   r  optionr   r   r   _get_field_optionsH  s    

z JiraAPIImport._get_field_optionsc                  C   s@  t dd* || _d}| jjd| d|  z| j|d ||D ]}zzb| j r|W W | W W 5 Q R  dS | jjd|d	  d
 |	t
|d }	|	 rW W .qJ| | |	t
|d d }
|
jdd |
|d< |		d}|d d rv| jjdt|d d   ztj| jjd|d  d}| jjjrptj| jjd|d  d}d}|	|}| jjjj d|d  }tjj||| jjjjd}|stj| j||| jjjjd}||_||_d|_|  |j  W nD tk
r> } z$| jjdt d| t j!dd W 5 d }~X Y nX |d d D ](}|drL| "|d | j# |
 qL| j$dr|
	d}| rt%| t&|d Z}| j'|d }|D ]2}| "|d | j# |
 | "|d! | j# |
 qt()|| W 5 Q R X |d" d# D ](}|dr| "|d | j# |
 q|d d$rp| "|d d$| j#|
 |d d%r| "|d d%i | j#|
 |d d&i r| "|d d&i | j#|
 |  j*| +|d' d( ||7  _*|
	d)}| rt%| | jjd* t&|d+}| j,|d D ]r}| "|d | j# |
 | "|d! | j# |
 |-t(.|d,  | j j/d7  _/|  j*| +|d- ||7  _*q:W 5 Q R X | j0|d }|
	d.}t&|d }t()|| W 5 Q R X | j1|d }|D ]}| "|| j# |
 q|
	d/}t&|d }t()|| W 5 Q R X t23|
|	 |jdd |		d0}t
|	4dd1|d< t&|d "}t()|| | j j/d7  _/W 5 Q R X W n(   |d7 }| jj5d2|  Y nX W 5 | jj  X qJW n.   | jj5d3|d4  d5 |d7 }Y nX |6| W 5 Q R X d S )6NFr  r   u   Задачи с u    до r   r(   u   Задача 'r   'r}  Texist_okr   rN  rh   r   u%   Обработка вложений: zsecure/attachmentzip/z.zipzsecure/issueAttachments/r  r   r  r  r  r$  r)   r   rU  zworklog.jsonrY   r8  rL  rM  rP  rQ  rR  rS  r   zcomments.jsonu#   Дампим комментарииr  r2  r   zproperties.jsonzwatchers.json	info.jsonr   u0   Не удалось скачать задачу u2   Не получить задачи проекта r   u    с сервера)7r   r@   r+   rU   rB  Zget_all_project_issuesr.   r/   r-   r[   r&   r\   r  mkdirr   re   r  r   r  rI   rJ   r   r   rK   r  rM   r   r  r   r0   r,   r  r3   r4   r7  rH   rO   r  r]   Zissue_get_worklogr^   r`   r2   r  Zget_commentsr4  r5  rz  Zget_issue_propertiesZget_watchersrd   rg   r  	exceptionr  ) r   r  maxr  r  r@   tasks_that_are_doner  r   Ztask_dirZtask_dir_tmpr  r!  r  Zattach_file_pathr   r  r-  r  Zworklog_file_namerb   Zworklogsr@  rA  Zcomments_file_namer)  Z
propertiesZproperties_fileZwatchersr   Zwatchers_fileZ	info_pathr   r   r   _dump_task_part_  s    



 


2


   


*


zJiraAPIImport._dump_task_partr  rD  c                 C   s  t |d  d}|jdd | j }d}t|| j d }|sFd}d}g }d}	||	kr|| | }	|	|krr|}	| jj| jd| || |	||t	| j
j| j
j| j
j| j
j| j
j| j
jjd	|d
d}
||
 |
  | jjd|
  |d7 }qR|d }| jjd| d |D ]2}
|
j|d |
 r| jjd tdq| jjd | sv|| 7 }q\| jjd|  | j|d | |d dD ]}| j r|  S t |d  d}| j!  d}| j!  d}|" rt#$|| t%|dR}t%|d<}t%|d&}|D ]}||kr.|&| q.W 5 Q R X W 5 Q R X W 5 Q R X |'  t#(|| q|S )Nr  z/tasksTr  r   r(   r   Z
dump_tasks)r  r)  tokenpassword
verify_sslrJ   )r  r  r  r  r@   r  targetr   r7   u$   Форкаем _dump_task_part proc=r<  u%   Таймаут дампа задач u    минут)timeoutu$   Возможно завис дампu"   Ошибка дампа задачuK   Закончили параллельное скачивание задачu   Ошибок: r   r   r   r0  zusers.json.newr1  r  ))r   r  ru   Queuer}   ry   rt   Threadr  r   r@   r  r)  r  r  r  rI   rJ   r   r  r+   rU   rB  r   is_aliver.  TimeoutErroremptyrM   r   r  r-   r[   rZ   r\   rd   r  r]   r4  unlinkrg   )r   r  rD  r  r  r  stepr  r  Z	max_limitr  r  r   Zissue_users_pathZall_users_pathZall_users_path_newf1f2Zf3liner   r   r   _dump_tasks  sv    
 	



 

,zJiraAPIImport._dump_tasksc              	   C   sZ   t |d  d}| r$t| t|d"}| j|d }t|| W 5 Q R X d S )Nr  r  rY   r   )	r   r\   re   r  r]   r@   Z!get_project_issue_security_schemer^   r`   )r   r  r  rb   perm_schr   r   r   #_dump_project_issue_security_scheme	  s    
z1JiraAPIImport._dump_project_issue_security_schemec           &      C   s  dd l }| jd d d }|d s*tddd |d D | _| j d	}|jd
d t| j}t	d| }| jD ]d}
z| j
 rW  d S | j jd7  _| jjd|d  d |t|d }|d}| s| jj|d }	t||d  d|	d< tj|	d d
d | jjd t|	d  d}
|
 rVt|
 t|
d}| j|	d }zx|d D ]j}|d d dkr|| jjjr|d dr| |d d | j  n| |d d | j  q|W n&   | jjd| tjd  Y nX t || W 5 Q R X | !|	 | jjjr|	d  sh| j"|	d gd d! |	d"< n| jjd# t|	d  d$}| rt| t|dD}| jj#|	d d%D ]*}| j jd7  _|$t%|d&  qW 5 Q R X n~| j&| jj' d'|	d   |	d"< d|	d" krB|	d" d |	d" d(|	d"< n.| j&| jj' d)|	d" d   |	d" d*< |	d+}|r| jd,|  | j&|d- }|(  | |	d+< | |	d+ | j) | j  | jjd. t|	d  d/}| rt| t|dD}| jj*|	d d%D ]*}| j jd7  _|$t%|d&  q$W 5 Q R X | jjd0 t|	d  d1}| rt| t|d}| jjj+|	d d2, D ]}| j
 r W 5 Q R  W  d S | j jd7  _| j&| }| jjjrt|d3g D ]d}|d d4kr| jjj-|d5 d|d5 d6d7|d8< |d8 r| |d8 | j) | j  q|$t%|d&  qW 5 Q R X | jjd9 t|	d  d:}| rt./| |  t|	d  d;}| rt./| |  z| jj0|	d d2D ]}| j
 r4 W W  d S | j jd7  _| jjd<|d   | jj1|d |d*< | j&|d* d= d-  |d* d=< | jjjrd>d | j2|d D |d* d?< |	d |d@< |t|d }|jd
d |d}t|dA}t || W 5 Q R X z@|d dBkrZ| jjdC | j3|d D ]}| j
 rl W  W W  d S | j jd7  _|t|d }|jd
d |d}| r| jjdD|d  dE qJg |dF< | jjdD|d   | j4|d D ]6} | j
 r  W  W W  d S |dF 5| d  qt|dA}t || W 5 Q R X qJW n |j6j7k
r }! z^|!j8j9dGkrdHt|!ksdIt|!kr| jt|! n| jjdJ| tjd
d
dK  W 5 d }!~!X Y n*   | jjdJ| tjd
d
dK  Y nX qW n6   |  j:d7  _:| jjdL|	 tjd
d
dK Y nX | jjdM |	d  dN}"t|"db}| j;|	d D ]J}#| j
 	r W 5 Q R  W  d S | j jd7  _|$t%|#d&  	q~W 5 Q R X | jjdO |	d  dP}$t|$d}| j<|	d D ]r}%| j
 
r0 W 5 Q R  W  d S |%d+
rX| |%d+ | j) | j  | j jd7  _|$t%|%d&  
qW 5 Q R X dQtj=dR< | j j>|7  _>| jj?d
dS | jj@A  dTtj=dR< t.B|	d | t||	d< |	d  dU}t|dA}t |	| W 5 Q R X n t|dV}tC|}	W 5 Q R X | jD|	d }| jjdW|  |  j:| jE|	|dX7  _:W qv   |  j:d7  _:|dYr| jjdZ|d  d[tjd
d
dK n| jjd\| tjd
d
dK Y qvX qv| jjd] d S )^Nr   r   r   	isCheckedu5   Не выбрали ни одного проекта!c                 S   s$   g | ]}|d  r|d r|d  qS )r5   r  r   )r   rI  r   r   r   r   	  s       z0JiraAPIImport._dump_projects.<locals>.<listcomp>childrenr   Tr  P   r(   u   Дампим "r   r  r   r  r   r}  r  u   Дампим права/permissions.jsonr  permissionsholderr  r   	parameteru0   Не удалось сдампить права rn  r  ZworkflowSchemer  u/   Дампим типы задач проектаr  r  r2  z$rest/projectconfig/1/workflowscheme/)r   rB  zrest/api/2/workflowscheme/rB  r  u1   Ищем руководителя проекта r   u*   Дампим статусы проектаr  u$   Дампим роли проектаr  )r   r  r  Z	actorUserrB   )r/  rQ  r,  u&   Дампим доски проектаz/boards/z	/sprints/u   Доска r   c                 S   s   g | ]}|qS r   r   r   r   r   r   r   	  s     Zquick_filtersr  rY   )ZscrumZsimpleu&   Дампим спринты доскиu   Спринт u    уже скачанr  i  z"The board does not support sprintsuD   Данная доска не поддерживает спринтыu4   Не удалось сдампить спринты r  u   Не удалось сдампить доски, возможно проект не поддерживает доски или у вас нет прав project_info=u(   Дампим версии проектаr  u!   Дампим компонентыr  r   NO_CACHEr  1r~  r1  u*   Дампим задачи проекта: r  r  u   Проект "u   " является приватным и не может быть импортирован, отключите приватность в Jira или предоставьте пользователю праваu2   Не удалось получить проект u0   Закончили дампить проекты)Frequestsr   r0   rS   r+   rZ   r[   r  r   r}   r-   rz  rU   rB  r&   r\   r@   rI   r  re   makedirsr   r  r]   Zget_project_permission_schemerJ   rM   r7  r,   r3   r4   r^   r`   r  Zget_project_schemeZget_issue_for_projectr4  r5  jira_requestr  raise_for_statusrH   get_statusesZget_project_rolesrk   r   rd   rmtreeZget_all_agile_boardsZget_agile_board_configurationZget_board_quick_filtersZget_sprintsZget_sprint_issuesr   
exceptionsZ	HTTPErrorZresponser  r2   Zget_project_versions_paginatedZget_project_componentsenvironprogressr   r.   r/   rg   r_   Zget_project_issues_countr  )&r   r  rootZprojects_dirrD  r  r   r  Z	data_filer  permission_filerb   r  	perm_dictZ
types_filer  r  r8   Zstatuses_filer6  Z	role_fileZrole_urir  r  Z
boards_dirr  r  r  Zboard_info_filer   Zsprint_pathZsprint_file_pathr   r-  Zversion_list_filenamer[  Zcomponents_list_filenamer\  r   r   r   _dump_projects	  s   





"



$


"



 "



$


*  ""

$$zJiraAPIImport._dump_projectsc                 C   s`  | j  }t|\}}}|| d }| j d| d | j jd | j jd |   |  | _	| j jd | 
| jjd | j jd | 
| jjd	 | j jd
 | 
| jjd | 
| jjd | jjjr| j jd | 
| jjd | j jd | 
| jjd dtjd< d| j _| j   t  dtjd< | j jd |   dS )u_   
        Загружаем все данные из апи, кроме аттачей
        d   u   Свободно места: rY  u$   Скачаем данные из APIu9   Дампим глобальные настройки Jirau'   Дампим пользователейr   u   Дампим статусыrt  u    Дампим типы задачrl  issue_security_schemesu*   Дампим бизнес процессыrp  u7   Дампим схемы бизнес процессовrw  r   r     r  u   Дампим проектыN)r+   rZ   rd   
disk_usager,   rU   rB  r  r  rO   r{  r@   r   r  Zget_issue_typesZget_issue_security_schemesrI   rJ   Zget_workflowsZget_workflow_schemere   r  r  r   r   r  )r   Zdownload_pathrT  usedfreeZfree_percentr   r   r   download_data	  s6    




zJiraAPIImport.download_datac                 C   s`   dt jd< tjj| jjdddgd}|jrRt|j	|j d |_
|jdd	 t  d
t jd< d S )Nr   r  rq  rz  r  )r   rh   r  Tr  r  )re   r  rK   Z	CmfImportrM   r+   r   rz  r}   rq  r  r   r   )r   r+   r   r   r   r  
  s    
zJiraAPIImport._calc_progressc                 C   st   dd l }|jdddd| jj d| jjj d| d| d	tj d
gddt	ddt	ddd}| j
d|j  |S )Nr   r  r  r  z&cmf_import = models.CmfImport.get(id="z#");cmf_import.process_cross_links("r  r  r  r  Tr  r  r  r  r  )r  r  r+   r   r@   rI   r  rx   r  r]   r,   r  )r   
model_namer  r  r  r   r   r   _execute_cross_links
  s    
.  z"JiraAPIImport._execute_cross_linksc                 C   sh  ddl }dddgdddgg}dD ]2}| jjd	| d
| j  d}tt| }|j|d}|sdq g }t| jD ]0}| 	||}	|t
j7 }||	 ||krr qqr| j  ||k r$|d t|D ]V}	|	 }
|
dk	r|  j|
7  _||	 | 	||}|t
j7 }|| | j  qqt|D ]&}	|	  |	jr,|  j|	j7  _q,q | jjd dS )uU   
        Заменим ссылки на задачи и документы
        r   Nr+   r  r  r   T)r  r   r  r   u   Обрабатываем u+    в несколько процессов: r   r(   u2   Закончили обработку ссылок)r  r+   rU   rB  r{   varsrK   r  r  r  rx   r  r   r  r  r   r  r2   r  r  r  )r   r  r   r  r  rn   Zobj_cntr  r  r  r  r   r   r   r   _process_cross_links 
  sB    








z"JiraAPIImport._process_cross_linksc              
   C   s  ddl m} t|d  d}| }dd |d D }t|d}t|}W 5 Q R X | jjjj d	|d
  }t	j
j|d}	|	st	j
|d| jd}	|d |	_|	jdd |d D ]}
zd	| jjjj d	|
d
  d	}||
d rt	jj||
d  dd}n| jjd|
  W qddd| dg}ddddg}t	jj||d}|st	jj||	|d}|s| jjd|
 tjddd |  jd7  _W q|jr|j | |_n||_|
d d  d!kr| jjjj d	|
d d! d
  }t	jj|d}|j| |jdd nD|
d d  d"kr<d|_|jdd n|
d d  d#krdd|_|jdd n|
d d  d$krd|_|jdd n|
d d  d%krd|_|jdd n|
d d  d&krV| jjj r|
d d'r| !|
d d' }n| !|
d d( }|r(|j"| |jdd n*| jjd)|
 tjddd |  jd7  _n|
d d  d*kr&| jjj rd	| jjjj d	|
d d'  d	}n&d	| jjjj d	|
d d* d  d	}t	j#jddd| dgd+}|r|j"| |jdd n*| jjd,|
 tjddd |  jd7  _n2| jjd-|
d d   tjddd |  jd7  _W q   | jjd.|
 tjddd |  jd7  _Y qX q|	S )/Nr   )get_datar  r  c                 S   s   i | ]}|d  |d qS )Z	jira_coderE   r   r   r   r   r   rK  R
  s      z6JiraAPIImport._process_permissions.<locals>.<dictcomp>Zproject_perm_permissionr1  r   r   rv  T)r   r  r+   r   r   r  Z
permission)rE   Zinclude_deletedu4   Не нашли соответствия права r   rX  rY  r   access_membersaccess_project_rolerW  )permr   rh   u*   Не найдено правило для r  r(   r  r  ZprojectRoleZapplicationRoleZprojectLeadrR  rP  r   r   r  u9   Не нашли пользователя для прав rl  r   u-   Не нашли группу для прав u'   Неизвестный тип прав u6   Не удалось загрузить правило )$Zcmf.system_datar  r   r]   r^   r_   r+   r   r   rK   ZCmfProjectPermSchemerM   r   r   ZCmfProjectPermPermissionrU   r`  ZCmfProjectPermSchemeRuler,   r3   r4   r2   r   r  r  r   Zaccess_local_userZaccess_ownerZaccess_responsibleZaccess_authorr@   rI   rJ   r   r  rd  )r   r  r  r  Zsystem_dataZmap_permrb   Zperm_scheme_dictr   Zperm_schemer  r  r   rh   Zscheme_ruleZrole_ext_idr  r   rl  r   r   r   r  N
  s    
"$&$z"JiraAPIImport._process_permissionsc                 C   sX  t ddB d}| }|dkr&q@| j r4q@| jjd|d   | jj|d dtj	d}z|
  t|d	 }t|d
"}|jddD ]}|| qW 5 Q R X | jjd|d	   tjj|d d}	d|	_|	  W nJ tk
r2 }
 z*|d7 }| jjd|d  d|  W 5 d }
~
X Y nX W 5 Q R X q|| W 5 Q R X d S )NFr  r   r  u    Пытаемся скачать r!  T)streamr  r  zwb+i    )Z
chunk_sizeu   Сохранили в r   r  r(   u+   Неудалось скачать файл r$  )r   rM   r+   r-   rU   rB  r@   r  rx   ZIMPORT_DOWNLOAD_TIMEOUTr  r   r]   Ziter_contentr4  rK   r  r  r   r0   r  r  )r   download_queueerror_queuer  r  r8   Z	save_pathrb   chunkZdownload_job_objr-  r   r   r   _download_file_worker
  s.    
<z#JiraAPIImport._download_file_workerc           	      C   s  | j jd | j }| j }d}| j dg dd| j jjjgdddgg}tj	j
|dsp| j jd	 |S g }t| jD ]J}| jj| jd
| ||dd}|| |  | j jd|  q~tj	j|ddgdD ],}| j  r q||j|j|jd q|D ]}|d q| s@|D ]}|  q0| j  rP|S | j jd | s| j  rx|S || 7 }q^|S )Nu=   Скачиваем все найденные вложенияr   zplugin.plugin.source_hashr   r   r  Fr   u   Вложений нетr  )r  r  r  u    Запустили задачу r!  r  rW  )r   r!  r  r  u%   Закончили скачивать)r+   rU   rB  ru   r  r   r   r   rK   r  r  r  ry   rt   r  r  r   r  Zslistr-   r  r   r!  r  r  r   rM   )	r   r  r  r8   r   Zprocsr  r  r  r   r   r   download_files
  sD    





zJiraAPIImport.download_filesc                 C   s
  |  dD ]}| j r d S | jjjj d|d  }tjj|d}|s\tj|| jdd}|jrx|d |_	|d|_
||_|jdd	 |d
 D ]p}| jjjj d|d  }tjj|d}|stj|| jdd}||_|jr|d |_	|d|_
|jdd	 qq
d S )Nr  r   r   rv  Trd  r   r   r   Zlevels)r   r+   r-   r   r   rK   r  rM   r  r   r~   r_  r   rX  r   )r   r  r   Z
scheme_objro  Zlevel_ext_idZ	level_objr   r   r   _process_security_levels
  s,    


z&JiraAPIImport._process_security_levels)rl   c                 C   s  |d d  dd}|d| jjjj 7 }|d d|g g g g dd	d	d
}|d d d dkrrd|d< d|d< nR|d d d dkrd|d< d|d< n*|d d d dkrd|d< d|d< n|d d d dkrd|d< d|d< n|d d d dkrd|d< d|d< d|d< n|d d d dkr`d|d< d|d< dd  |d d! D |d"< nd|d d d d#krd|d< d|d< n:|d d d d$krd%|d< d&|d< d|d'< d(|d< d)d  |d d! D |d"< n|d d d d*krd+|d< d+|d< d|d'< dg|d,< n|d d d d-krFd|d< d|d< d.|d< n~|d d d d/krd+|d< d+|d< d|d'< d.g|d,< nD|d d d d0krd%|d< d&|d< d|d'< d(|d< i |d"< nd }|S )1Nr   r   customfieldcfr  r   T)r  r  r  Zlogic_prefixF)r   r  
class_nameZvisible_filterrequiredZrequiredChangedZschemar  z:com.atlassian.jira.plugin.system.customfieldtypes:textareaZCmfStr
field_typeZCmfTextwidgetz:com.atlassian.jira.plugin.system.customfieldtypes:datetimeZCmfDateTimez<com.atlassian.jira.plugin.system.customfieldtypes:datepickerZCmfDatez7com.atlassian.jira.plugin.system.customfieldtypes:floatZ
CmfNumericZ	CmfNumberz<com.atlassian.jira.plugin.system.customfieldtypes:userpickerZCmfRelationr   rn   z8com.atlassian.jira.plugin.system.customfieldtypes:selectZ	CmfChoicec                 S   s   i | ]}|d  |d qS r   r   r   r   optr   r   r   rK    s      z,JiraAPIImport._map_field.<locals>.<dictcomp>r  choicesz;com.atlassian.jira.plugin.system.customfieldtypes:textfieldz=com.atlassian.jira.plugin.system.customfieldtypes:multiselectCmfM2MZCmfCascadeChoiceleftZCUSTOM_CHOICE_MODELc                 S   s   i | ]}|d  |d qS r  r   r  r   r   r   rK    s      zAcom.atlassian.jira.plugin.system.customfieldtypes:multiuserpickerZCmfGenericM2MrK   z=com.atlassian.jira.plugin.system.customfieldtypes:grouppickerrd  zBcom.atlassian.jira.plugin.system.customfieldtypes:multigrouppickerzAcom.atlassian.jira.plugin.system.customfieldtypes:cascadingselect)r  r+   r   r   )r   rl   r  r8   r   r   r   
_map_field
  st    
 

zJiraAPIImport._map_fieldc              	      s  d | _ | jD ]:}| jtjj d D ]"}|d d |kr"| j| |d< q"q| jd | jjjj	}tj
j|dgd}|stj
|| jddd	}|jdd
 tjj|dgd}|stjd| j|dd}d|_|jdd
 |jr*tjj|dgd}|stjd| j|d}|jr*||_||_d|_|jdd
 | jjjddsBd S |jj}dd tj D }g }	g }
| jtjj d D ]}|dr|d d s| jd|d   qx| |}|d d dd}|d| jjjj	 7 }|s| jjd|d  tjd |  jd7  _qx|d dkr2|
| |d r`| jd|d  d |d   qx||kr|| d! |d! kr| jjd|d  d"|d!  d#|| d!  tjd | jd$| d%|d  d& || |d< qx| jd'| d%|d  d( ||d) |< |d* d+ d | |	| qx||_|  |j  |	rtj  t !d tj"d,di d-d tj D }|	D ]<}|d d dd}|d| jjjj	 7 }|| |d< qd8t#d. fd/d0|
D ]D}tjj$|d d1  }t%t|j&  |d d2 D ]}| qq| jjj}|d3 D ].}|d4 tjjkr:| jtjj d |d< q:|| j_d5t'j(d6< | j  | jj  d7t'j(d6< d S )9Nrh   r   r   u0   Обработка кастомных полейr   r  TzJira import custom fields)r   r+   r  r   r   z"Jira import custom fields (UiForm))r  r+   r   r   r   )r  r+   r   Zimport_custom_fieldsFc                 S   s   i | ]}|d  |qS ro   r   r   rb   r   r   r   rK  Z  s      z8JiraAPIImport._process_custom_fields.<locals>.<dictcomp>r  u!   Поле не кастомное r   r  r  r  u+   Неудалось смапить поле rn  r(   r  r  r   u	   Поле u    уже смаплено  в r  u:   ,                                         widget поля u5    не совпадает с существующим u   Поле с именем (u4   ) есть в таблице, мапим в негоu(   Создаем поле с именем u   ), и мапим в негоZ	ui_fieldsZui_group_fieldsr9  r  c                 S   s   i | ]}|d  |qS r   r   r  r   r   r   rK    s      )option_dictc                    sf    j | d d}|s" | d d}| d |_|r8|j|_|jrF|  |  dg D ]}|| qRd S )Nr   r;   Zdisabledr  )rM   Z
cmf_hiddenr   Zchoice_parent_idrg  r   )r  Z
parent_optr  ZchildZchoice_modelcreate_optionr   r   r    s    
z;JiraAPIImport._process_custom_fields.<locals>.create_optionro   r  r   r   r   r  r  )N))r?   rc   r   rK   r   r   r+   r,   r   r   r  rM   r   Z	CmfUiFormr  r  ZCmfUiFormSchemeRuler  r  r   r   ui_form_jsonrp   r  r  r3   r4   r2   r   r.   r/   Zcustom_field_syncZgeventr  Zcustom_field_sync_update_modelsr   rh   r  rn   re   r  )r   Zext_field_namerl   r   r  Zui_view_formr  r  Zshop_fieldsZ
new_fieldsZ
m2m_fieldsZui_fieldr   Zfield_classr  r   sr   r  r   _process_custom_fields5  s    

    


 







z$JiraAPIImport._process_custom_fieldsc                 C   s  dt jd< d| j_d| j_| j  t  dt jd< tdD ]}|   | j	s< qTq<tdD ]}| 
 }|s\ qrq\|  j	|7  _	|   |   |   |   |   |   |   |   |   | jjjr|   |   |   |   |   n|   |   |   |   | j r,d S d| j_| j  t  |  D ]>}|d dd	 | jD krnqL| jjd
|d  d qL| jj d| j	 t!j"ddd | j d| jj  | j	 S )Nr   r  r   r     r  r   c                 S   s   g | ]}|d  qS r  r   r   r   r   r   r     s     z0JiraAPIImport.process_import.<locals>.<listcomp>u'   Импортирован проект 'r   r  u#   Ошибок обнаружено: Tr  u-   Импортировано объектов: )#re   r  r+   rq  r  r   r   r  r  r2   r  r  rs  rz  ro  r  r  r@   rI   rJ   rv  rx  r  r  r  r`  rh  r-   r   rS   rU   rB  r,   r3   r4   )r   r  r8   r   r   r   r   process_import  s\    





zJiraAPIImport.process_import)T)r   r   )T)TT)N)r   r  )mr"   r#   r$   r%   r}   r'   r   r&   r   r   rG   propertyrc   rj   rm   staticmethodr   r   r   rq   rT   r   r   r   r   r   r   r   r   r   r=   r   r  r.  r'  r   rR  rm  rs  rz  rK   r  r  r  r  r  r  r  r  r  r  r  r  r  r+  r   r   r7  rB  rE  rJ  rN  r  r]  r`  r  r  rh  ro  rv  rx  r{  r   r   r  r  r  r  r  r&  rg  re  r  r   r  rC  r?  rH  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  __classcell__r   r   rV   r   r>   ;   s   
	 

	/3  


A 
9
c

E'.
t$
=
+
/
?$
 

U5
$

H
7
$	
=r= U!
.T$;rr>   )*Zcmf.appr   Zcmf.includeZdataclassesrK  pathlibr   rd   Ztempfiler   rD  typingr   r   r   r	   r
   r   r   Zbs4r   r   Zcmf.data_providers.baser   r   Z
cmf.modelsr   Z%common.models.cmf_plugin_import_mixinr   Zmodules.jira.apir   Zmodules.jira.base_importr   rK   r   r   r   r   Z	dataclassr   r=   r>   r   r   r   r   <module>   s(   $ 