U
    {yd                     @   s   d dl Z d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dl	Z	d dl
mZ d dlmZmZ d dlZd dlmZ d dlZd dlZd d	lmZ d d
lT d dlmZ d dlmZ d dlmZ G dd dejZdS )    N)copydeepcopy)Decimal)datetimetimezone)Logger)Path)DictIterator)cached_property)commit_all_ds)*)fields
cmf_import)
log_configc                       sT  e Zd Zejjd ZdZeedddZ	e
jdfeeddd	Zd
d ZedddZdd Z fddZdd Zdd Zdd Zed=ddZedd Zeejdd Zd>ed d!d"Zd#d$ Zedd%d&Z d'd( Z!d)d* Z"d+d, Z#eejddd-d.Z$d/d0 Z% fd1d2Z& fd3d4Z'd5d6 Z(e)dd7d8Z*d9d: Z+d;d< Z,  Z-S )?	CmfImport)CSV_SETTING_NAMEcsv_import_settings_fieldsloggerZCSV)returnc           	      C   s  G dd dt j}t j}tjr$t j}|  }t |}|| d|_|j	st
tj|}|jddd |d}tj|ddd	d
d}|| |t tj || |d}tj|ddd	d
d}|| |t tj ||  || tj|d
d	|dS )uE    Журнал логирования процесса импорта c                   @   s   e Zd Zdd ZdS )z)CmfImport.logger.<locals>.AnonymousFilterc                 S   s   t |ddS )N	anonymousF)getattr)selfrecord r   ./common/models/cmf_import.pyfilter   s    z0CmfImport.logger.<locals>.AnonymousFilter.filterN)__name__
__module____qualname__r   r   r   r   r   AnonymousFilter   s   r!   FT)parentsexist_okzdata.logzutf-8Di@KL    )filenameencodingZwhenmaxBytesbackupCountzanonymous_data.log)r)   r(   level)loggingZFilterINFOconfigDEBUGget_files_dirZ	getLoggerZsetLevelZ	propagatehandlersr   
UPLOAD_DIRjoinpathmkdirr   ZMyLogsHandlerZsetFormatterZ	FormatterZ	FORMATTERZ
addHandlerZ	addFilterZ
get_logger)	r   r!   r*   log_pathZapp_logZpath_logZfile_name_logZfile_handlerZanonymous_handlerr   r   r   r      sB    







zCmfImport.loggerF)msgr*   c                 K   s"   | j j||fdd|ii| dS )uI  Логируем как обычные лог для веб так и со скрытыми данными для возможности отправки в СТП

        Args:
            msg (str): сообщение
            level (int, optional): уровень логирования. Defaults to logging.INFO.
        Zextrar   N)r   log)r   r5   r*   r   kwargsr   r   r   r6   @   s    zCmfImport.logc                 O   sF   t j|  tj}t jt jtj|s8| j	
d t jd|S )uK   
        Возвращает путь текущему файлу
        initfiles)ospathjoinr/   r   LOG_NAMEexistsr-   r1   r   info)r   argsr7   	file_pathr   r   r   get_log_filenameI   s    zCmfImport.get_log_filenamec                    s`   t j tj}t jt jtj|s8j	
d d  fddt  D }|S )u   
        Список файлов после logrotate, чтобы можно было загрузить историю
        r8   Tc                    sB   g | ]:}t jt j |rtd |rt jd |qS )z%(?:data|anonymous_data)\.log(\.\d+)?$r9   )r:   r;   isfiler<   rematchr/   ).0fdir_pathr   r   r   
<listcomp>Z   s    z/CmfImport.get_log_filenames.<locals>.<listcomp>)r:   r;   r<   r/   r   r=   r>   r-   r1   r   r?   listdir)r   r@   r7   rA   Z	file_listr   rH   r   get_log_filenamesR   s    
zCmfImport.get_log_filenamesc              	   O   s   dd l }t| d}|d}tj|r6t| ||d,}| 	 D ]}|
dtj}|| qLW 5 Q R X tjd|  dS )Nr   Tzall_logs.zipwz/filesr9   )zipfiler   r/   r2   r:   r;   r>   removeZipFilerL   replacer-   r1   writer<   )r   r@   r7   rN   rI   Zzip_pathZmyziprA   r   r   r   get_all_logs_by_zip_   s    

zCmfImport.get_all_logs_by_zipc                    s$   | j s| jjdd| _ t   d S )NZOPENstatus_type)statusworkflowget_default_statussuper_calc_statusr   	__class__r   r   rZ   k   s    zCmfImport._calc_statusc                 C   s  | j d rd S | jr(| j | j d< d S | jdkr6d S | j d dtj tjjd | j d dtj	 tj	jd | j d dtj
 tj
jd | j d dtj tjjd | j d dtj tjjd | j d d	tj tjjd d S )
NZmodels_fieldsexcelu   Поля контактов)Zcaptionr   iconu   Поля задачu   Поля заметокu   Поля проектовu"   Поля списков задачu   Поля групп)json_settingspluginZcalc_models_settingstypeappendmodels	CmfPersonZimport_shop_fieldsr_   CmfTaskZ
CmfComment
CmfProjectCmfListCmfPersonGroupr[   r   r   r   _calc_models_settingsp   s>    

zCmfImport._calc_models_settingsc                 K   sH   |  ddddg | jd r d S | jj| d| jd< | j| _|   d S Nr`   attachmentsrb   ra   import_settingsr   )load_fieldsr`   ra   calc_import_settingssaver   r7   r   r   r   ro      s    
zCmfImport.calc_import_settingsc                 K   sH   |  ddddg | jd r d S | jj| d| jd< | j| _|   d S rk   )rn   r`   ra   tmplt_import_settingsrp   rq   r   r   r   rr      s    
zCmfImport.tmplt_import_settings c                 C   s4   d|  d d} td|  d|  t|  d S )Nu1   Ошибка при импорте данных. .ue   . Проверьте загружаемый файл и повторите попытку снова.zCmfImportError: : )stripr+   errorZ	cmf_alert)r5   er   r   r   
_log_error   s    zCmfImport._log_errorc                 C   s:   |  dg | jd D ]}|d | jkr|d   S qg S )Nr`   rm   namer   )rn   r`   r   )r   settingsr   r   r   r      s
    z$CmfImport.csv_import_settings_fieldsc              
   O   s(  dt jd< t  t  tjj| ddddgd}|d |j	j
d	d
d|_|  t  | }|r|jd |j	j
dd|_|  n0|jjdkr|jd |j	j
dd|_|  |jjd jtjd}W 5 Q R X W 5 Q R X tjj|tjjj|jdtt j!dd| ddd dS )uJ   
        Запускаем в фоне импорт данных
        1importrW   rV   Zprogresszplugin.plugin.*idr   u%   ***Запускаем импорт***ZIN_PROGRESSimport-started)rU   status_codeu2   ***Импорт успешно завершен***ZCLOSEDrT   u7   ***Импорт завершен с ошибками***import-errorr   r   z/files/u+   Импорт данных завершен z%d.%m.%Y %H:%Mz
, <a href=u(   >Журнал логирования</a>T)objZ	person_idrz   r5   Zis_notify_current_personN)"r:   environcmfutildisable_acldisable_notifyrd   r   getr6   rW   rX   rV   rp   commit_with_eventprocess_importr   r?   rU   r0   ZbaseFilenamerQ   r-   r1   Z	CmfNotifyZplace_notifygZcurrent_personr   valuerz   r   nowr   utc)Z	import_id_args_kwargsr   resr4   r   r   r   start   s0    


,zCmfImport.startN)rowc              	   C   s4  d}|dkr|j }|sg }i }t| jD ]Z\}}|d r(|| r(|d d dd |jkr^q(|d d |kr(|| ||d d < q(|s|S || d}	tt|	d	}
|D ]}|j|}t	|tj
st	|tjr,d}| D ]8}|jd
dd|| gd	d|| ggdgd}|r qq|r,t|	|| qt	|tjr<qt|	|||  || }t	|tjr~t|d}t|tjj}t	|tjrq||d|g q| jd r|j|dgd}|s0|d	}|r|j|dgdd}|r|
|	_|j|	jdgd}|r$|	j dttjd|	_|	}|  |S )u  
        Импортируем объект

        :param list row: Строка из файла
        :param models.* model: Модель для создания объекта
        :param list _filter: Для фильтрации объекта
        :param str key: Ключевое имя для сопоставления модели

        :return: Новый или найденный объект
        :rtype: Optional[models.*]
        Nmodel_fieldfield_qualnamert   r   verbose_name
field_namer   codeORrz   ==z**)r   r   ZRUunionT)r   r   Zinclude_deleted)rz   r   _z%Y%m%d-%H%M%S)r   	enumerater   split
class_namestrr   r   r   
issubclassZCmfRelationZCmfGenericRelationZrelated_modelssetattrZ
CmfRelBaseZCmfPhonephonenumbersparseZformat_numberZPhoneNumberFormatZINTERNATIONALZ
CmfNumericrc   r`   r   rz   r   r   r   r   rp   )r   r   model_filterkeyresult_fieldsisettingZnew_objZnew_coder   ZfieldZrel_objZ	rel_modelvalZphoner   r   r   r   _process_obj   sp    

zCmfImport._process_objc              
   C   s   |  dddddg | jrz0t  | jj| dW  5 Q R  W S Q R X W n tk
r } zl| dt  }t| t	| dt
|jd	  }| j| || _| jjd
d| _|   W Y dS d}~X Y nX | jdkr|  S dS )u9   
        Импортируем файл в бд
        r^   rm   	first_rowra   rl   r   ru   
r   r   FN)rn   ra   r   enable_import_moder   	Exception	traceback
format_excr   r   	format_tb__traceback__r   	exception	error_msgrW   rX   rV   rp   rb   _process_import_excel)r   errerr_msg
error_textr   r   r   r     s     
&
zCmfImport.process_importc           4      C   sf  ddl m} | jr| jd s dS d}zi }i }i }i }i }i }i }	|  }
t|
D ](\}}|dkrr| jd rrqTi }i }d}d}t| jD ],\}}|d r|| sq|d d d	d }||i }|d d
 d	}|d }|| }|dkr|}qn|dkr|}qt	|dkrP|dkr2|||d < q||i }|| ||d < nZ|dkrt|dkrt|
|| }n6|dkr||| }n|dkrd||| i}|||< |||< q|di }|dd}||d}|s|| tj|\}}|||d< || tj|\}}|di }|drD|d |d< |di }|r|	|}|s|dkrtjjddtjjddd}ntjjddtjjddd}||	|< ||d < ||d< ||d!< ||d"< d#d$d%|jgdd%|d gg}|| tj||\} }| j| |   | ||d < |rH||g }!|!|  |!||< |rt||d g }"|"| |"||d < |d&i d'd}#|#r|# d(|jj }$||$i }%||%d!< |#|%d'< |%d)g }&|&|  |&|%d)< |%||$< ||d}'|'sftjjd#d*d%|jgd'd%d+ggd,}(d#d*d%|(jgd'd%d-gg})d-||(d.d/}*|j| tj|*|)d0\}'}|'||d< || jtj|'| d1 qT|  D ]\}$}+tjjd#d*d%|+d! jgd'd%d2ggd,}(d#d*d%|(jgd'd%|+d' gdd3d4gg})tjj|)d,},|,stj|+d! |(|+d' tj!jd5dd6},|,  |+d) D ]} |,| _"| jd7d8 q"q|  D ]>\}}-||}.|.rH|-D ]}/|/j"#  |.|/_"|/  qdqH|  D ]N\}}0||}.|.r|0D ].}1||1}/|/r|/j"#  |.|/_"|/  qqW n~ t$k
r` }2 z^t%|2 d9| d:|2 d;t&'|2j(d<  }3| j)*|3 |3| _+| j,j-d=d>| _.|   W Y dS d?}2~2X Y nX d7S )@u   
        Запуск импорта из CSV

        :return: Истина в случае успешного импорта
        r   )CmfPluginImportMixinrm   Fr   rs   r   r   rt   r   parent_taskZchild_tasks   	executorslogic_prefixrg   priorityrV   rU   zproject.agiler   rf   Z
cmf_authorZ	cmf_ownerzproject.basez
task.base%baser   )Z	cmf_modelprefixZactivityztask.agile:softdev%ZsoftdevrW   projectparentZANDZ	parent_idr   rh   rz   r   tasksZtree_parent_idZSprintsr   u
   ДоскаZkanban)rz   r   tree_parentZ	list_type)r   r   
obj_fields
filter_obj)leftrightZEpics=z	task.epicztask.epic:default)r   r   rz   Z
logic_typeT)Z	only_datau   Строка ru   r   r   r   r   N)/Z%common.models.cmf_plugin_import_mixinr   rl   r`   Z	_get_filer   r   r   r   lenZget_logic_prefixZget_priorityZget_status_type
setdefaultZprocess_any_table_fieldsrd   rg   re   CmfWorkflowZcalc_workflowZCmfActivityr   rf   r   rc   rp   r   r   Z	CmfFolderrh   r   ZCmfListCmfTaskitemsCmfLogicTyper   loadr   r   r   r   r   r   r   r   rW   rX   rV   )4r   r   Z
number_rowZall_parent_taskZall_child_tasksZall_epic_tasksZ	all_tasksZ
cash_boardZcash_projectZcash_workflow_taskZ	file_readr   executorr   Zparent_task_codeZchild_tasks_codeidxr   Z
model_nameZ_rowZfields_namer   r   Zproject_fieldsr   r   r   Zexecutor_objZtask_fieldsZvalue_statusrW   Zfilter_taskZtask_objZtmp_parent_tasksZtmp_child_tasksZ	epic_nameZepic_keyZtmp_dataZ	tmp_tasksZlist_objZparent_folderr   r   dataZepic_objr   r   Zsub_taskZ
tasks_codeZsub_task_coder   r   r   r   r   r   2  sL   








 



 




"zCmfImport._process_import_excelc                 C   s>  |  |tj}|r|sdS d}| jD ] }|d r$|d d dkr$|}q$|dg tjj|jt|d |j	jdgd}|stj|t|d |j	| d	}|
  |jjd
d t|jjD ]*}|jj| jkrq|  |jj| q| j |tjtjjdd|jgdd|jggd}|ddg ||_||_|jr:|
  |S )u;   
        Импорт товара в сделку
        Nr   r   zCmfProduct.pricescurrencyfile_valprice)
product_idr   Zcurrency_idr   )productr   r   r   T)forceZdeal_idr   r   )r   r   r   Zsales_order)r   rd   Z
CmfProductr   rn   ZCmfPricer   r   r   r   rp   Zpricesr   r   Z
deal_itemsr   r   deleterO   ZCmfDealItemsr   dealr   
is_changed)r   r   r   r   Zprice_settingr{   r   itemr   r   r   _process_product  s@    
 zCmfImport._process_productc           	      C   s  |s|r|j sd S d }| jD ] }|d r|d d dkr|}q|sFd S g }tjj|d d}|sxtj|d d}|  |dd|jg |r|dd|jg n|dd|j jg | |tj	|}|
d	d
g |j s|r||_ n|j |_ |js||_|jr
|  |S )Nr   r   zCmfDeal.pipeliner   )rz   Zpipeline_idr   Z
company_idcompanypipeline)r   r   rd   ZCmfPipeliner   rp   rc   r   r   ZCmfDealrn   r   r   )	r   r   Zcontactr   Zpipeline_settingr{   r   r   r   r   r   r   _process_deal  s8    
zCmfImport._process_dealc                 C   s   dd l }tj  |  }||\}}}t|| d }|dkrZ| jd| d dS t	j
j| jdgd}|jjd	krd
|_|jr|  t  dS dS )Nr   d      u8   Импорт прерван, место на диске z% < 20%TrV   r~   )import-cancelingimport-canceledr   import-stoppingr   F)shutilr   r   Zimport_heartbeatget_download_path
disk_usageintr   rw   rd   r   r   r   rV   r   r   rp   r   )r   r   Zdownload_pathtotalusedfreeZfree_percentr   r   r   r   	is_cancel7  s     
zCmfImport.is_cancelc                  O   sF  t  2 t   tjj|d ddgd}|jjdkr|jjdkrf|jj	dd|_|
  t  td	D ]t}td
 tjj|jddgd}|jjdkr q|dkrnd|_|jj	dd|_|
   W 5 Q R  W 5 Q R  dS qnn|jj	dd|_|
  t  zzd}|jd dD ]L}tt| jdd|gdD ]*}|jd|  |jdd t  q<q|j  |jD ]*}|jd|  |jdd t  q||jj	dd|_|jd W nX tk
r } z8|jd|  t| t||_|jj	dd|_W 5 d}~X Y nX W 5 |
  X W 5 Q R X W 5 Q R X dS )u/   
        Отменяем импорт
        r   rV   rW   r~   )r   r   r   r   r   <   r   r   ;   u5   Не удалось остановить импортr   Nu#   ***** Отменяем импорт)rg   ZCmfAttachmentZCmfImportDownloadr   ZCmfProjectPermSchemeZCmfSchemeWfZCmfStatusCodeZCmfProjectRoler   re   ri   r   r   r   u   Удаляем T)ZTEXKOM_db_deleter   u,   Успешно отменили импортu   Ошибка удаления )r   r   r   rd   r   r   rV   r   rW   rX   rp   r   rangeZgeventsleepr   r   r   r?   varslistr   Zimport_objectsr   r   r   r   r   )r@   r7   r   r   r   Zcls_namerx   r   r   r   cancelI  sR    




&zCmfImport.cancelc                 O   s    |  ddddg | jj| dS )uk   
        Предварительный просмотр получившихся объектов
        r`   rl   rb   ra   r   )rn   ra   preview)r   r@   r7   r   r   r   r     s    zCmfImport.previewc                    s   t   dddddg S )Nr`   rl   rb   rV   ra   )rY   save_preload_fieldsr[   r\   r   r   r     s    zCmfImport.save_preload_fieldsc                    s   |    t jf |S N)rj   rY   rp   rq   r\   r   r   rp     s    zCmfImport.savec                 C   s@   | j r
d S | jr,| jjdkr,tjjdd| _ tjjdd| _ d S )NZCmfPluginJirazimport.system.jira:defaultr   zimport.system:default)rW   ra   r   rd   r   r   r[   r   r   r   _calc_workflow  s
    zCmfImport._calc_workflowc                 C   sN   |  dg ttj| jj  d}|| jjjj	}|j
ddd |S )Nzplugin.source_hashZ	_raw_dataT)r#   r"   )rn   r   r-   r1   r2   ra   Zui_namelowersource_hashr   r3   )r   r   r   r   r   r     s
    zCmfImport.get_download_pathc                 C   s   | j |  d S r   )ra   download_datar[   r   r   r   r    s    zCmfImport.download_datac                 C   s   | j |  d S r   )ra   download_filesr[   r   r   r   r    s    zCmfImport.download_files)rs   )NN).r   r   r    r   r   Zui_meta_skipr   r   r   r   r+   r,   r   r   r6   rB   r   rL   rS   rZ   rj   ro   rr   staticmethodry   r   Z
celery_appZtaskr   r   r   boolr   r   r   r   r   r   r   rp   r   r   r   r  r  __classcell__r   r   r\   r   r      sH   %		

!N B!#4r   )Zcsvr   r   Zdecimalr   r   r   r+   r   r   pathlibr   typingr	   r
   r   r   ZxlrdZopenpyxlZcmf.data_providers.baser   Zcmf.includeZcmfr   Zcommon.fieldsr   Zcmf.modules.logsr   r   r   r   r   r   <module>   s"   