U
Edt @ s d dl Z d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dl Z d dl
m Z d dl m
Z
mZ d dlZd dlmZ d dlZd dlZd d lmZ d d
lT d d lmZ d d lmZ d d
lmZ G dd dej Z dS ) N)copydeepcopy)Decimal)datetimetimezone)Logger)Path)DictIterator)cached_property)
commit_all_ds)*)fields
cmf_import)
log_configc s4 e Zd Zejjd ZdZeedddZ dd Z
e ddd Z fd
d Z
d d
Zdd Zdd Zed7ddZedd Zeejdd Zd8e ddd Zd d Zedd d Zd!d" Zd#d$ Z d%d& Z eejddd'd(Z d)d* Z fd+d,Z fd-d.Z!d/d0 Z"e#dd1d2Z$d3d4 Z%d5d6 Z& Z'S )9 CmfImport)CSV_SETTING_NAMEcsv_import_settings_fieldsloggerZCSV)returnc C s( t j}tjrt j}tj| dd|dS )uE Журнал логирования процесса импорта i@KL )Z backupCountZmaxByteslevel)loggingINFOconfigDEBUGr Z
get_logger
get_files_dir)selfr r ./common/models/cmf_import.pyr s zCmfImport.loggerc O s8 | tj }tjtj | s4| j d |S )uK
Возвращает путь текущему файлу
init)
r r LOG_NAMEospathexistsr
UPLOAD_DIRr info)r argskwargs file_pathr r r get_log_filename" s zCmfImport.get_log_filenamec O sx | tj }tjtj | s4| j d g }t
tj| D ]&}| tjrL| | | qL|S )u
Список файлов после logrotate, чтобы можно было загрузить историю
r! )
r r r" r# r$ r% r r& r r' listdir
startswithappend)r r( r) r* res file_namer r r get_log_filenames+ s zCmfImport.get_log_filenamesc s$ | j s| jjdd| _ t d S )NZOPEN status_type)statusworkflowget_default_statussuper _calc_statusr __class__r r r8 8 s zCmfImport._calc_statusc C s | j d rd S | jr(| j | j d< d S | jdkr6d S | j d dtj tjjd | j d dtj tj jd | j d dtj
tj
jd | j d dtj tj jd | j d dtj tj jd | j d d tj
tj
jd d S )
NZ
models_fieldsexcelu Поля контактов)Zcaptionr iconu Поля задачu Поля заметокu Поля проектовu" Поля списков задачu Поля групп)
json_settingspluginZcalc_models_settingstyper. models CmfPersonZimport_shop_fieldsr= CmfTaskZ
CmfComment
CmfProjectCmfListZCmfPersonGroupr9 r r r _calc_models_settings= s>
z CmfImport._calc_models_settingsc K sH | ddddg | jd r d S | jj| d| jd< | j| _| d S Nr> attachmentsr@ r? import_settingsr ) load_fieldsr> r? calc_import_settingssaver r) r r r rK X s
z CmfImport.calc_import_settingsc K sH | ddddg | jd r d S | jj| d| jd< | j| _| d S rG )rJ r> r? tmplt_import_settingsrL rM r r r rN b s
z CmfImport.tmplt_import_settings c C s4 d| d d} td| d| t| d S )Nu1 Ошибка при импорте данных. .ue . Проверьте загружаемый файл и повторите попытку снова.zCmfImportError: : )stripr errorZ cmf_alert)msger r r
_log_errorj s zCmfImport._log_errorc C s: | dg | jd D ] }|d | jkr|d S qg S )Nr> rI namer )rJ r> r )r settingsr r r r p s
z$CmfImport.csv_import_settings_fieldsc
O s* dt jd< t t tjj| ddddgd}|j d |j
j d d
d |_ |
t | }|r|j d |j
j d
d|_ |
n0|j jd
kr|j d |j
j dd|_ |
|jjd jtjd}W 5 Q R X W 5 Q R X tjj|tjjj|j dt t j dd| ddd dS )uJ
Запускаем в фоне импорт данных
1importr5 r4 progresszplugin.plugin.*idr u% ***Запускаем импорт***Z IN_PROGRESSimport-started)r3 status_codeu2 ***Импорт успешно завершен***ZCLOSEDr2 u7 ***Импорт завершен с ошибками*** import-errorr_ r z/files/u+ Импорт данных завершен z%d.%m.%Y %H:%Mz
, Журнал логированияT)objZ person_idrW rT Zis_notify_current_personN)!r# environcmfutil disable_acldisable_notifyrA r getr r' r5 r6 r4 rL commit_with_eventprocess_importr3 ZhandlersZ baseFilenamereplacer r& Z CmfNotifyZ place_notifygZcurrent_personr] valuerW r nowr utc)Z import_id_args_kwargsrb r/ Zlog_pathr r r startx s0
,zCmfImport.startN)rowc C s4 d}|dkr|j }|sg }i }t| jD ]Z\}}|d r(|| r(|d d dd |jkr^q(|d d |kr(|| ||d d < q(|s|S || d} tt| d }
|D ]} |j| } t | tj
st | tj r,d}
| D ]8}|jd
d d || gd d || ggd
gd}
|
r qq|
r,t
| | |
qt | tjr rw rW r rm r rn rL )r rr model_filterkeyresult_fieldsisettingZnew_objZnew_coderv ZfieldZrel_objZ rel_modelvalZphonerw r r r _process_obj sp
zCmfImport._process_objc
C s | dddddg | jrz0t | jj| dW 5 Q R W S Q R X W n tk
r } zl| dt }t| t | dt
|j d }| j
| || _| jjd
d | _| W Y d S d
}~X Y nX | jdkr| S d
S )u9
Импортируем файл в бд
r< rI first_rowr? rH r rQ
r` ra FN)rJ r? rd Zenable_import_moderi Exception traceback
format_excr r format_tb
__traceback__r exception error_msgr5 r6 r4 rL r@ _process_import_excel)r errerr_msg
error_textr r r ri s
&
zCmfImport.process_importc 4 C sf ddl m} | jr | jd s dS d}zi }i }i }i }i }i }i } | }
t|
D ](\}} |dkrr| jd rrqTi } i }
d}d}t| jD ],\}}|d r| | sq|d d d d }|
|i }|d d
d }|d }| | }|d kr|}qn|d kr |}qt |d
krP|dkr2|| |d
< q||i }| | ||d
< nZ|dkrt|dkrt|
| | }n6|dkr| | | }n |dkrd| | | i}|||< ||
|< q|
di }|
dd}||d}|s || tj|\}}|||d< || tj| \}}|
di } | drD| d | d< | di } | r| |} | s|dkrtjjddtjjdd d } n tjjdd tjjd d d } | | |< | | d < | | d< || d!< || d"< d#d$d%|jgdd%| d gg} || tj| | \} }| j| | | || d < |rH||g }!|!| |!||< |rt|| d g }"|"| |"|| d < |
d&i d'd}#|#r|# d(|jj }$||$i }%||%d!< |#|%d'< |%d)g }&|&| |&|%d)< |%||$< ||d}'|'sftj jd#d*d%|jgd'd%d+ggd,}(d#d*d%|(jgd'd%d-gg})d-||(d.d/}*|j| tj |*|)d0\}'}|'||d< || j tj |'| d1 qT| D ]\}$}+tj jd#d*d%|+d! jgd'd%d2ggd,}(d#d*d%|(jgd'd%|+d' gdd3d4gg})tjj|)d,},|,stj|+d! |(|+d' tj!jd5d d6},|, |+d) D ]} |,| _"| jd7d8 q"q| D ]>\}}-||}.|.rH|-D ] }/|/j"# |.|/_"|/ qdqH| D ]N\}}0||}.|.r|0D ].}1||1}/|/r|/j"# |.|/_"|/ qqW n~ t$k
r` }2 z^t%|2 d9| d:|2 d;t&'|2j(d< }3| j)*|3 |3| _+| j,j-d=d>| _.| W Y dS d?}2~2X Y nX d7S )@u
Запуск импорта из CSV
:return: Истина в случае успешного импорта
r )CmfPluginImportMixinrI Fr rO rs rt rP rv parent_taskZ child_tasks executors logic_prefixrD priorityr4 r3 z
project.agilerw rC Z
cmf_authorZ cmf_ownerz project.basez
task.base%baserw )Z cmf_modelprefixZactivityztask.agile:softdev%Zsoftdevr5 projectparentZANDZ parent_idry rE rW r| tasksZtree_parent_idZSprints)rz u
ДоскаZkanban)rW r tree_parentZ list_type)r r
obj_fields
filter_obj)leftrightZEpics=z task.epicztask.epic:default)r r rW Z
logic_typeT)Z only_datau
Строка rQ r r r` ra N)/Z%common.models.cmf_plugin_import_mixinr rH r> Z _get_filer} r r~ rg lenZget_logic_prefixZ get_priorityZget_status_type
setdefaultZprocess_any_table_fieldsrA rD rB CmfWorkflowZ
calc_workflowZ CmfActivityr] rC r r. rL rw rl Z CmfFolderrE r ZCmfListCmfTaskitemsZ CmfLogicTyper loadr r r r r r r r r5 r6 r4 )4r r Z
number_rowZall_parent_taskZall_child_tasksZall_epic_tasksZ all_tasksZ
cash_boardZ cash_projectZcash_workflow_taskZ file_readrr executorr Zparent_task_codeZchild_tasks_codeidxr Z
model_nameZ_rowZ fields_namerv rl Zproject_fieldsr r r| Z executor_objZ task_fieldsZ value_statusr5 Z filter_taskZtask_objZtmp_parent_tasksZtmp_child_tasksZ epic_nameZepic_keyZtmp_dataZ tmp_tasksZlist_objZ
parent_folderr r dataZepic_objr r Zsub_taskZ
tasks_codeZ
sub_task_coder r r r r r sL
" z CmfImport._process_import_excelc C s> | |tj}|r|sdS d}| jD ] }|d r$|d d dkr$|}q$|dg tjj|jt|d |j jdgd}|stj|t|d |j | d }|
|j j d
d t
|jjD ]*}|jj| jkrq| |jj| q| j |tjtjjd d
|jgdd
|jggd}|ddg ||_||_|jr:|
|S )u;
Импорт товара в сделку
Nrs rt zCmfProduct.pricescurrencyfile_valprice)
product_idr Z currency_idr )productr r r T)forceZdeal_idry r )r r r Z sales_order)r rA Z
CmfProductr rJ ZCmfPricerg r] r r rL Zpricesr r Z
deal_itemsrl r deleteremoveZ CmfDealItemsru dealr
is_changed)r rr r r Z
price_settingrX r itemr r r _process_product s@
zCmfImport._process_productc C s |s|r|j sd S d }| jD ] }|d r |d d dkr |}q |sFd S g }tjj|d d}|sxtj|d d}| |dd|jg |r|dd|jg n|dd|j jg | |tj |}|
d d
g |j s|r||_ n|j |_ |j s||_ |j r
| |S ) Nrs rt zCmfDeal.pipeliner )rW Z pipeline_idry Z
company_idcompanypipeline)
r r rA Z CmfPipelinerg rL r. r] r ZCmfDealrJ r r ) r rr Zcontactr Zpipeline_settingrX r r r r r r
_process_deal s8
zCmfImport._process_dealc C s< t jj| jdgd}|jjdkr8d|_|jr4| dS dS )Nr4 r\ )import-cancelingimport-canceledr` import-stoppingr TF)rA r rg r] r4 rw r rL )r rb r r r is_cancel s zCmfImport.is_cancelc O s0 t t tjj|d ddgd}|jjdkr|jjdkrf|jj dd|_|
t t d D ]t}t
d
tjj|jddgd}|jjd kr q|d krnd
|_|jj dd|_|
W 5 Q R W 5 Q R dS qnn |jj d d|_|
t zzd}|jd |j t|j}|rd| }|jD ]N}| j|7 _|
t|tjrnqB|jd| | t qB|jj dd|_|jd W nX tk
r } z8|j d| t| t ||_|jj dd|_W 5 d}~X Y nX W 5 |
X W 5 Q R X W 5 Q R X dS )u/
Отменяем импорт
r] r4 r5 r\ )r^ r r^ r ra < r r ; u5 Не удалось остановить импортr` Nu# ***** Отменяем импортd u Удаляем r u, Успешно отменили импортu Ошибка удаления ) rd re rf rA r rg r4 rw r5 r6 rL rh rangeZgeventsleepr] r r r' Zimport_objectsr r r[
isinstancerB r r r r r )r( r) rb r rr totalsteprU r r r cancel sV
&zCmfImport.cancelc O s | ddddg | jj| dS )uk
Предварительный просмотр получившихся объектов
r> rH r@ r? r )rJ r? preview)r r( r) r r r r C s zCmfImport.previewc s t dddddg S )Nr> rH r@ r4 r? )r7 save_preload_fieldsr9 r: r r r J s z CmfImport.save_preload_fieldsc s | t jf |S N)rF r7 rL rM r: r r rL M s zCmfImport.savec C s@ | j r
d S | jr,| jjdkr,tjjdd| _ tjjdd| _ d S )NZ
CmfPluginJirazimport.system.jira:defaultr zimport.system:default)r5 r? r rA r rg r9 r r r _calc_workflowQ s
zCmfImport._calc_workflowc C sN | dg ttj| jj d}|| jjjj }|j
ddd |S )Nzplugin.source_hashZ _raw_dataT)exist_okparents) rJ r r r& Zjoinpathr? Zui_namelower source_hashrl mkdir)r r/ r r r get_download_pathY s
zCmfImport.get_download_pathc C s | j | d S r )r?
download_datar9 r r r r ` s zCmfImport.download_datac C s | j | d S r )r? download_filesr9 r r r r b s zCmfImport.download_files)rO )NN)(__name__
__module__ __qualname__r r Z ui_meta_skipr r r r r+ listr1 r8 rF rK rN staticmethodrV r Z
celery_appZtaskrq r ri boolr r r r r r r rL r r r r r
__classcell__r r r: r r sD
N B!# 5 r ) Zcsvr r Zdecimalr r r r r r pathlibr typingr r
r r ZxlrdZopenpyxlZcmf.data_providers.baser Z cmf.includeZcmfr Z
common.fieldsr Zcmf.modules.logsr r r r r r s"