U
    whpS                    @   s   d dl Z d dlZd dlmZ d dlT d dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ e ZG dd dejjjZdS )    N)
namedtuple)*)SQLAlchemyDataDriver)BeautifulSoupc                   @   s  e Zd ZdZdZddddddd	d
dddddddddddddddgZdZdddgZee	ddd Z
ed!d" Zed#d$ Zed%d& Zed'd( Zedid*d+Zedjd,d-Zed.d/ Zed0d1 Zedkd3d4Zed5d6 Zed7d8 Zeejjd9d:d;Zedld=d>Zeedd?d@dA ZedBdC ZeedddDdEdFdGdH ZedIdJ Z edKdLdMdNdOZ!edmdPdQZ"edndSdTZ#ed2d2dUgd2dRdVd<gfdWdXdYdZZ$ed[d\d]d^d_Z%ed[d`dadbZ&edodcddZ'edpdedfZ(eedgdh Z)d2S )qCmfFullSearchuC   
    Сервис полнотекстового поиска.
    russiannameZtext_rendertextZ
text_drafttags	parent_idtree_parent_id
project_idcmf_created_atcmf_modified_atcodeZcmf_deletedZcmf_archivedZloginZemailZphoneZphone_internalZphone_mobileZphone_2Zphone_assistant
ip_addressZemail_2ZbirthdayTfulltext_searchrun_force_reindexindex_statsr	   c                 C   sR   | rNt | dd} t| dkrNtdt|  d |  d d jdd} | S )	Nhtml.parser
i  z)CmfFullSearch.clean_text: trunc text len z to 1mbi  ignoreerrors)r   get_textlengdebugencodedecoder    r!   ./cmf/models/cmf_full_search.py
clean_text0   s    zCmfFullSearch.clean_textc                 C   s   t |ddjdd S )Nr   r   r   r   )r   r   r   r    )clsZtext_with_htmlr!   r!   r"   
strip_html@   s    zCmfFullSearch.strip_htmlc                    s@   dd t k rgS  fddtdt D }|S )Ni d   c                    s    g | ]}||    qS r!   r!   ).0iZoverlapr	   Ztext_len_maxr!   r"   
<listcomp>T   s     z-CmfFullSearch._strip_size.<locals>.<listcomp>r   )r   range)r$   r	   resr!   r)   r"   _strip_sizeD   s    "zCmfFullSearch._strip_sizec                 C   s   d dd td|D S )N c                 S   s   g | ]}t |d k r|qS )2   )r   r'   wr!   r!   r"   r*   Y   s      z+CmfFullSearch._strip_50.<locals>.<listcomp>z\sjoinresplitr$   r	   r!   r!   r"   	_strip_50W   s    zCmfFullSearch._strip_50c                 C   s   d dd td|D S )Nr.   c                 S   s   g | ]}|qS r!   r!   r0   r!   r!   r"   r*   `   s     z1CmfFullSearch._strip_not_word.<locals>.<listcomp>z[ \s<>|&\/%@!`+=;.]r2   r6   r!   r!   r"   _strip_not_word[   s    zCmfFullSearch._strip_not_wordFc                 C   s^  t   }td|  |s$tddd l}| jj}|| }|j}|	 }|
|jjg|jj|k|jjdk }	||	 }
|
r|
d }| |jj|kjd|j d}|| n@td|  |  }| j||d|j dd}|| |r0ddlm} |tjjd	|id
d t   | dkrZtdt   |   d S )Nzcmf_full_search: mark_dirty empty obj_idr   Tis_dirtydirty_atz#cmf_full_search: mark_dirty insert )idobj_idr;   r<   part_no)schedule_deferred_jobr>      )kwargsZ	countdowng?zPROF mark_dirty() got )timer   r   
ValueError
sqlalchemydpdata_driverdp_model_cls	__table__Sessionselectcr=   wherer>   r?   with_for_updateexecutefirstupdatevaluesfuncnowgen_idinsertcmf.includer@   modelsZ	CmfPersonZcelery_full_search_index)r$   r>   
fast_indexZprof_stsaddsa_modeltablesget_stmtget_resid_update_stmtinsert_stmtr@   r!   r!   r"   mark_dirty_deferb   sT    





zCmfFullSearch.mark_dirty_deferc                 C   s$   |r| j ||dS d|itj|< d S )NrY   rY   )rd   r   deferred_fullsearch_dirty_list)r$   r>   rY   forcer!   r!   r"   
mark_dirty   s    zCmfFullSearch.mark_dirtyc                 C   sf   t js
d S t j}t|dkr.tdt|  t| D ] }|| }| j||dd q:i t _d S )Nr/   ua   DEV: CmfFullSearch помечено грязными слишком много объектов: rY   re   )r   rf   r   	cmf_alertsortedkeysrd   get)r$   Z
dirty_listr>   rB   r!   r!   r"   apply_deferred_dirty   s    z"CmfFullSearch.apply_deferred_dirtyc                 C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kjd|j d}
||
 | j  d S )Nr9   r   Fr:   )rD   rE   rF   rG   rH   rI   rJ   rK   rL   r=   rM   r>   r?   rN   rO   rP   rQ   rR   rS   rT   commit)r$   r>   rZ   r[   r\   r]   r^   r_   r`   ra   rb   r!   r!   r"   
mark_clean   s2    




zCmfFullSearch.mark_cleanNc           (   %   C   s  |}|  |}| |}|r@| |}| |}|  |d }n
d g}d }|rh|}|  |} | | } nd }d } |r|}|  |}!| |!}!nd }d }!|r| |}"|"d d }"|  |"}#nd }"d }#|r| |d d }|  |}$nd }$|r|}|}%nd }d }%| jd|||||||||	||||||
||||d ||||| ||!|"|#||$|||%d" t|dkrt|dd  dD ]b\}&}'|  |'}| j|&|||||||||	||||||
||||'||d d d d d d d ||$|d d d" qr| j|t|d | j  d S )Nr   i )"r?   r>   	obj_modelobj_parent_idobj_tree_parent_idobj_project_idobj_created_atobj_modified_atobj_codeobj_deletedobj_archivedobj_owner_nameobj_author_nameobj_modified_by_nameobj_responsible_namesobj_hrefobj_logic_type_codeobj_activity_codeobj_status_typeobj_texttext_for_vecobj_namename_for_vecobj_tagstags_for_vecobj_result_textresult_text_for_vecobj_commentscomments_for_vecobj_addon_fieldscf_text_for_vecobj_user_ratingobj_key_phraseskey_phrases_for_vec   )delete_from_partno)	r8   r7   r-   _text_search_sql_insertr   	enumerate_text_search_sql_delete_partnorF   rn   )(r$   
model_namer>   rv   r   r	   r   Zcomments_textrx   rw   r}   rq   r~   r   r   ry   rz   r{   r|   rr   rs   rt   ru   r   r   r   r   r   r   r   Zobj_text_listr   r   r   Zobj_comments_textZcomments_text_for_vecr   r   r?   Zobj_text_partr!   r!   r"   index_object   s    







                 

                 
zCmfFullSearch.index_objectc           	      C   sh   |st ddd l}| jj}|| }|j}| }|||j	j
|k|j	j|k}|| d S )Nr9   r   )rD   rE   rF   rG   rH   rI   rJ   deleterM   rL   r>   r?   rO   )	r$   r>   r   rZ   r[   r\   r]   r^   Zdel_stmtr!   r!   r"   r   @  s    



z,CmfFullSearch._text_search_sql_delete_partnoc#           -   (   C   sf  t d|  |stddd l}#| jj}$|$| }%|%j}&|$ }'|#	|&j
jg|&j
j|k|&j
j|k }(t|'|(})|)rt|)dkrtd| d| ddd	 |)d d }*|& |&j
j|*kj|d
|||||||||||||#jj| j||#jj| j||#jj| j||#jj| j||#jj| j||#jj| j||	|
|||||||||| |!|#jj| j|"d"}+|'|+ nt d|  |  }*|& j|*d
|||||||||||||||#jj| j||#jj| j||#jj| j||#jj| j||#jj| j||#jj| j||	|
|||||||||| |!|#jj| j|"d$},|'|, |*S )Nz)cmf_full_search: _text_search_sql_insert r9   r   r   u   Для объекта z	(part_no=u   ) задублировались записи в поиске. Обратитесь в техническую поддержку!TabortF)"r>   r;   rp   r   r   r   r   r   r   rq   rr   rs   rt   ru   name_tsvectortext_tsvectortags_tsvectorresult_text_tsvectorcomments_tsvectoraddon_fields_tsvectorrv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   key_phrases_tsvectorz0cmf_full_search: _text_search_sql_insert insert )$r=   r;   r?   r>   rp   r   r   r   r   r   r   rq   rr   rs   rt   ru   r   r   r   r   r   r   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   r   )r   r   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r=   rM   r>   r?   rN   listrO   r   ri   rQ   rR   sqlrS   Zto_tsvector_fts_configrU   rV   )-r$   r?   r>   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   r!   r!   r"   r   S  s    



()
z%CmfFullSearch._text_search_sql_insert)objc                 C   s>   |j s
d S | jD ](}|dkrq||jkr|| jr dS qd S )Nr   T)full_searchrequired_fieldsfieldsZ
is_changed)r$   r   
field_namer!   r!   r"   is_obj_need_reindex  s    
z!CmfFullSearch.is_obj_need_reindexr&   c              
   C   s   t jj D ]}|jsq|j}|r,||kr,qd}t|}|j}|rJdg}t		 }	|j
|||| gdgddd}
|
svq|
D ]&}|rtjj|jjdd qz|  qz|t|
7 }| j  td| d| d	t		 |	 d
d qJqd S )Nr   r=   r   T)r   sliceorder_byinclude_archivedinclude_deleted)rg   zCmfFullSearch.reindex_models: :, z0.3z sec)cmfrX   	CmfEntityiter_subclassesr   
class_namecmfutilget_model_by_namefull_search_fieldsrC   r   r   rh   r=   valuefull_search_indexr   rF   rn   r   r   )r$   Zmodels_listZcommit_everyZlazyZ	model_clsr   offsetmodelZ
field_listtsobj_listr   r!   r!   r"   reindex_models  s8    



zCmfFullSearch.reindex_models)	only_oncec                  C   sf   t d tj  tj} | jj}|| }|j	}|
 }| |jjdkjdd}|| d S )NzRun run_force_reindexr   Tr;   )r   r   rX   CmfAccessListcheck_admin_moder   rF   rG   rH   rI   rJ   rQ   rM   rL   r?   rR   rO   )r$   r[   r\   r]   r^   rb   r!   r!   r"   r     s    


zCmfFullSearch.run_force_reindexc                 C   s(   t j  |  }| jdd}||dS )NTr   )totaldirty_count)rX   r   r   count)r$   r   r   r!   r!   r"   r     s    
zCmfFullSearch.index_statsz	@minutely   )r   Z
system_jobZschedulepriorityc            	      C   s  t d tj r"t d d S td t } d}tjj	dgdddgddd	gd
ddt
j
 t
jdd gddd gggddgd}|sqi }|D ]0}t|j}||krg ||< || |j q| D ]\}}t d| dt|  t| |D ]b}|j|jdd|gd	d}|d7 }|sFt d|  tj| qt d|j  |  qqt |  dkrt d| d qt d| d td q8t dt |    d S )NzStart cron_index_dirtyuJ   Не индексируем поскольку запущен импорт   r   r>   r?   =r;   TORr<   <   )Zminutes   )r   filterr   zcron_index_dirty process r.   r=   )r   r   r   r   u.   Объект не найдет в бд obj_id=zobj_id=   ua   cron_index_dirty превышен лимит времени 20 секунд обработано u    объектовu&   cron_index_dirty обработано u    объектов, sleep 1zEnd cron_index_dirty at )r   r   r   Zenable_import_modeZimport_is_runningrC   sleeprX   r   slistdatetimerT   Z	timedeltaZget_model_by_idr>   appenditemsr   printrl   r   ro   r=   r   )	stnZobj_id_listZobj_ids_by_modelr>   r   Zid_listra   r   r!   r!   r"   cron_index_dirty  sV    





zCmfFullSearch.cron_index_dirtyc                 C   sN   |S ]:}|sq|d dkr4t|dks|d dkr4q|| qd|}|S )u   
        Подчистка оригинального квери, который ввел пользователь:
        - удаление стоп-слов
        r.   r   -r   )r5   r   r   r3   )r$   search_queryZclean_search_query_listr1   Zclean_search_queryr!   r!   r"   _clean_search_queryV  s     
z!CmfFullSearch._clean_search_querystrztuple[str, set[str]])r   returnc                 C   s   t  }| d}t|dkr$| |fS d} |dd D ]Z}|dkrBq4td|d}|d dkrj||d  t|dkr4|  |d  |d  } q4|  } | |fS )	zExtrats tags from the given search_query and returns its reminder and a set of extracted tags

        Args:
            search_query (str)

        Returns:
            tuple[str, list[str]]: search_query reminder and a set of extractd tags
        #r    Nz(\W)r      r   )setr5   r   r4   addstrip)r   r
   Zsharp_splittedtokenZ
sub_tokensr!   r!   r"   _extract_tagsj  s    

zCmfFullSearch._extract_tagsc
           R      K   s  t |dkrtddd |d kr$d}d|
kr4d|
d< d|
krDd|
d< |sPd	d
g}tjjdd}|dkrttd|  dt_|}t }|
	d}|r|
| | |\}}t||}||
d< |}| }|}t|}|dkrd}t|}d	|d	< |dr|d d }g }g }g }g }g }g }g }g }g }g }g }g }g }g } g }!g }"g }#g }$g }%g }&g }'g }(g })g }*g }+g },g }-g }.|dkrdd tjj D }/n8t|tjkrd}d}dd tjj D }/n|g}/|
	d}0| |0|
d< |
	dd}1| j|dd|	d}2| j|d|	d}3| j|d|	d}4| j|dd|	d}5|4t_tdd|3}6|6t_tdd|3}7|dks||d krL|dkr|1sL| j|/fttjj d!d"gdd#|d$|
}&| j|/fd!d"gdd#|d%|
}'nt| j!d&|/|3fd!d"g|d'|2|d(|
}|1sL| j!d|/|4f|d!d"gd)|d*|
}| j!d+|/|3f|d!d"gd)|d*|
}|dks`|d!kr|dkr|1s| jd!gfttjj |d,|d-|
}(| jd!gf|d,|d.|
})nN|dkr| j!d&d!g|3f|d'|2|d/|
}| j!d&d!g|4f|d0|2d1|
}|1r|d&kr| j!|d!g|3f|d2d3|
}| j!|d!g|6fd4|d5|
}| j!|d!g|7fd6d7i|
}| j!|d!g|4f|d)d3|
} | j!d+d!g|3f|d)d3|
}"t"t |t |t |t |t |t | t |"fd8k r| j!|d!g|5f|d)d3|
d	d9 }$|dks|d"kr
|dkrn|1s
| jd"gfttjj |d,|d-|
}*| jd"gf|d,|d.|
}+n|dkr| j!d&d"g|3f|d'|2|d/|
}| j!d&d"g|4f|d0|2d1|
}| j!d:d"g|3f|d'|2|d/|
},| j!d:d"g|4f|d0|2d1|
}-|1r|d&kr
| j!|d"g|3f|d2d3|
}| j!|d"g|6fd4|d5|
}| j!|d"g|7fd6d7i|
}| j!|d"g|4f|d)d3|
}!| j!d+d"g|3f|d)d3|
}#t"t |t |t |t |t |t |!t |#t |,t |-f	d8k r
| j!|d"g|5f|d)d3|
d	d9 }%g }8d;|# krt$d<d=}9tjj%d>d?d@dAdBgdCd?dD|# gd>dD|# ggdED ]F}:|:j&rt|:j&d d nd};|8'|9|:j(|:j)|:j*|:j+|;dFd
d	dG qZg }<t }=dHdI }>d}?t,dJ d	}@||||&|'|||||,|-||||||| |!|"|#|$|%|(|*|)|+|8g}At-|A
r |?r^|>|8|<|= |>||<|= |>||<|= |>|,|<|= |>||<|= |>||<|= d}?|>||<|= |>||<|= |>||<|= |>||<|= |>||<|= |>||<|= |>|,|<|= |>|,|<|= |>||<|= |>||<|= |>||<|= |>|8|<|= |>||<|= |>||<|= |>||<|= |>||<|= |>||<|= |>||<|= |>| |<|= |>|!|<|= |>|"|<|= |>|-|<|= |>|#|<|= |>|$|<|= |>|%|<|= |>||<|= |>|(|<|= |>|)|<|= |>|*|<|= |>|+|<|= |>|&|<|= |>|'|<|= |@dK7 }@|@dLkrd;.dMd |AD }Bt,dN|B  
q qqt,dO |<|d	 |dK  }<|sli }C|r4|
	dd}D|
	dd}Ei }F|<D ]}G|F/|Gj)g '|Gj( 
qn|F0 D ]}t1t|Gj) j2
r|dPg }HdPdQ|F| gdRdSdgg}In|}HdTdQ|F| g}It|j|H|I|D|EdU}J|JD ]0}Kt1t|Gj) j2r |K|C|Kj3j4< n
|K|C|Kj4< 
q
qntdV t,dW g }L|<D ]}G|C	|Gj(}K|KrN|Kj4j5|Gj6|Kj7j5|Gj8 dX|Gj9 d;|Gj:dYd;|Gj;dZd[tj 	|Gj9|Gj:|Gj;d\}Mzf|s|Kj<ddd] n||Kst=|D ].}N|N>d^d	 }Nt?|C|Gj( |Nd }O|O|M|N< q|L'|M W n t=k
r.   Y nX qNt,d_ |sht@tjjA|tB|Ld d8 d`da |LS i }Pg }Q|<D ]b}G|Q'|Gj( |Gj(|Gj6|Gj*|Gj8 dX|Gj9 d;|Gj:dYd;|Gj;dZd[tj 	|Gj9|Gj:|Gj;d\|P|Gj(< qx|Pt_Ct,d_ |QS )bNi   uY   Превышена максимально допустимая длина запроса!Tr   r   archivedFdeletedr   r&   r      u   Идет процесс индексации, могут быть доступны не все результаты поиска. Осталось объектов: tag_nameANYr	   ZModelc                 S   s   g | ]}|j r|jqS r!   r   r   r'   mr!   r!   r"   r*     s      z1CmfFullSearch.fulltext_search.<locals>.<listcomp>commentsc                 S   s   g | ]}|j r|jqS r!   r   r   r!   r!   r"   r*     s      r   tree_parent_filtertitles_only)synonyms
stop_words	query_raw)r   r   )r   
first_wordr   z([&] )([^!])z<1> \2z<2> \2)CmfTaskCmfDocumentr   r   ZFS)related_usermodel_name_not_inr   labelr   )r   r   r   r   r   N)r   r   r   text_stop_words
like_queryS)r   r   r   r   addon_fieldsZFA)r   r   r   r   )r   r   r   )r   r   r   r   ZNs)r   r   r   A)r   r   ZH1)r   r   r   ZH2r   rA   key_phrasesr.   SearchResultTuplez<obj_id,obj_model,obj_code,title,headline,label,rank,age_daysr>   rv   r   rp   r   r   r   )r   r   ZCODE)r>   rp   rv   titleheadliner   rankage_daysc                 S   s<   | r8|  d}|d |kr || ||d  dS q dS )Nr   TF)popr   r   )r   resultskip_idsrr!   r!   r"   add_if_exists  s    

z4CmfFullSearch.fulltext_search.<locals>.add_if_existszfulltext_search Start mixingr   i  c                 S   s   g | ]}t t|qS r!   )r   r   )r'   fr!   r!   r"   r*   8  s     uX   Баг в поиске, много данных, либо не идет вычитка: zfulltext_search End mixingcmf_ver_headINZcmf_ver_curz==r=   )r   r   r   r   uY   DEV: FATAL. Укажите в запросе поиска список полей fields=z"fulltext_search Start check accessz ||| z.6fz.0fz words=)r=   r   r   r  r   r  r  )ZTEXKOM_skip_failread_auditZTEXKOM_ppp_project_simplecheck.zfulltext_search END)r   obj_dict)rB   )Dr   ri   rX   r   r   Zcmf_noter   FSTr   rl   r   r   r   unionlowerr   Zninjaendswithr   r   r   r   Z
CmfComment_get_all_branchesprepare_search_queryFSTQr4   subZFSTQHfilter_oncer   current_userr   search_oncesumr   r   r   r   r   r>   rp   rv   r   r   anyr3   
setdefaultrk   varsZcmf_verr  r=   r   r  r   r  r   r  r  Z_acl_check_readZCmfPermissionErrorr5   getattrr@   _do_calc_statisticsZ
dumps_dictfulltext_search_headlines)Rr$   r   r   r   Zonly_idsr   r   Zno_analitycsZcheck_accessr   rB   r   Zorig_field_namer
   r   Zextracted_tagsZorig_search_queryZlike_search_queryZfullsearch_sliceZother_res_name_foundZother_res_text_foundZother_res_cf_foundZtask_res_name_foundZdocument_res_name_foundZtask_res_name_syn_foundZdocument_res_name_syn_foundZtask_res_all_foundZdocument_res_all_foundZtask_res_strong_position_h1Zdocument_res_strong_position_h1Ztask_res_strong_position_h2Zdocument_res_strong_position_h2Ztask_res_syn_foundZdocument_res_syn_foundZtask_res_cf_foundZdocument_res_cf_foundZtask_res_first_wordZdocument_res_first_wordZother_filter_rel_foundZother_filter_foundZtask_filter_rel_foundZtask_filter_foundZdocument_filter_rel_foundZdocument_filter_foundZdocument_res_key_phrases_foundZ"document_res_key_phrases_syn_foundZaddon_res_foundZother_model_namesr   r   r   Ztsquery_without_synZtsquery_with_synZtsquery_first_wordZtsquery_strong_position_h1Ztsquery_strong_position_h2Zres_code_foundr  Zany_res_code_foundr  r  r  r
  Zdo_topZ
iter_countZsearch_foundsZ
found_lensZobjectsZis_archivedZ
is_deletedZids_by_modelr	  Z_filedsZ_filterr   r   r,   r  Zfieldattrr!  Z	result_idr!   r!   r"   r     s   






		  




     	
 	



       	 




            





,



,
zCmfFullSearch.fulltext_searchr   c           '      K   s  |d krdg}|sddg}|
d kr>|dkr>ddht |@ r>d}
|dd }|d	}|d
d }|dd }|dd }|d}|d}|dd }|d}|d}|dd }|d|}|p|}|d}|rdd|nd}|dpg }| |\}}|dkrd }|	dkr&d }	d }|	rTtd|	sLtd|	rT|	 }|dkrptd| dd d| }d| }|d krd!}| d"}d} |
rd#|
 d"} |d }!|d$ |d  }"d}#|rt	|}d%| d&}#t
jjj d' d}$|dkrd(}$d)jf |||| |#||d*}%t
jjj |%||!|"t|t||||	|$|||||||||||||d+|}&t|&S ),Nnull000r   r&   r	   r   r   r   r   modified_by_nameresponsible_namelogic_type_codestatus_typemodified_at_beforemodified_at_afterr   r   user_rating
owner_nameauthor_namer    AND obj_tree_parent_id IN ('{}')','r   r   z^[a-zA-Z]+-[0-9]+$z^[0-9]+$)r	   r   r
   r   r   r   6   Недопустимое значение field_name: Tr   obj_r   r   Z	_tsvectorz || r   $obj_modified_at > now() - interval '
 days' ANDz"set gin_fuzzy_search_limit=100000;@B u]  
            SELECT
                subs.*,
                ts_headline(
                    'russian',
                    substring(subs.{search_field} FROM 0 FOR 100000),
                    :tsquery,
                    'MaxFragments=1,MaxWords=25,MinWords=24'
                ) AS headline
            FROM (
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        ts_rank_cd({tsvector_field}, :tsquery, 2|8)
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        {tsvector_field} {addon_tsvector} @@ :tsquery
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                        and ( :user_rating is null or obj_user_rating >= :user_rating)
                    ORDER BY rank desc
                    LIMIT :slice_for OFFSET :slice_from
                )
            UNION
                /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        100
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                        ( {search_field} ILIKE '%' || :like_query || '%'
                            or (:obj_code is not null and obj_code ILIKE '%' || :obj_code)
                            or obj_id = :like_query
                        )
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                        and ( :user_rating is null or obj_user_rating >= :user_rating)

                        /*and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))*/
                    /*ORDER BY obj_id desc -- не будем делать, т.к. тормозит*/
                    LIMIT :slice_for/10
                    OFFSET :slice_from/10
                )
            ) as subs
            ORDER BY subs.rank desc
            LIMIT :slice_for
            OFFSET :slice_from;
        )r   search_fieldtsvector_fieldaddon_tsvectorage_days_subqueryheadline_fieldtags_filter)tsquery
slice_from	slice_formodel_name_inr   r   r   r   
max_partnor   r+  r,  r$  r%  r&  r'  r(  r)  r   r   rv   r*  )r   rl   formatr3   _build_tags_filterr4   matchupperri   intrX   r   rF   rG   rJ   rO   tupler   )'r$   r   r=  Ztsquery_strr   r  r   r   r   r   Zaddon_field_namerB   r   r$  r%  r&  r'  r(  r)  r   r   r*  r+  r,  r   
tags_namesr9  tags_paramsrv   r4  r8  r5  r6  r;  r<  r7  r>  r   
found_objsr!   r!   r"   r    s     







"





_hzCmfFullSearch.search_oncer#  r   z
str | None)r   c           "      K   s  | d}	| d}
| d}| d}| d}| d}| d}| d}| d	}| d
}| dd }| d|}|p|}| d}|rdd|nd}| dpg }| |\}}d}|r| }}d}|dkrtd| dd d| }|d }|d |d  }d}|r0t|}d| d}|dkr>dnd}d| d | d!| d"| d#| d$| d$| d%}tjjj	
 |||t|t||||	|||
||||||||d&|} t| }!|!S )'Nr   r$  r%  r&  r'  r(  r)  r   r   r*  r+  r,  r   r-  r.  r   r   z
            AND ( :responsible_name IS NULL OR obj_responsible_names ILIKE '%' || :responsible_name || '%' )
            AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
        z
            AND (
                :responsible_name IS NULL
                OR obj_responsible_names ILIKE '%' || :responsible_name || '%'
                OR obj_owner_name = :owner_name
            )
        )r	   r   r
   r/  Tr   r0  r   r   r1  r2  r	   r3  a  
            SELECT
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                z as z,
                substring(z` FROM 0 FOR 240) as headline
            FROM cmf_full_search
            WHERE
                a  
                obj_model IN :model_name_in AND obj_model NOT IN :model_name_not_in
                AND part_no <= :max_partno
                AND ( :parent_id IS NULL OR obj_parent_id = :parent_id )
                AND ( :author_name IS NULL OR obj_author_name = :author_name )
                AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                z
                a  
                AND ( :logic_type_code IS NULL OR obj_logic_type_code = :logic_type_code )
                AND ( :status_type IS NULL OR obj_status_type = :status_type )
                AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                AND ( :archived IS NULL OR obj_archived = :archived )
                AND ( :deleted IS NULL OR obj_deleted = :deleted )
                and ( :user_rating is null or obj_user_rating >= :user_rating)
            ORDER BY rank desc
            LIMIT :slice_for OFFSET :slice_from;
        )r;  r<  r=  r   r   r>  r   r+  r,  r$  r%  r&  r'  r(  r)  r   r   r*  )rl   r?  r3   r@  ri   rC  rX   r   rF   rG   rJ   rO   rD  r   )"r$   r=  r   r   r   r  r   r   rB   r   r$  r%  r&  r'  r(  r)  r   r   r*  r+  r,  r   rE  r9  rF  Zowner_and_responsible_filterr4  r;  r<  r7  r>  r   rG  Zall_objsr!   r!   r"   r    s    











			!zCmfFullSearch.filter_oncez	list[str]ztuple[str, dict[str, str]])rE  r   c                 C   sH   i }d}t | D ].\}}d|d  }|||< | d| d}q||fS )Nr   tagr   z AND obj_tags ILIKE '%' || :z || '%')r   )rE  rF  r9  r   rH  Z	tag_paramr!   r!   r"   r@    s    z CmfFullSearch._build_tags_filter)r   c                 C   s0   | sg S t jjj dd| i}dd |D S )Na  
                WITH tree_parents AS (
                    WITH RECURSIVE r AS (
                        SELECT obj_id, obj_code, obj_tree_parent_id
                        FROM cmf_full_search
                        WHERE obj_tree_parent_id = :tree_parent_id

                        UNION

                        SELECT cfs.obj_id, cfs.obj_code, cfs.obj_tree_parent_id
                        FROM cmf_full_search AS cfs
                        JOIN r ON cfs.obj_tree_parent_id = r.obj_id
                    )
                    SELECT obj_id FROM r
                    WHERE r.obj_id IN (SELECT obj_tree_parent_id FROM r)

                    UNION

                    SELECT :tree_parent_id
                )
                SELECT * FROM tree_parents;
            r   c                 S   s   g | ]}|d  qS )r   r!   )r'   r	  r!   r!   r"   r*   /  s     z3CmfFullSearch._get_all_branches.<locals>.<listcomp>)rX   r   rF   rG   rJ   rO   )r   Zrecordsr!   r!   r"   r    s    zCmfFullSearch._get_all_branchesc              
   C   sv  dt _|dd}tdd|}td|}d}d}	d}
|D ]}t|dkrPq<t|dkr|dkrfq<|d	kr~|
d| 7 }
q<|dkrq<|d
kr|
d7 }
q<|dkr|
d7 }
q<t|dkrq<|d dkrt|dkrq<|
d|dd   7 }
|	d|dd   7 }	q<|rq<|dd}|d7 }|dkr.d}|
rB|
d dkrJ|
d7 }
t|dkr| j||d}t|dkr|
|d  7 }
n|
dd| d 7 }
|r|dkr q|dkr< qq<q<|rt|	dkr|	d dkr|	dd  }	|	S |
dddddd d!d 	 }
|
r<|
d dkr<|
dd  }
|
r\|
d dkr\|
dd  }
|
r||
d dkr||
dd  }
|
r|
dd  dkr|
d d }
z.t
jjj d"d#|
i}t|d d }
W n tjjk
r^ } zlt
jjj  |s$td$|  td%|
 d| dt j  t
jjj d&d#|i}t|d d }
W 5 d }~X Y nX t|d'|
 |
t _|
S )(Nr   zwww.u   [^-A-Za-zА-Яа-я0-9()|&!' ]r.   z(,| |&|\||\(|\))r   r   )r   !z()&|)oru   или|z |)u   иand&z &r   z& !rI  r      F)rK  rM  z& )r   z( z | z )
   )rM  rK  z OR z or z AND z & z and z!select to_tsquery('russian', :q);qu;   Ошибочный синтаксиса в запросе: u@   DEV: Ошибочный синтаксиса в запросе: z+select websearch_to_tsquery('russian', :q);z->)r   r  replacer4   r  r5   r   prepare_wordr3   r   rX   
CmfSynonymrF   rG   rJ   rO   r   rE   excZProgrammingErrorZrollbackri   r   r  )r$   r   r   r   r   r   Zsearch_query_allowed_symbtokensZ
word_countZstopsrQ  tZ	sug_wordsr:  er!   r!   r"   r  1  s    

("z"CmfFullSearch.prepare_search_queryc                 C   s  t  jd| d7  _|d tjkr0d}tj}n
d}tj}g }t|s||}g }t	|}|rt  jd| d7  _|
| t  jd7  _d}	|D ]}
|	d	kr qHt|
d	krq|
d |d kr|
d
 |d
 krqd|
kr|
dd}
t  jd|
 d7  _|
|
 |	d
7 }	qt  jd|
 d7  _|
|
 |	d
7 }	qg }tjjj dd|i}d}	|D ]\}}|	d
kr qt|d	krqn|d |d ks|d
 |d
 krn|dd}t  jd| d7  _|
| |	d
7 }	qnt|t|B |hB }n|h}t }|D ]D}
t|
d d	 D ]*}||j t  jd|j d7  _q2q||B }t }|rtjjddt||hB gddgdgdd
gd}|D ]\}|jr|jjdd d D ]6}| dd}t  jd| d7  _|| qʐq||B t|B }t|S )Nz|w:z: r   enruzaddNinjaRevers r   zspellError, r   r   r.   z<->z	addSpell z
            SELECT
                name, similarity(:word, name) as sim
            FROM cmf_synonym
            WHERE
                :word % name
            ORDER BY "sim" desc
            LIMIT 5;
             wordzaddSpellTrgm z
normalize r   r  r	   Zorderno)r   r   r   r   ,r   zsynAdd )r   r  stringascii_lettersr   Zdictionary_enZdictionary_ruZdictionary_checkZsuggestZninja_reversr   r   rR  rX   rT  rF   rG   rJ   rO   r   morphparser   Znormal_formr   r	   r   r5   r   )r$   r[  r   langZ
dictionaryZfiltered_suggestions3ZsuggestionsZfiltered_suggestionsZnwr(   r1   Zfiltered_suggestions2Zsuggestions2_listZsugg_Zall_suggestionsZnormalized_wordsZsynonym_wordsZsynonym_listZsynonymr^   r!   r!   r"   rS    s    


 



	 
$
 zCmfFullSearch.prepare_wordc              	   C   s   t | dk rd S g }|d d D ]b}|d dsB|d dsBq i }|d |d< |dd rp|d d |d< nd |d< || q t }||_d	|_| |_t	j
j|_d
|_t  |  W 5 Q R X d S )Nr   r   r=   zCmfDocument:zCmfTask:r>   parentr   searchF)r   
startswithrl   r   rX   ZCmfSearchStatr   actionr   r   r  r=   Z	person_idZ
aggregatedr   Zdisable_aclZsave)r   r  Zst_dictZrecZst_recstatr!   r!   r"   r     s(    

z!CmfFullSearch._do_calc_statistics)F)FF)NNNFFNNNNNNNNNNNNNNNNN)Nr&   F)FNNFNF)NNr   NNNN)TFFF)T)*__name__
__module____qualname____doc__r   r   Z	api_allowZapi_methodsstaticmethodr   r#   classmethodr%   r-   r7   r8   rd   rh   rm   ro   r   r   r   r   rX   Z	BaseModelr   r   Zcmf_deferred_jobr   r   r   r   r   r   r  r  r@  r  r  rS  r   r!   r!   r!   r"   r      s                    



1

                      z

 !
6
!               5             Lw"fnr   )r   rC   collectionsr   rW   Zcmf.data_providers.sqlalchemyr   Zcmf.fields.cmf_full_searchr   Zenchantr]  Z	pymorphy3r4   rE   Zbs4r   ZMorphAnalyzerr_  r   Zcmf_full_searchr   r!   r!   r!   r"   <module>   s   
