U
    Gh                    @   s  d dl Z d dlZd dlmZ d dlmZmZmZmZ d dl	m
Z
mZmZ d dl	Z	d dlZd dlZd dlZd dlT d dlmZ d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ e Zdd	d
dddddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~ddddddddddddddddddddddddddddddddgZdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd ddddddddd	d
dddddddddddddddddddgZe ee Z!eddd d!d"d#d$d%d&d'g	Z"e
G d(d) d)Z#e
G d*d+ d+Z$e
G d,d- d-Z%G d.d/ d/ejj&j'Z'dS (0      N)
namedtuple)ListSetDictOptional)	dataclassfieldfields)*)SQLAlchemyDataDriver)BeautifulSoup   иu   вu   воu   неu   чтоu   онu   наu   яu   сu   соu   какu   аu   тоu   всеu   онаu   такu   егоu   ноu   даu   тыu   кu   уu   жеu   выu   заu   быu   поu   толькоu   ееu   мнеu   былоu   вотu   отu   меняu   ещеu   нетu   оu   изu   емуu   теперьu
   когдаu   дажеu   нуu
   вдругu   лиu   еслиu   уже   илиu   ниu   бытьu   былu   негоu   доu   васu   нибудьu
   опятьu   ужu   вамu   ведьu   тамu
   потомu   себяu   ничегоu   ейu
   можетu   ониu   тутu   гдеu   естьu   надоu   нейu   дляu   мыu   тебяu   ихu   чемu   былаu   самu   чтобu   безu
   будтоu   чегоu   разu   тожеu   себеu   подu
   будетu   жu
   тогдаu   ктоu   этотu   тогоu   потомуu
   этогоu
   какойu   совсемu   нимu
   здесьu   этомu   одинu
   почтиu   мойu   темu
   чтобыu   нееu   сейчасu   былиu   кудаu
   зачемu   всехu   никогдаu
   можноu   приu   наконецu   дваu   обu   другойu   хотьu
   послеu   надu   большеu   тотu
   черезu   этиu   насu   проu
   всегоu   нихu
   какаяu
   многоu
   развеu   триu   этуu   мояu   впрочемu   хорошоu   своюu   этойu
   передu   иногдаu
   лучшеu   чутьu   томu   нельзяu
   такойu   имu
   болееu   всегдаu   конечноu   всюu
   междуimeZmyZmyselfZweZourZoursZ	ourselvesZyouZyourZyoursZyourselfZ
yourselvesheZhimZhisZhimselfZsheZherZhersZherselfitZitsZitselfZtheyZthemZtheirZtheirsZ
themselvesZwhatwhichZwhoZwhomthisthatZtheseZthoseamisZareZwasZwerebeZbeenZbeingZhaveZhasZhadZhavingZdoZdoesZdidZdoingaZanZtheandZbutiforZbecauseasZuntilwhileZofZatZbyforwithZaboutZagainstZbetweenZintoZthroughZduringbeforeZafterZaboveZbelowtofromZupZdowninoutZonZoffZoverZunderZagainZfurtherZthenoncehereZthereZwhenwherewhyZhowallanyZbothZeachZfewZmoreZmostotherZsomeZsuchnoZnornotZonlyZownZsameZsoZthanZtooZverystZcanZwillZjustZdonZshouldnowSearchResultobj_id	obj_modelobj_codetitleheadlinebreadcrumbslabelrankage_daysc                   @   s   e Zd ZU dZeedZee e	d< eedZ
ee e	d< eedZee e	d< eedZee e	d< eedZee e	d< eedZee e	d< eedZee e	d	< eedZee e	d
< eee  dddZdS )FullSearchResultsuP   
    Контейнер для всех результатов поиска
    default_factory	doc_nosyndoc_syndoc_first_word
task_nosyntask_syntask_first_word	other_syn
attach_synreturnc                    s    fddt  D S )Q   
        Возвращает все списки результатов
        c                    s   g | ]}t  |jqS  getattrname.0r   selfrJ   ./cmf/models/cmf_full_search.py
<listcomp>K   s     z5FullSearchResults.get_all_results.<locals>.<listcomp>r	   rP   rJ   rP   rR   get_all_resultsG   s    z!FullSearchResults.get_all_resultsN)__name__
__module____qualname____doc__r   listr?   r   r2   __annotations__r@   rA   rB   rC   rD   rE   rF   rU   rJ   rJ   rJ   rR   r<   )   s   
r<   c                   @   s   e Zd ZU eedZee ed< eedZ	ee ed< eedZ
ee ed< eedZee ed< eedZee ed< eedZee ed< eee  dd	d
ZdS )FullSearchEmptyQueryResultsr=   doc_empty_query_related_userdoc_empty_querytask_empty_query_related_usertask_empty_queryother_empty_query_related_userother_empty_queryrG   c                    s    fddt  D S )rI   c                    s   g | ]}t  |jqS rJ   rK   rN   rP   rJ   rR   rS   ]   s     z?FullSearchEmptyQueryResults.get_all_results.<locals>.<listcomp>rT   rP   rJ   rP   rR   rU   Y   s    z+FullSearchEmptyQueryResults.get_all_resultsN)rV   rW   rX   r   rZ   r]   r   r2   r[   r^   r_   r`   ra   rb   rU   rJ   rJ   rJ   rR   r\   N   s   
r\   c                   @   sn   e Zd ZU eed< eed< eed< eed< ee ed< eed< eed< eed< eed	< ee ed
< eed< dS )FullSearchQueryParams
model_name
field_nameorig_field_namelike_search_queryother_model_names
stop_wordstsquery_without_syntsquery_with_syntsquery_first_wordfullsearch_slicetopN)rV   rW   rX   strr[   r   intboolrJ   rJ   rJ   rR   rc   _   s   
rc   c                    @   s  e Zd ZdZdZddddddd	d
dddddddddddddddgZdZdddgZee	ddd Z
ed!d" Zed#d$ Zed%d& Zed'd( Zed)d* Zedd,d-Zedd.d/Zedd0d1Zed2d3 Zed4d5 Zedd8d9Zed:d; Zed<d= Zed>d? Zeejjd@dAdBZeddDdEZeeddFddGdHZ edIdJ Z!eedddKdLdMdNdO Z"edPdQ Z#edRdSdTdUdVZ$ee	e	e	e	e%e	 e	e	e	e	e%e& e'e(dWdXdYZ)ee	e	e	e	e%e	 e	e	e	e	e%e& e'e(dWdZd[Z*edd\d]Z+ed^d_ Z,ed`da Z-ee%e%e.dbdcddZ/eddfdgZ0ed6d6d6dhgd6dedidCgfdjdjdkdldmZ1edndodpdqdrZ2edRdodsdtduZ3edndvdwdxZ4eddydzZ5edd{d|Z6eed}d~ Z7d6S )CmfFullSearchuC   
    Сервис полнотекстового поиска.
    russianrM   Ztext_rendertextZ
text_drafttags	parent_idtree_parent_id
project_idcmf_created_atcmf_modified_atcodeZcmf_deletedZcmf_archivedloginZemailZphoneZphone_internalZphone_mobileZphone_2Zphone_assistant
ip_addressZemail_2ZbirthdayTfulltext_searchrun_force_reindexindex_statsrt   c                 C   sR   | rNt | dd} t| dkrNtdt|  d |  d d jdd} | S )	Nhtml.parser
i  z)CmfFullSearch.clean_text: trunc text len z to 1mbi  ignoreerrors)r   get_textlengdebugencodedecoder   rJ   rJ   rR   
clean_text   s    zCmfFullSearch.clean_textc                 C   s   t |ddjdd S )Nr   r   r   r   )r   r   r   r   )clsZtext_with_htmlrJ   rJ   rR   
strip_html   s    zCmfFullSearch.strip_htmlc                    s@   dd t k rgS  fddtdt D }|S )Ni d   c                    s    g | ]}||    qS rJ   rJ   )rO   r   Zoverlaprt   Ztext_len_maxrJ   rR   rS      s     z6CmfFullSearch._split_size_to_parts.<locals>.<listcomp>r   )r   range)r   rt   resrJ   r   rR   _split_size_to_parts   s    "z"CmfFullSearch._split_size_to_partsc                 C   s   d dd td|D S )N c                 S   s   g | ]}t |d k r|qS )2   )r   rO   wrJ   rJ   rR   rS      s      z+CmfFullSearch._strip_50.<locals>.<listcomp>z\sjoinresplitr   rt   rJ   rJ   rR   	_strip_50   s    zCmfFullSearch._strip_50c                 C   s   d dd td|D S )Nr   c                 S   s   g | ]}|  tkr|qS rJ   )lowerALL_STOP_WORDSr   rJ   rJ   rR   rS      s      z1CmfFullSearch._strip_not_word.<locals>.<listcomp>z[ \s<>|&\/%@!`+=;.]r   r   rJ   rJ   rR   _strip_not_word   s    zCmfFullSearch._strip_not_wordc                 C   sB   dd l }d}d| d| }| jj |dd|i }|S )Nr   zCmfFullSearchLock:_zKSELECT pg_try_advisory_xact_lock(('x' || md5(:lock_key))::bit(64)::bigint);lock_key)
sqlalchemydpZ_ddSessionexecutert   Zscalar)r   r3   sapart_nor   r   rJ   rJ   rR   _fullsearch_obj_lock_get_pg   s     
z)CmfFullSearch._fullsearch_obj_lock_get_pgFc                 C   s  t   }td|  |s$tddd l}| jj}|| }|j}|	 }| 
|}	|	sptd| d d S d}
||jjg|jj|k|jj|
k }|| }|r|d }| |jj|kjd|j d}|| n@td|  |  }| j||d|j |
d	}|| |rXdd
lm} |tjjd|idd t   | dkrtdt   |   d S )Nzcmf_full_search: mark_dirty empty obj_idr   )cmf_full_search: _text_search_sql_insert  lockedTis_dirtydirty_atz#cmf_full_search: mark_dirty insert )idr3   r   r   r   )schedule_deferred_jobr3      )kwargsZ	countdowng?zPROF mark_dirty() got )timer   r   
ValueErrorr   r   data_driverdp_model_cls	__table__r   r   selectcr   r(   r3   r   with_for_updater   firstupdatevaluesfuncr1   gen_idinsertcmf.includer   modelsZ	CmfPersonZcelery_full_search_index)r   r3   
fast_indexprof_str   ddsa_modeltabler/   r   r   get_stmtget_resid_update_stmtinsert_stmtr   rJ   rJ   rR   mark_dirty_defer   s^    






zCmfFullSearch.mark_dirty_deferc                 C   s$   |r| j ||dS d|itj|< d S )Nr   r   )r   r   deferred_fullsearch_dirty_list)r   r3   r   forcerJ   rJ   rR   
mark_dirty   s    zCmfFullSearch.mark_dirtyc                 C   st   |ri t _d S t jsd S t j}t|dkr<tdt|  t| D ] }|| }| j||dd qHi t _d S )Nr   ua   DEV: CmfFullSearch помечено грязными слишком много объектов: r   r   )r   r   r   	cmf_alertsortedkeysr   get)r   Z
only_cleanZ
dirty_listr3   r   rJ   rJ   rR   apply_deferred_dirty   s    z"CmfFullSearch.apply_deferred_dirtyc                 C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kjd|j d}
||
 | j  d S )Nr   r   Fr   )r   r   r   r   r   r   r   r   r   r   r(   r3   r   r   r   r   r   r   r   r1   commitr   r3   r   r   r   r   r/   r   r   r   r   rJ   rJ   rR   
mark_clean  s2    




zCmfFullSearch.mark_cleanc                 C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kj|j tjdd d}
||
 | j  d S )Nr   r      )Zhours)delay_error_reindex)r   r   r   r   r   r   r   r   r   r   r(   r3   r   r   r   r   r   r   r   r1   datetime	timedeltar   r   rJ   rJ   rR   mark_delay_error_reindex.  s0    




z&CmfFullSearch.mark_delay_error_reindexN0_full_indexc$           <   *   C   s  |dkr |  | | j  d S | |d d }|}$d}%|rF|%|7 }%|	rV|%d|	 7 }%|
rf|%d|
 7 }%| |%}&| |&}&|dkr|r| |}'| |'}(|d d  d|(d  }'| |'})n|d d  }'|'g}(| |'})|r|}| |}*| |*}*nd }d }*|r.| |}|d d }| |}+nd }d }+|r^| |},|,d d },| |,}-nd },d }-|r| |d d }| |}.nd }.|!r|!}!|!}/nd }!d }/n<|dkrd g}(d })d }d }*d }d }+d },d }-d }.d }!d }/d }'d}0|r|}0|!r|!d d d |0 }0d}1|r*|1d|d d	  7 }1|rD|1d
|d d  7 }1|1d| 7 }1|rb|1d| 7 }1| |0}0| |1}1| |0}2| |1}3|2d}4|3d}5|4|5 }6|6dk rn|6d }7|4dkr|5dkrd|2 d d }2d|3 d d }3nd|2 d d|5  }2|3}3nV|4dk rF|2}2d|3 d d|4  }3n,d|2 d d }2d|3 d d }3|d |0 d |1 }8|8d d }8| |8}9| j	d||||||||||||||||||||'|)|$|&||*||+|8|9|,|-||.| |!|/|"|#d' t
|(dkrt|(dd  dD ]\}:};|d d  d|; }'| |'})| j	|:||||||||||||||||||||'|)d d |$d d d d d d d ||.| d d |"|#d' q| j|t
|(d | j  d S )NZ	9_disable   r   r   r   i Z1_namei  u
    Тэги i>  i  i*  i:  )'r   r3   r4   obj_parent_idobj_tree_parent_idobj_project_idobj_company_idobj_created_atobj_modified_atr5   obj_deletedobj_archivedobj_owner_nameobj_author_nameobj_modified_by_nameobj_responsible_namesobj_hrefobj_logic_type_codeobj_activity_codeobj_status_typeobj_texttext_for_vecobj_namename_for_vecobj_tagstags_for_vecobj_result_textresult_text_for_vecobj_ml_textml_text_for_vecobj_commentscomments_for_vecobj_addon_fieldsobj_addon_fields_for_vecobj_user_ratingobj_key_phraseskey_phrases_for_vecobj_breadcrumbsobj_related_person_loginsr   )'r   r3   r4   r   r   r   r   r   r   r5   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  )delete_from_partno)'_text_search_sql_delete_obj_with_childsr   r   r   r   r   countr   r   _text_search_sql_insertr   	enumerate_text_search_sql_delete_partno)<r   rd   r3   r5   rM   rt   Ztext_prefixZtext_suffixZfull_search_typeZcompany_nameZproject_nameZmodel_verbose_namer   Zcomments_textr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r   Z
smart_namer   r   Zobj_text_listr   r   r   Zobj_comments_textZcomments_text_for_vecr  r  Zobj_ml_text_mainZobj_ml_text_suffixZml_text_main_for_vecZml_text_suffix_for_vecZmain_token_lenZsuffix_token_lenZtotal_token_lenZoversize_lenr   r   r   Zobj_text_partrJ   rJ   rR   index_objectK  sn   (























                   

                   
zCmfFullSearch.index_objectc                 C   sf   |st ddd l}| jj}|| }|j}| }|||j	j
|k|j	j|kB }|| d S Nr   r   )r   r   r   r   r   r   r   deleter(   r   r   r3   r   )r   r3   r   r   r   r   r/   del_stmtrJ   rJ   rR   r  :  s    



z5CmfFullSearch._text_search_sql_delete_obj_with_childsc           	      C   sh   |st ddd l}| jj}|| }|j}| }|||j	j
|k|j	j|k}|| d S r  )r   r   r   r   r   r   r   r  r(   r   r3   r   r   )	r   r3   r  r   r   r   r   r/   r  rJ   rJ   rR   r  N  s    



z,CmfFullSearch._text_search_sql_delete_partnoc(           3   .   C   s  t d|  |stddd l}(| jj})|)| }*|*j}+|) },| 	|}-|-sht d| d d S |(
|+jjg|+jj|k|+jj|k }.t|,|.}/|/rt|/dkrtd| d| dd	d
 |/d d }0|+ |+jj|0kj|d||||||||!||||||	|(jj| j||(jj| j||(jj| j||(jj| j||(jj| j||(jj| j| |(jj| j|"|
|||||||||||#|$|(jj| j|%|&|'|(jj| j|'d(}1|,|1 nt d|  |  }0|+ j|0d||||||||||!||||||	|(jj| j||(jj| j||(jj| j||(jj| j||(jj| j||(jj| j| |(jj| j|"|
|||||||||||#|$|(jj| j|%|&|'|(jj| j|'d*}2|,|2 |0S )Nr   r   r   r   r   u   Для объекта z	(part_no=u   ) задублировались записи в поиске. Обратитесь в техническую поддержку!TabortF)(r3   r   r4   r   r   r   r   r   r   r   r   r   r   r   r   r   name_tsvectortext_tsvectorml_text_tsvectortags_tsvectorresult_text_tsvectorcomments_tsvectoraddon_fields_tsvectorr5   r   r   r   r   r   r   r   r   r   r   r  r  key_phrases_tsvectorr  r  "obj_related_person_logins_tsvectorz0cmf_full_search: _text_search_sql_insert insert )*r   r   r   r3   r4   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r5   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  )r   r   r   r   r   r   r   r   r   r   r   r   r   r(   r3   r   r   rZ   r   r   r   r   r   sqlr   Zto_tsvector_fts_configr   r   )3r   r   r3   r4   r   r   r   r   r   r   r5   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r   r   r   r   r/   r   r   r   r   r   r   rJ   rJ   rR   r
  a  s    




./
z%CmfFullSearch._text_search_sql_insert)objc                 C   s>   |j s
d S | jD ](}|dkrq||jkr|| jr dS qd S )Nrz   T)full_searchrequired_fieldsr	   Z
is_changed)r   r  re   rJ   rJ   rR   is_obj_need_reindex  s    
z!CmfFullSearch.is_obj_need_reindexr   c              
   C   s   t jj D ]}|jsq|j}|r,||kr,qd}t|}|j}|rJdg}t		 }	|j
|||| gdgddd}
|
svq|
D ]&}|rtjj|jjdd qz|  qz|t|
7 }| j  td| d| d	t		 |	 d
d qJqd S )Nr   r   ry   T)r	   sliceorder_byinclude_archivedinclude_deleted)r   zCmfFullSearch.reindex_models: :, z0.3z sec)cmfr   	CmfEntityiter_subclassesr  
class_namecmfutilget_model_by_namefull_search_preload_fieldsr   rZ   rr   r   r   valuefull_search_indexr   r   r   r   r   )r   Zmodels_listZcommit_everyZlazyZ	model_clsrd   offsetmodelZ
field_listtsobj_listr  rJ   rJ   rR   reindex_models  s8    



zCmfFullSearch.reindex_models)	only_oncec                 C   sz   | r|st ddd td tj  tj}|jj}|	|}|j
}| }| |jjdkjdd}|| d S )Nu   Поиск после обновления должен работать сразу, с приемлемым качеством без переиндексацииTr  zRun run_force_reindexr   r   )r   r   r   r   CmfAccessListcheck_admin_moderr   r   r   r   r   r   r   r(   r   r   r   r   )Z
from_patchZi_do_manual_migrationr   r   r   r   r/   r   rJ   rJ   rR   r   $  s    


zCmfFullSearch.run_force_reindexc                 C   s(   t j  |  }| jdd}||dS )NTr7  )totaldirty_count)r   r8  r9  r	  )r   r:  r;  rJ   rJ   rR   r   =  s    
zCmfFullSearch.index_statsz	@minutely   )r6  Z
system_jobZschedulepriorityc                  C   s  t d tj r"t d d S td t } d}g }tjj	dgdddgdd	dd
gddd
ggdddt
j
 t
jdd gddd ggdddt
j
 gddd gggdgddgd}|sqi }|D ]0}t|j}||krg ||< || |j q| D ]\}}t d| dt|  t| |D ]}|j|jdd|gd
d}	|d7 }|	sxt d|  tj| q.t d|	j  z|	  W nt tk
r
 }
 zTt d| d|	 d|
  t t  |||
g t  tj| t  W 5 d }
~
X Y nX q.qt |  dkrvt
j
  j}|dkr`|dk r`t |  dkrvqnt d| d  qt  t d!| d" td q<|rt d# |D ]\}}
t | d$|
  q|d d t d%t |    d S )&NzStart cron_index_dirtyuJ   Не индексируем поскольку запущен импорт   r   r3   r   =ORr   TZis_dirty_patchr   <   )Zminutesr   z	-dirty_at   )r	   filterr#  r"  zcron_index_dirty process r   r   )r	   rD  r%  r   u.   Объект не найдет в бд obj_id=zobj_id=u#   ERROR! При индексации u"    произошла ошибка:    iX  ua   cron_index_dirty превышен лимит времени 20 секунд обработано u    объектовu&   cron_index_dirty обработано u    объектов, sleep 1u2   При индексации были ошибки:: zEnd cron_index_dirty at ) r   r   r,  Zenable_import_modeZimport_is_runningr   sleepr   rr   Zslistr   r1   r   Zget_model_by_idr3   appenditemsr   printr   r.  r   r   r0  	Exception	traceback
format_excZcmf_rollbackr   Z
cmf_commitZ
astimezonehour)stn
exceptionsZobj_id_listZobj_ids_by_modelr3   r2  Zid_listr   r  erN  rJ   rJ   rR   cron_index_dirtyI  s    





zCmfFullSearch.cron_index_dirtyc                 C   sN   |S ]:}|sq|d dkr4t|dks|d dkr4q|| qd|}|S )u   
        Подчистка оригинального квери, который ввел пользователь:
        - удаление стоп-слов
        r   r   -r   )r   r   rH  r   )r   search_queryZclean_search_query_listr   Zclean_search_queryrJ   rJ   rR   _clean_search_query  s     
z!CmfFullSearch._clean_search_queryro   ztuple[str, set[str]])rU  rH   c                 C   s   t  }| d}t|dkr$| |fS d} |dd D ]Z}|dkrBq4td|d}|d dkrj||d  t|dkr4|  |d  |d  } q4|  } | |fS )	zExtrats tags from the given search_query and returns its reminder and a set of extracted tags

        Args:
            search_query (str)

        Returns:
            tuple[str, list[str]]: search_query reminder and a set of extractd tags
        #r   r   Nz(\W)r      r>  )setr   r   r   addstrip)rU  ru   Zsharp_splittedtokenZ
sub_tokensrJ   rJ   rR   _extract_tags  s    

zCmfFullSearch._extract_tags)rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   resultsc                 K   s  |dkrd S |dks|dkrb| j dgfttjj|d|
d||_| j dgf|d|
d||_|dksr|dkr| j dgfttjj|d|
d||_| j dgf|d|
d||_|dks|dkr| j |fttjjddgd	d
|
d||_	| j |fddgd	d
|
d||_
d S )Nr   ANYCmfTaskZFA)related_person_loginre   r9   r"  )re   r9   r"  CmfDocumentr`  rb  rt   ZFS)ra  model_name_not_inre   r9   r"  )rd  re   r9   r"  )filter_oncero   r   current_userr|   r_   r`   r]   r^   ra   rb   r   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   r^  r   rJ   rJ   rR   &_fulltext_search_execute_empty_queries  s    


	z4CmfFullSearch._fulltext_search_execute_empty_queriesc                 K   s  |dks|dkr8| j |||f|
dddgd|d||_|dksH|dkrj| j |dg|f|
d|d||_|dksz|dkr| j |dg|f|
d	d
||_| j |dg|f|
dd
||_t|jt|j dk r| j |dg|	f|
dd
|dd |_|dks|dkr| j |dg|f|
d	d
||_| j |dg|f|
dd
||_t|jt|j dk r| j |dg|	f|
dd
|dd |_	d S )Nr_  rc  r`  rb  CmfAttachmentS)r"  rd  r9   
like_query)r"  r9   rk  A)r"  r9   rC  r   r   )
search_oncerE   rF   rB   rC   r   rD   r?   r@   rA   rg  rJ   rJ   rR    _fulltext_search_execute_queries!  s     
 z.CmfFullSearch._fulltext_search_execute_queriesc           1         s   t |dkrtddd |d kr$d}dkr4dd< dkrDdd< |sPd	d
g}tjjdd}|dkrttd|  dt_|}t }	d}|rt
|tr|D ]}|| qn
|| | |\}}t||}|d< |}| }|}t|}|dkrd}t|}d	|d	< |dr0|d d }|dkrRdd tjj D }n8t|tjkrd}d}dd tjj D }n|g}	d}| |d< | j|dd|	d}| j|d|	d}| j|d|	d}| j|dd|	d}|t_t|||||||||||
d}g }t }|
rdd tjj D }t }|dkrFg S |dkrTg S |dksh|dkr| jddg|fd	d
gd ||d!dd"} ng } |dks|d#kr| jdd#g|fd	d
gd ||d!dd"}!ng }!d$}"|dkrd%}"|dks|d&krD|g}#|dkr|}#| jd|#|fdd#d'gd	|"gd ||d(}$ng }$t | d)krrtd*t |   t }t }g } fd+d, d-d. }%d	d	d	d/d0d0d1}&d	d$d2}'t| d d3 d4d5 d6| d3d   } t|!d d3 d7d5 d6|!d3d   }!t |}( ||&|   ||&|   ||&|!  ||&|! |%||'|$  ||&|   ||&|   ||&|!  ||&|! t |})|)d8ks|)|(krqq|d d8 }| j |d	d8g|||||f}*t | d)krtd9t |   |*S |dkr,t! }+t }| j"f t#|d:|+i t | d)krJtd;t |   t }d	},t$|+% r|+j&d<f|+j'd<f|+j(d<f|+j)d<f|+j*d<f|+j+d<ff}-| ,|-|| |,d<7 },|,d=krVd>-d?d |+% D }.td@|.  qqVt | d)krtdAt |   | j |||||||fS t. }+t }| j/f t#|d:|+i t | d)kr~tdBt |   t01|+D ]D}/t2|+|/j3}0t|0d d3 dCd5 d6|0d3d   }0t4|+|/j3|0 qt }d	},t$|+% r|+j5d3f|+j6d3f|+j7d<f|+j8d<f|+j9d<f|+j5d<f|+j9d<f|+j7d<f|+j8d<f|+j:d<f|+j6d<f|+j:d<f|+j7d<f|+j8d<f|+j;d<f|+j<d<fg}-| ,|-|| |,d<7 },|,d=krd>-dDd |+% D }.td@|.  qqt | d)krtdEt |   | j |||||||fS )FNi   uY   Превышена максимально допустимая длина запроса!Tr  r   archivedFdeletedr   r   r7     u   Идет процесс индексации, могут быть доступны не все результаты поиска. Осталось объектов: tag_namer_  ml_textZModelc                 S   s   g | ]}|j r|jqS rJ   r  r+  rO   mrJ   rJ   rR   rS     s      z1CmfFullSearch.fulltext_search.<locals>.<listcomp>commentsc                 S   s   g | ]}|j r|jqS rJ   ru  rv  rJ   rJ   rR   rS     s      rw   tree_parent_filter)synonymsri   	query_raw)rz  r{  )rz  
first_wordr{  )rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   c                 S   s   g | ]}|j r|jqS rJ   ru  rv  rJ   rJ   rR   rS     s      r`  rM   ZTOPi  )r"  r9   text_stop_wordsrk  r;   include_attachmentrb  rX  r   rc  ri  )rd  r"  r9   r}  rk  g?z'PROF fulltext_search TOP25 selects got c                    s   t tjj}td }|s d S |d}d}|jrD||jkrDd}n|jrX|j|krXd}| tj	|j
< || || d kr | ||S | | ||  d7  < d S )Nrecent_projectsr   r   r   bZ_limitr   )ro   r   rf  r|   rY  popr  r   upperfulltext_search_debug_labelr3   rH  )r   countersr4  
user_loginr  r  Z	cur_classappend_if_existsr   rJ   rR   r  "  s    

z7CmfFullSearch.fulltext_search.<locals>.append_if_existsc                 S   sP   |d |d krd S |sd S | d}dtj|j< | | |d  d7  < d S )Nr	  total_limitr   r,   r   )r  r   r  r3   rH  )r   r  r4  r  rJ   rJ   rR   append_if_exists_other8  s    

z=CmfFullSearch.fulltext_search.<locals>.append_if_exists_otherr<     )r   r  r   Za_limitZb_limitZc_limit)r	  r     c                 S   s   | j S Nr;   r   rJ   rJ   rR   <lambda>G      z/CmfFullSearch.fulltext_search.<locals>.<lambda>)keyc                 S   s   | j S r  r  r  rJ   rJ   rR   r  H  r  rC  z2PROF fulltext_search TOP25 mixing and prepare got r^  z/PROF fulltext_search empty_queries selects got r   r   r   c                 S   s   g | ]}t t|qS rJ   ro   r   rO   frJ   rJ   rR   rS   |  s     uX   Баг в поиске, много данных, либо не идет вычитка: z:PROF fulltext_search empty_queries mixing and prepare got z&PROF fulltext_search main selects got c                 S   s   | j S r  r  r  rJ   rJ   rR   r    r  c                 S   s   g | ]}t t|qS rJ   r  r  rJ   rJ   rR   rS     s     z1PROF fulltext_search main mixing and prepare got )=r   r   r   rr   r	  Zcmf_noter   FSTrY  r   
isinstancerZ   rZ  r]  unionr   r,  Zninjaendswithr(  r)  r*  r-  Z
CmfComment_get_all_branchesprepare_search_queryFSTQrc   r   rm  r   r   _prepare_final_resultr\   rh  varsr+   rU   r_   r`   r]   r^   ra   rb   _add_if_existsr   r<   rn  dataclassesr	   rL   rM   setattrrB   r?   rF   rE   rC   r@   rD   rA   )1r   rd   re   rU  only_idsr"  r	   no_analitycscheck_accessr{  rn   r   r;  rf   ru   rr  Z_tagZextracted_tagsorig_search_queryrg   rm   rh   rw   ri   rj   rk   rl   Zsearch_paramsfinal_resultskip_idsZall_model_namesr   Zresult_tasksZresult_docsZother_sliceZother_model_filterZresult_otherr  r  Zcounters_otherZres_count_beforeZres_count_afterresultr^  Z
iter_countZprocessing_groupZ
found_lensr  datarJ   r  rR   r~   p  s   








        

  
  
$$

$
zCmfFullSearch.fulltext_searchc                 C   s   d| kr| S |  dd S )Nz@#@#@#r   )r   )r   rJ   rJ   rR   _remove_suffix_from_headline  s    z*CmfFullSearch._remove_suffix_from_headlinec                 K   s
  ||d |d  }|ri }	g }
|D ]}|
 |j |jrF| |jnd}d}|jdk	rb|jd}|j}|jtjkr|d tj|j  }|j|j	|j
| d| d	| d	|jd
dtj 	t|j||j|jd|	|j< q$|	t_td |
S i }|r|dd}|dd}i }|D ]}||jg  |j q| D ]}tt|j jrx|dg }dd|| gdddgg}n|}dd|| g}|dkr|dddg }t|j||||d}|D ]0}tt|j jr|||jj< n
|||j< qq>ntd td g }|D ]}||j}|r|jr<| |jnd}d}|jdk	rZ|jd}|j}|jtjkr|d tj|j  }|jdkr|jsq|j |jj!|jj"j#| d| d	| d	|jd
dtj 	||j|j|jj#|j	|j$|j%|j&t|jd}nN|jj#|j	|j"j#| d| d	| d	|jd
dtj 	||j|jt|jd}zf|sh|j'ddd  n||svt(|D ].}|)d!d }t*||j |d}|||< qz| | W n t(k
r   Y nX qtd |st+tj,j-|t.|dd" d#d$ |S )%u5   
        Итоговая обработка
        r   r   r   NoneNz.6frT  z ||| r   z.0fz words=)r   rM   r{   r7   r8   r9   r:   r;   zfulltext_search ENDro  Frp  cmf_ver_headINZcmf_ver_curz==Tr   ri  urlurl_previewurl_preview_img)r	   rD  r$  r%  uY   DEV: FATAL. Укажите в запросе поиска список полей fields=z"fulltext_search Start check access)r   rM   r{   r7   r9   r:   r;   Z	attach_idZattach_nameZ
attach_urlZattach_url_previewZattach_url_preview_imgr8   )r   rM   r{   r7   r9   r:   r;   r8   )ZTEXKOM_skip_failread_auditZTEXKOM_ppp_project_simplecheck.rC  )rU  obj_dict)r   )/rH  r3   r7   r  r:   r9   r   r  r   r6   r5   r;   r  Zjsonloadsr8   fulltext_search_headlinesr   r   
setdefaultr4   r   r  r   Zcmf_verr,  r-  rZ   r  r   r+  parentrv   rM   r{   r/  r  r  r  Z_acl_check_readZCmfPermissionErrorr   rL   r   rr   _do_calc_statisticsZ
dumps_dict)r   r  r"  r  r	   r  r  r  r   r  Z	result_idrr7   Zformated_rankr9   r  ZobjectsZis_archivedZ
is_deletedZids_by_modelrd   _fieldsZ_filterr4  r   r  r   attrrJ   rJ   rR   r    s    

$







$
$



z#CmfFullSearch._prepare_final_result)r4  r  r  c                 C   sj   |D ]`\}}t |D ]N}|s  q|d j|kr:|d q||d  ||d j |d qqd S )Nr   )r   r3   r  rH  rZ  )r   r4  r  r  Zlstr	  r   rJ   rJ   rR   r  X  s    
zCmfFullSearch._add_if_existsr   c           -      K   s  |d krdg}|sddg}| dd }| d}| dd }| dd }| dd }| d	}| d
}| dd }| d}| d}| dd }| d|}|p|}| d}|rdd|nd}d}|
rd}| dpg }| |\}}| |\}}|dkrd }|	dkrd }	d } |	rJtd|	sBtd|	rJ|	 } |dkrftd| dd d| }!d| }"|dkrd}"| d}#|d }$|d  |d  }%d}&|rt	|}d!| d"}&t
jjj d# d}'|d$krd%}'td&d'|}(td&d(|})d)}*|d*krd+}*d,jf ||!|#|&|"||*||d-	}+t
jjj |+||(|)|$|%t|t||||	|'|||||||||||| |d.||},t|,S )/Nnull000r   r   rv   modified_by_nameresponsible_namelogic_type_codestatus_typemodified_at_beforemodified_at_afterro  rp  user_rating
owner_nameauthor_namery   AND obj_tree_parent_id IN ('{}')','r   zOOR (obj_model = 'CmfAttachment' and obj_parent_id ilike :model_name_in || ':%')rr  z^[a-zA-Z0-9]+-[0-9]+$z^[0-9]+$)rt   rs  rM   ru   rx  Zaddon_fieldsZkey_phrases6   Недопустимое значение field_name: Tr  obj_rM   r   Z	_tsvectorr   $obj_modified_at > now() - interval '
 days' ANDz"set gin_fuzzy_search_limit=100000;rt   @B z([&] )([^!])z<1> \2z<2> \20)rt   rs  rx  z1|16u"  
            SELECT
                subs.*,
                ts_headline(
                    'russian',
                    substring(subs.{search_field} FROM 0 FOR 100000),
                    :tsquery,
                    'MaxFragments=1,MaxWords=25,MinWords=24'
                ) AS headline
            FROM (
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        obj_breadcrumbs as breadcrumbs,
                        obj_project_id as obj_project_id,
                        obj_related_person_logins as obj_related_person_logins,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 3*365 then 'R' else '' end) as label,
                        (ts_rank_cd({tsvector_field}, :tsquery, {ts_rank_normalize}))
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 3*365 then
                                    3*((3*365-EXTRACT(days from (now()-obj_modified_at)))/30+1) else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        (({tsvector_field} @@ :tsquery)  or ( {search_field} ILIKE '%' || :like_query || '%' ))
                        and ((obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in) {include_attachment_filter})
                        and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {rpl_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                        and ( :user_rating is null or obj_user_rating >= :user_rating)
                    ORDER BY rank desc
                    LIMIT :slice_for OFFSET :slice_from
                )
            UNION
                /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        obj_breadcrumbs as breadcrumbs,
                        obj_project_id as obj_project_id,
                        obj_related_person_logins as obj_related_person_logins,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        100
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                        ((:obj_code is not null and obj_code ILIKE '%' || :obj_code)
                            or obj_id = :like_query
                        )
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {rpl_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                    /*ORDER BY obj_id desc -- не будем делать, т.к. тормозит*/
                    LIMIT :slice_for/10
                    OFFSET :slice_from/10
                )
            ) as subs
            ORDER BY subs.rank desc
            LIMIT :slice_for
            OFFSET :slice_from;
        )	ry  search_fieldtsvector_fieldage_days_subqueryheadline_fieldtags_filterts_rank_normalizeinclude_attachment_filter
rpl_filter)tsquerytsquery_str_h1tsquery_str_h2
slice_from	slice_formodel_name_inrd  r9   r}  rk  
max_partnorv   r  r  r  r  r  r  r  r  ro  rp  r5   r  )r   formatr   _build_tags_filter#_build_related_person_logins_filterr   matchr  r   rp   r   rr   r   r   r   r   subtuplerZ   )-r   re   r  Ztsquery_strrd  r;   r9   r"  r}  rk  r~  ra  r   rv   r  r  r  r  r  r  ro  rp  r  r  r  ry  r  
tags_namesr  tags_paramsr  
rpl_paramsr5   r  r  r  r  r  r  r  r  r  r  r  
found_objsrJ   rJ   rR   rm  e  s    




	


"





	

bmzCmfFullSearch.search_oncer  r   z
str | None)related_userra  c	           %      K   s  |	 d}
|	 d}|	 d}|	 d}|	 d}|	 d}|	 d}|	 d}|	 d	}|	 d
}|	 dd }|	 d|}|p|}|	 d}|rdd|nd}|	 dpg }| |\}}| |\}}d}|rtd | }}d}|dkrtd| dd d| }|d }|d |d  }d} |rHt|}d| d} |dkrVdnd}!d | d!| d"| d#|  d$| d%| d%| d%| d&}"tjj	j
 |"||t|t|||!|
|||||||||||d'||}#t|#}$|$S )(Nrv   r  r  r  r  r  r  ro  rp  r  r  r  ry  r  r  r   rr  z
            AND ( :responsible_name IS NULL OR obj_responsible_names ILIKE '%' || :responsible_name || '%' )
            AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
        z:Param related_user is deprecated. Use related_person_loginz
            AND (
                :responsible_name IS NULL
                OR obj_responsible_names ILIKE '%' || :responsible_name || '%'
                OR obj_owner_name = :owner_name
            )
        )rt   rs  rM   ru   rx  r  Tr  r  r   r   r  r  rt   r  a  
            SELECT
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                obj_breadcrumbs as breadcrumbs,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                z as z,
                substring(z` FROM 0 FOR 240) as headline
            FROM cmf_full_search
            WHERE
                a  
                obj_model IN :model_name_in AND obj_model NOT IN :model_name_not_in
                AND part_no <= :max_partno
                AND ( :parent_id IS NULL OR obj_parent_id = :parent_id )
                AND ( :author_name IS NULL OR obj_author_name = :author_name )
                AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                z
                a  
                AND ( :logic_type_code IS NULL OR obj_logic_type_code = :logic_type_code )
                AND ( :status_type IS NULL OR obj_status_type = :status_type )
                AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                AND ( :archived IS NULL OR obj_archived = :archived )
                AND ( :deleted IS NULL OR obj_deleted = :deleted )
                and ( :user_rating is null or obj_user_rating >= :user_rating)
            ORDER BY rank desc
            LIMIT :slice_for OFFSET :slice_from;
        )r  r  r  rd  r9   r  rv   r  r  r  r  r  r  r  r  ro  rp  r  )r   r  r   r  r  r   rp   r   rr   r   r   r   r   r  rZ   )%r   r  r  ra  re   rd  r;   r9   r"  r   rv   r  r  r  r  r  r  ro  rp  r  r  r  ry  r  r  r  r  r  Zowner_and_responsible_filterr  r  r  r  r  r  r  Zall_objsrJ   rJ   rR   re  X  s    















#zCmfFullSearch.filter_oncez	list[str]ztuple[str, dict[str, str]])r  rH   c                 C   s8   i }d}| r0d}dd dd | D  d|d< ||fS )	Nr   z AND obj_tags ~* :tags_re(^| )(|c                 s   s   | ]}t |V  qd S r  r   escape)rO   rr  rJ   rJ   rR   	<genexpr>  s     z3CmfFullSearch._build_tags_filter.<locals>.<genexpr>)( |$)Ztags_re)r   )r  r  r  rJ   rJ   rR   r    s     z CmfFullSearch._build_tags_filter)r  rH   c                 C   s2   i }d}| r*d}t | } d|  d|d< ||fS )Nr   z; AND obj_related_person_logins ~* :related_person_logins_rer  r  Zrelated_person_logins_rer  )r  r  r  rJ   rJ   rR   r    s    
z1CmfFullSearch._build_related_person_logins_filterrG   c                 C   s0   | sg S t jjj dd| i}dd |D S )Na  
                WITH tree_parents AS (
                    WITH RECURSIVE r AS (
                        SELECT obj_id, obj_code, obj_tree_parent_id
                        FROM cmf_full_search
                        WHERE obj_tree_parent_id = :tree_parent_id

                        UNION

                        SELECT cfs.obj_id, cfs.obj_code, cfs.obj_tree_parent_id
                        FROM cmf_full_search AS cfs
                        JOIN r ON cfs.obj_tree_parent_id = r.obj_id
                    )
                    SELECT obj_id FROM r
                    WHERE r.obj_id IN (SELECT obj_tree_parent_id FROM r)

                    UNION

                    SELECT :tree_parent_id
                )
                SELECT * FROM tree_parents;
            rw   c                 S   s   g | ]}|d  qS )r   rJ   )rO   r  rJ   rJ   rR   rS     s     z3CmfFullSearch._get_all_branches.<locals>.<listcomp>)r   rr   r   r   r   r   )rw   ZrecordsrJ   rJ   rR   r    s    zCmfFullSearch._get_all_branchesc              
   C   sT  dt _|dd}tdd|}td|}d}d}	d}
|D ]}t|dkrPq<t|dkr|dkrfq<|d	kr~|
d| 7 }
q<|dkrq<|d
kr|
d7 }
q<|dkr|
d7 }
q<t|dkrq<|d dkrt|dkrq<|
d|dd   7 }
|	d|dd   7 }	q<|rq<|dd}|d7 }|dkr.d}|
rB|
d dkrJ|
d7 }
t|dkr| j||d}t|dkr|
|d  7 }
n|
dd| d 7 }
|r|dkr q|dkr< qq<q<|rt|	dkr|	d dkr|	dd  }	|	S |
dddddd d!d 	 }
|
r<|
d dkr<|
dd  }
|
r\|
d dkr\|
dd  }
|
r||
d dkr||
dd  }
|
r|
dd  dkr|
d d }
z.t
jjj d"d#|
i}t|d d }
W nn tjjk
r< } zJt
jjj  |std$ t
jjj d%d#|i}t|d d }
W 5 d }~X Y nX t|d&|
 |
t _|
S )'Nr   zwww.u   [^-A-Za-zА-Яа-я0-9()|&!' ]r   z(,| |&|\||\(|\))r   r   )rT  !z()&|)r   r   r  z |)r   r   &z &r>  z& !r  rT  r  F)r  r  z& )rz  z( z | z )
   )r  r  z OR z or z AND z & z and z!select to_tsquery('russian', :q);qu7   Ошибочный синтаксис в запросеz+select websearch_to_tsquery('russian', :q);z->)r   r  replacer   r  r   r   prepare_wordr   r[  r   
CmfSynonymr   r   r   r   rZ   r   excZProgrammingErrorZrollbackr   rJ  r  )r   rU  rz  ri   r|  r{  Zsearch_query_allowed_symbtokensZ
word_countZstopsr  r0   Z	sug_wordsr  rR  rJ   rJ   rR   r    s    

("z"CmfFullSearch.prepare_search_queryc                 C   s  t  jd| d7  _|d tjkr0d}tj}n
d}tj}g }t|s||}g }t	|}|rt  jd| d7  _|
| t  jd7  _d}	|D ]}
|	d	kr qHt|
d	krq|
d |d kr|
d
 |d
 krqd|
kr|
dd}
t  jd|
 d7  _|
|
 |	d
7 }	qt  jd|
 d7  _|
|
 |	d
7 }	qg }tjjj dd|i}d}	|D ]\}}|	d
kr qt|d	krqn|d |d ks|d
 |d
 krn|dd}t  jd| d7  _|
| |	d
7 }	qnt|t|B |hB }n|h}t }|D ]D}
t|
d d	 D ]*}||j t  jd|j d7  _q2q||B }t }|rtjjddt||hB gddgdgdd
gd}|D ]\}|jr|jjdd d D ]6}| dd}t  jd| d7  _|| qʐq||B t|B }t|S )Nz|w:rF  r   enruzaddNinjaRevers r'  zspellError, r>  r   r   z<->z	addSpell z
            SELECT
                name, similarity(:word, name) as sim
            FROM cmf_synonym
            WHERE
                :word % name
            ORDER BY "sim" desc
            LIMIT 5;
             wordzaddSpellTrgm z
normalize rM   r  rt   Zorderno)rD  r	   r#  r"  ,rB  zsynAdd )r   r  stringascii_lettersr,  Zdictionary_enZdictionary_ruZdictionary_checkZsuggestZninja_reversrH  r   r  r   r  r   r   r   r   rY  morphparserZ  Znormal_formrZ   rt   r/  r   r[  )r   r  rz  langZ
dictionaryZfiltered_suggestions3ZsuggestionsZfiltered_suggestionsZnwr   r   Zfiltered_suggestions2Zsuggestions2_listZsuggr   Zall_suggestionsZnormalized_wordsZsynonym_wordsZsynonym_listZsynonymr/   rJ   rJ   rR   r  z  s    


 



	 
$
 zCmfFullSearch.prepare_wordc              	   C   s   t | dk rd S g }|d d D ]b}|d dsB|d dsBq i }|d |d< |dd rp|d d |d< nd |d< || q t }||_d	|_| |_t	j
j|_d
|_t  |  W 5 Q R X d S )NrX  rC  r   zCmfDocument:zCmfTask:r3   r  rx   searchF)r   
startswithr   rH  r   ZCmfSearchStatr4  actionrU  r   rf  r   Z	person_idZ
aggregatedr,  Zdisable_aclZsave)rU  r  Zst_dictZrecZst_recstatrJ   rJ   rR   r    s(    

z!CmfFullSearch._do_calc_statistics)F)FF)F)NNNr   NNNNNFFNNNNNNNNNNNNNNNNNNNN)Nr   F)FF)FNNFNFN)NNr   NNNFN)TFFF)T)8rV   rW   rX   rY   r  r   Z	api_allowZapi_methodsstaticmethodro   r   classmethodr   r   r   r   r   r   r   r   r   r   r  r  r  r
  r(  r   Z	BaseModelr!  r5  Zcmf_deferred_jobr   r   rS  rV  r]  r   rp   rq   r<   rh  rn  r~   r  r  r   r  rm  re  r  r  r  r  r  r  rJ   rJ   rJ   rR   rr   m   s                    




	9

                                o


 !
X
!GN               O

                s~	"hnrr   )(r   r   collectionsr   typingr   r   r   r   r  r   r   r	   rL  sys	itertoolsr   Zcmf.data_providers.sqlalchemyr   Zcmf.fields.cmf_full_searchr(  Zenchantr  Z	pymorphy3r   r   Zbs4r   ZMorphAnalyzerr  ZRUSSIAN_STOP_WORDSZENGLISH_STOP_WORDSrY  r   r2   r<   r\   rc   Zcmf_full_searchrr   rJ   rJ   rJ   rR   <module>   sR    3 ?       $