U
    i                    @   s  d dl Z d dlZd dlmZ d dlmZmZmZmZ d dl	m
Z
mZmZ d dl	Z	d dlZd dlZd dlZd dlT d dlmZ d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ e Zdd	d
dddddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~ddddddddddddddddddddddddddddddddgZdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd ddddddddd	d
dddddddddddddddddddgZe ee Z!ddd d!d"d#d$gZ"ed%d&d'd(d)d*d+d,d-d.g	Z#e
G d/d0 d0Z$e
G d1d2 d2Z%e
G d3d4 d4Z&G d5d6 d6ejj'j(Z(dS (7      N)
namedtuple)ListSetDictOptional)	dataclassfieldfields)*)SQLAlchemyDataDriver)BeautifulSoup   иu   вu   воu   неu   чтоu   онu   наu   яu   сu   соu   какu   аu   тоu   всеu   онаu   такu   егоu   ноu   даu   тыu   кu   уu   жеu   выu   заu   быu   поu   толькоu   ееu   мнеu   былоu   вотu   отu   меняu   ещеu   нетu   оu   изu   емуu   теперьu
   когдаu   дажеu   нуu
   вдругu   лиu   еслиu   уже   илиu   ниu   бытьu   былu   негоu   доu   васu   нибудьu
   опятьu   ужu   вамu   ведьu   тамu
   потомu   себяu   ничегоu   ейu
   можетu   ониu   тутu   гдеu   естьu   надоu   нейu   дляu   мыu   тебяu   ихu   чемu   былаu   самu   чтобu   безu
   будтоu   чегоu   разu   тожеu   себеu   подu
   будетu   жu
   тогдаu   ктоu   этотu   тогоu   потомуu
   этогоu
   какойu   совсемu   нимu
   здесьu   этомu   одинu
   почтиu   мойu   темu
   чтобыu   нееu   сейчасu   былиu   кудаu
   зачемu   всехu   никогдаu
   можноu   приu   наконецu   дваu   обu   другойu   хотьu
   послеu   надu   большеu   тотu
   черезu   этиu   насu   проu
   всегоu   нихu
   какаяu
   многоu
   развеu   триu   этуu   мояu   впрочемu   хорошоu   своюu   этойu
   передu   иногдаu
   лучшеu   чутьu   томu   нельзяu
   такойu   имu
   болееu   всегдаu   конечноu   всюu
   междуimeZmyZmyselfZweZourZoursZ	ourselvesZyouZyourZyoursZyourselfZ
yourselvesheZhimZhisZhimselfZsheZherZhersZherselfitZitsZitselfZtheyZthemZtheirZtheirsZ
themselvesZwhatwhichZwhoZwhomthisthatZtheseZthoseamisZareZwasZwerebeZbeenZbeingZhaveZhasZhadZhavingZdoZdoesZdidZdoingaZanZtheandZbutiforZbecauseasZuntilwhileZofZatZbyforwithZaboutZagainstZbetweenZintoZthroughZduringbeforeZafterZaboveZbelowtofromZupZdowninoutZonZoffZoverZunderZagainZfurtherZthenoncehereZthereZwhenwherewhyZhowallanyZbothZeachZfewZmoreZmostotherZsomeZsuchnoZnornotZonlyZownZsameZsoZthanZtooZverystZcanZwillZjustZdonZshouldnowtextml_textnametagscommentsZaddon_fieldsZkey_phrasesSearchResultobj_id	obj_modelobj_codetitleheadlinebreadcrumbslabelrankage_daysc                   @   s   e Zd ZU dZeedZee e	d< eedZ
ee e	d< eedZee e	d< eedZee e	d< eedZee e	d< eedZee e	d< eedZee e	d	< eedZee e	d
< eee  dddZdS )FullSearchResultsuP   
    Контейнер для всех результатов поиска
    default_factory	doc_nosyndoc_syndoc_first_word
task_nosyntask_syntask_first_word	other_syn
attach_synreturnc                    s    fddt  D S )Q   
        Возвращает все списки результатов
        c                    s   g | ]}t  |jqS  getattrr4   .0r   selfrO   ./cmf/models/cmf_full_search.py
<listcomp>M   s     z5FullSearchResults.get_all_results.<locals>.<listcomp>r	   rT   rO   rT   rV   get_all_resultsI   s    z!FullSearchResults.get_all_resultsN)__name__
__module____qualname____doc__r   listrD   r   r7   __annotations__rE   rF   rG   rH   rI   rJ   rK   rY   rO   rO   rO   rV   rA   +   s   
rA   c                   @   s   e Zd ZU eedZee ed< eedZ	ee ed< eedZ
ee ed< eedZee ed< eedZee ed< eedZee ed< eee  dd	d
ZdS )FullSearchEmptyQueryResultsrB   doc_empty_query_related_userdoc_empty_querytask_empty_query_related_usertask_empty_queryother_empty_query_related_userother_empty_queryrL   c                    s    fddt  D S )rN   c                    s   g | ]}t  |jqS rO   rP   rR   rT   rO   rV   rW   _   s     z?FullSearchEmptyQueryResults.get_all_results.<locals>.<listcomp>rX   rT   rO   rT   rV   rY   [   s    z+FullSearchEmptyQueryResults.get_all_resultsN)rZ   r[   r\   r   r^   ra   r   r7   r_   rb   rc   rd   re   rf   rY   rO   rO   rO   rV   r`   P   s   
r`   c                   @   sn   e Zd ZU eed< eed< eed< eed< ee ed< eed< eed< eed< eed	< ee ed
< eed< dS )FullSearchQueryParams
model_name
field_nameorig_field_namelike_search_queryother_model_names
stop_wordstsquery_without_syntsquery_with_syntsquery_first_wordfullsearch_slicetopN)rZ   r[   r\   strr_   r   intboolrO   rO   rO   rV   rg   a   s   
rg   c                    @   s  e Zd ZdZdZddddddd	d
dddddddddddddddgZdZdddgZee	ddd Z
ed!d" Zed#d$ Zed%d& Zed'd( Zed)d* Zedd,d-Zedd.d/Zedd0d1Zed2d3 Zed4d5 Zedd8d9Zed:d; Zed<d= Zed>d? Zeejjd@dAdBZeddDdEZeeddFddGdHZ edIdJ Z!eedddKdLdMdNdO Z"edPdQ Z#edRdSdTdUdVZ$ee	e	e	e	e%e	 e	e	e	e	e%e& e'e(dWdXdYZ)ee	e	e	e	e%e	 e	e	e	e	e%e& e'e(dWdZd[Z*edd\d]Z+ed^d_ Z,ed`da Z-ee%e%e.dbdcddZ/eddfdgZ0ed6d6d6dhgd6dedidCgfdjdjdkdldmZ1edndodpdqdrZ2edRdodsdtduZ3edndvdwdxZ4eddydzZ5edd{d|Z6eed}d~ Z7d6S )CmfFullSearchuC   
    Сервис полнотекстового поиска.
    russianr4   Ztext_renderr2   Z
text_draftr5   	parent_idtree_parent_id
project_idcmf_created_atcmf_modified_atcodeZcmf_deletedZcmf_archivedloginZemailZphoneZphone_internalZphone_mobileZphone_2Zphone_assistant
ip_addressZemail_2ZbirthdayTfulltext_searchrun_force_reindexindex_statsr2   c                 C   sR   | rNt | dd} t| dkrNtdt|  d |  d d jdd} | S )	Nhtml.parser
i  z)CmfFullSearch.clean_text: trunc text len z to 1mbi  ignoreerrors)r   get_textlengdebugencodedecoder   rO   rO   rV   
clean_text   s    zCmfFullSearch.clean_textc                 C   s   t |ddjdd S )Nr   r   r   r   )r   r   r   r   )clsZtext_with_htmlrO   rO   rV   
strip_html   s    zCmfFullSearch.strip_htmlc                    s@   dd t k rgS  fddtdt D }|S )Ni d   c                    s    g | ]}||    qS rO   rO   )rS   r   Zoverlapr2   Ztext_len_maxrO   rV   rW      s     z6CmfFullSearch._split_size_to_parts.<locals>.<listcomp>r   )r   range)r   r2   resrO   r   rV   _split_size_to_parts   s    "z"CmfFullSearch._split_size_to_partsc                 C   s   d dd td|D S )N c                 S   s   g | ]}t |d k r|qS )2   )r   rS   wrO   rO   rV   rW      s      z+CmfFullSearch._strip_50.<locals>.<listcomp>z\sjoinresplitr   r2   rO   rO   rV   	_strip_50   s    zCmfFullSearch._strip_50c                 C   s   d dd td|D S )Nr   c                 S   s   g | ]}|  tkr|qS rO   )lowerALL_STOP_WORDSr   rO   rO   rV   rW      s      z1CmfFullSearch._strip_not_word.<locals>.<listcomp>z[ \s<>|&\/%@!`+=;.]r   r   rO   rO   rV   _strip_not_word   s    zCmfFullSearch._strip_not_wordc                 C   sB   dd l }d}d| d| }| jj |dd|i }|S )Nr   zCmfFullSearchLock:_zKSELECT pg_try_advisory_xact_lock(('x' || md5(:lock_key))::bit(64)::bigint);lock_key)
sqlalchemydpZ_ddSessionexecuter2   Zscalar)r   r8   sapart_nor   r   rO   rO   rV   _fullsearch_obj_lock_get_pg   s     
z)CmfFullSearch._fullsearch_obj_lock_get_pgFc                 C   s  t   }td|  |s$tddd l}| jj}|| }|j}|	 }| 
|}	|	sptd| d d S d}
||jjg|jj|k|jj|
k }|| }|r|d }| |jj|kjd|j d}|| n@td|  |  }| j||d|j |
d	}|| |rXdd
lm} |tjjd|idd t   | dkrtdt   |   d S )Nzcmf_full_search: mark_dirty empty obj_idr   )cmf_full_search: _text_search_sql_insert  lockedT)is_dirtydirty_atz#cmf_full_search: mark_dirty insert )idr8   r   r   r   )schedule_deferred_jobr8      )kwargsZ	countdowng?zPROF mark_dirty() got )timer   r   
ValueErrorr   r   data_driverdp_model_cls	__table__r   r   selectcr   r(   r8   r   with_for_updater   firstupdatevaluesfuncr1   gen_idinsertcmf.includer   modelsZ	CmfPersonZcelery_full_search_index)r   r8   
fast_indexprof_str   ddsa_modeltabler/   r   r   get_stmtget_resid_update_stmtinsert_stmtr   rO   rO   rV   mark_dirty_defer   s^    






zCmfFullSearch.mark_dirty_deferc                 C   s:   |r| j ||dS d|itj|< ttjdkr6td d S )Nr   r   i'  u   DEV: WARNING! Большое количество блокировок может падать на некоторых инсталяциях)r   r   deferred_fullsearch_dirty_listr   	cmf_alert)r   r8   r   forcerO   rO   rV   
mark_dirty   s
    zCmfFullSearch.mark_dirtyc                 C   st   |ri t _d S t jsd S t j}t|dkr<tdt|  t| D ] }|| }| j||dd qHi t _d S )Nr   ua   DEV: CmfFullSearch помечено грязными слишком много объектов: r   r   )r   r   r   r   sortedkeysr   get)r   Z
only_cleanZ
dirty_listr8   r   rO   rO   rV   apply_deferred_dirty  s    z"CmfFullSearch.apply_deferred_dirtyc                 C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kjdd|j d}
||
 | j  d S )Nr   r   F)r   is_dirty_patchr   )r   r   r   r   r   r   r   r   r   r   r(   r8   r   r   r   r   r   r   r   r1   commitr   r8   r   r   r   r   r/   r   r   r   r   rO   rO   rV   
mark_clean  s4    



	
zCmfFullSearch.mark_cleanc                 C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kj|j tjdd d}
||
 | j  d S )Nr   r      )Zhours)delay_error_reindex)r   r   r   r   r   r   r   r   r   r   r(   r8   r   r   r   r   r   r   r   r1   datetime	timedeltar   r   rO   rO   rV   mark_delay_error_reindex4  s0    




z&CmfFullSearch.mark_delay_error_reindexN0_full_indexc$           <   *   C   s  |dkr |  | | j  d S | |d d }|}$d}%|rF|%|7 }%|	rV|%d|	 7 }%|
rf|%d|
 7 }%|!rv|%d|! 7 }%| |%}&| |&}&|dkr|r| |}'| |'}(|d d  d|(d  }'| |'})n|d d  }'|'g}(| |'})|r|}| |}*| |*}*nd }d }*|r@| |}|d d }| |}+nd }d }+|rp| |},|,d d },| |,}-nd },d }-|r| |d d }| |}.nd }.|!r|!}!|!}/nd }!d }/n<|dkrd g}(d })d }d }*d }d }+d },d }-d }.d }!d }/d }'d}0|r|}0|!r|!d d d |0 }0d}1|r<|1d|d d	  7 }1|rV|1d
|d d  7 }1|1d| 7 }1|rt|1d| 7 }1| |0}0| |1}1| |0}2| |1}3|2d}4|3d}5|4|5 }6|6dk rn|6d }7|4dkr.|5dkrd|2 d d }2d|3 d d }3nd|2 d d|5  }2|3}3nV|4dk rX|2}2d|3 d d|4  }3n,d|2 d d }2d|3 d d }3|d |0 d |1 }8|8d d }8| |8}9| j	d||||||||||||||||||||'|)|$|&||*||+|8|9|,|-||.| |!|/|"|#d' t
|(dkrt|(dd  dD ]\}:};|d d  d|; }'| |'})| j	|:||||||||||||||||||||'|)d d |$d d d d d d d ||.| d d |"|#d' q&| j|t
|(d | j  d S )NZ	9_disable   r   r   r   i Z1_namei  u
    Тэги i>  i  i*  i:  )'r   r8   r9   obj_parent_idobj_tree_parent_idobj_project_idobj_company_idobj_created_atobj_modified_atr:   obj_deletedobj_archivedobj_owner_nameobj_author_nameobj_modified_by_nameobj_responsible_namesobj_hrefobj_logic_type_codeobj_activity_codeobj_status_typeobj_texttext_for_vecobj_namename_for_vecobj_tagstags_for_vecobj_result_textresult_text_for_vecobj_ml_textml_text_for_vecobj_commentscomments_for_vecobj_addon_fieldsobj_addon_fields_for_vecobj_user_ratingobj_key_phraseskey_phrases_for_vecobj_breadcrumbsobj_related_person_loginsr   )'r   r8   r9   r   r   r   r   r   r   r:   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  )delete_from_partno)'_text_search_sql_delete_obj_with_childsr   r   r   r   r   countr   r   _text_search_sql_insertr   	enumerate_text_search_sql_delete_partno)<r   rh   r8   r:   r4   r2   Ztext_prefixZtext_suffixZfull_search_typeZcompany_nameZproject_nameZmodel_verbose_namer   Zcomments_textr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r   Z
smart_namer   r   Zobj_text_listr   r   r   Zobj_comments_textZcomments_text_for_vecr  r  Zobj_ml_text_mainZobj_ml_text_suffixZml_text_main_for_vecZml_text_suffix_for_vecZmain_token_lenZsuffix_token_lenZtotal_token_lenZoversize_lenr   r   r   Zobj_text_partrO   rO   rV   index_objectQ  sr   (























                   

                   
zCmfFullSearch.index_objectc                 C   sf   |st ddd l}| jj}|| }|j}| }|||j	j
|k|j	j|kB }|| d S Nr   r   )r   r   r   r   r   r   r   deleter(   r   r   r8   r   )r   r8   r   r   r   r   r/   del_stmtrO   rO   rV   r
  B  s    



z5CmfFullSearch._text_search_sql_delete_obj_with_childsc           	      C   sh   |st ddd l}| jj}|| }|j}| }|||j	j
|k|j	j|k}|| d S r  )r   r   r   r   r   r   r   r  r(   r   r8   r   r   )	r   r8   r	  r   r   r   r   r/   r  rO   rO   rV   r  V  s    



z,CmfFullSearch._text_search_sql_delete_partnoc(           3   /   C   s  t d|  |stddd l}(| jj})|)| }*|*j}+|) },| 	|}-|-sht d| d d S |(
|+jjg|+jj|k|+jj|k }.t|,|.}/|/rt|/dkrtd| d| dd	d
 |/d d }0|+ |+jj|0kj|dd||||||||!||||||	|(jj| j||(jj| j||(jj| j||(jj| j||(jj| j||(jj| j| |(jj| j|"|
|||||||||||#|$|(jj| j|%|&|'|(jj| j|'d)}1|,|1 nt d|  |  }0|+ j|0dd||||||||||!||||||	|(jj| j||(jj| j||(jj| j||(jj| j||(jj| j||(jj| j| |(jj| j|"|
|||||||||||#|$|(jj| j|%|&|'|(jj| j|'d+}2|,|2 |0S )Nr   r   r   r   r   u   Для объекта z	(part_no=u   ) задублировались записи в поиске. Обратитесь в техническую поддержку!TabortF))r8   r   r   r9   r   r   r   r   r   r   r  r   r   r   r   r   r   name_tsvectortext_tsvectorml_text_tsvectortags_tsvectorresult_text_tsvectorcomments_tsvectoraddon_fields_tsvectorr:   r   r   r   r   r   r   r   r   r   r   r  r  key_phrases_tsvectorr  r  "obj_related_person_logins_tsvectorz0cmf_full_search: _text_search_sql_insert insert )+r   r   r   r   r8   r9   r   r   r   r   r   r   r  r   r   r   r   r   r   r  r  r  r  r  r  r  r:   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  )r   r   r   r   r   r   r   r   r   r   r   r   r   r(   r8   r   r   r^   r   r   r   r   r   sqlr   Zto_tsvector_fts_configr   r   )3r   r   r8   r9   r   r   r   r   r   r   r:   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r   r   r   r   r/   r   r   r   r   r   r   rO   rO   rV   r  i  s    




/0
z%CmfFullSearch._text_search_sql_insert)objc                 C   s>   |j s
d S | jD ](}|dkrq||jkr|| jr dS qd S )Nr|   T)full_searchrequired_fieldsr	   Z
is_changed)r   r   ri   rO   rO   rV   is_obj_need_reindex   s    
z!CmfFullSearch.is_obj_need_reindexr   c              
   C   s   t jj D ]}|jsq|j}|r,||kr,qd}t|}|j}|rJdg}t		 }	|j
|||| gdgddd}
|
svq|
D ]&}|rtjj|jjdd qz|  qz|t|
7 }| j  td| d| d	t		 |	 d
d qJqd S )Nr   r   r{   T)r	   sliceorder_byinclude_archivedinclude_deleted)r   zCmfFullSearch.reindex_models: :, z0.3z sec)cmfr   	CmfEntityiter_subclassesr!  
class_namecmfutilget_model_by_namefull_search_preload_fieldsr   r^   rv   r   r   valuefull_search_indexr   r   r   r   r   )r   Zmodels_listZcommit_everyZlazyZ	model_clsrh   offsetmodelZ
field_listtsobj_listr   rO   rO   rV   reindex_models  s8    



zCmfFullSearch.reindex_models)	only_oncec           
      C   s   | r|st ddd td tj  tj}|jj}|	|}|j
}| }d}| rZd}td| dd	dd
 d| d| d}||}	t|	}	|  t|	rtdd	dd
 qpt  qqpd S )Nu   Поиск после обновления должен работать сразу, с приемлемым качеством без переиндексацииTr  zRun run_force_reindexr   r   zrun_force_reindex Process z=true: r   )endflushz
                WITH cte AS (
                SELECT
                    id as id
                FROM   cmf_full_search
                WHERE  z=false AND part_no=0
                ORDER BY dirty_at DESC
                LIMIT  1000
                )
                UPDATE cmf_full_search s
                SET
                    zt = true
                FROM   cte
                WHERE  cte.id = s.id
                RETURNING s.id;
            .)r   r   r   r   CmfAccessListcheck_admin_moderv   r   r   r   r   r   printr   r^   r   r   )
Z
from_patchZi_do_manual_migrationr   r   r   r   r/   Zdirty_fieldr  r   rO   rO   rV   r   .  s2    



zCmfFullSearch.run_force_reindexc                 C   s(   t j  |  }| jdd}||dS )NTr   )totaldirty_count)r   r<  r=  r  )r   r@  rA  rO   rO   rV   r   n  s    
zCmfFullSearch.index_statsz	@minutely   )r8  Z
system_jobZschedulepriorityc                  C   s  t d tj r"t d d S td t } d}g }tjj	dgdddgdd	dd
gddd
ggdddt
j
 t
jdd gddd ggdddt
j
 gddd gggdgddgd}|sqi }|D ]0}t|j}||krg ||< || |j q| D ]\}}t d| dt|  t| |D ]}|j|jdd|gd
d}	|d7 }|	sxt d|  tj| q.t d|	j  z|	  W nt tk
r
 }
 zTt d| d|	 d|
  t t  |||
g t  tj| t  W 5 d }
~
X Y nX q.qt |  dkrvt
j
  j}|dkr`|dk r`t |  dkrvqnt d| d  qt  t d!| d" td q<|rt d# |D ]\}}
t | d$|
  q|d d t d%t |    d S )&NzStart cron_index_dirtyuJ   Не индексируем поскольку запущен импорт   r   r8   r   =ORr   Tr   r   <   )Zminutesr   z	-dirty_at   )r	   filterr%  r$  zcron_index_dirty process r   r   )r	   rJ  r'  r   u.   Объект не найдет в бд obj_id=zobj_id=u#   ERROR! При индексации u"    произошла ошибка:    iX  ua   cron_index_dirty превышен лимит времени 20 секунд обработано u    объектовu&   cron_index_dirty обработано u    объектов, sleep 1u2   При индексации были ошибки:: zEnd cron_index_dirty at ) r   r   r.  Zenable_import_modeZimport_is_runningr   sleepr   rv   Zslistr   r1   r   Zget_model_by_idr8   appenditemsr   r>  r   r0  r   r   r2  	Exception	traceback
format_excZcmf_rollbackr   Z
cmf_commitZ
astimezonehour)stn
exceptionsZobj_id_listZobj_ids_by_modelr8   r4  Zid_listr   r   erS  rO   rO   rV   cron_index_dirtyz  s    





zCmfFullSearch.cron_index_dirtyc                 C   sN   |S ]:}|sq|d dkr4t|dks|d dkr4q|| qd|}|S )u   
        Подчистка оригинального квери, который ввел пользователь:
        - удаление стоп-слов
        r   r   -r   )r   r   rN  r   )r   search_queryZclean_search_query_listr   Zclean_search_queryrO   rO   rV   _clean_search_query  s     
z!CmfFullSearch._clean_search_queryrs   ztuple[str, set[str]])rZ  rM   c                 C   s   t  }| d}t|dkr$| |fS d} |dd D ]Z}|dkrBq4td|d}|d dkrj||d  t|dkr4|  |d  |d  } q4|  } | |fS )	zExtrats tags from the given search_query and returns its reminder and a set of extracted tags

        Args:
            search_query (str)

        Returns:
            tuple[str, list[str]]: search_query reminder and a set of extractd tags
        #r   r   Nz(\W)r      rD  )setr   r   r   addstrip)rZ  r5   Zsharp_splittedtokenZ
sub_tokensrO   rO   rV   _extract_tags  s    

zCmfFullSearch._extract_tags)rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   resultsc                 K   s  |dkrd S |dks|dkrb| j dgfttjj|d|
d||_| j dgf|d|
d||_|dksr|dkr| j dgfttjj|d|
d||_| j dgf|d|
d||_|dks|dkr| j |fttjjddgd	d
|
d||_	| j |fddgd	d
|
d||_
d S )Nr   ANYCmfTaskZFA)related_person_loginri   r>   r$  )ri   r>   r$  CmfDocumentre  rg  r2   ZFS)rf  model_name_not_inri   r>   r$  )ri  ri   r>   r$  )filter_oncers   r   current_userr~   rc   rd   ra   rb   re   rf   r   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rc  r   rO   rO   rV   &_fulltext_search_execute_empty_queries  s    


	z4CmfFullSearch._fulltext_search_execute_empty_queriesc                 K   s  |dks|dkr8| j |||f|
dddgd|d||_|dksH|dkrj| j |dg|f|
d|d||_|dksz|dkr| j |dg|f|
d	d
||_| j |dg|f|
dd
||_t|jt|j dk r| j |dg|	f|
dd
|dd |_|dks|dkr| j |dg|f|
d	d
||_| j |dg|f|
dd
||_t|jt|j dk r| j |dg|	f|
dd
|dd |_	d S )Nrd  rh  re  rg  CmfAttachmentS)r$  ri  r>   
like_query)r$  r>   rp  A)r$  r>   rI  r   r   )
search_oncerJ   rK   rG   rH   r   rI   rD   rE   rF   rl  rO   rO   rV    _fulltext_search_execute_queriesS  s     
 z.CmfFullSearch._fulltext_search_execute_queriesc           1         s&  t |dkrtddd |d kr$d}d|kr4d|d< d|krDd|d< |sPd	d
g}tjjdd}|dkrttd|  dt_|}t }|	d}|rt
|tr|D ]}|| qn
|| | |\}}t||}||d< |}| }|}t|}|dkrd}t|}d	|d	< |dr0|d d }|dkrRdd tjj D }n8t|tjkrd}d}dd tjj D }n|g}|	d}| ||d< t d|kr|d rt|d | j|dd|	d}| j|d|	d}| j|d|	d}| j|dd|	d}|t_t|||||||||||
d}g }t }|
rdd tjj D }t }|dkrlg S |dkrzg S |dks|dkr| jd dg|fd	d
gd!||d"dd#|} ng } |dks|d$kr| jd d$g|fd	d
gd!||d"dd#|}!ng }!d%}"|dkrd&}"|dks(|d'krj|g}#|dkr<|}#| jd |#|fdd$d(gd	|"gd!||d)|}$ng }$t | d*krtd+t |   t }t }g } fd,d- d.d/ }%d	d	d	d0d1d1d2}&d	d%d3}'t| d d4 d5d6 d7| d4d   } t|!d d4 d8d6 d7|!d4d   }!t |}( ||&|   ||&|   ||&|!  ||&|! |%||'|$  ||&|   ||&|   ||&|!  ||&|! t |})|)d9ks|)|(kr$qq$|d d9 }| j |d	d9g|||||f|}*t | d*krtd:t |   |*S |dkrRt! }+t }| j"f t#|d;|+i| t | d*krptd<t |   t }d	},t$|+% r|+j&d=f|+j'd=f|+j(d=f|+j)d=f|+j*d=f|+j+d=ff}-| ,|-|| |,d=7 },|,d>kr|d?-d@d |+% D }.tdA|.  qq|t | d*kr8tdBt |   | j |||||||f|S t. }+t }| j/f t#|d;|+i| t | d*krtdCt |   t01|+D ]D}/t2|+|/j3}0t|0d d4 dDd6 d7|0d4d   }0t4|+|/j3|0 qt }d	},t$|+% r|+j5d4f|+j6d4f|+j7d=f|+j8d=f|+j9d=f|+j5d=f|+j9d=f|+j7d=f|+j8d=f|+j:d=f|+j6d=f|+j:d=f|+j7d=f|+j8d=f|+j;d=f|+j<d=fg}-| ,|-|| |,d=7 },|,d>kr d?-dEd |+% D }.tdA|.  qq t | d*krtdFt |   | j |||||||f|S )GNi   uY   Превышена максимально допустимая длина запроса!Tr  r   archivedFdeletedr   r   r?     u   Идет процесс индексации, могут быть доступны не все результаты поиска. Осталось объектов: tag_namerd  r3   ZModelc                 S   s   g | ]}|j r|jqS rO   r!  r-  rS   mrO   rO   rV   rW     s      z1CmfFullSearch.fulltext_search.<locals>.<listcomp>r6   c                 S   s   g | ]}|j r|jqS rO   ry  rz  rO   rO   rV   rW     s      ry   tree_parent_filterrecent_projects)synonymsrm   	query_raw)r~  r  )r~  
first_wordr  )rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   c                 S   s   g | ]}|j r|jqS rO   ry  rz  rO   rO   rV   rW     s      re  r4   ZTOPi  )r$  r>   text_stop_wordsrp  r@   include_attachmentrg  r]  r   rh  rn  )ri  r$  r>   r  rp  g?z'PROF fulltext_search TOP25 selects got c                    s   t tjj}|sd S |d}d}|jr8||jkr8d}n|jrL|jkrLd}| tj|j	< || || d kr~ | ||S | 
| ||  d7  < d S )Nr   r   r   bZ_limitr   )rs   r   rk  r~   popr  r   upperfulltext_search_debug_labelr8   rN  )r   countersr6  
user_loginr   Z	cur_classappend_if_existsr}  rO   rV   r  X  s    

z7CmfFullSearch.fulltext_search.<locals>.append_if_existsc                 S   sP   |d |d krd S |sd S | d}dtj|j< | | |d  d7  < d S )Nr  total_limitr   r,   r   )r  r   r  r8   rN  )r   r  r6  r   rO   rO   rV   append_if_exists_otherm  s    

z=CmfFullSearch.fulltext_search.<locals>.append_if_exists_otherrB     )r   r  r   Za_limitZb_limitZc_limit)r  r     c                 S   s   | j S Nr@   r   rO   rO   rV   <lambda>|      z/CmfFullSearch.fulltext_search.<locals>.<lambda>)keyc                 S   s   | j S r  r  r  rO   rO   rV   r  }  r  rI  z2PROF fulltext_search TOP25 mixing and prepare got rc  z/PROF fulltext_search empty_queries selects got r   r   r   c                 S   s   g | ]}t t|qS rO   rs   r   rS   frO   rO   rV   rW     s     uX   Баг в поиске, много данных, либо не идет вычитка: z:PROF fulltext_search empty_queries mixing and prepare got z&PROF fulltext_search main selects got c                 S   s   | j S r  r  r  rO   rO   rV   r    r  c                 S   s   g | ]}t t|qS rO   r  r  rO   rO   rV   rW     s     z1PROF fulltext_search main mixing and prepare got )=r   r   r   rv   r  Zcmf_noter   FSTr^  r   
isinstancer^   r_  rb  unionr   r.  Zninjaendswithr*  r+  r,  r/  Z
CmfComment_get_all_branchesprepare_search_queryFSTQrg   r   rr  r   r   _prepare_final_resultr`   rm  varsr+   rY   rc   rd   ra   rb   re   rf   _add_if_existsr   rA   rs  dataclassesr	   rQ   r4   setattrrG   rD   rK   rJ   rH   rE   rI   rF   )1r   rh   ri   rZ  only_idsr$  r	   no_analitycscheck_accessr  rr   r   rA  rj   r5   rw  Z_tagZextracted_tagsorig_search_queryrk   rq   rl   ry   rm   rn   ro   rp   Zsearch_paramsfinal_resultskip_idsZall_model_namesr   Zresult_tasksZresult_docsZother_sliceZother_model_filterZresult_otherr  r  Zcounters_otherZres_count_beforeZres_count_afterresultrc  Z
iter_countZprocessing_groupZ
found_lensr  datarO   r  rV   r     s   








        

  
  
$$

$
zCmfFullSearch.fulltext_searchc                 C   s   d| kr| S |  dd S )Nz@#@#@#r   )r   )r   rO   rO   rV   _remove_suffix_from_headline  s    z*CmfFullSearch._remove_suffix_from_headlinec                 K   s  ||d |d  }|ri }	g }
|D ]}|
 |j |jrF| |jnd}d}|jdk	rb|jd}|j}|jtjkr|d tj|j  }|j|j	|j
| d| d	| d	|jd
dtj 	t|j||j|jd|	|j< q$|	t_td |
S i }|r|dd}|dd}i }|D ]}||jg  |j q| D ]}tt|j jrx|dg }dd|| gdddgg}n|}dd|| g}|dkr|dddg }t|j||||d}|D ]8}tt|j jr|jr|||jj< n
|||j< qq>ntd td g }|D ]}||j}|r|jrD| |jnd}d}|jdk	rb|jd}|j}|jtjkr|d tj|j  }|jdkr
|jsq|j |jj!|jj"j#| d| d	| d	|jd
dtj 	||j|j|jj#|j	|j$|j%|j&t|jd}nN|jj#|j	|j"j#| d| d	| d	|jd
dtj 	||j|jt|jd}zf|sp|j'ddd  n||s~t(|D ].}|)d!d }t*||j |d}|||< q| | W n t(k
r   Y nX qtd |st+tj,j-|t.|dd" d#d$ |S )%u5   
        Итоговая обработка
        r   r   r   NoneNz.6frY  z ||| r   z.0fz words=)r   r4   r}   r<   r=   r>   r?   r@   zfulltext_search ENDrt  Fru  cmf_ver_headINZcmf_ver_curz==Tr   rn  urlurl_previewurl_preview_img)r	   rJ  r&  r'  uY   DEV: FATAL. Укажите в запросе поиска список полей fields=z"fulltext_search Start check access)r   r4   r}   r<   r>   r?   r@   Z	attach_idZattach_nameZ
attach_urlZattach_url_previewZattach_url_preview_imgr=   )r   r4   r}   r<   r>   r?   r@   r=   )ZTEXKOM_skip_failread_auditZTEXKOM_ppp_project_simplecheckr;  rI  )rZ  obj_dict)r   )/rN  r8   r<   r  r?   r>   r   r  r   r;   r:   r@   r  Zjsonloadsr=   fulltext_search_headlinesr   r   
setdefaultr9   r   r  r   Zcmf_verr.  r/  r^   r  r   r-  parentrx   r4   r}   r1  r  r  r  Z_acl_check_readZCmfPermissionErrorr   rQ   r   rv   _do_calc_statisticsZ
dumps_dict)r   r  r$  r  r	   r  r  r  r   r  Z	result_idrr<   Zformated_rankr>   r   ZobjectsZis_archivedZ
is_deletedZids_by_modelrh   _fieldsZ_filterr6  r   r  r   attrrO   rO   rV   r    s    

$







$
$



z#CmfFullSearch._prepare_final_result)r6  r  r  c                 C   sj   |D ]`\}}t |D ]N}|s  q|d j|kr:|d q||d  ||d j |d qqd S )Nr   )r   r8   r  rN  r_  )r   r6  r  r  Zlstr  r   rO   rO   rV   r    s    
zCmfFullSearch._add_if_existsr   c           -      K   s  |d krdg}|sddg}| dd }| d}| dd }| dd }| dd }| d	}| d
}| dd }| d}| d}| dd }| d|}|p|}| d}|rdd|nd}d}|
rd}| dpg }| |\}}| |\}}|dkrd }|	dkrd }	d } |	rJtd|	sBtd|	rJ|	 } |tkrft	d| dd d| }!d| }"|dkrd}"| d}#|d }$|d |d  }%d}&|rt
|}d | d!}&tjjj d" d}'|d#krd$}'td%d&|}(td%d'|})d(}*|d)krd*}*d+jf ||!|#|&|"||*||d,	}+tjjj |+||(|)|$|%t|t||||	|'|||||||||||| |d-||},t|,S ).Nnull000r   r   rx   modified_by_nameresponsible_namelogic_type_codestatus_typemodified_at_beforemodified_at_afterrt  ru  user_rating
owner_nameauthor_namer|   AND obj_tree_parent_id IN ('{}')','r   zOOR (obj_model = 'CmfAttachment' and obj_parent_id ilike :model_name_in || ':%')rw  z^[a-zA-Z0-9]+-[0-9]+$z^[0-9]+$6   Недопустимое значение field_name: Tr  obj_r4   r   Z	_tsvectorr   $obj_modified_at > now() - interval '
 days' ANDz"set gin_fuzzy_search_limit=100000;r2   @B z([&] )([^!])z<1> \2z<2> \20)r2   r3   r6   z1|16u"  
            SELECT
                subs.*,
                ts_headline(
                    'russian',
                    substring(subs.{search_field} FROM 0 FOR 100000),
                    :tsquery,
                    'MaxFragments=1,MaxWords=25,MinWords=24'
                ) AS headline
            FROM (
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        obj_breadcrumbs as breadcrumbs,
                        obj_project_id as obj_project_id,
                        obj_related_person_logins as obj_related_person_logins,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 3*365 then 'R' else '' end) as label,
                        (ts_rank_cd({tsvector_field}, :tsquery, {ts_rank_normalize}))
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 3*365 then
                                    3*((3*365-EXTRACT(days from (now()-obj_modified_at)))/30+1) else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        (({tsvector_field} @@ :tsquery)  or ( {search_field} ILIKE '%' || :like_query || '%' ))
                        and ((obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in) {include_attachment_filter})
                        and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {rpl_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                        and ( :user_rating is null or obj_user_rating >= :user_rating)
                    ORDER BY rank desc
                    LIMIT :slice_for OFFSET :slice_from
                )
            UNION
                /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        obj_breadcrumbs as breadcrumbs,
                        obj_project_id as obj_project_id,
                        obj_related_person_logins as obj_related_person_logins,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        100
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                        ((:obj_code is not null and obj_code ILIKE '%' || :obj_code)
                            or obj_id = :like_query
                        )
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {rpl_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                    /*ORDER BY obj_id desc -- не будем делать, т.к. тормозит*/
                    LIMIT :slice_for/10
                    OFFSET :slice_from/10
                )
            ) as subs
            ORDER BY subs.rank desc
            LIMIT :slice_for
            OFFSET :slice_from;
        )	r|  search_fieldtsvector_fieldage_days_subqueryheadline_fieldtags_filterts_rank_normalizeinclude_attachment_filter
rpl_filter)tsquerytsquery_str_h1tsquery_str_h2
slice_from	slice_formodel_name_inri  r>   r  rp  
max_partnorx   r  r  r  r  r  r  r  r  rt  ru  r:   r  )r   formatr   _build_tags_filter#_build_related_person_logins_filterr   matchr  ALLOWED_FIELDSr   rt   r   rv   r   r   r   r   subtupler^   )-r   ri   r  Ztsquery_strri  r@   r>   r$  r  rp  r  rf  r   rx   r  r  r  r  r  r  rt  ru  r  r  r  r|  r  
tags_namesr  tags_paramsr  
rpl_paramsr:   r  r  r  r  r  r  r  r  r  r  r  
found_objsrO   rO   rV   rr    s    




	


"





	

bmzCmfFullSearch.search_oncer  r   z
str | None)related_userrf  c	           %      K   s  |	 d}
|	 d}|	 d}|	 d}|	 d}|	 d}|	 d}|	 d}|	 d	}|	 d
}|	 dd }|	 d|}|p|}|	 d}|rdd|nd}|	 dpg }| |\}}| |\}}d}|rtd | }}d}|tkrtd| dd d| }|d }|d |d  }d} |rHt|}d| d} |dkrVdnd}!d| d | d!| d"|  d#| d$| d$| d$| d%}"tj	j
j |"||t|t|||!|
|||||||||||d&||}#t|#}$|$S )'Nrx   r  r  r  r  r  r  rt  ru  r  r  r  r|  r  r  r   rw  z
            AND ( :responsible_name IS NULL OR obj_responsible_names ILIKE '%' || :responsible_name || '%' )
            AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
        z:Param related_user is deprecated. Use related_person_loginz
            AND (
                :responsible_name IS NULL
                OR obj_responsible_names ILIKE '%' || :responsible_name || '%'
                OR obj_owner_name = :owner_name
            )
        r  Tr  r  r   r   r  r  r2   r  a  
            SELECT
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                obj_breadcrumbs as breadcrumbs,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                z as z,
                substring(z` FROM 0 FOR 240) as headline
            FROM cmf_full_search
            WHERE
                a  
                obj_model IN :model_name_in AND obj_model NOT IN :model_name_not_in
                AND part_no <= :max_partno
                AND ( :parent_id IS NULL OR obj_parent_id = :parent_id )
                AND ( :author_name IS NULL OR obj_author_name = :author_name )
                AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                z
                a  
                AND ( :logic_type_code IS NULL OR obj_logic_type_code = :logic_type_code )
                AND ( :status_type IS NULL OR obj_status_type = :status_type )
                AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                AND ( :archived IS NULL OR obj_archived = :archived )
                AND ( :deleted IS NULL OR obj_deleted = :deleted )
                and ( :user_rating is null or obj_user_rating >= :user_rating)
                AND obj_modified_at is not null
            ORDER BY obj_modified_at DESC
            LIMIT :slice_for OFFSET :slice_from;
        )r  r  r  ri  r>   r  rx   r  r  r  r  r  r  r  r  rt  ru  r  )r   r  r   r  r  r   r  rt   r   rv   r   r   r   r   r  r^   )%r   r  r  rf  ri   ri  r@   r>   r$  r   rx   r  r  r  r  r  r  rt  ru  r  r  r  r|  r  r  r  r  r  Zowner_and_responsible_filterr  r  r  r  r  r  r  Zall_objsrO   rO   rV   rj    s    















$zCmfFullSearch.filter_oncez	list[str]ztuple[str, dict[str, str]])r  rM   c                 C   s8   i }d}| r0d}dd dd | D  d|d< ||fS )	Nr   z AND obj_tags ~* :tags_re(^| |,)(|c                 s   s   | ]}t |V  qd S r  r   escape)rS   rw  rO   rO   rV   	<genexpr>  s     z3CmfFullSearch._build_tags_filter.<locals>.<genexpr>)( |,|$)Ztags_re)r   )r  r  r  rO   rO   rV   r    s     z CmfFullSearch._build_tags_filter)r  rM   c                 C   s2   i }d}| r*d}t | } d|  d|d< ||fS )Nr   z; AND obj_related_person_logins ~* :related_person_logins_rer  r  Zrelated_person_logins_rer  )r  r  r  rO   rO   rV   r    s    
z1CmfFullSearch._build_related_person_logins_filterrL   c                 C   s0   | sg S t jjj dd| i}dd |D S )Na  
                WITH tree_parents AS (
                    WITH RECURSIVE r AS (
                        SELECT obj_id, obj_code, obj_tree_parent_id
                        FROM cmf_full_search
                        WHERE obj_tree_parent_id = :tree_parent_id

                        UNION

                        SELECT cfs.obj_id, cfs.obj_code, cfs.obj_tree_parent_id
                        FROM cmf_full_search AS cfs
                        JOIN r ON cfs.obj_tree_parent_id = r.obj_id
                    )
                    SELECT obj_id FROM r
                    WHERE r.obj_id IN (SELECT obj_tree_parent_id FROM r)

                    UNION

                    SELECT :tree_parent_id
                )
                SELECT * FROM tree_parents;
            ry   c                 S   s   g | ]}|d  qS )r   rO   )rS   r  rO   rO   rV   rW   H  s     z3CmfFullSearch._get_all_branches.<locals>.<listcomp>)r   rv   r   r   r   r   )ry   ZrecordsrO   rO   rV   r  '  s    zCmfFullSearch._get_all_branchesc              
   C   sT  dt _|dd}tdd|}td|}d}d}	d}
|D ]}t|dkrPq<t|dkr|dkrfq<|d	kr~|
d| 7 }
q<|dkrq<|d
kr|
d7 }
q<|dkr|
d7 }
q<t|dkrq<|d dkrt|dkrq<|
d|dd   7 }
|	d|dd   7 }	q<|rq<|dd}|d7 }|dkr.d}|
rB|
d dkrJ|
d7 }
t|dkr| j||d}t|dkr|
|d  7 }
n|
dd| d 7 }
|r|dkr q|dkr< qq<q<|rt|	dkr|	d dkr|	dd  }	|	S |
dddddd d!d 	 }
|
r<|
d dkr<|
dd  }
|
r\|
d dkr\|
dd  }
|
r||
d dkr||
dd  }
|
r|
dd  dkr|
d d }
z.t
jjj d"d#|
i}t|d d }
W nn tjjk
r< } zJt
jjj  |std$ t
jjj d%d#|i}t|d d }
W 5 d }~X Y nX t|d&|
 |
t _|
S )'Nr   zwww.u   [^-A-Za-zА-Яа-я0-9()|&!' ]r   z(,| |&|\||\(|\))r   r   )rY  !z()&|)r   r   r  z |)r   r   &z &rD  z& !r  rY  r  F)r  r  z& )r~  z( z | z )
   )r  r  z OR z or z AND z & z and z!select to_tsquery('russian', :q);qu7   Ошибочный синтаксис в запросеz+select websearch_to_tsquery('russian', :q);z->)r   r  replacer   r  r   r   prepare_wordr   r`  r   
CmfSynonymr   r   r   r   r^   r   excZProgrammingErrorZrollbackr   r>  r  )r   rZ  r~  rm   r  r  Zsearch_query_allowed_symbtokensZ
word_countZstopsr  r0   Z	sug_wordsr  rW  rO   rO   rV   r  J  s    

("z"CmfFullSearch.prepare_search_queryc                 C   s  t  jd| d7  _|d tjkr0d}tj}n
d}tj}g }t|s||}g }t	|}|rt  jd| d7  _|
| t  jd7  _d}	|D ]}
|	d	kr qHt|
d	krq|
d |d kr|
d
 |d
 krqd|
kr|
dd}
t  jd|
 d7  _|
|
 |	d
7 }	qt  jd|
 d7  _|
|
 |	d
7 }	qg }tjjj dd|i}d}	|D ]\}}|	d
kr qt|d	krqn|d |d ks|d
 |d
 krn|dd}t  jd| d7  _|
| |	d
7 }	qnt|t|B |hB }n|h}t }|D ]D}
t|
d d	 D ]*}||j t  jd|j d7  _q2q||B }t }|rtjjddt||hB gddgdgdd
gd}|D ]\}|jr|jjdd d D ]6}| dd}t  jd| d7  _|| qʐq||B t|B }t|S )Nz|w:rL  r   enruzaddNinjaRevers r)  zspellError, rD  r   r   z<->z	addSpell z
            SELECT
                name, similarity(:word, name) as sim
            FROM cmf_synonym
            WHERE
                :word % name
            ORDER BY "sim" desc
            LIMIT 5;
             wordzaddSpellTrgm z
normalize r4   r  r2   Zorderno)rJ  r	   r%  r$  ,rH  zsynAdd )r   r  stringascii_lettersr.  Zdictionary_enZdictionary_ruZdictionary_checkZsuggestZninja_reversrN  r   r  r   r  r   r   r   r   r^  morphparser_  Znormal_formr^   r2   r1  r   r`  )r   r  r~  langZ
dictionaryZfiltered_suggestions3ZsuggestionsZfiltered_suggestionsZnwr   r   Zfiltered_suggestions2Zsuggestions2_listZsuggr   Zall_suggestionsZnormalized_wordsZsynonym_wordsZsynonym_listZsynonymr/   rO   rO   rV   r    s    


 



	 
$
 zCmfFullSearch.prepare_wordc              	   C   s   t | dk rd S g }|d d D ]b}|d dsB|d dsBq i }|d |d< |dd rp|d d |d< nd |d< || q t }||_d	|_| |_t	j
j|_d
|_t  |  W 5 Q R X d S )Nr]  rI  r   zCmfDocument:zCmfTask:r8   r  rz   searchF)r   
startswithr   rN  r   ZCmfSearchStatr6  actionrZ  r   rk  r   Z	person_idZ
aggregatedr.  Zdisable_aclZsave)rZ  r  Zst_dictZrecZst_recstatrO   rO   rV   r  "	  s(    

z!CmfFullSearch._do_calc_statistics)F)FF)F)NNNr   NNNNNFFNNNNNNNNNNNNNNNNNNNN)Nr   F)FF)FNNFNFN)NNr   NNNFN)TFFF)T)8rZ   r[   r\   r]   r  r"  Z	api_allowZapi_methodsstaticmethodrs   r   classmethodr   r   r   r   r   r   r   r   r   r   r  r
  r  r  r*  r   Z	BaseModelr#  r7  Zcmf_deferred_jobr   r   rX  r[  rb  r   rt   ru   rA   rm  rs  r   r  r  r   r  rr  rj  r  r  r  r  r  r  rO   rO   rO   rV   rv   o   s                    




	9

                                q


 !>
Y
!GN               R

                s 
"hnrv   ))r   r   collectionsr   typingr   r   r   r   r  r   r   r	   rQ  sys	itertoolsr   Zcmf.data_providers.sqlalchemyr   Zcmf.fields.cmf_full_searchr*  Zenchantr  Z	pymorphy3r   r   Zbs4r   ZMorphAnalyzerr  ZRUSSIAN_STOP_WORDSZENGLISH_STOP_WORDSr^  r   r  r7   rA   r`   rg   Zcmf_full_searchrv   rO   rO   rO   rV   <module>   sT    3 ?        $