
    *`mhS                        d dl Z d dlZd dlmZ d dlT d dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ  ej                    Z G d dej        j        j                  ZdS )    N)
namedtuple)*)SQLAlchemyDataDriver)BeautifulSoupc                   Z   e Zd ZdZdZg dZdZg dZede	fd            Z
ed             Zed	             Zed
             Zed             Zed6d            Zed7d            Zed             Zed             Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d8d            Zed             Zed             Zedej        j        fd            Zed9d            Ze ed          d                         Zed             Ze edddd          d                         Zed              Z ed:d%            Z!e	 d;d&            Z"e	 	 d<d(            Z#eddd)gdd'd*dgf	 d=d-            Z$ed>d1            Z%ed?d2            Z&ed@d3            Z'edAd4            Z(eed5                         Z)dS )BCmfFullSearchuC   
    Сервис полнотекстового поиска.
    russian)nametext_rendertext
text_drafttags	parent_idtree_parent_id
project_idcmf_created_atcmf_modified_atcodecmf_deletedcmf_archivedloginemailphonephone_internalphone_mobilephone_2phone_assistant
ip_addressemail_2birthdayT)fulltext_searchrun_force_reindexindex_statsr   c                 ,   | rt          | d                              d          } t          |           dk    r[t                              dt          |            d           |                                 d d                             d          } | S )	Nhtml.parser
i  z)CmfFullSearch.clean_text: trunc text len z to 1mbi  ignoreerrors)r   get_textlengdebugencodedecode)r   s    ./cmf/models/cmf_full_search.py
clean_textzCmfFullSearch.clean_text0   s      	H }55>>tDDD4yy7""VCIIVVVWWW{{}}YhY/66h6GG    c                     t          |d                              d                              d                                          S )Nr%   r&   r'   r(   )r   r*   r.   r/   )clstext_with_htmls     r0   
strip_htmlzCmfFullSearch.strip_html@   s>    ^];;DDTJJQQYaQbbiikkkr2   c                     ddt                    k     rgS fdt          dt                              D             }|S )Ni d   c                 0    g | ]}||z   z            S  r:   ).0ioverlapr   text_len_maxs     r0   
<listcomp>z-CmfFullSearch._strip_size.<locals>.<listcomp>T   s,    YYY!tAanW,,-YYYr2   r   )r+   range)r4   r   resr=   r>   s    ` @@r0   _strip_sizezCmfFullSearch._strip_sizeD   s_     t99|##6MYYYYYYuQD		<7X7XYYY
r2   c                 f    d                     d t          j        d|          D                       S )N c                 8    g | ]}t          |          d k     |S )2   )r+   r;   ws     r0   r?   z+CmfFullSearch._strip_50.<locals>.<listcomp>Y   s#    IIIqSVVb[[[[[r2   z\sjoinresplitr4   r   s     r0   	_strip_50zCmfFullSearch._strip_50W   s/    xxIIBHUD$9$9IIIJJJr2   c                 f    d                     d t          j        d|          D                       S )NrD   c                     g | ]}|S r:   r:   rG   s     r0   r?   z1CmfFullSearch._strip_not_word.<locals>.<listcomp>`   s    KKKqKKKr2   z[ \s<>|&\/%@!`+=;.]rI   rM   s     r0   _strip_not_wordzCmfFullSearch._strip_not_word[   s2    
 xxKKBH-CT$J$JKKKLLLr2   Fc                 :   t          j                     }t                              d|            |st          d          dd l}| j        j        }|                    |           }|j        }|	                                }|
                    |j        j        g                              |j        j        |k                                  |j        j        dk                                              }	|                    |	                                          }
|
r|
d         }|                                                    |j        j        |k                                  d|j                                                  }|                    |           nt                              d|            |                                 }|                                                    ||d|j                                        d          }|                    |           |r%ddlm}  |t6          j        j        d	|id
           t          j                     |z
  dk    r3t                              dt          j                     |z
              d S d S )Nzcmf_full_search: mark_dirty empty obj_idr   Tis_dirtydirty_atz#cmf_full_search: mark_dirty insert )idobj_idrU   rV   part_no)schedule_deferred_jobrX      )kwargs	countdowng?zPROF mark_dirty() got )timer,   r-   
ValueError
sqlalchemydpdata_driverdp_model_cls	__table__SessionselectcrW   whererX   rY   with_for_updateexecutefirstupdatevaluesfuncnowgen_idinsertcmf.includerZ   models	CmfPersoncelery_full_search_index)r4   rX   
fast_indexprof_stsaddsa_modeltablesget_stmtget_resid_update_stmtinsert_stmtrZ   s                  r0   mark_dirty_deferzCmfFullSearch.mark_dirty_deferb   sb   )++	7v77888 	._---V??3''"JJLL IIuwzl##uw~/00uw!+,, ""	 	 ))H%%++-- 	#!*CU57:,--V!W[[]]     IIk""""GGB&BBCCC**,,CV!!W[[]]     IIk""" 	u999999!!&"2"KU]_eTfrstttt9;; 4''GGDTY[[7-BDDEEEEE ('r2   c                 Z    |r|                      ||          S d|it          j        |<   d S )Nrv   rv   )r   r,   deferred_fullsearch_dirty_list)r4   rX   rv   forces       r0   
mark_dirtyzCmfFullSearch.mark_dirty   s<     	G'':'FFF4@*3M(000r2   c                 b   t           j        sd S t           j        }t          |          dk    rt          dt          |                      t	          |                                          D ]4}||         }|                     ||                    d                     5i t           _        d S )NrF   ua   DEV: CmfFullSearch помечено грязными слишком много объектов: rv   r   )r,   r   r+   	cmf_alertsortedkeysr   get)r4   
dirty_listrX   r\   s       r0   apply_deferred_dirtyz"CmfFullSearch.apply_deferred_dirty   s    / 	F5
z??R  Lz}  I  {J  {J  L  L  M  M  MZ__..// 	N 	NF'F  FJJ|4L4L MMMM+-(((r2   c                    |st          d          dd l}| j        j        }|                    |           }|j        }|                                }|                    |j        j	        g          
                    |j        j        |k              
                    |j        j        dk                                              }|                    |                                          }|r~|d         }	|                                
                    |j        j	        |	k                                  d|j                                                  }
|                    |
           | j                                         d S )NrS   r   FrT   )r_   r`   ra   rb   rc   rd   re   rf   rg   rW   rh   rX   rY   ri   rj   rk   rl   rm   rn   ro   commit)r4   rX   rx   ry   rz   r{   r|   r}   r~   r   r   s              r0   
mark_cleanzCmfFullSearch.mark_clean   sH    	._---V??3''"JJLL IIuwzl##uw~/00uw!+,, ""	 	 ))H%%++-- 
	#!*CU57:,--V"W[[]]     IIk"""r2   Nc                    |}|                      |          }|                     |          }|rF|                     |          }|                     |          }|                      |d                   }nd g}d }|r-|}|                      |          } |                     |           } nd }d } |r-|}|                      |          }!|                     |!          }!nd }d }!|r5|                     |          }"|"d d         }"|                      |"          }#nd }"d }#|r3|                     |          d d         }|                      |          }$nd }$|r|}|}%nd }d }% | j        d'i ddd|d|d|d|d|d	|d
|d|d|	d|d|d|d|d|d|
d|d|d|d|d         d|d|d|d|d| d|d|!d|"d|#d |d!|$d"|d#|d$|% t	          |          d%k    rt          |d%d          d%          D ]\  }&}'|                      |'          } | j        d'i d|&d|d|d|d|d|d	|d
|d|d|	d|d|d|d|d|d|
d|d|d|d|'d|d|dd dd dd dd dd dd dd d |d!|$d"|d#d d$d  |                     |t	          |          &           | j                                         d S )(Nr   i rY   rX   	obj_modelobj_parent_idobj_tree_parent_idobj_project_idobj_created_atobj_modified_atobj_codeobj_deletedobj_archivedobj_owner_nameobj_author_nameobj_modified_by_nameobj_responsible_namesobj_hrefobj_logic_type_codeobj_activity_codeobj_status_typeobj_texttext_for_vecobj_namename_for_vecobj_tagstags_for_vecobj_result_textresult_text_for_vecobj_commentscomments_for_vecobj_addon_fieldscf_text_for_vecobj_user_ratingobj_key_phraseskey_phrases_for_vec   )delete_from_partnor:   )	rQ   rN   rB   _text_search_sql_insertr+   	enumerate_text_search_sql_delete_partnora   r   )(r4   
model_namerX   r   r
   r   r   comments_textr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   obj_text_listr   r   r   obj_comments_textcomments_text_for_vecr   r   rY   obj_text_parts(                                           r0   index_objectzCmfFullSearch.index_object   s   > **400}}\22 	 }}T**HOOH55M..}Q/?@@LL!FML 	 H..x88L==66LLHL 	'-O"%"5"5o"F"F"%--0C"D"D"O"& 	) #m < < 1'6' :$'$7$78I$J$J!! $$(! 	#"}}-=>>wwG!112BCCOO"O 	'-O"1 #O"&## 	
 	
 	
AA 	
ff 	


 	
'-	
<N<N	
_m_m	
)>	
;J?	
 X	
 ,7;	
 FR\	
 cqbp	
 ,O		
 CWBV		

 #8"7	

 CK(	
 !4 3	
 HYGX	
 kzjy	
 #1%%	
 5AL	
 X	
 -9L	
 X	
 -9L	
 ,O	
 BUAT	
 +*	
 >S=R	
 .-	
 @O	
 ,O	
 ,O	
 BUAT	
 	
" }!!*3M!""4Eq*I*I  &"22=AA++   GG FF V`V` "/-DVDVgugu#1>CR? &X 4?; NZ\ kyjx %4O	 K_J^	
 +@*?
 KS( )<(; PaO` sB  sB +] :F &X 59D "T 15 %)D ?Cd "& 9= &6%5 HW %4O %)D ?Cd  " 	**6c-FXFX*YYYr2   c                    |st          d          dd l}| j        j        }|                    |           }|j        }|                                }|                    |                              |j	        j
        |k                                  |j	        j        |k              }|                    |           d S )NrS   r   )r_   r`   ra   rb   rc   rd   re   deleterh   rg   rX   rY   rj   )	r4   rX   r   rx   ry   rz   r{   r|   del_stmts	            r0   r   z,CmfFullSearch._text_search_sql_delete_partno@  s     	._--- V??3''"JJLL IIeU57>V+,,U57?&8899 	 	
		(r2   c#           	      n	   t                               d|            |st          d          dd l}#| j        j        }$|$                    |           }%|%j        }&|$                                }'|#	                    |&j
        j        g                              |&j
        j        |k                                  |&j
        j        |k                                              }(t!          |'                    |(                    })|)rt%          |)          dk    rt'          d| d| dd	           |)d         d         }* |&                                                    |&j
        j        |*k              j        d0i d
|ddd|d|d|d|d|d|d|d|d|d|d|d|d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|	d |
d!|d"|d#|d$|d%|d&|d'|d(|d)|d*| d+|!d,|#j        j                            | j        |"          }+|'                    |+           nt                               d-|            |                                 }* |&                                j        d0i d.|*ddd/|d
|d|d|d|d|d|d|d|d|d|d|d|d|d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|#j        j                            | j        |          d|	d |
d!|d"|d#|d$|d%|d&|d'|d(|d)|d*| d+|!d,|#j        j                            | j        |"          },|'                    |,           |*S )1Nz)cmf_full_search: _text_search_sql_insert rS   r   r   u   Для объекта z	(part_no=u   ) задублировались записи в поиске. Обратитесь в техническую поддержку!TabortrX   rU   Fr   r   r   r   r   r   r   r   r   r   r   r   name_tsvectortext_tsvectortags_tsvectorresult_text_tsvectorcomments_tsvectoraddon_fields_tsvectorr   r   r   r   r   r   r   r   r   r   r   r   r   key_phrases_tsvectorz0cmf_full_search: _text_search_sql_insert insert rW   rY   r:   )r,   r-   r_   r`   ra   rb   rc   rd   re   rf   rg   rW   rh   rX   rY   ri   listrj   r+   r   rl   rm   sqlrn   to_tsvector_fts_configrp   rq   )-r4   rY   rX   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rx   ry   rz   r{   r|   r}   r~   r   r   r   s-                                                r0   r   z%CmfFullSearch._text_search_sql_insertS  s   " 	
DFDDEEE 	._--- V??3''"JJLL IIuwzl##U57>V+,,U57?g-.._	 	 qyy**++ Z	#7||a  O6  O  OG  O  O  O  W[  \  \  \  \!*Q-CuwzS())% % % "6% #U	%
 (i% &X% &X% &X% %4O% ".% &6%5% #0-% (:'9% $2>% $2>%  %4O!%" #%&+"9"9#/<"X"X"X#%$ #%&+"9"9#/<"X"X"X%%& #%&+"9"9#/<"X"X"X'%( *,)@)@Re)f)f)f)%* ')fk&=&=coO_&`&`&`+%, +-&+*A*A#/Sb*c*c*c-%. &X/%0 !,1%2 ".3%4 $2>5%6 %4O7%8 *>)=9%: +@*?;%< &X=%> )<(;?%@ '8&7A%B %4OC%D %4OE%F %4OG%H *,)@)@Re)f)f)fI% P IIk""""GGOvOOPPP**,,C& & &s& #U& $G	&
 "6& (i& &X& &X& &X& %4O& ".& &6%5& #0-& (:'9& $2>&  $2>!&" %4O#&$ #%&+"9"9#/<"X"X"X%&& #%&+"9"9#/<"X"X"X'&( #%&+"9"9#/<"X"X"X)&* *,)@)@Re)f)f)f+&, ')fk&=&=coO_&`&`&`-&. +-&+*A*A#/Sb*c*c*c/&0 &X1&2 !,3&4 ".5&6 $2>7&8 %4O9&: *>)=;&< +@*?=&> &X?&@ )<(;A&B '8&7C&D %4OE&F %4OG&H %4OI&J *,)@)@Re)f)f)fK& R IIk"""
r2   objc                 l    |j         sd S | j        D ]"}|dk    r	||j        v r||         j        r dS #d S )Nr   T)full_searchrequired_fieldsfields
is_changed)r4   r   
field_names      r0   is_obj_need_reindexz!CmfFullSearch.is_obj_need_reindex  s_     	F - 	 	J...SZ''C
O,F'tt		 	r2   r8   c           
         t           j        j                                        D ]$}|j        s|j        }|r||vrd}t                              |          }|j        }|rdg}	 t          j	                    }	|
                    ||||z   gdgdd          }
|
sn|
D ]D}|r,t          j                            |j        j        d           0|                                 E|t!          |
          z  }| j                                         t&                              d| d| d	t          j	                    |	z
  d
d           &d S )Nr   rW   Tr   )r   sliceorder_byinclude_archivedinclude_deleted)r   zCmfFullSearch.reindex_models: :, z0.3z sec)cmfrs   	CmfEntityiter_subclassesr   
class_namecmfutilget_model_by_namefull_search_fieldsr^   r   r   r   rW   valuefull_search_indexr+   ra   r   r,   r-   )r4   models_listcommit_everylazy	model_clsr   offsetmodel
field_listtsobj_listr   s               r0   reindex_modelszCmfFullSearch.reindex_models  s   -==?? 	l 	lI( "-J z<<F--j99E1J $"V
lY[[ ::%!6,#67./%)$( &     # 0 0C 0,77D7QQQQ--////#h--'jjjfjjPTPYP[P[^`P`jjjjkkk'l +	l 	lr2   )	only_oncec                     t                               d           t          j                                         t          j        } | j        j        }|                    |           }|j	        }|
                                }|                                                    |j        j        dk                                  d          }|                    |           d S )NzRun run_force_reindexr   TrU   )r,   r-   rs   CmfAccessListcheck_admin_moder   ra   rb   rc   rd   re   rl   rh   rg   rY   rm   rj   )r4   ry   rz   r{   r|   r   s         r0   r"   zCmfFullSearch.run_force_reindex  s     	
()))--///"V??3''"JJLLllnn**57?a+?@@GGQUGVV			+r2   c                     t           j                                         |                                 }|                     d          }||dS )NTr   )totaldirty_count)rs   r   r   count)r4   r   r   s      r0   r#   zCmfFullSearch.index_stats  sN    --///		iii.. &
 
 	
r2   z	@minutely   )r   
system_jobschedulepriorityc                     t                               d           t          j                                        rt                               d           d S t          j        d           t          j                    } d}	 t          j        	                    dgg dg dd	d
dt          j
                                        t          j        d          z
  gg dggddg          }|sni }|D ]J}t                              |j                  }||vrg ||<   ||                             |j                   K|                                D ]\  }}t                               d| dt#          |                      t%          |           |D ]}|                    |j        dd|gd          }|dz  }|s=t                               d|            t          j                            |           ft                               d|j                    |                                 t          j                    | z
  dk    rt                               d| d           n4t                               d| d           t          j        d           t                               dt          j                    | z
              d S )NzStart cron_index_dirtyuJ   Не индексируем поскольку запущен импорт   r   TrX   )rY   =r   )rU   r  TORrV   <   )minutes)rV   r  N   )r   filterr   zcron_index_dirty process rD   rW   r  )r   r  r   r   u.   Объект не найдет в бд obj_id=zobj_id=   ua   cron_index_dirty превышен лимит времени 20 секунд обработано u    объектовu&   cron_index_dirty обработано u    объектов, sleep 1zEnd cron_index_dirty at )r,   r-   r   enable_import_modeimport_is_runningr^   sleeprs   r   slistdatetimero   	timedeltaget_model_by_idrX   appenditemsr+   printr   r   r   rW   r   )	stnobj_id_listobj_ids_by_modelrX   r   id_listr   r   s	            r0   cron_index_dirtyzCmfFullSearch.cron_index_dirty  s    	
()))%7799 	GG`aaaF
1Y[[%	 .44XJ*=*=*=*A*A*A+/-7h>O>S>S>U>UX`XjstXuXuXu>u,v,C,C,C+,.( ./G 5 	5 	5K  !% > >//>> 000.0$U+ '..v}===="2"8"8":": , ,wJEJJCLLJJKKKg" , ,C))5+CdTWY\L^pt)uuCFA ! VQT V VWWW,77<<< GG.cf..///))++++, y{{R"$$  Q|}  Q  Q  Q  R  R  RGGZQZZZ[[[JqMMMK%	N 	
;49;;r>;;<<<<<r2   c                     |S )u   
        Подчистка оригинального квери, который ввел пользователь:
        - удаление стоп-слов
        )rL   r+   r  rJ   )r4   search_queryclean_search_query_listrH   clean_search_querys        r0   _clean_search_queryz!CmfFullSearch._clean_search_queryV  s
     r2   r  strreturntuple[str, set[str]]c                    t                      }|                     d          }t          |          dk    r| |fS d} |dd         D ]m}|dk    r	t          j        d|d          }|d         dk    r|                    |d                    t          |          dk    r|  |d          |d          } n|                                 } | |fS )	zExtrats tags from the given search_query and returns its reminder and a set of extracted tags

        Args:
            search_query (str)

        Returns:
            tuple[str, list[str]]: search_query reminder and a set of extractd tags
        #r    Nz(\W)r      r   )setrL   r+   rK   addstrip)r  r   sharp_splittedtoken
sub_tokenss        r0   _extract_tagszCmfFullSearch._extract_tagsj  s     uu%++C00~!##%%#ABB' 
	O 
	OE{{'5!44J!}""A''' :!##".N
1Nz!}NN#))++T!!r2   c
                    t          |          dk    rt          dd           |d}d|
vrd|
d<   d|
vrd|
d<   |sd	d
g}t          j                            d          }|dk    rt          d|            dt          _        |}t                      }|
	                    d          }|rEt          |t                    r|D ]}|                    |           n|                    |           |                     |          \  }}t          |                    |                    }||
d<   |}|                                }|}t                               |          }|dk    rd}t          |          }d	|d	<   |                    d          r
|d d         }g }g }g }g }g }g }g }g }g }g }g }g }g } g }!g }"g }#g }$g }%g }&g }'g }(g })g }*g }+g },g }-g }.g }/|dk    r.d t&          j        j                                        D             }0n[t                               |          t          j        u r2d}d}d t&          j        j                                        D             }0n|g}0|
	                    d          }1|                     |1          |
d<   |
	                    dd          }2|                     |dd|	          }3|                     |d|	          }4|                     |d|	          }5|                     |dd|	          }6|5t          _        t7          j        dd|4          }7|7t          _        t7          j        dd|4          }8|dk    s|dvr|dk    rJ|2sG | j        |0ft?          t          j         j!                  d d!gdd"|d#|
}' | j        |0fd d!gdd"|d$|
}(nH | j"        d%|0|4fd d!g|d&|3|d'|
}|2s. | j"        d|0|5f|d d!gd(|d)|
} | j"        d*|0|4f|d d!gd(|d)|
}|dk    s|d k    rz|dk    rG|2sC | j        d gft?          t          j         j!                  |d+|d,|
}) | j        d gf|d+|d-|
}*n-|dk    r+ | j"        d%d g|4f|d&|3|d.|
} | j"        d%d g|5f|d/|3d0|
}|2r|d%k    r | j"        |d g|4f|d1d2|
} | j"        |d g|7fd3|d4|
} | j"        |d g|8fd5d6i|
} | j"        |d g|5f|d(d2|
}! | j"        d*d g|4f|d(d2|
}#tG          t          |          t          |          t          |          t          |          t          |          t          |!          t          |#          f          d7k     r | j"        |d g|6f|d(d2|
d	d8         }%|dk    s|d!k    r|dk    rG|2sC | j        d!gft?          t          j         j!                  |d+|d,|
}+ | j        d!gf|d+|d-|
},nu|dk    rV | j"        d%d!g|4f|d&|3|d.|
} | j"        d%d!g|5f|d/|3d0|
} | j"        d9d!g|4f|d&|3|d.|
}- | j"        d9d!g|5f|d/|3d0|
}.|2r|d%k    r | j"        |d!g|4f|d1d2|
} | j"        |d!g|7fd3|d4|
} | j"        |d!g|8fd5d6i|
}  | j"        |d!g|5f|d(d2|
}" | j"        d*d!g|4f|d(d2|
}$tG          t          |          t          |          t          |          t          |          t          |           t          |"          t          |$          t          |-          t          |.          f	          d7k     r | j"        |d!g|6f|d(d2|
d	d8         }&g }9d:|$                                vrtK          d;d<          }:t          j        &                    g d=d>d?d@|$                                gdAd@|$                                ggB          D ]T};|;j'        r|;j'        d d         nd}<|9(                     |:|;j)        |;j*        |;j+        |;j,        |<dCd
d	D                     Ug }=t                      }>dE }?d}@t          -                    dF           d	}A||||'|(|||||-|.|||||| |!|"|#|$|%|&|)|+|*|,|9g}Bt]          |B          r<|@rP |?|9|=|>            |?||=|>            |?||=|>            |?|-|=|>            |?||=|>            |?||=|>           d}@ |?||=|>            |?||=|>            |?||=|>            |?||=|>            |?||=|>            |?||=|>            |?|-|=|>            |?|-|=|>            |?||=|>            |?| |=|>            |?||=|>            |?|9|=|>            |?||=|>            |?||=|>            |?||=|>            |?||=|>            |?||=|>            |?||=|>            |?|!|=|>            |?|"|=|>            |?|#|=|>            |?|.|=|>            |?|$|=|>            |?|%|=|>            |?|&|=|>            |?||=|>            |?|)|=|>            |?|*|=|>            |?|+|=|>            |?|,|=|>            |?|'|=|>            |?|(|=|>           |AdGz  }A|AdHk    r=d:/                    dI |BD                       }Ct          -                    dJ|C            nLt          -                    dK           |=|d	         |dG                  }=|si }D|r9|
	                    dd          }E|
	                    dd          }Fi }G|=D ]5}H|G0                    |Hj*        g           (                    |Hj)                   6|G1                                D ]}te          t                    |Hj*                 j3        r|dLgz   }IdLdM|G|         gg dNg}Jn|}IdOdM|G|         g}Jt                               |                              |I|J|E|FP          }K|KD ]@}Lte          t                    |Hj*                 j3        r|L|D|Lj4        j5        <   6|L|D|Lj5        <   Ant          dQ           t          -                    dR           g }M|=D ]}H|D	                    |Hj)                  }L|Lr|Lj5        j6        |Hj7        |Lj8        j6        |Hj9         dS|Hj:         d:|Hj;        dTd:|Hj<        dUdVt          j         	|Hj:        |Hj;        |Hj<        dW}N	 |s|L=                    ddX           n ||L          st|          |D ]>}O|O?                    dY          d	         }Ot          |D|Hj)                 |Od           }P|P|N|O<   ?|M(                    |N           # t|          $ r Y w xY wt          -                    dZ           |sCt          t          j        jB        |t           C                    |Md d7                   d[\           |MS i }Qg }R|=D ]z}H|R(                    |Hj)                   |Hj)        |Hj7        |Hj+        |Hj9         dS|Hj:         d:|Hj;        dTd:|Hj<        dUdVt          j         	|Hj:        |Hj;        |Hj<        dW|Q|Hj)        <   {|Qt          _D        t          -                    dZ           |RS )]Ni   uY   Превышена максимально допустимая длина запроса!Tr   r#  archivedFdeletedr   r8   r      u   Идет процесс индексации, могут быть доступны не все результаты поиска. Осталось объектов: tag_nameANYr   Modelc                 *    g | ]}|j         	|j        S r:   r   r   r;   ms     r0   r?   z1CmfFullSearch.fulltext_search.<locals>.<listcomp>  #     o o o!aban o o o or2   commentsc                 *    g | ]}|j         	|j        S r:   r5  r6  s     r0   r?   z1CmfFullSearch.fulltext_search.<locals>.<listcomp>  r8  r2   r   tree_parent_filtertitles_only)synonyms
stop_words	query_raw)r=  r?  )r=  
first_wordr?  z([&] )([^!])z<1> \2z<2> \2)CmfTaskCmfDocumentrA  rB  FS)related_usermodel_name_not_inr   labelr   )rE  r   rF  r   r
   N)rE  r   rF  text_stop_words
like_queryS)r   rE  rF  rI  addon_fieldsFA)rD  r   rF  r   )r   rF  r   )r   rF  rH  rI  Ns)r   rF  rH  A)r   rF  H1)rF  rI  rF  H2r  r[   key_phrasesrD   SearchResultTuplez<obj_id,obj_model,obj_code,title,headline,label,rank,age_days)rX   r   r   r   r   r  r   r  rX   )r   r  CODE)rX   r   r   titleheadlinerF  rankage_daysc                     | rS|                      d          }|d         |vr2|                    |           |                    |d                    dS | SdS )Nr   TF)popr  r&  )r   resultskip_idsrs       r0   add_if_existsz4CmfFullSearch.fulltext_search.<locals>.add_if_exists  sf      LLOOQ4x''MM!$$$LL1&&&4    5r2   zfulltext_search Start mixingr   i  c                 F    g | ]}t          t          |                    S r:   )r  r+   )r;   fs     r0   r?   z1CmfFullSearch.fulltext_search.<locals>.<listcomp><  s$    &J&J&Jqs3q66{{&J&J&Jr2   uX   Баг в поиске, много данных, либо не идет вычитка: zfulltext_search End mixingcmf_ver_headIN)cmf_ver_curz==TrW   )r   r  r   r   uY   DEV: FATAL. Укажите в запросе поиска список полей fields=z"fulltext_search Start check accessz ||| z.6fz.0fz words=)rW   r
   r   rU  rF  rV  rW  )TEXKOM_skip_failread_auditTEXKOM_ppp_project_simplecheck.zfulltext_search END)r  obj_dict)r\   )Er+   r   rs   r   r   cmf_noter,   FSTr%  r   
isinstancer   r&  r+  unionlowerr   ninjaendswithr   r   r   r   
CmfComment_get_all_branchesprepare_search_queryFSTQrK   subFSTQHfilter_oncer  current_userr
   search_oncesumr'  r   r  r   r  rX   r   r   r   r-   anyrJ   
setdefaultr   varscmf_verr`  rW   r   rT  r   rU  rF  rV  rW  _acl_check_readCmfPermissionErrorrL   getattrrZ   _do_calc_statistics
dumps_dictfulltext_search_headlines)Sr4   r   r   r  only_idsr   r   no_analitycscheck_accessr?  r\   r   orig_field_namer   r0  _tagextracted_tagsorig_search_querylike_search_queryfullsearch_sliceother_res_name_foundother_res_text_foundother_res_cf_foundtask_res_name_founddocument_res_name_foundtask_res_name_syn_founddocument_res_name_syn_foundtask_res_all_founddocument_res_all_foundtask_res_strong_position_h1document_res_strong_position_h1task_res_strong_position_h2document_res_strong_position_h2task_res_syn_founddocument_res_syn_foundtask_res_cf_founddocument_res_cf_foundtask_res_first_worddocument_res_first_wordother_filter_rel_foundother_filter_foundtask_filter_rel_foundtask_filter_founddocument_filter_rel_founddocument_filter_founddocument_res_key_phrases_found"document_res_key_phrases_syn_foundaddon_res_foundother_model_namesr   r<  r>  tsquery_without_syntsquery_with_syntsquery_first_wordtsquery_strong_position_h1tsquery_strong_position_h2res_code_foundrR  any_res_code_foundrU  rZ  r[  r]  do_top
iter_countsearch_founds
found_lensobjectsis_archived
is_deletedids_by_modelr\  _fileds_filterr   r   rA   rf  fieldattrr  	result_idsS                                                                                      r0   r!   zCmfFullSearch.fulltext_search  sj    |S  qy}~~~~LV##!&F:F"" %F9 	HE*00$0??  K  ~I  K  K  L  L  L$uu::j)) 	#(D)) #$ # #DHHTNNNN# """'*'8'8'F'F$nDJJ~..//!z(#))++
 )}}\22J;; w'' 	)#CRCJ$  "! "$"$&(#!#&(#*,'&(#*,'!# " "$!# "$&! ")+&-/*  o osz7K7[7[7]7] o o o&&z22f6GGGJ#J o osz7K7[7[7]7] o o o!+$455'*'<'<^'L'L#$jj66 --lUW[gp-qq
!66|e_h6ii33L4[d3ee 55lT^bnw5xx!%'VOYH[%\%\",%'VOYH[%\%\"*4N"N"N B&&" -<S_).%()<%=%=+4m*D#)".. . !. .* *9)*+4m*D#)".* * !* *& (7sv?PRe (>GPR_F`:JRUgq?P(> (> 7=	(> (>$
 # +:3?)(	, /+4m*D!#4	, 	, !	, 	,( *9&)+	* /+4m*D!#4	* 	* !	* 	*& *	"9"9 B&&" ,;CO"-%()<%=%=#-".- - !- -) )8")#-".	) )
 !) )% #e++*9#/&9+Ob +`>NVYkuCT+` +`X^+` +`' />cofykSc /FBRZ^pz/F /F E/F /F+ # %jF&:&:)8i[Re *Mm}>A*M *MEK*M *M& 3B#/*yk[u 3tFJWh3t 3tlr3t 3t/2A#/*yk[u 3V 3VFJ3VNT3V 3V/)8i[Rb *Mjz>A*M *MEK*M *M&(7&"+) /!) ) !) )%  344 788 233 ;<< ;<< 233 122
 
 
 
 />co&&K./ #3"%/ / %/ / A#/+ *"="= B&&" 0?&1%()<%=%=#-".1 1 !1 1- -<CO&-#-".	- -
 !- -) #e++.=cof}oWj /dBRZ]oyGX/d /d\b/d /d+ 3B#/&=/[k 3fFV^bPZ3f 3f^d3f 3f/ 6ES_]UbTcex 6mK[cf  yCPa6m 6mek6m 6m2 :IYfXgiy :Z:JRVDN:Z :ZRX:Z :Z6 # &jF&:&:-<S_Z-Zm .M  vF>A.M .MEK.M .M*6EcojS`Rac} 7tFJWh7t 7tlr7t 7t36EcojS`Rac} 7V 7VFJ7VNT7V 7V3-<S_Z-Zj .M  sC>A.M .MEK.M .M*,;CO&&+- /!- - !- -)  788 ;<< 677 ?@@ ?@@ 677 566 >?? BCC
    3B#/&*O.3 #3"%3 3 %3 3 A#3/ '--//// *#N!P !P '-&:&@&@RRRz30A0G0G0I0IJXWZ\m\s\s\u\uLvw 'A ' '  " AS@[c-6tt<<ac%%&7&7-40:/8,5% 	' 	' 	' 	 	 	 	 55	 	 	 	.///
-/CEW');$&=(*E/1S#%;,.M,.M#%;"$9$&=&(A"$9! -   ;	
  	nfh???168DDD5vxHHH<fhOOO2FHEEE2FHEEE  M-vx@@@M-vx@@@M5vxHHHM5vxHHHM168DDDM168DDDM8&(KKKM8&(KKKM968LLLM968LLLM.AAAM.&(;;; M.AAAM,fh???M,fh???M0&(CCCM168DDDM5vxHHHM,fh???M0&(CCCM+VX>>>M<fhOOOM/BBBM-vx@@@M168DDDM.AAAM/BBBM+VX>>>M3VXFFFM/BBBM0&(CCCM,fh???!OJD   XX&J&JM&J&J&JKK
  Du  D  D  E  E  E> 	
,---aq)* L	G x$jjU;;#ZZ	599
! N NA ++AK<<CCAHMMMM"."3"3"5"5 2 2JF||AK08 G &.)9 9$2D,z:R#SUpUpUp"q &!%t\*-E F&88DDII&&)4(2	  J    H  ( 2 2<<4< 2;>GC$4$788.1GCFOO	22& vwwwGG8999C  kk!(++  "fl ! #'(z$m$m$m$m!&$m$m$mqz$m$m$mfgfk$m$m!" !$%J   H+ 5//4pt/uuuu!-c!2!2 5"44%+ 3 3E$)KK$4$4Q$7E#*718+<eT#J#JD.2HUOO

8,,,,-    /> GG)***   %(<,=7K]K]^abecebe^fKgKghh   
 J$&!	 
	 
	AQX&&&h
 zeeee!&eeeqzeee^_^ceeJ3 3%ah// '@#	%&&&s   Bu
uur#  c                    |dg}|sddg}|
|dk    rddht          |          z  rd}
|                    dd           }|                    d	          }|                    d
d           }|                    dd           }|                    dd           }|                    d          }|                    d          }|                    dd           }|                    d          }|                    d          }|                    dd           }|                    d|          }|p|}|                    d          }|r(d                    d                    |                    nd}|                    d          pg }|                     |          \  }}|dk    rd }|	dk    rd }	d }|	r>t          j        d|	          st          j        d|	          r|	                                }|dvrt          d| d           d| }d| }|d k    rd!}| d"}d} |
rd#|
 d"} |d         }!|d$         |d         z
  }"d}#|rt          |          }d%| d&}#t          j        j        j                                                            d'           d}$|dk    rd(}$ d)j        d5i |||| |#||d*}%t          j        j        j                                                            |%i d+|d,|!d-|"d.t!          |          d/t!          |          d0|d1|d2|	d3|$d|d|d|d	|d
|d|d|d||||||d4|          }&t#          |&          S )6Nnull000r   r8   r   rA  rB  rK  r   modified_by_nameresponsible_namelogic_type_codestatus_typemodified_at_beforemodified_at_afterr-  r.  user_rating
owner_nameauthor_namer;   AND obj_tree_parent_id IN ('{}')','r#  r0  z^[a-zA-Z]+-[0-9]+$z^[0-9]+$)r   r
   r   r9  rK  rQ  6   Недопустимое значение field_name: Tr   obj_r
   r   	_tsvectorz || r   $obj_modified_at > now() - interval '
 days' ANDz"set gin_fuzzy_search_limit=100000;@B u]  
            SELECT
                subs.*,
                ts_headline(
                    'russian',
                    substring(subs.{search_field} FROM 0 FOR 100000),
                    :tsquery,
                    'MaxFragments=1,MaxWords=25,MinWords=24'
                ) AS headline
            FROM (
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        ts_rank_cd({tsvector_field}, :tsquery, 2|8)
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        {tsvector_field} {addon_tsvector} @@ :tsquery
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                        and ( :user_rating is null or obj_user_rating >= :user_rating)
                    ORDER BY rank desc
                    LIMIT :slice_for OFFSET :slice_from
                )
            UNION
                /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        100
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                        ( {search_field} ILIKE '%' || :like_query || '%'
                            or (:obj_code is not null and obj_code ILIKE '%' || :obj_code)
                            or obj_id = :like_query
                        )
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                        and ( :user_rating is null or obj_user_rating >= :user_rating)

                        /*and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))*/
                    /*ORDER BY obj_id desc -- не будем делать, т.к. тормозит*/
                    LIMIT :slice_for/10
                    OFFSET :slice_from/10
                )
            ) as subs
            ORDER BY subs.rank desc
            LIMIT :slice_for
            OFFSET :slice_from;
        )r;  search_fieldtsvector_fieldaddon_tsvectorage_days_subqueryheadline_fieldtags_filtertsquery
slice_from	slice_formodel_name_inrE  rF  rH  rI  
max_partno)r  r-  r.  r   r  r:   )r%  r   formatrJ   _build_tags_filterrK   matchupperr   intrs   r   ra   rb   re   rj   tupler   )'r4   r   r  tsquery_strrE  rW  rF  r   rH  rI  addon_field_namer\   r   r  r  r  r  r  r  r-  r.  r  r  r  r;  
tags_namesr  tags_paramsr   r  r  r  r  r  r  r  r  r   
found_objss'                                          r0   rv  zCmfFullSearch.search_once  s   
 $!* 	HE
 #
f(<(<)]A[^abo^p^pAp(<-JJ{D11	!::&899!::&8$?? **%6==jj55#ZZ(<=="JJ':;;::j$//**Y''jj//ZZd33
jj
;;.;
#ZZ(<==j|  E?FFuzzRdGeGefff  CEZZ
++1r
#&#9#9*#E#E [b  "OJ 	*28$9:FF 	*"(S^`jJkJk 	*!''))H```[z[[cghhhh*j**,
,,'N&111 	@?$4???N1X
!HuQx'	 	\8}}H [x [ [ [+3355==>fggg
 J^ ^| }f f~ #5(,,!2,&
 
}f fP ),8@@BBJJ3 Q
{Q
*Q
 Q
 U=11	Q

  '8!9!9Q
 UQ
 Q
 *Q
 *Q
 Q
 *Q
 ;Q
  0Q
  0Q
 Q
  ;!Q
" !"4#Q
$ "3  &-Q
 Q
 Q
. /Q
  
4 Jr2   r  r   rD  
str | Nonec                    |                     d          }	|                     d          }
|                     d          }|                     d          }|                     d          }|                     d          }|                     d          }|                     d          }|                     d	          }|                     d
          }|                     dd           }|                     d|          }|p|}|                     d          }|r(d                    d                    |                    nd}|                     d          pg }|                     |          \  }}d}|r|x}}d}|dvrt	          d| d           d| }|d         }|d         |d         z
  }d}|rt          |          }d| d}|dk    rdnd}d| d | d!| d"| d#| d$| d$| d%}t          j        j        j	        
                                                    |i d&|d'|d(t          |          d)t          |          d*|d+|d|	d|d|d|
d|d|d|d|d|d|d	|d
|i|          } t          |           }!|!S ),Nr   r  r  r  r  r  r  r-  r.  r  r  r  r;  r  r  r#  r0  z
            AND ( :responsible_name IS NULL OR obj_responsible_names ILIKE '%' || :responsible_name || '%' )
            AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
        z
            AND (
                :responsible_name IS NULL
                OR obj_responsible_names ILIKE '%' || :responsible_name || '%'
                OR obj_owner_name = :owner_name
            )
        )r   r
   r   r  Tr   r  r   r   r  r  r   r  a  
            SELECT
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                z as z,
                substring(z` FROM 0 FOR 240) as headline
            FROM cmf_full_search
            WHERE
                a  
                obj_model IN :model_name_in AND obj_model NOT IN :model_name_not_in
                AND part_no <= :max_partno
                AND ( :parent_id IS NULL OR obj_parent_id = :parent_id )
                AND ( :author_name IS NULL OR obj_author_name = :author_name )
                AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                z
                a  
                AND ( :logic_type_code IS NULL OR obj_logic_type_code = :logic_type_code )
                AND ( :status_type IS NULL OR obj_status_type = :status_type )
                AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                AND ( :archived IS NULL OR obj_archived = :archived )
                AND ( :deleted IS NULL OR obj_deleted = :deleted )
                and ( :user_rating is null or obj_user_rating >= :user_rating)
            ORDER BY rank desc
            LIMIT :slice_for OFFSET :slice_from;
        r  r  r  rE  rF  r  )r   r  rJ   r  r   r  rs   r   ra   rb   re   rj   r  r   )"r4   r  rD  r   rE  rW  rF  r   r\   r   r  r  r  r  r  r  r-  r.  r  r  r  r;  r  r  r  owner_and_responsible_filterr  r  r  r  r  r   r  all_objss"                                     r0   rt  zCmfFullSearch.filter_once  s    JJ{++	!::&899!::&899 **%677jj//#ZZ(<=="JJ':;;::j))**Y''jj//ZZd33
jj
;;.;
#ZZ(<==j|  E?FFuzzRdGeGefff  CEZZ
++1r
#&#9#9*#E#E [($  	,88J),( 555[z[[cghhhh*j**1X
!HuQx'	 	\8}}H [x [ [ [ *f 4 4WW!
   $0  (  # & .' ( ) * $+  B ),8@@BBJJ3 Q
*Q
Q
 U=11Q
  '8!9!9	Q

 UQ
 *Q
 Q
 *Q
 ;Q
  0Q
  0Q
 Q
 ;Q
 !"4Q
  !2Q
  !Q
" w#Q
$ ;%Q
 Q
& 'Q
  
* 
##r2   r  	list[str]tuple[str, dict[str, str]]c                 f    i }d}t          |           D ]\  }}d|dz    }|||<   | d| d}||fS )Nr#  tagr   z AND obj_tags ILIKE '%' || :z || '%')r   )r  r  r  r  r  	tag_params         r0   r  z CmfFullSearch._build_tags_filter  se    
++ 	Y 	YFAs%a!eI%(K	"(XXiXXXKKK''r2   c                     | sg S t           j        j        j                                                            dd| i          }d |D             S )Na  
                WITH tree_parents AS (
                    WITH RECURSIVE r AS (
                        SELECT obj_id, obj_code, obj_tree_parent_id
                        FROM cmf_full_search
                        WHERE obj_tree_parent_id = :tree_parent_id

                        UNION

                        SELECT cfs.obj_id, cfs.obj_code, cfs.obj_tree_parent_id
                        FROM cmf_full_search AS cfs
                        JOIN r ON cfs.obj_tree_parent_id = r.obj_id
                    )
                    SELECT obj_id FROM r
                    WHERE r.obj_id IN (SELECT obj_tree_parent_id FROM r)

                    UNION

                    SELECT :tree_parent_id
                )
                SELECT * FROM tree_parents;
            r   c                     g | ]
}|d          S )r   r:   )r;   r\  s     r0   r?   z3CmfFullSearch._get_all_branches.<locals>.<listcomp>3  s    &&&!&&&r2   )rs   r   ra   rb   re   rj   )r   recordss     r0   ro  zCmfFullSearch._get_all_branches  s`    
  	I&)5==??GG, ~./
 
2 '&g&&&&r2   c           	         dt           _        |                    dd          }t          j        dd|          }t          j        d|          }d}d}	d}
|D ][}t          |          dk    rt          |          dk    r|dv r/|d	v r	|
d| z  }
<|dk    rC|d
v r|
dz  }
M|dv r|
dz  }
Wt          |          dk    rk|d         dv r5t          |          dk    r|
d|dd           z  }
|	d|dd           z  }	|r|                    dd          }|dz  }|dk    rd}|
r
|
d         dvr|
dz  }
t          |          dk    rU|                     ||          }t          |          dk    r|
|d          z  }
n|
dd                    |          z   dz   z  }
|r|dk    r n
|dk    r n]|r)t          |	          dk    r|	d         dv r
|	dd          }	|	S |
                    dd                              dd                              dd                               d!d           	                                }
|
r|
d         dv r
|
dd          }
|
r|
d         dv r
|
dd          }
|
r|
d         dv r
|
dd          }
|
r|
dd          dv r
|
d d         }
	 t          j        j        j                                                            d"d#|
i          }t!          |          d         d         }
n# t"          j        j        $ r}t          j        j        j                                         |s4t+          d$|            t+          d%|
 d| dt           j                    t          j        j        j                                                            d&d#|i          }t!          |          d         d         }
Y d }~nd }~ww xY wt-          |d'|
           |
t           _        |
S )(Nr#  zwww.u   [^-A-Za-zА-Яа-я0-9()|&!' ]rD   z(,| |&|\||\(|\))r   r   )-!z()&|)oru   или|z |)u   иand&z &r   z& !r  r     F)r  r  z& )r=  z( z | z )
   )r  r  z OR z or z AND z & z and z!select to_tsquery('russian', :q);qu;   Ошибочный синтаксиса в запросе: u@   DEV: Ошибочный синтаксиса в запросе: z+select websearch_to_tsquery('russian', :q);z->)r,   rh  replacerK   rr  rL   r+   prepare_wordrJ   r'  rs   
CmfSynonymra   rb   re   rj   r   r`   excProgrammingErrorrollbackr   r  rq  )r4   r  r=  r>  r@  r?  search_query_allowed_symbtokens
word_countstopsr  t	sug_wordsr  es                  r0   rp  z"CmfFullSearch.prepare_search_query5  s    !))&44$&F+KSR^$_$_!-/HII 
 2	 2	A1vv{{1vv{{
??;;QLA88)))U
&&&U
 1vv{{tz!!q66Q;;]1QRR5]]"qu& 		#s##A!OJA~~  "Z//T	1vvzz,,Q,BB	y>>Q&&IaL**AA

9 5 55<<A jAoo R 	5zz!||aJ 6 6abb	LIIfe$$,,VU;;CCGUSS[[\cejkkqqss 	1##!""A 	1##!""A 	1##!""A 	233:%%#2#A	$'*6>>@@HHIpsvxyrz{{GWa #AA~. 		$ 		$ 		$ ,55777 nfXdffgggl]^llabllefejllmmm'*6>>@@HHAl#% %G Wa #AAAAAA		$ 	lD!$$$s   AK7 7OB7OOc                 $   t           xj        d| dz  c_        |d         t          j        v rd}t          j        }nd}t          j        }g }t                              |          s6|                    |          }g }t          	                    |          }|r.t           xj        d| dz  c_        |
                    |           t           xj        dz  c_        d}	|D ]}
|	d	k    r nt          |
          d	k    r|
d         |d         k    r|
d
         |d
         k    rCd|
v rJ|
                    dd          }
t           xj        d|
 dz  c_        |
                    |
           |	d
z  }	t           xj        d|
 dz  c_        |
                    |
           |	d
z  }	g }t          j        j        j                                                            dd|i          }d}	|D ]\  }}|	d
k    r nt          |          d	k    r!|d         |d         k    s|d
         |d
         k    rI|                    dd          }t           xj        d| dz  c_        |
                    |           |	d
z  }	t'          |          t'          |          z  |hz  }n|h}t'                      }|D ]_}
t(                              |
          d d	         D ]:}|                    |j                   t           xj        d|j         dz  c_        ;`||z  }t'                      }|rt          j                            ddt1          ||hz            gddgdgdd
g          }|D ]}|j        r|j        j                            d          d d         D ]X}|                                                    dd          }t           xj        d| dz  c_        |                    |           Y||z  t'          |          z  }t1          |          S )Nz|w:z: r   enruzaddNinjaRevers r   zspellError, r   r   rD   z<->z	addSpell z
            SELECT
                name, similarity(:word, name) as sim
            FROM cmf_synonym
            WHERE
                :word % name
            ORDER BY "sim" desc
            LIMIT 5;
             wordzaddSpellTrgm z
normalize r
   ra  r   orderno)r  r   r   r   ,r  zsynAdd )r,   rh  stringascii_lettersr   dictionary_endictionary_rudictionary_checksuggestninja_reversr  r+   r  rs   r  ra   rb   re   rj   r%  morphparser&  normal_formr   r   r   rL   r'  )r4   r  r=  lang
dictionaryfiltered_suggestions3suggestionsfiltered_suggestionsnwr<   rH   filtered_suggestions2suggestions2_listsugg_all_suggestionsnormalized_wordssynonym_wordssynonym_listsynonymr|   s                        r0   r  zCmfFullSearch.prepare_word  s   	t7f***D .JJD .J "''-- @	%$,,T22K#% %%d++B 0121111$++B///EE_$EEA   66Eq66Q;;Q447??qttAw!88		#u--AEE.....EE)00333FA*Q****$++A...Q %'! & 1 4 @ H H J J R R T 
! 
! A, 	 	a66Et99>>7d1g%%aDG););<<U33DEE5T5555EE)00666FA ""677#>S:T:TTX\W]]OO  $fO 55  	9 	9Akk!nnRaR( 9 9 !$$R^4448bn88889 *,<<  	-!,11&$M]ae`fMfHgHg9h:@&9I<E;9:1 2 ? ?L ( - -< -$\/55c::2A2> - -GGII--c59910%))!,,,,)M9C@U<V<VVO$$$r2   c                 |   t          |           dk     rd S g }|d d         D ]}|d                             d          s|d                             d          s9i }|d         |d<   |                    dd           r|d         d         |d<   nd |d<   |                    |           t                                          }||_        d	|_        | |_        t          j
        j        |_        d
|_        t                                          5  |                                 d d d            d S # 1 swxY w Y   d S )Nr$  r  rW   zCmfDocument:zCmfTask:rX   parentr   searchF)r+   
startswithr   r  rs   CmfSearchStatr   actionr  r,   ru  rW   	person_id
aggregatedr   disable_aclsave)r  rf  st_dictrecst_recstats         r0   r  z!CmfFullSearch._do_calc_statistics  s    |q  FCRC= 	# 	#CI((88 CI<P<PQ[<\<\ F"4yF8wwx&& ,'*8}T':|$$'+|$NN6""""##%%(*  "" 	 	IIKKK	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   D11D58D5)F)FF)NNNFFNNNNNNNNNNNNNNNNN)Nr8   F)r  r  r  r   )FNNFNF)NNr#  NNNN)rD  r  )r  r  r  r  )r  r  )TFFF)T)*__name__
__module____qualname____doc__r   r   	api_allowapi_methodsstaticmethodr  r1   classmethodr6   rB   rN   rQ   r   r   r   r   r   r   r   r   rs   	BaseModelr   r   cmf_deferred_jobr"   r#   r  r  r+  r!   rv  rt  r  ro  rp  r  r  r:   r2   r0   r   r      sw         K  O I  K 	 	 	 	 \	 l l [l   [$ K K [K M M [M /F /F /F [/Fb N N N [N
 
. 
. [
.   [:   !"7x x x [xt   [$ ~ ~ [~@ 	cj&: 	 	 	 [	 l l l [lB %%%  &% \ 	
 	
 [	
 VWXXX4= 4= YX \4=l " " ["& " " " \"B  gls s s [sj fhPTI  I  I  [I V  &*$+#hu #u u u [un 	( 	( 	( \	(  '  '  ' \ 'D d d d [dL k% k% k% [k%\    \  r2   r   )r  r^   collectionsr   rr   cmf.data_providers.sqlalchemyr   cmf.fields.cmf_full_searchr   enchantr  	pymorphy3rK   r`   bs4r   MorphAnalyzerr  r   cmf_full_searchr   r:   r2   r0   <module>rH     s      " " " " " "     > > > > > > ! ! ! !       				           	 	!!K K K K KCJ.< K K K K Kr2   