
    ci                       d dl Z d dlZd dlmZ d dlmZmZmZmZ d dl	m
Z
mZmZ d dl	Z	d dlZd dlZd dlZd dl d dlmZ d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ  ej8                         Zg dZg d	Z e eez         Z!g d
Z" edg d      Z#e
 G d d             Z$e
 G d d             Z%e
 G d d             Z& G d dej                  jN                  jP                        Z(y)    N)
namedtuple)ListSetDictOptional)	dataclassfieldfields)*)SQLAlchemyDataDriver)BeautifulSoup)   иu   вu   воu   неu   чтоu   онu   наu   яu   сu   соu   какu   аu   тоu   всеu   онаu   такu   егоu   ноu   даu   тыu   кu   уu   жеu   выu   заu   быu   поu   толькоu   ееu   мнеu   былоu   вотu   отu   меняu   ещеu   нетu   оu   изu   емуu   теперьu
   когдаu   дажеu   нуu
   вдругu   лиu   еслиu   уже   илиu   ниu   бытьu   былu   негоu   доu   васu   нибудьu
   опятьu   ужu   вамu   ведьu   тамu
   потомu   себяu   ничегоu   ейu
   можетu   ониu   тутu   гдеu   естьu   надоu   нейu   дляu   мыu   тебяu   ихu   чемu   былаu   самu   чтобu   безu
   будтоu   чегоu   разu   тожеu   себеu   подu
   будетu   жu
   тогдаu   ктоu   этотu   тогоu   потомуu
   этогоu
   какойu   совсемu   нимu
   здесьu   этомu   одинu
   почтиu   мойu   темu
   чтобыu   нееu   сейчасu   былиu   кудаu
   зачемu   всехu   никогдаu
   можноu   приu   наконецu   дваu   обu   другойu   хотьu
   послеu   надu   большеu   тотu
   черезu   этиu   насu   проu
   всегоu   нихu
   какаяu
   многоu
   развеu   триu   этуu   мояu   впрочемu   хорошоu   своюu   этойu
   передu   иногдаu
   лучшеu   чутьu   томu   нельзяu
   такойu   имu
   болееu   всегдаu   конечноu   всюu
   между)imemymyselfweourours	ourselvesyouyouryoursyourself
yourselveshehimhishimselfsheherhersherselfititsitselftheythemtheirtheirs
themselveswhatwhichwhowhomthisthatthesethoseamisarewaswerebebeenbeinghavehashadhavingdodoesdiddoingaantheandbutiforbecauseasuntilwhileofatbyforwithaboutagainstbetweenintothroughduringbeforeafterabovebelowtofromupdowninoutonoffoverunderagainfurtherthenonceheretherewhenwherewhyhowallanybotheachfewmoremostothersomesuchnonornotonlyownsamesothantooverystcanwilljustdonshouldnow)textml_textnametagscommentsaddon_fieldskey_phrasesSearchResult)	obj_id	obj_modelobj_codetitleheadlinebreadcrumbslabelrankage_daysc                   :   e Zd ZU dZ ee      Zee   e	d<    ee      Z
ee   e	d<    ee      Zee   e	d<    ee      Zee   e	d<    ee      Zee   e	d<    ee      Zee   e	d<    ee      Zee   e	d	<    ee      Zee   e	d
<   deee      fdZy)FullSearchResultsuP   
    Контейнер для всех результатов поиска
    default_factory	doc_nosyndoc_syndoc_first_word
task_nosyntask_syntask_first_word	other_syn
attach_synreturnc                 f    t        |       D cg c]  }t        | |j                         c}S c c}w uQ   
        Возвращает все списки результатов
        r
   getattrr   selfr	   s     ./cmf/models/cmf_full_search.pyget_all_resultsz!FullSearchResults.get_all_resultsI   '     8>d|Deejj)DDD   .N)__name__
__module____qualname____doc__r	   listr   r   r   __annotations__r   r   r   r   r   r   r   r        r   r   r   +   s     %*$$?ItL!?"'"=GT,=).t)DND&D
 &+4%@J\"@#(#>Hd< >*/*EOT,'E %*$$?ItL!? &+4%@J\"@Ed<&8!9 Er   r   c                       e Zd ZU  ee      Zee   ed<    ee      Z	ee   ed<    ee      Z
ee   ed<    ee      Zee   ed<    ee      Zee   ed<   deee      fdZy	)
FullSearchEmptyQueryResultsr   doc_empty_query_related_userdoc_empty_querytask_empty_query_related_usertask_empty_queryother_empty_queryr   c                 f    t        |       D cg c]  }t        | |j                         c}S c c}w r   r   r   s     r   r   z+FullSearchEmptyQueryResults.get_all_resultsY   r   r   N)r   r   r   r	   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   P   s     8=T7R $|"4R*/*EOT,'E8=d8S!4#5S+0+Fd<(F,1$,GtL)GEd<&8!9 Er   r   c                       e Zd ZU eed<   eed<   eed<   eed<   ee   ed<   eed<   eed<   eed<   eed	<   ee   ed
<   eed<   y)FullSearchQueryParams
model_name
field_nameorig_field_namelike_search_queryother_model_names
stop_wordstsquery_without_syntsquery_with_syntsquery_first_wordfullsearch_slicetopN)r   r   r   strr   r   intboolr   r   r   r   r   _   sH    OOCy O3i	Ir   r   c                      e Zd ZdZdZg dZdZg dZede	fd       Z
ed        Zed	        Zed
        Zed        Zed        ZedEd       ZedFd       ZedGd       Zed        Zed        Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dHd       Zed        Zed        Zed        Zedej6                  j8                  fd       ZedId       Ze ed      dJd              Z ed        Z!e edddd !      d"               Z"ed#        Z#edKd$       Z$ed%e	d&e	d'e	d(e	d)e%e	   d*e	d+e	d,e	d-e	d.e%e&   d/e'd0e(fd1       Z)ed%e	d&e	d'e	d(e	d)e%e	   d*e	d+e	d,e	d-e	d.e%e&   d/e'd0e(fd2       Z*e	 dLd3       Z+ed4        Z,ed5        Z-ed6e%d7e%d8e.fd9       Z/e	 	 dMd:       Z0edddd;gddd<dgf	 	 	 dNd=       Z1ed>        Z2edOd?       Z3edPd@       Z4edQdA       Z5edRdB       Z6edSdC       Z7eedD               Z8y)TCmfFullSearchuC   
    Сервис полнотекстового поиска.
    russian)r   text_renderr   
text_draftr   	parent_idtree_parent_id
project_idcmf_created_atcmf_modified_atcodecmf_deletedcmf_archivedloginemailphonephone_internalphone_mobilephone_2phone_assistant
ip_addressemail_2birthdayT)fulltext_searchrun_force_reindexindex_statsr   c                     | rnt        | d      j                  d      } t        |       dkD  rEt        j	                  dt        |        d       | j                         d d j                  d      } | S )	Nhtml.parser
i  z)CmfFullSearch.clean_text: trunc text len z to 1mbi  ignoreerrors)r   get_textlengdebugencodedecode)r   s    r   
clean_textzCmfFullSearch.clean_text   si      }5>>tDD4y7"CCI;gVW{{}Yh/66h6Gr   c                 t    t        |d      j                  d      j                  d      j                         S )Nr   r   r   r   )r   r   r   r   )clstext_with_htmls     r   
strip_htmlzCmfFullSearch.strip_html   s2    ^];DDTJQQYaQbiikkr   c                     d}d}t        |      |k  r|gS t        dt        |      |      D cg c]  }||||z   |z     }}|S c c}w )Ni d   r   )r   range)r   r   text_len_maxoverlapr   ress         r   _split_size_to_partsz"CmfFullSearch._split_size_to_parts   s\     t9|#6M7<QD	<7XY!tAanW,-YY
 Zs   Ac                     dj                  t        j                  d|      D cg c]  }t        |      dk  s| c}      S c c}w )N z\s2   )joinresplitr   r   r   ws      r   	_strip_50zCmfFullSearch._strip_50   s4    xxBHHUD$9IqSVb[IJJIs
   AAc                     dj                  t        j                  d|      D cg c]  }|j                         t        vs| c}      S c c}w )Nr
  z[ \s<>|&\/%@!`+=;.])r  r  r  lowerALL_STOP_WORDSr  s      r   _strip_not_wordzCmfFullSearch._strip_not_word   s;     xxBHH-CT$Jnqaggi_mNmnoons   A Ac                     dd l }d}d| d| }| j                  j                  j                         j	                  |j                  d      d|i      j                         }|S )Nr   zCmfFullSearchLock:_zKSELECT pg_try_advisory_xact_lock(('x' || md5(:lock_key))::bit(64)::bigint);lock_key)
sqlalchemydp_ddSessionexecuter   scalar)r   r   sapart_nor  r  s         r   _fullsearch_obj_lock_get_pgz)CmfFullSearch._fullsearch_obj_lock_get_pg   sk    'xq	:ffjj  "**277  4A  ,BD
 68 	 
r    c                 p   t        j                          }t        j                  d|        d|z   }|st        d      dd l}| j
                  j                  }|j                  |       }|j                  }|j                         }	| j                  |      }
|
st        j                  d| d       y d}|j                  |j                  j                  g      j                  |j                  j                  |k(        j                  |j                  j                   |k(        j#                         }|	j%                  |      j'                         }|rv|d   }|j)                         j                  |j                  j                  |k(        j+                  d|j,                  j/                               }|	j%                  |       nvt        j                  d	|        | j1                         }|j3                         j+                  ||d|j,                  j/                         ||
      }|	j%                  |       |r+ddlm}  |t8        j:                  j<                  d|id       t        j                          |z
  dkD  r.t        j                  dt        j                          |z
          y y )Nzcmf_full_search: mark_dirty zDEBUG_MSG: empty obj_idr   )cmf_full_search: _text_search_sql_insert  lockedT)is_dirtydirty_atz#cmf_full_search: mark_dirty insert )idr   r'  r(  r   obj_name)schedule_deferred_jobr      )kwargs	countdowng?zPROF mark_dirty() got )timer   r   
ValueErrorr  r  data_driverdp_model_cls	__table__r  r!  selectcr)  rp   r   r   with_for_updater  firstupdatevaluesfuncr   gen_idinsertcmf.includer+  models	CmfPersoncelery_full_search_index)r   r   
fast_index	debug_msgprof_str  ddsa_modeltabler   r  r   get_stmtget_resid_update_stmtinsert_stmtr+  s                     r   mark_dirty_deferzCmfFullSearch.mark_dirty_defer   s,   ))+	.vh78!I-	|--VV??3'""JJL--f5GG?xwOPIIuwwzzl#uww~~/0uww'12 "	 	 ))H%++-!*CU577::,-V!WW[[]    IIk"GG9&BC**,CV!!WW[[]#&    IIk"9!&"2"2"K"KU]_eTfrst99; 4'GG,TYY[7-B,CDE (r   c                     |r| j                  ||      S t        j                  j                  |      }|r
|d   dk(  rd}||dt        j                  |<   t	        t        j                        dk\  rt        d       y y )N)rA  rA  TrA  rB  i'  u   DEV: WARNING! Большое количество блокировок может падать на некоторых инсталяциях)rL  r   deferred_fullsearch_dirty_listgetr   	cmf_alert)r   r   rA  forcerB  prevs         r   
mark_dirtyzCmfFullSearch.mark_dirty   s     '':'FF//33F;D&,JBL[d3e((0q//0E9  f  g :r   c                 l   |ri t         _        y t         j                  sy t         j                  }t        |      dkD  rt        dt        |              t	        |j                               D ]9  }||   }| j                  ||j                  d      |j                  d             ; i t         _        y )Nr  ua   DEV: CmfFullSearch помечено грязными слишком много объектов: rA  rB  rN  )r   rO  r   rQ  sortedkeysrL  rP  )r   
only_clean
dirty_listr   r-  s        r   apply_deferred_dirtyz"CmfFullSearch.apply_deferred_dirty  s    /1A,//55
z?Ryz}  I  {J  zK  L  MZ__./ 	qF'F  FJJ|4LX^XbXbcnXo p	q ,.(r   c                    |st        d      dd l}| j                  j                  }|j	                  |       }|j
                  }|j                         }|j                  |j                  j                  g      j                  |j                  j                  |k(        j                  |j                  j                  dk(        j                         }|j                  |      j                         }|rv|d   }	|j!                         j                  |j                  j                  |	k(        j#                  dd|j$                  j'                               }
|j                  |
       | j                  j)                          y )Nr$  r   F)r'  is_dirty_patchr(  )r0  r  r  r1  r2  r3  r  r4  r5  r)  rp   r   r   r6  r  r7  r8  r9  r:  r   commitr   r   r  rD  rE  rF  r   rG  rH  rI  rJ  s              r   
mark_cleanzCmfFullSearch.mark_clean  s    |--VV??3'""JJL IIuwwzzl#uww~~/0uww!+, "	 	 ))H%++-!*CU577::,-V"#(WW[[]    IIk"r   c                 8   |st        d      dd l}| j                  j                  }|j	                  |       }|j
                  }|j                         }|j                  |j                  j                  g      j                  |j                  j                  |k(        j                  |j                  j                  dk(        j                         }|j                  |      j                         }|r|d   }	|j!                         j                  |j                  j                  |	k(        j#                  |j$                  j'                         t)        j*                  d      z         }
|j                  |
       | j                  j-                          y )Nr$  r      )hours)delay_error_reindex)r0  r  r  r1  r2  r3  r  r4  r5  r)  rp   r   r   r6  r  r7  r8  r9  r:  r   datetime	timedeltar]  r^  s              r   mark_delay_error_reindexz&CmfFullSearch.mark_delay_error_reindex<  s+   |--VV??3'""JJL IIuwwzzl#uww~~/0uww!+, "	 	 ))H%++-!*CU577::,-V(*8J8JQR8S(S    IIk"r   Nc$                 6
   |dk(  r,| j                  |       | j                  j                          y | j                  |      d d }|}$d}%|r|%|z  }%|	r|%d|	z   z  }%|
r|%d|
z   z  }%|!r|%d|!z   z  }%|$d|!z   z  }$| j	                  |%      }&| j                  |&      }&|dk(  r|rA| j                  |      }'| j                  |'      }(|d d  d|(d    }'| j	                  |'      })n|d d  }'|'g}(| j	                  |'      })|r%|}| j	                  |      }*| j                  |*      }*nd }d }*|r(| j                  |      }|d d }| j	                  |      }+nd }d }+|r(| j                  |      },|,d d },| j	                  |,      }-nd },d }-|r&| j                  |      d d }| j	                  |      }.nd }.|!r|!}!|!}/n#d }!d }/n|d	k(  rd g}(d })d }d }*d }d }+d },d }-d }.d }!d }/d }'d}0|r|}0|!r|!d d dz   |0z   }0d}1|r|1d|d d
 z   z  }1|r|1d|d d z   z  }1|1d|z   z  }1|r|1d|z   z  }1| j                  |0      }0| j                  |1      }1| j	                  |0      }2| j	                  |1      }3|2j                  d      }4|3j                  d      }5|4|5z   }6|6dk  rn|6dz
  }7|4dkD  rr|5dk\  rEdj                  |2j                         d d       }2dj                  |3j                         d d       }3ndj                  |2j                         d d|5z
         }2|3}3nq|4dk  r(|2}2dj                  |3j                         d d|4z
         }3nDdj                  |2j                         d d       }2dj                  |3j                         d d       }3|dz   |0z   dz   |1z   }8|8d d }8| j	                  |8      }9 | j                  d9i ddd|d|d|d|d|d|d|d|d|d|d|d|d|d|d|d |d!|d"|d#|d$'d%)d&|$d'|&d(|d)*d*|d++d,|8d-|9d.,d/-d0|d1.d2| d3|!d4/d5|"d6|# t        (      d7kD  rt        |(d7d  d7      D ]  \  }:};|d d  d|; }'| j	                  |'      }) | j                  d9i d|:d|d|d|d|d|d|d|d|d|d|d|d|d|d|d|d |d!|d"|d#|d$|'d%|)d,d d-d d&|$d'd d(d d)d d*d d+d d.d d/d d0|d1|.d2| d3d d4d d5|"d6|#  | j                  |t        |(      8       | j                  j                          y ):N	9_disable  r"  r
   DZQTN 0_full_indexr   i 1_namei  u
    Тэги i>  i  i*  i:  r   r   r   obj_parent_idobj_tree_parent_idobj_project_idobj_company_idobj_created_atobj_modified_atr   obj_deletedobj_archivedobj_owner_nameobj_author_nameobj_modified_by_nameobj_responsible_namesobj_hrefobj_logic_type_codeobj_activity_codeobj_status_typeobj_texttext_for_vecr*  name_for_vecobj_tagstags_for_vecobj_result_textresult_text_for_vecobj_ml_textml_text_for_vecobj_commentscomments_for_vecobj_addon_fieldsobj_addon_fields_for_vecobj_user_ratingobj_key_phraseskey_phrases_for_vecobj_breadcrumbsobj_related_person_loginsra  )delete_from_partnor   )'_text_search_sql_delete_obj_with_childsr  r]  r  r  r  countr  r  _text_search_sql_insertr   	enumerate_text_search_sql_delete_partno)<r   r   r   r   r   r   text_prefixtext_suffixfull_search_typecompany_nameproject_namemodel_verbose_namer  comments_textrt  rs  ry  rm  rz  r{  r  ru  rv  rw  rx  rn  ro  rp  rq  rr  r|  r  r  r  r  r  r*  
smart_namer  r}  obj_text_listr~  r  r  obj_comments_textcomments_text_for_vecr  r  obj_ml_text_mainobj_ml_text_suffixml_text_main_for_vecml_text_suffix_for_vecmain_token_lensuffix_token_lentotal_token_lenoversize_lenr  r  r   obj_text_parts<                                                               r   index_objectzCmfFullSearch.index_objectY  sx   P {*77?FFMMO mmK0$7 
$J#,,J#,,J#//J	O33H**:6}}\2~-==. # 8 8 B)%401=3C2DE"228<)%401!)
"228<#"228<"}}\:#"%--"@"1'6":&)&9&9/&J#"&&*#$'MM-$@!$5gv$>!(+(;(;<M(N%$(!(,%#&==1A#B7F#K +.+>+>?O+P(+/("1&5# #'&*#)!FMLHL"O"& $$(!'+$"O"&H #.u5;>NN#(8$(?"??,%4"@@cM11#(:"::==)9: ]]+=>"223CD!$!4!45G!H-33C8177<(+;; V#  +V3L&#u,+.884H4N4N4PQXRX4Y+Z(-0XX6L6R6R6TU[V[6\-]*+.884H4N4N4PQqR`aqRq4r+s(-C*&(';$),2H2N2N2PQoR`aoRo2p)q& (+xx0D0J0J0LWf0U'V$),2H2N2N2PQWRW2X)Y& "C'*::S@CUU!'6*--k: 	$## 	
A 	
f 	

 	
'	
<N	
_m	
 M	
)	
;J	
 	
 ,7	
 FR	
 cq	
 ,		
 CW		

 #8	

 CK	
 !4	
 HY	
 kz	
 	
 -9	
 	
 -9	
 	
 -9	
 ,	
 BU	
 $	
 6E	
 +	
 >S	
 .	
 Ia	
 ,	
  ,!	
  BU!	
" ,#	
$ '@%	
( }!*3M!"4Eq*I &)%401=/B"228<+++ G F V` "/DVgu GU#1CR & 4? NZ ky %4	 K_	
 +@
 KS )< Pa sB & 5A !% 7; & 59 " 15 %) ?C "& 9= &6 Qi %4  %)!  ?C!" %4#$ /H%	0 	**6c-FX*Yr   c                 p   |st        d      dd l}| j                  j                  }|j	                  |       }|j
                  }|j                         }|j                  |      j                  |j                  j                  |k(  |j                  j                  |k(  z        }|j                  |       y Nr$  r   )r0  r  r  r1  r2  r3  r  deleterp   r5  rm  r   r  )r   r   r  rD  rE  rF  r   del_stmts           r   r  z5CmfFullSearch._text_search_sql_delete_obj_with_childsK  s     |--VV??3'""JJL99U#))WW""f,WW^^v%'
 	
		(r   c                    |st        d      dd l}| j                  j                  }|j	                  |       }|j
                  }|j                         }|j                  |      j                  |j                  j                  |k(        j                  |j                  j                  |k\        }|j                  |       y r  )r0  r  r  r1  r2  r3  r  r  rp   r5  r   r   r  )	r   r   r  r  rD  rE  rF  r   r  s	            r   r  z,CmfFullSearch._text_search_sql_delete_partno_  s    |-- VV??3'""JJL IIeU577>>V+,U577??&889 	 	
		(r   c(           	         t         j                  d|        |st        d      dd l}(| j                  j
                  })|)j                  |       }*|*j                  }+|)j                         },| j                  |      }-|-st         j                  d| d       y |(j                  |+j                  j                  g      j                  |+j                  j                  |k(        j                  |+j                  j                  |k(        j!                         }.t#        |,j%                  |.            }/|/rt'        |/      dkD  rt)        d| d| dd	
       |/d   d   }0 |+j+                         j                  |+j                  j                  |0k(        j,                  d8i d|ddddd|d|d|d|d|d|d|d|!d|d|d|d|d|d|	d|(j.                  j0                  j3                  | j4                  |      d|(j.                  j0                  j3                  | j4                  |      d|(j.                  j0                  j3                  | j4                  |      d |(j.                  j0                  j3                  | j4                  |      d!|(j.                  j0                  j3                  | j4                  |      d"|(j.                  j0                  j3                  | j4                  |       d#|(j.                  j0                  j3                  | j4                  |"      d$|
d%|d&|d'|d(|d)|d*|d+|d,|d-|d.|d/|#d0|$d1|(j.                  j0                  j3                  | j4                  |%      d2|&d3|'d4|(j.                  j0                  j3                  | j4                  |'      }1|,j%                  |1       |0S t         j                  d5|        | j7                         }0 |+j9                         j,                  d8i d6|0ddddd7|d|d|d|d|d|d|d|d|d|!d|d|d|d|d|d|	d|(j.                  j0                  j3                  | j4                  |      d|(j.                  j0                  j3                  | j4                  |      d|(j.                  j0                  j3                  | j4                  |      d |(j.                  j0                  j3                  | j4                  |      d!|(j.                  j0                  j3                  | j4                  |      d"|(j.                  j0                  j3                  | j4                  |       d#|(j.                  j0                  j3                  | j4                  |"      d$|
d%|d&|d'|d(|d)|d*|d+|d,|d-|d.|d/|#d0|$d1|(j.                  j0                  j3                  | j4                  |%      d2|&d3|'d4|(j.                  j0                  j3                  | j4                  |'      }2|,j%                  |2       |0S )9Nr%  r$  r   r&  ra  u   Для объекта z	(part_no=u   ) задублировались записи в поиске. Обратитесь в техническую поддержку!Tabortr   r'  Fr\  r   r*  r}  r  r  r  r  r  rm  rn  ro  rp  rq  rr  name_tsvectortext_tsvectorml_text_tsvectortags_tsvectorresult_text_tsvectorcomments_tsvectoraddon_fields_tsvectorr   rs  rt  ru  rv  rw  rx  ry  rz  r{  r|  r  r  key_phrases_tsvectorr  r  "obj_related_person_logins_tsvectorz0cmf_full_search: _text_search_sql_insert insert r)  r   r   )r   r   r0  r  r  r1  r2  r3  r  r!  r4  r5  r)  rp   r   r   r6  r   r  r   rQ  r8  r9  sqlr:  to_tsvector_fts_configr;  r<  )3r   r   r   r   rm  rn  ro  rp  rq  rr  r   rs  rt  ru  rv  rw  rx  ry  rz  r{  r|  r}  r~  r  r  r*  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rD  rE  rF  r   r  rG  rH  rI  rJ  rK  s3                                                      r   r  z%CmfFullSearch._text_search_sql_insertr  s   & 	
;F8DE|-- VV??3'""JJL--f5GG?xwOP IIuwwzzl#U577>>V+,U577??g-._	 	 qyy*+7|a26()G9  MN  O  W[  \!*Q-CuwwzzS(), ", #	,
 $), (, &, &, !,, &, %4, "., &6, #0, (:,  $2!," $2#,$ $2%,& %4',( #%&&++"9"9#//<"X),* #%&&++"9"9#//<"X+,, &(VV[[%<%<S__o%^-,. #%&&++"9"9#//<"X/,0 *,)@)@Re)f1,2 ')ffkk&=&=cooO_&`3,4 +-&&++*A*A#//Sk*l5,6 &7,8 !,9,: ".;,< $2=,> %4?,@ *>A,B +@C,D &E,F )<G,H '8I,J %4K,L %4M,N %4O,P *,)@)@Re)fQ,R %4S,T /HU,V 8:vv{{7N7Ns`y7zW, ^ IIk"j 
g GGFvhOP**,C-- #- $)	-
 $- "- (- &- &- !,- &- %4- ".- &6- #0-  (:!-" $2#-$ $2%-& $2'-( %4)-* #%&&++"9"9#//<"X+-, #%&&++"9"9#//<"X--. &(VV[[%<%<S__o%^/-0 #%&&++"9"9#//<"X1-2 *,)@)@Re)f3-4 ')ffkk&=&=cooO_&`5-6 +-&&++*A*A#//Sk*l7-8 &9-: !,;-< ".=-> $2?-@ %4A-B *>C-D +@E-F &G-H )<I-J '8K-L %4M-N %4O-P %4Q-R *,)@)@Re)fS-T %4U-V /HW-X 8:vv{{7N7Ns`y7zY- ` IIk"
r   objc                     |j                   sy | j                  D ](  }|dk(  r	||j                  v s||   j                  s( y y )Nr   T)full_searchrequired_fieldsr
   
is_changed)r   r  r   s      r   is_obj_need_reindexz!CmfFullSearch.is_obj_need_reindex	  sJ     -- 	J..SZZ'C
O,F,F		r   r  c           
         t         j                  j                  j                         D ]*  }|j                  s|j
                  }|r||vr$d}t        j                  |      }|j                  }|rdg}	 t        j                         }	|j                  ||||z   gdgdd      }
|
s|
D ]J  }|r6t        j                  j                  |j                  j                  d       ;|j                          L |t!        |
      z  }| j"                  j%                          t&        j)                  d| d| d	t        j                         |	z
  d
d        y )Nr   r)  Tr   )r
   sliceorder_byinclude_archivedinclude_deleted)rR  zCmfFullSearch.reindex_models: :, z0.3z sec)cmfr>  	CmfEntityiter_subclassesr  
class_namecmfutilget_model_by_namefull_search_preload_fieldsr/  r   r   rT  r)  valuefull_search_indexr   r  r]  r   r   )r   models_listcommit_everylazy	model_clsr   offsetmodel
field_listtsobj_listr  s               r   reindex_modelszCmfFullSearch.reindex_models  sQ   --==? 	lI(("--Jz<F--j9E99J"V
YY[ ::%!6,#67./%)$( &   # 0C,,77D7Q--/	0
 #h-'8AfXRPTPYPYP[^`P`adOeeijk' 	lr   )	only_oncec                    | r|st        dd       t        j                  d       t        j                  j                          t        j                  }|j                  j                  }|j                  |      }|j                  }|j                         }d}| rd}t        d| dd	d
       	 d| d| d}|j                  |      }	t        |	      }	|j                          t!        |	      rt        dd	d
       nt                y [)Nu   Поиск после обновления должен работать сразу, с приемлемым качеством без переиндексацииTr  zRun run_force_reindexr'  r\  zrun_force_reindex Process z=true: r"  )endflushz
                WITH cte AS (
                SELECT
                    id as id
                FROM   cmf_full_search
                WHERE  z=false AND part_no=0
                ORDER BY dirty_at DESC
                LIMIT  1000
                )
                UPDATE cmf_full_search s
                SET
                    zt = true
                FROM   cte
                WHERE  cte.id = s.id
                RETURNING s.id;
            .)rQ  r   r   r>  CmfAccessListcheck_admin_moder   r  r1  r2  r3  r  printr  r   r]  r   )

from_patchi_do_manual_migrationr   rD  rE  rF  r   dirty_fieldr  r  s
             r   r   zCmfFullSearch.run_force_reindex7  s    3   |  EI  J	')--/""VV??3'""JJL !*K*;-w?RtT
 $} % !M "C  ))C.Cs)CHHJ3xcr.1 r   c                     t         j                  j                          | j                         }| j                  d      }||dS )NTr'  )totaldirty_count)r>  r  r  r  )r   r  r  s      r   r   zCmfFullSearch.index_statsw  sA    --/		iii. &
 	
r   z	@minutely   )r  
system_jobschedulepriorityc                  d	   t         j                  d       t        j                  j	                         rt         j                  d       y t        j                  d       t        j
                         } d}g }	 t        j                  j                  dgg ddg d	g d
gdddt        j                  j                         t        j                  d      z
  gg dgdddt        j                  j                         gg dggdgddg      }|snpi }|D ]H  }t        j                  |j                        }||vrg ||<   ||   j                  |j                         J |j!                         D ]D  \  }}t         j                  d| dt#        |              t%        |       |D ]	  }|j'                  |j(                  dd|gd      }	|dz  }|	st        j                  j'                  |g d      }
d|
j*                   d|
j,                   d|
j                   d |
j.                   d!|
j0                   
}t         j                  |t2        j4                  "       t        j                  j7                  |       t         j                  d#|	j*                          	 |	j9                           G t        j
                         | z
  d&kD  rut        j                  j                         jG                         jH                  }|dk\  r |dk  rt        j
                         | z
  d'kD  rnTt         j                  d(| d)       n:tE                t         j                  d*| d+       t        j                  d       |rQt         j                  d,t2        j4                  "       |D ]  \  }}t         j                  | d-|        ! |d   d   t         j                  d.t        j
                         | z
          y # t:        $ r}t         j                  d$| d|	 d%| t2        j4                  "       t         j                  t=        j>                                |j                  ||g       tA                t        j                  jC                  |       tE                Y d }~d }~ww xY w)/NzStart cron_index_dirtyuJ   Не индексируем поскольку запущен импорт   r   Tr   )r   =r   OR)r'  r  T)r\  r  Tr(  <   )minutes)r(  r  Nrc  )rc  r  Nz	-dirty_at   )r
   filterr  r  zcron_index_dirty process r
  r)  r  )r
   r  r  ra  )r)  r*  r   r(  r'  )r   r
   u   cron_index_dirty: Не индексируем поскольку объект не найден или удален в бд fullsearch id:z
 obj name:z	 obj id: z
 dirty_at:z
 is_dirty:)levelzobj_id=u#   ERROR! При индексации u"    произошла ошибка:    -   ua   cron_index_dirty превышен лимит времени 20 секунд обработано u    объектовu&   cron_index_dirty обработано u    объектов, sleep 1u2   При индексации были ошибки:: zEnd cron_index_dirty at )%r   r   r  enable_import_modeimport_is_runningr/  sleepr>  r   slistrd  r   re  get_model_by_idr   appenditemsr   r  rP  r  r)  r*  r(  r'  loggingERRORr  r  	Exception	traceback
format_exccmf_rollbackrf  
cmf_commit
astimezonehour)stn
exceptionsobj_id_listobj_ids_by_modelr   r  id_listrI  r  
search_rawerr_strer  s                 r   cron_index_dirtyzCmfFullSearch.cron_index_dirty  su    	
()%%779GG`a

1YY[
 ..44XJ*=+/1HJg*h+/-7h>O>O>S>S>UX`XjXjstXu>u,v,C+,
 *.-BCIZIZI^I^I`,a,N)*.(" 1<}-.G' 5 5K( !% >//> 00.0$U+ '..v}}=	> #3"8"8": %w3E7!CL>JKg" %C))5+K+KUY[^`cTdvz){CFA%+%9%9%=%=S  RF%=  &G
%11;z*J]J]I^^ghrhyhygz  {E  FP  FY  FY  EZ Z,,6,?,?+@#B w}}=,,TTUXY GGgcffX./%--/%%6 yy{R"$((,,.99;@@19 yy{R'",GG  AB  @C  CT  U  VLGG<QC?YZ[JJqMM P GGHQXQ^Q^G_$ 'Q3%r!&'Q-""	*499;r>*:;<; % %"EcU!C5Prstru v  F  L  L  M	 4 4 67"))3(3$,,EEcJ"%s    P  	R/	BR**R/c                     |S )u   
        Подчистка оригинального квери, который ввел пользователь:
        - удаление стоп-слов
        )r  r   r  r  )r   search_queryclean_search_query_listr  clean_search_querys        r   _clean_search_queryz!CmfFullSearch._clean_search_query  s
     r   c                 H   t               }| j                  d      }t        |      dk(  r| |fS d} |dd D ]X  }|dk(  r	t        j                  d|d      }|d   dk7  r|j	                  |d          t        |      dk(  sK|  |d    |d    } Z | j                         } | |fS )	zExtrats tags from the given search_query and returns its reminder and a set of extracted tags

        Args:
            search_query (str)

        Returns:
            tuple[str, list[str]]: search_query reminder and a set of extractd tags
        #ra  r"  Nz(\W)r      r  )setr  r   r  addstrip)r  r   sharp_splittedtoken
sub_tokenss        r   _extract_tagszCmfFullSearch._extract_tags  s     u%++C0~!#%%#AB' 
	OE{'5!4J!}"A' :!#".
1z!}oN
	O $))+T!!r   r   r   r   r   r   r   r   r   r   r   r   resultsc           	         |dk7  ry |dk(  s|dk(  r^ | j                   dgft        t        j                  j                        |d|
dd||_         | j                   dgf|d|
dd||_        |dk(  s|dk(  r^ | j                   dgft        t        j                  j                        |d|
dd||_         | j                   dgf|d|
dd||_        |dk(  s|d	vr! | j                   |fddgd
d|
dd||_	        y y )Nr"  ANYCmfTaskFA   )related_person_loginr   r   r  r   )r   r   r  r   CmfDocumentr*  r.  r   FS)model_name_not_inr   r   r  r   )
filter_oncer   r   current_userr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r'  r-  s                 r   &_fulltext_search_execute_empty_queriesz4CmfFullSearch._fulltext_search_execute_empty_queries  st   $ "*	"94CCOO5%()=)=%>%&5 5G1 (7s(%&( (G$ *"=3B3??4%()=)=%>%&4 4G0 '6coo'%&' 'G# *4N"N(7!)#,m"<!&) )G% #Or   c                    |dk(  s|dvr! | j                   |||f|
g dd|d||_        |dk(  s|dk(  r | j                   |dg|f|
d|d||_        |dk(  s|dk(  r | j                   |dg|f|
d	d
||_         | j                   |dg|f|
dd
||_        t        |j                        t        |j                        z   dk  r! | j                   |dg|	f|
dd
|dd |_        |dk(  s|dk(  r | j                   |dg|f|
d	d
||_         | j                   |dg|f|
dd
||_        t        |j                        t        |j                        z   dk  r" | j                   |dg|	f|
dd
|dd |_	        y y y )Nr)  r/  r*  r.  CmfAttachmentS)r  r1  r   
like_queryr8  )r  r   r:  r*  A)r  r   r  r   r,  r.  )
search_oncer   r   r   r   r   r   r   r   r   r4  s                 r    _fulltext_search_execute_queriesz.CmfFullSearch._fulltext_search_execute_queries]  sY   $ *4N"N !0!#	! '"M,	! 	!G *"?!0 ! " '," "G *	"9!0i[J] "Eeu69"E=C"EG  /szI;HX  E`p69 E=C EG 7%%&W-=-=)>>C*9#//K&+ ++ + A+' *"= /
]OM` !Ehx69!E=C!EG-cooj=/K[ Ecs69E=CEGO 7$$%GOO(<<rA)8"O&* +* * A*& B #>r   c                 ~  9: t        |      dkD  rt        dd       |d}d|vrd|d<   d|vrd|d<   |sd	d
g}t        j                  j	                  d      }|dkD  rt        d|        dt        _        |}t               }|j                  d      }|r:t        |t              r|D ]  }|j                  |        n|j                  |       | j                  |      \  }}t        |j                  |            }||d<   |}|j                         }|}t         j#                  |      }|dk(  rd}t        |      }d	|d	<   |j%                  d      r|d d }|dk(  rMt&        j                  j(                  j+                         D cg c]  }|j,                  s|j.                   }}nyt         j1                  |      t        j2                  u rQd}d}t&        j                  j(                  j+                         D cg c]  }|j,                  s|j.                   }}n|g}|j                  d      }| j5                  |      |d<   t               :d|v r|d   rt        |d         :| j7                  |dd|	      }| j7                  |d|	      }| j7                  |d|	      }| j7                  |dd|	      }|t        _        t;        |||||||||||
      }g }t               }|dk(  rK|
rHt=        j<                         }g }| j?                         } | D !cg c]  }!|!j@                  dk(  s|! }"}!| D !cg c]  }!|!j@                  dk(  s|! }#}!| D !cg c]  }!|!j@                  dvs|! }$}!d }%d	dd }&	 t        |      }' |%||&|"        |%||&|"        |%||&|$        |%||&|#        |%||&|#       t        |      }(|(d!k\  s|(|'k(  rnT|d d! }t=        j<                         |z
  d"kD  r-t        jC                  d#t=        j<                         |z
           | jD                  |d	d!g|||||fi |})|)S |dk(  rtG               }*t=        j<                         } | jH                  dBi tK        |      d$|*i| t=        j<                         |z
  d"kD  r-t        jC                  d%t=        j<                         |z
          t=        j<                         }d	}+tM        |*jO                               r|*jP                  d&f|*jR                  d&f|*jT                  d&f|*jV                  d&f|*jX                  d&ff},| j[                  |,||       |+d&z  }+|+d'kD  rXd(j]                  |*jO                         D -cg c]  }-t_        t        |-             c}-      }.t        jC                  d)|.        ntM        |*jO                               rt=        j<                         |z
  d"kD  r-t        jC                  d*t=        j<                         |z
           | jD                  |||||||fi |S |
rt&        j                  j(                  j+                         D cg c]  }|j,                  s|j.                   }/}t=        j<                         }|dk(  rg S |dk7  rg S |dk(  s|dk(  r  | j`                  d+dg|fd	d
gd,||d-dd.|}0ng }0|dk(  s|dk(  r  | j`                  d+dg|fd	d
gd,||d-dd.|}1ng }1d/}2|dk7  rd0}2|dk(  s|d1vr*|g}3|dk(  r|/}3 | j`                  d+|3|fg d2d	|2gd,||d3|}4ng }4t=        j<                         |z
  d"kD  r-t        jC                  d4t=        j<                         |z
          t=        j<                         }t               }g }9:fd59d6 }5d	d	d	d7d8d8d9}6d	d/d }7tc        |0d d: d; <      |0d:d  z   }0tc        |1d d: d= <      |1d:d  z   }1	 t        |      }' 9||6|0        9||6|0        9||6|1        9||6|1        |5||7|4        9||6|0        9||6|0        9||6|1        9||6|1       t        |      }(|(d!k\  s|(|'k(  rn||d d! } | jD                  |d	d!g|||||fi |})t=        j<                         |z
  d"kD  r-t        jC                  d>t=        j<                         |z
          |)S te               }*t=        j<                         } | jf                  dBi tK        |      d$|*i| t=        j<                         |z
  d"kD  r-t        jC                  d?t=        j<                         |z
          ti        jj                  |*      D ]F  }-tm        |*|-jn                        }8tc        |8d d: d@ <      |8d:d  z   }8tq        |*|-jn                  |8       H t=        j<                         }d	}+tM        |*jO                               rb|*jr                  d:f|*jt                  d:f|*jv                  d&f|*jx                  d&f|*jz                  d&f|*jr                  d&f|*jz                  d&f|*jv                  d&f|*jx                  d&f|*j|                  d&f|*jt                  d&f|*j|                  d&f|*jv                  d&f|*jx                  d&f|*j~                  d&f|*j                  d&fg},| j[                  |,||       |+d&z  }+|+d'kD  rXd(j]                  |*jO                         D -cg c]  }-t_        t        |-             c}-      }.t        jC                  d)|.        ntM        |*jO                               rbt=        j<                         |z
  d"kD  r-t        jC                  dAt=        j<                         |z
           | jD                  |||||||fi |S c c}w c c}w c c}!w c c}!w c c}!w c c}-w c c}w c c}-w )CNi   uY   Превышена максимально допустимая длина запроса!Tr  r"  archivedFdeletedr   r  r     u   Идет процесс индексации, могут быть доступны не все результаты поиска. Осталось объектов: tag_namer)  r   Modelr   r   tree_parent_filterrecent_projects)synonymsr   	query_raw)rG  rH  )rG  
first_wordrH  )r   r   r   r   r   r   r   r   r   r   r   r*  r.  )r.  r*  c                     |d   |d   k\  ry |sy |j                  d      }dt        j                  |j                  <   | j	                  |       |dxx   dz  cc<   y )Nr  total_limitr   	empty_topra  popr   fulltext_search_debug_labelr   r  r  countersr  r  s       r   append_if_exists_empty_topzACmfFullSearch.fulltext_search.<locals>.append_if_exists_empty_top*  sY    G$(??ll1o<G--cjj9

3!Q&!r   r  )r  rK  r  g?z3PROF fulltext_search empty_queries_top selects got r'  z/PROF fulltext_search empty_queries selects got ra  ri  r
  uX   Баг в поиске, много данных, либо не идет вычитка: z:PROF fulltext_search empty_queries mixing and prepare got r   TOPi  )r  r   text_stop_wordsr:  r   include_attachmentr  r,  r/  r7  )r1  r  r   rT  r:  z'PROF fulltext_search TOP25 selects got c                    t        t        j                  j                        }|sy |j	                  d      }d}|j
                  r||j
                  v rd}n|j                  r|j                  v rd}|j                         t        j                  |j                  <   ||   || d   k\  r
 | ||      S | j                  |       ||xx   dz  cc<   y )Nr   r5  rE   b_limitra  )r   r   r3  r   rN  r  ro  upperrO  r   r  )r  rQ  r  
user_loginr  	cur_classappend_if_existsrF  s         r   r\  z7CmfFullSearch.fulltext_search.<locals>.append_if_exists  s     !5!56
ll1o  	00Z3C`C`5` #I''C,>,>/,Q #I<EOO<M--cjj9I&(i[3G*HH ,C8DD

3#q(#r   c                     |d   |d   k\  ry |sy |j                  d      }dt        j                  |j                  <   | j	                  |       |dxx   dz  cc<   y )Nr  rK  r   rz   ra  rM  rP  s       r   append_if_exists_otherz=CmfFullSearch.fulltext_search.<locals>.append_if_exists_other  sY    G$(??ll1o<C--cjj9

3!Q&!r   r     )rE   rW  r5  a_limitb_limitc_limit   c                     | j                   S Nr   r   s    r   <lambda>z/CmfFullSearch.fulltext_search.<locals>.<lambda>  s
    !** r   )keyc                     | j                   S re  rf  rg  s    r   rh  z/CmfFullSearch.fulltext_search.<locals>.<lambda>  s
    

 r   z2PROF fulltext_search TOP25 mixing and prepare got z&PROF fulltext_search main selects got c                     | j                   S re  rf  rg  s    r   rh  z/CmfFullSearch.fulltext_search.<locals>.<lambda>  s
    !** r   z1PROF fulltext_search main mixing and prepare got r   )Ar   rQ  r>  r   r  cmf_noter   FSTr   rP  
isinstancer   r!  r&  unionr  r  ninjaendswithr  r  r  r  r  r  
CmfComment_get_all_branchesprepare_search_queryFSTQr   r/  filter_once_topr   r   _prepare_final_resultr   r5  varsrt   r   r   r   r   r   r   _add_if_existsr  r   r<  rV  r   r=  dataclassesr
   r   r   setattrr   r   r   r   r   r   r   r   );r   r   r   r  only_idsr  r
   no_analitycscheck_accessrH  r   r-  r  r   r   rB  _tagextracted_tagsorig_search_queryr   r   mr   r   r   r   r   r   search_paramsfinal_resultskip_idsrC  top_resultsr   top_results_taskstop_results_docstop_results_othersrR  counters_empty_topres_count_beforeres_count_afterresultr'  
iter_countprocessing_groupf
found_lensall_model_namesresult_tasksresult_docsother_sliceother_model_filterresult_otherr^  rQ  counters_otherdatar\  rF  s;                                                            @@r   r   zCmfFullSearch.fulltext_search  s    |S qy}~LV#!&F:F" %F9HE**00$0?  }  ~I  }J  K  L $u::j)(D)$ #DHHTN# "'*'8'8'F$nDJJ~./!z(#))+
 )}}\2"J; w'#CRJ* 7:zz7K7K7[7[7] o!abanan o o&&z2f6G6GGJ#J7:zz7K7K7[7[7] o!abanan o o!+$45'*'<'<^'L#$%&62C+D!&):";<O --lUW[gp-q
!66|e_h6i33L4[d3e 55lT^bnw5x!-!!+//! 3-1-
 5"siikGL--/K,7 Tq1;;);S T T+6Wa!++:VWW-8!jAKKOi<i!!j!j' ,-R!@#&|#4 *<9KM^_*<9KM^_*<9KM_`*<9KM]^*<9KM]^"%l"3"b(O?O,O  (,Lyy{W$s*Mdiik\cNcMdef.S..|aWhPVXikw  zF  Q  JP  QFM"13GiikG6C66hm9LhV]haghyy{W$s*I$))+X_J_I`abiikGJg--/0::A>--q1991=,,a0..2$  ""#3\8La
$!$@W@W@Y*Z13s1v;*Z![JGG x  zD  yE  H  I g--/0" yy{W$s*TUYU^U^U`cjUjTklm,3,,\5(FTegs  vB  M  FL  M  M
 69ZZ5I5I5Y5Y5[m_`_l_lq||mOmiikGZ'	%'	 U"jI&=.sv	{DW  A78gU\f;LW\qu Ay A  "U"jM&A-coof}oGZ A78gU\f;LW\quAyA ! KU"U"j8R&R&0\"&)8&.sv7IK^  XBm78oUdn;L X QW X
  "yy{W$s*A$))+PWBWAXYZ iikGuHL
)*' QQ#$qBH'(;N ",r"28LMP\]^]_P``L Ra6JKkZ[Z\o]K#&|#4  xF xF xE xE&|^\R xF xF xE xE"%l"3"b(O?O,O " (,L /S..|aWhPVXikw  zF  Q  JP  QFyy{W$s*LTYY[[bMbLcdeM
 $% ))+,,,^tM/B^G^W]^99; 3&GG<TYY[7=R<STU ##G, 	+A7AFF+D$r((<=QRHDGQVVT*	+
 ))+
'))+,##Q'""A&##Q'""A&!!1%##Q'!!1%##Q'""A&!$""A&!$##Q'""A&((!,''++ 0 /xH!OJD  XXG<S<S<U&Vqs3q6{&VW
tu  uA  D  E? '))+,B 99; 3&GGG		V]H]G^_`(s((uhPacoq}  I  BH  I  	I]	 !p
 !pR !UW!jt +[ n^ 'WsT   	nn?nnn!2n!<n&n&n+/n+3n0n5'n5!n:c                 6    d| vr| S | j                  d      d   S )Nz@#@#@#r   )r  )r}  s    r   _remove_suffix_from_headlinez*CmfFullSearch._remove_suffix_from_headline(  s#    8#O~~h'**r   c                    ||d   |d    }|rhi }	g }
|D ]<  }|
j                  |j                         |j                  r| j                  |j                        nd}d}|j                  |j                  d}|j
                  }|j                  t        j                  v r#|dz   t        j                  j                     z   }|j                  |j                  |j                  | d| d	| d	|j                  d
dt        j                   	t        j                  |j                        ||j                  |j                  d|	|j                  <   ? |	t        _        t        j#                  d       |
S i }|rR|j%                  dd      }|j%                  dd      }i }|D ]7  }|j'                  |j(                  g       j                  |j                         9 |j+                         D ]  }t-        t.              |   j0                  r|dgz   }dd||   gg dg}n
|}dd||   g}|dk(  r|g dz   }|dk(  r|dgz   }t2        j5                  |      j7                  ||||      }|D ]]  }t-        t.              j(                     j0                  r&|j0                  r|||j8                  j                  <   O|||j                  <   _  nt;        d       t        j#                  d       g }|D ]  }|j%                  |j                        }|s"|j                  r| j                  |j                        nd}d}|j                  |j                  d}|j
                  }|j                  t        j                  v r#|dz   t        j                  |j                     z   }|j<                  dk(  r|j>                  s|j@                  |j>                  jB                  |j>                  jD                  jF                  | d| d	| d	|j                  d
dt        j                   	||j                  |j                  |j                  jF                  |j                  |jH                  |jJ                  |jL                  t        j                  |j                        d}ntO        jP                  d|j                        d   }|j                  jF                  ||jD                  jF                  | d| d	| d	|j                  d
dt        j                   	||j                  |j                  t        j                  |j                        d}|j<                  dk(  r|jR                  jF                  |d<   	 |s|jU                  dd        n ||      stV        |D ]6  } |jP                  d!      d   }tY        ||j                     |d      }|||<   8 |j                  |        t        j#                  d       |s>t[        t.        j\                  j^                  |t2        ja                  |dd"       d#$       |S # tV        $ r Y \w xY w)%u5   
        Итоговая обработка
        r   ra  r"  NoneNz.6f-z ||| r
  z.0fz words=)r)  r   r   r   r   r   r   r   zfulltext_search ENDr?  Fr@  cmf_ver_headIN)cmf_ver_curz==Tr)  r8  )urlurl_previewurl_preview_imgr.  fullsearch_answer)r
   r  r  r  uY   DEV: FATAL. Укажите в запросе поиска список полей fields=z"fulltext_search Start check access)r)  r   r   r   r   r   r   	attach_idattach_name
attach_urlattach_url_previewattach_url_preview_imgr   rj  )r)  r   r   r   r   r   r   r   T)TEXKOM_skip_failread_auditTEXKOM_ppp_project_simplecheckr  r  )r  obj_dict)r-  )1r  r   r   r  r   r   r   rO  r)  r   r   r   rm  jsonloadsr   fulltext_search_headlinesr   rP  
setdefaultr   rW  rx  r>  cmf_verr  r  r   r  rQ  r  parentr   r   r   r  r  r  r  r  r  r  _acl_check_readCmfPermissionErrorr   r+  r   _do_calc_statistics
dumps_dict)r   r  r  r|  r
   r  r}  r~  r-  r  	result_idrr   formated_rankr   r  objectsis_archived
is_deletedids_by_modelr   _fields_filterr  r  r  r   r	   attrs                                r   rw  z#CmfFullSearch._prepare_final_result.  s   
 $E!HU1X6(*%I!   *KL::3;;AJJG[] &66%'(vvclM88q<<<!CK!*G*G*OOE((GGJJ#+*E%-!**UXIYY`abafaf`g h#'::amm#<"FF !

	7)!((3* +DA'GG)* **Z7KIu5JL! J''R8??IJ*//1 .
<
+33"n%55G .l:6NOQlmG"G!4j)ABG0%(QQG.%)<(==G"44Z@EE""%0$.	 F  $ .CF|AKK088S[[7: 0 0 3 34*-	.).4 qs	45 E	A++ahh'CKL::3;;AJJG[] &66%'(vvclM88q<<<!CK!*G*G*QQE>>_4 ::  "mm #

 #

 5 5'/jeWAm_AajjY\M]]defejejdk$l!& !$%JJ%(VV\\'(ww&)gg.1oo252E2E'+zz!--'@ H" HHZ9!<E "ffll % #'/jeWAm_AajjY\M]]defejejdk$l!& !$%JJ'+zz!--'@	 H >>]2474I4I4O4OH01'++tlp+q)#.00!' / +C 0 3&wqxx'8%F*./ JJx(}E	N 	
%& !$$88(9wGYGYZ]^a_aZbGcd
 
) *  s   8A0Y	YYr  r  r  c                    |D ]  \  }}t        |      D ]l  }|s |d   j                  |v r|j                  d       *|j                  |d          |j	                  |d   j                         |j                  d       n  y )Nr   )r  r   rN  r  r!  )r   r  r  r  lstr  r  s          r   ry  zCmfFullSearch._add_if_exists  s}    " 		JC5\ q6==H,GGAJ##CF+SV]]+
		r   c                 t   |dg}|sddg}|j                  dd       }|j                  d      }|j                  dd       }|j                  dd       }|j                  dd       }|j                  d	      }|j                  d
      }|j                  dd       }|j                  d      }|j                  d      }|j                  dd       }|j                  d|      }|xs |}|j                  d      }|r dj                  dj                  |            nd}d}|
rd}|j                  d      xs g }| j                  |      \  }}| j	                  |      \  }}|dk(  rd }|	dk(  rd }	d } |	r<t        j                  d|	      st        j                  d|	      r|	j                         } |t        vrt        d| d       d| }!d| }"|dk(  rd}"| d}#|d   }$|d   |d   z
  }%d}&|rt        |      }d | d!}&t        j                  j                  j                  j                         j!                  d"       d}'|d#k(  rd$}'t        j"                  d%d&|      }(t        j"                  d%d'|      })d(}*|d)v rd*}* d+j                  d9i ||!|#|&|"||*||d,	}+t        j                  j                  j                  j                         j!                  |+i d-|d.|(d/|)d0|$d1|%d2t%        |      d3t%        |      d4|d5|d6|	d7|'d|d|d|d|d|d||||||| |d8||      },t'        |,      S ):Nnull000r   r  r   modified_by_nameresponsible_namelogic_type_codestatus_typemodified_at_beforemodified_at_afterr?  r@  user_rating
owner_nameauthor_namerE   AND obj_tree_parent_id IN ('{}')','r"  zOOR (obj_model = 'CmfAttachment' and obj_parent_id ilike :model_name_in || ':%')rB  z^[a-zA-Z0-9]+-[0-9]+$z^[0-9]+$6   Недопустимое значение field_name: Tr  obj_r   r}  	_tsvectorra  $obj_modified_at > now() - interval '
 days' ANDz"set gin_fuzzy_search_limit=100000;r   @B z([&] )([^!])z<1> \2z<2> \20)r   r   r   z1|16u"  
            SELECT
                subs.*,
                ts_headline(
                    'russian',
                    substring(subs.{search_field} FROM 0 FOR 100000),
                    :tsquery,
                    'MaxFragments=1,MaxWords=25,MinWords=24'
                ) AS headline
            FROM (
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        obj_breadcrumbs as breadcrumbs,
                        obj_project_id as obj_project_id,
                        obj_related_person_logins as obj_related_person_logins,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 3*365 then 'R' else '' end) as label,
                        (ts_rank_cd({tsvector_field}, :tsquery, {ts_rank_normalize}))
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 3*365 then
                                    3*((3*365-EXTRACT(days from (now()-obj_modified_at)))/30+1) else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        (({tsvector_field} @@ :tsquery)  or ( {search_field} ILIKE '%' || :like_query || '%' ))
                        and ((obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in) {include_attachment_filter})
                        and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {rpl_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                        and ( :user_rating is null or obj_user_rating >= :user_rating)
                    ORDER BY rank desc
                    LIMIT :slice_for OFFSET :slice_from
                )
            UNION
                /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                (
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        obj_breadcrumbs as breadcrumbs,
                        obj_project_id as obj_project_id,
                        obj_related_person_logins as obj_related_person_logins,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        100
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                        ((:obj_code is not null and obj_code ILIKE '%' || :obj_code)
                            or obj_id = :like_query
                        )
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {rpl_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                    /*ORDER BY obj_id desc -- не будем делать, т.к. тормозит*/
                    LIMIT :slice_for/10
                    OFFSET :slice_from/10
                )
            ) as subs
            ORDER BY subs.rank desc
            LIMIT :slice_for
            OFFSET :slice_from;
        )	rE  search_fieldtsvector_fieldage_days_subqueryheadline_fieldtags_filterts_rank_normalizeinclude_attachment_filter
rpl_filtertsquerytsquery_str_h1tsquery_str_h2
slice_from	slice_formodel_name_inr1  r   rT  r:  
max_partno)r  r  r  r?  r@  r   r  r   )rP  formatr  _build_tags_filter#_build_related_person_logins_filterr  matchrY  ALLOWED_FIELDSrQ  r   r>  r   r  r1  r  r  subtupler   )-r   r   r  tsquery_strr1  r   r   r  rT  r:  rU  r-  r-  r   r  r  r  r  r  r  r?  r@  r  r  r  rE  r  
tags_namesr  tags_paramsr  
rpl_paramsr   r  r  r  r  r  r  r  r  r  r  r  
found_objss-                                                r   r<  zCmfFullSearch.search_once  s_   
 $!*HEJJ{D1	!::&89!::&8$? **%6=jj5#ZZ(<="JJ':;::j$/**Y'jj/ ZZd3
jj
;.;
#ZZ(<=j|?FFuzzRdGef  CE %'!(}%ZZ
+1r
#&#9#9*#E [!$!H!HI]!^
Jb "OJ288$<jIRXXVacmMn!'')H^+Nzl[cghj\*
|,'N&<y11X
!HuQx'	8}H"FxjPZ [++335==>fg
 J KHKH  88 &a aB FCkD #5(,!2,&!2)B$

CkZ )),,88@@BJJ3 Q
{Q
nQ
 nQ
 *	Q

 Q
 U=1Q
  '8!9Q
 UQ
 Q
 *Q
 *Q
 Q
 *Q
 ;Q
  0Q
   0!Q
" #Q
$ '"4!2  &1Q
2 3Q
4 5Q
 
: Jr   r  r   c	                    |	j                  d      }
|	j                  d      }|	j                  d      }|	j                  d      }|	j                  d      }|	j                  d      }|	j                  d      }|	j                  d      }|	j                  d	      }|	j                  d
      }|	j                  dd       }|	j                  d|      }|xs |}|	j                  d      }|r dj                  dj                  |            nd}|	j                  d      xs g }| j                  |      \  }}| j	                  |      \  }}d}|rt        d       |x}}d}|t        vrt        d| d       d| }|d   }|d   |d   z
  }d} |rt        |      }d| d} |dk(  rdnd}!d| d | d!| d"|  d#| d$| d$| d$| d%}"t        j                  j                  j                  j                         j                  |"i d&|d'|d(t        |      d)t        |      d*|d+|!d|
d|d|d|d|d|d|d|d|d|d	|d
|i||      }#t        |#      }$|$S ),Nr   r  r  r  r  r  r  r?  r@  r  r  r  rE  r  r  r"  rB  z
            AND ( :responsible_name IS NULL OR obj_responsible_names ILIKE '%' || :responsible_name || '%' )
            AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
        z:Param related_user is deprecated. Use related_person_loginz
            AND (
                :responsible_name IS NULL
                OR obj_responsible_names ILIKE '%' || :responsible_name || '%'
                OR obj_owner_name = :owner_name
            )
        r  Tr  r  r   ra  r  r  r   r  a  
            SELECT
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                obj_breadcrumbs as breadcrumbs,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                 as ,
                substring(` FROM 0 FOR 240) as headline
            FROM cmf_full_search
            WHERE
                a  
                obj_model IN :model_name_in AND obj_model NOT IN :model_name_not_in
                AND part_no <= :max_partno
                AND ( :parent_id IS NULL OR obj_parent_id = :parent_id )
                AND ( :author_name IS NULL OR obj_author_name = :author_name )
                AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                z
                a  
                AND ( :logic_type_code IS NULL OR obj_logic_type_code = :logic_type_code )
                AND ( :status_type IS NULL OR obj_status_type = :status_type )
                AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                AND ( :archived IS NULL OR obj_archived = :archived )
                AND ( :deleted IS NULL OR obj_deleted = :deleted )
                and ( :user_rating is null or obj_user_rating >= :user_rating)
                AND obj_modified_at is not null
            ORDER BY obj_modified_at DESC
            LIMIT :slice_for OFFSET :slice_from;
        r  r  r  r1  r   r  )rP  r  r  r  r  rQ  r  r   r>  r   r  r1  r  r  r  r   )%r   r  related_userr-  r   r1  r   r   r  r-  r   r  r  r  r  r  r  r?  r@  r  r  r  rE  r  r  r  r  r  owner_and_responsible_filterr  r  r  r  r  r  r  all_objss%                                        r   r2  zCmfFullSearch.filter_once  s    JJ{+	!::&89!::&89 **%67jj/#ZZ(<="JJ':;::j)**Y'jj/ZZd3
jj
;.;
#ZZ(<=j|?FFuzzRdGef  CEZZ
+1r
#&#9#9*#E [!$!H!HI]!^
J($ RS,88J),( ^+Nzl[cghj\*1X
!HuQx'	8}H"FxjPZ [ *f 4W!

 d<. 1'. ) ## $ .. /  #$ %	/"H )),,88@@BJJ3 Q
*Q
Q
 U=1Q
  '8!9	Q

 UQ
 *Q
 Q
 *Q
 ;Q
  0Q
  0Q
 Q
 ;Q
 !"4Q
  !2Q
  !Q
" w#Q
$ ;%Q
& 'Q
( )Q
 
, 
#r   c                 j   t        j                          }t        t        j                  j                        }| j                  |      \  }}d}d}|rt        |      }d| d}d}d}d}	d}
dj                  g d	| d
| d| d| d| d| d
| d| d| d| d| d
| d| d| d| d      }t        j                  j                  j                  j                         j                  ||	|
|d|      }t        |      }t        j                          |z
  }|S )Nr"  r,  r  r  EMTr  r   r  a  
            (SELECT
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                obj_breadcrumbs as breadcrumbs,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                r  r  r  zh
                obj_model = 'CmfTask' AND                 
                part_no = 0
                u  
                AND obj_modified_at is not null /* Временный хак пока не поправят багу TEM-1625047513 */
            ORDER BY obj_modified_at DESC
            LIMIT :slice_for OFFSET :slice_from)
            UNION
            (SELECT
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                obj_breadcrumbs as breadcrumbs,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                z[
                obj_model = 'CmfDocument' AND
                part_no = 0
                z
                obj_model != 'CmfDocument' AND
                obj_model != 'CmfTask' AND
                part_no = 0
                u   
                AND obj_modified_at is not null /* Временный хак пока не поправят багу TEM-1625047513 */
            ORDER BY obj_modified_at DESC
            LIMIT :slice_for OFFSET :slice_from);
            )r  r  r   )r/  r   r   r3  r   r  r   r  r>  r   r  r1  r  r  r   )r   rC  r-  r  r  r  r   r   r  r  r  r  r  r  profs                  r   rv  zCmfFullSearch.filter_once_topL  s   ))+ !5!56!$!H!HI]!^
J8}H"FxjPZ [$
	@ @ 
 @ @ #@ $0.@1@ (.@)@ ##@$@" #@"#@> ?@> #?@> $0.?@>1?@@ (.A@@)A@F ##G@F$G@L M@LM@h i@h #i@h $0.i@h1i@j (.k@j)k@p ##q@p$q@x y@xy@D )),,88@@BJJ3$"Q
 	Q
 
 
#yy{W$r   c                 X    i }d}| r!d}ddj                  d | D               d|d<   ||fS )Nr"  z AND obj_tags ~* :tags_re(^| |,)(|c              3   F   K   | ]  }t        j                  |        y wre  r  escape).0rB  s     r   	<genexpr>z3CmfFullSearch._build_tags_filter.<locals>.<genexpr>  s     8hQY89L8hs   !)( |,|$)tags_re)r  )r  r  r  s      r   r  z CmfFullSearch._build_tags_filter  sF     5K'/8h]g8h0h/iiq%rK	"K''r   c                 V    i }d}| r d}t        j                  |       } d|  d|d<   ||fS )Nr"  z; AND obj_related_person_logins ~* :related_person_logins_rer  r  related_person_logins_rer  )rZ  r  r  s      r   r  z1CmfFullSearch._build_related_person_logins_filter  sC     

VJ:.J7?
|85TJ12:%%r   c                     | sg S t         j                  j                  j                  j	                         j                  dd| i      }|D cg c]  }|d   	 c}S c c}w )Na  
                WITH tree_parents AS (
                    WITH RECURSIVE r AS (
                        SELECT obj_id, obj_code, obj_tree_parent_id
                        FROM cmf_full_search
                        WHERE obj_tree_parent_id = :tree_parent_id

                        UNION

                        SELECT cfs.obj_id, cfs.obj_code, cfs.obj_tree_parent_id
                        FROM cmf_full_search AS cfs
                        JOIN r ON cfs.obj_tree_parent_id = r.obj_id
                    )
                    SELECT obj_id FROM r
                    WHERE r.obj_id IN (SELECT obj_tree_parent_id FROM r)

                    UNION

                    SELECT :tree_parent_id
                )
                SELECT * FROM tree_parents;
            r   r   )r>  r   r  r1  r  r  )r   recordsr  s      r   rs  zCmfFullSearch._get_all_branches  s`    
 I&&))55==?GG, ~./
2 &&!&&&s   Ac                    dt         _        |j                  dd      }t        j                  dd|      }t        j
                  d|      }d}d}	d}
|D ]  }t        |      dk(  rt        |      dk(  r|dv r&|d	v r	|
d| z  }
3|dk(  r9|d
v r|
dz  }
C|dv r|
dz  }
Mt        |      dk(  r\|d   dv r&t        |      dk(  rr|
d|dd   z  }
|	d|dd   z  }	|r|j                  dd      }|dz  }|dkD  rd}|
r|
d   dvr|
dz  }
t        |      dkD  rE| j                  ||      }t        |      dk(  r
|
|d    z  }
n|
ddj                  |      z   dz   z  }
|r|dk\  r n
|dk\  r n |rt        |	      dkD  r|	d   dv r|	dd  }	|	S |
j                  dd      j                  dd      j                  dd       j                  d!d       j                         }
|
r|
d   dv r|
dd  }
|
r|
d   dv r|
dd  }
|
r|
d   dv r|
dd  }
|
r|
dd  dv r|
d d }
	 t        j                  j                  j                  j                         j                  d"d#|
i      }t!        |      d   d   }
t-        |d&|
       |
t         _        |
S # t"        j$                  j&                  $ r}t        j                  j                  j                  j)                          |st+        d$       t        j                  j                  j                  j                         j                  d%d#|i      }t!        |      d   d   }
Y d }~d }~ww xY w)'Nr"  zwww.u   [^-A-Za-zА-Яа-я0-9()|&!' ]r
  z(,| |&|\||\(|\))r   ra  )r  !z()&|)rK   r   r  z |)r   rH   &z &r  z& !r  r  rc  F)r  r  z& )rG  z( z | z )
   )r  r  z OR z or z AND z & z and z!select to_tsquery('russian', :q);qu7   Ошибочный синтаксис в запросеz+select websearch_to_tsquery('russian', :q);z->)r   rm  replacer  r  r  r   prepare_wordr  r"  r>  
CmfSynonymr  r1  r  r  r   r  excProgrammingErrorrollbackrQ  r  ru  )r   r  rG  r   rI  rH  search_query_allowed_symbtokens
word_countstopsr  r   	sug_wordsr  r  s                  r   rt  z"CmfFullSearch.prepare_search_query  s    !))&4$&FF+KSR^$_!-/HI 
 2	A1v{1v{
?;1QCLA8))r
&&r
 1v{tz!q6Q;s1QR5']"3qug&		#s#A!OJA~ "Z/T	1vz,,Q,B	y>Q&IaL>*A

9 55<<AjAo Re2	h 5z!|aJ 6ab	LIIfe$,,VU;CCGUS[[\cejkqqs1#!"A1#!"A1#!"A23:%#2A	$''**66>>@HHIpsvxyrz{GWa #A 	lD!$ ~~.. 	$  ,,557
 ST''**66>>@HHAl#%G Wa #A	$s   AJ M$BL==Mc                    t         xj                  d| dz  c_        |d   t        j                  v rd}t        j
                  }nd}t        j                  }g }t        j                  |      s|j                  |      }g }t        j                  |      }|r.t         xj                  d| dz  c_        |j                  |       t         xj                  dz  c_        d}	|D ]  }
|	d	k\  r nt        |
      d	k  r|
d   |d   k7  r|
d
   |d
   k7  r0d|
v rF|
j                  dd      }
t         xj                  d|
 dz  c_        |j                  |
       |	d
z  }	zt         xj                  d|
 dz  c_        |j                  |
       |	d
z  }	 g }t        j                  j                  j                   j#                         j%                  dd|i      }d}	|D ]w  \  }}|	d
k\  r nmt        |      d	k  r|d   |d   k(  s|d
   |d
   k(  s3|j                  dd      }t         xj                  d| dz  c_        |j                  |       |	d
z  }	y t'        |      t'        |      z  |hz  }n|h}t'               }|D ]a  }
t(        j+                  |
      d d	 D ]D  }|j-                  |j.                         t         xj                  d|j.                   dz  c_        F c ||z  }t'               }|rt        j                  j1                  ddt1        ||hz        gddgdgdd
g      }|D ]  }|j2                  s|j2                  j4                  j7                  d      d d D ]P  }|j9                         j                  dd      }t         xj                  d| dz  c_        |j-                  |       R  ||z  t'        |      z  }t1        |      S )Nz|w:r  r   enruzaddNinjaRevers r  zspellError, r  ra  r
  z<->z	addSpell z
            SELECT
                name, similarity(:word, name) as sim
            FROM cmf_synonym
            WHERE
                :word % name
            ORDER BY "sim" desc
            LIMIT 5;
             wordzaddSpellTrgm z
normalize r   r  r   orderno)r  r
   r  r  ,r  zsynAdd )r   rm  stringascii_lettersr  dictionary_endictionary_rudictionary_checksuggestninja_reversr  r   r  r>  r  r  r1  r  r  r   morphparser!  normal_formr   r   r  r  r"  )r   r  rG  lang
dictionaryfiltered_suggestions3suggestionsfiltered_suggestionsnwr   r  filtered_suggestions2suggestions2_listsuggr  all_suggestionsnormalized_wordssynonym_wordssynonym_listsynonymr   s                        r   r  zCmfFullSearch.prepare_wordQ	  s   	3tfB7f***D ..JD ..J "''-$,,T2K#% %%d+B?2$b11$++B/EE|$EA  6q6Q;Q447?qttAw!8		#u-AEEy2..E)003FA9QCr**$++A.Q%0 %'! & 1 1 4 4 @ @ H H J R R T 
! A, 	a6t9>7d1g%aDG);<<U3DEE}TF"55E)006FA	 ""67#>S:TTX\W]]O  $fO 5  	9Akk!nRa( 9 !$$R^^4:bnn%5R889	9 *,<< !,,11&$M]ae`fMfHg9h:@&9I<E;9:1 2 ?L ( -<<$\\//55c:2A> -GGI--c5971#R0%))!,	-- *M9C@U<VVO$$r   c                    t        |       dk  ry g }|d d D ]i  }|d   j                  d      s|d   j                  d      s,i }|d   |d<   |j                  dd       r|d   d   |d<   nd |d<   |j                  |       k t        j                         }||_        d	|_        | |_        t        j                  j                  |_        d
|_        t        j                         5  |j!                          d d d        y # 1 sw Y   y xY w)Nr  r  r)  zCmfDocument:zCmfTask:r   r  r   searchF)r   
startswithrP  r  r>  CmfSearchStatr  actionr  r   r3  r)  	person_id
aggregatedr  disable_aclsave)r  r  st_dictrecst_recstats         r   r  z!CmfFullSearch._do_calc_statistics	  s    |q CR= 	#CI((8CI<P<PQ[<\F"4yF8wwx&'*8}T':|$'+|$NN6"	# ##%(**  " 	IIK	 	 	s   &D  D	)Fr"  )FFr"  )F)NNNrk  NNNNNFFNNNNNNNNNNNNNNNNNNNN)Nr  F)FF)r  r   r   ztuple[str, set[str]])FNNFNFN)NNr"  NNNFN)r  
str | Noner-  rG  )r  	list[str]r   tuple[str, dict[str, str]])rZ  r   r   rI  )r   rH  )TFFF)T)9r   r   r   r   r  r  	api_allowapi_methodsstaticmethodr   r   classmethodr  r  r  r  r!  rL  rT  rZ  r_  rf  r  r  r  r  r  r>  	BaseModelr  r  cmf_deferred_jobr   r   r  r  r&  r   r   r   r   r5  r=  r   r  rw  r   ry  r<  r2  rv  r  r  rs  rt  r  r  r   r   r   r   r   m   sI    KO IK 	 	 	 l l  $ K K p p   :F :Fx g g . .  <  8  ' !""&Io ob  &  $ T Tl 	cjj&:&: 	 	 l lB %; & ;| 	
 	
 VWX]= Y ]=~ " "& " "B @@ @ 	@
 @  9@ @ !@ @  @ s)@ @ #@ @D LL L 	L
 L  9L L !L L  L s)L L #L L\  rvyI yIv + +
 Z Zx 
d 
$ 
# 
 
 fhnrp  p d  &*-1$+#h~ #~ +	~ ~@ ] ]~ ( ( & &  '  'D f fP k% k%\   r   r   ))rd  r/  collectionsr   typingr   r   r   r   rz  r   r	   r
   r  sys	itertoolsr=  cmf.data_providers.sqlalchemyr   cmf.fields.cmf_full_searchr  enchantr"  	pymorphy3r  r  bs4r   MorphAnalyzerr)  RUSSIAN_STOP_WORDSENGLISH_STOP_WORDSr   r  r  r   r   r   r   cmf_full_searchr   r   r   r   <module>r]     s      " , , 0 0   
   > !    	   	 	! n  A '*<<=_ . +  !E !E !EH E E E   n%CJJ..<< n%r   