B
    fp               @   sV  d dl Z d dlZd dlmZ d dlT d dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ e Ze	dZe	dZdd	 Zd
dddddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdpdqdrdsdsdtdudvdwdxdydzd{d|d}d~d~ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd ddddddddd	d
dddddddddddddddddddddddddd d!d"d"d#d$d$d%d&d'd(d)d)d)d*d+d,d-d.d.d/d0d1d2d3d4d5d6d7d7d8d9d:d:d;d<d=d>d>d>d?d@dAdBdBdCdDdEdFdGdGdHdIdIdIdJdKdLdMdNdOdOdPdQdQdQdRdSdSdTdUdVdWdWdXdYdYdZdZd[d\d]d]d^d_d`dadadbdbdcdcdddedfdgdhdidjdkdldmdndndodpdpdqdrdsdtdudvdwdwdxdxdydydzd{d|d|d|d}d}d~d~dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddÐdÐdÐdÐdÐdÐdÐdÐdĐdĐdŐdŐdŐdŐdŐdŐdƐdƐdƐdƐdƐdǐdǐdǐdǐdǐdǐdȐdȐdȐdȐdȐdȐdȐdȐdɐdɐdɐdɐdɐdɐdɐdɐdɐdɐdʐdʐdʐdʐdʐdʐdʐdʐdʐdːdːdːdːdːdːd̐d̐d̐d̐d̐d̐d̐d̐d̐d̐d͐d͐d͐d͐d͐d͐d͐d͐d͐d͐d͐dΐdΐdΐdΐdΐdΐdΐdΐdΐdΐdΐdΐdΐdΐdϐdϐdϐdϐdϐdϐdϐdϐdϐdϐdϐdϐdАZd
dѐdҐdӐdԐdՐd֐dאdؐdِdd+dېdܐdݐdސdߐddddddddddddddddddddddddddddddddd ddddddddd	d
ddiddddddddddddddddddddddd d!d"d"d#d$d%d&d'd(dd)d*d+d,dd-ddd.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>dd?d@dAdBdCdDdEdFdGdHdIdJdKdLdMddNdOdPdQdRdSdTdȐdUdVdWdXdXdYdZdZdѐd[d\d]d^d_d`dadbddݐdcdcdddedfdgdhdidjdkdlddmdnddodpdqdddrdrdsdsddtdudvdvdwdxdydydzd{d|d}d d~ddddddddddddddddddddddddddddddddddddddddd d ddd#dd%dddd)dd,dd-dddd/dd2dddddd6dd9d;d;ddddddd@ddddddBdBddCddDdEddFddddHdJddKddLdNdNdddOddVdVdddYd[dd]d`ddddddddddddddpdpdÐdÐdÐdĐdrdrdŐdŐdududƐdwdǐdzdzd{d}d}d~d~ddȐddddɐdɐdʐdddːddd̐d̐d͐d͐dΐdΐdΐddϐdАdАdАddѐddddddddddddddҐdҐdҐdҐdӐdӐddddddddddddԐdԐdԐdddddddddddddddddՐdՐddd֐ddddddddddddddddddddddddddddddddddddddddddddddÐdÐdÐdÐdÐdĐdĐdŐdŐdŐdŐdŐdƐdƐdǐdǐdǐdȐdȐdȐdȐdȐdȐdȐdɐdɐdɐdɐdɐdʐdʐdʐdːdːdːdːdːdːdːdːd̐d̐d̐d̐d̐d͐d͐d͐d͐d͐d͐d͐d͐d͐d͐d͐dΐdΐdΐdΐdΐdΐdΐdΐdΐdϐdϐdϐdϐdϐdϐdאdאdאdאdאdאdאdאdאdאdאdאdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdِdِdِdِdِdِdِdِdِdِdِdِdِdِdِdِdِdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdڐdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdېdd d d d d d d d d d d d d d d d d d d d d d d d d dܐZdݐdބ ZdZdZeeeeZeeeeZdd Zdd Z dfddZ!dfddZ"dfddZ#G dd dej$j%j&Z&dS (      N)
namedtuple)*)SQLAlchemyDataDriver)BeautifulSoupen_USru_RUc             C   s.   | d t jkrt|  S t|  S d S )Nr   )stringascii_lettersdictionary_enZcheck
capitalizedictionary_ru)word r   ./cmf/models/cmf_full_search.pydictionary_check   s    r   i'  i$  i#  iH   ib  iC  i  i  i}  i  i+  i  i  is  iC  i  i  i  ip  i  i  i  i  i  i  ia  i  i  iy  iC  i  i  i  i  i  i  i  i  i  i[  i?  i  i  i  i  i  i:  i  i  iO  i>  i  i  i  i  iZ  i  i  i  i  i9  i
  i
  i
  in
  im
  iU
  iJ
  i/
  i$
  i
  i	  i	  i	  i	  i[	  i?	  i0	  i  i  iD  i/  i$  i  i  i  i  i  i  i  iM  i?  i8  i2  i  i  iu  iR  iM  iC  i8  i6  i4  i   i  i  i  i  i  i  i  i  i  i  i  i  iw  ih  if  iY  iL  i<  i2  i#  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iy  ig  ie  iV  iP  iL  iA  i6  i0  i.  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iv  ij  ic  i^  i[  iZ  iU  iR  iJ  iH  iF  iC  i=  i9  i%  i!  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iw  in  ic  i^  i]  iZ  iN  iI  iF  iE  i8  i0  i)  i  i  i  i  i  i  i
  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i}  ix  is  iq  ip  io  im  il  if  id  ic  i\  i[  iX  iT  iO  iN  iL  iD  i?  i>  i9  i8  i6  i3  i2  i0  i.  i*  i'  i$  i"  i   i  i  i  i  i  i
  i  i  i  i  i                                                                                                                                                                          ~   {   z   y   v   u   s   q   p   o   m   l   k   j   i   g   f   e   c   a   `   _   ^   \   [   Z   Y   W   V   U   R   Q   N   M   J   I   G   F   D   C   B   A   @   ?   >   =   :   9   8   7   6   3   2   1   0   /   .   -   ,   +   *   (   '   %   $   #   "   !                                                                      
   	            (  u   стu   тоu   ноu   наu   поu   енu   ниu   коu   раu   овu   неu   роu   прu   алu   гоu   реu   осu   лиu   каu   воu   ерu   таu   отu   ваu   орu   олu   етu   теu   омu   анu   елu   одu   тьu   лаu   онu   леu   лоu   есu   атu   риu   льu   деu   огu   веu   ныu   тиu   заu   итu   скu   даu   акu   ойu   инu   меu   емu   илu   чеu   обu   асu   доu   моu   киu   трu   сяu   едu   соu   миu   ннu   сеu   амu   исu   авu   жеu   азu   маu   имu   твu   арu   виu   иеu   руu   боu   слu   изu   выu   диu   быu   ияu   икu   егu   пеu   ивu   чтu   всu   ейu   сьu   окu   ьнu   оеu   нуu   сиu   ихu   этu   хоu   адu   чаu   ожu   аяu   спu   ляu   евu   ичu   муu   чиu   шеu   тсu   днu   куu   удu   ыхu   саu   екu   беu   очu   ийu   паu   циu   крu   знu   опu   аеu   дуu   ииu   щеu   туu   ыеu   ыйu   ирu   тыu   озu   внu   ылu   рыu   ятu   утu   тнu   жиu   нтu   усu   ссu   шиu   идu   ымu   няu   гаu   свu   езu   луu   ктu   мыu   мнu   ееu   ужu   уюu   влu   улu   грu   пиu   ажu   брu   абu   учu   дрu   цеu   аюu   ечu   снu   ачu   гиu   чнu   суu   глu   люu   жнu   ахu   ебu   буu   рнu   урu   ьсu   ывu   щиu   ньu   звu   плu   апu   жаu   гдu   ряu   нсu   ькu   угu   блu   оиu   лсu   ждu   ашu   баu   биu   умu   ицu   епu   ртu   здu   ютu   зоu   ежu   врu   длu   укu   упu   клu   шаu   ешu   ошu   пуu   смu   агu   двu   вуu   кеu   ткu   ацu   йсu   ояu   айu   зиu   иоu   дыu   убu   нкu   ысu   ытu   игu   ехu   ющu   ещu   хаu   ндu   дсu   рмu   ялu   геu   янu   гуu   июu   лыu   ушu   рсu   фиu   тяu   цаu   зыu   вшu   охu   ишu   змu   фоu   ьшu   иаu   нцu   ксu   ямu   шкu   увu   рьu   зуu   ргu   ибu   рвu   ядu   срu   зеu   чуu   шлu   феu   мяu   явu   унu   шьu   лнu   ьюu   ркu   ьеu   чкu   дьu   ущu   щаu   ышu   ооu   еоu   ухu   бяu   вкu   сыu   ясu   втu   йнu   зрu   ецu   юдu   язu   бнu   лкu   бщu   узu   ржu   вяu   мпu   кнu   ллu   дяu   уеu   квu   счu   шнu   взu   юбu   оцu   ижu   ипu   яеu   ьяu   пыu   ащu   дкu   рдu   лжu   офu   аиu   хиu   ящu   яхu   хнu   ощu   злu   фаu   ырu   экu   вдu   шоu   бсu   ынu   зяu   шуu   згu   тлu   ьмu   впu   хрu   млu   ьтu   йтu   жуu   гнu   еаu   пяu   ыкu   ршu   нгu   бъu   еяu   ммu   оюu   яюu   ьзu   ыбu   цыu   ъеu   кцu   схu   сдu   ыпu   афu   чьu   якu   ауu   ючu   ищu   дпu   рхu   нчu   мсu   уаu   фрu   вьu   цоu   дцu   ычu   йчu   ефu   тдu   збu   хуu   зкu   лгu   хвu   пнu   ифu   вмu   нюu   джu   нфu   оэu   йдu   яжu   ппu   ыдu   еиu   штu   жкu   юсu   еюu   мкu   цуu   ячu   юрu   рбu   элu   дмu   йкu   сшu   рюu   сюu   дтu   рпu   щуu   хлu   тчu   ярu   ъяu   ррu   чшu   ьбu   ыгu   бкu   йшu   птu   мьu   рчu   тмu   гкu   зьu   пкu   рлu   еуu   ьцu   нщu   ягu   ьгu   кжu   псu   бхu   уйu   сбu   цкu   фуu   мбu   ьиu   пьu   лчu   яяu   сцu   оуu   тпu   цвu   яцu   ттu   энu   лтu   ызu   щнu   хеu   тюu   ябu   ддu   зжu   сфu   нрu   яйu   нвu   дшu   ввu   йоu   рцu   юнu   флu   аоu   жоu   хсu   рфu   дчu   ьчu   чоu   эрu   ффu   вчu   юзu   лдu   сёu   эфu   ьдu   бюu   бвu   жбu   уяu   тцu   нзu   фтu   вгu   хмu   члu   юмu   дхu   ймu   дгu   тбu   йцu   эмu   эпu   щьu   вхu   ётu   юкu   япu   жчu   оаu   ыжu   бмu   рзu   мрu   эсu   юлu   ёнu   йлu   ккu   юцu   вцu   ыяu   съu   щёu   чрu   дзu   хтu   кзu   дъu   швu   иуu   бьu   ёмu   дбu   юшu   чёu   жьu   мвu   южu   мчu   мцu   мгu   уцu   жсu   лбu   лпu   уиu   ююu   нжu   гчu   сгu   ьвu   еёu   зсu   пцu   лёu   яиu   ёлu   яшu   уфu   гсu   аэu   фсu   дюu   лмu   югu   рщu   вщu   ёрu   эдu   лзu   гмu   гвu   дёu   мфu   зъu   шпu   мэu   эвu   пчu   кмu   ьфu   бжu   тхu   ншu   эйu   кпu   уэu   гтu   нбu   ьоu   бдu   шмu   кгu   зчu   шёu   юхu   сжu   ббu   нёu   кшu   ззu   тщu   фыu   зцu   ггu   лхu   йеu   ааu   тёu   ьщu   ыиu   тфu   нхu   тгu   рэu   лвu   бшu   рёu   вёu   ьпu   лрu   зтu   ххu   йзu   эзu   чвu   шрu   ыщu   оёu   вбu   нпu   кдu   сэu   йбu   жлu   тъu   жмu   бзu   уоu   жрu   сзu   жгu   тзu   жёu   жжu   фьu   йфu   пшu   зюu   иэu   эхu   тэu   лшu   бэu   йрu   хгu   бтu   хшu   гбu   мтu   йгu   фнu   нэu   тшu   фгu   аёi!  iF  i  iL  iW  i  i  i	  i  i[  i  il  if  iD  iw  iR  i$  i  i  i  i  i  ix  i  i
  i
  ic
  i2
  i%
  i
  i	  i	  i	  i:	  i.	  i 	  i	  i	  i  i  i  i  i:  i  i  i  i  i  iJ  i=  i+  i*  i  i  i  i	  i  i  i  i  i  iw  iQ  iO  i-  i  i  i  i  i  i  i  i  iZ  iU  iH  iC  i>  i5  i3  i  i  i  i  i  i  i  i  i  i  i  i  im  ia  iS  iN  i-  i  i  i  i  i  i  i  i  i  i  i  i  i  i~  i{  il  if  ie  iY  iW  iL  iG  i@  i/  i"  i  i
  i	  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  it  ik  ih  i_  i\  iU  iT  iQ  iG  i>  i,  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iw  it  ih  i`  i_  iY  iV  iJ  iH  iF  iE  iC  iB  iA  i<  i:  i4  i/  i,  i+  i&  i!  i  i  i  i  i  i  i                                                                                               }   |   x   t   r   h   d   b   X   T   P   O   L   K   H   <   ;   4   )   &                  (  thheinZerZanreesZonstntenZatZedZndtoorZeaZtiarteZngZalitasisZhaetZseZouZoflesaveroZraZrihinemedecotaZecsiZllZsoZnaZliZlaelZmaZdiZicZrtnsZrsioZomZchZotcaZceZhobettfotsssnoeeZemZacZildaZniurwashZeiamtrZdtusloZpeZunZncZwiutZadZewZowgeZepZaiZlyZolZftoseoZefZprZweZdomoidZiemipafiZpoZctZwhZirZaygaZscZkeZevspZimopZdsZldZulZooZsuiaZghplZebZigviZivZwoZyoZrdZtwZbaZagZryZabZlsswZapZfeZtuZcifaZhtfravZegZgoZboZbuZtyZmpocZodZehZysZeyZrmZovgtZyaZckZgiZrnZgrZrcZblltZytZoaZyeobZdbZffZsfZrrZduZkiZucifafZdrZclexZsmZpiZsbZcrtlZoiruZupZbyZtcnnZakslZnfZueZdwZauppZugZrlZrgbrZcuZuaZdhZrkyiZluZumZbinynwZquZogZsnZmbZvaZdfddmsZgsZawZnhZpuhrsdtbptZnmZdcZguZtmZmuZnuZmmnleuZwnnbZrpZdmsrZudZuiZrfokZywZtfZiprwrbZohksdpZfuZyctpmtZdlZnkZccZubZrhZnpZjuZflZdnkaphhuZjoZlfZybrvZoeZibZikZypglZlpZymZlbZhsZdgZgnZeknrZpsZtdZlcskZyfZyhZvoZahZdyZlmZsyZnvZydZfsZsgZyrZylZwsZmyZoyknZizZxpZlwtnkoZaajaZzeZfcZgwtgZxtZfhZlrZjeZynZggZgfeqZhyZktZhcbsZhwZhncsZhmZnjZhhZwtgcZlhZejZfmZdvlvwrZgpfpZgbZgmZhlZlkcyZmcZygZxiZhbfwZgyZhpZmwZpmZzaZlgiwZxaZfbsvgdZixZajklZhfZhdZaesqZdjZfyazZlnZaofdkwZmfZmhZsjZufZtvZxcZyuZbbZwwZojZaxmrZwlZxeZkhZoxZuoZziZfgZihZtkiiiuZtjZmnZwykyZkffnZuyZpwZdkZrjukZkrZkuZwmZkmZmdmlZezZkbZwcZwdZhgZbtZzoZkcZpfZyvZpcpywbZykZcpZyjZkpZpbZcdZjiZuwZuhZwfZyyZwpZbcZaqcbZiqcmZmgZdqZbjZtzZkdpdZfjZcfZnzZcwZfvZvyZfkZozZzzZijZljZnqZuvZxoZpgZhkZkgZvsZhvZbmZhjZcngvZcgZwuZgjxhZgkZtqZcqZrqZbhZxsuzZwkZxuZuxZbdZbwZwgZmvZmjZpnZxmZoqZbvZxwZkkZbpzuZrzZxfmkzhZbnZzyZhqZwjZiyZdzZvrZzsZxyZcvZxbZxrZujZyqZvdZpkZvuZjrZzlZszZyzZlqZkjZbfZnxZqaZqiZkvZzwZwvuuZvtZvpZxdZgqZxlZvcczZlzZztZwzZsxZzbZvlZpvZfqZpjZzmZvwZcjZzcbgZjsZxgrxhzZxxZvmZxnZqwZjpZvnZzdZzrZfzZxvZzpZvhZvbZzfZgzZtxZvfZdxZqbZbkZzgZvgZjcZzkZznZuqZjmZvvZjdZmqZjhZqsZjtZjbZfxZpqZmzZyxZqtZwqZjjZjwZlxZgxZjnZzvZmxZjkZkqZxkZjfZqmZqhZjlZjgZvkZvjZkzZqcZxjZpzZqlZqoZjvZqfZqdZbzhxZzjZpxZqpZqeZqrZzqZjyZbqZxqZcxZkxZwxZqyZqvZqnZvxZbxZjzZvzZqgZqqZzxZxzZqkZvqZqjZqxZjxZjqZqzc                sZ   t dd| | krdS  fddt| | dd  D }t|dkrFdS t|t| d S )Nz
[\[\];',.] gHz>c                s    g | ]\}}  || d qS )r   )get).0ab)statr   r   
<listcomp>#   s    z$bigram_text_rank.<locals>.<listcomp>r   r   )r   subziplensum)textr|  Zstatsr   )r|  r   bigram_text_rank    s     r  u   йцукенгшщзхъфывапролджэячсмитьбюЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮz@qwertyuiop[]asdfghjkl;'zxcvbnm,.QWERTYUIOP[]ASDFGHJKL;'ZXCVBNM,.c                s\    t kr| d tjkr|  S  tkr<| d tjkr<|  S d fdd| D }| S )Nr   rw  c             3   s   | ]}  ||V  qd S )N)rx  )ry  x)dict_r   r   	<genexpr>3   s    z'ninja_translate_text.<locals>.<genexpr>)rus2engr   	printablelowereng2rusjoin)r  r  retr   )r  r   ninja_translate_text.   s    r  c             C   sB   | d t jkrt| t}n
t| t}tdd||kr:|S d S d S )Nr   z
[\[\];',.]rw  )r   r  r  r  r  r   r~  )r   r  r   r   r   ninja_revers6   s    
r  Fc             C   s   t | dk r| S t| t}t| t}t|t}t|t}|rpt|  d| dd||   d| dd||   	 t|rt|s|S t|rt|s|S d||  dkr|S d||  dkr|S | S d S )Nr   z
 rank_ru:  r   z rank_eng: g?)	r  r  r  r  r  bigram_stat_rubigram_stat_engprintr   )r   debugZ	trans_engZ	trans_rusZrank_ruZrank_engr   r   r   
ninja_once@   s"    



4r  c             C   s4   d| kr| S t | |}|| kr,|  d| S | S d S )N@|)r  )r   r  Z
ninja_wordr   r   r   	ninja_addU   s    
r  c                s   d  fdd| dD S )Nr  c                s   g | ]}t | qS r   )r  )ry  w)r  r   r   r}  _   s    zninja.<locals>.<listcomp>)r  split)r  r  r   )r  r   ninja^   s    r  c               @   s  e Zd ZdZdZddddddd	d
dddddddddddddddgZdZdddgZee	ddd Z
ed!d" Zed#d$ Zed%d& Zed'd( Zed)d* Zed+d, Zeded/d0Zed1d2 Zed3d4 Zeejjd5d6d7Zedfd9d:Zeedd;d<d= Zed>d? Zeeddd@dAdBdCdD ZedEdF ZedGdHdIdJdKZedgdLdMZ edNgd-dOdPd8gd-d-fdQdRZ!ed-d-dNgd-dOdPd8gfdSdTdUdVZ"edWdXdYdZd[Z#edWd\d]d^Z$edhd_d`Z%edidadbZ&eedcdd Z'd-S )jCmfFullSearchuC   
    Сервис полнотекстового поиска.
    russiannameZtext_renderr  Z
text_drafttags	parent_idtree_parent_id
project_idcmf_created_atcmf_modified_atcodeZcmf_deletedZcmf_archivedZloginZemailZphoneZphone_internalZphone_mobileZphone_2Zphone_assistantZ
ip_addressZemail_2ZbirthdayTfulltext_searchrun_force_reindexindex_stats)r  c             C   sR   | rNt | dd} t| dkrNtdt|  d |  d d jdd} | S )	Nzhtml.parser
i  z)CmfFullSearch.clean_text: trunc text len z to 1mbi  ignore)errors)r   get_textr  gr  encodedecode)r  r   r   r   
clean_text~   s    zCmfFullSearch.clean_textc             C   s   t |ddjdd S )Nzhtml.parserr  r  )r  )r   r  r  r  )clsZtext_with_htmlr   r   r   
strip_html   s    zCmfFullSearch.strip_htmlc                s@   dd t k rgS  fddtdt D }|S )Ni r   c                s    g | ]}||    qS r   r   )ry  i)overlapr  text_len_maxr   r   r}     s    z-CmfFullSearch._strip_size.<locals>.<listcomp>r   )r  range)r  r  resr   )r  r  r  r   _strip_size   s    "zCmfFullSearch._strip_sizec             C   s   d dd td|D S )Nr  c             S   s   g | ]}t |d k r|qS )r}   )r  )ry  r  r   r   r   r}     s    z+CmfFullSearch._strip_50.<locals>.<listcomp>z\s)r  r   r  )r  r  r   r   r   	_strip_50   s    zCmfFullSearch._strip_50c             C   s   d dd td|D S )Nr  c             S   s   g | ]}|qS r   r   )ry  r  r   r   r   r}     s    z1CmfFullSearch._strip_not_word.<locals>.<listcomp>z[ \s<>|&\/%@!`+=;.])r  r   r  )r  r  r   r   r   _strip_not_word   s    zCmfFullSearch._strip_not_wordc             C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kjd|j d}
||
 n0|  }	| j|	|d|j dd}|| d S )Nzempty obj_idr   T)is_dirtydirty_at)r  obj_idr  r  part_no)
ValueError
sqlalchemyr?  data_driverdp_model_cls	__table__Sessionselectcr  wherer  r  with_for_updateexecutefirstupdatevaluesfuncnowgen_idinsert)r  r  r   r1  sa_modeltablesget_stmtget_resid_update_stmtinsert_stmtr   r   r   
mark_dirty   s4    
zCmfFullSearch.mark_dirtyc             C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kjd|j d}
||
 | j  d S )Nzempty obj_idr   F)r  r  )r  r  r?  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  commit)r  r  r   r1  r  r  r  r  r  r  r  r   r   r   
mark_clean   s&    

zCmfFullSearch.mark_cleanNFc       %   "   C   s  |}|  |}| |}|r@| |}| |}|  |d }n
d g}d }|rh|}|  |}| |}nd }d }|r|}|  |}| |}nd }d }|r| |} | d d } |  | }!nd } d }!|r| |d d }|  |}"nd }"| jd|||||||||	||||||
||||d |||||||| |!||"d t|dkrxrt|dd  dD ]\\}#}$|  |$}| j|#|||||||||	||||||
||||$||d d d d d d d ||"d qXW | j|t|d | j  d S )Nr   i )r  r  	obj_modelobj_parent_idobj_tree_parent_idobj_project_idobj_created_atobj_modified_atobj_codeobj_deletedobj_archivedobj_owner_nameobj_author_nameobj_modified_by_nameobj_responsible_namesobj_hrefobj_logic_type_codeobj_activity_codeobj_status_typeobj_texttext_for_vecobj_namename_for_vecobj_tagstags_for_vecobj_result_textresult_text_for_vecobj_commentscomments_for_vecobj_custom_fieldscf_text_for_vecr   )delete_from_partno)	r  r  r  _text_search_sql_insertr  	enumerate_text_search_sql_delete_partnor?  r  )%r  
model_namer  r  r  r  r  Zcomments_textr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Zobj_text_listr  r  r  Zobj_comments_textZcomments_text_for_vecr  r  Zobj_text_partr   r   r   index_object   s|    










zCmfFullSearch.index_objectc       	      C   sh   |st ddd l}| jj}|| }|j}| }|||j	j
|k|j	j|k}|| d S )Nzempty obj_idr   )r  r  r?  r  r  r  r  deleter  r  r  r  r  )	r  r  r  r   r1  r  r  r  Zdel_stmtr   r   r   r  d  s    

z,CmfFullSearch._text_search_sql_delete_partnoc        *   #   C   s  |st ddd l} | jj}!|!| }"|"j}#|! }$| |#jj	g
|#jj|k
|#jj|k }%|$|% }&|&r:|&d }'|# 
|#jj	|'kj|d||||||||||||| jj| j|| jj| j|| jj| j|| jj| j|| jj| j|| jj| j||	|
|||||||||d}(|$|( n|  }'|# j|'d||||||||||||||| jj| j|| jj| j|| jj| j|| jj| j|| jj| j|| jj| j||	|
|||||||||d!})|$|) |'S )Nzempty obj_idr   F)r  r  r  r  r  r  r  r  r  r  r  r  r  r  name_tsvectortext_tsvectortags_tsvectorresult_text_tsvectorcomments_tsvectorcustom_fields_tsvectorr  r  r  r  r  r  r  r  r  r  r  )!r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )r  r  r?  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  sqlr  Zto_tsvector_fts_configr  r  )*r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r1  r  r  r  r  r  r  r  r  r   r   r   r  w  s    

z%CmfFullSearch._text_search_sql_insert)objc             C   s@   |j s
d S x0| jD ]&}|dkr q||jkr|| jrdS qW d S )Nr  T)full_searchrequired_fieldsfieldsZ
is_changed)r  r  
field_namer   r   r   is_obj_need_reindex  s    z!CmfFullSearch.is_obj_need_reindexr   c          
   C   s   xt jj D ]}|jsq|j}|r.||kr.qd}t|}|j}|rLdg}xt		 }	|j
|||| gdgddd}
|
szP x*|
D ]"}|rtj|jj q|  qW |t|
7 }| j  td| d| dt		 |	 d	d
 qNW qW d S )Nr   r  r  T)r
  sliceorder_byinclude_archivedinclude_deletedzCmfFullSearch.reindex_models: :z, z0.3z sec)cmfmodels	CmfEntityiter_subclassesr  
class_namecmfutilget_model_by_namefull_search_fieldstimelistr  r  r  valuefull_search_indexr  r?  r  r  r  )r  Zmodels_listZcommit_everyZlazyZ	model_clsr  offsetmodelZ
field_listr  obj_listr  r   r   r   reindex_models  s8    



zCmfFullSearch.reindex_models)	only_oncec               C   s   t j  t j S )N)r  CmfAccessListcheck_admin_moder  r!  r   r   r   r   r    s    
zCmfFullSearch.run_force_reindexc             C   s(   t j  |  }| jdd}||dS )NT)r  )totaldirty_count)r  r#  r$  count)r  r%  r&  r   r   r   r     s
    
zCmfFullSearch.index_statsz	@minutelyr   )r"  Z
system_jobZschedulepriorityc        	   
   C   s  t d tj r"t d d S td t } d}xxtjj	dgdddgd	d
dt
j
 t
jdd gd
dd gggddgd}|sP i }x8|D ]0}t|j}||krg ||< || |j qW x| D ]\}}t d| dt|  t| xn|D ]f}|j|jdd|gdd}|d7 }|sJt d|  tj| q t d|j  |  q W qW t |  dkrt d| d P t d| d td q<W t dt |    d S )NzStart cron_index_dirtyuJ   Не индексируем поскольку запущен импортr   r   r  r  =TORr  <r   )Zminutesr   )r
  filterr  zcron_index_dirty process r  r  )r
  r,  r  r   u.   Объект не найдет в бд obj_id=zobj_id=r   ua   cron_index_dirty превышен лимит времени 20 секунд обработано u    объектовu&   cron_index_dirty обработано u    объектов, sleep 1zEnd cron_index_dirty at )r  r  r  Zenable_import_modeZimport_is_runningr  Zsleepr  r  slistdatetimer  Z	timedeltaZget_model_by_idr  appenditemsr  r  rx  r  r  r  r  )	r   nZobj_id_listZobj_ids_by_modelr  r  Zid_listr  r  r   r   r   cron_index_dirty,  sP    






zCmfFullSearch.cron_index_dirtyc             C   sP   |S ]:}|sq|d dkr4t|dks|d dkr4q|| qW d|}|S )u   
        Подчистка оригинального квери, который ввел пользователь:
        - удаление стоп-слов
        r  r   -r   )r  r  r/  r  )r  search_queryZclean_search_query_listr  Zclean_search_queryr   r   r   _clean_search_queryc  s     
z!CmfFullSearch._clean_search_querystrztuple[str, set[str]])r4  returnc             C   s   t  }| d}t|dkr$| |fS d} xj|dd D ]Z}|dkrDq6td|d}|d dkrl||d  t|dkr6|  |d  |d  } q6W |  } | |fS )	zExtrats tags from the given search_query and returns its reminder and a set of extracted tags

        Args:
            search_query (str)

        Returns:
            tuple[str, list[str]]: search_query reminder and a set of extractd tags
        #r   rw  Nz(\W)r   r   r   )setr  r  r   addstrip)r4  r  Zsharp_splittedtokenZ
sub_tokensr   r   r   _extract_tagsw  s    

zCmfFullSearch._extract_tagsc       H      K   s  |d krd}d|krd|d< d|kr,d|d< |s8ddg}t jjdd}	|	d	kr\td
|	  dt_|}
t }|d}|r|| | 	|\}}t
||}||d< |}| }|}t|}|dkrd}t
|}d|d< |dr|d d }g }g }g }g }g }g }g }g }g }g }g }g }g }g }g }g } g }!g }"g }#g }$g }%g }&g }'g }(g })|dkrxdd tj j D }*n8t|t jkrd}d}dd tj j D }*n|g}*|d}+| |+|d< |dd},| j|ddd}-| j|dd}.| j|dd}/| j|ddd}0|/t_tdd|.}1|1t_tdd|.}2|dksL|dkr|dkr|,s| j|*fttjjddgdd |d!|}$| j|*fddgdd |d"|}%nt| jd#|*|.fddg|d$|-|d%|}|,s| jd|*|/f|ddgd&|d'|}| jd(|*|.f|ddgd&|d'|}|dks0|dkr|dkr|,s| jdgfttjj|d)|d*|}&| jdgf|d)|d+|}'nN|
dkr| jd#dg|.f|d$|-|d,|}| jd#dg|/f|d-|-d.|}|,r|d#kr| j|dg|.f|d/d0|}| j|dg|1fd1|d2|}| j|dg|2fd3d4i|}| j|dg|/f|d&d0|}| jd(dg|.f|d&d0|} t t!|t!|t!|t!|t!|t!|t!| fd5k r| j|dg|0f|d&d0|dd6 }"|dks|dkr|dkr>|,s| jdgfttjj|d)|d*|}(| jdgf|d)|d+|})nN|
dkr| jd#dg|.f|d$|-|d,|}| jd#dg|/f|d-|-d.|}|,r|d#kr| j|dg|.f|d/d0|}| j|dg|1fd1|d2|}| j|dg|2fd3d4i|}| j|dg|/f|d&d0|}| jd(dg|.f|d&d0|}!t t!|t!|t!|t!|t!|t!|t!|!fd5k r| j|dg|0f|d&d0|dd6 }#g }3d7|" kr(t#d8d9}4x~t jj$d:d;d<d=d>gd?d;d@|" gd:d@|" ggdAD ]F}5|5j%r|5j%d d	 nd}6|3&|4|5j'|5j(|5j)|5j*|6dBdddC qW g }7t }8dDdE }9d}:t+dF x<|s|s|s|$s|%s|s|s|s|s|s|s|s|s|s|s|s|s| s|!s|"s|#s|&s|(s|'s|)s|3	r|:r.|9|3|7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 d}:|9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9||7|8 |9| |7|8 |9|!|7|8 |9|"|7|8 |9|#|7|8 |9||7|8 |9|&|7|8 |9|'|7|8 |9|(|7|8 |9|)|7|8 |9|$|7|8 |9|%|7|8 qLqLW |7|d |dG  }7|sVi };|
r@|dd}<|dd}=i }>x$|7D ]}?|>,|?j(g &|?j' 	qW xX|>- D ]B}t|j
|dHdI|>| g|<|=dJ}@x|@D ]}A|A|;|Aj.< 
q$W 	qW nt/dK g }Bx|7D ]}?|?j'|?j0|?j)|?j1 dL|?j2 d7|?j3dMd7|?j4dNdOtj 	|?j2|?j3|?j4dP}C|;|?j'}A|A
rRyN|A5  x6|D ].}D|D6dQd }Dt7|;|?j' |Dd }E|E|C|D< 
qW |B&|C W n t8k
r   Y nX 
qRW t+dR |sRt9t jj:|t;|Bd d5 dSdT |BS i }Fg }Gxj|7D ]b}?|G&|?j' |?j'|?j0|?j)|?j1 dL|?j2 d7|?j3dMd7|?j4dNdOtj 	|?j2|?j3|?j4dP|F|?j'< qdW |Ft_<t+dR |GS )UNrw  archivedFdeletedr   r   T)r  r!   u   Идет процесс индексации, могут быть доступны не все результаты поиска. Осталось объектов: tag_nameANYr  ZModelc             S   s   g | ]}|j r|jqS r   )r  r  )ry  mr   r   r   r}    s    z1CmfFullSearch.fulltext_search.<locals>.<listcomp>commentsc             S   s   g | ]}|j r|jqS r   )r  r  )ry  rC  r   r   r   r}    s    r  tree_parent_filtertitles_only)synonyms
stop_words)rG  )rG  
first_wordz([&] )([^!])z<1> \2z<2> \2)CmfTaskCmfDocumentrJ  rK  ZFS)related_usermodel_name_not_inr  labelr  )rM  r  rN  r  r  N)rM  r  rN  text_stop_words
like_queryS)r  rM  rN  rQ  custom_fieldsZFA)rL  r  rN  r  )r  rN  r  )r  rN  rP  rQ  ZNs)r  rN  rP  A)r  rN  ZH1)rN  rQ  rN  ZH2r   r   r  SearchResultTuplez<obj_id,obj_model,obj_code,title,headline,label,rank,age_daysr  r  r  r  r  r*  r)  )r
  r,  ZCODE)r  r  r  titleheadlinerN  rankage_daysc             S   s@   x:| r:|  d}|d |kr|| ||d  dS qW dS )Nr   TF)popr/  r:  )r   resultskip_idsrr   r   r   add_if_exists  s    

z4CmfFullSearch.fulltext_search.<locals>.add_if_existszfulltext_search Start mixingr   r  IN)r
  r,  r  r  uY   DEV: FATAL. Укажите в запросе поиска список полей fields=z ||| z.6fz.0fz words=)r  r  r  rW  rN  rX  rY  .zfulltext_search END)r4  obj_dict)kwargs)=r  r  r'  Zcmf_noter  FSTr9  rx  r:  r=  r  unionr  r  endswithr  r  r  r  r  Z
CmfComment_get_all_branchesprepare_search_queryFSTQr   r~  ZFSTQHfilter_oncer6  current_userr  search_oncer  r  r;  r   r-  r  r/  r  r  r  r  r  
setdefaultkeysr  	cmf_alertrV  rW  rN  rX  rY  Z_acl_check_readr  getattrZCmfPermissionErrorZschedule_deferred_job_do_calc_statisticsZ
dumps_dictfulltext_search_headlines)Hr  r  r  r4  Zonly_idsr  r
  Zno_analitycsrb  r&  Zorig_field_namer  r@  Zextracted_tagsZorig_search_queryZlike_search_queryZfullsearch_sliceZother_res_name_foundZother_res_text_foundZother_res_cf_foundZtask_res_name_foundZdocument_res_name_foundZtask_res_name_syn_foundZdocument_res_name_syn_foundZtask_res_all_foundZdocument_res_all_foundZtask_res_strong_position_h1Zdocument_res_strong_position_h1Ztask_res_strong_position_h2Zdocument_res_strong_position_h2Ztask_res_syn_foundZdocument_res_syn_foundZtask_res_cf_foundZdocument_res_cf_foundZtask_res_first_wordZdocument_res_first_wordZother_filter_rel_foundZother_filter_foundZtask_filter_rel_foundZtask_filter_foundZdocument_filter_rel_foundZdocument_filter_foundZother_model_namesr  rF  rH  Ztsquery_without_synZtsquery_with_synZtsquery_first_wordZtsquery_strong_position_h1Ztsquery_strong_position_h2Zres_code_foundrU  Zany_res_code_foundrW  r[  r\  r^  Zdo_topZobjectsZis_archivedZ
is_deletedZids_by_modelr]  r   r  r  ra  Zfieldattrrq  Z	result_idr   r   r   r    s   

















&







,



,
zCmfFullSearch.fulltext_searchZnull000rw  r   c
       $      K   s  |
 dd }|
 d}|
 dd }|
 dd }|
 dd }|
 d}|
 d}|
 dd }|
 d	}|
 d
d }|
 d|}|p|}|
 d}|rdd|nd}|
 dpg }| |\}}|dkrd }|	dkrd }	|dkrtd| dd d| }d| }|dkrd}| d}|d }|d |d  }d}|rVt|}d| d}tjjj	
 d d} |dkr|d} d jf ||||||d!}!tjjj	
 |!|||t|t||||	| |||||||||||d"|}"t|"}#|#S )#Nr  modified_by_nameresponsible_namelogic_type_codestatus_typemodified_at_beforemodified_at_afterr>  r?  
owner_nameauthor_namerE  z AND obj_tree_parent_id IN ('{}')z','rw  r@  )r  r  r  rD  rS  u6   Недопустимое значение field_name: T)abortobj_r  r  Z	_tsvectorr   r   z$obj_modified_at > now() - interval 'z
 days' ANDz"set gin_fuzzy_search_limit=100000;r  i@B u  
            SELECT 
                subs.*, 
                ts_headline(
                    'russian', 
                    substring(subs.{search_field} FROM 0 FOR 100000), 
                    :tsquery, 
                    'MaxFragments=1,MaxWords=25,MinWords=24'
                ) AS headline 
            FROM (
                (    
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        ts_rank_cd({tsvector_field}, :tsquery, 2|8)
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        {tsvector_field} @@ :tsquery
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )
                    ORDER BY rank desc
                    LIMIT :slice_for OFFSET :slice_from
                )
            UNION
                /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                (    
                    SELECT
                        obj_id,
                        obj_code,
                        obj_model,
                        obj_name as title,
                        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
                        100
                        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
                        EXTRACT(days from (now()-obj_modified_at)) as age_days,
                        {search_field} as {search_field}
                    FROM cmf_full_search
                    WHERE
                        {age_days_subquery}
                        /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
                        ( {search_field} ILIKE '%' || :like_query || '%'
                            or obj_code = :like_query
                            or obj_id = :like_query
                        )
                        and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
                        and part_no <= :max_partno
                        and ( :parent_id is null or obj_parent_id = :parent_id )
                        AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
                        AND ( :author_name IS NULL OR obj_author_name = :author_name )
                        AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                        and ( :responsible_name is null or obj_responsible_names ILIKE '%' || :responsible_name || '%' )
                        {tags_filter}
                        {tree_parent_filter}
                        and ( :logic_type_code is null or obj_logic_type_code = :logic_type_code )
                        and ( :status_type is null or obj_status_type = :status_type )
                        AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                        AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                        and ( :archived is null or obj_archived = :archived )
                        AND ( :deleted IS NULL OR obj_deleted = :deleted )

                        /*and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))*/
                    /*ORDER BY obj_id desc -- не будем делать, т.к. тормозит*/
                    LIMIT :slice_for/10 
                    OFFSET :slice_from/10
                )
            ) as subs
            ORDER BY subs.rank desc
            LIMIT :slice_for 
            OFFSET :slice_from;
        )rE  search_fieldtsvector_fieldage_days_subqueryheadline_fieldtags_filter)tsquery
slice_from	slice_formodel_name_inrM  rN  rP  rQ  
max_partnor  ry  rz  rs  rt  ru  rv  rw  rx  r>  r?  )rx  formatr  _build_tags_filterrn  intr  r  r?  r  r  r  tupler  )$r  r  r  Ztsquery_strrM  rY  rN  r  rP  rQ  rb  r  rs  rt  ru  rv  rw  rx  r>  r?  ry  rz  rE  
tags_namesr  tags_paramsr}  r  r~  r  r  r  r  r  
found_objsall_objsr   r   r   rk    s    









^
zCmfFullSearch.search_oncez
str | None)rL  c       !      K   s  | d}	| d}
| d}| d}| d}| d}| d}| d}| d	}| d
d }| d|}|px|}| d}|rdd|nd}| dpg }| |\}}d}|r| }}d}|dkrtd| dd d| }|d }|d |d  }d}|r&t|}d| d}|dkr4dnd}d| d| d | d!| d"| d#| d#| d$}tjjj	
 |||t|t||||	|||
|||||d%|}t|} | S )&Nr  rs  rt  ru  rv  rw  rx  r>  r?  ry  rz  rE  z AND obj_tree_parent_id IN ('{}')z','rw  r@  z
            AND ( :responsible_name IS NULL OR obj_responsible_names ILIKE '%' || :responsible_name || '%' )
            AND ( :owner_name IS NULL OR obj_owner_name = :owner_name )
        z
            AND (
                :responsible_name IS NULL 
                OR obj_responsible_names ILIKE '%' || :responsible_name || '%' 
                OR obj_owner_name = :owner_name
            )
        )r  r  r  u6   Недопустимое значение field_name: T)r{  r|  r   r   z$obj_modified_at > now() - interval 'z
 days' ANDr  i@B a  
            SELECT 
                obj_id,
                obj_code,
                obj_model,
                obj_name as title,
                :label || (CASE WHEN EXTRACT(DAYS FROM (NOW() - obj_modified_at)) < 365 then 'R' else '' end) as label,
                100 / (EXTRACT(DAYS FROM (NOW() - obj_modified_at)) + 1) as rank,
                EXTRACT(DAYS FROM (NOW() - obj_modified_at)) as age_days,
                z as z,
                substring(z` FROM 0 FOR 240) as headline
            FROM cmf_full_search
            WHERE
                a  
                obj_model IN :model_name_in AND obj_model NOT IN :model_name_not_in
                AND part_no <= :max_partno
                AND ( :parent_id IS NULL OR obj_parent_id = :parent_id )
                AND ( :author_name IS NULL OR obj_author_name = :author_name )
                AND ( :modified_by_name IS NULL OR obj_modified_by_name = :modified_by_name)
                z
                aF  
                AND ( :logic_type_code IS NULL OR obj_logic_type_code = :logic_type_code )
                AND ( :status_type IS NULL OR obj_status_type = :status_type )
                AND ( :modified_at_before IS NULL OR obj_modified_at <= :modified_at_before )
                AND ( :modified_at_after IS NULL OR obj_modified_at >= :modified_at_after )
                AND ( :archived IS NULL OR obj_archived = :archived )
                AND ( :deleted IS NULL OR obj_deleted = :deleted )
            ORDER BY rank desc
            LIMIT :slice_for OFFSET :slice_from;
        )r  r  r  rM  rN  r  r  ry  rz  rs  rt  ru  rv  r>  r?  )rx  r  r  r  rn  r  r  r  r?  r  r  r  r  r  )!r  r  rL  r  rM  rY  rN  r  rb  r  rs  rt  ru  rv  rw  rx  r>  r?  ry  rz  rE  r  r  r  Zowner_and_responsible_filterr}  r  r  r  r  r  r  r  r   r   r   ri  T  sd    










 0zCmfFullSearch.filter_oncez	list[str]ztuple[str, dict[str, str]])r  r7  c             C   sL   i }d}x:t | D ].\}}d|d  }|||< | d| d}qW ||fS )Nrw  tagr   z AND obj_tags ILIKE '%' || :z || '%')r  )r  r  r  r1  r  Z	tag_paramr   r   r   r    s    z CmfFullSearch._build_tags_filter)r7  c             C   s0   | sg S t jjj dd| i}dd |D S )Na/  
                WITH tree_parents AS (
                    WITH RECURSIVE r AS (
                        SELECT obj_id, obj_code, obj_tree_parent_id
                        FROM cmf_full_search
                        WHERE obj_tree_parent_id = :tree_parent_id

                        UNION

                        SELECT cfs.obj_id, cfs.obj_code, cfs.obj_tree_parent_id
                        FROM cmf_full_search AS cfs
                        JOIN r ON cfs.obj_tree_parent_id = r.obj_id
                    )
                    SELECT obj_id FROM r
                    WHERE r.obj_id IN (SELECT obj_tree_parent_id FROM r)
                    
                    UNION
                    
                    SELECT :tree_parent_id
                )
                SELECT * FROM tree_parents;
            r  c             S   s   g | ]}|d  qS )r   r   )ry  r]  r   r   r   r}    s    z3CmfFullSearch._get_all_branches.<locals>.<listcomp>)r  r  r?  r  r  r  )r  Zrecordsr   r   r   rf    s    
zCmfFullSearch._get_all_branchesc          
   C   sn  dt _|dd}tdd|}td|}d}d}d}	x|D ]z}
t|
dkrTq@t|
dkr|
dkrjq@|
d	kr|	d|
 7 }	q@|
dkrq@|
d
kr|	d7 }	q@|
dkr|	d7 }	q@t|
dkrq@|
d dkrt|
dkrq@|	d|
dd   7 }	|d|
dd   7 }q@|rq@|
dd}
|d7 }|dkr2d}|	rF|	d dkrN|	d7 }	t|
dkr| j|
|d}t|dkr|	|d  7 }	n|	dd| d 7 }	|r|dkrP |dkr@P q@q@W |rt|dkr|d dkr|dd  }|S |	dddddd d!d 	 }	|	r:|	d dkr:|	dd  }	|	rZ|	d dkrZ|	dd  }	|	rz|	d dkrz|	dd  }	|	r|	dd  dkr|	d d }	y.t
jjj d"d#|	i}t|d d }	W n tjjk
rV } zft
jjj  td$|  td%|	 d| dt j  t
jjj d&d#|i}t|d d }	W d d }~X Y nX t|d'|	 |	t _|	S )(Nrw  zwww.u   [^-A-Za-zА-Яа-я0-9()|&!' ]r  z(,| |&|\||\(|\))r   r   )r3  !z()&|)r   u   илиr  z |)u   иand&z &r   z& !r  r3  r   F)r  r  z& )rG  z( z | z )r   )r  r  z OR z or z AND z & z and z!select to_tsquery('russian', :q);qu;   Ошибочный синтаксиса в запросе: u@   DEV: Ошибочный синтаксиса в запросе: z+select websearch_to_tsquery('russian', :q);z->)r  rc  replacer   r~  r  r  prepare_wordr  r;  r  
CmfSynonymr?  r  r  r  r  r  excZProgrammingErrorZrollbackrn  r  rh  )r  r4  rG  rH  rI  Zsearch_query_allowed_symbtokensZ
word_countZstopsr  tZ	sug_wordsr  er   r   r   rg    s    
(
"z"CmfFullSearch.prepare_search_queryc             C   s&  t  jd| d7  _|d tjkr.d}t}nd}t}g }t|s||}g }t|}|r~t  jd| d7  _|	| t  jd7  _d}	x|D ]}
|	d	krP t
|
d	krq|
d |d kr|
d
 |d
 krqd|
kr|
dd}
t  jd|
 d7  _|	|
 |	d
7 }	qt  jd|
 d7  _|	|
 |	d
7 }	qW g }tjjj dd|i}d}	x|D ]\}}|	d
kr|P t
|d	krqh|d |d ks|d
 |d
 krh|dd}t  jd| d7  _|	| |	d
7 }	qhW t|t|B |hB }n|h}t }xP|D ]H}
x@t|
d d	 D ]*}||j t  jd|j d7  _q.W qW ||B }t }|rtjjddt||hB gddgdgdd
gd}xh|D ]`}|jrxP|jjdd d D ]6}| dd}t  jd| d7  _|| qW qW ||B t|B }t|S )Nz|w:z: r   r   r)  zaddNinjaRevers z, zspellError, r   r   r  z<->z	addSpell z
            SELECT
                name, similarity(:word, name) as sim
            FROM cmf_synonym
            WHERE
                :word % name
            ORDER BY "sim" desc
            LIMIT 5;
             r   zaddSpellTrgm z
normalize r  r_  r  Zorderno)r,  r
  r  r  ,r   zsynAdd )r  rc  r   r	   r
   r   r   Zsuggestr  r/  r  r  r  r  r?  r  r  r  r9  morphparser:  Znormal_formr  r  r  r  r;  )r  r   rG  langZ
dictionaryZfiltered_suggestions3ZsuggestionsZfiltered_suggestionsr0  r  r  Zfiltered_suggestions2Zsuggestions2_listZsugg_Zall_suggestionsZnormalized_wordsZsynonym_wordsZsynonym_listZsynonymr  r   r   r   r  [  s    



 




$

$
zCmfFullSearch.prepare_wordc             C   s   t | dk rd S g }xr|d d D ]b}|d dsD|d dsDq"i }|d |d< |dd rr|d d |d< nd |d< || q"W t }||_d	|_| |_t	j
j|_d
|_|  d S )Nr   r   r  zCmfDocument:zCmfTask:r  parentr  searchF)r  
startswithrx  r/  r  ZCmfSearchStatr   actionr4  r  rj  r  Z	person_idZ
aggregatedZsave)r4  ra  Zst_dictZrecZst_recr|  r   r   r   rp    s&    
z!CmfFullSearch._do_calc_statistics)NNNFFNNNNNNNNNNNNNNN)Nr   F)FNNF)TFF)T)(__name__
__module____qualname____doc__r  r	  Z	api_allowZapi_methodsstaticmethodr6  r  classmethodr  r  r  r  r  r  r  r  r  r  r  Z	BaseModelr  r!  Zcmf_deferred_jobr  r  r2  r5  r=  r  rk  ri  r  rf  rg  r  rp  r   r   r   r   r  i   s   

*                   Ru!6!    
 3j"dnr  )'r.  r  collectionsr   Zcmf.includeZcmf.data_providers.sqlalchemyr   Zcmf.fields.cmf_full_searchr  Zenchantr   Z	pymorphy2r   r  Zbs4r   ZMorphAnalyzerr  ZDictr
   r   r   r  r  r  Zrus_keyZeng_keydictr  r  r  r  r  r  r  r  r
  Zcmf_full_searchr  r   r   r   r   <module>   s\   

                  N
	