U
    e                    @   s  d dl Z d dlZd dlT d dlmZ d dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ e	 ZedZedZddd	d
dddddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdmdndodpdpdqdrdsdtdudvdwdxdydzd{d{d|d}d~ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd ddddddddd	d
ddddddddddddddddddddddddddd d!d!d"d#d$d%d&d&d&d'd(d)d*d+d+d,d-d.d/d0d1d2d3d4d4d5d6d7d7d8d9d:d;d;d;d<d=d>d?d?d@dAdBdCdDdDdEdFdFdFdGdHdIdJdKdLdLdMdNdNdNdOdPdPdQdRdSdTdTdUdVdVdWdWdXdYdZdZd[d\d]d^d^d_d_d`d`dadbdcdddedfdgdhdidjdkdkdldmdmdndodpdqdrdsdtdtdududvdvdwdxdydydydzdzd{d{d|d}d~ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddÐdÐdÐdÐdÐdĐdĐdĐdĐdĐdĐdŐdŐdŐdŐdŐdŐdŐdŐdƐdƐdƐdƐdƐdƐdƐdƐdƐdƐdǐdǐdǐdǐdǐdǐdǐdǐdǐdȐdȐdȐdȐdȐdȐdɐdɐdɐdɐdɐdɐdɐdɐdɐdɐdʐdʐdʐdʐdʐdʐdʐdʐdʐdʐdʐdːdːdːdːdːdːdːdːdːdːdːdːdːdːd̐d̐d̐d̐d̐d̐d̐d̐d̐d̐d̐d̐d͐ZddΐdϐdАdѐdҐdӐdԐdՐd֐dd(dؐdِdڐdېdܐdݐdސdߐddddddddddddddddddddddddddddddddd dddddddddfd	d
ddddddddddddddddddddddddd d!d"d#d$d%dd&d'd(d)dd*ddd+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd<d=d>d?d@dAdBdCdDdEdFdGdHdIdJddKdLdMdNdOdPdQdŐdRdSdTdUdUdVdWdWdΐdXdYdZd[d\d]d^d_ddڐd`d`dadbdcdddedfdgdhdiddjdkddldmdndddododpdpddqdrdsdsdtdudvdvdwdxdydzdd{d d|dd}d}d~dddddddd
ddddddddddddddddddddddddddddddd dd"dddd&dd)dd*dddd,dd/dddddd3dd6d8d8ddddddd=dddddd?d?dd@ddAdBddCddddEdGddHddIdKdKdddLddSdSdddVdXddZd]ddddddddddddddmdmdddddododddrdrdÐdtdĐdwdwdxdzdzd{d{d|dŐd}dddƐdƐdǐdddȐdddɐdɐdʐdʐdːdːdːdd̐d͐d͐d͐ddΐddddddddddddddϐdϐdϐdϐdАdАddddddddddddѐdѐdѐdddddddddddddddddҐdҐdddӐddddddddddddddddddddddddddddddddddddddddddddddddddddddddddÐdÐdĐdĐdĐdŐdŐdŐdŐdŐdŐdŐdƐdƐdƐdƐdƐdǐdǐdǐdȐdȐdȐdȐdȐdȐdȐdȐdɐdɐdɐdɐdɐdʐdʐdʐdʐdʐdʐdʐdʐdʐdʐdʐdːdːdːdːdːdːdːdːdːd̐d̐d̐d̐d̐d̐dԐdԐdԐdԐdԐdԐdԐdԐdԐdԐdԐdԐdՐdՐdՐdՐdՐdՐdՐdՐdՐdՐd֐d֐d֐d֐d֐d֐d֐d֐d֐d֐d֐d֐d֐d֐d֐d֐d֐dאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdאdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdؐdd d d d d d d d d d d d d d d d d d d d d d d d d dِZdڐdۄ ZdZdZeeeeZeeeeZdސd߄ Zdd Zdd ZG dd dejj j!Z!e"j#$e!j% dS (      N)*)SQLAlchemyDataDriver)BeautifulSoupen_USru_RUi'  i$  i#  iH   ib  iC  i  i  i}  i  i+  i  i  is  iC  i  i  i  ip  i  i  i  i  i  i  ia  i  i  iy  iC  i  i  i  i  i  i  i  i  i  i[  i?  i  i  i  i  i  i:  i  i  iO  i>  i  i  i  i  iZ  i  i  i  i  i9  i
  i
  i
  in
  im
  iU
  iJ
  i/
  i$
  i
  i	  i	  i	  i	  i[	  i?	  i0	  i  i  iD  i/  i$  i  i  i  i  i  i  i  iM  i?  i8  i2  i  i  iu  iR  iM  iC  i8  i6  i4  i   i  i  i  i  i  i  i  i  i  i  i  i  iw  ih  if  iY  iL  i<  i2  i#  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iy  ig  ie  iV  iP  iL  iA  i6  i0  i.  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iv  ij  ic  i^  i[  iZ  iU  iR  iJ  iH  iF  iC  i=  i9  i%  i!  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iw  in  ic  i^  i]  iZ  iN  iI  iF  iE  i8  i0  i)  i  i  i  i  i  i  i
  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i}  ix  is  iq  ip  io  im  il  if  id  ic  i\  i[  iX  iT  iO  iN  iL  iD  i?  i>  i9  i8  i6  i3  i2  i0  i.  i*  i'  i$  i"  i   i  i  i  i  i  i
  i  i  i  i  i                                                                                                                                                                          ~   {   z   y   v   u   s   q   p   o   m   l   k   j   i   g   f   e   c   a   `   _   ^   \   [   Z   Y   W   V   U   R   Q   N   M   J   I   G   F   D   C   B   A   @   ?   >   =   :   9   8   7   6   3   2   1   0   /   .   -   ,   +   *   (   '   %   $   #   "   !                                                                      
   	            (  u   стu   тоu   ноu   наu   поu   енu   ниu   коu   раu   овu   неu   роu   прu   алu   гоu   реu   осu   лиu   каu   воu   ерu   таu   отu   ваu   орu   олu   етu   теu   омu   анu   елu   одu   тьu   лаu   онu   леu   лоu   есu   атu   риu   льu   деu   огu   веu   ныu   тиu   заu   итu   скu   даu   акu   ойu   инu   меu   емu   илu   чеu   обu   асu   доu   моu   киu   трu   сяu   едu   соu   миu   ннu   сеu   амu   исu   авu   жеu   азu   маu   имu   твu   арu   виu   иеu   руu   боu   слu   изu   выu   диu   быu   ияu   икu   егu   пеu   ивu   чтu   всu   ейu   сьu   окu   ьнu   оеu   нуu   сиu   ихu   этu   хоu   адu   чаu   ожu   аяu   спu   ляu   евu   ичu   муu   чиu   шеu   тсu   днu   куu   удu   ыхu   саu   екu   беu   очu   ийu   паu   циu   крu   знu   опu   аеu   дуu   ииu   щеu   туu   ыеu   ыйu   ирu   тыu   озu   внu   ылu   рыu   ятu   утu   тнu   жиu   нтu   усu   ссu   шиu   идu   ымu   няu   гаu   свu   езu   луu   ктu   мыu   мнu   ееu   ужu   уюu   влu   улu   грu   пиu   ажu   брu   абu   учu   дрu   цеu   аюu   ечu   снu   ачu   гиu   чнu   суu   глu   люu   жнu   ахu   ебu   буu   рнu   урu   ьсu   ывu   щиu   ньu   звu   плu   апu   жаu   гдu   ряu   нсu   ькu   угu   блu   оиu   лсu   ждu   ашu   баu   биu   умu   ицu   епu   ртu   здu   ютu   зоu   ежu   врu   длu   укu   упu   клu   шаu   ешu   ошu   пуu   смu   агu   двu   вуu   кеu   ткu   ацu   йсu   ояu   айu   зиu   иоu   дыu   убu   нкu   ысu   ытu   игu   ехu   ющu   ещu   хаu   ндu   дсu   рмu   ялu   геu   янu   гуu   июu   лыu   ушu   рсu   фиu   тяu   цаu   зыu   вшu   охu   ишu   змu   фоu   ьшu   иаu   нцu   ксu   ямu   шкu   увu   рьu   зуu   ргu   ибu   рвu   ядu   срu   зеu   чуu   шлu   феu   мяu   явu   унu   шьu   лнu   ьюu   ркu   ьеu   чкu   дьu   ущu   щаu   ышu   ооu   еоu   ухu   бяu   вкu   сыu   ясu   втu   йнu   зрu   ецu   юдu   язu   бнu   лкu   бщu   узu   ржu   вяu   мпu   кнu   ллu   дяu   уеu   квu   счu   шнu   взu   юбu   оцu   ижu   ипu   яеu   ьяu   пыu   ащu   дкu   рдu   лжu   офu   аиu   хиu   ящu   яхu   хнu   ощu   злu   фаu   ырu   экu   вдu   шоu   бсu   ынu   зяu   шуu   згu   тлu   ьмu   впu   хрu   млu   ьтu   йтu   жуu   гнu   еаu   пяu   ыкu   ршu   нгu   бъu   еяu   ммu   оюu   яюu   ьзu   ыбu   цыu   ъеu   кцu   схu   сдu   ыпu   афu   чьu   якu   ауu   ючu   ищu   дпu   рхu   нчu   мсu   уаu   фрu   вьu   цоu   дцu   ычu   йчu   ефu   тдu   збu   хуu   зкu   лгu   хвu   пнu   ифu   вмu   нюu   джu   нфu   оэu   йдu   яжu   ппu   ыдu   еиu   штu   жкu   юсu   еюu   мкu   цуu   ячu   юрu   рбu   элu   дмu   йкu   сшu   рюu   сюu   дтu   рпu   щуu   хлu   тчu   ярu   ъяu   ррu   чшu   ьбu   ыгu   бкu   йшu   птu   мьu   рчu   тмu   гкu   зьu   пкu   рлu   еуu   ьцu   нщu   ягu   ьгu   кжu   псu   бхu   уйu   сбu   цкu   фуu   мбu   ьиu   пьu   лчu   яяu   сцu   оуu   тпu   цвu   яцu   ттu   энu   лтu   ызu   щнu   хеu   тюu   ябu   ддu   зжu   сфu   нрu   яйu   нвu   дшu   ввu   йоu   рцu   юнu   флu   аоu   жоu   хсu   рфu   дчu   ьчu   чоu   эрu   ффu   вчu   юзu   лдu   сёu   эфu   ьдu   бюu   бвu   жбu   уяu   тцu   нзu   фтu   вгu   хмu   члu   юмu   дхu   ймu   дгu   тбu   йцu   эмu   эпu   щьu   вхu   ётu   юкu   япu   жчu   оаu   ыжu   бмu   рзu   мрu   эсu   юлu   ёнu   йлu   ккu   юцu   вцu   ыяu   съu   щёu   чрu   дзu   хтu   кзu   дъu   швu   иуu   бьu   ёмu   дбu   юшu   чёu   жьu   мвu   южu   мчu   мцu   мгu   уцu   жсu   лбu   лпu   уиu   ююu   нжu   гчu   сгu   ьвu   еёu   зсu   пцu   лёu   яиu   ёлu   яшu   уфu   гсu   аэu   фсu   дюu   лмu   югu   рщu   вщu   ёрu   эдu   лзu   гмu   гвu   дёu   мфu   зъu   шпu   мэu   эвu   пчu   кмu   ьфu   бжu   тхu   ншu   эйu   кпu   уэu   гтu   нбu   ьоu   бдu   шмu   кгu   зчu   шёu   юхu   сжu   ббu   нёu   кшu   ззu   тщu   фыu   зцu   ггu   лхu   йеu   ааu   тёu   ьщu   ыиu   тфu   нхu   тгu   рэu   лвu   бшu   рёu   вёu   ьпu   лрu   зтu   ххu   йзu   эзu   чвu   шрu   ыщu   оёu   вбu   нпu   кдu   сэu   йбu   жлu   тъu   жмu   бзu   уоu   жрu   сзu   жгu   тзu   жёu   жжu   фьu   йфu   пшu   зюu   иэu   эхu   тэu   лшu   бэu   йрu   хгu   бтu   хшu   гбu   мтu   йгu   фнu   нэu   тшu   фгu   аёi!  iF  i  iL  iW  i  i  i	  i  i[  i  il  if  iD  iw  iR  i$  i  i  i  i  i  ix  i  i
  i
  ic
  i2
  i%
  i
  i	  i	  i	  i:	  i.	  i 	  i	  i	  i  i  i  i  i:  i  i  i  i  i  iJ  i=  i+  i*  i  i  i  i	  i  i  i  i  i  iw  iQ  iO  i-  i  i  i  i  i  i  i  i  iZ  iU  iH  iC  i>  i5  i3  i  i  i  i  i  i  i  i  i  i  i  i  im  ia  iS  iN  i-  i  i  i  i  i  i  i  i  i  i  i  i  i  i~  i{  il  if  ie  iY  iW  iL  iG  i@  i/  i"  i  i
  i	  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  it  ik  ih  i_  i\  iU  iT  iQ  iG  i>  i,  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  i  iw  it  ih  i`  i_  iY  iV  iJ  iH  iF  iE  iC  iB  iA  i<  i:  i4  i/  i,  i+  i&  i!  i  i  i  i  i  i  i                                                                                               }   |   x   t   r   h   d   b   X   T   P   O   L   K   H   <   ;   4   )   &                  (  thheinZerZanreesZonstntenZatZedZndtoorZeaZtiarteZngZalitasisZhaetZseZouZoflesaveroZraZrihinemedecotaZecsiZllZsoZnaZliZlaelZmaZdiZicZrtnsZrsioZomZchZotcaZceZhobettfotsssnoeeZemZacZildaZniurwashZeiamtrZdtusloZpeZunZncZwiutZadZewZowgeZepZaiZlyZolZftoseoZefZprZweZdomoidZiemipafiZpoZctZwhZirZaygaZscZkeZevspZimopZdsZldZulZooZsuiaZghplZebZigviZivZwoZyoZrdZtwZbaZagZryabZlsswZapZfeZtuZcifaZhtfravZegZgoZboZbuZtyZmpocZodZehZysZeyZrmZovgtZyaZckZgiZrnZgrrcZblltZytZoaZyeobZdbZffZsfZrrZduZkiZucifafZdrZclexZsmZpiZsbZcrtlZoiruZupZbyZtcnnZakslZnfZueZdwZauppZugZrlZrgbrZcuZuaZdhZrkyiZluZumZbinynwZquZogZsnZmbZvaZdfddmsZgsZawZnhZpuhrsdtbptZnmZdcZguZtmZmuZnuZmmnleuZwnnbZrpZdmsrZudZuiZrfokZywZtfZiprwrbZohksdpZfuZyctpmtZdlZnkZccZubZrhZnpZjuZflZdnkaphhuZjoZlfZybrvZoeZibZikZypglZlpZymZlbZhsZdgZgnZeknrZpsZtdZlcskZyfZyhZvoZahZdyZlmZsyZnvZydZfsZsgZyrZylZwsZmyZoyknZizZxpZlwtnkoZaajaZzeZfcZgwtgZxtZfhZlrZjeZynZggZgfeqZhyZktZhcbsZhwZhncsZhmZnjZhhZwtgcZlhZejZfmZdvlvwrZgpfpZgbZgmZhlZlkcyZmcZygZxiZhbfwZgyZhpZmwZpmZzaZlgiwZxaZfbsvgdZixZajklZhfZhdZaesqZdjZfyazZlnZaofdkwZmfZmhZsjZufZtvZxcZyuZbbZwwZojZaxmrZwlZxeZkhZoxZuoZziZfgZihZtkiiiuZtjZmnZwykyZkffnZuyZpwZdkZrjukZkrZkuZwmZkmZmdmlZezZkbZwcZwdZhgZbtZzoZkcZpfZyvZpcpywbZykZcpZyjZkpZpbZcdZjiZuwZuhZwfZyyZwpZbcZaqcbZiqcmZmgZdqZbjZtzZkdpdZfjZcfZnzZcwZfvZvyZfkZozZzzZijZljZnqZuvZxoZpgZhkZkgZvsZhvZbmZhjZcngvZcgZwuZgjxhZgkZtqZcqZrqZbhZxsuzZwkZxuZuxZbdZbwZwgmvZmjZpnZxmZoqZbvZxwZkkZbpzuZrzZxfmkzhZbnZzyZhqZwjZiyZdzZvrZzsZxyZcvxbZxrZujZyqZvdZpkZvuZjrZzlZszZyzZlqZkjZbfZnxZqaZqiZkvZzwZwvuuZvtZvpZxdZgqZxlZvcczZlzZztZwzZsxZzbZvlZpvZfqZpjZzmZvwZcjZzcbgZjsZxgrxhzZxxZvmZxnZqwZjpZvnZzdZzrZfzZxvZzpZvhZvbzfgzZtxZvfZdxZqbZbkZzgZvgZjcZzkZznZuqZjmZvvZjdZmqZjhZqsZjtZjbZfxZpqZmzZyxZqtZwqZjjZjwZlxZgxZjnZzvZmxZjkZkqZxkZjfZqmZqhZjlZjgZvkZvjZkzZqcZxjZpzZqlZqoZjvZqfZqdZbzhxZzjZpxZqpZqeZqrZzqZjyZbqZxqZcxZkxZwxZqyZqvZqnZvxZbxZjzZvzZqgZqqZzxxzZqkZvqZqjZqxZjxZjqZqzc                    sD    fddt | | dd  D }t|dkr0dS t|t| d S )Nc                    s0   g | ](\}}|  r|  r || d qS )r   )isalphaget).0abstat ./cmf/models/cmf_full_search.py
<listcomp>   s       z$bigram_text_rank.<locals>.<listcomp>r   r   gHz>)ziplensum)textrz  Zstatsr{  ry  r|  bigram_text_rank   s     r  u   йцукенгшщзхъфывапролджэячсмитьбюЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮz@qwertyuiop[]asdfghjkl;'zxcvbnm,.QWERTYUIOP[]ASDFGHJKL;'ZXCVBNM,.c                    s   d  fdd| D S )N c                 3   s   | ]}  ||V  qd S N)ru  )rv  xdict_r{  r|  	<genexpr>$   s     z'ninja_translate_text.<locals>.<genexpr>)join)r  r  r{  r  r|  ninja_translate_text#   s    r  c                 C   sX   t | t}t | t}t|t}t|t}d||  dkr<|S d||  dkrP|S | S d S )Nr   g?)r  rus2engeng2rusr  bigram_stat_rubigram_stat_eng)wordZ	trans_engZ	trans_rusZrank_ruZrank_engr{  r{  r|  
ninja_once&   s    



r  c                 C   s   d dd | dD S )N c                 S   s   g | ]}t |qS r{  )r  rv  wr{  r{  r|  r}  4   s     zninja.<locals>.<listcomp>)r  splitr  r{  r{  r|  ninja3   s    r  c                   @   sb  e Zd ZdZdZddddddd	d
dddddddddddddddgZeedddZe	dd Z
e	dd  Ze	d!d" Ze	d#d$ Ze	d%d& Ze	dHd)d*Ze	d+d, Ze	d-d. Ze	ejjd/d0d1Ze	dId3d4Zeejd5d5d6d7d8 Ze	d9d: Ze	d(d;d2gfd<d=Ze	d>gd'd?d;d2gd'd'fd@dAZe	dJdBdCZe	dKdDdEZe	dFdG Zd'S )LCmfFullSearchuC   
    Сервис полнотекстового поиска.
    russiannameZtext_renderr  Z
text_drafttagsZ	parent_idZtree_parent_idZroot_parent_idcmf_created_atcmf_modified_atcodeZcmf_deletedZcmf_archivedZloginZemailZphoneZphone_internalZphone_mobileZphone_2Zphone_assistantZ
ip_addressZemail_2Zbirthdayr  c                 C   sR   | rNt | dd} t| dkrNtdt|  d |  d d jdd} | S )	Nhtml.parser
i  z)CmfFullSearch.clean_text: trunc text len z to 1mbi  ignore)errors)r   get_textr  gdebugencodedecoder  r{  r{  r|  
clean_textM   s    zCmfFullSearch.clean_textc                 C   s   t |ddS )Nr  r  )r   r  )clsZtext_with_htmlr{  r{  r|  
strip_html]   s    zCmfFullSearch.strip_htmlc                    s@   dd t k rgS  fddtdt D }|S )Ni r   c                    s    g | ]}||    qS r{  r{  )rv  iZoverlapr  Ztext_len_maxr{  r|  r}  q   s     z-CmfFullSearch._strip_size.<locals>.<listcomp>r   )r  range)r  r  resr{  r  r|  _strip_sizea   s    "zCmfFullSearch._strip_sizec                 C   s   d dd td|D S )Nr  c                 S   s   g | ]}t |d k r|qS )rs   )r  r  r{  r{  r|  r}  v   s      z+CmfFullSearch._strip_50.<locals>.<listcomp>z\sr  r   r  r  r  r{  r{  r|  	_strip_50t   s    zCmfFullSearch._strip_50c                 C   s   d dd td|D S )Nr  c                 S   s   g | ]}|qS r{  r{  r  r{  r{  r|  r}  }   s     z1CmfFullSearch._strip_not_word.<locals>.<listcomp>z[ \s<>|&\/%@!`+=;.]r  r  r{  r{  r|  _strip_not_wordx   s    zCmfFullSearch._strip_not_wordc                 C   s   |st ddd l}| jj}|| }|j}| }||jj	g
|jj|k
|jjdk }|| }|r|d }	| 
|jj	|	kjd|j d}
||
 n0|  }	| j|	|d|j dd}|| d S )Nempty obj_idr   T)is_dirtydirty_at)r	  obj_idr  r  part_no)
ValueError
sqlalchemyr7  data_driverdp_model_cls	__table__Sessionselectcr	  wherer  r  with_for_updateexecutefirstupdatevaluesfuncnowgen_idinsert)r  r  r   r)  sa_modeltablesget_stmtget_resid_update_stmtinsert_stmtr{  r{  r|  
mark_dirty   sD    




zCmfFullSearch.mark_dirtyNFc           !      C   s  |}|  |}| |}|r@| |}| |}|  |d }n
d g}d }|rh|}|  |}| |}nd }d }|r|}|  |}| |}nd }d }|r| |}|d d }|  |}nd }d }| jd|||||||||	||||
||||d |||||||||d t|dkr~t|dd  dD ]T\}} |  | }| j||||||||||	||||
|||| ||d d d d d d d d q(| j|t|d d S )Nr   i )r  r  	obj_modelobj_parent_idobj_tree_parent_idobj_root_parent_idobj_created_atobj_modified_atobj_codeobj_deletedobj_archivedobj_owner_nameobj_responsible_namesobj_hrefobj_logic_type_codeobj_activity_codeobj_status_typeobj_texttext_for_vecobj_namename_for_vecobj_tagstags_for_vecobj_result_textresult_text_for_vecobj_commentscomments_for_vecr   )delete_from_partno)r  r  r  _text_search_sql_insertr  	enumerate_text_search_sql_delete_partno)!r  
model_namer  r  r  r  r  Zcomments_textr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Zobj_text_listr  r  r  Zobj_comments_textZcomments_text_for_vecr  Zobj_text_partr{  r{  r|  index_object   s    







              

              
zCmfFullSearch.index_objectc           	      C   sh   |st ddd l}| jj}|| }|j}| }|||j	j
|k|j	j|k}|| d S )Nr  r   )r  r  r7  r  r  r  r  deleter  r  r  r  r  )	r  r  r  r   r)  r  r  r  Zdel_stmtr{  r{  r|  r     s    



z,CmfFullSearch._text_search_sql_delete_partnoc           &      C   s  |st ddd l}| jj}|| }|j}| } ||jj	g
|jj|k
|jj|k }!| |! }"|"r$|"d }#| 
|jj	|#kj|d||||||||||||jj| j||jj| j||jj| j||jj| j||jj| j||	|
|||||||d}$| |$ n|  }#| j|#d||||||||||||||jj| j||jj| j||jj| j||jj| j||jj| j||	|
|||||||d}%| |% |#S )Nr  r   F)r  r  r  r  r  r  r  r  r  r  r  r  r  name_tsvectortext_tsvectortags_tsvectorresult_text_tsvectorcomments_tsvectorr  r  r  r  r  r  r  r  r  )r	  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )r  r  r7  r  r  r  r  r  r  r	  r  r  r  r  r  r  r  r  sqlr  Zto_tsvector_fts_configr  r  )&r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r)  r  r  r  r  r  r  r  r  r{  r{  r|  r    s    



!"
z%CmfFullSearch._text_search_sql_insert)objc                 C   s>   |j s
d S | jD ](}|dkrq||jkr|| jr dS qd S )Nr  T)full_searchrequired_fieldsfieldsZ
is_changed)r  r  
field_namer{  r{  r|  is_obj_need_reindexr  s    
z!CmfFullSearch.is_obj_need_reindexr   c              
   C   s   t jj D ]}|jsq|j}d}t }t|}|j	}	|rDdg}	|j
|	||| gdgd}
|
sdq|
D ]"}|rtj|jj qh|  qh|t|
7 }| j  td| d| dt | dd	 q"qd S )
Nr   r	  r  )r  sliceorder_byzCmfFullSearch.reindex_models: :, z0.3z sec)cmfmodels	CmfEntityiter_subclassesr  
class_nametimecmfutilZget_model_by_namefull_search_fieldslistr  r  r	  valuefull_search_indexr  r7  commitr  r  )r  Zmodels_listZcommit_everyZlazyZ	model_clsr  offsetr   modelZ
field_listobj_listr  r{  r{  r|  reindex_models~  s0    
 


zCmfFullSearch.reindex_modelsT)Z	only_onceZsystem_taskc            
   
   O   sV  t d td t }tjjdgdddgddd	tj tj	d
d gddd gggddgd}|sjq:i }|D ]0}t
|j}||krg ||< || |j qr| D ]J\}}t d| dt|  |j|jdd|gd}|D ]}	|	  qqtjj  t | dkr$t d q:t d td qt dt |   d S )NzStart cron_index_dirtyr   r  r  =TORr  <r   )Zminutesr   r   )r  filterr   zcron_index_dirty process r  r	  INr  r  r   uK   cron_index_dirty превышен лимит времени 20 секундuB   cron_index_dirty обработано 25 объектов, sleep 0r   zEnd cron_index_dirty at )r  r  r	  sleepr  r  Zslistdatetimer  Z	timedeltar
  Zget_model_by_idr  appenditemsr  r  r  r  r7  r  )
argskwargsr   Zobj_id_listZobj_ids_by_modelr  r  Zid_listr  r  r{  r{  r|  cron_index_dirty  s@    


	

zCmfFullSearch.cron_index_dirtyc                 C   sN   |S ]:}|sq|d dkr4t|dks|d dkr4q|| qd|}|S )u   
        Подчистка оригинального квери, который ввел пользователь:
        - удаление стоп-слов
        r  r   -r   )r  r  r  r  )r  search_queryZclean_search_query_listr  Zclean_search_queryr{  r{  r|  _clean_search_query  s     
z!CmfFullSearch._clean_search_queryr   c           *      K   s  dt _|}t|}|}| |}	||kr>d|kr>| d| }|dkrJd}t|}
d|
d< |drp|d d }g }g }g }g }g }g }g }g }g }g }g }g }g }g }|dkrd	d
 tjj	 D }n|g}| j
|ddd}| j
|dd}| j
|dd}tdd|}tdd|}|dks*|dkrt| jd||fddg|
d||	d|}| jd||f|
ddgd|	d|}|dks|dkrJ|dkr| jddg|f|
d||	d|}| jddg|f|
d|d|}| j|dg|f|
dd|}| j|dg|fd|	d |}| j|dg|fd!d"i|}| j|dg|f|
dd|}|dks^|dkr |dkr| jddg|f|
d||	d|}| jddg|f|
d|d|}| j|dg|f|
dd|}| j|dg|fd|	d |}| j|dg|fd!d"i|}| j|dg|f|
dd|}d }d| kr`tjjd#d$gd%d&d'| gd#d'| ggd(}|r|s|j|j d)d*dd+gS |jS g } t }!d,d- }"d}#t d. |s|s|s|s|s|s|s|s|s|s|s|s|s|rT|#r8|"|| |! |"|| |! |"|| |! |"|| |! d}#|"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! |"|| |! qtd/D ]<}$td0D ]}%|"|| |! q|"|| |! trttd}&|&d |!krt| |& |!|&d  trtd}&|&d |!kr| |& |!|&d  trtd}&|&d |!kr| |& |!|&d  trtd}&|&d |!kr| |& |!|&d  trtd}&|&d |!kr| |& |!|&d  qq| |d |d1  } |sd2d
 | D }'t d3 |'S i }(g })| D ]b}&|)|&j |&j|&j|&j|&j  d|&j! d|&j"d4d|&j#d5d6t j 	|&j!|&j"|&j#d7|(|&j< q|(t _$t d3 |)S )8Nr  r   OR ANYr  r   ZModelc                 S   s   g | ]}|j r|jqS r{  )r  r  )rv  mr{  r{  r|  r}    s      z1CmfFullSearch.fulltext_search.<locals>.<listcomp>FT)synonyms
stop_wordsr(  z([&] )([^!])z<1> \2z<2> \2)CmfTaskCmfDocumentr  r+  r,  N)model_name_not_inr   labeltext_stop_words
like_queryS)r   r.  r/  r1  )r   r/  r0  r1  ZNs)r   r/  r0  A)r   r/  ZH1)r/  r1  r/  ZH2r  r  r  r  r  r  ZCODEr   )r	  r  headliner/  rankage_daysc                 S   s<   | r8|  d}|d |kr || ||d  dS q dS )Nr   TF)popr  add)r  resultskip_idsrr{  r{  r|  add_if_existsi  s    

z4CmfFullSearch.fulltext_search.<locals>.add_if_existszfulltext_search Start mixingr   r   r   c                 S   sV   g | ]N}|j |j|j|j d |j d |jdd |jddtj 	|j|j|jdqS )r  .6f.0f words=r	  r  r  r4  r/  r5  r6  )	r  titler  r4  r/  r5  r6  r  FST)rv  r;  r{  r{  r|  r}    s   ,zfulltext_search ENDr=  r>  r?  r@  )%r  rB  r  r#  r  endswithr  r  r  r  prepare_search_queryr   subsearch_oncestripr  ru  r  r  setr  r  Zres_strong_position_h1r7  r  r8  Zres_strong_position_h2Zres_all_foundZres_name_syn_foundZres_syn_foundrA  r  r4  r/  r5  r6  fulltext_search_headlines)*r  r  r  r"  Zonly_idsr   r  Zorig_field_nameZorig_search_queryZclean_orig_search_queryZfullsearch_sliceZother_res_name_foundZother_res_text_foundZtask_res_name_foundZdocument_res_name_foundZtask_res_name_syn_foundZdocument_res_name_syn_foundZtask_res_all_foundZdocument_res_all_foundZtask_res_strong_position_h1Zdocument_res_strong_position_h1Ztask_res_strong_position_h2Zdocument_res_strong_position_h2Ztask_res_syn_foundZdocument_res_syn_foundZother_model_namesr)  Ztsquery_without_synZtsquery_with_synZtsquery_strong_position_h1Ztsquery_strong_position_h2Zany_res_code_foundr9  r:  r<  Zdo_top_Ztttr;  r  rI  Z	result_idr{  r{  r|  fulltext_search  s   

   
     
    		










	
,
zCmfFullSearch.fulltext_searchZnull000r  c
                 C   s  |dkrd }|	dkrd }	|dkr2t d| dd d| }
d| }|dkrRd}| d	}|d
 }|d |d
  }d}|rt|}d| d}tjjj d d
}|dkrd}djf |
|||d}tjjj ||||t	|t	||||	|d	}t
|}|S )Nr  )r  r  r  u6   Недопустимое значение field_name: T)abortZobj_r  r  Z	_tsvectorr   r   z$obj_modified_at > now() - interval 'z
 days' ANDz"set gin_fuzzy_search_limit=100000;r  i@B uL	  SELECT subs.*, ts_headline('russian', substring(subs.{search_field} FROM 0 FOR 100000), :tsquery, 'MaxFragments=1,MaxWords=25,MinWords=24') as headline FROM (
(    SELECT
        obj_id,
        obj_code,
        obj_name as title,
        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
        ts_rank_cd({tsvector_field}, :tsquery, 2|8)
        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
        EXTRACT(days from (now()-obj_modified_at)) as age_days,
        {search_field} as {search_field}
    FROM cmf_full_search
    WHERE
      {age_days_subquery}
      {tsvector_field} @@ :tsquery
      and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
      and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))
      and part_no <= :max_partno
    ORDER BY rank desc
    LIMIT :slice_for OFFSET :slice_from
)
UNION
/* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
(    SELECT
        obj_id,
        obj_code,
        obj_name as title,
        :label || (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then 'R' else '' end) as label,
        100
        * (CASE WHEN EXTRACT(days from (now()-obj_modified_at)) < 365 then
                    (365-EXTRACT(days from (now()-obj_modified_at)))/30+1 else 1 end) as rank,
        EXTRACT(days from (now()-obj_modified_at)) as age_days,
        {search_field} as {search_field}
    FROM cmf_full_search
    WHERE
      {age_days_subquery}
      /* Дополнительно проверяем полное совпадение по ilike с высоким рангом */
      {search_field} ILIKE '%' || :like_query || '%'
      and obj_model IN :model_name_in and obj_model NOT IN :model_name_not_in
      and part_no <= :max_partno
      /*and (obj_text is null or obj_text = '' or :text_stop_words is null or text_tsvector @@ to_tsquery('russian', :text_stop_words))*/
    /*ORDER BY obj_id desc -- не будем делать, т.к. тормозит*/
    LIMIT :slice_for/10 OFFSET :slice_from/10
)
) as subs
ORDER BY subs.rank desc
LIMIT :slice_for OFFSET :slice_from;)search_fieldtsvector_fieldage_days_subqueryheadline_field)	tsquery
slice_from	slice_formodel_name_inr.  r/  r0  r1  
max_partno)	cmf_alertintr  r  r7  r  r  r  formattupler  )r  r  rT  Ztsquery_strr.  r6  r/  r   r0  r1  rM  rP  rN  rR  rS  rO  rU  r  Z
found_objsZall_objsr{  r{  r|  rF    sR    


/5zCmfFullSearch.search_oncec              
   C   s  dt _| }tdd|}td|}d}d}d}|D ]\}	t|	dkrLq8t|	dkr|	dkrbq8|	dkrz|d|	 7 }q8|	dkrq8|	d	kr|d
7 }q8|	dkr|d7 }q8t|	dkr|d|	 7 }q8|	d dkr|d|	dd   7 }|d|	dd   7 }q8|rq8|	dd}	| j|	|d}
|d7 }|dkr6d}|rJ|d dkrR|d7 }t|
dkrp||
d  7 }n|dd	|
 d 7 }|dkr8 qq8q8|rt|dkr|d dkr|dd  }|S |ddddddd d}|d dkr|dd  }z.t
jjj d!d"|i}t|d d }W n tjjk
r } zft
jjj  td#|  td$| d| dt j  t
jjj d%d"|i}t|d d }W 5 d }~X Y nX t|d&| |S )'Nr  u   [^-A-Za-zА-Яа-я0-9()|&! ]z(,| |&|\||\(|\))r   r   )r!  !z()&|r  )r   u   или|z |)u   иand&z &r   z& z& !rZ  r!  r*  r   F)r[  r]  z( z | z )r   )r]  r[  r$  z or z AND z & z and z!select to_tsquery('russian', :q);qu;   Ошибочный синтаксиса в запросе: u@   DEV: Ошибочный синтаксиса в запросе: z+select websearch_to_tsquery('russian', :q);z->)r  rB  lowerr   rE  r  r  replaceprepare_wordr  r  
CmfSynonymr7  r  r  r  r  r  excZProgrammingErrorZrollbackrV  print)r  r"  r(  r)  Zsearch_query_allowed_symbtokensZ
word_countZstopsr_  tZ	sug_wordsrQ  er{  r{  r|  rD  D  s    

$ "z"CmfFullSearch.prepare_search_queryc                 C   s  t  jd| d7  _|d tjkr.d}t}nd}t}g }||st  jd7  _||}g }d}|D ]}	|dkr| qt|	dkrqj|	d |d kr|	d |d krqjd	|	kr|		d	d
}	t  jd|	 d7  _|
|	 |d7 }qjt  jd|	 d7  _|
|	 |d7 }qjg }
tjjj dd|i}d}|D ]\}}|dkrR qt|dkrdq:|d |d ks|d |d kr:|	d	d
}t  jd| d7  _|

| |d7 }q:t|t|
B |hB }n|h}t }|D ]D}	t|	d d D ]*}||j t  jd|j d7  _qq||B }t }|rtjjddt||hB gddgdgddgd}|D ]\}|jrt|jjdd d D ]6}| 	d	d
}t  jd| d7  _|| qqt||B t|B }t|S )Nz|w:z: r   r   r!  zspellError, r   r   r  z<->z	addSpell r  z
            SELECT
                name, similarity(:word, name) as sim
            FROM cmf_synonym
            WHERE
                :word % name
            ORDER BY "sim" desc
            LIMIT 5;
             r  zaddSpellTrgm z
normalize r  r  r  Zorderno)r  r  r  r   ,r   zsynAdd )r  rB  stringZascii_lettersdictionary_endictionary_rucheckZsuggestr  ra  r  r  rc  r7  r  r  r  rH  morphparser8  Znormal_formr  r  r  r  rG  )r  r  r(  langZ
dictionaryZfiltered_suggestions3ZsuggestionsZfiltered_suggestionsr  r  Zfiltered_suggestions2Zsuggestions2_listZsuggrJ  Zall_suggestionsZnormalized_wordsr(  Zsynonym_wordsZsynonym_listZsynonymr  r{  r{  r|  rb    s    
 


	 
$
 zCmfFullSearch.prepare_wordc                 C   s   t | j d S r  )Zcmf_deferred_taskr   )r  r{  r{  r|  minutely_hook  s    zCmfFullSearch.minutely_hook)NNNFFNNNNNNNNNNNN)Nr   F)TF)T) __name__
__module____qualname____doc__r  r  staticmethodstrr  classmethodr  r  r  r  r  r  r  r  r  r  Z	BaseModelr  r  Z
celery_appZtaskr   r#  rK  rF  rD  rb  rq  r{  r{  r{  r|  r  >   s                     




)                            K

i-
      aUhr  )&r  r	  Zcmf.includeZcmf.data_providers.sqlalchemyr   Zcmf.fields.cmf_full_searchr  Zenchantrj  Z	pymorphy2r   r  Zbs4r   ZMorphAnalyzerrn  ZDictrk  rl  r  r  r  Zrus_keyZeng_keydictr~  r  r  r  r  r  r  Zcmf_full_searchr  ZAPPZHOOK_CRON_MINUTELYr  rq  r{  r{  r{  r|  <module>   sd   

                  L       P