
    E&jb~                         d dl  d dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d d	lmZ  G d
 d      Z G d d      Z G d d      Zy)    )*)timeN)timezone)OrderedDict)ImmutableDict)CmfCacheClusterVersionStorage)CmfCacheClusterCalcCmfCacheClusterOutdated)CmfCacheVerLogc                       e Zd ZdZddZd ZddZddZd Zd Z	d	 Z
edd
       Zd Zd ZddZed        ZddZddZddZy)	CmfCache2u?  
    Кеш 2.0
    https://bcrm.carbonsoft.ru/project/Document/DOC-016279#spec-cach-ev2-0
    Один на APP

    # TODO1: авто-поля
        # Авто-загружаемые поля
        for field_name, field in model.fields.items():
            if field.load_mode == 'always' and field_name not in obj_fields:
                obj_fields.append(field_name)
    
    # TODO0: sget/slist

    # TODO0: order_by in filter
    # TODO0: filter in kwargs

    # TODO1: хранить data отдельно, чтобы не грузить ее из редиса, пока не проверили версии

    # Формат данных в кеше:
    # key = {
    #   'model': <имя модели>
    #   'data_clusters': [[<кластера данных>, <версия>], ]
    #   'data': <данные>

    # Хранение статистики
    # key_wo_versions = {
    #   'query_count': <количество запросов>
    #   'miss_query_versions_count': <сколько раз не смогли найти по key (смена версий в query)>
    #   'miss_data_versions_count': <сколько раз нашли по key, но отбросили из-за data_clusters>
    #   'miss_bucket_collision': <сколько раз нашли не тот > # ??? Мы вообще можем это понять
    # ??? cluster = <количество миссов из-за того что версия обновилась (можем считать только для данных, но мб сделать спец.дебаг режим)>
    c                     || _         || _        t        d| j                   | j                        | _        t	        | j                        | _        || _        || _        d | _        d | _	        t               | _        y )Nobj_storage)use_bucketsbucket_size)r   r   r   obj_versionsr   verlogDEBUGDEBUG_CLUSTER_VERSIONS_MISSredis_storageredis_instanceInmemoryStorageinmemory_storage)selfr   r   r   r   s        ./cmf/util/cmf_cache_2.py__init__zCmfCache2.__init__3   sz    &&
 :-UYUeUesw  tD  tD  E$T%6%67
+F(!" / 1    c                     t               }| j                  j                  |       t        d|      | _        t               |z
  dkD  rt        dt               |z
          y y )Nstorageg?z PROF CmfCache2 init_cache() got )r   r   	init_storRedisStorager   print)r   r   prof_sts      r   
init_cachezCmfCache2.init_cacheC   sX    &##N3))^D6Gc!4TVg5E4FGH "r   Nc	           
      v   	 |}	| j                  |||||||| j                        }
d}d}	|r8| j                  j                  |d|
       }d}	|du r| j	                  d       	 |du r | j
                  j                  d|
       }d}	|s2| j	                          |	r| j                  j                  |d|
        y|j                  |d   k7  r4| j	                  |	       |	r| j                  j                  |d|
        yd}|d   D ]  \  }}|t        j                  v rt        |      | j                  j                  |      }||k7  sD| j                  s5| j	                  |	       |	r| j                  j                  |d|
         yd} | j                  rQd	|v rM|d	   D ]E  \  }}|t        j                  v rt        |      | j                  j                  |      }||k7  sDd}G |r?t        d
       | j	                  |	       |	r| j                  j                  |d|
        yt        j                  r]t               }|d   D ]  \  }}|t        j                   |<    t               |z
  dkD  r#t        j#                  dt               |z
          |r"|	s | j                  j%                  |d|
 |       | j'                  |	       |d   S # t        $ rV}t        d|j(                   d       | j	                  	       |	r| j                  j                  |d
        Y d }~yd }~ww xY w)Nmodeloperate
obj_fields
obj_filter	obj_sliceobj_order_byobj_group_bykey_wo_versions.Fobj:T)cache_inmemoryr'   data_clustersquery_clustersver_missgQ?z+PROF cache2_collect_touched_cache_clusters datau9   Пропускаем запрос из кеша. Cluster 8    обновлен в текущей транзакции)_calc_query_hashr   r   get_itemprofiler_redis_data_missr   del_item
class_namegdeferred_cache2_inc_versionsr
   r   get_versionr"   %cache2_collect_touched_cache_clustersr   cache2_touched_cache_clustersdebugadd_itemprofiler_redis_data_hitcluster_name)r   r'   r(   r)   r*   r+   r,   r-   r0   from_inmemorykeyr4   r3   rC   cluster_versioncur_verr#   es                     r   	cache_getzCmfCache2.cache_getK   s   U	*M''eWQ[hrBKZf  vBHLHhHh ( jC D!M,,55etC5\J $3;111Fs{))22T#<@ %--/ ))225D,G 4=0--]-K ))225D,GH15o1F $-o1#A#AA1,??++77E"g-  ;;55]5S( 11::5D,O"#H$ //4D4L59:J5K (1L/#q'E'EE5lCC"//;;LIG&'1 $(( j!--]-K ))225D,G 66&59/5J T1L/DSA33LAT6G#d*GGI$&SZJZI[\] m%%..uSElDI(((F<& 	MannM]  ^V  W  X)))G%%..uSElC	sC   B:K =AK A
K A K AK ,AK 1B'K 	L8"AL33L8c	           
      D   	 | j                  |||||||| j                        }	d|j                  i}
t        j                  | j
                  | j                  |||||      }g }t        |      D ]T  }| j                  j                  |      }|j                  ||g       t        j                  sB|t        j                  |<   V ||
d<   | j                  rvt        j                  | j
                  | j                  ||||||      }g }t        |      D ]0  }| j                  j                  |      }|j                  ||g       2 ||
d<   ||
d<   | j                  j!                  d|	 |
       y	# t"        $ r#}t%        d
|j&                   d       Y d }~yd }~ww xY w)Nr&   r'   )r(   
query_datar)   r*   r1   r(   r)   r*   r,   r-   r2   r4   r/   TzSkip add to cache. Cluster r5   F)r6   r   r:   r	   calc_query_clusters_with_datar   r   sortedr   r=   appendr;   r>   r?   calc_query_clustersr   set_itemr
   r"   rC   )r   r'   r(   rK   r)   r*   r+   r,   r-   rE   	cache_objdata_cluster_namesr1   rC   verquery_cluster_namesr2   rH   s                     r   	cache_addzCmfCache2.cache_add   s   "	''eWQ[hrBKZf  vBHLHhHh ( jC ))I "5!R!RSWScSceieueuw|  GN  [e  r|;E"GM &'9 : H''33LA$$lC%89::DGA33LAH *7Io&//&9&M&MdN^N^`d`p`prw  BI  V`;ET`o{'}#!#$*+>$? ?L++77EC"))<*=>? /=	*+ *If''$sei@& 	//??wxy	s   B/E3 2C E3 3	F<FFc                 8    | j                   j                          y N)r   drop_allr   s    r   
drop_cachezCmfCache2.drop_cache   s    ##%r   c                 B   | j                   r| j                  j                  |      }t        j                  j                  |       |D ]T  }|t        j                  vrt        |g      t        j                  |<   3t        j                  |   j                  |       V y rX   )r   r   _bucket_hashr;   r<   add'deferred_cache2_inc_versions_by_subjectset)r   rE   verlog_subject_ids
subject_ids       r   !_add_deferred_cache2_inc_versionsz+CmfCache2._add_deferred_cache2_inc_versions   s    
 ##005C	&&**3/, 	OJ!J!JJHKSE
99*E99*EII#N		Or   c           	      J   t               |dk7  rj                  g d       fd}fd} |d        |d        |d        |d        |d	        |d
        |d       	 j                  d       |dv r_|dk(  r| j                  dj                          | j                  d      D ]$  }| j                  | dj                   d       & y |dv r| j                  dj                          | j                  d      D ]@  }j                  d      D ])  \  }}| j                  | dj                   d|        + B | j                  d      D ]$  }| j                  | dj                   d       & y |dv ry t        d|      )Ndelete)z
project.idzresponsible.idzcmf_author.idzcmf_owner.idzwaiting_for.idzexecutors.idzspectators.idc                    t        |       sy t        |       }|r$j                  t        |j                               |j
                  r<|j                  r/j                  t        |j                  j                               y y y rX   hasattrgetattrr^   strid
is_changedold)
field_name
field_dataobjra   s     r   _add_fkz-CmfCache2.cache_apply_update.<locals>._add_fk   sj    3
+ j1J"&&s:=='9:$$"&&s:>>+<+<'=> *8$r   c                 0   t        |       sy t        |       }|r+|D ]&  }j                  t        |j                               ( |j
                  rC|j                  r6|j                  D ]&  }j                  t        |j                               ( y y y rX   rg   )rn   ro   srp   ra   s      r   _add_m2mz.CmfCache2.cache_apply_update.<locals>._add_m2m   s    3
+ j1J# 6A&**3qtt956$$# 6A&**3qtt956 *8$r   project
cmf_author	cmf_ownerresponsiblewaiting_for	executors
spectatorsTGLOBAL)createre   zID.ALL)type.z.insdel)updateCUR)rl   CHANGED)rO   removezTODO operate=)	r`   load_fieldsr^   rc   rk   _get_obj_cache_clustersr:   items	Exception)	r   r(   rp   rq   rt   clusterrn   _ra   s	     `     @r   cache_apply_updatezCmfCache2.cache_apply_update   s    !Uh OO O P	?		6 		 ""8, **("66SVVH~GYZ77%7H q
 66'!CNNCSSZ7[]opq 
" 22S>CUV77%7H {%(YY$Y%? {MJ::gYaGWWXYcXd;egyz{{  77)7L q66'!CNNCSSZ7[]opq,,nG:.//r   c                 *   |dv sJ dd l }|dv rdg}ng }dg|j                  D ]  }t        ||      s|dk(  rt        ||      }|j                  st
        j                  d| d       Ht        ||j                  j                        r|dv r2|r0|j                  s$|j                  t        |j                               |dv s|j                  s|j                  r.|j                  t        |j                  j                               |j                  s|j                  t        |j                  j                               't        ||j                  j                         r|sK|D ]  }|dv rB|j                  r|j#                  |d	      s$|j                  t        |j                               |dv sN|j                  s[|j$                  D ]&  }|j                  t        |j                               ( |j&                  D ]&  }|j                  t        |j                               (  t        ||j                  j(                        r|dv r(|r&|j                  s|j                  t        |             |dv sh|j                  sv|j                  r$|j                  t        |j                               |j                  s|j                  t        |j                               t+        d
 ||              |S )N)r   r   r~   r   )r   r~   r~   parentuT   CMF_CACHE Error: не можем посчитать кластер, т.к. поле u    не загружено!)r   r~   rO   zTODO )
cmf.fieldscache_cluster_fieldsrh   ri   
is_definedr;   r@   
isinstancefieldsCmfRelationBaserl   rO   rj   rk   rm   new
CmfM2MBase
in_changeschanges_appendedchanges_removedCmfTUUIDr   )clsrp   r   cmfrescache_cluster_fieldro   r   s           r   r   z!CmfCache2._get_obj_cache_clusters!  s    0000 	>!'C C$,#Hs/G/G#H /	<3 349LPX9X &9:J((n  pC  oD  D]  ^  _*cjj&@&@A>)!**?*?

3z}}#56--!,,%>>JJs:>>+<+<'=>%>>JJs:>>+<+<'=>J

(=(=>#- 	@>1#-#8#8
@U@UV]_g@h #

3wzz? ;#55)44/9/J/J !@G$'JJs7::$?!@/9/I/I !@G$'JJs7::$?!@	@ J

(;(;<>)!**?*?

3z?3--!,,%>>JJs:>>':;%>>JJs:>>':;%Z(8'9 :;;_/	<` 
r   c                 L   t         j                  r| j                  j                  t         j                         | j                  j                  dt        t         j                  j                        t         j                         t         j                  j                         D ]`  \  }}|j                  d      rd}n*|j                  d      rd}n|dk(  rd}nt        d|      | j                  j                  |||       b i t         _	        t               t         _        y )Npersonz
CmfPerson:zCmfProject:ru   r|   globalu/   Нет очереди VerLog для subject_id=)r;   r<   r   inc_version_listr   update_clustersrj   current_userrk   r_   r   
startswithr   r`   )r   rb   keyscl_types       r   cache_commit_hookzCmfCache2.cache_commit_hook`  s     ))..q/M/MN KK''#ann6G6G2H!JhJhi$%$M$M$S$S$U 
G 
D((6&G**=9'G8+&G#&V:-$XYY++GZF
G 571),&r   c                 @    i t         _        t               t         _        y rX   )r;   r_   r`   r<   rZ   s    r   cache_rollback_hookzCmfCache2.cache_rollback_hook  s    461),&r   c	                 `   | j                  |j                   d| |||||      }	| j                  ||||||      }
|r| j                  rcd|	dd z   dz   t	        j
                  |	j                               j                         z   }|j                  dd	      }|j                  d
d      }|S |j                   d	t	        j
                  |	j                               j                         z   }|S | j                  rld|	dd z   |
dd z   dz   t	        j
                  |	|
z   j                               j                         z   }|j                  dd	      }|j                  d
d      }|S |j                   d	t	        j
                  |	|
z   j                               j                         z   }|S )Nr   )r)   r*   r+   r,   r-   rL   zHASH:r   i  z|md5:
: r   iT     )	_calc_query_hash_rawr:   _calc_query_hash_versionsr   hashlibmd5encode	hexdigestreplace)r   r'   r(   r)   r*   r.   r+   r,   r-   query_key_rawversions_key_rawrE   s               r   r6   zCmfCache2._calc_query_hash  s   11U5E5E4Fay2Q^hu?HWcr~ 2 @99%]gt~BN]i : k zz")M!C,@"@7"JW[[YfYmYmYoMpMzMzM|"|"1"9"9$"D"1"9"9#s"C #" &+%5%5$6a"87;;}G[G[G];^;h;h;j"j""::M!C003CAc3JJWTkk=3C#C"K"K"MNXXZ[C++dC(C++c3'C 
 %%&a(7;;HX8X7`7`7b+c+m+m+ooC
r   c                    	 dd l 	|}|r|ddj                  |      z   z  }|dt        |      z   z  }|r|ddj                  |      z   z  }|r|ddj                  |      z   z  }	fd|dz    |      z   }|S )	Nr   zfields=,zslice=zorder=zgroup=c                 `   t        |       }d}|t        t        t        fv r~|t        k(  rh| dd dk(  r`	 t        j
                  j                  |       }|j                  t        j                        }|j                  dd      }|} t        d|        t        |       }|S |t        k(  r| D ]  }| |      z   dz   } |S |t        k(  r t        |       D ]  }| |      z   dz   } |S |t         k(  s	|t"        k(  r>t        | j%                               D ]   \  }}||z   }|r| d |      z   }|dz   }" |S | d	}|S |t&        u rt        |       }|S |t(        j(                  k(  r | j                  dd      } t        |       }|S |t(        j*                  k(  rt        |       }|S t-        |j.                  j0                        r$| j2                  r | j4                        }|S d
}|S t7        | d      rt        | j8                        }|S t        d      # t        $ r}Y d }~d }~ww xY w)N r      20)secondmicrosecondz!!!!! Date:r   r   Nonez...rk   u   TODO0: разобраться)#r   rj   intfloatdateutilparserparse
astimezoner   utcr   r"   r   listr`   rN   dictr   r   booldatetimedate
issubclassr   CmfTyper   valuerh   rk   __name__r   r   pickledumpsr   )	valtr   val1rH   vk_all2strr   s	          r   r   z0CmfCache2._calc_query_hash_raw.<locals>._all2str  sa   S	ACS#u%%8AaD 0&oo33C8"oohll;"ll!lC!}c2 #hR JQ d 2A+c1C2N JK c 2A+c1C2H JE da=0"399;/ (DAq'C!$Qi(1+5!Ci(B J7 4 J3 d#h0 J/ h'''kkk:#h( J' hmm##h$ J# Aszz112 >>"399-C J  C J 3%cff+C J $$CDDS & s   AH 	H-(H-|)r   joinrj   )
r   prefixr)   r*   r+   r,   r-   rs   r   r   s
           @@r   r   zCmfCache2._calc_query_hash_raw  s    SXXj111A	XI&&CHH\222ACHH\222A:	x Ghz**r   c           
          t        j                  | j                  | j                  ||||||      }d}t	        |      D ]L  }	| j
                  j                  |	      }
|d|	 d|
 z  }t        j                  s:|
t        j                  |	<   N |S )NrL   r   r   r   )
r	   rP   r   r   rN   r   r=   r;   r>   r?   )r   r'   r(   r)   r*   r,   r-   clustersr   r   rT   s              r   r   z#CmfCache2._calc_query_hash_versions  s    &::4;K;KTM]M]_dnu  CM7AP\kwy h' 	?G##//8CQwiq&&C 66;>//8	? 
r   c                 `   	 dt         vrt               t         _        d}|rd}| dt         j                  vrdt         j                  | d<   nt         j                  | dxx   dz  cc<   | dt         j                  vrdt         j                  | d<   y y # t        $ r
}Y d }~y d }~ww xY w)Nprofiler_dataredis_cacheinmemory_cache_hit   _missr   r;   r   r   r   r   r0   rH   rE   s       r   rB   z!CmfCache2.profiler_redis_data_hit  s    	a'"&& "CU$<q.,-AOOse4L)OOse4L)Q.)U%=/-.AOOse5M* 0  		   B 	B-(B-c                 `   	 dt         vrt               t         _        d}|rd}| dt         j                  vrdt         j                  | d<   nt         j                  | dxx   dz  cc<   | dt         j                  vrdt         j                  | d<   y y # t        $ r
}Y d }~y d }~ww xY w)Nr   r   r   r   r   r   r   r   r   s       r   r8   z"CmfCache2.profiler_redis_data_miss  s    	a'"&& "CU%=/-.AOOse5M*OOse5M*a/*U$<q.,-AOOse4L) /  		r   )Ti  FF)NNNF)NNN)r   )FNNN)NN)F)r   
__module____qualname____doc__r   r$   rI   rV   r[   rc   r   classmethodr   r   r   r6   r   r   rB   r8    r   r   r   r      s}    @2 IVr#J&OE0N ; ;|!/F/4 J JX /".r   r   c                   $    e Zd Zd Zd Zd Zd Zy)r!   c                 &    d| | _         || _        y )NRedisStorage_)storage_namer   )r   namer   s      r   r   zRedisStorage.__init__7  s    +D62,r   c                     | j                   j                  | j                  |      }|s|S t        j                  |      S rX   )r   hgetr   r   loads)r   rE   r   s      r   r7   zRedisStorage.get_item;  s7    !!&&t'8'8#>J||C  r   c                 v    | j                   j                  | j                  |t        j                  |            S rX   )r   hsetr   r   r   )r   rE   r4   s      r   rQ   zRedisStorage.set_itemA  s,    ""''(9(93T@RSSr   c                 L    | j                   j                  | j                        S rX   )r   re   r   rZ   s    r   rY   zRedisStorage.drop_allD  s    ""))$*;*;<<r   N)r   r   r   r   r7   rQ   rY   r   r   r   r!   r!   !  s    ,-!T=r   r!   c                   .    e Zd ZdZd Zd Zd Zd Zd Zy)r   u   
    g.in_memory_cache и g.in_memory_cache_skip_models - больше не нужен, т.к. используем кластера
    TODO: memory limits
    TODO: metrics
    TODO: readlime-метрика размера кеша
    c                     i | _         y rX   )cacherZ   s    r   r   zInmemoryStorage.__init__O  s	    
r   c                     |j                   | j                  vr!t               | j                  |j                   <   | j                  |j                      S rX   )r:   r   r   )r   r'   s     r   _model_cachezInmemoryStorage._model_cacheR  s?    4::-+6=DJJu''(zz%**++r   c                     |j                   st        dd       y | j                  |      }|j                  |d      }|dur|j	                  |       |S )NuX   DEV: inmemory get_item вызван для модели без флага cache_inmemoryTabort.)r0   	cmf_alertr   getmove_to_end)r   r'   rE   model_cacher   s        r   r7   zInmemoryStorage.get_itemW  sT    ##py}~''.ooc3'c>##C(
r   c                     |j                   st        dd       y | j                  |      }|||<   |j                   rd}t        |      kD  r|j	                  d      \  }}y y )NuX   DEV: inmemory add_item вызван для модели без флага cache_inmemoryTr   i  F)last)r0   r  r   lenpopitem)r   r'   rE   r4   r  	LRU_LIMITevicted_keyr   s           r   rA   zInmemoryStorage.add_itema  sm    ##py}~''.C I{i'(00e0<NK (r   c                 ~    |j                   st        dd       y| j                  |      }|j                  |d       y)u   Вызывается, когда ловим, что данные устарели. TODO1: по таймеру перепроверять данные в кешеuX   DEV: inmemory del_item вызван для модели без флага cache_inmemoryTr   N)r0   r  r   pop)r   r'   rE   r  s       r   r9   zInmemoryStorage.del_itemp  s8    ##py}~''.T"r   N)	r   r   r   r   r   r   r7   rA   r9   r   r   r   r   r   H  s     ,
=#r   r   )cmf.includer   r   r   randomdateutil.parserr   r   r   collectionsr   werkzeug.datastructuresr   &cmf.util.cache_cluster_version_storager   cmf.util.cache_cluster_calcr	   r
   cmf.util.cache_verlogr   r   r!   r   r   r   r   <module>r     sN            # 1 P T 0L. L.^$= $=N.# .#r   