B
    fk_                @   sn  d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d dl	Z
d dlZ
d dlZ
d dlmZ d dl
mZmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d dl m!Z! d dl"m#Z# ddl$m%Z%m&Z&m'Z' ddl(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 G dd de.Z7G dd dZ8dS )    N)defaultdict
namedtuple)copy)List)OrderedDict)IntegrityError)
ForeignKeyfuncinspect)json)Config)command)UNIQUE_VIOLATION)errors)close_all_sessions)
CmfRelBaseCmfType)fields)models   )CmfOrmErrorCmfOrmUniqueErrorCmfOrmIntegrityError)BaseModelMetaDEFAULT_DATASOURCE	BaseModelCmfGM2MModel   )BaseDataDriver
BaseMapperMapperinc_select_count)imutable_deep_copy)cache_obj_lock_release_allemit_delayed_eventsc                   s  e Zd ZdZi Ze Zejj	j
eedZe ZdZdd Z fddZedd	 Zed
d Zedd ZedzddZd{ddZdd Zdd Zdd Zdd Zdd Zedd Zdd Zd d! Zd"d# Zd$d% Z d&d' Z!e"d(d) Z#e"d*d+ Z$e%d,d-Z&d|d/d0Z'e"d}e(e& d2d3d4Z)d~d5d6Z*e"d7d8 Z+ej,dd1d1fd9d:Z-e"d;d< Z.e"d=d> Z/d?d@ Z0e"ddAdBZ1de&dCdDdEZ2d1dd1dFdGdHZ3dddd1dd1d.ddd1dI
dJdKZ4dddLdMdNZ5ddd1dd1d1dOdPdQZ6ddRdSZ7dTdU Z8dVdW Z9d1dXdYdZZ:d[d\ Z;dd]d^Z<d1d1d_e=d`dadbZ>ddcdddeZ? fdfdgZ@ fdhdiZAdjdk ZBdldm ZCdndo ZDdpdq ZEe fdrdsZFdtdu ZGdvdw ZHdxdy ZI  ZJS )SQLAlchemyDataDriveru  
    dataproviders['default'] = {
        'type': 'sqlalchemy',
        'sqlachemy.url': 'postgresql://user:password@host/db',
        'sqlachemy.echo': True,  # Достаточно для отладки запросов в develop mode
    }

    # Для отладки в продакшене логгеры:
    sqlalchemy.engine - controls SQL echoing. set to
        logging.INFO for SQL query output, logging.
        DEBUG for query + result set output.
        These settings are equivalent to echo=True and echo="debug" on create_engine.echo, respectively.
    sqlalchemy.pool - controls connection pool logging.
         set to logging.INFO to log connection invalidation and recycle events;
         set to logging.DEBUG to additionally log all pool checkins and checkouts.
         These settings are equivalent to pool_echo=True and pool_echo="debug" on create_engine.echo_pool, respectively.
    sqlalchemy.dialects - controls custom logging for SQL dialects, to the extend that logging
         is used within specific dialects, which is generally minimal.
    sqlalchemy.orm - controls logging of various ORM functions to the extent that logging
         is used within the ORM, which is generally minimal.
         Set to logging.INFO to log some top-level information on mapper configurations.

    Example:
        logging.basicConfig()
        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
    )Zclass_registryZmetadataNc             C   s   t jj| jdS )N)Zbind)
sqlalchemyormZsessionmakerengine)self r*   "./cmf/data_providers/sqlalchemy.py_session_factoryC   s    z%SQLAlchemyDataDriver._session_factoryc                sL   t  j|| tj| jfdd | D | _tj| 	 | _
|   d S )Nc             S   s*   i | ]"\}}| d r||d ddqS )zsqlalchemy. r   )
startswithreplace).0kvr*   r*   r+   
<dictcomp>g   s   z1SQLAlchemyDataDriver.__init__.<locals>.<dictcomp>)super__init__r&   Zengine_from_configconfigitemsr(   r'   Zscoped_sessionr,   Session	init_meta)r)   argskwargs)	__class__r*   r+   r5   b   s    zSQLAlchemyDataDriver.__init__c             C   s  g }x
|j  D ]\}}t|tr&q|js4|js4qd |g}i }|jrNd|d< |jr|jrx|jD ]}|dkr|tj	|| qb|dkr|tj	d|j
 d| d| |fd|did	| qb|tj	d|j
 d| d| |fd
|i| qbW q|tj	|| qW |S )NTuniquedefaultZgin_trgmZix__ZginZgin_trgm_ops)postgresql_usingZpostgresql_opsr@   )r   r7   
issubclassr   indexr=   Zindex_usingappendr&   ZIndex	tablename)cls	cmf_modelZindexescmf_field_name	cmf_fieldr:   r;   Zidx_typer*   r*   r+   Z__get_indexesp   s:    

z"SQLAlchemyDataDriver.__get_indexesc             C   s    | j |}|r| j | |S )N)cached_queriesgetmove_to_end)rE   keyretr*   r*   r+   	get_query   s    zSQLAlchemyDataDriver.get_queryc             C   s   || j |< | jsZtdtd d d }|d }|d }|dk rHd}|dkrTd}|| _t| j | jkr| j jdd\}}td	|  d S )
NSC_PAGE_SIZESC_PHYS_PAGESi   i         F)ZlastzDEV: set_query evicted key )rI   cached_queries_limitossysconflenpopitemprint)rE   queryrL   Zsys_memsize_mbZchunk_countZqueries_limitZevicted_keyr?   r*   r*   r+   	set_query   s    
zSQLAlchemyDataDriver.set_queryc             C   s4  |j }|jrdS t|tjr(t }nt|tjr>tj}nt|tj	r^t
|j|j}nt|tjr|t|jg}|jr|t|j tj||j|jdS t|tjrt }n\t|tjrt|j}n@t|tjrt|j}n$t|tjrdS t|tjrdS t|tjr,dS t|tjr>dS t|tjrPdS t|tj rltj!dd}nt|tj"rt# }nt|tj$rt }n~t|tj%rdS t|tj&rdS t|tj'rtj(}nDt|tj)rtj*}n.t|tj+rddl,m-} |}nt.d| tj|||j/|j|jdS )	u0   Создадим SA поле для моделиN)primary_keynullableT)Ztimezoner   )TSVECTORu"   Не найден тип поля )r>   r\   r[   )0
class_nameZvirtualrA   r   Z	CmfBigIntr&   ZBIGINTCmfIntZIntegerZ
CmfNumericZNumericZ	precisionZscaleCmfTUUIDStringZ
max_lengthZforeign_keyrC   r   ZColumnr[   r\   ZCmfTextZTEXTZCmfStrZCmfBytesZLargeBinaryCmfRelationCmfGenericRelationZCmfGenericM2MZCmfDateRangeZCmfM2MCmfDateTime	TIMESTAMPZCmfDateZDATEZCmfJson
CmfBackrefCmfGenericBackrefCmfBoolZBooleanZCmfTimeZTimeZCmfTsVectorZsqlalchemy.dialects.postgresqlr]   	Exceptionr>   )rE   Z
_cmf_modelrH   Z_db_field_namecolumn_nameZsa_typer:   r]   r*   r*   r+   _make_sa_column   sn    


z$SQLAlchemyDataDriver._make_sa_columnc             C   s   t | jj||d}|S )N)schema)r
   r(   Zget_columns)r)   
table_namerl   columnsr*   r*   r+   inspect_table_columns  s    z*SQLAlchemyDataDriver.inspect_table_columnsc             C   s   |  |}| jj| j|jS )N)dp_modelr(   Zdialect	has_table	__table__)r)   rF   sa_modelr*   r*   r+   rq   	  s    
zSQLAlchemyDataDriver.has_tablec             C   s   |  d |j}t|tjr$| d}|  }d| d| d| }|jsn|jd kr\d| | d|j d}|| |	  d S )NZTZzalter table z add column  uH   Значение по умолчанию не может быть None: z	 default z	 NOT NULL)
rk   typerA   r   rd   r8   r\   r>   executecommit)r)   rm   rj   Zcmf_field_typeZsa_column_typessqlr*   r*   r+   add_custom_column  s    



z&SQLAlchemyDataDriver.add_custom_columnc          
   C   s   dddd}t jt jdddd|d< t jt jdddd|d	< t jt jd
dd|d< t jt jddd|d< t|tf|}tj|j	j
}|  }y|| |  W n@ tk
r } z"|j  td| d|  W dd}~X Y nX dS )u5   
        Создаем М2М таблицу
        Tzcustom.r-   )abstract
__module____qualname__u   ID Объекта)captionr\   rB   left_idu   ID Элементаright_idu   Имя объекта)r~   r\   Zleft_name_cacheu   Имя элементаZright_name_cacheu-   Ошбика создания таблицы z: N)r   Fieldr`   	CmfStr256ru   r   r&   rl   CreateTablerp   rr   r8   rv   rw   ri   transactionrollbackrX   )r)   
model_namer;   rF   smtprx   er*   r*   r+   add_custom_m2m_model  s    

z)SQLAlchemyDataDriver.add_custom_m2m_modelc          	   C   s   dddd}t jt jddddddd|d	< t jt jd
dd|d< t jt jdddddd|d< t jt jdddddd|d< t jt jddddd|d< t jt jdddddd|d< t|tf|}|S )NTzcustom.r-   )r{   r|   r}   u)   Идентификатор объектаu3   Автоматически генерируетсяF)r~   commentr\   r[   readonlyvisibleidu   Имя объекта)r~   rB   nameu   Сортировкаr   )r~   rB   r   r\   r>   Zordernou   Кодu3   Код в реальном мире из жизни)r~   r=   r   r   r\   codeu:   Код родителя в каскадном выборе)r~   r   r\   rB   Zchoice_parent_idu   СкрытьZ
cmf_hidden)	r   r   r`   r   r_   ZCmfStr64rh   ru   r   )r)   r   r;   rF   r*   r*   r+   gen_custom_choice_model.  sT    z,SQLAlchemyDataDriver.gen_custom_choice_modelc          
   C   s~   |  |}tj|jj}|  }y|| |  W n@ t	k
rx } z"|j
  td| d|  W dd}~X Y nX dS )ur   
        Создаем пользовательский справочник выбора таблицу
        u-   Ошбика создания таблицы z: N)r   r&   rl   r   rp   rr   r8   rv   rw   ri   r   r   rX   )r)   r   rF   r   rx   r   r*   r*   r+   add_custom_choice_modela  s    


z,SQLAlchemyDataDriver.add_custom_choice_modelc             C   s   |j }| j|}|s|j}g }g }ddi}x<|j D ].\}}	|}
| ||	|
}|dkr\q8|| q8W || 	| || |t
|d}|dd |D  t|| jf|}|| j|< |S )u]   
        Вернём SA модель для CMF модели.
        TODO: db_name
        Zextend_existingTN)Z__tablename____table_args__c             S   s   i | ]}||j qS r*   )r   )r0   cr*   r*   r+   r3     s    z5SQLAlchemyDataDriver.dp_model_cls.<locals>.<dictcomp>)__name__models_registryrJ   rD   r   r7   rk   rC   extend"_SQLAlchemyDataDriver__get_indexestupleupdateru   sa_base_model)rE   rF   r   rs   rD   Z
sa_columnsr   Z__table_kwargs__rG   rH   Zsa_field_nameZ	sa_columnZsa_model_kwargsr*   r*   r+   dp_model_clsp  s,    
z!SQLAlchemyDataDriver.dp_model_clsc             C   s
   |  |S )N)r   )r)   rF   r*   r*   r+   rp     s    zSQLAlchemyDataDriver.dp_modelc             C   s   t t|jS )uG   
        Вернём CMF модель для SA модели.
        )getattrr   r   )r)   rs   r*   r*   r+   rF     s    zSQLAlchemyDataDriver.cmf_modelc             C   s   t |tjS )N)
isinstancer%   r   )r)   Zdp_instancer*   r*   r+   is_instance  s    z SQLAlchemyDataDriver.is_instancec             C   s4   i }x|D ]}|j j||j jj< q
W tj|d|S )N
table_type)rp   rr   fullnamer&   r'   polymorphic_union)r)   Z
cmf_modelsaliasZtablesmodelr*   r*   r+   _union_models  s    
z"SQLAlchemyDataDriver._union_modelsc	                sF   d fdd	}	i }
x|D ]}|j j|
|j jj< qW |	|
d|S )Np_unionTc       
   
      s  t j }i i x| D ]x}| | tt jjrB | |< i }x@jD ]6}rb|jkrbqN|	|j |||j< |j
|j< qNW |< qW  fddg }x|  D ]\}	|d k	r.|t jjfdd|D t jt jj|	|g gdtj djjk q|t jjfdd|D gdtj djjk qW t jj| |S )Nc                sf   y| |  S  t k
r`    r>tjtj |  | S tjtj |  | S Y nX d S )N)KeyErrorr&   ry   castZnulllabelZtype_coerce)r   table)
cast_nullscolnamemapstypesr*   r+   col  s    zPSQLAlchemyDataDriver._union_models2.<locals>.hack_polymorphic_union.<locals>.colc                s   g | ]} |qS r*   r*   )r0   r   )r   r   r*   r+   
<listcomp>  s    zWSQLAlchemyDataDriver._union_models2.<locals>.hack_polymorphic_union.<locals>.<listcomp>)Zfrom_obj_idc                s   g | ]} |qS r*   r*   )r0   r   )r   r   r*   r+   r     s    )r&   utilZ
OrderedSetr   ry   ZSelectr   r   rL   addru   r7   rC   ZselectZliteral_columnZ_quote_ddl_exprr   wherer   r   Zcorrelate_exceptZ	union_all)
Z	table_mapZtypecolnameZ	aliasnamer   ZcolnamesrL   mr   resultZtype_)left_field_name
left_tablesub_ffl)r   r   r   r   r   r+   hack_polymorphic_union  sF    


	


zCSQLAlchemyDataDriver._union_models2.<locals>.hack_polymorphic_unionr   )r   T)rp   rr   r   Zlateral)r)   r   right_tabler   r   right_modelmodels_is_unionr   r   Z	join_subqr   r*   )r   r   r   r+   _union_models2  s
    <
z#SQLAlchemyDataDriver._union_models2c             C   sX   d}| d |kr&| d | d | d fS | d |krH| d | d | d fS t d| |d S )N)><z==z>=z<==INz!=z><z<>zNOT INLIKEzNOT LIKEILIKEz	NOT ILIKEz
SIMILAR TOzNOT SIMILAR TOEXISTSz
NOT EXISTS	MEMBER_OFz@@r   r   r   zInvalid filter operation)r   )paramsZop_listr*   r*   r+   _parse_params  s    z"SQLAlchemyDataDriver._parse_paramsc             C   s  ddl m}m} t|tjr:|jdkr2||jd |j}n\t|t	rt	 }xF|D ]>}t|tjr|jdkrv||jd |
|j qP|
| qPW |}|dkrd }| dkr||kS | dkr||kS | dkr||k S | d	kr||kS | d
k r||kS | dkr||kS | dkr||S | dkr.||S | dkrB||S | dkrX|| S | dkrl||S | dkr|| S | dkr||S | dkr|| S || |S d S )Nr   )print_debugr6   .u<   collect_filter_exp: Значение не загруженоZNULLr   z>=r   z<=)z==r   )z!=z><z<>r   r   z@@zNOT INr   zNOT LIKEr   z	NOT ILIKE)cmf.includer   r6   r   r   r   _valueDEBUGvaluelistrC   in_ZlikeZilikeop)operfieldvalr   r6   new_valr2   r*   r*   r+   _expression   sZ    


















z SQLAlchemyDataDriver._expressionJoinDatazJleft_table, right_table, alias, left_field_name, right_model, right_modelsTc          
   C   s0  |r|j }|j}nT|j|}|s.| || t|tjtjfrLt|tjrft	d|j  d|j  d|j  d}|r| d| n|}|
 }	|	s|rt	d| d|j  d|j  dd S |	d	 }
t|	d
krtjj|
jj|d}n2|d k	r| j|d |||
|	d|d}n| |	|}| |||||
|	}|S )NuX   Недопустимый тип поля для вложенной фильтрации .up   . Вложенная фильтрация возможно только для CmfRelation и CmfGenericRelationZ_sub__uj   Не возможно построить запрос для вложенной фильтрации по u   , т.к. у поля u#    не указаны related_modelsr   r   )r   T)r   r   )r^   instance_classr   rJ   _raise_invalid_fieldrA   rb   rc   CmfSubclassedGenericRelationr   related_modelsrV   r&   r'   Zaliasedrp   rr   r   r   r   )r)   r   r   
field_name
from_aliasis_models_required	field_clsr   Z
alias_partr   r   r   	join_datar*   r*   r+   _calc_join_data4  s4    
z$SQLAlchemyDataDriver._calc_join_dataF)joinsc             C   sf   |r
t  ndd | j D }xB|D ]:}|j|kr4q$| |jt|jj|j	 d|jjj
k} q$W | S )uP  
        TODO cmf_deleted
        Все данные для необходимых присоединений в joins, добавим недостающие присоединения в запрос.
        froms могут быть от филдов, при этом без джойна, чтобы принудительно создать join нужен force = True.
        Или более продвинутый способ определения наличия нужного джойна.
        Сейчас при построении основного запроса, джойны принудительно добавляются.
        А при расчёте фильтров, когда нет алиасов от полей, джойн делается только если нет нужного алиаса.
        c             S   s"   h | ]}t |tjjjr|jqS r*   )r   r&   ry   
selectableAliasr   )r0   r   r*   r*   r+   	<setcomp>a  s   z3SQLAlchemyDataDriver._make_joins.<locals>.<setcomp>r   )set	statementlocate_all_fromsr   Z	outerjoinr   r   r   r   r   r   )rY   r   forceZavailable_joinsr   r*   r*   r+   _make_joinsW  s    


&z SQLAlchemyDataDriver._make_joinsc             C   sh   d }g }|d krd }|j j}n|j}x>|dD ]0}| ||||}|| |j}|j}|j}q0W |S )Nr   )	rp   rr   r   splitr   rC   r   r   r   )r)   	join_pathr   r   r   r   r   r   r*   r*   r+   _calc_joins_by_patho  s    


z(SQLAlchemyDataDriver._calc_joins_by_pathc             C   s0   t |jdd}t|  d|j d| d S )N,z,
u+    не существует у модели u   , но есть: 
)strr   r/   r   r^   )r   r   Zfields_r*   r*   r+   r     s    z)SQLAlchemyDataDriver._raise_invalid_fieldc	       0   
   C   sz	  |s|d fS t |d trhg }	x<|D ]4}
| j|
||||||d\}}|d krNq$|	| q$W |||	 fS |d dkr| j|dd  ||||||dS |d dkr| j|dd  |||tj|||dS |d dkrt|dks|d d	k rtd
|||d }|d fS | 	|\}}}t
t|tr2|j}t
t|trJ|jj}t |ttfrg }xz|D ]r}t
t|trt
t|jtr||jjj n||j n*t
t|tr||jj n
|| qdW |}|d}|j|d }d}t|dkrZt
|tjtjtjfrZ|d }d|dd  ||g}d}|d kr|jj}nrt|dkr|d\}}}| j|||d}| ||}|d }|j}|j|}|j}d}n|d kr|jj}|dkr|dkr|jd}|std| d|j  |d kr| !|| t
|tjr|dkr\t |ts\t |ts\td||||d	kr~|d k	r~td||||dkr| j"|d||d}|d krtd| d|j#r8| t$j%}|d	kr|j&'|(|j)|k|j*|j k+ }|| fS |j&'|(|j)|k|j*|j k| ,d|j-|+ }n|. }| |}|j/r`|j0}|j1}n"|j2rv|j1}|j0}ntd|||d	kr|j&'|(||k+ }|| fS |j&'|(||k| ,d||+ }||dkr|n| fS |d kr| j"|d||d}|d kr$td| d!| j3| d d d |d"}|j} |j}!|j&'|!j4j}"|j#r| t$j%}#|#j}$|"|$|#j-|!j4jk}"|"(|#j)|k|#j*|j k}"nh|j2rd#}%d$}&nd$}%d#}&|. }| |}#|#j}$t5|#|%}'t5|#|&}(|"|$|(|!j4jk}"|"(|'|k}"t6|!j4d%r(|s(d%d&dg})ng })t6|!j4d'rN|sN|)d'd&dgg})|r\|)|g})| j|)| | j|"|!||d\}"})|"(|)}"|"+ }"|7d(r|" }"||"fS td)|||n$t
|tjtjfr|dkr|7d(rd*nd}dd|g}|d krtd+|||| j"|d||d}|d kr8td| d,| j3| d d d |d"}*|*j}+|*j},|j8 d-}-t5|,j4|-}.|j&'|.}"|-d&|g})t6|,j4d%r|)d%d&dgg})|r|)|g})| j|)|+|+j|"|,||d\}"})|"(|)}"|"+ }"|7d(r|" }"||"fS t
|tj9r| d-}|d.kr| j3| d d d |d"}|j} |j}!|j&'|!j4j}"| t$j%}#|#j}$|"|$|#j-|!j4jk}"|"(t| ,d|#j)|| ,d|#j:||#j*d/k}"| j"||||d}|t| ,d||| ,d||"fS |dk	r2t$j;j<|j ||ddd0gd1}/|/	sd2g}/| j"|d||d}|| ,d||/fS | j"||||d}|d k	rdtd| d3| d4|| ,|||fS d S )5Nr   )field_tableinclude_deletedinclude_dummyZANDr   OR)cmdr   r   r   order_by   )r   z==ut   Операция order_by в фильтре должна быть в формате: ["order_by", "=", [поля..]]r   r   Fr   )r   TANYz@@r   u	   Поле u    не найдено у )r   zNOT INu   Правое значение для IN, NOT IN для m2m должно быть list, т.е. в квадратных скобках, например "field", "IN", "[obj.id]"u   Недопустимая операция над m2m полем (сравнение = только на None), допустимые IN, NOT IN, = None)r   zNOT INr   z==)
with_aliasu   В таблице u3    нет поля id для фильтрации m2mr   uD   Надо указать для поля либо left либо right)r   z
NOT EXISTSuG    нет поля id для фильтрации в подзапросе)r   r   r   cmf_deletedz==is_dummyzNOT uh   Недопустимая операция над m2m полем, допустимые IN, NOT IN, = Nonez
NOT EXISTSul   Недопустимая операция над backref полем, допустимые EXISTS, NOT EXISTSu7    нет поля id для фильтрации backrefr   r   Z
rg_membersi'  )r   r   Zsearch_queryZonly_idssliceZNOTFOUNDu    нет поля u    для фильтрации)=r   r   collect_filter_exprC   r&   or_rV   ri   order_queryr   rA   ru   r   r   r   r   r   r   r   rJ   
CmfM2MBaserf   rg   joinrp   rr   
rpartitionr   r   r   r   r   r^   r   _query_columnZnested_fieldsr   ZRelationCachesessionrY   filterZ	parent_idZparent_fieldexistsr   Zchild_idm2m_model_clsrightr   r   leftr   r   r   hasattrr.   backrefCmfRelationBaseZparent_codeZCmfFullSearchZfulltext_search)0r)   Z	in_filterr   rs   rY   r   r   r   r   Z
filter_resr2   Zexpr   r	  r  Z_rightiZ
left_partsr   r   r   r?   r   r   Zfield_column	m2m_modelZm2m_subqueryZdp_m2m_modelZdp_m2m_field_selfZdp_m2m_field_theirZself_columnZsub_dataZ	sub_modelZ	sub_tableZ	sub_queryZm2m_sa_modelZ	m2m_tablemy_id_field_namesub_id_field_namemy_sa_fieldsub_sa_fieldZsub_filter_expZbackref_dataZbackref_modelZbackref_tableZbackref_column_nameZbackref_columnZ	found_idsr*   r*   r+   r     s   







"

























z'SQLAlchemyDataDriver.collect_filter_expc             C   s
   |  dS )NZ_m2m_directr*   )r   r*   r*   r+   _m2m_depth_alias_name  s    z*SQLAlchemyDataDriver._m2m_depth_alias_namec             C   s
   |  dS )NZ_m2m_idr*   )r   r*   r*   r+   _m2m_alias_name  s    z$SQLAlchemyDataDriver._m2m_alias_namec             C   sl   | dg }t|tr t|}|r8t|d ts8|g}x.| D ]"\}}||jkrB||d|g qBW |S )u   
        Преобразуем обычные kwargs в продвинутый фильтр для однообразной логики фильтрования
        :param model:
        :param kwargs:
        :return:
        r  r   z==)	rJ   r   r   r   loadsr   r7   r   rC   )r)   r   r;   smart_filterr1   r2   r*   r*   r+   _get_filter  s    


z SQLAlchemyDataDriver._get_filterc             C   s   |r8x2|j  D ]$}t|tjjjr| j|jkr|} qW d|kr|d\}}|dkrzt	| j
|d}t	t|||}qt	| j
|d}nt	| j
|d}|dk	r|r|t|ddS |S dS )u   Для полей "основной" таблицы не нужен префикс таблицы, для этого нужно указать with_alias=Falser   )sumZavgcountminmaxNr   )r   r   r   r&   ry   r   r   r   r   r   r   r	   r   r   r/   )r   rj   rY   r   Zfrom_Z	func_nameZcolumn_name_shortcolumnr*   r*   r+   r    s    z"SQLAlchemyDataDriver._query_column)parent_joinc          	   C   s  |r
|j n|jj}x| D ]\}}| j||t|d}	|	dk	rL||	 q|r|r^|jpbg n|g}
g }xD|
D ]<}|j	|}|rnt
|tjtjfrnt
|tjsn|| qnW |sq|d }|j}|r|jnd}| j||||d|d}|sq|| | j||||j|d qW dS )u2   Соберём поля и таблицы по ffl)r   Nr   F)r   r   )r  )r   rp   rr   r7   r  boolrC   Zright_modelsr   rJ   rA   rb   rc   r   r   r   r   _select_joined_columnsr   )r)   Zfflr   rn   r   r  r   r   r   r  r   Z
fields_clsZrelated_modelZfield_cls_tmpr   Z
left_modelr   r   r*   r*   r+   r    s2    

z+SQLAlchemyDataDriver._select_joined_columns)r   group_byr   c            O   s  |  |}|  }	ddlm}
 |
jd|j |d}t| |}|rZ| }|	|	}nht

 }g }g }| |||| |	j| }| j||dd}t

 | dkr| 	d }t| || t|dkr|j}x$tt|D ]}|| ||| < qW | ||}|r0| j||||||d\}}||}|rg }x\|D ]T}||jjkrf||jj|  n,| d	|jjkr@||jj| d	  q@W |j| }|S )
Nr   )	CMF_CACHEzquery: )full_fields_loadT)r   g{Gz?)r   r   r   )rp   r8   r   r!  hashr   ru   rN   Z_cloneZwith_sessiontimer  rY   r   rZ   rV   r[   ranger  r   r  rr   r   rC   r   )r)   r   r"  r   r   r   r:   r;   rs   rx   r!  rL   rY   Zstart_calc_timer   rn   Zclone_qpkeyr  r  
filter_exprj   r*   r*   r+   _create_query  sF    




z"SQLAlchemyDataDriver._create_query)
r   r"  r   
for_updatemapperr   load_m2mr   aggregate_selectr   c   
         O   s   t |t }t|}| |}| |}| j|f||||	|
d| }|j||||
d}|rl|j| }|
r|	 }td}|j
|||d}|S |r| }|	 }|j
|||d}|r|r| j|||||d |r|  |S dS )u   Грубо селектим всю таблицу, фильтры:
          - в kwargs: поле -> значение
        )r"  r   r   r   r,  )r   r,  Zsimple)r"  r   )r   r   N)r   r   
get_mappermap_args
map_kwargsr(  with_labelsr   r   alllist_to_cmfwith_for_update	_load_m2mpost_mapping_hook)r)   r   r   r"  r   r)  r*  r   r+  r   r,  r   r:   r;   mapper_is_my_argsqZsql_resZres_objsr*   r*   r+   r   '  s2    




zSQLAlchemyDataDriver.list)r"  r   c               s  |  |}| |}| j|fd|i| }|r| |  fdd|D }|jj|jj |t	
 fj| d}	t|j|	}
g }xH|
D ]@}i }x t|D ]\}}|| ||< qW |d |d< || qW |S |jj|jj t	
 gd}	|j|	 }|S )u   Грубо селектим всю таблицу, фильтры:
          - в kwargs: поле -> значение
        r"  c                s   g | ]} j j| qS r*   )rr   r   )r0   rj   )rs   r*   r+   r   Z  s    z.SQLAlchemyDataDriver.count.<locals>.<listcomp>Nr   r  )r.  r/  r(  r0  rp   r   select_fromZfromsZwith_only_columnsr	   r  r   r   r   r  rv   	enumeraterC   Zscalar)r)   r   r"  r   r:   r;   r7  r8  rn   Zcount_qres_listZresrowZrow_resr  rj   r  r*   )rs   r+   r  I  s.    



"zSQLAlchemyDataDriver.count)r   r"  r)  r*  r   r   c            O   s   t |t }
t|}| |}| |	}	| j|f||||d|	}| }|j||ddgd}|rp| }|	 }|r|j
|||d}| j|g||||d |
r|  |S dS )u   
        Выбрать один объект по фильтру:
          - в args - первичный ключ(опционально)
          - в kwargs - другие поля: поле -> значение)r"  r   r   r   r   )r   )r"  r   )r*  r   r   N)r   r   r-  r.  r/  r(  r0  r   r3  firstobject_to_cmfr4  r5  )r)   r   r   r"  r)  r*  r   r   r:   r;   r6  r8  sa_instanceZres_objr*   r*   r+   rJ   m  s"    


zSQLAlchemyDataDriver.getc       =   
      s  |r|sdS t |t  fdddd }t|}tt}tt}	tt}
x.|D ]&}|| | |	|j | qPW x|	D ]x}tt|}xh|D ]`}|| sqt||d}|rt|t	j
rt|t	jr||  qt|t	jt	jfr|
| | qW qW x.| D ] \}}g }xB|D ]8}tt|j}|j	|}|rt|t	jr rt||}|jdk	r|jdk	r||j qd|_|jdkrd|_| }t||}|j}|dkr0td||n`t||d}|dk	r|r|| qt||d | }t||d}|dkr0td|||r|dd	 }|| | | qW x| D ]\}}tt|d}|sqft  d
dt|g}| j||| ||d}xT|D ]L}|}x<|| D ]0} rt||}||_||_nt||| qW qW qfW |r| j||| |||d qW xL|
 D ]>\}}x0|D ]&}|j}|	| }t||}t|t	jr|jrt }|j d} | dfdd|D g}!tt|| f| i i}"xz| D ]n}#|!}$d|#j	kr|s|!dddgg}$d|#j	kr|s|$dddgg}$| j|#|"|$||d}%|#|%||#j< qW |sxd	dlm }&m!}' |&|'j"d|j d|j d qLnVt#|dkrt  t$t%|& ' }(n,t  | (t)j*+dd | D d' }(tt})x&|(D ]}*t|*| }+|)|+ |* qW g },x|D ]}|)|}- rt||}|-sLg |_|jdkrg |_q|,|-|| |_|jdkrvt-|j|_|,.|j n<|-st||g  q|,|-|| }t||| |,.| qW |,rr| j|,|| |||d qLt|t	jrL|j/rd}.d}/nd}.d}/|0 }0| 1|0}1t|1|.}2t|1|/}3| (|2|32|23fd d|D }4tt}5tt}6t  xH|4D ]@\}7}8|8rx|7sqx|6|8 |7 |5|8dd	  |8 qxW xB|5 D ]4\}}9tt|}t  d
d|9g}|st4|drdddg|g}|s,t4|dr,dddg|g}| j||| ||dd!},|,rb| j|,|| |||d x|,D ]}x|6| D ]x}7xp||7 D ]d}: r|:| };|;jdkrg |;_|;j| n0t|:|d}|dkrg }t|:|| || qW qzW qhW qW xn|D ]f} rN|| }<|<jdkr,g |<_|<jdkrDt-|<j|<_|<5  nt||ddkrt||g  qW qLW q:W dS )"u   
        Прогрузка М2М и М2О полей
        :param objs: список обьектов
        :param full_fields_load: поля обьектов
        :return:
        Nc                s    r| j jS | j S )N)r   r   )Zobj_)is_heavyr*   r+   obj_id  s    z.SQLAlchemyDataDriver._load_m2m.<locals>.obj_idc              S   s   dd } t | S )Nc               S   s   t tS )N)r   r   r*   r*   r*   r+   id_to_list_factory  s    zWSQLAlchemyDataDriver._load_m2m.<locals>.model_to_id_factory.<locals>.id_to_list_factory)r   )rB  r*   r*   r+   model_to_id_factory  s    z;SQLAlchemyDataDriver._load_m2m.<locals>.model_to_id_factory.u#   Не загружено id поле:r   r   r   )r"  r*  r  )r   r   r   c                s   g | ]} |qS r*   r*   )r0   o)rA  r*   r+   r     s    z2SQLAlchemyDataDriver._load_m2m.<locals>.<listcomp>r   z==Fr   )r"  r  r   r   )r   r6   zORM: skip load field r   z die no related modelsr   c             S   s   i | ]\}}|j |qS r*   )r   )r0   r1   r   r*   r*   r+   r3   ;  s    z2SQLAlchemyDataDriver._load_m2m.<locals>.<dictcomp>r   r   r   c                s   g | ]} |qS r*   r*   )r0   rE  )rA  r*   r+   r   q  s    )r"  r  r*  r+  )6r   r    r   r   rC   r^   r   r   rA   r   r   r  r   ZCmfBackrefBaser7   rJ   r   Z_oldZid_fieldnamer   setattr	partitionr!   r4  r  dictr"   r   r(  r   rD   r   r   r6   r   rV   nextitervaluesr1  rY   r&   r'   r   r2  r   r   r	  r  rp   r  r   r
  Zapply_changes)=r)   Zobjsr"  r*  r   r   rC  Zto_load_mappingZmy_objs_by_idZmy_objs_by_model_nameZmodels_by_rel_fieldsobjr   r   r   r   Zmodel_to_idZalready_loaded_obj_listr   Zid_field_nameZid_fieldZid_field_valueZfield_valueZrel_model_nameZid_to_obj_listZ_filterZ	load_objsZload_objZ	loaded_idZmodels_listZobj_listZm2mqueryZbackref_field_nameZfilter_qZ	field_fflZ	rel_modelZmodel_query_filterr8  r   r6   r;  Zres_dictr   Zm2m_idZsub_objsZref_obj_listr  r  r  rs   r  r  Z	m2m_queryZid_by_modelsZmy_id_by_sub_idZmy_idZsub_idZsub_id_listZmy_objZmy_fieldZ	obj_fieldr*   )r@  rA  r+   r4    s   


























"

zSQLAlchemyDataDriver._load_m2mc             C   s:   g }x0|D ](}t |tjr(||j q
|| q
W |S )N)r   r   r   rC   r   )r)   r:   Znew_argsargr*   r*   r+   r.    s    
zSQLAlchemyDataDriver.map_argsc             C   s>   i }x4|  D ](\}}t|tjr.|j||< q|||< qW |S )N)r7   r   r   r   r   )r)   r;   Z
new_kwargsr1   r2   r*   r*   r+   r/    s    zSQLAlchemyDataDriver.map_kwargs)	no_reloadc         
   O   s  |  |}| |}|  }| |j}| }	t }
|
||	 y||	 |j	|	gd W n t
k
r } z4|j  t|jttrt||t||W dd}~X Y n0 tk
r } z|j  |W dd}~X Y nX d|_|rdS |
j|	|d}|
  d|_|S )um   
        no_reload - не загружать объект из базы и не возвращать
        )objectsNT)
cmf_entity)r.  r/  r8   rp   r<   r   r-  cmf_to_objectr   flushr   r   r   r   origr   lookupr   r   r   ri   is_newr>  r5  )r)   instancerN  r:   r;   r7  _kwargsrx   rs   r?  r*  r   rL  r*   r*   r+   create  s2    





zSQLAlchemyDataDriver.createc          
      s   |  |}| |}|  }|  j}t j}|  t fdd|D }	||	|	}
y|
|
 W n0 tk
r } z|j  |W dd}~X Y nX  S )u*   TODO логическое удалениеc             3   s   | ]}t  |V  qd S )N)r   )r0   f)rV  r*   r+   	<genexpr>  s    z.SQLAlchemyDataDriver.delete.<locals>.<genexpr>N)r.  r/  r8   rp   r<   ru   r[   r   rY   rJ   deleteri   r   r   )r)   rV  r:   r;   r7  rW  rx   rs   r&  get_argsr?  r   r*   )rV  r+   r[    s    



zSQLAlchemyDataDriver.deletec             K   s`   |  |}|  }||}| ||}|rP| j||||||d\}}	||	}|jdd}
|
S )N)r   r   Zfetch)Zsynchronize_session)rp   r8   rY   r  r   r  r[  )r)   r   r   r   r;   rs   rx   rY   r  r'  Z
delete_cntr*   r*   r+   bulk_delete  s    


z SQLAlchemyDataDriver.bulk_delete)r   r   )rK  c            O   s   ddl m} | |}|  }	|	|}
| ||}|r\| j||||
||d\}
}|
|}
|
j}|
j	}||
||j}x|D ]}||}qW ||}|	|}dd |D S )Nr   )r   )r   r   c             S   s   g | ]}|d  qS )r   r*   )r0   r<  r*   r*   r+   r     s    z4SQLAlchemyDataDriver.bulk_update.<locals>.<listcomp>)r&   r   rp   r8   rY   r  r   r  Z
_criterionZ_join_entitiesrK  Z	returningr   r9  r   rv   )r)   r   rK  r   r   r:   r;   r   rs   rx   rY   r  r'  Zfilter_criteriaZjoin_clausesZstmtZjoin_clauser   r*   r*   r+   bulk_update  s"    





z SQLAlchemyDataDriver.bulk_update)r"  c         
      s.  |  |}| |}|  }| t }t j}|  t fdd|D }|||}	t	
 }
|
 |	 y|j|	gd W n~ tk
r } z0|j  t|jttrt |t |W d d }~X Y n2 tk
r } z|j  |W d d }~X Y nX  j}|
j|	 |d}|
  ||_|S )Nc             3   s   | ]}t  |V  qd S )N)r   )r0   rY  )rV  r*   r+   rZ    s    z.SQLAlchemyDataDriver.update.<locals>.<genexpr>)rO  )rP  r"  )r.  r/  r8   rp   ru   r[   r   rY   rJ   r   r-  rQ  rR  r   r   r   r   rS  r   rT  r   r   r   ri   rU  r>  r5  )r)   rV  r"  r:   r;   rx   rs   r&  r\  r?  r*  r   rU  rL  r*   )rV  r+   r     s0    




zSQLAlchemyDataDriver.updatec                s   t    d S )N)r4   before_request)r)   )r<   r*   r+   r_  :  s    z#SQLAlchemyDataDriver.before_requestc                s   t    d S )N)r4   before_first_request)r)   )r<   r*   r+   r`  =  s    z)SQLAlchemyDataDriver.before_first_requestc             C   sB   ddl m} |  |  |    | j  t  t  d S )Nr   )r!  )	r   r!  cache_unlockZcache_transaction_startr8   rw   remover#   r$   )r)   r!  r*   r*   r+   rw   @  s    
zSQLAlchemyDataDriver.commitc             C   s:   ddl m} t  |  |    | j  t  d S )Nr   )r!  )r   r!  r#   ra  r8   r   rb  r$   )r)   r!  r*   r*   r+   r   K  s    
zSQLAlchemyDataDriver.rollbackc             O   s   |   j||S )N)r8   rY   )r)   r:   r;   r*   r*   r+   rY   S  s    zSQLAlchemyDataDriver.queryc             O   s   |   j||S )N)r8   rY   )r)   r:   r;   r*   r*   r+   query_deprecatedV  s    z%SQLAlchemyDataDriver.query_deprecatedc                s   t  | d S )N)r4   make_models)rE   r   )r<   r*   r+   rd  Y  s    z SQLAlchemyDataDriver.make_modelsc             C   sb   x\t tD ]P}tt|}t|tr
|jr*q
|jr>| j|jkr>q
|jsP| jtkrPq
| 	| q
W dS )u   Создадим схемуN)
dirr   r   r   r   r{   Zdata_sourcesr   r   rp   )r)   r   rF   r*   r*   r+   r9   ^  s    
zSQLAlchemyDataDriver.init_metac             C   sd   |   }|d |  | j| j tjt	 d}t
|}|d| jd  t|d d S )Nz'CREATE EXTENSION IF NOT EXISTS pg_trgm;zalembic.inizsqlalchemy.urlhead)r8   rv   rw   db_metaZ
create_allr(   rT   pathr  getcwdr   Zset_main_optionr6   r   Zstamp)r)   rx   Zalembic_config_fileZalembic_configr*   r*   r+   init_dbp  s    
zSQLAlchemyDataDriver.init_dbc             C   s   t   | j  d S )N)r   r(   Zdispose)r)   r*   r*   r+   r   |  s    z'SQLAlchemyDataDriver.close_all_sessions)N)N)NTNN)F)N)NT)N)FF)FF)Kr   r|   r}   __doc__r   r&   ZMetaDatarg  ZextZdeclarativeZdeclarative_baser   r   rI   rS   r,   r5   classmethodr   rN   rZ   rk   ro   rq   rz   r   r   r   r   rp   rF   r   r   r   staticmethodr   r   r   r   r   r   r   r   r   and_r   r  r  r  r  r  r(  r   r  rJ   r4  r.  r/  rX  r[  r]  rH  r^  r   r_  r`  rw   r   rY   rc  rd  r9   rj  r   __classcell__r*   r*   )r<   r+   r%      s   
*	K
3'C2

#
  !3
 $
  $		$
%r%   c               @   s   e Zd Zdd ZdS )SAModelAccessorc             C   s   |dkrd S t tt|S )N)Z_pytestfixturefunctionZ__test__)r%   r   r   r   )r)   itemr*   r*   r+   __getattr__  s    zSAModelAccessor.__getattr__N)r   r|   r}   rr  r*   r*   r*   r+   rp    s   rp  )9rT   r$  collectionsr   r   r   typingr   r   Zsqlalchemy.ext.declarativer&   Zsqlalchemy.ormZsqlalchemy.sql.selectableZsqlalchemy.excr   r   r	   r
   Zflaskr   Zalembic.configr   Zalembicr   Zpsycopg2.errorcodesr   Zpsycopg2r   r   Zcmf.fields.base_fieldsr   r   Zcmfr   r   r   Z
base_errorr   r   r   Zmodels.base_modelr   r   r   r   baser   r   r    r!   Zutil.immutablesr"   Zutil.cmfutilr#   r$   r%   rp  r*   r*   r*   r+   <module>   sL               o