U
    EhU                    @   s  d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlZd dlZd dlZd dlmZ d dlmZmZmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z: G dd de2Z;G dd dZ<dS )    N)defaultdict
namedtuple)copy)List)OrderedDict)	timedelta)IntegrityError)
ForeignKeyfuncinspect)jsong)Config)command)UNIQUE_VIOLATION)errors)close_all_sessions)
CmfRelBaseCmfType)fields)models   )CmfOrmErrorCmfOrmUniqueErrorCmfOrmIntegrityError)BaseModelMetaDEFAULT_DATASOURCE	BaseModelCmfGM2MModel   )BaseDataDriver
BaseMapperMapperinc_select_count)imutable_deep_copy)cache_obj_lock_release_allemit_delayed_eventsc                       s  e Zd ZdZi Ze Zejj	j
eedZe ZdZdd Z fddZedd	 Zed
d Zedd Zed|ddZd}ddZdd Zdd Zdd Zdd Zdd Zedd Zdd Zd d! Zd"d# Zd$d% Z d&d' Z!e"d(d) Z#e"d*d+ Z$e%d,d-Z&d~d/d0Z'e"de(e& d2d3d4Z)dd5d6Z*e"d7d8 Z+ej,dd1d1fd9d:Z-e"d;d< Z.e"d=d> Z/d?d@ Z0e"ddAdBZ1de&dCdDdEZ2d1dd1dFdGdHZ3dddd1dd1d.ddd1dI
dJdKZ4dddLdMdNZ5ddd1dd1d1dOdPdQZ6ddRdSZ7dTdU Z8dVdW Z9d1dXdYdZZ:d[d\ Z;dd]d^Z<d1d1d_e=d`dadbZ>ddcdddeZ?ddcdfdgZ@ fdhdiZAdjdk ZBdldm ZCdndo ZDdpdq ZEddrdsZFe fdtduZGdvdw ZHdxdy ZIdzd{ ZJ  ZKS )SQLAlchemyDataDriveru  
    dataproviders['default'] = {
        'type': 'sqlalchemy',
        'sqlachemy.url': 'postgresql://user:password@host/db',
        'sqlachemy.echo': True,  # Достаточно для отладки запросов в develop mode
    }

    # Для отладки в продакшене логгеры:
    sqlalchemy.engine - controls SQL echoing. set to
        logging.INFO for SQL query output, logging.
        DEBUG for query + result set output.
        These settings are equivalent to echo=True and echo="debug" on create_engine.echo, respectively.
    sqlalchemy.pool - controls connection pool logging.
         set to logging.INFO to log connection invalidation and recycle events;
         set to logging.DEBUG to additionally log all pool checkins and checkouts.
         These settings are equivalent to pool_echo=True and pool_echo="debug" on create_engine.echo_pool, respectively.
    sqlalchemy.dialects - controls custom logging for SQL dialects, to the extend that logging
         is used within specific dialects, which is generally minimal.
    sqlalchemy.orm - controls logging of various ORM functions to the extent that logging
         is used within the ORM, which is generally minimal.
         Set to logging.INFO to log some top-level information on mapper configurations.

    Example:
        logging.basicConfig()
        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
    )Zclass_registryZmetadataNc                 C   s   t jj| jdS )N)Zbind)
sqlalchemyormZsessionmakerengineself r-   "./cmf/data_providers/sqlalchemy.py_session_factoryE   s    z%SQLAlchemyDataDriver._session_factoryc                    sL   t  j|| tj| jfdd | D | _tj| 	 | _
|   d S )Nc                 S   s*   i | ]"\}}| d r|d dd|qS )zsqlalchemy. r   )
startswithreplace).0kvr-   r-   r.   
<dictcomp>i   s   
 z1SQLAlchemyDataDriver.__init__.<locals>.<dictcomp>)super__init__r(   Zengine_from_configconfigitemsr*   r)   Zscoped_sessionr/   Session	init_metar,   argskwargs	__class__r-   r.   r8   d   s    zSQLAlchemyDataDriver.__init__c                 C   st  g }|j  D ] \}}t|tr$q|js2|js2qd |g}g }|jrNt|j}|jr|r|jrd|krtd| d|j	 d|D ]}|dkr|
tj|d|ji q|dkr|
tjd|j d| d| |d	|d
id q|
tjd|j d| d| |||jd qq|
tj|d|ji qt|drp|jrp|
tjd|j df|jdddi |S )Ndefaultu   В поле .uU    указана уникальность, но не указан индекс defaultuniqueZgin_trgmZix__ZginZgin_trgm_ops)postgresql_usingZpostgresql_ops)rF   rD   orderno_partition_byZ_orderno_partition_by_clusterordernoF)rH   )r   r:   
issubclassr   indexrD   index_usinglistr   
class_nameappendr(   ZIndex	tablenamehasattrrG   )cls	cmf_modelZindexescmf_field_name	cmf_fieldr>   rK   Zidx_typer-   r-   r.   Z__get_indexesr   sV    

 

 z"SQLAlchemyDataDriver.__get_indexesc                 C   s    | j |}|r| j | |S N)cached_queriesgetmove_to_end)rQ   keyretr-   r-   r.   	get_query   s    zSQLAlchemyDataDriver.get_queryc                 C   s   || j |< | jsZtdtd d d }|d }|d }|dk rHd}|dkrTd}|| _t| j | jkr| j jdd\}}td	|  d S )
NSC_PAGE_SIZESC_PHYS_PAGESi   i         F)lastzDEV: set_query evicted key )rV   cached_queries_limitossysconflenpopitemprint)rQ   queryrY   Zsys_memsize_mbZchunk_countZqueries_limitZevicted_keyrE   r-   r-   r.   	set_query   s    
zSQLAlchemyDataDriver.set_queryc                 C   s4  |j }|jrdS t|tjr(t }nt|tjr>tj}nt|tj	r^t
|j|j}nt|tjr|t|jg}|jr|t|j tj||j|jdS t|tjrt }n\t|tjrt|j}n@t|tjrt|j}n$t|tjrdS t|tjrdS t|tjr,dS t|tjr>dS t|tjrPdS t|tj rltj!dd}nt|tj"rt# }nt|tj$rt }n~t|tj%rdS t|tj&rdS t|tj'rtj(}nDt|tj)rtj*}n.t|tj+rddl,m-} |}nt.d| tj|||j/|j|jdS )	u0   Создадим SA поле для моделиN)primary_keynullableT)timezoner   )TSVECTORu"   Не найден тип поля )rB   rj   ri   )0rM   virtualrI   r   Z	CmfBigIntr(   ZBIGINTCmfIntIntegerZ
CmfNumericZNumericZ	precisionZscaleCmfTUUIDStringZ
max_lengthZforeign_keyrN   r	   ZColumnri   rj   ZCmfTextZTEXTZCmfStrZCmfBytesZLargeBinaryCmfRelationCmfGenericRelationCmfGenericM2MCmfDateRangeCmfM2MCmfDateTime	TIMESTAMPZCmfDateZDATEZCmfJson
CmfBackrefCmfGenericBackrefCmfBoolZBooleanZCmfTimeZTimeZCmfTsVectorZsqlalchemy.dialects.postgresqlrl   	ExceptionrB   )rQ   Z
_cmf_modelrT   Z_db_field_namecolumn_nameZsa_typer>   rl   r-   r-   r.   _make_sa_column   sv    
  

z$SQLAlchemyDataDriver._make_sa_columnc                 C   s   t | jj||d}|S )N)schema)r   r*   Zget_columns)r,   
table_namer   columnsr-   r-   r.   inspect_table_columns  s    z*SQLAlchemyDataDriver.inspect_table_columnsc                 C   s   |  |}| jj| j|jS rU   )dp_modelr*   Zdialect	has_table	__table__)r,   rR   sa_modelr-   r-   r.   r     s    
zSQLAlchemyDataDriver.has_tablec                 C   s   |  d |j}t|tjr$| d}td |  }d| d| d| }d| d| d| }|js|jd krzd| | d|j d}td	 |	| td
 |
  td d S )NZTZzadd_custom_column open sessionzalter table z add column IF NOT EXISTS  uH   Значение по умолчанию не может быть None: z	 default z	 NOT NULLzadd_custom_column run executezadd_custom_column run commitzadd_custom_column done)r~   typerI   r   rw   rf   r;   rj   rB   executecommit)r,   r   r}   Zcmf_field_typeZsa_column_typessqlr-   r-   r.   add_custom_column  s     



z&SQLAlchemyDataDriver.add_custom_columnc              
   C   st  dddd}t jt jdddd|d< t jt jdddd|d	< t jt jd
dd|d< t jt jddd|d< t|tf|}ddl}t|	dd | 
|rtd| d dS ddl}t|	dd | 
|rtd| d dS tj|jj}|  }z.td || td |  td W nB tk
rn } z"|j  td| d|  W 5 d}~X Y nX dS )u5   
        Создаем М2М таблицу
        Tcustom.r0   abstract
__module____qualname__u   ID Объекта)captionrj   rJ   left_idu   ID Элементаright_id   Имя объекта)r   rj   Zleft_name_cacheu   Имя элементаZright_name_cacher   N皙?g      ?u$   add_custom_m2m_model Таблица uT    уже создана в БД. Вероятно, на другом инстансеz!add_custom_m2m_model create tablezadd_custom_m2m_model run commitzadd_custom_m2m_model commituB   add_custom_m2m_model Ошбика создания таблицы : )r   Fieldrp   	CmfStr256r   r   randomtimesleepZuniformr   rf   r(   r   CreateTabler   r   r;   r   r   r|   transactionrollback)r,   
model_namer?   rR   r   smtpr   er-   r-   r.   add_custom_m2m_model.  s6    



z)SQLAlchemyDataDriver.add_custom_m2m_modelc              	   C   s   dddd}t jt jddddddd|d	< t jt jd
dd|d< t jt jdddddd|d< t jt jdddddd|d< t jt jddddd|d< t jt jdddddd|d< t|tf|}|S )NTr   r0   r   u)   Идентификатор объектаu3   Автоматически генерируетсяF)r   commentrj   ri   readonlyvisibleidr   )r   rJ   nameu   Сортировкаr   )r   rJ   r   rj   rB   rH   u   Кодu3   Код в реальном мире из жизни)r   rD   r   r   rj   codeu:   Код родителя в каскадном выборе)r   r   rj   rJ   Zchoice_parent_idu   СкрытьZ
cmf_hidden)	r   r   rp   r   rn   ZCmfStr64r{   r   r   )r,   r   r?   rR   r-   r-   r.   gen_custom_choice_modelR  s`    
	




	z,SQLAlchemyDataDriver.gen_custom_choice_modelc              
   C   s~   |  |}tj|jj}|  }z|| |  W n@ t	k
rx } z"|j
  td| d|  W 5 d}~X Y nX dS )ur   
        Создаем пользовательский справочник выбора таблицу
        u-   Ошбика создания таблицы r   N)r   r(   r   r   r   r   r;   r   r   r|   r   r   rf   )r,   r   rR   r   r   r   r-   r-   r.   add_custom_choice_model  s    


z,SQLAlchemyDataDriver.add_custom_choice_modelc                 C   s   |j }| j|}|s|j}g }g }ddi}|j D ].\}}	|}
| ||	|
}|dkrZq6|| q6|| 	| || |t
|d}|dd |D  t|| jf|}|| j|< |S )u]   
        Вернём SA модель для CMF модели.
        TODO: db_name
        Zextend_existingTN)Z__tablename____table_args__c                 S   s   i | ]}|j |qS r-   r   )r3   cr-   r-   r.   r6     s      z5SQLAlchemyDataDriver.dp_model_cls.<locals>.<dictcomp>)__name__models_registryrW   rO   r   r:   r~   rN   extend"_SQLAlchemyDataDriver__get_indexestupleupdater   sa_base_model)rQ   rR   r   r   rO   Z
sa_columnsr   Z__table_kwargs__rS   rT   Zsa_field_nameZ	sa_columnZsa_model_kwargsr-   r-   r.   dp_model_cls  s0    
z!SQLAlchemyDataDriver.dp_model_clsc                 C   s
   |  |S rU   )r   )r,   rR   r-   r-   r.   r     s    zSQLAlchemyDataDriver.dp_modelc                 C   s   t t|jS )uG   
        Вернём CMF модель для SA модели.
        )getattrr   r   )r,   r   r-   r-   r.   rR     s    zSQLAlchemyDataDriver.cmf_modelc                 C   s   t |tjS rU   )
isinstancer'   r   )r,   Zdp_instancer-   r-   r.   is_instance  s    z SQLAlchemyDataDriver.is_instancec                 C   s0   i }|D ]}|j j||j jj< qtj|d|S )N
table_type)r   r   fullnamer(   r)   polymorphic_union)r,   Z
cmf_modelsaliasZtablesmodelr-   r-   r.   _union_models  s    z"SQLAlchemyDataDriver._union_modelsc	                    sB   d fdd	}	i }
|D ]}|j j|
|j jj< q|	|
d|S )Np_unionTc           
   
      s  t j }i i | D ]t}| | tt jjs0 | |< i }jD ]6}r^|jkr^qJ|	|j |||j< |j
|j< qJ|< q fddg }|  D ]\}	|d k	r$|t jjfdd|D t jt jj|	|g gdtj djjk q|t jjfdd|D gdtj djjk qt jj| |S )	NTc                    sp   z| |  W S  t k
rj    rDtjtj |  |  Y S tjtj |  |  Y S Y nX d S rU   )KeyErrorr(   r   castZnulllabelZtype_coerce)r   table)
cast_nullscolnamemapstypesr-   r.   col   s    "zPSQLAlchemyDataDriver._union_models2.<locals>.hack_polymorphic_union.<locals>.colc                    s   g | ]} |qS r-   r-   r3   r   r   r   r-   r.   
<listcomp>  s     zWSQLAlchemyDataDriver._union_models2.<locals>.hack_polymorphic_union.<locals>.<listcomp>Zfrom_obj_idc                    s   g | ]} |qS r-   r-   r   r   r-   r.   r     s     )r(   utilZ
OrderedSetr   r   ZSelectr   r   rY   addr   r:   rN   selectZliteral_columnZ_quote_ddl_exprr   wherer   r   Zcorrelate_exceptZ	union_all)
Z	table_mapZtypecolnameZ	aliasnamer   ZcolnamesrY   mr   resultZtype_left_field_name
left_tablesub_ffl)r   r   r   r   r   r.   hack_polymorphic_union  s`    



	
	 zCSQLAlchemyDataDriver._union_models2.<locals>.hack_polymorphic_unionr   )r   T)r   r   r   Zlateral)r,   r   right_tabler   r   right_modelmodels_is_unionr   r   Z	join_subqr   r-   r   r.   _union_models2  s       Mz#SQLAlchemyDataDriver._union_models2c                 C   sX   d}| d |kr&| d | d | d fS | d |krH| d | d | d fS t d| |d S )N)><==>=<==IN!=><<>NOT INLIKENOT LIKEILIKE	NOT ILIKEz
SIMILAR TOzNOT SIMILAR TOEXISTS
NOT EXISTS	MEMBER_OF@@r   r   r   zInvalid filter operation)r   )paramsZop_listr-   r-   r.   _parse_params'  s    z"SQLAlchemyDataDriver._parse_paramsc                 C   s  ddl m}m} t|tjr:|jdkr2||jd |j}nXt|t	rt	 }|D ]>}t|tjr|jdkrt||jd |
|j qN|
| qN|}|dkrd }| dkr||kS | dkr||kS | dkr||k S | d	kr||kS | d
kr||kS | dkr ||kS | dkr||S | dkr(||S | dkr<||S | dkrR|| S | dkrf||S | dkr||| S | dkr||S | dkr|| S || |S d S )Nr   print_debugr9   .u<   collect_filter_exp: Значение не загруженоZNULLr   r   r   r   )r   r   )r   r   r   r   r   r   r   r   r   r   r   )cmf.includer   r9   r   r   r   _valueDEBUGvaluerL   rN   in_ZlikeZilikeop)operfieldvalr   r9   new_valr5   r-   r-   r.   _expression4  sZ    
















z SQLAlchemyDataDriver._expressionJoinDatazJleft_table, right_table, alias, left_field_name, right_model, right_modelsTc              
   C   s0  |r|j }|j}nT|j|}|s.| || t|tjtjfrLt|tjrft	d|j  d|j  d|j  d}|r| d| n|}|
 }	|	s|rt	d| d|j  d|j  dd S |	d	 }
t|	d
krtjj|
jj|d}n2|d k	r| j|d |||
|	d|d}n| |	|}| |||||
|	}|S )NuX   Недопустимый тип поля для вложенной фильтрации rC   up   . Вложенная фильтрация возможно только для CmfRelation и CmfGenericRelationZ_sub__uj   Не возможно построить запрос для вложенной фильтрации по u   , т.к. у поля u#    не указаны related_modelsr   r   r   T)r   r   )rM   instance_classr   rW   _raise_invalid_fieldrI   rr   rs   CmfSubclassedGenericRelationr   related_modelsrd   r(   r)   aliasedr   r   r   r   r  )r,   r   r   
field_name
from_aliasis_models_required	field_clsr   Z
alias_partr   r   r   	join_datar-   r-   r.   _calc_join_datah  s:    

z$SQLAlchemyDataDriver._calc_join_dataF)joinsc                 C   sb   |r
t  ndd | j D }|D ]:}|j|kr2q"| |jt|jj|j	 d|jjj
k} q"| S )uP  
        TODO cmf_deleted
        Все данные для необходимых присоединений в joins, добавим недостающие присоединения в запрос.
        froms могут быть от филдов, при этом без джойна, чтобы принудительно создать join нужен force = True.
        Или более продвинутый способ определения наличия нужного джойна.
        Сейчас при построении основного запроса, джойны принудительно добавляются.
        А при расчёте фильтров, когда нет алиасов от полей, джойн делается только если нет нужного алиаса.
        c                 S   s"   h | ]}t |tjjjr|jqS r-   )r   r(   r   
selectableAliasr   )r3   r   r-   r-   r.   	<setcomp>  s   z3SQLAlchemyDataDriver._make_joins.<locals>.<setcomp>r   )set	statementlocate_all_fromsr   	outerjoinr   r   r   r   r   r   )rg   r  forceZavailable_joinsr  r-   r-   r.   _make_joins  s    

z SQLAlchemyDataDriver._make_joinsc                 C   sd   d }g }|d krd }|j j}n|j}|dD ]0}| ||||}|| |j}|j}|j}q.|S )NrC   )	r   r   r   splitr  rN   r   r   r   )r,   	join_pathr   r   r	  r  r  r  r-   r-   r.   _calc_joins_by_path  s    

z(SQLAlchemyDataDriver._calc_joins_by_pathc                 C   s0   t |jdd}t|  d|j d| d S )N,z,
u+    не существует у модели u   , но есть: 
)strr   r2   r   rM   )r  r   Zfields_r-   r-   r.   r    s    z)SQLAlchemyDataDriver._raise_invalid_fieldc	           1   
   C   s
  |s|d fS t |d trdg }	|D ]4}
| j|
||||||d\}}|d krLq"|	| q"|||	 fS |d dkr| j|dd  ||||||dS |d dkr| j|dd  |||tj|||dS |d dkr
t|dks|d d	krtd
| j||d |d}|d fS | 	|\}}}t
t|tr0|j}t
t|trH|jj}t |ttfrg }|D ]r}t
t|trt
t|jtr||jjj n||j n*t
t|tr||jj n
|| q`|}|d}|j|d }d}t|dkrTt
|tjtjtjfrT|d }d|dd  ||g}d}|d kr|jj}nrt|dkr|d\}}}| j|||d}| ||}|d }|j}|j|}|j}d}n|d kr|jj}|dkr|dkr|jd}|std| d|j |d kr|  || t
|tjr|dkrVt |tsVt |tsVtd||||d	krx|d k	rxtd||||dkrddl!m"} | j#|d||d}|d krtd| d|j$r| t%j&}|d	kr|j'(|)|j*|k|j+|jk, }|| fS |j-sF|j'(|)|j*|k|j+|jk| .d |j/|, }ntj01tj2j3|j*g|gd!4|j*5| .d |j/|5|j+|jk}|6|t|j7d |k}||d krt|j7d d knt|j7d d kfS n&|8 }| |}|j9r|j:}|j;}n"|j<r|j;}|j:}ntd"|||d	krT|j'(|)||k, }|| fS |j=s|j'(|)||k| .d ||, }n~tj01tj2j3|g|gd!4|5| .d ||}|6|t|j7d |k}||d krt|j7d d knt|j7d d kfS ||d kr|n| fS |d#kr| j#|d||d}|d krPtd| d$| j>| d d d |d%} | j}!| j}"|j'(|"j7j}#|j$r| t%j&}$|$j}%|#|%|$j/|"j7jk}#|#)|$j*|k|$j+|jk}#nh|j<rd&}&d'}'nd'}&d&}'|8 }| |}$|$j}%t?|$|&}(t?|$|'})|#|%|)|"j7jk}#|#)|(|k}#t@|"j7d(rT|sTd(d)dg}*ng }*t@|"j7d*rz|sz|*d*d)dgg}*|r|*|g}*| j|*|!|!j|#|"||d\}#}*|#)|*}#|#, }#|Ad+r|# }#||#fS td,|||n$t
|tjtjf	r&|dkr |Ad+rd-nd}dd |g}|d#kr8td.|||| j#|d||d}|d krdtd| d/| j>| d d d |d%}+|+j},|+j}-|jB d0}.t?|-j7|.}/|j'(|/}#|.d)|g}*t@|-j7d(r|*d(d)dgg}*|r|*|g}*| j|*|,|,j|#|-||d\}#}*|#)|*}#|#, }#|Ad+	r|# }#||#fS t
|tjC
r| d0}|d1k
r| j>| d d d |d%} | j}!| j}"|j'(|"j7j}#| t%j&}$|$j}%|#|%|$j/|"j7jk}#|#)t| .d |$j*|| .d |$jD||$j+d2k}#| j#||||d}|t| .d ||| .d ||#fS |dk
r^t%jEjF|j||ddd3gd4}0|0
s:d5g}0| j#|d||d}|| .d ||0fS | j#||||d}|d k
rtd| d6| d7|| .|||fS d S )8Nr   )field_tableinclude_deletedinclude_dummyZANDr   OR)cmdr  r  r  order_by   )r   r   ut   Операция order_by в фильтре должна быть в формате: ["order_by", "=", [поля..]]r   r   rC   Fr   )r   TANYr   r   u	   Поле u    не найдено у )r   r   u   Правое значение для IN, NOT IN для m2m должно быть list, т.е. в квадратных скобках, например "field", "IN", "[obj.id]"u   Недопустимая операция над m2m полем (сравнение = только на None), допустимые IN, NOT IN, = None)r   r   r   r   r9   
with_aliasu   В таблице u3    нет поля id для фильтрации m2mr   r   uD   Надо указать для поля либо left либо right)r   r   uG    нет поля id для фильтрации в подзапросе)r  r   r   cmf_deletedr   is_dummyzNOT uh   Недопустимая операция над m2m полем, допустимые IN, NOT IN, = Noner   ul   Недопустимая операция над backref полем, допустимые EXISTS, NOT EXISTSu7    нет поля id для фильтрации backrefr   r   Z
rg_membersi'  )r   r  Zsearch_queryZonly_idssliceZNOTFOUNDu    нет поля u    для фильтрации)Gr   rL   collect_filter_exprN   r(   or_rd   r   order_queryr   rI   r   r   r   r   r   r   r  r   rW   
CmfM2MBasery   rz   joinr   r   
rpartitionr  r  r   r   rM   r  r   r9   _query_columnZnested_fieldsr   ZRelationCachesessionrg   filterZ	parent_idZparent_fieldexistsZORM_RELCACHE_EXISTS_VIA_JOINr   Zchild_idr)   r  r   r   Zdistinctr   r  r   m2m_model_clsrightr   r   leftZORM_M2M_EXISTS_VIA_JOINr  r   rP   r1   backrefCmfRelationBaseZparent_codeZCmfFullSearchZfulltext_search)1r,   Z	in_filterr   r   rg   r!  r  r  r  Z
filter_resr5   Zexpr   r9  r8  Z_rightiZ
left_partsr  r)  r  rE   r  r  r9   Zfield_column	m2m_modelZm2m_subqueryZdp_m2m_modelZdp_m2m_field_selfZdp_m2m_field_theirZself_columnZsub_dataZ	sub_modelZ	sub_tableZ	sub_queryZm2m_sa_modelZ	m2m_tablemy_id_field_namesub_id_field_namemy_sa_fieldsub_sa_fieldZsub_filter_expZbackref_dataZbackref_modelZbackref_tableZbackref_column_nameZbackref_columnZ	found_idsr-   r-   r.   r-    s2   

      




"      






 
6



	 2





     

   


  

     






z'SQLAlchemyDataDriver.collect_filter_expc                 C   s
   |  dS )NZ_m2m_directr-   r  r-   r-   r.   _m2m_depth_alias_name  s    z*SQLAlchemyDataDriver._m2m_depth_alias_namec                 C   s
   |  dS )NZ_m2m_idr-   rB  r-   r-   r.   _m2m_alias_name  s    z$SQLAlchemyDataDriver._m2m_alias_namec                 C   sh   | dg }t|tr t|}|r8t|d ts8|g}| D ]"\}}||jkr@||d|g q@|S )u   
        Преобразуем обычные kwargs в продвинутый фильтр для однообразной логики фильтрования
        :param model:
        :param kwargs:
        :return:
        r5  r   r   )	rW   r   r  r   loadsrL   r:   r   rN   )r,   r   r?   smart_filterr4   r5   r-   r-   r.   _get_filter  s    


z SQLAlchemyDataDriver._get_filterc                 C   s   |r4|j  D ]$}t|tjjjr| j|jkr|} qd|kr|d\}}|dkrvt	| j
|d}t	t|||}qt	| j
|d}nt	| j
|d}|dk	r|r|t|ddS |S dS )u   Для полей "основной" таблицы не нужен префикс таблицы, для этого нужно указать with_alias=Falser  )sumZavgcountminmaxNrC   )r  r  r   r(   r   r  r  r   r  r   r   r
   r   r  r2   )r   r}   rg   r)  Zfrom_Z	func_nameZcolumn_name_shortcolumnr-   r-   r.   r3    s    z"SQLAlchemyDataDriver._query_columnparent_joinc              	      s   |r
|j n|jj}| D ]\}| j|t|d}|dk	rJ|| q|r|r\|jp`g n|g}	 fdd|	D }
|
s|q|
d   j}|r|j	nd}| j
|||d|d}|sq|| | j||||j|d qdS )	u2   Соберём поля и таблицы по fflr(  Nc                    s>   g | ]6}|j   rt t jt jfrt t js qS r-   )r   rW   rI   rr   rs   r  )r3   Zrelated_modelr  r  r-   r.   r   .  s
   z?SQLAlchemyDataDriver._select_joined_columns.<locals>.<listcomp>r   F)r
  r   rM  )r   r   r   r:   r3  boolrN   Zright_modelsr  r   r  _select_joined_columnsr   )r,   Zfflr  r   r   rN  r   r   rL  r  Z
fields_clsZ
left_modelr	  r  r-   rO  r.   rQ  #  s6         
z+SQLAlchemyDataDriver._select_joined_columns)r  group_byr  c                O   s  |  |}|  }	ddlm}
 |
jd|j |d}t| |}|rZ| }|	|	}nht

 }g }g }| |||| |	j| }| j||dd}t

 | dkr| 	d }t| || t|dkr|j}tt|D ]}|| ||| < q| ||}|r,| j||||||d\}}||}|rg }|D ]T}||jjkr`||jj|  n,| d	|jjkr:||jj| d	  q:|j| }|S )
Nr   	CMF_CACHEzquery: full_fields_loadT)r  g{Gz?r  r  r   )r   r;   r   rT  hashr   r   r[   Z_cloneZwith_sessionr   rQ  rg   r  rh   rd   ri   rangerG  r-  r5  r   r   rN   rR  )r,   r   rV  r  rR  r  r>   r?   r   r   rT  rY   rg   Zstart_calc_timer  r   Zclone_qpkeyr<  rF  
filter_expr}   r-   r-   r.   _create_queryA  sH    




z"SQLAlchemyDataDriver._create_query)
r"  rV  r,  
for_updatemapperr  load_m2mrR  aggregate_selectr  c       
         O   s   t |t }t|}| |}| |}| j|f||||	|
d| }| j||||
|d}|rn|j| }|
r|	 }td}|j
|||d}|S |r| }|	 }|j
|||d}|r|r| j|||||d |r|  |S dS )   Грубо селектим всю таблицу, фильтры:
          - в kwargs: поле -> значение
        )rV  r  r  rR  r`  )r,  r`  r   ZsimplerV  r   rW  N)r   r!   
get_mappermap_args
map_kwargsr\  with_labelsr/  r,  alllist_to_cmfwith_for_update	_load_m2mpost_mapping_hook)r,   r   r"  rV  r,  r]  r^  r  r_  rR  r`  r  r>   r?   mapper_is_my_argsqZsql_resZres_objsr-   r-   r.   rL   t  sJ    


    


  zSQLAlchemyDataDriver.list)rV  rR  c                   s
  |  |}| |}| j|fd|i| }|r| |  fdd|D }|jj|jj |t	
 fj| d}	t|j|	}
g }|
D ]<}i }t|D ]\}}|| ||< q|d |d< || q|S |jj|jj t	
 gd}	|j|	 }|S )ra  rV  c                    s   g | ]} j j| qS r-   )r   r   )r3   r}   r   r-   r.   r     s     z.SQLAlchemyDataDriver.count.<locals>.<listcomp>Nr%  rI  )rd  re  r\  rf  r   r  select_fromfromsZwith_only_columnsr
   rI  rR  r"  rL   r4  r   	enumeraterN   Zscalar)r,   r   rV  rR  r>   r?   rm  rn  r   Zcount_qres_listresrowZrow_resr<  r}   rI  r-   ro  r.   rI    s>    


	
"zSQLAlchemyDataDriver.count)r"  rV  r]  r^  r  r  c                O   s   t |t }
t|}| |}| |	}	| j|f||||d|	}| }| j||ddg|d}|rr| }|	 }|r|j
|||d}| j|g||||d |
r|  |S dS )u   
        Выбрать один объект по фильтру:
          - в args - первичный ключ(опционально)
          - в kwargs - другие поля: поле -> значение)rV  r  r  r   r   )r,  r   rb  )r^  r  r  N)r   r!   rc  rd  re  r\  rf  r/  ri  firstobject_to_cmfrj  rk  )r,   r   r"  rV  r]  r^  r  r  r>   r?   rl  rn  sa_instanceZres_objr-   r-   r.   rW     s(    


zSQLAlchemyDataDriver.getc           =   
      s,  |r|sdS t |t  fdddd }t|}tt}tt}	tt}
|D ]&}|| | |	|j | qN|	D ]t}tt|}|D ]`}|| sqt||d}|rt|t	j
rt|t	jr||  qt|t	jt	jfr|
| | qqz| D ]
\}}g }|D ]8}tt|j}|j	|}|r
t|t	jr
 rt||}|jdk	rt|jdk	r
||j q
d|_|jdkrd|_| }t||}|j}|dkrtd||n`t||d}|dk	r|r
|| q
t||d | }t||d}|dkrtd|||r
|dd	 }|| | | q
| D ]\}}tt|d}|slqNt  d
dt|g}| j||| ||d}|D ]H}|}|| D ]0} rt||}||_||_nt||| qqqN|r| j||| |||d q|
 D ]\}}|D ]}|j}|	| }t||}t|t	jr|jrt }|j d} | dfdd|D g}!tt|| f| i i}"| D ]r}#|!}$d|#j	kr|s|!dddgg}$d|#j	kr|s|$dddgg}$| j|#|"|$||d}%| j|%|#d||#j< q|sHd	dlm }&m!}' |&|'j"d|j d|j d qnVt#|dkrrt  t$t%|& ' }(n,t  | (t)j*+dd | D d' }(tt})|(D ]}*t|*| }+|)|+ |* qg },|D ]}|)|}- rNt||}|-sg |_|jdkrg |_q|,|-|| |_|jdkr@t-|j|_|,.|j n<|-sdt||g  q|,|-|| }t||| |,.| q|,r | j|,|| |||d qt|t	jr|j/rd}.d }/nd }.d}/|0 }0| 1|0}1t|1|.}2t|1|/}3| (|2|32|23fd!d|D }4tt}5tt}6t  |4D ]@\}7}8|8r>|7sVq>|6|8 |7 |5|8dd	  |8 q>|5 D ](\}}9tt|}t  d
d|9g}|st4|drdddg|g}|st4|drdddg|g}| j||| ||dd"},|,r"| j|,|| |||d |,D ]}|6| D ]t}7||7 D ]d}: rt|:| };|;jdkrfg |;_|;j| n0t|:|d}|dkrg }t|:|| || qBq6q&q|D ]f} r|| }<|<jdkrg |<_|<jdkrt-|<j|<_|<5  nt||ddkrt||g  qqqdS )#u   
        Прогрузка М2М и М2О полей
        :param objs: список обьектов
        :param full_fields_load: поля обьектов
        :return:
        Nc                    s    r| j jS | j S rU   )r   r   )Zobj_)is_heavyr-   r.   obj_id  s    z.SQLAlchemyDataDriver._load_m2m.<locals>.obj_idc                  S   s   dd } t | S )Nc                   S   s   t tS rU   )r   rL   r-   r-   r-   r.   id_to_list_factory  s    zWSQLAlchemyDataDriver._load_m2m.<locals>.model_to_id_factory.<locals>.id_to_list_factory)r   )r{  r-   r-   r.   model_to_id_factory  s    z;SQLAlchemyDataDriver._load_m2m.<locals>.model_to_id_factory.u#   Не загружено id поле:r   r   r   )rV  r^  r5  rW  r   c                    s   g | ]} |qS r-   r-   r3   orz  r-   r.   r   f  s     z2SQLAlchemyDataDriver._load_m2m.<locals>.<listcomp>r*  r   Fr+  )rV  r5  r  r  r$  r   zORM: skip load field rC   z die no related modelsr   c                 S   s   i | ]\}}||j qS r-   )r  )r3   r4   r   r-   r-   r.   r6     s      z2SQLAlchemyDataDriver._load_m2m.<locals>.<dictcomp>r   r   r   c                    s   g | ]} |qS r-   r-   r~  r  r-   r.   r     s     )rV  r5  r^  r_  )6r   r"   r   rL   rN   rM   r   r   rI   r   r   r;  r0  ZCmfBackrefBaser:   rW   r   Z_oldZid_fieldnamer   setattr	partitionr#   rj  r:  dictr$   r  r\  r/  rO   r   r   r9   r   rd   nextitervaluesrg  rg   r(   r)   r   rh  r   r   r9  r7  r   r5  r   rP   Zapply_changes)=r,   ZobjsrV  r^  r  r  r|  Zto_load_mappingZmy_objs_by_idZmy_objs_by_model_nameZmodels_by_rel_fieldsobjr   r   r  r  Zmodel_to_idZalready_loaded_obj_listr   Zid_field_nameZid_fieldZid_field_valueZfield_valueZrel_model_nameZid_to_obj_listZ_filterZ	load_objsZload_objZ	loaded_idZmodels_listZobj_listZm2mqueryZbackref_field_nameZfilter_qZ	field_fflZ	rel_modelZmodel_query_filterrn  r   r9   rs  Zres_dictr   Zm2m_idZsub_objsZref_obj_listr>  r?  r=  r   r@  rA  Z	m2m_queryZid_by_modelsZmy_id_by_sub_idZmy_idZsub_idZsub_id_listZmy_objZmy_fieldZ	obj_fieldr-   )ry  rz  r.   rj    s   









  


 







   

zSQLAlchemyDataDriver._load_m2mc                 C   s6   g }|D ](}t |tjr&||j q|| q|S rU   )r   r   r   rN   r   )r,   r>   Znew_argsargr-   r-   r.   rd    s    zSQLAlchemyDataDriver.map_argsc                 C   s:   i }|  D ](\}}t|tjr,|j||< q|||< q|S rU   )r:   r   r   r   r   )r,   r?   Z
new_kwargsr4   r5   r-   r-   r.   re    s    
zSQLAlchemyDataDriver.map_kwargs)	no_reloadc             
   O   s  |  |}| |}|  }| |j}| }	t }
|
||	 z||	 |j	|	gd W n t
k
r } z4|j  t|jttrt||t||W 5 d}~X Y n0 tk
r } z|j  |W 5 d}~X Y nX d|_|rdS |
j|	|d}|
  d|_|S )um   
        no_reload - не загружать объект из базы и не возвращать
        ZobjectsNT)
cmf_entity)rd  re  r;   r   rA   r!   rc  cmf_to_objectr   flushr   r   r   r   origr   lookupr   r   r   r|   is_newrw  rk  )r,   instancer  r>   r?   rm  _kwargsr   r   rx  r^  r   r  r-   r-   r.   create  s2    





zSQLAlchemyDataDriver.createc              
      s   |  |}| |}|  }|  j}t j}|  t fdd|D }	||	|	}
z|
|
 W n0 tk
r } z|j  |W 5 d}~X Y nX  S )u*   TODO логическое удалениеc                 3   s   | ]}t  |V  qd S rU   r   r3   fr  r-   r.   	<genexpr>4  s     z.SQLAlchemyDataDriver.delete.<locals>.<genexpr>N)rd  re  r;   r   rA   r   ri   r   rg   rW   deleter|   r   r   )r,   r  r>   r?   rm  r  r   r   rZ  get_argsrx  r   r-   r  r.   r  ,  s    



zSQLAlchemyDataDriver.deletec                 K   s`   |  |}|  }||}| ||}|rP| j||||||d\}}	||	}|jdd}
|
S )NrW  Zfetch)Zsynchronize_session)r   r;   rg   rG  r-  r5  r  )r,   r   r  r  r?   r   r   rg   rF  r[  Z
delete_cntr-   r-   r.   bulk_delete=  s     

     

z SQLAlchemyDataDriver.bulk_deleterW  )r  c                O   s   ddl m} | |}|  }	|	|}
| ||}|r\| j||||
||d\}
}|
|}
|
j}|
j	}||
||j}|D ]}||}q||}|	|}dd |D S )Nr   r   rW  c                 S   s   g | ]}|d  qS )r   r-   )r3   ru  r-   r-   r.   r   _  s     z4SQLAlchemyDataDriver.bulk_update.<locals>.<listcomp>)r(   r   r   r;   rg   rG  r-  r5  Z
_criterionZ_join_entitiesr  	returningr   rp  r   r   )r,   r   r  r  r  r>   r?   r   r   r   rg   rF  r[  Zfilter_criteriaZjoin_clausesstmtZjoin_clauser   r-   r-   r.   bulk_updateI  s$    





z SQLAlchemyDataDriver.bulk_updaterU  c             
      s>  ddl m} |  }| t }t j}| t fdd|D }	i }
t j	 D ]\}}t
|tjrpqZt
|tjr~qZt
|tjrqZt
|tjrqZt
|tjrqZt
|tjrqZt
|tjrqZt |}t|tjr|jdksZ|jrqZ|js|jsqZ| }|dkr(tdt d| d| ||
|<  | jrZd | _qZ|
s`td	  d
  S |||
}|t||d |	d k|j }z$t!t"|#|dkst$dW n t%k
r } z2|j&'  t|j(t)*t+rt, |t- |W 5 d }~X Y n2 tk
r8 } z|j&'  |W 5 d }~X Y nX  S )Nr   r  c                 3   s   | ]}t  |V  qd S rU   r  r  r  r-   r.   r  g  s     z1SQLAlchemyDataDriver.update_v2.<locals>.<genexpr>.u%   Попытка update field=Ellipsis r   FzDEV: warning! UPDATE u*    не содержит изменений.r   uA   Доп.защита, что только 1 объект в update).r(   r   r;   r   r   ri   rd  r   r   r:   rI   rr   rs   rv   rt   ry   ru   rz   r   r   r   r   rm   Z
is_changedZ	orm_dirtyZ	db_formatr|   r  rf   r  r   r  r   rd   rL   r   AssertionErrorr   r   r   r  r   r  r   r   r   )r,   r  rV  r>   r?   r   r   r   rZ  r  r  r4   r5   Zvvr  r   r-   r  r.   	update_v2a  sb    



$$

zSQLAlchemyDataDriver.update_v2c             
      s\  ddl m} |jr,| j f|dd i|S | |}| |}|  }| t }t j	}| t
 fdd|D }	|||	}
t }| |
 z|j|
gd W n tk
r } z0|j  t|jttrt |t |W 5 d }~X Y n2 tk
r2 } z|j  |W 5 d }~X Y nX  j}|j|
 |d}|  ||_|S )Nr   r'  rV  c                 3   s   | ]}t  |V  qd S rU   r  r  r  r-   r.   r    s     z.SQLAlchemyDataDriver.update.<locals>.<genexpr>r  )r  rV  )r   r9   ZORM_RAW_UPDATE_V2r  rd  re  r;   r   r   ri   r   rg   rW   r!   rc  r  r  r   r   r   r   r  r   r  r   r   r   r|   r  rw  rk  )r,   r  rV  r>   r?   r9   r   r   rZ  r  rx  r^  r   r  r  r-   r  r.   r     s6    




zSQLAlchemyDataDriver.updatec                    s   t    d S rU   )r7   before_requestr+   r@   r-   r.   r    s    z#SQLAlchemyDataDriver.before_requestc                 C   s   ddl m} |  |  t }|    t | dkrjddl m}m} ||j	dt |   t }| j
  t | dkrddl m}m} ||j	dt |   tdd	 t  d S )
Nr   rS  g333333?r   zPROF Session.commit() got r   zPROF Session.remove() got TZnear_commit_or_rollback)r   rT  cache_unlockZcache_transaction_startr   r;   r   r   r9   r   remover%   r&   )r,   rT  Zprof_str   r9   r-   r-   r.   r     s    

zSQLAlchemyDataDriver.commitc                 C   s>   ddl m} tdd |  |    | j  t  d S )Nr   rS  Tr  )r   rT  r%   r  r;   r   r  r&   )r,   rT  r-   r-   r.   r     s    

zSQLAlchemyDataDriver.rollbackc                 O   s   |   j||S rU   r;   rg   r=   r-   r-   r.   rg     s    zSQLAlchemyDataDriver.queryc                 O   s   |   j||S rU   r  r=   r-   r-   r.   query_deprecated  s    z%SQLAlchemyDataDriver.query_deprecatedc                    s  ddl m} g }|s& jpg }| }|r.|}|rD|rD|sD|dg7 }td fdd}|D ]$}	d}
|	dr~|	d	d
 }	d}
||	}|jjd j	|}|d
kr|jjd j	|	}|d
kr|j
D ]}|d |	kr|d } qq|d
krtd|	 dddlm} |jrft|jtjjr*|d }n<t|jtjjrF|d }n t|jtjjrf|tdd }|
rvt|}||}qZ|S )u3   Сортировка по логике моделиr   )
QueryProxyr   )cmf_column_namec                    s`   d}|  d\}}}|s|} qD|} |r8| d| d}q| d}q|sT j d}| |  S )Nr0   rC   rE   Z_sub_)r  rO   )r  Zalias_prefixZp1rE   Zp2r$  r-   r.   calc_dp_column_name   s    z=SQLAlchemyDataDriver.order_query.<locals>.calc_dp_column_nameF-r   NTr   expru   Не найдено поле uK    или тип поля не поддерживает сортировку.r'  r0   )Zminutes)Zcmf.data_providers.baser  Zorderingr   r  r1   r  rq  r   rW   Zcolumn_descriptionsr   r   r9   ZORM_ORDERBY_IDX_SKIPr   r   r(   r   ro   rq   ZDateTimer   Zdescr"  )r,   rg   r"  r,  r`  r   r  Zorder_fieldsr  rY   Z
order_descZdp_column_nameZ	dp_columnZcolumn_descriptionr9   r-   r$  r.   r/    sL    







z SQLAlchemyDataDriver.order_queryc                    s   t  | d S rU   )r7   make_models)rQ   r   r@   r-   r.   r  C  s    z SQLAlchemyDataDriver.make_modelsc                 C   s^   t tD ]P}tt|}t|tr|jr(q|jr<| j|jkr<q|jsN| jtkrNq| 	| qdS )u   Создадим схемуN)
dirr   r   r   r   r   Zdata_sourcesr   r   r   )r,   r   rR   r-   r-   r.   r<   H  s    
zSQLAlchemyDataDriver.init_metac                 C   s   |   }t| }d| kr(td|d |  | j| j	 t
jt
 d}t|}|d| jd  t|d d S )NZalembic_versionzDatabase has alembic versionz'CREATE EXTENSION IF NOT EXISTS pg_trgm;zalembic.inizsqlalchemy.urlhead)r;   r   Z
connectionZget_table_namesFileExistsErrorr   r   db_metaZ
create_allr*   rb   pathr1  getcwdr   Zset_main_optionr9   r   Zstamp)r,   r   Z	inspectorZalembic_config_fileZalembic_configr-   r-   r.   init_dbZ  s    
zSQLAlchemyDataDriver.init_dbc                 C   s   t   | j  d S rU   )r   r*   Zdisposer+   r-   r-   r.   r   l  s    z'SQLAlchemyDataDriver.close_all_sessions)N)N)NTNN)F)N)NT)N)FF)FF)NNNN)Lr   r   r   __doc__r   r(   ZMetaDatar  extZdeclarativeZdeclarative_baser   r   rV   ra   r/   r8   classmethodr   r[   rh   r~   r   r   r   r   r   r   r   r   rR   r   r   r   staticmethodr   r   r   r  r  r   r  r  r  and_r-  rC  rD  rG  r3  rQ  r\  rL   rI  rW   rj  rd  re  r  r  r  r  r  r  r   r  r   r   rg   r  r/  r  r<   r  r   __classcell__r-   r-   r@   r.   r'       s    
5

K
$3
&S

1

#


  <

3      "$
  $		$
=(
Xr'   c                   @   s   e Zd Zdd ZdS )SAModelAccessorc                 C   s   |dkrd S t tt|S )N)Z_pytestfixturefunctionZ__test__)r'   r   r   r   )r,   itemr-   r-   r.   __getattr__r  s    zSAModelAccessor.__getattr__N)r   r   r   r  r-   r-   r-   r.   r  q  s   r  )=rb   r   collectionsr   r   r   typingr   r   Zdatetimer   Zsqlalchemy.typesr(   Zsqlalchemy.ext.declarativeZsqlalchemy.ormZsqlalchemy.sql.selectableZsqlalchemy.excr   r	   r
   r   Zflaskr   r   Zalembic.configr   Zalembicr   Zpsycopg2.errorcodesr   Zpsycopg2r   r   Zcmf.fields.base_fieldsr   r   Zcmfr   r   r   Z
base_errorr   r   r   Zmodels.base_modelr   r   r   r   baser    r!   r"   r#   Zutil.immutablesr$   Zutil.cmfutilr%   r&   r'   r  r-   r-   r-   r.   <module>   sT                 _