
    ג[i 7                     l    d Z ddlZddlmZmZmZmZmZ ddl	m
Z
 ddlmZ ddlmZ  G d d          ZdS )	u  
Data Service Layer

This module handles loading and saving tabular data from MongoDB collections.
It provides the bridge between MongoDB documents and pandas DataFrames for UI editing.

Key responsibilities:
- Load collection documents as normalized DataFrames
- Handle schema normalization (missing fields → None)
- Convert ObjectId to string for UI display
- Track changes between original and edited data
- Apply changes back to MongoDB with user confirmation
    N)DictListAnyTupleOptional)ObjectId)datetime)JSONFlattenerc                      e Zd ZdZd Zdeeef         deeef         fdZde	eeef                  de	e         fdZ
	 	 dd
edededej        fdZ	 	 dd
ededede	eeef                  fdZdej        dej        dee	e         e	e         e	e         f         fdZdej        dej        deeef         fdZd
edej        dej        deeef         fdZdS )DataServicez
    Service for loading and saving collection data in tabular format.
    
    This service normalizes MongoDB documents into pandas DataFrames,
    tracks changes, and applies updates back to MongoDB.
    c                 :    || _         t                      | _        dS )z
        Initialize data service with MongoDB connection.
        
        Args:
            mongo_service: MongoService instance
        N)mongor
   	flattener)selfmongo_services     1/var/www/html/IGF-ODF-V3/services/data_service.py__init__zDataService.__init__   s     #
&    docreturnc                 ^   i }|                                 D ]\  }}t          |t                    rt          |          ||<   -t          |t                    r|                                ||<   Zt          |t                    r|||<   ut          |t                    r|||<   |||<   |S )a   
        Normalize a MongoDB document for DataFrame display.
        
        Converts ObjectId to string and handles nested structures.
        
        Args:
            doc: MongoDB document
            
        Returns:
            Normalized document with ObjectId as string
        )items
isinstancer   strr	   	isoformatdictlist)r   r   
normalizedkeyvalues        r   _normalize_documentzDataService._normalize_document(   s     
))++ 	( 	(JC%** ("%e**
3E8,, 
("'//"3"3
3E4(( ("'
3E4(( ("'
3"'
3r   	documentsc                     t                      }|D ])}|                    |                                           *t          |          }d|v r+|                    d           |                    dd           |S )z
        Get union of all fields across documents.
        
        Args:
            documents: List of MongoDB documents
            
        Returns:
            Sorted list of unique field names
        _idr   )setupdatekeyssortedremoveinsert)r   r"   
all_fieldsr   fieldss        r   _get_all_fieldszDataService._get_all_fieldsH   s     UU
 	* 	*Cchhjj)))) 
## F? 	$MM%   MM!U###r     r   collection_namelimitskipc                 2     j                             |||          }|st          j        dg          S  fd|D             }                     |          }t          j        ||          }|                    t          j        |          d          }|S )a  
        Load MongoDB collection as a normalized pandas DataFrame.
        
        All documents are normalized to have the same columns.
        Missing fields are filled with None.
        ObjectId fields are converted to strings.
        
        Args:
            collection_name: Name of the collection
            limit: Maximum number of documents to load
            skip: Number of documents to skip
            
        Returns:
            DataFrame with normalized documents
        r$   )columnsc                 :    g | ]}                     |          S  )r!   ).0r   r   s     r   
<listcomp>z<DataService.load_collection_as_dataframe.<locals>.<listcomp>~   s'    NNNS433C88NNNr   N)r   sample_documentspd	DataFramer-   wherenotnull)r   r/   r0   r1   r"   normalized_docsr+   dfs   `       r   load_collection_as_dataframez(DataService.load_collection_as_dataframea   s    , J//MM	 	1<0000 ONNNINNN ))/::
 \/:>>> XXbjnnd++	r     c                    | j                             |||          }|sg S g }|D ]}d|v r3t          |d         t                    rt	          |d                   |d<   | j                            |          }|                                D ]Y\  }}	t          |	t                    rt	          |	          ||<   -t          |	t                    r|		                                ||<   Z|
                    |           |S )a0  
        Load MongoDB records and flatten them for comparison with JSON records.
        
        This method:
        1. Loads documents from MongoDB
        2. Flattens each document using JSONFlattener
        3. Converts ObjectId to string
        4. Returns list of flattened dictionaries
        
        Args:
            collection_name: Name of the collection
            limit: Maximum number of documents to load
            skip: Number of documents to skip
            
        Returns:
            List of flattened document dictionaries
        r$   )r   r8   r   r   r   r   flattenr   r	   r   append)
r   r/   r0   r1   r"   flattened_recordsr   	flattenedr   r    s
             r   load_flattened_recordsz"DataService.load_flattened_records   s   0 J//MM	 	I 	0 	0C| -
3u:x @ @ - U__E
 ..s33I (oo// 7 7
UeX.. 7%(ZZIcNNx00 7%*__%6%6IcN$$Y////  r   original_df	edited_dfc                 t   g }g }g }t                      }d|j        v r?t          |d                                                             t                              }t                      }d|j        v r?t          |d                                                             t                              }t          ||z
            }|                                D ]\  }}	|	                                }
d |
                                D             }
|
	                    d          }|r|dk    st          j        |          r,|
                    dd           |                    |
           t	          |          |vr,|
                    dd           |                    |
           ||d         |k             }|j        s|j        d                                         }d |                                D             }d}i }|
                                D ]W\  }}|dk    r|	                    |          }t          j        |          rt          j        |          rJ||k    rd}|||<   X|                                D ]}|dk    r||
vrd}d||<   |r|                    ||d	           |||fS )
a  
        Identify new, updated, and deleted documents.
        
        Args:
            original_df: Original DataFrame from MongoDB
            edited_df: Edited DataFrame from UI
            
        Returns:
            Tuple of (new_docs, updated_docs, deleted_ids)
        r$   c                 B    i | ]\  }}t          j        |          ||S r5   r9   notnar6   kvs      r   
<dictcomp>z1DataService._identify_changes.<locals>.<dictcomp>   s+    IIIARXa[[I1IIIr    Nr   c                 B    i | ]\  }}t          j        |          ||S r5   rK   rM   s      r   rP   z1DataService._identify_changes.<locals>.<dictcomp>   s,    $[$[$[darxXY{{$[Q$[$[$[r   FT)r$   updates)r%   r3   dropnaastyper   r   iterrowsto_dictr   getr9   isnapoprC   emptyilocr'   )r   rG   rH   new_docsupdated_docsdeleted_idsoriginal_ids
edited_idsidxrowrow_dictdoc_idoriginal_roworiginal_dictchangedrS   r   r    original_values                      r   _identify_changeszDataService._identify_changes   s     uuK'' 	H{5188::AA#FFGGL UU
I%% 	DYu-4466==cBBCCJ <*455 "**,, 8	 8	HC{{}}H JI)9)9IIIH\\%((F 0Vr\ 0RWV__ 0 UD)))))))VL0 * UD)))))))
  +;u+=+GH#) $0$5a$8$@$@$B$BM$[$[m6I6I6K6K$[$[$[M $G G&.nn&6&6 1 1
U%< %$)6):):3)?)? 75>> 1bgn.E.E 1$"n4 1&*G+0GCL  -1133 0 0%< 0Cx,? 0&*G+/GCL $++#)'.- -   
 {22r   c                     |                      ||          \  }}}t          |          t          |          t          |          |||dS )z
        Preview changes between original and edited DataFrames.
        
        Args:
            original_df: Original DataFrame
            edited_df: Edited DataFrame
            
        Returns:
            Dictionary with change summary
        )	new_countupdated_countdeleted_countr]   r^   r_   )rj   len)r   rG   rH   r]   r^   r_   s         r   preview_changeszDataService.preview_changes  s^     /3.D.D/
 /
+,
 X .. -- (&
 
 	
r   c           
         | j                             |          }|dddS |                     ||          \  }}}ddddg d}	 |rp	 |                    |          }	t	          |	j                  |d<   nB# t          $ r5}
|d	                             d
t          |
                      Y d}
~
nd}
~
ww xY w|D ]}	 |d         }|d         }t          |t                    rt          |          }d |                                D             }d |                                D             }i }|r||d<   |r||d<   |r(|                    d|i|           |dxx         dz  cc<   # t          $ r8}
|d	                             d| dt          |
                      Y d}
~
d}
~
ww xY w|D ]}	 t          |t                    rt          |          }|                    d|i           |dxx         dz  cc<   O# t          $ r8}
|d	                             d| dt          |
                      Y d}
~
d}
~
ww xY wnG# t          $ r:}
d|d<   |d	                             dt          |
                      Y d}
~
nd}
~
ww xY w|S )a   
        Apply changes from edited DataFrame back to MongoDB.
        
        This method:
        1. Identifies new, updated, and deleted documents
        2. Inserts new documents
        3. Updates existing documents (only changed fields)
        4. Deletes removed documents
        
        Args:
            collection_name: Name of the collection
            original_df: Original DataFrame from MongoDB
            edited_df: Edited DataFrame from UI
            
        Returns:
            Dictionary with operation results
        NFz0Collection not found or not connected to MongoDB)successerrorTr   )rr   insertedupdateddeletederrorsrt   rw   zInsert error: r$   rS   c                     i | ]
\  }}|||S )Nr5   rM   s      r   rP   z-DataService.apply_changes.<locals>.<dictcomp>w  s#    "U"U"UDAqq"U1a"U"U"Ur   c                     i | ]
\  }}||dS )NrQ   r5   rM   s      r   rP   z-DataService.apply_changes.<locals>.<dictcomp>x  s#    $T$T$Ttq!!$TQ$T$T$Tr   z$setz$unsetru      zUpdate error for z: rv   zDelete error for rr   zGeneral error: )r   get_collectionrj   insert_manyro   inserted_ids	ExceptionrC   r   r   r   r   
update_one
delete_one)r   r/   rG   rH   
collectionr]   r^   r_   resultsinsert_resulte
update_docre   rS   set_updatesunset_updates	update_ops                    r   apply_changeszDataService.apply_changes7  s   . Z..??
 	 K   /3.D.D/
 /
+,
 
 
6	A HH$.$:$:8$D$DM*-m.H*I*IGJ''  H H HH%,,-Fc!ff-F-FGGGGGGGGH + U U
U'.F(3G "&#.. 2!)&!1!1 #V"UGMMOO"U"U"UK$T$Tw}}$T$T$TM "I" 8,7	&)$ <.;	(+  0"--"FO%    	***a/***  U U UH%,,-S-S-S3q66-S-STTTTTTTTU & 
U 
U	U!&#.. 2!)&!1!1))5&/:::I&&&!+&&&&  U U UH%,,-S-S-S3q66-S-STTTTTTTTU
U  	A 	A 	A!&GIH$$%?s1vv%?%?@@@@@@@@	A s   I ,A5 4I 5
B4?+B/*I /B44I <B*E'&I '
F)1.F$I $F))I 1AG=<I =
H?.H:5I :H??I 
J0JJN)r.   r   )r@   r   )__name__
__module____qualname____doc__r   r   r   r   r!   r   r-   intr9   r:   r?   rF   r   rj   rp   r   r5   r   r   r   r      s        ) ) )tCH~ $sCx.    @d38n)= $s)    8 	) )) ) 	)
 
) ) ) )\ 	0! 0!0! 0! 	0!
 
d38n	0! 0! 0! 0!d[3\[3 <[3 
tDz4:tCy0	1	[3 [3 [3 [3z
\
 <
 
c3h	
 
 
 
8dd \d <	d
 
c3hd d d d d dr   r   )r   pandasr9   typingr   r   r   r   r   bsonr   r	   services.json_flattenerr
   r   r5   r   r   <module>r      s         3 3 3 3 3 3 3 3 3 3 3 3 3 3             1 1 1 1 1 1E E E E E E E E E Er   