
    /_i]                     J   d dl Z d dlmZ d dlZ	 d dlmZ d dlmc m	Z
 d dlmZmZ dZdZdZdZdZe j(                  j*                  e j(                  j                  gZ e j.                  d	
      d        Z e j.                  d	
      d        Zd Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*d Z+d Z,e j(                  j[                  d !      d"        Z.e j(                  j[                  d#!      d$        Z/d% Z0d& Z1d' Z2y# e$ r dZdZ
Y w xY w)(    N)	timedelta)InMemoryKmsClientverify_file_encryptedzencrypted_table.in_mem.parquets   0123456789112345
footer_keys   1234567890123450col_keymodule)scopec                      t         j                  j                  t        j                  g d      t        j                  g d      t        j                  g d      d      } | S )N)         )abc)xyz)paTablefrom_pydictarray)
data_tables    d/var/www/html/land_sniper/venv/lib/python3.12/site-packages/pyarrow/tests/parquet/test_encryption.pyr   r   0   sG    %%XXi XXo&XXo&' J
     c                  L    t        j                  t        t        ddgi      } | S )Nr   r   )r   column_keys)peEncryptionConfigurationFOOTER_KEY_NAMECOL_KEY_NAME)basic_encryption_configs    r   r!   r!   :   s,     88"3*

 #"r   c                 f    t        j                  |       }d }t        j                  |      }||fS )z
    Sets up and returns the KMS connection configuration and crypto factory
    based on provided KMS configuration parameters.
    custom_kms_confc                     t        |       S Nr   kms_connection_configurations    r   kms_factoryz1setup_encryption_environment.<locals>.kms_factoryK        !=>>r   )r   KmsConnectionConfigCryptoFactory)r$   kms_connection_configr*   crypto_factorys       r   setup_encryption_environmentr0   D   s7    
 22?S? %%k2N .00r   c                     ||j                  d      ||j                  d      i}t        |      \  }}	t        | ||||	       ||	fS )zL
    Writes an encrypted parquet file based on the provided parameters.
    UTF-8)decoder0   write_encrypted_parquet)
pathr   footer_key_namecol_key_namer   r   encryption_configr$   r.   r/   s
             r   write_encrypted_filer9   T   sc     	**73gnnW-O -I-)> D*.?1>C !.00r   c           	      v   | t         z  }t        j                  t        t        ddgidt        d      d      }|j                  du sJ t        ||t        t        t        t        |      \  }}t        |       t        j                  t        d      	      }t        ||||      }|j                  |      sJ y
)DWrite an encrypted parquet, verify it's encrypted, and then read it.r   r   
AES_GCM_V1      @minutes   r   r   encryption_algorithmcache_lifetimedata_key_length_bitsFrC   N)PARQUET_NAMEr   r   r   r    r   uniform_encryptionr9   
FOOTER_KEYCOL_KEYr   DecryptionConfigurationread_encrypted_parquetequalstempdirr   r5   r8   r.   r/   decryption_configresult_tables           r   !test_encrypted_parquet_write_readrQ   j   s    \!D
 22"3*
 * - " //5888,@j/<W-)> $ 22 -/)!6HL\***r   c           	      ^   | t         z  }t        j                  t        ddt	        d      d      }|j
                  du sJ t        ||t        t        t        d|      \  }}t        |       t        j                  t	        d            }t        ||||      }|j                  |      sJ y	)
r;   Tr<   r=   r>   r@   )r   rG   rB   rC   rD   r   rE   N)rF   r   r   r   r   rG   r9   r    rH   r   rJ   rK   rL   rM   s           r   )test_uniform_encrypted_parquet_write_readrS      s    \!D 22") - " //4777,@j/<S-)> $ 22 -/)!6HL\***r   c                     |j                  ||      }|J t        j                  | |j                  |      5 }|j	                  |       d d d        y # 1 sw Y   y xY w)N)encryption_properties)file_encryption_propertiespqParquetWriterschemawrite_table)r5   tabler8   r.   r/   rV   writers          r   r4   r4      se    !/!J!J0"2%111			%,,"<
> "AG5!" " "s   AAc                 2   |j                  ||      }|J t        j                  | |      }|j                  dk(  sJ t        j                  | |      }t        |j                        dk(  sJ t        j                  | |      }|j                  d      S )Ndecryption_propertiesr   Tuse_threads)	file_decryption_propertiesrW   read_metadatanum_columnsread_schemalennamesParquetFileread)r5   rO   r.   r/   rb   metarY   results           r   rK   rK      s    !/!J!J0"2%111$>@Dq   ^^$>@Fv||!!!^^$>@F;;4;((r   c           	      
   | t         z  }t        j                  t        t        ddgidt        d      d      }t        ||t        t        t        t        |       t        |       t        t        t        j                  d      t        t        j                  d      i      \  }}t        j                  t        d      	      }t        j                  t        d
      5  t!        ||||       ddd       y# 1 sw Y   yxY w)zYWrite an encrypted parquet, verify it's encrypted,
    and then read it using wrong keys.r   r   r<   r=   r>   r@   rA   r2   rE   zIncorrect master key usedmatchN)rF   r   r   r   r    r   r9   rH   rI   r   r0   r3   rJ   pytestraises
ValueErrorrK   )rN   r   r5   r8   wrong_kms_connection_configwrong_crypto_factoryrO   s          r   +test_encrypted_parquet_write_read_wrong_keyrt      s     \!D
 22"3*
 * - " z?L#W.?A $8T0j''0V 95!5
 22 -/	z)E	F "#%@ 	"" " "s   !C99Dc                     t        | |       t        j                  t        d      5  t	        j
                  | t        z        j                          ddd       y# 1 sw Y   yxY w)zmWrite an encrypted parquet, verify it's encrypted,
    but then try to read it without decryption properties.no decryptionrm   N)rQ   ro   rp   IOErrorrW   rh   rF   ri   rN   r   s     r   0test_encrypted_parquet_read_no_decryption_configry      sL     &gz:	w&6	7 6
w-.3356 6 6s   +AA%c                     t        | |       t        j                  t        d      5  t	        j
                  | t        z         ddd       y# 1 sw Y   yxY w)zwWrite an encrypted parquet, verify it's encrypted,
    but then try to read its metadata without decryption properties.rv   rm   N)rQ   ro   rp   rw   rW   rc   rF   rx   s     r   9test_encrypted_parquet_read_metadata_no_decryption_configr{      sE     &gz:	w&6	7 1
</01 1 1   AAc                     t        | |       t        j                  t        d      5  t	        j
                  | t        z         ddd       y# 1 sw Y   yxY w)zuWrite an encrypted parquet, verify it's encrypted,
    but then try to read its schema without decryption properties.rv   rm   N)rQ   ro   rp   rw   rW   re   rF   rx   s     r   7test_encrypted_parquet_read_schema_no_decryption_configr~      sC     &gz:	w&6	7 /
w-./ / /r|   c           
          | dz  }t        j                  t              }t        j                  t
        d      5  t        ||t        t        t        d|       ddd       y# 1 sw Y   yxY w)MWrite an encrypted parquet, but give only footer key,
    without column key.z)encrypted_table_no_col_key.in_mem.parquetr   z4Either column_keys or uniform_encryption must be setrm   r   N)	r   r   r   ro   rp   OSErrorr9   r    rH   rN   r   r5   r8   s       r   'test_encrypted_parquet_write_no_col_keyr      sn     @@D 22"$ 
w%
& A 	T:'.?	A	A A As   A""A+c           
          | dz  }t        j                  t        t        ddgid      }t	        j
                  t        d      5  t        ||t        t        t        d|       d	d	d	       y	# 1 sw Y   y	xY w)
r   z=encrypted_table_col_key_and_uniform_encryption.in_mem.parquetr   r   T)r   r   rG   z2Cannot set both column_keys and uniform_encryptionrm   r   N)	r   r   r   r    ro   rp   r   r9   rH   r   s       r   ;test_encrypted_parquet_write_col_key_and_uniform_encryptionr     s     TTD 22"3*
  ! 
wR
T A 	T:'.?	AA A As   A,,A5c                     | dz  }|}t        j                         }d }t        j                  |      }t        j                  t
        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)<Write an encrypted parquet, but raise KeyError in KmsClient.(encrypted_table_kms_error.in_mem.parquetc                     t        |       S r&   r'   r(   s    r   r*   z;test_encrypted_parquet_write_kms_error.<locals>.kms_factory-  s     !!=>>r   r   rm   N)r   r,   r-   ro   rp   KeyErrorr4   rN   r   r!   r5   r8   r.   r*   r/   s           r   &test_encrypted_parquet_write_kms_errorr   $  sy     ??D/ 224?
 %%k2N	x|	4 Gj2C 5~	GG G G   A((A1c                     | dz  }|}t        j                         } G d dt         j                        fd}t        j                  |      }t	        j
                  t        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)r   r   c                   "    e Zd ZdZd Zd Zd Zy)Jtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClientzVA KmsClient implementation that throws exception in
        wrap/unwrap calls
        c                 P    t         j                  j                  |        || _        y)z%Create an InMemoryKmsClient instance.N)r   	KmsClient__init__configselfr   s     r   r   zStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.__init__G  s    LL!!$' DKr   c                     t        d      )NCannot Wrap Keyrq   r   	key_bytesmaster_key_identifiers      r   wrap_keyzStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.wrap_keyL  s    .//r   c                     t        d      )NzCannot Unwrap Keyr   r   wrapped_keyr   s      r   
unwrap_keyzUtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.unwrap_keyO  s    011r   N__name__
__module____qualname____doc__r   r   r    r   r   ThrowingKmsClientr   B  s    		!
	0	2r   r   c                      |       S r&   r   )r)   r   s    r   r*   zDtest_encrypted_parquet_write_kms_specific_error.<locals>.kms_factoryR  s     !=>>r   r   rm   N)r   r,   r   r-   ro   rp   rq   r4   )	rN   r   r!   r5   r8   r.   r*   r/   r   s	           @r   /test_encrypted_parquet_write_kms_specific_errorr   9  s     ??D/ 2242BLL 2 ? %%k2N	z):	; Gj2C 5~	GG G Gs   +BBc                     | dz  }|}t        j                         }d }t        j                  |      }t        j                  t
        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)z@Write an encrypted parquet, but raise ValueError in kms_factory.0encrypted_table_kms_factory_error.in_mem.parquetc                     t        d      )NCannot create KmsClientr   r(   s    r   r*   zCtest_encrypted_parquet_write_kms_factory_error.<locals>.kms_factoryf  s    233r   r   rm   N)r   r,   r-   ro   rp   rq   r4   r   s           r   .test_encrypted_parquet_write_kms_factory_errorr   ]  s~     GGD/ 2244 %%k2N	z6
8 G 	 j2C 5~	GG G Gr   c                     | dz  }|}t        j                         } G d d      fd}t        j                  |      }t        j                  t
              5  t        |||||       ddd       y# 1 sw Y   yxY w)z_Write an encrypted parquet, but use wrong KMS client type
    that doesn't implement KmsClient.r   c                   "    e Zd ZdZd Zd Zd Zy)Otest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClientz4This is not an implementation of KmsClient.
        c                 &    |j                   | _        y r&   )r$   master_keys_mapr   s     r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.__init__  s    #)#9#9D r   c                      y r&   r   r   s      r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.wrap_key      r   c                      y r&   r   r   s      r   r   zZtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.unwrap_key  r   r   Nr   r   r   r   WrongTypeKmsClientr   {  s    		:		r   r   c                      |       S r&   r   )r)   r   s    r   r*   zHtest_encrypted_parquet_write_kms_factory_type_error.<locals>.kms_factory  s    !">??r   N)r   r,   r-   ro   rp   	TypeErrorr4   )	rN   r   r!   r5   r8   r.   r*   r/   r   s	           @r   3test_encrypted_parquet_write_kms_factory_type_errorr   q  s     GGD/ 224 @ %%k2N	y	! Gj2C 5~	GG G Gs   A33A<c            
      J   d } t        j                  t        t        ddgidddt	        d      dd	
      } | |       t        j                  t              }t        ddgi|_        d|_        d|_        d|_        t	        d      |_	        d|_
        d	|_         | |       y )Nc                 0   t         | j                  k(  sJ ddg| j                  t           k(  sJ d| j                  k(  sJ | j
                  sJ | j                  rJ t        d      | j                  k(  sJ | j                  rJ d| j                  k(  sJ y )Nr   r   AES_GCM_CTR_V1      $@r>      )r   r   r   r    rB   plaintext_footerdouble_wrappingr   rC   internal_key_materialrD   )r8   s    r   !validate_encryption_configurationzZtest_encrypted_parquet_encryption_configuration.<locals>.validate_encryption_configuration  s    "3">">>>>Sz.::<HHHH#4#I#IIII 1111$4444&*;*J*JJJJ$::::'<<<<<r   r   r   r   TFr   r>   r   )r   r   rB   r   r   rC   r   rD   r   )r   r   r   r    r   r   rB   r   r   rC   r   rD   )r   r8   encryption_config_1s      r   /test_encrypted_parquet_encryption_configurationr     s    = 22"!C:0- .# 	 &&7844"$'3c3Z&B#/?,+/(*/')24)@&05-/2,%&9:r   c                      t        j                  t        d            } t        d      | j                  k(  sJ t        j                         }t        d      |_        t        d      |j                  k(  sJ y )Nr   r>   rE   )r   rJ   r   rC   )rO   decryption_config_1s     r   /test_encrypted_parquet_decryption_configurationr     sl    22 .0T"&7&F&FFFF446)24)@&T"&9&H&HHHHr   c                      d } t        j                  dddddd      } | |       t        j                         }d|_        d|_        d|_        ddd|_         | |       y )	Nc                     d| j                   k(  sJ d| j                  k(  sJ d| j                  k(  sJ ddd| j                  k(  sJ y )N	Instance1URL1MyTokenkey_material_1key_material_2key1key2kms_instance_idkms_instance_urlkey_access_tokenr$   )r.   s    r   validate_kms_connection_configzPtest_encrypted_parquet_kms_configuration.<locals>.validate_kms_connection_config  sb    3CCCCC.?????1BBBBB)3CD%556 	7 6r   r   r   r   r   r   r   r   )r   r,   r   r   r   r$   )r   r.   kms_connection_config_1s      r   (test_encrypted_parquet_kms_configurationr     s    7 22#"$$
	 ##89 446.9+/5,/8,  /+ ##:;r   zNPlaintext footer - reading plaintext column subset reads encrypted columns too)reasonc                 >   | t         z  }t        j                  t        t        ddgidd      }t        j
                  t        t        j                  d      t        t        j                  d      i      }d }t        j                  |      }t        |||||       y	)
zWrite an encrypted parquet, with plaintext footer
    and with single wrapping,
    verify it's encrypted, and then read plaintext columns.r   r   TF)r   r   r   r   r2   r#   c                     t        |       S r&   r'   r(   s    r   r*   zStest_encrypted_parquet_write_read_plain_footer_single_wrapping.<locals>.kms_factory  r+   r   N)rF   r   r   r   r    r,   rH   r3   rI   r-   r4   rN   r   r5   r8   r.   r*   r/   s          r   >test_encrypted_parquet_write_read_plain_footer_single_wrappingr     s     \!D
 22"3*
  22Z..w7'..1
? %%k2ND*.?1>Cr   z'External key material not supported yetc                     | t         z  }t        j                  t        i d      }t        j                  t        t
        j                  d      i      }d }t        j                  |      }t        |||||       y)zWrite an encrypted parquet, with external key
    material.
    Currently it's not implemented, so should throw
    an exceptionF)r   r   r   r2   r#   c                     t        |       S r&   r'   r(   s    r   r*   z:test_encrypted_parquet_write_external.<locals>.kms_factory  r+   r   N)	rF   r   r   r   r,   rH   r3   r-   r4   r   s          r   %test_encrypted_parquet_write_externalr     s|     \!D 22"#%
 22(**;*;G*DE? %%k2ND*.?1>Cr   c           	      x   | t         z  }t        ||t        t        t        t
        |      \  }}t        |       t        j                  t        d            }t        d      D ]T  }|j                  ||      }|J t        j                  ||      }	|	j                  d      }
|j                  |
      rTJ  y)	z`Write an encrypted parquet, verify it's encrypted,
    and then read it multithreaded in a loop.r=   r>   rE   2   Nr^   Tr`   )rF   r9   r   r    rH   rI   r   r   rJ   r   rangerb   rW   rh   ri   rL   )rN   r   r!   r5   r.   r/   rO   irb   rk   rP   s              r   test_encrypted_parquet_loopr      s     \!D
 -Aj/<W-!)> $22 -/ 2Y 	/%3%N%N!#4&6")555(BD{{t{4  ...	/r   c           	      R   | t         z  }t        ||t        t        t        t
        |      \  }}t        |       t        j                  t        d            }|j                  ||      }~t        j                  ||      }|j                  d      }	|j                  |	      sJ y)z^
    Test that decryption properties can be used if the crypto factory is no longer alive
    r=   r>   rE   r^   Tr`   N)rF   r9   r   r    rH   rI   r   r   rJ   r   rb   rW   rh   ri   rL   )
rN   r   r!   r5   r.   r/   rO   rb   rk   rP   s
             r   %test_read_with_deleted_crypto_factoryr   =  s     \!D,@j/<W-!)> $ 22 -/!/!J!J0"2^^$>@F;;4;0L\***r   c           	      j   | t         z  }t        ||t        t        t        t
        |      \  }}t        j                  t        d            }|j                  ||      }t        j                  ||      }|j                  |      sJ t        j                  | |      }|j                  |      sJ y)z>Write an encrypted parquet then read it back using read_table.r=   r>   rE   r^   N)rF   r9   r   r    rH   rI   r   rJ   r   rb   rW   
read_tablerL   )	rN   r   r!   r5   r.   r/   rO   rb   rP   s	            r   !test_encrypted_parquet_read_tabler   U  s    \!D -Aj/<W-!)> 22 -/!/!J!J0"2 ===WXL \*** =='ACL\***r   )3ro   datetimer   pyarrowr   pyarrow.parquetparquetrW   pyarrow.parquet.encryption
encryptionr    pyarrow.tests.parquet.encryptionr   r   ImportErrorrF   rH   r   rI   r    markparquet_encryption
pytestmarkfixturer   r!   r0   r9   rQ   rS   r4   rK   rt   ry   r{   r~   r   r   r   r   r   r   r   r   r   xfailr   r   r   r   r   r   r   r   <module>r      s  "   2 ++
2 0 

 KK""
KK
 h   h#  #1 1,+>+6")" "F61/A"A(G*!GHG(GB ;FI<:  2 3C3CN CDC EC4/:+0+{  	B	Bs   D 	D"!D"