
    /_i                       d dl m Z mZmZ d dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZmZmZ d dlmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z# ejH                  jK                  ejH                  jM                  e'            Z( G d d	e      Z)ejT                  d
        Z+ejT                  d        Z,ejT                  d        Z-ejT                  d        Z.ejT                  d        Z/ejT                  d        Z0ejT                  d        Z1ejT                  d        Z2ejT                  d        Z3dZ4ejT                  d        Z5ejT                  d        Z6ejT                  d        Z7ejT                  d        Z8ejT                  d        Z9 ejT                   ejt                  dd       ejt                  dd       ejt                  dd       ejt                  d d!ejv                  jx                  "       ejt                  d#d$ejv                  jz                  "       ejt                  d%d&ejv                  j|                  "       ejt                  d'd(ejv                  jl                  "       ejt                  d)d*       ejt                  d+d,       ejt                  d-d.       ejt                  d/d0       ejt                  d1d2       ejt                  d3d4ejv                  jx                  "      g5      d6        Z?ejT                  d7        Z@ejT                  d8        ZAejT                  d9        ZBejT                  d:        ZCd; ZDd< ZEd= ZFd> ZGd? ZHejv                  jx                  d@        ZIdA ZJdB ZKdC ZLdD ZMdE ZNdF ZOdG ZPdH ZQdI ZRdJ ZSdK ZTdL ZUdM ZVdN ZWdO ZXdP ZYdQ ZZdR Z[dS Z\dT Z]dU Z^dV Z_dW Z`ejv                  j                  ejv                  j                  dXdde`fddYe`fdZdej                  fdZd[ej                  fg      d\               Zcd] Zdd^ Zeejv                  j                  ejv                  j                  d_dde`fddYe`fdZdej                  fdZd[ej                  fg      d`               Zgejv                  j                  ejv                  j                  dadde`e`fddYe`e`fdZdej                  ej                  fdZd[ej                  ej                  fg      db               Zhdc Zidd Zjde Zkdf Zldg Zmejv                  jz                  dh        Znejv                  jx                  di        Zoejv                  jx                  dj        Zpejv                  jx                  dk        Zqejv                  j|                  dl        Zrejv                  jl                  dm        Zsejv                  j                  dndoedpfdqedrfdsedrfdtedrfduedvfdwedxfdyedxfdzed{fd|ed{fd}ed~fdxedxfd{ed{fdedfg      d        Ztd Zuejv                  j                  ddpdxdrd{e'g      d        Zvejv                  jx                  d        Zwejv                  jz                  d        Zxd Zyd Zzd Z{d Z|d Z}d Z~d Zd Zd Zd Zd Zejv                  jx                  d        Zejv                  jx                  d        Zejv                  jx                  d        Zejv                  jx                  d        Zd Zejv                  jx                  d        Zejv                  jx                  d        Zejv                  j                  ejv                  j                  ejv                  jx                  d                      Zejv                  jx                  ejv                  j                  d      ejv                  j                   e       d      d                      Zd Zd Zy)    )datetimetimezone	timedeltaN)urlopen)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_userrunning_on_musllinux)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_files)find_free_portc                   z    e Zd ZddZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zy)DummyHandlerc                     || _         y N)_value)selfvalues     T/var/www/html/land_sniper/venv/lib/python3.12/site-packages/pyarrow/tests/test_fs.py__init__zDummyHandler.__init__/   s	        c                 `    t        |t              r| j                  |j                  k(  S t        S r   
isinstancer   r   NotImplementedr   others     r   __eq__zDummyHandler.__eq__2   &    e./;;%,,..r!   c                 `    t        |t              r| j                  |j                  k7  S t        S r   r#   r&   s     r   __ne__zDummyHandler.__ne__7   r)   r!   c                      y)Ndummy r   s    r   get_type_namezDummyHandler.get_type_name<   s    r!   c                     |S r   r.   r   paths     r   normalize_pathzDummyHandler.normalize_path?   s    r!   c                 r   g }|D ]  }d|v r*|j                  t        |t        j                               1d|v r*|j                  t        |t        j                               _d|v r*|j                  t        |t        j
                               d|v r|j                  t                      t         |S )Nfiledirnotfoundbadtype)appendr   r   File	DirectoryNotFoundobjectIOError)r   pathsinfor3   s       r   get_file_infozDummyHandler.get_file_infoB   s     	D~HT8==9:$HT8+=+=>?t#HT8+<+<=>d"FH%	 r!   c                 .   |j                   dk7  r#|j                  rg S t        |j                         t        dt        j
                  d      t        dt        j                        g}|j                  r |t        dt        j
                  d      gz  }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr   r   r;   r<   	recursive)r   selectorinfoss      r   get_file_info_selectorz#DummyHandler.get_file_info_selectorR   s    	)''	'(9(9::_hmm#>&(:(:;
 0(--cJ E r!   c                 >    |dk(  r|du sJ y |dk(  r|du sJ y t         )NrM   Tnon-recursiveF)r?   )r   r3   rM   s      r   
create_dirzDummyHandler.create_dirb   s3    ;$$$_$%%%Mr!   c                     |dk(  sJ y )N
delete_dirr.   r2   s     r   rU   zDummyHandler.delete_dirj   s    |###r!   c                 @    |j                  d      st        |dk(  sJ y )N/delete_dir_contents)strip
ValueError)r   r3   missing_dir_oks      r   rX   z DummyHandler.delete_dir_contentsm   s"    zz#,,,,r!   c                      y r   r.   r/   s    r   delete_root_dir_contentsz%DummyHandler.delete_root_dir_contentsr   s    r!   c                     |dk(  sJ y )Ndelete_filer.   r2   s     r   r_   zDummyHandler.delete_fileu   s    }$$$r!   c                      |dk(  sJ |dk(  sJ y )N	move_frommove_tor.   r   srcdests      r   movezDummyHandler.movex   s    k!!!y   r!   c                      |dk(  sJ |dk(  sJ y )Ncopy_file_fromcopy_file_tor.   rc   s      r   	copy_filezDummyHandler.copy_file|   s    &&&&~%%%r!   c                 r    d|v rt        |      | dj                  d      }t        j                  |      S )Nr8   z:input_streamutf8rL   encodepaBufferReaderr   r3   datas      r   open_input_streamzDummyHandler.open_input_stream   s;    #D))}%,,V4t$$r!   c                 r    d|v rt        |      | dj                  d      }t        j                  |      S )Nr8   z:input_filerl   rm   rq   s      r   open_input_filezDummyHandler.open_input_file   s;    #D)){#**62t$$r!   c                 H    d|v rt        |      t        j                         S Nr8   rL   ro   BufferOutputStreamr   r3   metadatas      r   open_output_streamzDummyHandler.open_output_stream   $    #D))$$&&r!   c                 H    d|v rt        |      t        j                         S rw   rx   rz   s      r   open_append_streamzDummyHandler.open_append_stream   r}   r!   N)*   )__name__
__module____qualname__r    r(   r+   r0   r4   rB   rP   rS   rU   rX   r]   r_   rf   rj   rs   ru   r|   r   r.   r!   r   r   r   .   s\    

  $-
%!&%%'
'r!   r   c                 8    t        t               fddd      S )Nc                 *    | z  j                         S r   as_posixptempdirs    r   <lambda>zlocalfs.<locals>.<lambda>       'A+//1 r!   Tfspathfnallow_move_dirallow_append_to_filedictr   requestr   s    `r   localfsr      s    1!	 r!   c                 \    t        t        t        t                           fddd      S )Nc                 *    | z  j                         S r   r   r   s    r   r   zpy_localfs.<locals>.<lambda>   r   r!   Tr   )r   r   r	   r   r   s    `r   
py_localfsr      s)    _%6781!	 r!   c                 2    t        t               d dd      S )Nc                     | S r   r.   r   s    r   r   zmockfs.<locals>.<lambda>        r!   Tr   )r   r   r   s    r   mockfsr      s    !	 r!   c                 V    t        t        t        t                           d dd      S )Nc                     | S r   r.   r   s    r   r   zpy_mockfs.<locals>.<lambda>   r   r!   Tr   )r   r   r	   r   r   s    r   	py_mockfsr      s(    _%678!	 r!   c                 <    t        t        d      fddd      S )NTuse_mmapc                 *    | z  j                         S r   r   r   s    r   r   z#localfs_with_mmap.<locals>.<lambda>   r   r!   r   r   r   s    `r   localfs_with_mmapr      s!    D)1!	 r!   c                 N    t        t        t        |      |d         d dd      S )Nr   c                     | S r   r.   r   s    r   r   z!subtree_localfs.<locals>.<lambda>   r   r!   Tr   )r   r   str)r   r   r   s      r   subtree_localfsr      s*    S\74=9!	 r!   c              #     K   | j                   j                  j                  d       ddlm} |d   \  }}d} || d| ddt        d	
      d      }	 |j                  |       t        ||j                  dd       |j                  |       y # t        $ r%}t        j                  d| d|        Y d }~Vd }~ww xY ww)Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousretry_time_limit
project_idzCould not create directory in : Fr   )configpyarrowrequires
pyarrow.fsr   r   rS   OSErrorpytestskipr   __add__rU   )r   
gcs_serverr   hostportbucketr   es           r   gcsfsr      s     NN##E*(L)JD$"F	!F!D6*"2.$
B@
f ~~"	  MM&  @4RD1#>??@s0   ACB &-C	CB<7C<CCc              #     K   | j                   j                  j                  d       ddlm} |d   \  }}}}d} |||| d| ddd	      }|j                  |       t        ||j                  d
d
       |j                  |       y w)Ns3r   S3FileSystemr   r   r   r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFr   )	r   r   r   r   r   rS   r   r   rU   )	r   	s3_serverr   r   r   r   r   r   r   s	            r   s3fsr      s     NN##D)')2<)@&D$
J"F	!F!D6*""
B MM&
~~"	  MM&s   B	Bc                 R    d}t        t        ||d         |j                  dd      S )Nzpyarrow-filesystem/prefix/r   Fr   )r   r   r   )r   r   prefixs      r   subtree_s3fsr     s/    )FVT$Z0~~"	 r!   a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]

        },
        {
            "Effect": "Deny",
            "Action": [
                "s3:DeleteObject"
            ],
            "Resource": [
                "arn:aws:s3:::no-delete-bucket*"
            ]
        }
    ]
}c              #   "  K   | j                   j                  j                  d       ddlm} |d   \  }}}}| d| }d}d}	 |||||||      }
|
j                  |	       t        |
|	j                  d	d	
       |
j                  |	       y w)Nazurer   AzureFileSystemr   r   r   r   account_nameaccount_keyblob_storage_authoritydfs_storage_authorityblob_storage_schemedfs_storage_schemeTr   )	r   r   r   r   r   rS   r   r   rU   )r   azure_serverr   r   r   r   r   azurite_authorityazurite_scheme	containerr   s              r   azurefsr   7  s     NN##G,*,8,F)D$k&$(N%I	l%00A/@-;,:
<B MM)
  !	  MM)s   BBc                     | j                   j                  j                  d       t        j                         st        j                  d       ddlm} |\  }}} ||||      }t        |d dd      S )	NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                     | S r   r.   r   s    r   r   zhdfs.<locals>.<lambda>a  r   r!   Tr   )
r   r   r   ro   have_libhdfsr   r   r   r   r   )r   hdfs_connectionr   r   r   r   r   s          r   r   r   T  sg    NN##F+??+,+&D$	$T	5B!	 r!   c                     t        j                  d      }|j                  d      }t        t	        t        |            fddd      S )Nfsspecr6   c                 *    | z  j                         S r   r   r   s    r   r   z#py_fsspec_localfs.<locals>.<lambda>m  r   r!   Tr   )r   importorskip
filesystemr   r   r   r   r   r   r   s    `  r   py_fsspec_localfsr   g  sF      *F			6	"Bb)*1!	 r!   c                     t        j                  dd      }|j                  dk(  rt        j                  d       |j	                  d      }t        t        t        |            d dd	      S )
Nr   z0.7.5)
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemmemoryc                     | S r   r.   r   s    r   r   z$py_fsspec_memoryfs.<locals>.<lambda>|  r   r!   Tr   )r   r   __version__r   r   r   r   r   r   s       r   py_fsspec_memoryfsr   s  sb      g>FW$BC			8	$Bb)*!	 r!   c           
   #   8  K   t        j                  d      }|d   \  }}}}d}|j                  ||t        d| d|             }t	        t        |            }|j                  |       t        ||j                  dd	
       |j                  |       y w)Nr   r   r   zhttp://r   )endpoint_url)keysecretclient_kwargsFTr   )	r   r   r   r   r   r   rS   r   rU   )	r   r   r   r   r   r   r   r   r   s	            r   py_fsspec_s3fsr    s     v&D)2<)@&D$
J"F			'$q(?@ 
 
B
 
mB'	(BMM&
~~!	  MM&s   BBr   zLocalFileSystem())idr   zLocalFileSystem(use_mmap=True)r   z$SubTreeFileSystem(LocalFileSystem())r   r   )r  marksr   r   r   r   r   r   r   z_MockFileSystem()r   z-PyFileSystem(ProxyHandler(LocalFileSystem()))r   z-PyFileSystem(ProxyHandler(_MockFileSystem()))r   z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))r   z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))r  z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 8    | j                  | j                        S r   )getfixturevalueparamr   s    r   filesystem_configr
    s    x ""7==11r!   c                     | d   S Nr   r.   r
  s    r   r   r     s    T""r!   c                     | d   S )Nr   r.   r  s    r   r   r     s    X&&r!   c                     | d   S )Nr   r.   r  s    r   r   r     s    -..r!   c                     | d   S )Nr   r.   r  s    r   r   r     s    344r!   c                    t        | j                  t              sJ t        | j                  t              sJ | j                  dk\  sJ | j                  t        j                  | j                  j                         dz        k(  sJ | j                  j                  }|J |j                  d       t        d      k(  sJ y )Nr   g    eA)r$   mtimer   mtime_nsintr   approx	timestamptzinfo	utcoffsetr   )	file_infor  s     r   check_mtimer    s    ioox000i((#..."""!!#c)"+ + + + __##FD!Yq\111r!   c                 <    | j                   J | j                  J y r   )r  r  r  s    r   check_mtime_absentr    s$    ??"""%%%r!   c                 J    | j                   t        |        y t        |        y r   )r  r  r  r  s    r   check_mtime_or_absentr    s    9%Ir!   c                 P    | j                   dk(  rt        j                  d       y y )Npy::fsspec+('s3', 's3a')zNot working with fsspec's s3fsreason)	type_namer   xfailr   s    r   skip_fsspec_s3fsr'    s!    	||11<= 2r!   c                 P    | j                   dk(  rt        j                  |       y y )Nabfsr"  )r$  r   r   )r   r#  s     r   
skip_azurer*  	  s    	||v6" r!   c                    ddl m} t        | t        dd       | d   \  }}}} |dd| d| d      }|j	                  d	       t        j                  t        j                  d
      5  |j	                  d       d d d        t        j                  t        j                  d      5  |j                  d       d d d        t        j                  t        d      5  |j                  dd       d d d        t        j                  t        j                  d      5  |j                  d      5 }|j                  d       d d d        d d d        |j                  d      5 }|j                  d       d d d        t        j                  t        d      5  |j                  d       d d d        y # 1 sw Y   9xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   }xY w# 1 sw Y   y xY w)Nr   r   test_fs_limited_user
limited123r   r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucketzRequest ID:zexisting-bucket/test-filezexisting-bucket/test-file-copyznon-existing-bucket/test-file   testzno-delete-bucket/test-file)r   r   r
   _minio_limited_policyrS   r   raisesro   ArrowIOErrorrU   r   rj   r|   writer_   )r   r   r   r   _r   fs          r   +test_s3fs_limited_permissions_create_bucketr7    s   'y*?5|E .D$1	)!F!D6*	
B MM()	r.M	N $
l#$ 
r.C	D )
'() 
wm	4 T
02RST 
rm	< ""#BC 	qGGG	
 
		;	< 	 
wm	4 5
345 5#$ $) )T T	 	 
 5 5s`   *F3)G GG%0GG%#G1G=3F= G
GG"	G%%G.1G:=Hc                  \   t        j                  dt        j                        } t	        d      }|j
                  dk(  sJ |j                  dk(  sJ |j                  t        j                  k(  sJ |j                  J t        |       t	        dt        j                  dd      }|j
                  dk(  sJ |j                  dk(  sJ |j                  t        j                  k(  sJ |j                  dk(  sJ |j                  d	k(  sJ t        |       t	        d
t        j                  |       }|j
                  d
k(  sJ |j                  d
k(  sJ |j                  t        j                  k(  sJ |j                  J |j                   | k(  sJ |j                  dk(  sJ t        |       y )NR]foo/barbarzfoo/baz.txtrF   g  `A)typerG   r  zbaz.txtl    t<|+. foo)r<  r  l    4 |+. )r   fromtimestampr   utcr   r3   	base_namer<  r   UnknownrG   r  r;   r  r  r<   r  )dtrA   s     r   test_file_info_constructorrC  2  s   			
HLL	9BID99	!!!>>U"""99(((((99tMC&(D99%%%>>Y&&&99%%%99==////E 2 2"=D99>>U"""99*****99::==////r!   c                  t    t        j                  t              5  t                d d d        y # 1 sw Y   y xY wr   )r   r2  	TypeErrorr   r.   r!   r   'test_cannot_instantiate_base_filesystemrF  O  s(    	y	!   s   .7c                     t               } t               }t               }| j                  |       sJ | j                  |      sJ t        j                  t
              5  | j                  d       d d d        | | cxk(  r|k(  sJ  J | dk7  sJ ||k(  sJ |t               k7  sJ t        d|       t        d|       k(  sJ t        d|       t        d|      k7  sJ t        d|       t        d|       k7  sJ y # 1 sw Y   xY w)Nstring   /basez/other)r   r   equalsr   r2  rE  r   )fs0fs1fs2s      r   test_filesystem_equalsrO  T  s    

C

C

C::c??::c??	y	! 

8#!8O8#::/####Wc*.?.MMMMWc*.?.MMMMWc*.?#.NNNN s   C77D c                     t        j                  t        d      5  | j                  d        d d d        | J y # 1 sw Y   xY w)Nzgot NoneTyper.  )r   r2  rE  rK  r&  s    r   test_filesystem_equals_nonerQ  h  s<    	y	7 
		$ >> s	   ;Ac                  P   t               } t        d|       }|j                  dk(  sJ |j                  | k(  sJ t	        |      j                  d      sJ t        dt                     }|j                  dk(  sJ |j                  | k(  sJ t	        |      j                  d      sJ y )NrJ  z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathbase_fsrepr
startswith)r   subfss     r   test_subtree_filesystemrX  o  s    Ggw/E??h&&&==G###;!! #I J J J .0ABE??....==G###;!! #J K K Kr!   c                     | j                   j                  d      d   dk(  rt        j                  d       |j	                  |       }|j                  |      }t        |t              sJ |j                  |       sJ y )N::mock"MockFileSystem is not serializabler"  )	r$  splitr   r%  dumpsloadsr$   r   rK  )r   pickle_module
serializedrestoreds       r   test_filesystem_picklingrd    sm    	||$#v-@A$$R(J"":.Hh
+++??2r!   c                    | j                   j                  d      d   dk(  rt        j                  d       t	        |         |d      } |d      } |d      }| j                  |       | j                  |      5  	 d d d        | j                  |      5 }|j                  d	       d d d        |j                  |j                  |             }|j                  |||g      \  }}	}
|j                  t        j                  k(  sJ |	j                  t        j                  k(  sJ |
j                  t        j                  k(  sJ y # 1 sw Y   xY w# 1 sw Y   xY w)
NrZ  r[  r\  r]  r"  	a/aa/aaa/a/bbc.txtr0  )r$  r^  r   r%  r'  rS   r|   r4  r`  r_  rB   r<  r   r<   r;   )r   r   ra  aaabbcfprc  aaa_infobb_infoc_infos              r   ,test_filesystem_is_functional_after_picklingrp    s0   	||$#v-@AR

C	BwAMM#			r	" 			q	! R
 ""=#6#6r#:;H ( 6 6R| DHgv==H.....<<8==(((;;(--'''  s   =EEEEc                  p    t               } | j                  dk(  sJ t               } | j                  dk(  sJ y )Nlocalr\  )r   r$  r   r&  s    r   test_type_namers    s6    		B<<7"""		B<<6!!!r!   c                 0    | j                  d      dk(  sJ y )Nr=  )r4   r&  s    r   test_normalize_pathru    s     U#u,,,r!   c           	           G d d      }dd |       t               i g d t        j                         g}|D ]5  }t        j                  t
              5  | j                  |       d d d        7 y # 1 sw Y   BxY w)Nc                       e Zd Zy)-test_non_path_like_input_raises.<locals>.PathN)r   r   r   r.   r!   r   Pathrx    s    r!   ry     g?c                       y)Nrz  r.   r.   r!   r   r   z1test_non_path_like_input_raises.<locals>.<lambda>  s    r!   )tuplepathlibry  r   r2  rE  rS   )r   ry  invalid_pathsr3   s       r   test_non_path_like_input_raisesr    sr      TVUWb"i\\^%M  ]]9% 	 MM$	  	  	  	 s   A00A9	c                     |d      } |d      } |d      } |d      }| j                  |       | j                  |      5  	 d d d        | j                  |      5 }|j                  d       d d d        | j                  ||||g      \  }}}	}
|j                  |k(  sJ dt        |      v sJ |j                  dk(  sJ | j                  dk(  r|j                  t        j                  k(  s0J |j                  t        j                  k(  sJ d	t        |      v sJ |j                  J t        |       |j                  t        |      k(  sJ |j                  d
k(  sJ |j                  dk(  sJ |j                  t        j                   k(  sJ dt        |      v sJ |j                  dk(  sJ | j                  dvrt#        |       |	j                  t        |      k(  sJ |	j                  dk(  sJ |	j                  dk(  sJ |	j                  t        j                   k(  sJ dt        |	      v sJ |	j                  dk(  sJ | j                  dvrt#        |	       |
j                  t        |      k(  sJ |
j                  dk(  sJ |
j                  dk(  sJ |
j                  t        j                  k(  sJ |
j                  J |
j$                  J dt        |
      v sJ t'        |
       | j                  |      }|j                  |j                  k(  sJ |j                  |j                  k(  sJ y # 1 sw Y   xY w# 1 sw Y   xY w)Nrf  rg  rh  zzzr0  ri   r!  zFileType.Directoryrj  zFileType.Filer   )zpy::fsspec+memoryr!  txtrI  zFileType.NotFound)rS   r|   r4  rB   r3   rU  	extensionr$  r<  r   r=   r<   rG   r  r   r@  r;   r  r  r  )r   r   ri  rj  rk  r  rl  rm  rn  ro  zzz_info	aaa_info2s               r   test_get_file_infor    s6   

C	BwA
-CMM#			r	" 			q	! R
 +-*:*:CQ;L*M'Hgvx==CDN"""###	||11}} 1 1111}} 2 2222#tH~555==   (#<<3r7"""$$$"""<<8==(((d7m+++<<1	||LLG;;#a&   w&&&u$$$;;(--'''d6l***;;!	||LLF==CH$$$&&&###==H-----==   >>!!!$x.000x    %I==INN***==INN***c  s   L9M9MMc                 ~    |d      } |d      } |d      } |d      } |d      } |d      }	 | j                  |       | j                  |      5  	 d d d        | j                  |      5  	 d d d        | j                  |       | j                  |      5  	 d d d        | j                  |       t        |dd	      }|j                  |k(  sJ | j	                  |      }	| j
                  d
k(  rt        |	      dk(   nt        |	      dk(  sJ |	D ]	  }
|
j                  j                  |      s6|
j                  j                  |      s|
j                  j                  |      r|
j                  t        j                  k(  sJ |
j                  j                  d      j                  |      s*|
j                  j                  d      j                  |      r|
j                  t        j                  k(  sJ t        d|
j                         t        |
        t        |d      }| j	                  |      }	| j
                  d
k(  rt        |	      dk(  sJ t        |	      dk(  sJ | j!                  |       y # 1 sw Y   .xY w# 1 sw Y    xY w# 1 sw Y   xY w# | j!                  |       w xY w)Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)rK   rM   r!  rI     rW   zunexpected path rM      )rS   r|   r   rJ   rB   r$  lenr3   endswithr<  r   r;   rstripr<   rZ   r  rU   )r   r   rJ   file_afile_bdir_afile_cdir_brN   rO   rA   s              r    test_get_file_info_with_selectorr    sr   o&H./F./F,-E9:F,-E- 
h""6* 		""6* 		
e""6* 		
e  %*.0  H,,,  *<<55J!Ou:?"? 		(D		""6*dii.@.@.HII&&v.yyHMM111))""3'007))""3'007yyH$6$6666 #3DII;!?@@!$'		(  E:  *<<55u:?"?u:?"? 	hW	 		 		 	L 	hsT   "J) JJ) /J1*J) JGJ) JJ) JJ) J&!J) )J<c                 X   t        |         |d      }t        j                  t        j                        5  | j                  |       d d d        | j                  |       | j                  |        |d      }| j                  |d       | j                  |       y # 1 sw Y   XxY w)Nztest-directory/zdeeply/nested/test-directory/Tr  )r'  r   r2  ro   r3  rU   rS   )r   r   ds      r   test_create_dirr  *  s     R !A	r	' 
a MM!MM!./AMM!tM$MM! s   B  B)c                    t        |         |d      } |d      }| j                  |       | j                  |       t        j                  t
        j                        5  | j                  |       d d d        t        j                  t
        j                        5  | j                  |       d d d        y # 1 sw Y   GxY w# 1 sw Y   y xY w)N
directory/directory/nested/)r'  rS   rU   r   r2  ro   r3  r   r   r  nds       r   test_delete_dirr  ;  s    R|A	#	$BMM"MM!	r	' 
b	r	' 
a   s   !B9C9CCc                 d   t        |         |d      } |d      }| j                  |       | j                  |       | j                  |       | j                  |      }|j                  t
        j                  k(  sJ  |d      } |d      } |d      }| j                  |       | j                  |       | j                  |      5 }|j                  d       d d d        | j                  |       | j                  |      }|j                  t
        j                  k(  sJ y # 1 sw Y   KxY w)Nr  r  
directory2zdirectory2/nestedzdirectory2/nested/target-file   data)	r'  rS   rU   rB   r<  r   r=   r|   r4  )r   r   r  r  dir_infor6  ss          r   $test_delete_dir_with_explicit_subdirr  I  s   
 R|A	#	$B MM!MM"MM!"H==H----- 	|A	#	$B./AMM!MM"			q	! Q	 MM!"H==H----- s   
D&&D/c                 |   t        |         |d      } |d      }| j                  |       | j                  |       t        j                  t
        j                        5  | j                  |       d d d        | j                  |d       t        j                  t
        j                        5  | j                  |       d d d        | j                  |       t        j                  t
        j                        5  | j                  |       d d d        y # 1 sw Y   xY w# 1 sw Y   dxY w# 1 sw Y   y xY w)Nr  r  T)r[   )r'  rS   rX   r   r2  ro   r3  rU   r  s       r   test_delete_dir_contentsr  i  s    R|A	#	$BMM"1	r	' 
b2d3	r	' #
r"#MM!	r	' 
a  # # s$   !D1D&?D2D#&D/2D;c                    | d   }| d   } |d      } |d      }|j                  |       t        j                  t        j                        5  |j                  d       d d d        t        j                  t        j                        5  |j                  d       d d d        t        j                  t        j                        5  |j                  d       d d d        |j                  dd	       |j                  dd	       |j                  dd	       t        j                  t        j                        5  |j                  |       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)
Nr   r   r  r  r  rW   //Taccept_root_dir)rS   r   r2  ro   ArrowInvalidrX   r3  rU   )r   r   r   r  r  s        r   _check_root_dir_contentsr  {  s@   	BHF|A	#	$BMM"	r	' #
r"#	r	' $
s#$	r	' %
t$% 2t43546	r	' 
a # #$ $% % s0   EE&	E2?E>E#&E/2E;>Fc                 0    t        |        t        |       y r   )r  )r   r   s     r   test_delete_root_dir_contentsr    s    V$Y'r!   c                      |d      } |d      }| j                  |      5  	 d d d        | j                  ||       | j                  |       | j                  |       y # 1 sw Y   >xY w)Nztest-copy-source-fileztest-copy-target-file)r|   rj   r_   r   r   r  ts       r   test_copy_filer    sf    &'A&'A			q	!  LLANN1NN1 s   A!!A*c                    t        | d        |d      } |d      }| j                  |       |ra| j                  ||       t        j                  t
        j                        5  | j                  |       d d d        | j                  |       y t        j                  t
        j                        5  | j                  ||       d d d        y # 1 sw Y   ZxY w# 1 sw Y   y xY w)N.Not implemented yet in for Azure. See GH-40025zsource-dir/ztarget-dir/)r*  rS   rf   r   r2  ro   r3  rU   )r   r   r   r  r  s        r   test_move_directoryr    s    rCD 	}A}AMM!
1]]2??+ 	MM!	
a]]2??+ 	GGAqM	 			 		 	s   %C4CCC%c                 x   t        |        t        | d        |d      } |d      }| j                  |      5  	 d d d        | j                  ||       t	        j
                  t        j                        5  | j                  |       d d d        | j                  |       y # 1 sw Y   jxY w# 1 sw Y   'xY w)Nr  ztest-move-source-fileztest-move-target-file)	r'  r*  r|   rf   r   r2  ro   r3  r_   r  s       r   test_move_filer    s     R rCD&'A&'A			q	!  GGAqM	r	' 
qNN1  s   B$8B0$B-0B9c                     |d      }| j                  |      5  	 d d d        | j                  |       t        j                  t        j
                        5  | j                  |       d d d         |d      }| j                  |        |d      }| j                  |      5 }|j                  d       d d d        | j                  |       y # 1 sw Y   xY w# 1 sw Y   sxY w# 1 sw Y   3xY w)Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-filer  )	r|   r_   r   r2  ro   r3  rS   r4  rU   )r   r   r   r  r6  r  s         r   test_delete_filer    s    ()A			q	!  NN1	r	' 
q 	#$AMM!/0A			q	! Q	 MM!   s#   CC$C(CC%(C1c                     | S r   r.   )vs    r   identityr    s    Hr!   )compressionbuffer_size
compressor@   gzip   c                     |d      }ddz  }| j                  |      5 }|j                   ||             d d d        | j                  |||      5 }|j                         }d d d        |k(  sJ y # 1 sw Y   =xY w# 1 sw Y   xY w)Nzopen-input-streams   some data for reading
   )r|   r4  rs   read)	r   r   r  r  r  r   rr   r  results	            r   test_open_input_streamr    s     	"#A%+D			q	! "Q	
4 !" 
		ak	: a T>>" " s   A3A?3A<?Bc                     |d      }ddz  }| j                  |      5 }|j                  |       d d d        t        d      dz  }| j                  |      5 }|j	                         }d d d        |k(  sJ | j                  |      5 }|j                  |       |j	                         }d d d        |||d  k(  sJ y # 1 sw Y   xY w# 1 sw Y   bxY w# 1 sw Y   ,xY w)Nzopen-input-file	   some data   r  )r|   r4  r  ru   r  seek)r   r   r   rr   r  	read_fromr6  r  s           r   test_open_input_filer    s     !A$D			q	! Q	 L!C'I			A	 !T>>			A	 !	y T)*%%%%   s#   B>C
	"C>C
CCc                      |d      }t        j                  t              5  | j                  |       d d d        y # 1 sw Y   y xY w)Nzopen-input-stream-not-found)r   r2  rL   rs   )r   r   r   s      r    test_open_input_stream_not_foundr  
  s=    ,-A	(	)  
Q     s	   =A)r  r  decompressorc                      |d      }ddz  }| j                  |||      5 }|j                  |       d d d        | j                  |||      5 }|j                  t	        |            |k(  sJ 	 d d d        y # 1 sw Y   FxY w# 1 sw Y   y xY w)Nzopen-output-streams   some data for writingr  )r|   r4  rs   r  r  )r   r   r  r  r  r   rr   r6  s           r   test_open_output_streamr    s     	#$A#d*D			q+{	; q	 
		ak	: )avvc$i D((() ) ) )s   A8 B8BB)r  r  r  r  c                 .    |d      } |d      }| j                  |      5 }	|	j                  |       d d d        |rh| j                  |||      5 }
|
j                  d       d d d        | j                  |      5 }
|
j	                         }d d d         |      }|dk(  sJ y t        j                  t        j                        5  | j                  |||       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   rxY w# 1 sw Y   y xY w)Nzopen-append-streams   already existing)r  r  s   
newly addeds   already existing
newly added)	r|   r4  r   rs   r  r   r2  ro   ArrowNotImplementedError)r   r   r  r  r  r  r   r   initialr  r6  r  s               r   test_open_append_streamr  '  s"    	#$A,-G			q	! Q	 ""1+/: # < 	&?@GG$%	& !!!$ 	VVXF	 f%9999]]2667 	;!!!.9 " ;	; 	; 	& 	&	 		; 	;s/   C'C3=C?	D'C03C<?DDc                     |d      }ddi}d}| j                  ||      5 }|j                  |       d d d        | j                  |      5 }|j                         |k(  sJ |j	                         }d d d        | j
                  dv sd| j
                  v r=t        | d       |j                         D ]  \  }}|   |j                         k(  rJ  y i k(  sJ y # 1 sw Y   xY w# 1 sw Y   vxY w)	Nzopen-output-stream-metadataContent-Typezx-pyarrow/testr  )r{   )r   r   r)  r\  zWAzure filesystem currently only returns system metadata not user metadata. See GH-40026)	r|   r4  rs   r  r{   r$  r*  itemsrn   )	r   r   r   r{   rr   r6  got_metadatakr  s	            r    test_open_output_stream_metadatar  I  s   ,-A 01HD			q8		4 	 
		a	  $Avvx4zz|$ 
||,,",,0F %	& NN$ 	1DAq?ahhj000	1 r!!! $ $s   C&C(C%(C1c                      t        d       t        j                  t              5  t        d       d d d        y # 1 sw Y   y xY w)NFr   )xxx)r   r   r2  rE  r.   r!   r   test_localfs_optionsr  `  s3    U#	y	! #E"# # #s	   <Ac                    | d   }t               5  |j                  d       d d d        t               5  |j                  d       d d d        t               5  |j                  dd       d d d        t               5  |j	                  d       d d d        t               5  |j                  d       d d d        t               5  |j                  dd       d d d        t               5  |j                  dd       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   kxY w# 1 sw Y   y xY w)Nr   z/non/existent/filez/non/existent/dirFr  z/non/existentz/xxx)r   rs   r|   rS   rU   r_   rf   rj   )r   r   s     r   test_localfs_errorsr  h  s4    
B		  3
123		  4
234		  <
)U;<		  +
)*+		  ,
*+,		  )
()		  .
_f-. .3 34 4< <+ +, ,) ). .sR   DDD>D+"D7E+EDDD(+D47E EEc                    | d   }t        j                  t              }|j                  }|j	                  |j                         |j                         g      \  }}|j                  |j                         j                  k(  sJ |j                  |j                         j                  k(  sJ t        |       |j                  |j                         j                  k(  sJ t        |       y r  )r}  ry  __file__parentrB   r   rG   statst_sizer  st_mtime_nsr  )r   r   	file_pathdir_pathr  r  s         r   test_localfs_file_infor  |  s    	BX&IH,,i.@.@.B.6.?.?.A.C DY>>Y^^-55555!1!=!====	 ; ;;;;r!   c                     t        j                  dt        j                        }t	        |      }|j                  d      5  	 d d d        |j                  dg      \  }|j                  |k(  sJ y # 1 sw Y   /xY w)Nr9  r=  )r   r>  r   r?  r   r|   rB   r  )r   rB  r   rA   s       r   test_mockfs_mtime_roundtripr    sn    			
HLL	9B		B			u	% ug&FT:: s   A11A:c           	      J   ddl m} t        j                         } |dd|dddd	      }t	        ||      sJ |j
                  dk(  sJ |j                  dk(  sJ | j                  | j                  |            |k(  sJ  |       }t	        ||      sJ | j                  | j                  |            |k(  sJ  |d
      }t	        ||      sJ | j                  | j                  |            |k(  sJ  |ddd      }t	        ||      sJ | j                  | j                  |            |k(  sJ t        j                  t              5   |d       d d d        t        j                  t              5   |d
d       d d d        t        j                  t              5   |d
d       d d d        t        j                  t              5   |t        j                                d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   _xY w# 1 sw Y   y xY w)Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainACLr  default_metadataaccess)r  r  )r   r  acct)r   r  )r  )r   r   r   nowr$   r  r   r`  r_  r   r2  rZ   )ra  r   rB  r   s       r   test_gcs_optionsr    s   (	B	E.F35/9%9I"3
5B b-(((%%333==----}222672===	Bb-(((}222672===		&Bb-(((}222672===	0D9E)G 
HBb-(((}222672===	z	" -8,-	z	" =8<=	z	" EVDE	z	" B(,,.AB B- -= =E EB Bs0   
G58H$HH5G>H
HH"c                    ddl m}m}m}m}  |dddddd	      }t        ||      sJ |j                  dk(  sJ | j                  | j                  |            |k(  sJ  |d
ddd      }t        ||      sJ | j                  | j                  |            |k(  sJ  | |d            }t        ||      sJ  | |d            }t        ||      sJ  |d
      }t        ||      sJ | j                  | j                  |            |k(  sJ ||k7  sJ  |d      }t        ||      sJ | j                  | j                  |            |k(  sJ  |d      }t        ||      sJ | j                  | j                  |            |k(  sJ  |dddd      }t        ||      sJ | j                  | j                  |            |k(  sJ ||k7  sJ  |d      }t        ||      sJ | j                  | j                  |            |k(  sJ | j                  | j                  |             |       k7  sJ  |dd      }t        ||      sJ | j                  | j                  |            |k(  sJ  |ddd      }t        ||      sJ | j                  | j                  |            |k(  sJ  |dd      }t        ||      sJ | j                  | j                  |            |k(  sJ  |dd      }t        ||      sJ | j                  | j                  |            |k(  sJ ||k7  sJ  |dd       }t        ||      sJ | j                  | j                  |            |k(  sJ  |d!"      }t        ||      sJ | j                  | j                  |            |k(  sJ | |d#"      k7  sJ | |       k7  sJ t        j                  t              5   |d$       d d d        t        j                  t              5   |d%       d d d        t        j                  t              5   |dd&       d d d        t        j                  t              5   |dd'       d d d        t        j                  t              5   |ddd()       d d d        t        j                  t              5   |ddd*       d d d        t        j                  t              5   |d(d+       d d d        t        j                  t              5   |d,d-g.       d d d        t        j                  t              5   | |              d d d        y # 1 sw Y   qxY w# 1 sw Y   SxY w# 1 sw Y   4xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)/Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyr  r  token	us-east-2r  r  )r   r   session_tokenregionr   r   rolesessionr  d   )role_arnsession_nameexternal_idload_frequencyr  )max_attempts)retry_strategy)r
  Tr  )background_writesr  r  r  )r  r  )allow_delayed_open)r   r   )r   r   )check_directory_existence_before_creationg      ?g      ?)request_timeoutconnect_timeout)r   force_virtual_addressingzca.pem)tls_ca_file_pathzother_ca.pem)r   )r   )r   r  )r   r  arn)r   r   r
  )r   r   r   )r
  r   r=  r;  r  )r   r   r  r   r  r$   r  r`  r_  r   r2  rZ   )ra  r   r  r   r  r   rN  s          r   test_s3_optionsr    s   - - 
h$+K$8H
JB b,'''99###}222672===	vI"&s
<Bb,'''}222672=== 
1qA
CBb,'''	0a@
BBb,'''

'Cc<(((}22378C???"99		%Bb,'''}222672===		-Bb,'''}222672===
0D9E)GHC c<(((}22378C???"99		.Bb,'''}222672===}22267<>III	D	MBb,'''}222672===	D@D
FBb,'''}222672===	c4	@Bb,'''}222672===
tS
ACc<(((}22378C???"99	(8SW	XBb,'''}222672===	x	0Bb,'''}222672===~>>>>	z	" *)*	z	" *)*	z	" A@A	z	" A@A	z	" 
Hu	

 
z	" 
H	

 
z	" 5et45	z	" 6uen56	z	" 7O$567 7)* ** *A AA A
 

 
5 56 67 7sl   *
U#
U0 U=,V
VV#2V/V;W#U-0U:=V
VV #V,/V8;WWc                    ddl m} dddd}d}ddd	d}d
} ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      }t        ||      sJ |j                  |j	                  |            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k(  sJ |j                  |j	                  |            |	k(  sJ |j                  |j	                  |	            |k(  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } ||      }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ  ||      } |       }	||	k7  sJ |j                  |j	                  |            |	k7  sJ |j                  |j	                  |	            |k7  sJ t        j                  t              5   |d       d d d        t        j                  t              5   |ddd       d d d        t        j                  t              5   |ddd       d d d        t        j                  t              5   |ddd       d d d        t        j                  t        j                        5   |d       d d d        t        j                  t        j                        5   |dddd       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   txY w# 1 sw Y   y xY w)Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999r  i  zhttps://localhost:8080)proxy_options)r   r  #  r  )r   r   )r   r   )r   r   zhttpsB://localhost:9000httpA)r   r   r$   r`  r_  r   r2  rE  KeyErrorro   r  )
monkeypatchra  r   proxy_opts_1_dictproxy_opts_1_strproxy_opts_2_dictproxy_opts_2_strr   rM  rN  s
             r   test_s3_proxy_optionsr$  "  s/   ' $*;M.#*KN/ 
$5	6Bb,'''}222672===	$5	6Bb,'''}222672=== 
$4	5Bb,'''}222672===	$4	5Bb,'''}222672=== %6
7C
%6
7C#::}22378C???}22378C???
%6
7C
%6
7C#::}22378C???}22378C??? %5
6C
%5
6C#::}22378C???}22378C???
%5
6C
%5
6C#::}22378C???}22378C??? %6
7C
%5
6C#::}22378C???}22378C???
%6
7C
%5
6C#::}22378C???}22378C??? %6
7C
%6
7C#::}22378C???}22378C???
%6
7C
%5
6C#::}22378C???}22378C???
%5
6C
%6
7C#::}22378C???}22378C???
%5
6C
%5
6C#::}22378C???}22378C??? %6
7C
.C#::}22378C???}22378C???
%5
6C
.C#::}22378C???}22378C???
%6
7C
.C#::}22378C???}22378C???
%5
6C
.C#::}22378C???}22378C??? 
y	! @#>?@ 
x	  HK#FGH 
x	  Fgt#DEF 
x	  Lfk#JKL 
r	' >#<=> 
r	' 3g{,0$2 	33 3@ @H HF FL L> >3 3sH   
^-^^&	^2
^>6_
^^#&^/2^;>_
_c                     ddl m}   | dd      }d}t        j                  t        |      5 }|j                  d       d d d        d	t        j                        v ry  | d
d      }|j                  d       y # 1 sw Y   =xY w)Nr   r   z
eu-north-1T)r  r   a  When getting information for bucket 'voltrondata-labs-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-2'.|NETWORK_CONNECTIONr.  voltrondata-labs-datasetsNETWORK_CONNECTIONr  )r   r   r   r2  r   rB   r   r   )r   r   msgexcs       r   test_s3fs_wrong_regionr*    s    ' 
\T	:B!C
 
wc	* 6c
456 s399~-	[D	9B016 6s   A<<Bc                 h   ddl m}  |d      }t        ||      sJ | j                  | j	                  |            |k(  sJ  |dd      }t        ||      sJ | j                  | j	                  |            |k(  sJ ||k7  sJ  |ddd	d
dd      }t        ||      sJ | j                  | j	                  |            |k(  sJ ||k7  sJ  |dd      }t        ||      sJ | j                  | j	                  |            |k(  sJ ||k7  sJ  |dddd      }t        ||      sJ | j                  | j	                  |            |k(  sJ ||k7  sJ  |dd      }t        ||      sJ | j                  | j	                  |            |k(  sJ ||k7  sJ t        j                  t        d      5   |dd       d d d        t        j                  t        d      5   |dd       d d d        d}t        j                  t        |      5   |ddd       d d d        t        j                  t        d      5   |ddd       d d d        t        j                  t        |      5   |ddd       d d d        t        j                  t              5   |ddd       d d d        t        j                  t              5   |        d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   pxY w# 1 sw Y   y xY w)Nr   r   zfake-account-name)r   fakeaccountkey)r   r   zfake-accountfakeaccountzfake-blob-authorityzfake-dfs-authorityr  r   fakesastoken)r   	sas_tokenzfake-tenant-idzfake-client-idzfake-client-secret)r   	tenant_id	client_idclient_secret)r   r1  zclient_id must be specifiedr.  )r   r0  )r   r2  zInvalid Azure credential configuration: For ManagedIdentityCredential, provide only client_id. For ClientSecretCredential, provide tenant_id, client_id, and client_secret.)r   r1  r2  )r   r0  r2  )r   r0  r1  )r   r   r/  )	r   r   r$   r`  r_  r   r2  rZ   rE  )	ra  r   rM  rN  fs3fs4fs5fs6invalid_msgs	            r   test_azurefs_optionsr8    s`   *
':
;Cc?+++}22378C???
':&68Cc?+++}22378C???#::
~=1F0D.5-4	6C
 c?+++}22378C???#::
':$24Cc?+++}22378C???#::
(""*	C c?+++}22378C???#::
("C c?+++}22378C???#::	z)F	G 
,&	

 
z)F	G 
,.	

	W  
z	5 
,&.	

 
z)F	G 
,&.	

 
z	5 
,&&	

 
z	" 2%8m"0	22 
y	!  W
 

 

 

 

 
2 2 sT   8K&K,K8L4L!LL(K),K58LLLL%(L1c                 $   ddl m} t        j                         st	        j
                  d       | \  }}}d}d}d}d}	 |||d|||	      }
|j                  |	j                  ||d|||            }|j                  |	j                  ||d
|||            }|j                  |	j                  ||d
|dz   ||            } |||      }|j                  d| d|       } |||d      } |||dd      } |||dt        j                  d            } |||dd      } |||ddddi      }|
|k(  sJ ||k(  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k7  sJ ||k(  sJ ||k7  sJ ||k7  sJ t	        j                  t              5   |        d d d        t	        j                  t              5  |j                  d       d d d        |
||||||||||fD ]'  }|j                  |j                  |            |k(  r'J  | \  }}} ||||      }|j                  t        d            sJ |j                  d| d| d|       }|j                  t        d            sJ y # 1 sw Y   xY w# 1 sw Y   xY w)Nr   r   r      i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}libhdfs)r   replicationr  default_block_sizemerz  zhdfs://r   	localuser)r   
cache_path)r   kerb_ticketcache_path2
hdfs_tokenabcd)r   rA  
extra_confr  rW   z/?user=)r   r   ro   r   r   r   from_uriformatr}  ry  r2  rE  r`  r_  rB   r   )r   ra  r   r   r   r   r<  r  r=  urihdfs1hdfs2hdfs3hdfs4hdfs5hdfs6hdfs7hdfs8hdfs9hdfs10hdfs11r   r   s                          r   test_hdfs_optionsrT  %  s&   +??+,&D$KK$$C T4i)4+0BDE %%cjjdI{K9K' E %%cjjdD+{4F' E %%cjjdD+/;8J' E T4(E%%vQtf&=>ET4k:ET4k)57ET4k)0l)CEEdD{*79FdD{*6*6)?AF E>>E>>E>>E>>E>>E>>E>>E>>E>>E>>U??U??	y	! 	y	! %!!!$% eUE5%ff& B""=#6#6r#:;rAAAB 'D$D$T2Dl3/000$$
$qgdV,D l3/000# % %s   I:J:JJ)rH  expected_klassexpected_pathzmock:r  zmock:foo/barr:  zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/éfile:/rW   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 ^    t        j                  |       \  }}t        ||      sJ ||k(  sJ y r   )r   rF  r$   )rH  rU  rV  r   r3   s        r   test_filesystem_from_urirY  m  s6    & ""3'HBb.)))=   r!   c                  6   t        j                  d      \  } }t        | t              sJ |dk(  sJ t        j                  d      \  } }t        | t              sJ |dk(  sJ t               j                  d      \  } }t        | t              sJ |dk(  sJ y )NrW  rW   )rH  )r   rF  r$   r   )r   r3   s     r    test_filesystem_from_uri_callingr[    s    ""8,HBb/***3;; ""x0HBb/***3;;  ))h)7HBb/***3;;r!   r3   c                     t        j                  |       }t        j                  |      \  }} t	        |t
              sJ | |j                         j                         j                         k(  sJ y r   )	r}  ry  r   rF  r$   r   resolveabsoluter   )r3   r   r   s      r    test_filesystem_from_path_objectr_    s\    
 	TA""1%HBb/***199;'')224444r!   c           	      F   ddl m} | d   \  }}}}d| d| d| d| d	}t        j                  |      \  }}t	        ||      sJ |dk(  sJ |j                  |       |j                  |g      \  }	|	j                  |k(  sJ |	j                  t        j                  k(  sJ y )	Nr   r   r   s3://r   z0@mybucket/foo/bar?scheme=http&endpoint_override=z&allow_bucket_creation=Truemybucket/foo/bar)r   r   r   rF  r$   rS   rB   r3   r<  r   r<   )
r   r   r   r   r   r   rH  r   r3   rA   s
             r   test_filesystem_from_uri_s3rc    s    ')2<)@&D$
J*Qzl +!F!D6)DFC ""3'HBb,'''%%%%MM$tf%FT9999*****r!   c                 B   ddl m} | d   \  }}dd| d| dz   dz   }t        j                  |      \  }}t	        ||      sJ |d	k(  sJ |j                  |       |j                  |g      \  }|j                  |k(  sJ |j                  t        j                  k(  sJ y )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idrb  )r   r   r   rF  r$   rS   rB   r3   r<  r   r<   )r   r   r   r   rH  r   r3   rA   s           r   test_filesystem_from_uri_gcsrf    s    (L)JD$<TF!D6KL=>C ""3'HBb-(((%%%%MM$tf%FT9999*****r!   c                     t               } t        |       }t        |t              sJ |j                  dk(  sJ |j                  | u sJ t        j                  t              5  t        d        d d d        y # 1 sw Y   y xY wNz	py::dummy)r   r   r$   r$  handlerr   r2  rE  )ri  r   s     r   test_py_filesystemrj    sp    nG	g	Bb,'''<<;&&&::   	y	! T  s   "A77B c                  (   t        d      } t        d      }t        d      }t        |       }t        |       }t        |      }t        |      }||usJ ||usJ ||usJ ||k(  sJ ||k7  sJ ||k(  sJ |t               k7  sJ |t               k7  sJ y )Nrz  r:  )r   r   r   r>   )handler1handler2handler3rM  rN  r3  r4  s          r   test_py_filesystem_equalityro    s    AHAHAH
x
 C
x
 C
x
 C
x
 Cc>>c>>c>>#::#::#::/####&(??r!   c                     t               }t        |      }| j                  |      }| j                  |      }t	        |t
              sJ ||k(  sJ |j                  |k(  sJ |j                  dk(  sJ y rh  )r   r   r_  r`  r$   r   ri  r$  )ra  ri  r   rb  rc  s        r   test_py_filesystem_picklingrq    sw    nG	g	B$$R(J"":.Hh
+++r>>w&&&,,,r!   c                  R   t               } t        |       }t        |t              sJ t        j                  |       }d }  |       J d } |       J t               } t        |       }t        j                  |       }d }  |       |j
                  u sJ  |       J d } |       J y r   )r   r   r$   weakrefrefri  )ri  r   wrs      r   test_py_filesystem_lifetimerv    s    nG	g	Bb,'''	W	BG4	B4<< nG	g	B	W	BG42::4	B4<<r!   c                     t               } t        |       }|j                  dg      \  }|j                  dk(  sJ |j                  t
        j                  k(  sJ |j                  dg      \  }|j                  dk(  sJ |j                  t
        j                  k(  sJ |j                  dg      \  }|j                  dk(  sJ |j                  t
        j                  k(  sJ t        j                  t              5  |j                  dg       d d d        t        j                  t              5  |j                  dg       d d d        y # 1 sw Y   >xY w# 1 sw Y   y xY w)Nzsome/dirz	some/filer8   r9   r  )r   r   rB   r3   r<  r   r<   r;   r=   r   r2  rE  r?   )ri  r   rA   s      r    test_py_filesystem_get_file_inforx    s4   nG	g	Bzl+FT99
"""99*****{m,FT99###99%%%zl+FT99
"""99)))))	y	! &
)%& 
w	 "
%!" "& &" "s   ;E/EEE c                  Z   t               } t        |       }t        d      }|j                  |      }t	        |      dk(  sJ |d   j
                  dk(  sJ |d   j                  t        j                  k(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  t        j                  k(  sJ |d   j                  J t        dd	
      }|j                  |      }t	        |      dk(  sJ |d   j
                  dk(  sJ |d   j
                  dk(  sJ |d   j
                  dk(  sJ t        d      }t        j                  t              5  |j                  |       d d d        t        dd	      }|j                  |      g k(  sJ y # 1 sw Y   -xY w)NrD   )rJ   r:  r   rE   rF   rz  rH   T)rJ   rM   r  rI   r8   )rJ   rK   )r   r   r   rB   r  r3   r<  r   r;   rG   r<   r   r2  rL   )ri  r   rN   rO   s       r   )test_py_filesystem_get_file_info_selectorrz    s   nG	g	BY/HX&Eu:??8==O+++8==HMM)))8==C8==----8==H.....8==   Y$?HX&Eu:??8==O+++8==----8==3333Z0H	(	) #
"# ZFHH%+++	# #s   #F!!F*c                  X   t               } t        |       }|j                  dd       |j                  dd       t        j                  t
              5  |j                  d       d d d        |j                  d       |j                  d       d	D ]H  }t        j                  t              5  |j                  |       d d d        |j                  |d
       J |j                  d       |j                  dd       |j                  dd       y # 1 sw Y   xY w# 1 sw Y   `xY w)NrM   Tr  rR   FfoobarrU   rX   )r  rW   r  r  r_   ra   rb   rh   ri   )r   r   rS   r   r2  r?   rU   rX   rZ   r_   rf   rj   )ri  r   r3   s      r   test_py_filesystem_opsr}  <  s    nG	g	BMM+M.MM/UM3	w	  
h  MM,01 ;]]:& 	)""4(	)
tT:; NN=!GGK#LL!>2   	) 	)s   D/D D D)	c                      t        t                     } | j                  d      5 }|j                         dk(  sJ 	 d d d        t	        j
                  t              5  | j                  d       d d d        y # 1 sw Y   =xY w# 1 sw Y   y xY w)Nsomefiles   somefile:input_streamr8   )r   r   rs   r  r   r2  rL   r   r6  s     r   test_py_open_input_streamr  P  s}    	ln	%B			j	) 4Qvvx33334	(	) )
Z() )4 4) )   A8B8BBc                      t        t                     } | j                  d      5 }|j                         dk(  sJ 	 d d d        t	        j
                  t              5  | j                  d       d d d        y # 1 sw Y   =xY w# 1 sw Y   y xY w)Nr  s   somefile:input_filer8   )r   r   ru   r  r   r2  rL   r  s     r   test_py_open_input_filer  Y  s}    	ln	%B			J	' 21vvx11112	(	) '
:&' '2 2' 'r  c                      t        t                     } | j                  d      5 }|j                  d       d d d        y # 1 sw Y   y xY wNr  r  )r   r   r|   r4  r  s     r   test_py_open_output_streamr  b  ?    	ln	%B			z	* a	     A  A	c                      t        t                     } | j                  d      5 }|j                  d       d d d        y # 1 sw Y   y xY wr  )r   r   r   r4  r  s     r   test_py_open_append_streamr  i  r  r  c                     ddl m}  t        j                  j	                  d      xs d} | d      }|j
                  |k(  sJ  | dd      }|j                  t        d	            }t        |      dkD  sJ d
}|j                  |      5 }|j                         }d|v sJ |d   dk(  sJ |d   dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r   PYARROW_TEST_S3_REGIONz	us-east-1Tr  r  )r   r  z"voltrondata-labs-datasets/nyc-taxizCvoltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquetr  zLast-Modifieds   2022-07-12T23:32:00ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetr  rB   r   r  rs   r{   )r   default_regionr   entriesr   r6  mds          r   test_s3_real_awsr  p  s     (jjnn%=> "! 		%B99&&&	[	9B|, . /Gw<!
OC			c	" FaZZ\###/"&==== &zEEEEF F Fs   +C  C	c                     t        j                  d      \  } }| j                  dk(  sJ | j                  |      5 }d|j	                  d      v sJ 	 d d d        t        j                  d      \  } }| j                  dk(  sJ t        j                  t        d      5  t        j                  d	       d d d        t        j                  d
      \  } }| j                  dk(  sJ t        j                  d      \  } }| j                  |      }t        j                  t              5  |j                          d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2r  zBucket '.*' not foundr.  z s3://x-arrow..nonexistent-bucketz0s3://x-arrow-nonexistent-bucket?region=us-east-3z	us-east-3zMs3://x-arrow-nonexistent-bucket/T.md?region=us-east-2&allow_delayed_open=true)
r   rF  r  rs   r  r   r2  r?   r|   close)r   r3   r6  streams       r   !test_s3_real_aws_region_selectionr    sN    ""#BCHB99###			d	# Gq8AFF2JFFFG ""8:HB99### 
w&=	> @>?@""#UVHB99###
 ""WYHB""4(F	w	  1G G@ @ s#   EE'EE
EE"c                  $   ddl m}   | d      dk(  sJ  | d      dk(  sJ t        j                  t        d      5   | d	       d d d        t        j                  t        d      5   | d
       d d d        y # 1 sw Y   6xY w# 1 sw Y   y xY w)Nr   resolve_s3_regionr&  r  zmf-nwp-modelsr  zNot a valid bucket namer.  r:  z	s3:bucket)r   r  r   r2  rZ   r  s    r   test_resolve_s3_regionr    s    ,89[HHH_-<<<	z)B	C %)$%	z)B	C '+&' '% %' 's   	A:(	B:BBc           
      `   |d   }|d   } |d      }|j                  |      5 }|j                  d       d d d        | \  }}}	}
d|	 d|
 d| d| d| 
}t        |d	z        }t        ||       t	               }|j                  |      5 }|j                         dk(  sJ 	 d d d        t        |d
z        }t        |||       |j                  |      5 }|j                         dk(  sJ 	 d d d        t        |dz        }t        |      }t        ||       |j                  |      5 }|j                         dk(  sJ 	 d d d        t        |dz        }t        |||       |j                  |      5 }|j                         dk(  sJ 	 d d d        t        |dz        }t        ||dd       |j                  |      5 }|j                         dk(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   RxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r   rh  r0  ra  r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtrz  F
chunk_sizeuse_threads)r|   r4  r   r   r   rs   r  r   )s3_connectionr   r   r   r   r3   r6  r   r   r   r   
source_urilocal_path1r   local_path2local_path3destination_urilocal_path4local_path5s                      r   test_copy_filesr    sN   	dB(^F '?D			t	$ 	 *7&D$
J

|1ZL$
)$q	8 
 g/0Kz;'G		"	";	/ #1vvx7"""# g/0Kt[B7		"	";	/ #1vvx7"""# g/0K%k2Oz?+		"	";	/ #1vvx7"""# g/0Kz;wG		"	";	/ #1vvx7"""# g/0Kz;1%H		"	";	/ #1vvx7"""# #U # ## ## ## ## #sG   G%G2G?,H8HH$%G/2G<?H	HH!$H-c                    t               | dz  }|j                          j                  t        |dz              5 }|j	                  d       d d d        j                  t        |dz              5 }|j	                  d       d d d        fd}| dz  }|j                          t        t        |      t        |              ||       | dz  }|j                          t        t        |      t        |      	        ||       | d
z  }|j                          t        t        |            }t        t        |            }t        ||        ||       | dz  }	|	j                          t        ||	        ||	       | dz  }
|
j                          t        ||
dd        ||
       y # 1 sw Y   fxY w# 1 sw Y   <xY w)Nsourcefile1   test1file2   test2c                 &   j                  t        | dz              5 }|j                         dk(  sJ 	 d d d        j                  t        | dz              5 }|j                         dk(  sJ 	 d d d        y # 1 sw Y   FxY w# 1 sw Y   y xY w)Nr  r  r  r  )rs   r   r  )destination_dirr6  r   s     r   check_copied_filesz5test_copy_files_directory.<locals>.check_copied_files  s    &&s?W+D'EF 	(!668x'''	(&&s?W+D'EF 	(!668x'''	( 	(	( 	(	( 	(s   A;B;BBdestination1destination2)r  r  destination3destination4destination5rz  Fr  )r   mkdirr|   r   r4  r   r   )r   
source_dirr6  r  destination_dir1destination_dir2destination_dir3r  r  destination_dir4destination_dir5r   s              @r   test_copy_files_directoryr    s   G 8#J		#	#C
W(<$=	> !			#	#C
W(<$=	> !	( /s:$4 56'( /s:$4 5!(J'( / Z1J%c*:&;<Oz?+'( /z+,'( /z+uM'(W  s   F45G4F>Gc                  T    d} t        j                  t        j                  d| g       y )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecodes    r   test_s3_finalizer    s$    D* 3>>467r!   c                  T    d} t        j                  t        j                  d| g       y )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('voltrondata-labs-datasets')
        r  r  r  s    r    test_s3_finalize_region_resolverr  ;  s$    D 3>>467r!   c                  T    d} t        j                  t        j                  d| g       y )Na  if 1:
        import threading
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)
        threads = []
        fn = lambda: FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        for i in range(4):
            thread = threading.Thread(target = fn)
            threads.append(thread)
            thread.start()

        for thread in threads:
            thread.join()

        finalize_s3()
        r  r  r  s    r   test_concurrent_s3fs_initr  O  s$    
D" 3>>467r!   zUatexit(ensure_s3_finalized) will be called too late with bundled aws-sdk-cpp 1.11.587r"  z?Leaking S3ClientFinalizer causes segfault on musl based systemsc            	      H   	 t        j                  ddg       t               } dddd|  dt        j                  j                  t        d      g}t        j                  |t         j                  	      }	 d
|  d}t        j                         }d }t        j                         |dz   k  r(	 t        |      5 }|j                  dk(  sJ 	 d d d        nt        j"                  d|d|        |j%                          |j'                         dk(  sJ y # t        $ r t        j                  d       Y w xY w# 1 sw Y   pxY w# t        $ r!}|}t        j                   d       Y d }~nd }~ww xY wt        j                         |dz   k  r⌻# |j%                          w xY w)Nuwsgiz	--versionz"uwsgi not installed on this Pythonz-iz--httpz
127.0.0.1:z--wsgi-filezwsgi_examples.py)stdinzhttp://127.0.0.1:z/s3/r     g?zCould not fetch r      )r  r  rL   r   r   r   r  r3   joinherePopenDEVNULLtimer   statusr   sleepfail	terminatewait)r   argsprocurl
start_timeerrorrespr   s           r   test_uwsgi_integrationr  h  ss   :w45 DT8z$%8277<<.@ACDD
(:(:;D!$t,YY[
iikJN* S\ .T;;#---.
 KK*3'E7;<99;"3  :89:. .   

3  iikJN* 	s_   D 66F -E 8D:E F D76D7:E?E 	E0E+&F +E00F F!c                  ^   	 ddl m}  ddlm} t        j                  d      \  }}t        t                            }||k(  sJ |dk(  sJ d}t        j                  d|       \  }}t        t                             }||k(  sJ y # t        $ r t        j                  d       Y w xY w)	Nr   )r   )MemoryFileSystemzfsspec not installedz$fsspec+memory://path/to/data.parquetz/path/to/data.parquetzfile:///tmp/my.filezfsspec+)fsspec.implementations.localr   fsspec.implementations.memoryr  ImportErrorr   r   r   rF  r   r   )r   r  r   r3   expected_fsrH  r5  s          r   test_fsspec_filesystem_from_urir    s    ,@B ""#IJHB}-=-?@AK****  C'#0EB}_->?@K  ,*+,s   B B,+B,c                     t        j                  d       	 ddlm}  t        j                  d      \  }}t        t                             }||k(  sJ |dk(  sJ y # t        $ r t        j
                  d       Y `w xY w)Nr   r   )HfFileSystemzhuggingface_hub not installedzFhf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquetzAdatasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet)
r   r   huggingface_hubr  r  r   r   rF  r   r   )r  r   r3   r  s       r   $test_huggingface_filesystem_from_urir    s    
!50 ""PHB }\^<=KVVVV  5345s   A A>=A>)r   r   r   r  r  r}  urllib.requestr   r  r  r  r   rs  r   ro   pyarrow.tests.test_ior   pyarrow.tests.utilr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   pyarrow.utilr   r3   dirnameabspathr  r  r   fixturer   r   r   r   r   r   r   r   r   r1  r   r   r   r   r  r	  markr   r   r   r
  r   r   r   r   r  r  r  r'  r*  r7  rC  rF  rO  rQ  rX  rd  rp  rs  ru  r  r  r  r  r  r  r  r  r  r  r  r  r  r  parametrizecompressr  r  r  
decompressr  r  r  r  r  r  r  r  r  r$  r*  r8  rT  rY  r[  r_  rc  rf  rj  ro  rq  rv  rx  rz  r}  r  r  r  r  r  r  r  r  r  r  r  	processes	threadingr  r   skipifr  r  r  r.   r!   r   <module>r     sW	  $ 3 2  	  "  
     76 6$ $ $ $ ( 
wwrwwx01f'$ f'R              :  4   @  8  $      , FLL FLL+ FLL1 FLLkknn
 FLLkkoo
 FLLkk
 FLLkk
 FLL FLL: FLL: FLLB FLLE FLL=kknnk: :v2w:v2 # # ' ' / / 5 5	2&
>
#
  5  5F:
O(K (,"- 8+v5 p".@$,(
	(($ 0	tX	r8	t}}%	dmm$	
 
&&  2	tX	r8	t'	doo&		) 	) @	tXx(	r8X&	t}}doo6	dmmT__5	; ;0".#.( "B "BJ d7 d7N I3 I3X 2 2. Z Zz D1 D1N C or"_i0oy13!?NC$#&oz24!?OD/3*-*<=!F $!%$!" 
iX.5	5 + +$ + +&(	-*".,:3()' F F.  B ' ' 1# 1#h1)h 8 86 8 8& 8   8,  6 7(* 4T UU7 
@(Wr!   