a ij6"@sddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZddZdaGd d d e Zed gd Gd ddejZGdddeZdS)N)suppress)cached_propertywraps)parse_qs)AbstractFileSystem)"get_package_version_without_importinfer_storage_options mirror_fromtokenizecstfdd}|S)Nc spz|i|WStyj}zB|js(|j^}}t|trTd|vrTttj||nWYd}~n d}~00dS)Nzdoes not exist)OSErrorargs isinstancestrFileNotFoundErrorerrnoENOENT)r kwargs exceptionmessagefuncZ/home/ghrups/robot-bench/.venv/lib/python3.9/site-packages/fsspec/implementations/arrow.pywrappers z wrap_exceptions..wrapper)r)rrrrrwrap_exceptionss rcseZdZdZdZfddZeddZeddZ e d d Z d-d d Z ddZ ddZddZeddZeddZeddZed.ddZed/dd Zed0d!d"Zed1d#d$Zed%d&Zed'd(Zd2fd)d* Zfd+d,ZZS)3ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem. Parameters ---------- fs : pyarrow.fs.FileSystem /c s$tda||_tjfi|dS)NZpyarrow)rPYARROW_VERSIONfssuper__init__)selfrr __class__rrr 3szArrowFSWrapper.__init__cCs|jjSN)r type_namer!rrrprotocol9szArrowFSWrapper.protocolcCsdt|jj|jjS)NZhdfs_)r rhostportr&rrrfsid=szArrowFSWrapper.fsidcCs*t|}|d}|dr&|dd}|S)Npathz//)r startswith)clsr+opsrrr_strip_protocolAs   zArrowFSWrapper._strip_protocolFc s~|}ddlm}z"fddj||D}Wn*ttfybj|fi|g}Yn0|rl|Sdd|DSdS)Nr) FileSelectorcsg|]}|qSr) _make_entry.0entryr&rr Osz%ArrowFSWrapper.ls..cSsg|] }|dqS)namerr3rrrr6X)r0 pyarrow.fsr1r get_file_inforNotADirectoryErrorinfo)r!r+detailrr1entriesrr&rlsJs    zArrowFSWrapper.lscKs$||}|j|g\}||Sr$)r0rr:r2)r!r+rr<rrrr<Zs zArrowFSWrapper.infocCs6||}z||Wnty,YdS0dSdS)NFT)r0r<rr!r+rrrexists_s   zArrowFSWrapper.existscCsnddlm}|j|jurd}n<|j|jur0d}n*|j|jurVttjt tj|j nd}|j |j ||j dS)Nr)FileType directoryfileother)r7sizetypemtime)r9rBrGZ DirectoryFileZNotFoundrrrosstrerrorr+rFrH)r!r<rBkindrrrr2hs    zArrowFSWrapper._make_entryc Ks||d}||d}||d}|dtd}zJ||d}t||Wdn1sr0Y|j ||WnHt yt t |j |Wdn1s0YYn0Wdn1s0YdS)Nrrbz.tmp.wb)r0rstrip_opensecretsZ token_hexopenshutil copyfileobjrmove BaseExceptionrr delete_file)r!path1path2rZlstreamZ tmp_fnameZrstreamrrrcp_file{s*  *zArrowFSWrapper.cp_filecKs2||d}||d}|j||dS)Nr)r0rPrrV)r!rYrZrrrrmvszArrowFSWrapper.mvcCs||}|j|dSr$)r0rrXr@rrrrm_files zArrowFSWrapper.rm_fileNcCsF||d}||r6|r,|j|qBtdn |j|dS)Nrz0Can't delete directories without recursive=False)r0rPisdirr delete_dir ValueErrorrX)r!r+ recursiveZmaxdepthrrrrms   zArrowFSWrapper.rmrMTc Ks|dkr |r|jj}qR|jj}n2|dkr2|jj}n |dkrD|jj}ntd|i}|dksb|sttdddkrd|d<||fi|}t |||||fi|S) NrMrOabz'unsupported mode for Arrow filesystem: .r compression) rZopen_input_fileZopen_input_streamZopen_output_streamZopen_append_streamr`intrsplit ArrowFile) r!r+mode block_sizeseekablermethod_kwargsstreamrrrrQs     zArrowFSWrapper._opencKs2||}|r|j|ddn|jj|dddS)NT)exist_okFra)r0makedirsr create_dir)r!r+Zcreate_parentsrrrrmkdirs zArrowFSWrapper.mkdircCs||}|jj|dddS)NTrq)r0rrs)r!r+rprrrrrs zArrowFSWrapper.makedirscCs||}|j|dSr$)r0rr_r@rrrrmdirs zArrowFSWrapper.rmdircCs||}|j|jSr$)r0rr:rHr@rrrmodifieds zArrowFSWrapper.modifiedc s&|dv|d<tj|fddd|S)N)Nrrl)startend)rcat_file)r!r+rwrxrr"rrrys zArrowFSWrapper.cat_filec s"d|d<tj||fi|dS)NFrl)rget_file)r!ZrpathZlpathrr"rrrzszArrowFSWrapper.get_file)F)FN)rMNT)T)F)NN)__name__ __module__ __qualname____doc__Z root_markerr propertyr'rr* classmethodr0r?r<rAr2rr[r\r]rbrQrtrrrurvryrz __classcell__rrr"rr(sB               rro) readseektellwritereadablewritablecloserFrlc@s&eZdZdddZddZddZdS) riNcKs.||_||_||_||_||_|_||_dSr$)r+rjrro blocksizerkr)r!rror+rjrkrrrrr s  zArrowFile.__init__cCs|Sr$rr&rrr __enter__szArrowFile.__enter__cGs|Sr$)r)r!r rrr__exit__szArrowFile.__exit__)N)r{r|r}r rrrrrrris rics2eZdZdZdZd fdd Zed d ZZS) HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem to connect it's interface with fsspecZhdfsdefaultrNc  s:ddlm}|||||||d} tjfd| i|dS)a; Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication replication: int set replication factor of file for write operations. default value is 3. extra_conf: None or dict Passed on to HadoopFileSystem r)r)r(r)user kerb_ticket replication extra_confrN)r9rrr ) r!r(r)rrrrrrrr"rrr s zHadoopFileSystem.__init__cCst|}i}|ddr$|d|d<|ddr<|d|d<|ddrT|d|d<|ddrt|d}|ddrt|dd|d<|S)Nr(usernamerr)Z url_queryrr)rgetrrg)r+r/outZqueriesrrr_get_kwargs_from_urls%s         z&HadoopFileSystem._get_kwargs_from_urls)rrNNrN) r{r|r}r~r'r staticmethodrrrrr"rrs'r)riorJrRrT contextlibr functoolsrr urllib.parserZ fsspec.specrZ fsspec.utilsrrr r rrrIOBaserirrrrrs&   1