Voice et bot modif

venv/lib/python3.12/site-packages/fsspec/implementations/arrow.py

@@ -242,7 +242,9 @@ class ArrowFile(io.IOBase):
 
     @property
     def size(self):
-        return self.stream.size()
+        if self.stream.seekable():
+            return self.stream.size()
+        return None
 
     def __exit__(self, *args):
         return self.close()

venv/lib/python3.12/site-packages/fsspec/implementations/cache_metadata.py

@@ -165,7 +165,7 @@ class CacheMetadata:
 
         The actual closing of the file is the responsibility of the caller.
         """
-        # File must be writeble, so in self.cached_files[-1]
+        # File must be writable, so in self.cached_files[-1]
         c = self.cached_files[-1][path]
         if c["blocks"] is not True and len(c["blocks"]) * f.blocksize >= f.size:
             c["blocks"] = True

venv/lib/python3.12/site-packages/fsspec/implementations/cached.py

@@ -427,6 +427,7 @@ class CachingFileSystem(ChainedFileSystem):
     def __getattribute__(self, item):
         if item in {
             "load_cache",
+            "_get_cached_file_before_open",
             "_open",
             "save_cache",
             "close_and_update",
@@ -473,9 +474,12 @@ class CachingFileSystem(ChainedFileSystem):
         }:
             # all the methods defined in this class. Note `open` here, since
             # it calls `_open`, but is actually in superclass
-            return lambda *args, **kw: getattr(type(self), item).__get__(self)(
-                *args, **kw
-            )
+            if hasattr(type(self), item):
+                return lambda *args, **kw: getattr(type(self), item).__get__(self)(
+                    *args, **kw
+                )
+            # method is in the whitelist but not defined on this subclass;
+            # fall through to delegate to the wrapped filesystem below
         if item in ["__reduce_ex__"]:
             raise AttributeError
         if item in ["transaction"]:
@@ -678,46 +682,12 @@ class WholeFileCacheFileSystem(CachingFileSystem):
             out = out[paths[0]]
         return out
 
-    def _open(self, path, mode="rb", **kwargs):
-        path = self._strip_protocol(path)
-        if "r" not in mode:
-            hash = self._mapper(path)
-            fn = os.path.join(self.storage[-1], hash)
-            user_specified_kwargs = {
-                k: v
-                for k, v in kwargs.items()
-                # those kwargs were added by open(), we don't want them
-                if k not in ["autocommit", "block_size", "cache_options"]
-            }
-            return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
-        detail = self._check_file(path)
-        if detail:
-            detail, fn = detail
-            _, blocks = detail["fn"], detail["blocks"]
-            if blocks is True:
-                logger.debug("Opening local copy of %s", path)
-
-                # In order to support downstream filesystems to be able to
-                # infer the compression from the original filename, like
-                # the `TarFileSystem`, let's extend the `io.BufferedReader`
-                # fileobject protocol by adding a dedicated attribute
-                # `original`.
-                f = open(fn, mode)
-                f.original = detail.get("original")
-                return f
-            else:
-                raise ValueError(
-                    f"Attempt to open partially cached file {path}"
-                    f" as a wholly cached file"
-                )
-        else:
-            fn = self._make_local_details(path)
-        kwargs["mode"] = mode
-
+    def _get_cached_file_before_open(self, path, **kwargs):
+        fn = self._make_local_details(path)
         # call target filesystems open
         self._mkcache()
         if self.compression:
-            with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2:
+            with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2:
                 if isinstance(f, AbstractBufferedFile):
                     # want no type of caching if just downloading whole thing
                     f.cache = BaseCache(0, f.cache.fetcher, f.size)
@@ -735,7 +705,87 @@ class WholeFileCacheFileSystem(CachingFileSystem):
         else:
             self.fs.get_file(path, fn)
         self.save_cache()
-        return self._open(path, mode)
+
+    def _open(self, path, mode="rb", **kwargs):
+        path = self._strip_protocol(path)
+        # For read (or append), (try) download from remote
+        if "r" in mode or "a" in mode:
+            if not self._check_file(path):
+                if self.fs.exists(path):
+                    self._get_cached_file_before_open(path, **kwargs)
+                elif "r" in mode:
+                    raise FileNotFoundError(path)
+
+            detail, fn = self._check_file(path)
+            _, blocks = detail["fn"], detail["blocks"]
+            if blocks is True:
+                logger.debug("Opening local copy of %s", path)
+            else:
+                raise ValueError(
+                    f"Attempt to open partially cached file {path}"
+                    f" as a wholly cached file"
+                )
+
+        # Just reading does not need special file handling
+        if "r" in mode and "+" not in mode:
+            # In order to support downstream filesystems to be able to
+            # infer the compression from the original filename, like
+            # the `TarFileSystem`, let's extend the `io.BufferedReader`
+            # fileobject protocol by adding a dedicated attribute
+            # `original`.
+            f = open(fn, mode)
+            f.original = detail.get("original")
+            return f
+
+        hash = self._mapper(path)
+        fn = os.path.join(self.storage[-1], hash)
+        user_specified_kwargs = {
+            k: v
+            for k, v in kwargs.items()
+            # those kwargs were added by open(), we don't want them
+            if k not in ["autocommit", "block_size", "cache_options"]
+        }
+        return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
+
+    async def _cat_file(self, path, start=None, end=None, **kwargs):
+        logger.debug("async cat_file %s", path)
+        path = self._strip_protocol(path)
+        sha = self._mapper(path)
+        fn = self._check_file(path)
+
+        if not fn:
+            fn = os.path.join(self.storage[-1], sha)
+            await self.fs._get_file(path, fn, **kwargs)
+
+        with open(fn, "rb") as f:  # noqa ASYNC230
+            if start:
+                f.seek(start)
+            size = -1 if end is None else end - f.tell()
+            return f.read(size)
+
+    async def _cat_ranges(
+        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
+    ):
+        logger.debug("async cat ranges %s", paths)
+        lpaths = []
+        rset = set()
+        download = []
+        rpaths = []
+        for p in paths:
+            fn = self._check_file(p)
+            if fn is None and p not in rset:
+                sha = self._mapper(p)
+                fn = os.path.join(self.storage[-1], sha)
+                download.append(fn)
+                rset.add(p)
+                rpaths.append(p)
+            lpaths.append(fn)
+        if download:
+            await self.fs._get(rpaths, download, on_error=on_error)
+
+        return LocalFileSystem().cat_ranges(
+            lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
+        )
 
 
 class SimpleCacheFileSystem(WholeFileCacheFileSystem):
@@ -841,46 +891,6 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         else:
             raise ValueError("path must be str or dict")
 
-    async def _cat_file(self, path, start=None, end=None, **kwargs):
-        logger.debug("async cat_file %s", path)
-        path = self._strip_protocol(path)
-        sha = self._mapper(path)
-        fn = self._check_file(path)
-
-        if not fn:
-            fn = os.path.join(self.storage[-1], sha)
-            await self.fs._get_file(path, fn, **kwargs)
-
-        with open(fn, "rb") as f:  # noqa ASYNC230
-            if start:
-                f.seek(start)
-            size = -1 if end is None else end - f.tell()
-            return f.read(size)
-
-    async def _cat_ranges(
-        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
-    ):
-        logger.debug("async cat ranges %s", paths)
-        lpaths = []
-        rset = set()
-        download = []
-        rpaths = []
-        for p in paths:
-            fn = self._check_file(p)
-            if fn is None and p not in rset:
-                sha = self._mapper(p)
-                fn = os.path.join(self.storage[-1], sha)
-                download.append(fn)
-                rset.add(p)
-                rpaths.append(p)
-            lpaths.append(fn)
-        if download:
-            await self.fs._get(rpaths, download, on_error=on_error)
-
-        return LocalFileSystem().cat_ranges(
-            lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
-        )
-
     def cat_ranges(
         self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
     ):
@@ -894,37 +904,16 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
             paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
         )
 
-    def _open(self, path, mode="rb", **kwargs):
-        path = self._strip_protocol(path)
+    def _get_cached_file_before_open(self, path, **kwargs):
         sha = self._mapper(path)
-
-        if "r" not in mode:
-            fn = os.path.join(self.storage[-1], sha)
-            user_specified_kwargs = {
-                k: v
-                for k, v in kwargs.items()
-                if k not in ["autocommit", "block_size", "cache_options"]
-            }  # those were added by open()
-            return LocalTempFile(
-                self,
-                path,
-                mode=mode,
-                autocommit=not self._intrans,
-                fn=fn,
-                **user_specified_kwargs,
-            )
-        fn = self._check_file(path)
-        if fn:
-            return open(fn, mode)
-
         fn = os.path.join(self.storage[-1], sha)
         logger.debug("Copying %s to local cache", path)
-        kwargs["mode"] = mode
 
         self._mkcache()
         self._cache_size = None
+
         if self.compression:
-            with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2:
+            with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2:
                 if isinstance(f, AbstractBufferedFile):
                     # want no type of caching if just downloading whole thing
                     f.cache = BaseCache(0, f.cache.fetcher, f.size)
@@ -941,7 +930,39 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
                     f2.write(data)
         else:
             self.fs.get_file(path, fn)
-        return self._open(path, mode)
+
+    def _open(self, path, mode="rb", **kwargs):
+        path = self._strip_protocol(path)
+        sha = self._mapper(path)
+
+        # For read (or append), (try) download from remote
+        if "r" in mode or "a" in mode:
+            if not self._check_file(path):
+                # append does not require an existing file but read does
+                if self.fs.exists(path):
+                    self._get_cached_file_before_open(path, **kwargs)
+                elif "r" in mode:
+                    raise FileNotFoundError(path)
+
+        fn = self._check_file(path)
+        # Just reading does not need special file handling
+        if "r" in mode and "+" not in mode:
+            return open(fn, mode)
+
+        fn = os.path.join(self.storage[-1], sha)
+        user_specified_kwargs = {
+            k: v
+            for k, v in kwargs.items()
+            if k not in ["autocommit", "block_size", "cache_options"]
+        }  # those were added by open()
+        return LocalTempFile(
+            self,
+            path,
+            mode=mode,
+            autocommit=not self._intrans,
+            fn=fn,
+            **user_specified_kwargs,
+        )
 
 
 class LocalTempFile:

venv/lib/python3.12/site-packages/fsspec/implementations/dirfs.py

@@ -97,6 +97,9 @@ class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
     def rm(self, path, *args, **kwargs):
         return self.fs.rm(self._join(path), *args, **kwargs)
 
+    def delete(self, path, recursive=False, maxdepth=None):
+        return self.fs.delete(self._join(path), recursive=recursive, maxdepth=maxdepth)
+
     async def _cp_file(self, path1, path2, **kwargs):
         return await self.fs._cp_file(self._join(path1), self._join(path2), **kwargs)
 
@@ -137,6 +140,18 @@ class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
     def pipe_file(self, path, *args, **kwargs):
         return self.fs.pipe_file(self._join(path), *args, **kwargs)
 
+    def write_text(
+        self, path, value, encoding=None, errors=None, newline=None, **kwargs
+    ):
+        return self.fs.write_text(
+            self._join(path),
+            value,
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+            **kwargs,
+        )
+
     async def _cat_file(self, path, *args, **kwargs):
         return await self.fs._cat_file(self._join(path), *args, **kwargs)
 

venv/lib/python3.12/site-packages/fsspec/implementations/http.py

@@ -41,6 +41,9 @@ class HTTPFileSystem(AsyncFileSystem):
     match on the result. If simple_link=True, anything of the form
     "http(s)://server.com/stuff?thing=other"; otherwise only links within
     HTML href tags will be used.
+
+    URLs are passed unfiltered to aiohttp, so all addresses are accessible. Where URLs are
+    supplied by a user, the calling application may wish to filter to prevent scanning.
     """
 
     protocol = ("http", "https")
@@ -560,7 +563,9 @@ class HTTPFileSystem(AsyncFileSystem):
 
         session = await self.set_session()
 
-        async with session.put(url, data=value, headers=headers, **kwargs) as r:
+        async with session.put(
+            self.encode_url(url), data=value, headers=headers, **kwargs
+        ) as r:
             r.raise_for_status()
 
 

venv/lib/python3.12/site-packages/fsspec/implementations/local.py

@@ -204,8 +204,9 @@ class LocalFileSystem(AbstractFileSystem):
                 os.remove(p)
 
     def unstrip_protocol(self, name):
+        protocol = self.protocol if isinstance(self.protocol, str) else self.protocol[0]
         name = self._strip_protocol(name)  # normalise for local/win/...
-        return f"file://{name}"
+        return f"{protocol}://{name}"
 
     def _open(self, path, mode="rb", block_size=None, **kwargs):
         path = self._strip_protocol(path)
@@ -253,14 +254,12 @@ class LocalFileSystem(AbstractFileSystem):
     @classmethod
     def _strip_protocol(cls, path):
         path = stringify_path(path)
-        if path.startswith("file://"):
-            path = path[7:]
-        elif path.startswith("file:"):
-            path = path[5:]
-        elif path.startswith("local://"):
-            path = path[8:]
-        elif path.startswith("local:"):
-            path = path[6:]
+        protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol
+        prefixes = (protocol + sep for protocol in protos for sep in ("://", ":"))
+        for prefix in prefixes:
+            if path.startswith(prefix):
+                path = path.removeprefix(prefix)
+                break
 
         path = make_path_posix(path)
         if os.sep != "/":

venv/lib/python3.12/site-packages/fsspec/implementations/reference.py

@@ -157,6 +157,13 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         if self.engine == "pyarrow" and find_spec("pyarrow") is None:
             raise ImportError("engine choice `pyarrow` is not installed.")
 
+        # Apply `lru_cache` decorator manually per instance.
+        # This way `self` reference is not held on class level.
+        # WARNING: However, this means that self and its members are not reflected
+        # in the cache key, so we expect they won't be mutated once a value is cached.
+        self.listdir = lru_cache()(self.listdir)
+        self._key_to_record = lru_cache(maxsize=4096)(self._key_to_record)
+
     def __getattr__(self, item):
         if item in ("_items", "record_size", "zmetadata"):
             self.setup()
@@ -219,7 +226,6 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
         return LazyReferenceMapper(root, fs, **kwargs)
 
-    @lru_cache
     def listdir(self):
         """List top-level directories"""
         dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z"))
@@ -331,7 +337,6 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         # URL, offset, size
         return selection[:3]
 
-    @lru_cache(4096)
     def _key_to_record(self, key):
         """Details needed to construct a reference for one key"""
         field, chunk = key.rsplit("/", 1)

venv/lib/python3.12/site-packages/fsspec/implementations/sftp.py

@@ -135,8 +135,8 @@ class SFTPFileSystem(AbstractFileSystem):
             paths = [stat["name"] for stat in stats]
             return sorted(paths)
 
-    def put(self, lpath, rpath, callback=None, **kwargs):
-        rpath = self._strip_protocol(rpath)
+    def put_file(self, lpath, rpath, callback=None, **kwargs):
+        self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True)
         logger.debug("Put file %s into %s", lpath, rpath)
         self.ftp.put(lpath, rpath)
 

venv/lib/python3.12/site-packages/fsspec/implementations/tar.py

@@ -122,3 +122,14 @@ class TarFileSystem(AbstractArchiveFileSystem):
         if details["type"] != "file":
             raise ValueError("Can only handle regular files")
         return self.tar.extractfile(path)
+
+    def close(self):
+        """Commits any write changes to the file. Done on ``del`` too."""
+        self.tar.close()
+
+    def __del__(self):
+        if hasattr(self, "tar"):
+            self.close()
+            del self.tar
+        if hasattr(self, "of") and hasattr(self.of, "__exit__"):
+            self.of.__exit__(None, None, None)

venv/lib/python3.12/site-packages/fsspec/implementations/webhdfs.py

@@ -7,6 +7,7 @@ import shutil
 import tempfile
 import uuid
 from contextlib import suppress
+from datetime import datetime
 from urllib.parse import quote
 
 import requests
@@ -268,6 +269,23 @@ class WebHDFS(AbstractFileSystem):
         info["name"] = path
         return self._process_info(info)
 
+    def created(self, path):
+        """Return the created timestamp of a file as a datetime.datetime"""
+        # The API does not provide creation time, so we use modification time
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve creation time (modification time).")
+
+    def modified(self, path):
+        """Return the modified timestamp of a file as a datetime.datetime"""
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve modification time.")
+
     def ls(self, path, detail=False, **kwargs):
         out = self._call("LISTSTATUS", path=path)
         infos = out.json()["FileStatuses"]["FileStatus"]

venv/lib/python3.12/site-packages/fsspec/implementations/zip.py

@@ -78,6 +78,8 @@ class ZipFileSystem(AbstractArchiveFileSystem):
         if hasattr(self, "zip"):
             self.close()
             del self.zip
+        if hasattr(self, "of") and hasattr(self.of, "__exit__"):
+            self.of.__exit__(None, None, None)
 
     def close(self):
         """Commits any write changes to the file. Done on ``del`` too."""
@@ -138,14 +140,17 @@ class ZipFileSystem(AbstractArchiveFileSystem):
         if maxdepth is not None and maxdepth < 1:
             raise ValueError("maxdepth must be at least 1")
 
+        def to_parts(_path: str):
+            return list(filter(None, _path.replace("\\", "/").split("/")))
+
+        if not isinstance(path, str):
+            path = str(path)
+
         # Remove the leading slash, as the zip file paths are always
         # given without a leading slash
         path = path.lstrip("/")
-        path_parts = list(filter(lambda s: bool(s), path.split("/")))
-
-        def _matching_starts(file_path):
-            file_parts = filter(lambda s: bool(s), file_path.split("/"))
-            return all(a == b for a, b in zip(path_parts, file_parts))
+        path_parts = to_parts(path)
+        path_depth = len(path_parts)
 
         self._get_dirs()
 
@@ -157,21 +162,22 @@ class ZipFileSystem(AbstractArchiveFileSystem):
             return result if detail else [path]
 
         for file_path, file_info in self.dir_cache.items():
-            if not (path == "" or _matching_starts(file_path)):
+            if len(file_parts := to_parts(file_path)) < path_depth or any(
+                a != b for a, b in zip(path_parts, file_parts)
+            ):
+                # skip parent folders and mismatching paths
                 continue
 
             if file_info["type"] == "directory":
-                if withdirs:
-                    if file_path not in result:
-                        result[file_path.strip("/")] = file_info
+                if withdirs and file_path not in result:
+                    result[file_path.strip("/")] = file_info
                 continue
 
             if file_path not in result:
                 result[file_path] = file_info if detail else None
 
         if maxdepth:
-            path_depth = path.count("/")
             result = {
-                k: v for k, v in result.items() if k.count("/") - path_depth < maxdepth
+                k: v for k, v in result.items() if k.count("/") < maxdepth + path_depth
             }
         return result if detail else sorted(result)