Voice et bot modif

venv/lib/python3.12/site-packages/fsspec/_version.py

@@ -1,5 +1,6 @@
-# file generated by setuptools-scm
+# file generated by vcs-versioning
 # don't change, don't track in version control
+from __future__ import annotations
 
 __all__ = [
     "__version__",
@@ -10,25 +11,14 @@ __all__ = [
     "commit_id",
 ]
 
-TYPE_CHECKING = False
-if TYPE_CHECKING:
-    from typing import Tuple
-    from typing import Union
-
-    VERSION_TUPLE = Tuple[Union[int, str], ...]
-    COMMIT_ID = Union[str, None]
-else:
-    VERSION_TUPLE = object
-    COMMIT_ID = object
-
 version: str
 __version__: str
-__version_tuple__: VERSION_TUPLE
-version_tuple: VERSION_TUPLE
-commit_id: COMMIT_ID
-__commit_id__: COMMIT_ID
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
 
-__version__ = version = '2026.1.0'
-__version_tuple__ = version_tuple = (2026, 1, 0)
+__version__ = version = '2026.4.0'
+__version_tuple__ = version_tuple = (2026, 4, 0)
 
 __commit_id__ = commit_id = None

venv/lib/python3.12/site-packages/fsspec/asyn.py

@@ -263,9 +263,22 @@ async def _run_coros_in_chunks(
             break
 
         done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
+        first_exc = None
         while done:
-            result, k = await done.pop()
-            results[k] = result
+            task = done.pop()
+            try:
+                result, k = await task
+                results[k] = result
+            except Exception as exc:
+                if first_exc is None:
+                    first_exc = exc
+
+        if first_exc is not None:
+            for task in pending:
+                task.cancel()
+            if pending:
+                await asyncio.gather(*pending, return_exceptions=True)
+            raise first_exc
 
     return results
 
@@ -795,11 +808,18 @@ class AsyncFileSystem(AbstractFileSystem):
                 else:
                     return {}
         elif "/" in path[:min_idx]:
+            first_wildcard_idx = min_idx
             min_idx = path[:min_idx].rindex("/")
-            root = path[: min_idx + 1]
+            root = path[
+                : min_idx + 1
+            ]  # everything up to the last / before the first wildcard
+            prefix = path[
+                min_idx + 1 : first_wildcard_idx
+            ]  # stem between last "/" and first wildcard
             depth = path[min_idx + 1 :].count("/") + 1
         else:
             root = ""
+            prefix = path[:min_idx]  # stem up to the first wildcard
             depth = path[min_idx + 1 :].count("/") + 1
 
         if "**" in path:
@@ -810,6 +830,10 @@ class AsyncFileSystem(AbstractFileSystem):
             else:
                 depth = None
 
+        # Pass the filename stem as prefix= so backends that support it such as
+        # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard.
+        if prefix:
+            kwargs["prefix"] = prefix
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs
         )

venv/lib/python3.12/site-packages/fsspec/caching.py

@@ -25,7 +25,7 @@ else:
 T = TypeVar("T")
 
 
-logger = logging.getLogger("fsspec")
+logger = logging.getLogger("fsspec.caching")
 
 Fetcher = Callable[[int, int], bytes]  # Maps (start, end) to bytes
 MultiFetcher = Callable[[list[int, int]], bytes]  # Maps [(start, end)] to bytes
@@ -214,7 +214,7 @@ class MMapCache(BaseCache):
         if self.multi_fetcher:
             logger.debug(f"MMap get blocks {ranges}")
             for idx, r in enumerate(self.multi_fetcher(ranges)):
-                (sstart, send) = ranges[idx]
+                sstart, send = ranges[idx]
                 logger.debug(f"MMap copy block ({sstart}-{send}")
                 self.cache[sstart:send] = r
         else:
@@ -391,19 +391,8 @@ class BlockCache(BaseCache):
         if start >= self.size or start >= end:
             return b""
 
-        # byte position -> block numbers
-        start_block_number = start // self.blocksize
-        end_block_number = end // self.blocksize
-
-        # these are cached, so safe to do multiple calls for the same start and end.
-        for block_number in range(start_block_number, end_block_number + 1):
-            self._fetch_block_cached(block_number)
-
         return self._read_cache(
-            start,
-            end,
-            start_block_number=start_block_number,
-            end_block_number=end_block_number,
+            start, end, start // self.blocksize, (end - 1) // self.blocksize
         )
 
     def _fetch_block(self, block_number: int) -> bytes:
@@ -439,6 +428,8 @@ class BlockCache(BaseCache):
         """
         start_pos = start % self.blocksize
         end_pos = end % self.blocksize
+        if end_pos == 0:
+            end_pos = self.blocksize
 
         self.hit_count += 1
         if start_block_number == end_block_number:
@@ -662,12 +653,12 @@ class KnownPartsOfAFile(BaseCache):
         pass
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
+        logger.debug("Known parts request %s %s", start, stop)
         if start is None:
             start = 0
         if stop is None:
             stop = self.size
         self.total_requested_bytes += stop - start
-
         out = b""
         started = False
         loc_old = 0
@@ -698,11 +689,13 @@ class KnownPartsOfAFile(BaseCache):
             elif loc0 <= stop <= loc1:
                 # end block
                 self.hit_count += 1
-                return out + self.data[(loc0, loc1)][: stop - loc0]
+                out = out + self.data[(loc0, loc1)][: stop - loc0]
+                return out
             loc_old = loc1
         self.miss_count += 1
         if started and not self.strict:
-            return out + b"\x00" * (stop - loc_old)
+            out = out + b"\x00" * (stop - loc_old)
+            return out
         raise ValueError
 
 
@@ -710,7 +703,7 @@ class UpdatableLRU(Generic[P, T]):
     """
     Custom implementation of LRU cache that allows updating keys
 
-    Used by BackgroudBlockCache
+    Used by BackgroundBlockCache
     """
 
     class CacheInfo(NamedTuple):

venv/lib/python3.12/site-packages/fsspec/compression.py

@@ -163,6 +163,20 @@ try:
 
     register_compression("zstd", zstd.ZstdFile, "zst")
 except ImportError:
+    try:
+        import zstandard as zstd
+
+        def zstandard_file(infile, mode="rb"):
+            if "r" in mode:
+                cctx = zstd.ZstdDecompressor()
+                return cctx.stream_reader(infile)
+            else:
+                cctx = zstd.ZstdCompressor(level=10)
+                return cctx.stream_writer(infile)
+
+        register_compression("zstd", zstandard_file, "zst")
+    except ImportError:
+        pass
     pass
 
 

venv/lib/python3.12/site-packages/fsspec/config.py

@@ -30,18 +30,18 @@ def set_conf_env(conf_dict, envdict=os.environ):
     envdict : dict-like(str, str)
         Source for the values - usually the real environment
     """
+    envdict = dict(envdict)
     kwarg_keys = []
     for key in envdict:
         if key.startswith("FSSPEC_") and len(key) > 7 and key[7] != "_":
+            try:
+                value = json.loads(envdict[key])
+                envdict[key] = value
+            except json.decoder.JSONDecodeError:
+                value = envdict[key]
             if key.count("_") > 1:
                 kwarg_keys.append(key)
                 continue
-            try:
-                value = json.loads(envdict[key])
-            except json.decoder.JSONDecodeError as ex:
-                warnings.warn(
-                    f"Ignoring environment variable {key} due to a parse failure: {ex}"
-                )
             else:
                 if isinstance(value, dict):
                     _, proto = key.split("_", 1)

venv/lib/python3.12/site-packages/fsspec/implementations/arrow.py

@@ -242,7 +242,9 @@ class ArrowFile(io.IOBase):
 
     @property
     def size(self):
-        return self.stream.size()
+        if self.stream.seekable():
+            return self.stream.size()
+        return None
 
     def __exit__(self, *args):
         return self.close()

venv/lib/python3.12/site-packages/fsspec/implementations/cache_metadata.py

@@ -165,7 +165,7 @@ class CacheMetadata:
 
         The actual closing of the file is the responsibility of the caller.
         """
-        # File must be writeble, so in self.cached_files[-1]
+        # File must be writable, so in self.cached_files[-1]
         c = self.cached_files[-1][path]
         if c["blocks"] is not True and len(c["blocks"]) * f.blocksize >= f.size:
             c["blocks"] = True

venv/lib/python3.12/site-packages/fsspec/implementations/cached.py

@@ -427,6 +427,7 @@ class CachingFileSystem(ChainedFileSystem):
     def __getattribute__(self, item):
         if item in {
             "load_cache",
+            "_get_cached_file_before_open",
             "_open",
             "save_cache",
             "close_and_update",
@@ -473,9 +474,12 @@ class CachingFileSystem(ChainedFileSystem):
         }:
             # all the methods defined in this class. Note `open` here, since
             # it calls `_open`, but is actually in superclass
-            return lambda *args, **kw: getattr(type(self), item).__get__(self)(
-                *args, **kw
-            )
+            if hasattr(type(self), item):
+                return lambda *args, **kw: getattr(type(self), item).__get__(self)(
+                    *args, **kw
+                )
+            # method is in the whitelist but not defined on this subclass;
+            # fall through to delegate to the wrapped filesystem below
         if item in ["__reduce_ex__"]:
             raise AttributeError
         if item in ["transaction"]:
@@ -678,46 +682,12 @@ class WholeFileCacheFileSystem(CachingFileSystem):
             out = out[paths[0]]
         return out
 
-    def _open(self, path, mode="rb", **kwargs):
-        path = self._strip_protocol(path)
-        if "r" not in mode:
-            hash = self._mapper(path)
-            fn = os.path.join(self.storage[-1], hash)
-            user_specified_kwargs = {
-                k: v
-                for k, v in kwargs.items()
-                # those kwargs were added by open(), we don't want them
-                if k not in ["autocommit", "block_size", "cache_options"]
-            }
-            return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
-        detail = self._check_file(path)
-        if detail:
-            detail, fn = detail
-            _, blocks = detail["fn"], detail["blocks"]
-            if blocks is True:
-                logger.debug("Opening local copy of %s", path)
-
-                # In order to support downstream filesystems to be able to
-                # infer the compression from the original filename, like
-                # the `TarFileSystem`, let's extend the `io.BufferedReader`
-                # fileobject protocol by adding a dedicated attribute
-                # `original`.
-                f = open(fn, mode)
-                f.original = detail.get("original")
-                return f
-            else:
-                raise ValueError(
-                    f"Attempt to open partially cached file {path}"
-                    f" as a wholly cached file"
-                )
-        else:
-            fn = self._make_local_details(path)
-        kwargs["mode"] = mode
-
+    def _get_cached_file_before_open(self, path, **kwargs):
+        fn = self._make_local_details(path)
         # call target filesystems open
         self._mkcache()
         if self.compression:
-            with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2:
+            with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2:
                 if isinstance(f, AbstractBufferedFile):
                     # want no type of caching if just downloading whole thing
                     f.cache = BaseCache(0, f.cache.fetcher, f.size)
@@ -735,7 +705,87 @@ class WholeFileCacheFileSystem(CachingFileSystem):
         else:
             self.fs.get_file(path, fn)
         self.save_cache()
-        return self._open(path, mode)
+
+    def _open(self, path, mode="rb", **kwargs):
+        path = self._strip_protocol(path)
+        # For read (or append), (try) download from remote
+        if "r" in mode or "a" in mode:
+            if not self._check_file(path):
+                if self.fs.exists(path):
+                    self._get_cached_file_before_open(path, **kwargs)
+                elif "r" in mode:
+                    raise FileNotFoundError(path)
+
+            detail, fn = self._check_file(path)
+            _, blocks = detail["fn"], detail["blocks"]
+            if blocks is True:
+                logger.debug("Opening local copy of %s", path)
+            else:
+                raise ValueError(
+                    f"Attempt to open partially cached file {path}"
+                    f" as a wholly cached file"
+                )
+
+        # Just reading does not need special file handling
+        if "r" in mode and "+" not in mode:
+            # In order to support downstream filesystems to be able to
+            # infer the compression from the original filename, like
+            # the `TarFileSystem`, let's extend the `io.BufferedReader`
+            # fileobject protocol by adding a dedicated attribute
+            # `original`.
+            f = open(fn, mode)
+            f.original = detail.get("original")
+            return f
+
+        hash = self._mapper(path)
+        fn = os.path.join(self.storage[-1], hash)
+        user_specified_kwargs = {
+            k: v
+            for k, v in kwargs.items()
+            # those kwargs were added by open(), we don't want them
+            if k not in ["autocommit", "block_size", "cache_options"]
+        }
+        return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
+
+    async def _cat_file(self, path, start=None, end=None, **kwargs):
+        logger.debug("async cat_file %s", path)
+        path = self._strip_protocol(path)
+        sha = self._mapper(path)
+        fn = self._check_file(path)
+
+        if not fn:
+            fn = os.path.join(self.storage[-1], sha)
+            await self.fs._get_file(path, fn, **kwargs)
+
+        with open(fn, "rb") as f:  # noqa ASYNC230
+            if start:
+                f.seek(start)
+            size = -1 if end is None else end - f.tell()
+            return f.read(size)
+
+    async def _cat_ranges(
+        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
+    ):
+        logger.debug("async cat ranges %s", paths)
+        lpaths = []
+        rset = set()
+        download = []
+        rpaths = []
+        for p in paths:
+            fn = self._check_file(p)
+            if fn is None and p not in rset:
+                sha = self._mapper(p)
+                fn = os.path.join(self.storage[-1], sha)
+                download.append(fn)
+                rset.add(p)
+                rpaths.append(p)
+            lpaths.append(fn)
+        if download:
+            await self.fs._get(rpaths, download, on_error=on_error)
+
+        return LocalFileSystem().cat_ranges(
+            lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
+        )
 
 
 class SimpleCacheFileSystem(WholeFileCacheFileSystem):
@@ -841,46 +891,6 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         else:
             raise ValueError("path must be str or dict")
 
-    async def _cat_file(self, path, start=None, end=None, **kwargs):
-        logger.debug("async cat_file %s", path)
-        path = self._strip_protocol(path)
-        sha = self._mapper(path)
-        fn = self._check_file(path)
-
-        if not fn:
-            fn = os.path.join(self.storage[-1], sha)
-            await self.fs._get_file(path, fn, **kwargs)
-
-        with open(fn, "rb") as f:  # noqa ASYNC230
-            if start:
-                f.seek(start)
-            size = -1 if end is None else end - f.tell()
-            return f.read(size)
-
-    async def _cat_ranges(
-        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
-    ):
-        logger.debug("async cat ranges %s", paths)
-        lpaths = []
-        rset = set()
-        download = []
-        rpaths = []
-        for p in paths:
-            fn = self._check_file(p)
-            if fn is None and p not in rset:
-                sha = self._mapper(p)
-                fn = os.path.join(self.storage[-1], sha)
-                download.append(fn)
-                rset.add(p)
-                rpaths.append(p)
-            lpaths.append(fn)
-        if download:
-            await self.fs._get(rpaths, download, on_error=on_error)
-
-        return LocalFileSystem().cat_ranges(
-            lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
-        )
-
     def cat_ranges(
         self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
     ):
@@ -894,37 +904,16 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
             paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
         )
 
-    def _open(self, path, mode="rb", **kwargs):
-        path = self._strip_protocol(path)
+    def _get_cached_file_before_open(self, path, **kwargs):
         sha = self._mapper(path)
-
-        if "r" not in mode:
-            fn = os.path.join(self.storage[-1], sha)
-            user_specified_kwargs = {
-                k: v
-                for k, v in kwargs.items()
-                if k not in ["autocommit", "block_size", "cache_options"]
-            }  # those were added by open()
-            return LocalTempFile(
-                self,
-                path,
-                mode=mode,
-                autocommit=not self._intrans,
-                fn=fn,
-                **user_specified_kwargs,
-            )
-        fn = self._check_file(path)
-        if fn:
-            return open(fn, mode)
-
         fn = os.path.join(self.storage[-1], sha)
         logger.debug("Copying %s to local cache", path)
-        kwargs["mode"] = mode
 
         self._mkcache()
         self._cache_size = None
+
         if self.compression:
-            with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2:
+            with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2:
                 if isinstance(f, AbstractBufferedFile):
                     # want no type of caching if just downloading whole thing
                     f.cache = BaseCache(0, f.cache.fetcher, f.size)
@@ -941,7 +930,39 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
                     f2.write(data)
         else:
             self.fs.get_file(path, fn)
-        return self._open(path, mode)
+
+    def _open(self, path, mode="rb", **kwargs):
+        path = self._strip_protocol(path)
+        sha = self._mapper(path)
+
+        # For read (or append), (try) download from remote
+        if "r" in mode or "a" in mode:
+            if not self._check_file(path):
+                # append does not require an existing file but read does
+                if self.fs.exists(path):
+                    self._get_cached_file_before_open(path, **kwargs)
+                elif "r" in mode:
+                    raise FileNotFoundError(path)
+
+        fn = self._check_file(path)
+        # Just reading does not need special file handling
+        if "r" in mode and "+" not in mode:
+            return open(fn, mode)
+
+        fn = os.path.join(self.storage[-1], sha)
+        user_specified_kwargs = {
+            k: v
+            for k, v in kwargs.items()
+            if k not in ["autocommit", "block_size", "cache_options"]
+        }  # those were added by open()
+        return LocalTempFile(
+            self,
+            path,
+            mode=mode,
+            autocommit=not self._intrans,
+            fn=fn,
+            **user_specified_kwargs,
+        )
 
 
 class LocalTempFile:

venv/lib/python3.12/site-packages/fsspec/implementations/dirfs.py

@@ -97,6 +97,9 @@ class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
     def rm(self, path, *args, **kwargs):
         return self.fs.rm(self._join(path), *args, **kwargs)
 
+    def delete(self, path, recursive=False, maxdepth=None):
+        return self.fs.delete(self._join(path), recursive=recursive, maxdepth=maxdepth)
+
     async def _cp_file(self, path1, path2, **kwargs):
         return await self.fs._cp_file(self._join(path1), self._join(path2), **kwargs)
 
@@ -137,6 +140,18 @@ class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
     def pipe_file(self, path, *args, **kwargs):
         return self.fs.pipe_file(self._join(path), *args, **kwargs)
 
+    def write_text(
+        self, path, value, encoding=None, errors=None, newline=None, **kwargs
+    ):
+        return self.fs.write_text(
+            self._join(path),
+            value,
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+            **kwargs,
+        )
+
     async def _cat_file(self, path, *args, **kwargs):
         return await self.fs._cat_file(self._join(path), *args, **kwargs)
 

venv/lib/python3.12/site-packages/fsspec/implementations/http.py

@@ -41,6 +41,9 @@ class HTTPFileSystem(AsyncFileSystem):
     match on the result. If simple_link=True, anything of the form
     "http(s)://server.com/stuff?thing=other"; otherwise only links within
     HTML href tags will be used.
+
+    URLs are passed unfiltered to aiohttp, so all addresses are accessible. Where URLs are
+    supplied by a user, the calling application may wish to filter to prevent scanning.
     """
 
     protocol = ("http", "https")
@@ -560,7 +563,9 @@ class HTTPFileSystem(AsyncFileSystem):
 
         session = await self.set_session()
 
-        async with session.put(url, data=value, headers=headers, **kwargs) as r:
+        async with session.put(
+            self.encode_url(url), data=value, headers=headers, **kwargs
+        ) as r:
             r.raise_for_status()
 
 

venv/lib/python3.12/site-packages/fsspec/implementations/local.py

@@ -204,8 +204,9 @@ class LocalFileSystem(AbstractFileSystem):
                 os.remove(p)
 
     def unstrip_protocol(self, name):
+        protocol = self.protocol if isinstance(self.protocol, str) else self.protocol[0]
         name = self._strip_protocol(name)  # normalise for local/win/...
-        return f"file://{name}"
+        return f"{protocol}://{name}"
 
     def _open(self, path, mode="rb", block_size=None, **kwargs):
         path = self._strip_protocol(path)
@@ -253,14 +254,12 @@ class LocalFileSystem(AbstractFileSystem):
     @classmethod
     def _strip_protocol(cls, path):
         path = stringify_path(path)
-        if path.startswith("file://"):
-            path = path[7:]
-        elif path.startswith("file:"):
-            path = path[5:]
-        elif path.startswith("local://"):
-            path = path[8:]
-        elif path.startswith("local:"):
-            path = path[6:]
+        protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol
+        prefixes = (protocol + sep for protocol in protos for sep in ("://", ":"))
+        for prefix in prefixes:
+            if path.startswith(prefix):
+                path = path.removeprefix(prefix)
+                break
 
         path = make_path_posix(path)
         if os.sep != "/":

venv/lib/python3.12/site-packages/fsspec/implementations/reference.py

@@ -157,6 +157,13 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         if self.engine == "pyarrow" and find_spec("pyarrow") is None:
             raise ImportError("engine choice `pyarrow` is not installed.")
 
+        # Apply `lru_cache` decorator manually per instance.
+        # This way `self` reference is not held on class level.
+        # WARNING: However, this means that self and its members are not reflected
+        # in the cache key, so we expect they won't be mutated once a value is cached.
+        self.listdir = lru_cache()(self.listdir)
+        self._key_to_record = lru_cache(maxsize=4096)(self._key_to_record)
+
     def __getattr__(self, item):
         if item in ("_items", "record_size", "zmetadata"):
             self.setup()
@@ -219,7 +226,6 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
         return LazyReferenceMapper(root, fs, **kwargs)
 
-    @lru_cache
     def listdir(self):
         """List top-level directories"""
         dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z"))
@@ -331,7 +337,6 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         # URL, offset, size
         return selection[:3]
 
-    @lru_cache(4096)
     def _key_to_record(self, key):
         """Details needed to construct a reference for one key"""
         field, chunk = key.rsplit("/", 1)

venv/lib/python3.12/site-packages/fsspec/implementations/sftp.py

@@ -135,8 +135,8 @@ class SFTPFileSystem(AbstractFileSystem):
             paths = [stat["name"] for stat in stats]
             return sorted(paths)
 
-    def put(self, lpath, rpath, callback=None, **kwargs):
-        rpath = self._strip_protocol(rpath)
+    def put_file(self, lpath, rpath, callback=None, **kwargs):
+        self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True)
         logger.debug("Put file %s into %s", lpath, rpath)
         self.ftp.put(lpath, rpath)
 

venv/lib/python3.12/site-packages/fsspec/implementations/tar.py

@@ -122,3 +122,14 @@ class TarFileSystem(AbstractArchiveFileSystem):
         if details["type"] != "file":
             raise ValueError("Can only handle regular files")
         return self.tar.extractfile(path)
+
+    def close(self):
+        """Commits any write changes to the file. Done on ``del`` too."""
+        self.tar.close()
+
+    def __del__(self):
+        if hasattr(self, "tar"):
+            self.close()
+            del self.tar
+        if hasattr(self, "of") and hasattr(self.of, "__exit__"):
+            self.of.__exit__(None, None, None)

venv/lib/python3.12/site-packages/fsspec/implementations/webhdfs.py

@@ -7,6 +7,7 @@ import shutil
 import tempfile
 import uuid
 from contextlib import suppress
+from datetime import datetime
 from urllib.parse import quote
 
 import requests
@@ -268,6 +269,23 @@ class WebHDFS(AbstractFileSystem):
         info["name"] = path
         return self._process_info(info)
 
+    def created(self, path):
+        """Return the created timestamp of a file as a datetime.datetime"""
+        # The API does not provide creation time, so we use modification time
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve creation time (modification time).")
+
+    def modified(self, path):
+        """Return the modified timestamp of a file as a datetime.datetime"""
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve modification time.")
+
     def ls(self, path, detail=False, **kwargs):
         out = self._call("LISTSTATUS", path=path)
         infos = out.json()["FileStatuses"]["FileStatus"]

venv/lib/python3.12/site-packages/fsspec/implementations/zip.py

@@ -78,6 +78,8 @@ class ZipFileSystem(AbstractArchiveFileSystem):
         if hasattr(self, "zip"):
             self.close()
             del self.zip
+        if hasattr(self, "of") and hasattr(self.of, "__exit__"):
+            self.of.__exit__(None, None, None)
 
     def close(self):
         """Commits any write changes to the file. Done on ``del`` too."""
@@ -138,14 +140,17 @@ class ZipFileSystem(AbstractArchiveFileSystem):
         if maxdepth is not None and maxdepth < 1:
             raise ValueError("maxdepth must be at least 1")
 
+        def to_parts(_path: str):
+            return list(filter(None, _path.replace("\\", "/").split("/")))
+
+        if not isinstance(path, str):
+            path = str(path)
+
         # Remove the leading slash, as the zip file paths are always
         # given without a leading slash
         path = path.lstrip("/")
-        path_parts = list(filter(lambda s: bool(s), path.split("/")))
-
-        def _matching_starts(file_path):
-            file_parts = filter(lambda s: bool(s), file_path.split("/"))
-            return all(a == b for a, b in zip(path_parts, file_parts))
+        path_parts = to_parts(path)
+        path_depth = len(path_parts)
 
         self._get_dirs()
 
@@ -157,21 +162,22 @@ class ZipFileSystem(AbstractArchiveFileSystem):
             return result if detail else [path]
 
         for file_path, file_info in self.dir_cache.items():
-            if not (path == "" or _matching_starts(file_path)):
+            if len(file_parts := to_parts(file_path)) < path_depth or any(
+                a != b for a, b in zip(path_parts, file_parts)
+            ):
+                # skip parent folders and mismatching paths
                 continue
 
             if file_info["type"] == "directory":
-                if withdirs:
-                    if file_path not in result:
-                        result[file_path.strip("/")] = file_info
+                if withdirs and file_path not in result:
+                    result[file_path.strip("/")] = file_info
                 continue
 
             if file_path not in result:
                 result[file_path] = file_info if detail else None
 
         if maxdepth:
-            path_depth = path.count("/")
             result = {
-                k: v for k, v in result.items() if k.count("/") - path_depth < maxdepth
+                k: v for k, v in result.items() if k.count("/") < maxdepth + path_depth
             }
         return result if detail else sorted(result)

venv/lib/python3.12/site-packages/fsspec/parquet.py

@@ -1,7 +1,6 @@
 import io
 import json
 import warnings
-from typing import Literal
 
 import fsspec
 
@@ -25,7 +24,6 @@ class AlreadyBufferedFile(AbstractBufferedFile):
 
 def open_parquet_files(
     path: list[str],
-    mode: Literal["rb"] = "rb",
     fs: None | fsspec.AbstractFileSystem = None,
     metadata=None,
     columns: None | list[str] = None,
@@ -54,8 +52,6 @@ def open_parquet_files(
     ----------
     path: str
         Target file path.
-    mode: str, optional
-        Mode option to be passed through to `fs.open`. Default is "rb".
     metadata: Any, optional
         Parquet metadata object. Object type must be supported
         by the backend parquet engine. For now, only the "fastparquet"
@@ -150,16 +146,16 @@ def open_parquet_files(
         AlreadyBufferedFile(
             fs=None,
             path=fn,
-            mode=mode,
+            mode="rb",
             cache_type="parts",
             cache_options={
                 **options,
-                "data": data.get(fn, {}),
+                "data": ranges,
             },
-            size=max(_[1] for _ in data.get(fn, {})),
+            size=max(_[1] for _ in ranges),
             **kwargs,
         )
-        for fn in data
+        for fn, ranges in data.items()
     ]
 
 
@@ -167,7 +163,7 @@ def open_parquet_file(*args, **kwargs):
     """Create files tailed to reading specific parts of parquet files
 
     Please see ``open_parquet_files`` for details of the arguments. The
-    difference is, this function always returns a single ``AleadyBufferedFile``,
+    difference is, this function always returns a single ``AlreadyBufferedFile``,
     whereas `open_parquet_files`` always returns a list of files, even if
     there are one or zero matching parquet files.
     """
@@ -197,7 +193,7 @@ def _get_parquet_byte_ranges(
     if isinstance(engine, str):
         engine = _set_engine(engine)
 
-    # Pass to specialized function if metadata is defined
+    # Pass to a specialized function if metadata is defined
     if metadata is not None:
         # Use the provided parquet metadata object
         # to avoid transferring/parsing footer metadata
@@ -212,63 +208,54 @@ def _get_parquet_byte_ranges(
             filters=filters,
         )
 
-    # Get file sizes asynchronously
-    file_sizes = fs.sizes(paths)
-
     # Populate global paths, starts, & ends
-    result = {}
-    data_paths = []
-    data_starts = []
-    data_ends = []
-    add_header_magic = True
     if columns is None and row_groups is None and filters is None:
         # We are NOT selecting specific columns or row-groups.
         #
         # We can avoid sampling the footers, and just transfer
         # all file data with cat_ranges
-        for i, path in enumerate(paths):
-            result[path] = {}
-            data_paths.append(path)
-            data_starts.append(0)
-            data_ends.append(file_sizes[i])
-        add_header_magic = False  # "Magic" should already be included
+        result = {path: {(0, len(data)): data} for path, data in fs.cat(paths).items()}
     else:
         # We ARE selecting specific columns or row-groups.
         #
+        # Get file sizes asynchronously
+        file_sizes = fs.sizes(paths)
+        data_paths = []
+        data_starts = []
+        data_ends = []
         # Gather file footers.
         # We just take the last `footer_sample_size` bytes of each
         # file (or the entire file if it is smaller than that)
-        footer_starts = []
-        footer_ends = []
-        for i, path in enumerate(paths):
-            footer_ends.append(file_sizes[i])
-            sample_size = max(0, file_sizes[i] - footer_sample_size)
-            footer_starts.append(sample_size)
-        footer_samples = fs.cat_ranges(paths, footer_starts, footer_ends)
+        footer_starts = [
+            max(0, file_size - footer_sample_size) for file_size in file_sizes
+        ]
+        footer_samples = fs.cat_ranges(paths, footer_starts, file_sizes)
 
         # Check our footer samples and re-sample if necessary.
-        missing_footer_starts = footer_starts.copy()
-        large_footer = 0
+        large_footer = []
         for i, path in enumerate(paths):
             footer_size = int.from_bytes(footer_samples[i][-8:-4], "little")
             real_footer_start = file_sizes[i] - (footer_size + 8)
             if real_footer_start < footer_starts[i]:
-                missing_footer_starts[i] = real_footer_start
-                large_footer = max(large_footer, (footer_size + 8))
+                large_footer.append((i, real_footer_start))
         if large_footer:
             warnings.warn(
                 f"Not enough data was used to sample the parquet footer. "
                 f"Try setting footer_sample_size >= {large_footer}."
             )
-            for i, block in enumerate(
-                fs.cat_ranges(
-                    paths,
-                    missing_footer_starts,
-                    footer_starts,
-                )
-            ):
+            path0 = [paths[i] for i, _ in large_footer]
+            starts = [_[1] for _ in large_footer]
+            ends = [file_sizes[i] - footer_sample_size for i, _ in large_footer]
+            data = fs.cat_ranges(path0, starts, ends)
+            for i, (path, start, block) in enumerate(zip(path0, starts, data)):
                 footer_samples[i] = block + footer_samples[i]
-                footer_starts[i] = missing_footer_starts[i]
+                footer_starts[i] = start
+        result = {
+            path: {(start, size): data}
+            for path, start, size, data in zip(
+                paths, footer_starts, file_sizes, footer_samples
+            )
+        }
 
         # Calculate required byte ranges for each path
         for i, path in enumerate(paths):
@@ -284,9 +271,6 @@ def _get_parquet_byte_ranges(
             data_paths += [path] * len(path_data_starts)
             data_starts += path_data_starts
             data_ends += path_data_ends
-            result.setdefault(path, {})[(footer_starts[i], file_sizes[i])] = (
-                footer_samples[i]
-            )
 
         # Merge adjacent offset ranges
         data_paths, data_starts, data_ends = merge_offset_ranges(
@@ -295,19 +279,14 @@ def _get_parquet_byte_ranges(
             data_ends,
             max_gap=max_gap,
             max_block=max_block,
-            sort=False,  # Should already be sorted
+            sort=True,
         )
 
-        # Start by populating `result` with footer samples
-        for i, path in enumerate(paths):
-            result[path] = {(footer_starts[i], footer_ends[i]): footer_samples[i]}
+        # Transfer the data byte-ranges into local memory
+        _transfer_ranges(fs, result, data_paths, data_starts, data_ends)
 
-    # Transfer the data byte-ranges into local memory
-    _transfer_ranges(fs, result, data_paths, data_starts, data_ends)
-
-    # Add b"PAR1" to header if necessary
-    if add_header_magic:
-        _add_header_magic(result)
+    # Add b"PAR1" to headers
+    _add_header_magic(result)
 
     return result
 
@@ -362,7 +341,7 @@ def _transfer_ranges(fs, blocks, paths, starts, ends):
 
 def _add_header_magic(data):
     # Add b"PAR1" to file headers
-    for path in list(data.keys()):
+    for path in list(data):
         add_magic = True
         for k in data[path]:
             if k[0] == 0 and k[1] >= 4:
@@ -419,9 +398,6 @@ class FastparquetEngine:
 
         self.fp = fp
 
-    def _row_group_filename(self, row_group, pf):
-        return pf.row_group_filename(row_group)
-
     def _parquet_byte_ranges(
         self,
         columns,
@@ -465,6 +441,10 @@ class FastparquetEngine:
             # Input row_groups contains row-group indices
             row_group_indices = row_groups
             row_groups = pf.row_groups
+        if column_set is not None:
+            column_set = [
+                _ if isinstance(_, list) else _.split(".") for _ in column_set
+            ]
 
         # Loop through column chunks to add required byte ranges
         for r, row_group in enumerate(row_groups):
@@ -472,13 +452,12 @@ class FastparquetEngine:
             # specific row-groups
             if row_group_indices is None or r in row_group_indices:
                 # Find the target parquet-file path for `row_group`
-                fn = self._row_group_filename(row_group, pf)
+                fn = pf.row_group_filename(row_group)
 
                 for column in row_group.columns:
-                    name = column.meta_data.path_in_schema[0]
-                    # Skip this column if we are targeting a
-                    # specific columns
-                    if column_set is None or name in column_set:
+                    name = column.meta_data.path_in_schema
+                    # Skip this column if we are targeting specific columns
+                    if column_set is None or _cmp(name, column_set):
                         file_offset0 = column.meta_data.dictionary_page_offset
                         if file_offset0 is None:
                             file_offset0 = column.meta_data.data_page_offset
@@ -512,9 +491,6 @@ class PyarrowEngine:
 
         self.pq = pq
 
-    def _row_group_filename(self, row_group, metadata):
-        raise NotImplementedError
-
     def _parquet_byte_ranges(
         self,
         columns,
@@ -527,6 +503,7 @@ class PyarrowEngine:
         if metadata is not None:
             raise ValueError("metadata input not supported for PyarrowEngine")
         if filters:
+            # there must be a way!
             raise NotImplementedError
 
         data_starts, data_ends = [], []
@@ -550,6 +527,10 @@ class PyarrowEngine:
                     if not isinstance(ind, dict)
                 ]
                 column_set |= set(md_index)
+        if column_set is not None:
+            column_set = [
+                _[:1] if isinstance(_, list) else _.split(".")[:1] for _ in column_set
+            ]
 
         # Loop through column chunks to add required byte ranges
         for r in range(md.num_row_groups):
@@ -559,22 +540,33 @@ class PyarrowEngine:
                 row_group = md.row_group(r)
                 for c in range(row_group.num_columns):
                     column = row_group.column(c)
-                    name = column.path_in_schema
-                    # Skip this column if we are targeting a
-                    # specific columns
-                    split_name = name.split(".")[0]
-                    if (
-                        column_set is None
-                        or name in column_set
-                        or split_name in column_set
-                    ):
-                        file_offset0 = column.dictionary_page_offset
-                        if file_offset0 is None:
-                            file_offset0 = column.data_page_offset
-                        num_bytes = column.total_compressed_size
+                    name = column.path_in_schema.split(".")
+                    # Skip this column if we are targeting specific columns
+                    if column_set is None or _cmp(name, column_set):
+                        meta = column.to_dict()
+                        # Any offset could be the first one
+                        file_offset0 = min(
+                            _
+                            for _ in [
+                                meta.get("dictionary_page_offset"),
+                                meta.get("data_page_offset"),
+                                meta.get("index_page_offset"),
+                            ]
+                            if _ is not None
+                        )
                         if file_offset0 < footer_start:
                             data_starts.append(file_offset0)
                             data_ends.append(
-                                min(file_offset0 + num_bytes, footer_start)
+                                min(
+                                    meta["total_compressed_size"] + file_offset0,
+                                    footer_start,
+                                )
                             )
+
+        data_starts.append(footer_start)
+        data_ends.append(footer_start + len(footer))
         return data_starts, data_ends
+
+
+def _cmp(name, column_set):
+    return any(all(a == b for a, b in zip(name, _)) for _ in column_set)

venv/lib/python3.12/site-packages/fsspec/registry.py

@@ -66,7 +66,11 @@ known_implementations = {
     },
     "adl": {
         "class": "adlfs.AzureDatalakeFileSystem",
-        "err": "Install adlfs to access Azure Datalake Gen1",
+        "err": (
+            "Azure Data Lake Storage Gen1 is retired and no longer supported. Please "
+            "install adlfs and use the `az://` protocol to access Azure Blob Storage "
+            "and Azure Data Lake Storage Gen2 instead."
+        ),
     },
     "arrow_hdfs": {
         "class": "fsspec.implementations.arrow.HadoopFileSystem",

venv/lib/python3.12/site-packages/fsspec/spec.py

@@ -611,6 +611,7 @@ class AbstractFileSystem(metaclass=_Cached):
         min_idx = min(idx_star, idx_qmark, idx_brace)
 
         detail = kwargs.pop("detail", False)
+        withdirs = kwargs.pop("withdirs", True)
 
         if not has_magic(path):
             if self.exists(path, **kwargs):
@@ -639,7 +640,9 @@ class AbstractFileSystem(metaclass=_Cached):
             else:
                 depth = None
 
-        allpaths = self.find(root, maxdepth=depth, withdirs=True, detail=True, **kwargs)
+        allpaths = self.find(
+            root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs
+        )
 
         pattern = glob_translate(path + ("/" if ends_with_sep else ""))
         pattern = re.compile(pattern)

venv/lib/python3.12/site-packages/fsspec/utils.py

@@ -566,6 +566,16 @@ def merge_offset_ranges(
                 )
             )
         )
+    remove = []
+    for i, (path, start, end) in enumerate(zip(paths, starts, ends)):
+        if any(
+            e is not None and p == path and start >= s and end <= e and i != i2
+            for i2, (p, s, e) in enumerate(zip(paths, starts, ends))
+        ):
+            remove.append(i)
+    paths = [p for i, p in enumerate(paths) if i not in remove]
+    starts = [s for i, s in enumerate(starts) if i not in remove]
+    ends = [e for i, e in enumerate(ends) if i not in remove]
 
     if paths:
         # Loop through the coupled `paths`, `starts`, and
@@ -587,7 +597,7 @@ def merge_offset_ranges(
                 new_starts.append(starts[i])
                 new_ends.append(ends[i])
             else:
-                # Merge with previous block by updating the
+                # Merge with the previous block by updating the
                 # last element of `ends`
                 new_ends[-1] = ends[i]
         return new_paths, new_starts, new_ends