Voice et bot modif

This commit is contained in:
pi 2026-06-16 17:09:34 +00:00
parent 189d56026b
commit 7333a22bcd
10774 changed files with 634644 additions and 933308 deletions

View file

@ -2,7 +2,7 @@
A platform independent file lock that supports the with-statement.
.. autodata:: filelock.__version__
:no-value:
:no-value:
"""
@ -14,7 +14,24 @@ from typing import TYPE_CHECKING
from ._api import AcquireReturnProxy, BaseFileLock
from ._error import Timeout
if TYPE_CHECKING:
from ._async_read_write import (
AsyncAcquireReadWriteReturnProxy,
AsyncReadWriteLock,
)
from ._read_write import ReadWriteLock
else:
try:
from ._async_read_write import AsyncAcquireReadWriteReturnProxy, AsyncReadWriteLock
from ._read_write import ReadWriteLock
except ImportError: # sqlite3 may be unavailable if Python was built without it or the C library is missing
AsyncAcquireReadWriteReturnProxy = None
AsyncReadWriteLock = None
ReadWriteLock = None
from ._soft import SoftFileLock
from ._soft_rw import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock, SoftReadWriteLock
from ._unix import UnixFileLock, has_fcntl
from ._windows import WindowsFileLock
from .asyncio import (
@ -54,15 +71,21 @@ else:
__all__ = [
"AcquireReturnProxy",
"AsyncAcquireReadWriteReturnProxy",
"AsyncAcquireReturnProxy",
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncFileLock",
"AsyncReadWriteLock",
"AsyncSoftFileLock",
"AsyncSoftReadWriteLock",
"AsyncUnixFileLock",
"AsyncWindowsFileLock",
"BaseAsyncFileLock",
"BaseFileLock",
"FileLock",
"ReadWriteLock",
"SoftFileLock",
"SoftReadWriteLock",
"Timeout",
"UnixFileLock",
"WindowsFileLock",

View file

@ -4,20 +4,30 @@ import contextlib
import inspect
import logging
import os
import sys
import time
import warnings
from abc import ABCMeta, abstractmethod
from dataclasses import dataclass
from threading import local
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any, TypeVar
from weakref import WeakValueDictionary
from ._error import Timeout
from ._util import break_lock_file
#: Sentinel indicating that no explicit file permission mode was passed.
#: When used, lock files are created with 0o666 (letting umask and default ACLs control the final permissions)
#: and fchmod is skipped so that POSIX default ACL inheritance is preserved.
_UNSET_FILE_MODE: int = -1
if TYPE_CHECKING:
import sys
from collections.abc import Callable
from types import TracebackType
from ._read_write import ReadWriteLock
from ._soft_rw import SoftReadWriteLock
if sys.version_info >= (3, 11): # pragma: no cover (py311+)
from typing import Self
else: # pragma: no cover (<py311)
@ -26,6 +36,19 @@ if TYPE_CHECKING:
_LOGGER = logging.getLogger("filelock")
# On Windows os.path.realpath calls CreateFileW with share_mode=0, which blocks concurrent DeleteFileW and causes
# livelocks under threaded contention with SoftFileLock. os.path.abspath is purely string-based and avoids this.
_canonical = os.path.abspath if sys.platform == "win32" else os.path.realpath
class _ThreadLocalRegistry(local):
def __init__(self) -> None:
super().__init__()
self.held: dict[str, int] = {}
_registry = _ThreadLocalRegistry()
# This is a helper class which is returned by :meth:`BaseFileLock.acquire` and wraps the lock to make sure __enter__
# is not called twice when entering the with statement. If we would simply return *self*, the lock would be acquired
@ -33,10 +56,10 @@ _LOGGER = logging.getLogger("filelock")
class AcquireReturnProxy:
"""A context-aware object that will release the lock file when exiting."""
def __init__(self, lock: BaseFileLock) -> None:
self.lock = lock
def __init__(self, lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock) -> None:
self.lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock = lock
def __enter__(self) -> BaseFileLock:
def __enter__(self) -> BaseFileLock | ReadWriteLock | SoftReadWriteLock:
return self.lock
def __exit__(
@ -67,6 +90,12 @@ class FileLockContext:
#: Whether the lock should be blocking or not
blocking: bool
#: The default polling interval value.
poll_interval: float
#: The lock lifetime in seconds; ``None`` means the lock never expires.
lifetime: float | None = None
#: The file descriptor for the *_lock_file* as it is returned by the os.open() function, not None when lock held
lock_file_fd: int | None = None
@ -78,26 +107,35 @@ class ThreadLocalFileContext(FileLockContext, local):
"""A thread local version of the ``FileLockContext`` class."""
_T = TypeVar("_T", bound="BaseFileLock")
class FileLockMeta(ABCMeta):
_instances: WeakValueDictionary[str, BaseFileLock]
def __call__( # noqa: PLR0913
cls,
cls: type[_T],
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = 0o644,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = True, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
**kwargs: Any, # capture remaining kwargs for subclasses # noqa: ANN401
) -> BaseFileLock:
) -> _T:
if is_singleton:
instance = cls._instances.get(str(lock_file)) # type: ignore[attr-defined]
instance = cls._instances.get(str(lock_file))
if instance:
params_to_check = {
"thread_local": (thread_local, instance.is_thread_local()),
"timeout": (timeout, instance.timeout),
"mode": (mode, instance.mode),
"mode": (mode, instance._context.mode), # noqa: SLF001
"blocking": (blocking, instance.blocking),
"poll_interval": (poll_interval, instance.poll_interval),
"lifetime": (lifetime, instance.lifetime),
}
non_matching_params = {
@ -106,7 +144,7 @@ class FileLockMeta(ABCMeta):
if passed_param != set_param
}
if not non_matching_params:
return cast("BaseFileLock", instance)
return instance # ty: ignore[invalid-return-type] # https://github.com/astral-sh/ty/issues/3231
# parameters do not match; raise error
msg = "Singleton lock instances cannot be initialized with differing arguments"
@ -125,22 +163,31 @@ class FileLockMeta(ABCMeta):
"thread_local": thread_local,
"blocking": blocking,
"is_singleton": is_singleton,
"poll_interval": poll_interval,
"lifetime": lifetime,
**kwargs,
}
present_params = inspect.signature(cls.__init__).parameters # type: ignore[misc]
present_params = inspect.signature(cls.__init__).parameters
init_params = {key: value for key, value in all_params.items() if key in present_params}
instance = super().__call__(lock_file, **init_params)
if is_singleton:
cls._instances[str(lock_file)] = instance # type: ignore[attr-defined]
cls._instances[str(lock_file)] = instance
return cast("BaseFileLock", instance)
return instance
class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
"""Abstract base class for a file lock object."""
"""
Abstract base class for a file lock object.
Provides a reentrant, cross-process exclusive lock backed by OS-level primitives. Subclasses implement the actual
locking mechanism (:class:`UnixFileLock <filelock.UnixFileLock>`, :class:`WindowsFileLock
<filelock.WindowsFileLock>`, :class:`SoftFileLock <filelock.SoftFileLock>`).
"""
_instances: WeakValueDictionary[str, BaseFileLock]
@ -153,26 +200,39 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = 0o644,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = True, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
) -> None:
"""
Create a new lock object.
:param lock_file: path to the file
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in \
the acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it \
to a negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
:param mode: file permissions for the lockfile
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to \
``False`` then the lock will be reentrant across threads.
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the
acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a
negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
:param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and
default ACLs, preserving POSIX default ACL inheritance in shared directories.
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to
``False`` then the lock will be reentrant across threads. When ``True`` (the default), **all fields of the
lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``,
``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change
the value seen by another thread; threads that did not perform the write continue to see the value supplied
at construction time. If you need configuration values to be visible across threads, construct the lock
with ``thread_local=False``.
:param blocking: whether the lock should be blocking or not
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created \
per lock file. This is useful if you want to use the lock object for reentrant locking without needing \
to pass the same object around.
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock
file. This is useful if you want to use the lock object for reentrant locking without needing to pass the
same object around.
:param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value
in the acquire method, if no poll_interval value (``None``) is given.
:param lifetime: maximum time in seconds a lock can be held before it is considered expired. When set, a waiting
process will break a lock whose file modification time is older than ``lifetime`` seconds. ``None`` (the
default) means locks never expire.
"""
self._is_thread_local = thread_local
@ -185,29 +245,37 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
"timeout": timeout,
"mode": mode,
"blocking": blocking,
"poll_interval": poll_interval,
"lifetime": lifetime,
}
self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)(**kwargs)
def is_thread_local(self) -> bool:
""":return: a flag indicating if this lock is thread local or not"""
""":returns: a flag indicating if this lock is thread local or not"""
return self._is_thread_local
@property
def is_singleton(self) -> bool:
""":return: a flag indicating if this lock is singleton or not"""
"""
:returns: a flag indicating if this lock is singleton or not
.. versionadded:: 3.13.0
"""
return self._is_singleton
@property
def lock_file(self) -> str:
""":return: path to the lock file"""
""":returns: path to the lock file"""
return self._context.lock_file
@property
def timeout(self) -> float:
"""
:return: the default timeout value, in seconds
:returns: the default timeout value, in seconds
.. versionadded:: 2.0.0
"""
return self._context.timeout
@ -223,7 +291,12 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
@property
def blocking(self) -> bool:
""":return: whether the locking is blocking or not"""
"""
:returns: whether the locking is blocking or not
.. versionadded:: 3.14.0
"""
return self._context.blocking
@blocking.setter
@ -236,10 +309,73 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
"""
self._context.blocking = value
@property
def poll_interval(self) -> float:
"""
:returns: the default polling interval, in seconds
.. versionadded:: 3.24.0
"""
return self._context.poll_interval
@poll_interval.setter
def poll_interval(self, value: float) -> None:
"""
Change the default polling interval.
:param value: the new value, in seconds
"""
self._context.poll_interval = value
@property
def lifetime(self) -> float | None:
"""
:returns: the lock lifetime in seconds, or ``None`` if the lock never expires
.. versionadded:: 3.24.0
"""
return self._context.lifetime
@lifetime.setter
def lifetime(self, value: float | None) -> None:
"""
Change the lock lifetime.
:param value: the new value in seconds, or ``None`` to disable expiration
"""
self._context.lifetime = value
@property
def mode(self) -> int:
""":return: the file permissions for the lockfile"""
return self._context.mode
""":returns: the file permissions for the lockfile"""
return 0o644 if self._context.mode == _UNSET_FILE_MODE else self._context.mode
@property
def has_explicit_mode(self) -> bool:
""":returns: whether the file permissions were explicitly set"""
return self._context.mode != _UNSET_FILE_MODE
def _open_mode(self) -> int:
""":returns: the mode for os.open() — 0o666 when unset (let umask/ACLs decide), else the explicit mode"""
return 0o666 if self._context.mode == _UNSET_FILE_MODE else self._context.mode
def _try_break_expired_lock(self) -> None:
"""Remove the lock file if its modification time exceeds the configured :attr:`lifetime`."""
if (lifetime := self._context.lifetime) is None:
return
with contextlib.suppress(OSError):
# lstat, not stat: an attacker with write access to the lock directory can replace a held
# lock file with a symlink pointing at an old file, making stat() report the target's stale
# mtime so a waiter breaks a live lock and two processes hold it at once. lstat reads the
# symlink's own mtime, matching the O_NOFOLLOW reads elsewhere.
mtime = os.lstat(self.lock_file).st_mtime
if time.time() - mtime < lifetime:
return
break_lock_file(self.lock_file, mtime)
@abstractmethod
def _acquire(self) -> None:
@ -254,39 +390,66 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
@property
def is_locked(self) -> bool:
"""
:return: A boolean indicating if the lock file is holding the lock currently.
:returns: A boolean indicating if the lock file is holding the lock currently.
.. versionchanged:: 2.0.0
This was previously a method and is now a property.
"""
return self._context.lock_file_fd is not None
@property
def lock_counter(self) -> int:
""":return: The number of times this lock has been acquired (but not yet released)."""
""":returns: The number of times this lock has been acquired (but not yet released)."""
return self._context.lock_counter
@staticmethod
def _check_give_up( # noqa: PLR0913
lock_id: int,
lock_filename: str,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
start_time: float,
) -> bool:
if blocking is False:
_LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename)
return True
if cancel_check is not None and cancel_check():
_LOGGER.debug("Cancellation requested for lock %s on %s", lock_id, lock_filename)
return True
if 0 <= timeout < time.perf_counter() - start_time:
_LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename)
return True
return False
def acquire(
self,
timeout: float | None = None,
poll_interval: float = 0.05,
poll_interval: float | None = None,
*,
poll_intervall: float | None = None,
blocking: bool | None = None,
cancel_check: Callable[[], bool] | None = None,
) -> AcquireReturnProxy:
"""
Try to acquire the file lock.
:param timeout: maximum wait time for acquiring the lock, ``None`` means use the default :attr:`~timeout` is and
if ``timeout < 0``, there is no timeout and this method will block until the lock could be acquired
:param poll_interval: interval of trying to acquire the lock file
if ``timeout < 0``, there is no timeout and this method will block until the lock could be acquired
:param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default
:attr:`~poll_interval`
:param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead
:param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the
first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.
first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.
:param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll
iteration. When triggered, raises :class:`~Timeout` just like an expired timeout.
:returns: a context object that will unlock the file when the context is exited
:raises Timeout: if fails to acquire lock within the timeout period
:return: a context object that will unlock the file when the context is exited
.. code-block:: python
@ -303,8 +466,8 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
.. versionchanged:: 2.0.0
This method returns now a *proxy* object instead of *self*,
so that it can be used in a with statement without side effects.
This method returns now a *proxy* object instead of *self*, so that it can be used in a with statement
without side effects.
"""
# Use the default timeout, if no timeout is provided.
@ -319,40 +482,81 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
warnings.warn(msg, DeprecationWarning, stacklevel=2)
poll_interval = poll_intervall
poll_interval = poll_interval if poll_interval is not None else self._context.poll_interval
# Increment the number right at the beginning. We can still undo it, if something fails.
self._context.lock_counter += 1
lock_id = id(self)
lock_filename = self.lock_file
canonical = _canonical(lock_filename)
would_block = self._context.lock_counter == 1 and not self.is_locked and timeout < 0 and blocking
if would_block and (existing := _registry.held.get(canonical)) is not None and existing != lock_id:
self._context.lock_counter -= 1
msg = (
f"Deadlock: lock '{lock_filename}' is already held by a different "
f"FileLock instance in this thread. Use is_singleton=True to "
f"enable reentrant locking across instances."
)
raise RuntimeError(msg)
start_time = time.perf_counter()
try:
while True:
if not self.is_locked:
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
self._acquire()
if self.is_locked:
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
break
if blocking is False:
_LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename)
raise Timeout(lock_filename) # noqa: TRY301
if 0 <= timeout < time.perf_counter() - start_time:
_LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename)
raise Timeout(lock_filename) # noqa: TRY301
msg = "Lock %s not acquired on %s, waiting %s seconds ..."
_LOGGER.debug(msg, lock_id, lock_filename, poll_interval)
time.sleep(poll_interval)
except BaseException: # Something did go wrong, so decrement the counter.
self._poll_until_acquired(
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
poll_interval=poll_interval,
start_time=start_time,
)
except BaseException:
self._context.lock_counter = max(0, self._context.lock_counter - 1)
if self._context.lock_counter == 0:
_registry.held.pop(canonical, None)
raise
if self._context.lock_counter == 1:
_registry.held[canonical] = lock_id
return AcquireReturnProxy(lock=self)
def _poll_until_acquired(
self,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
poll_interval: float,
start_time: float,
) -> None:
lock_id = id(self)
lock_filename = self.lock_file
while True:
if not self.is_locked:
self._try_break_expired_lock()
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
self._acquire()
if self.is_locked:
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
return
if self._check_give_up(
lock_id,
lock_filename,
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
start_time=start_time,
):
raise Timeout(lock_filename)
msg = "Lock %s not acquired on %s, waiting %s seconds ..."
_LOGGER.debug(msg, lock_id, lock_filename, poll_interval)
time.sleep(poll_interval)
def release(self, force: bool = False) -> None: # noqa: FBT001, FBT002
"""
Releases the file lock. Please note, that the lock is only completely released, if the lock counter is 0.
Also note, that the lock file itself is not automatically deleted.
Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file
itself may be deleted automatically, the behavior is platform-specific.
:param force: If true, the lock counter is ignored and the lock is released in every case/
:param force: If true, the lock counter is ignored and the lock is released in every case.
"""
if self.is_locked:
@ -364,13 +568,14 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
_LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename)
self._release()
self._context.lock_counter = 0
_registry.held.pop(_canonical(lock_filename), None)
_LOGGER.debug("Lock %s released on %s", lock_id, lock_filename)
def __enter__(self) -> Self:
"""
Acquire the lock.
:return: the lock object
:returns: the lock object
"""
self.acquire()
@ -398,6 +603,7 @@ class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
__all__ = [
"_UNSET_FILE_MODE",
"AcquireReturnProxy",
"BaseFileLock",
]

View file

@ -0,0 +1,216 @@
"""Async wrapper around :class:`ReadWriteLock` for use with ``asyncio``."""
from __future__ import annotations
import asyncio
import functools
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING
from ._read_write import ReadWriteLock
if TYPE_CHECKING:
import os
from collections.abc import AsyncGenerator, Callable
from concurrent import futures
from types import TracebackType
class AsyncAcquireReadWriteReturnProxy:
"""Context-aware object that releases the async read/write lock on exit."""
def __init__(self, lock: AsyncReadWriteLock) -> None:
self.lock = lock
async def __aenter__(self) -> AsyncReadWriteLock:
return self.lock
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock.release()
class AsyncReadWriteLock:
"""
Async wrapper around :class:`ReadWriteLock` for use in ``asyncio`` applications.
Because Python's :mod:`sqlite3` module has no async API, all blocking SQLite operations are dispatched to a thread
pool via ``loop.run_in_executor()``. Reentrancy, upgrade/downgrade rules, and singleton behavior are delegated
to the underlying :class:`ReadWriteLock`.
:param lock_file: path to the SQLite database file used as the lock
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:param is_singleton: if ``True``, reuse existing :class:`ReadWriteLock` instances for the same resolved path
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
:param executor: executor for ``run_in_executor``. When ``None`` a dedicated single-thread executor is created
and owned by this lock, ensuring every operation runs on the same thread (required for SQLite affinity); it
is shut down by :meth:`close`. A caller-supplied executor is used as-is and never shut down here, so when no
executor is passed remember to call :meth:`close` to release the owned one.
.. versionadded:: 3.21.0
"""
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
loop: asyncio.AbstractEventLoop | None = None,
executor: futures.Executor | None = None,
) -> None:
self._lock = ReadWriteLock(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
self._loop = loop
self._owns_executor = executor is None
self._executor = executor or ThreadPoolExecutor(max_workers=1)
@property
def lock_file(self) -> str:
""":returns: the path to the lock file."""
return self._lock.lock_file
@property
def timeout(self) -> float:
""":returns: the default timeout."""
return self._lock.timeout
@property
def blocking(self) -> bool:
""":returns: whether blocking is enabled by default."""
return self._lock.blocking
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
""":returns: the event loop (or ``None`` for the running loop)."""
return self._loop
@property
def executor(self) -> futures.Executor:
""":returns: the executor used for ``run_in_executor`` (a dedicated single-thread one if none was supplied)."""
return self._executor
async def _run(self, func: Callable[..., object], *args: object, **kwargs: object) -> object:
loop = self._loop or asyncio.get_running_loop()
return await loop.run_in_executor(self._executor, functools.partial(func, *args, **kwargs))
async def acquire_read(self, timeout: float = -1, *, blocking: bool = True) -> AsyncAcquireReadWriteReturnProxy:
"""
Acquire a shared read lock.
See :meth:`ReadWriteLock.acquire_read` for full semantics.
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as an async context manager to release the lock
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_read, timeout, blocking=blocking)
return AsyncAcquireReadWriteReturnProxy(lock=self)
async def acquire_write(self, timeout: float = -1, *, blocking: bool = True) -> AsyncAcquireReadWriteReturnProxy:
"""
Acquire an exclusive write lock.
See :meth:`ReadWriteLock.acquire_write` for full semantics.
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as an async context manager to release the lock
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_write, timeout, blocking=blocking)
return AsyncAcquireReadWriteReturnProxy(lock=self)
async def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
See :meth:`ReadWriteLock.release` for full semantics.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
await self._run(self._lock.release, force=force)
@asynccontextmanager
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self._lock.timeout
if blocking is None:
blocking = self._lock.blocking
await self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
@asynccontextmanager
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self._lock.timeout
if blocking is None:
blocking = self._lock.blocking
await self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
async def close(self) -> None:
"""
Release the lock (if held) and close the underlying SQLite connection.
After calling this method, the lock instance is no longer usable.
"""
await self._run(self._lock.close)
if self._owns_executor:
self._executor.shutdown(wait=False)
def __del__(self) -> None:
# Safety net: if close() was never called, still shut down the executor we created so its worker thread does
# not outlive the lock. A caller-supplied executor is left untouched. shutdown(wait=False) never blocks.
if getattr(self, "_owns_executor", False):
self._executor.shutdown(wait=False)
__all__ = [
"AsyncAcquireReadWriteReturnProxy",
"AsyncReadWriteLock",
]

View file

@ -1,7 +1,5 @@
from __future__ import annotations
from typing import Any
class Timeout(TimeoutError): # noqa: N818
"""Raised when the lock could not be acquired in *timeout* seconds."""
@ -10,7 +8,7 @@ class Timeout(TimeoutError): # noqa: N818
super().__init__()
self._lock_file = lock_file
def __reduce__(self) -> str | tuple[Any, ...]:
def __reduce__(self) -> tuple[type[Timeout], tuple[str]]:
return self.__class__, (self._lock_file,) # Properly pickle the exception
def __str__(self) -> str:
@ -21,7 +19,7 @@ class Timeout(TimeoutError): # noqa: N818
@property
def lock_file(self) -> str:
""":return: The path of the file lock."""
""":returns: The path of the file lock."""
return self._lock_file

View file

@ -0,0 +1,369 @@
from __future__ import annotations
import atexit
import logging
import os
import pathlib
import sqlite3
import threading
import time
from contextlib import contextmanager, suppress
from typing import TYPE_CHECKING, Literal
from weakref import WeakValueDictionary
from ._api import AcquireReturnProxy
from ._error import Timeout
if TYPE_CHECKING:
from collections.abc import Generator
_LOGGER = logging.getLogger("filelock")
_all_connections: set[sqlite3.Connection] = set()
_all_connections_lock = threading.Lock()
def _cleanup_connections() -> None:
with _all_connections_lock:
for con in list(_all_connections):
with suppress(Exception):
con.close()
_all_connections.clear()
atexit.register(_cleanup_connections)
# sqlite3_busy_timeout() accepts a C int, max 2_147_483_647 on 32-bit. Use a lower value to be safe (~23 days).
_MAX_SQLITE_TIMEOUT_MS = 2_000_000_000 - 1
def timeout_for_sqlite(timeout: float, *, blocking: bool, already_waited: float) -> int:
if blocking is False:
return 0
if timeout == -1:
return _MAX_SQLITE_TIMEOUT_MS
if timeout < 0:
msg = "timeout must be a non-negative number or -1"
raise ValueError(msg)
remaining = max(timeout - already_waited, 0) if timeout > 0 else timeout
timeout_ms = int(remaining * 1000)
if timeout_ms > _MAX_SQLITE_TIMEOUT_MS or timeout_ms < 0:
_LOGGER.warning("timeout %s is too large for SQLite, using %s ms instead", timeout, _MAX_SQLITE_TIMEOUT_MS)
return _MAX_SQLITE_TIMEOUT_MS
return timeout_ms
class _ReadWriteLockMeta(type):
"""
Metaclass that handles singleton resolution when is_singleton=True.
Singleton logic lives here rather than in ReadWriteLock.get_lock so that ``ReadWriteLock(path)`` transparently
returns cached instances without a 2-arg ``super()`` call that type checkers cannot verify.
"""
_instances: WeakValueDictionary[pathlib.Path, ReadWriteLock]
_instances_lock: threading.Lock
def __call__(
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
) -> ReadWriteLock:
if not is_singleton:
return super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
normalized = pathlib.Path(lock_file).resolve()
with cls._instances_lock:
if normalized not in cls._instances:
instance = super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
cls._instances[normalized] = instance
else:
instance = cls._instances[normalized]
if instance.timeout != timeout or instance.blocking != blocking:
msg = (
f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking},"
f" cannot be changed to timeout={timeout}, blocking={blocking}"
)
raise ValueError(msg)
return instance
class ReadWriteLock(metaclass=_ReadWriteLockMeta):
"""
Cross-process read-write lock backed by SQLite.
Allows concurrent shared readers or a single exclusive writer. The lock is reentrant within the same mode (multiple
``acquire_read`` calls nest, as do multiple ``acquire_write`` calls from the same thread), but upgrading from read
to write or downgrading from write to read raises :class:`RuntimeError`. Write locks are pinned to the thread that
acquired them.
By default, ``is_singleton=True``: calling ``ReadWriteLock(path)`` with the same resolved path returns the same
instance. The lock file must use a ``.db`` extension (SQLite database).
:param lock_file: path to the SQLite database file used as the lock
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:param is_singleton: if ``True``, reuse existing instances for the same resolved path
.. versionadded:: 3.21.0
"""
_instances: WeakValueDictionary[pathlib.Path, ReadWriteLock] = WeakValueDictionary()
_instances_lock = threading.Lock()
@classmethod
def get_lock(
cls, lock_file: str | os.PathLike[str], timeout: float = -1, *, blocking: bool = True
) -> ReadWriteLock:
"""
Return the singleton :class:`ReadWriteLock` for *lock_file*.
:param lock_file: path to the SQLite database file used as the lock
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: the singleton lock instance
:raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values
"""
return cls(lock_file, timeout, blocking=blocking)
def __init__(
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True, # noqa: ARG002 # consumed by _ReadWriteLockMeta.__call__
) -> None:
self.lock_file = os.fspath(lock_file)
self.timeout = timeout
self.blocking = blocking
self._transaction_lock = threading.Lock() # serializes the (possibly blocking) SQLite transaction work
self._internal_lock = threading.Lock() # protects _lock_level / _current_mode updates and rollback
self._lock_level = 0
self._current_mode: Literal["read", "write"] | None = None
self._write_thread_id: int | None = None
self._con = sqlite3.connect(self.lock_file, check_same_thread=False)
with _all_connections_lock:
_all_connections.add(self._con)
def _acquire_transaction_lock(self, *, blocking: bool, timeout: float) -> None:
if not blocking:
acquired = self._transaction_lock.acquire(blocking=False)
elif timeout == -1:
acquired = self._transaction_lock.acquire(blocking=True)
else:
acquired = self._transaction_lock.acquire(blocking=True, timeout=timeout)
if not acquired:
raise Timeout(self.lock_file) from None
def _validate_reentrant(self, mode: Literal["read", "write"]) -> AcquireReturnProxy:
opposite = "write" if mode == "read" else "read"
direction = "downgrade" if mode == "read" else "upgrade"
if self._current_mode != mode:
msg = (
f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): "
f"already holding a {opposite} lock ({direction} not allowed)"
)
raise RuntimeError(msg)
if mode == "write" and (cur := threading.get_ident()) != self._write_thread_id:
msg = (
f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) "
f"from thread {cur} while it is held by thread {self._write_thread_id}"
)
raise RuntimeError(msg)
self._lock_level += 1
return AcquireReturnProxy(lock=self)
def _configure_and_begin(
self, mode: Literal["read", "write"], timeout: float, *, blocking: bool, start_time: float
) -> None:
waited = time.perf_counter() - start_time
timeout_ms = timeout_for_sqlite(timeout, blocking=blocking, already_waited=waited)
self._con.execute(f"PRAGMA busy_timeout={timeout_ms};").close()
# Use legacy journal mode (not WAL) because WAL does not block readers when a concurrent EXCLUSIVE
# write transaction is active, making read-write locking impossible without modifying table data.
# MEMORY is safe here since no actual writes happen — crashes cannot corrupt the DB.
# See https://sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions
#
# Set here (not in __init__) because this pragma itself may block on a locked database,
# so it must run after busy_timeout is configured above.
self._con.execute("PRAGMA journal_mode=MEMORY;").close()
# Recompute remaining timeout after the potentially blocking journal_mode pragma.
waited = time.perf_counter() - start_time
if (recomputed := timeout_for_sqlite(timeout, blocking=blocking, already_waited=waited)) != timeout_ms:
self._con.execute(f"PRAGMA busy_timeout={recomputed};").close()
stmt = "BEGIN EXCLUSIVE TRANSACTION;" if mode == "write" else "BEGIN TRANSACTION;"
self._con.execute(stmt).close()
if mode == "read":
# A SELECT is needed to force SQLite to actually acquire the SHARED lock on the database.
# https://www.sqlite.org/lockingv3.html#transaction_control
self._con.execute("SELECT name FROM sqlite_schema LIMIT 1;").close()
def _acquire(self, mode: Literal["read", "write"], timeout: float, *, blocking: bool) -> AcquireReturnProxy:
with self._internal_lock:
if self._lock_level > 0:
return self._validate_reentrant(mode)
start_time = time.perf_counter()
self._acquire_transaction_lock(blocking=blocking, timeout=timeout)
try:
return self._do_acquire_inner(mode, timeout, blocking=blocking, start_time=start_time)
except sqlite3.OperationalError as exc:
if "database is locked" not in str(exc):
raise
raise Timeout(self.lock_file) from None
finally:
self._transaction_lock.release()
def _do_acquire_inner(
self,
mode: Literal["read", "write"],
timeout: float,
*,
blocking: bool,
start_time: float,
) -> AcquireReturnProxy:
# Double-check: another thread may have acquired the lock while we waited on _transaction_lock.
with self._internal_lock:
if self._lock_level > 0:
return self._validate_reentrant(mode)
self._configure_and_begin(mode, timeout, blocking=blocking, start_time=start_time)
with self._internal_lock:
self._current_mode = mode
self._lock_level = 1
if mode == "write":
self._write_thread_id = threading.get_ident()
return AcquireReturnProxy(lock=self)
def acquire_read(self, timeout: float = -1, *, blocking: bool = True) -> AcquireReturnProxy:
"""
Acquire a shared read lock.
If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a
read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed).
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("read", timeout, blocking=blocking)
def acquire_write(self, timeout: float = -1, *, blocking: bool = True) -> AcquireReturnProxy:
"""
Acquire an exclusive write lock.
If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant).
Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not allowed).
Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises
:class:`RuntimeError`.
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("write", timeout, blocking=blocking)
def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
When the lock level reaches zero the underlying SQLite transaction is rolled back, releasing the database lock.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
should_rollback = False
with self._internal_lock:
if self._lock_level == 0:
if force:
return
msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held"
raise RuntimeError(msg)
if force:
self._lock_level = 0
else:
self._lock_level -= 1
if self._lock_level == 0:
self._current_mode = None
self._write_thread_id = None
should_rollback = True
if should_rollback:
self._con.rollback()
@contextmanager
def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self.timeout
if blocking is None:
blocking = self.blocking
self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
self.release()
@contextmanager
def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self.timeout
if blocking is None:
blocking = self.blocking
self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
self.release()
def close(self) -> None:
"""
Release the lock (if held) and close the underlying SQLite connection.
After calling this method, the lock instance is no longer usable.
"""
self.release(force=True)
self._con.close()
with _all_connections_lock:
_all_connections.discard(self._con)

View file

@ -1,17 +1,36 @@
from __future__ import annotations
import os
import socket
import sys
import time
from contextlib import suppress
from errno import EACCES, EEXIST
from errno import EACCES, EEXIST, EPERM, ESRCH
from pathlib import Path
from ._api import BaseFileLock
from ._util import ensure_directory_exists, raise_on_not_writable_file
from ._util import break_lock_file, ensure_directory_exists, raise_on_not_writable_file
_WIN_SYNCHRONIZE = 0x100000
_WIN_ERROR_INVALID_PARAMETER = 87
_WIN_PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
_MALFORMED_LOCK_AGE_THRESHOLD = 2.0
_MAX_LOCK_FILE_SIZE = 1024
class SoftFileLock(BaseFileLock):
"""Simply watches the existence of the lock file."""
"""
Portable file lock based on file existence.
Unlike :class:`UnixFileLock <filelock.UnixFileLock>` and :class:`WindowsFileLock <filelock.WindowsFileLock>`, this
lock does not use OS-level locking primitives. Instead, it creates the lock file with ``O_CREAT | O_EXCL`` and
treats its existence as the lock indicator. This makes it work on any filesystem but leaves stale lock files behind
if the process crashes without releasing the lock.
To mitigate stale locks, the lock file contains the PID and hostname of the holding process. On contention, if the
holder is on the same host and its PID no longer exists, the stale lock is broken automatically.
"""
def _acquire(self) -> None:
raise_on_not_writable_file(self.lock_file)
@ -22,26 +41,201 @@ class SoftFileLock(BaseFileLock):
| os.O_EXCL # together with above raise EEXIST if the file specified by filename exists
| os.O_TRUNC # truncate the file to zero byte
)
o_nofollow = getattr(os, "O_NOFOLLOW", None)
if o_nofollow is not None:
if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None:
flags |= o_nofollow
try:
file_handler = os.open(self.lock_file, flags, self._context.mode)
except OSError as exception: # re-raise unless expected exception
file_handler = os.open(self.lock_file, flags, self._open_mode())
except OSError as exception:
if not (
exception.errno == EEXIST # lock already exist
or (exception.errno == EACCES and sys.platform == "win32") # has no access to this lock
exception.errno == EEXIST or (exception.errno == EACCES and sys.platform == "win32")
): # pragma: win32 no cover
raise
self._try_break_stale_lock()
else:
self._write_lock_info(file_handler)
self._context.lock_file_fd = file_handler
def _try_break_stale_lock(self) -> None:
with suppress(OSError, ValueError):
content, mtime = _read_lock_file(self.lock_file)
holder = _parse_lock_holder(content)
if holder is None:
# Unparsable: wrong line count, a non-integer PID or creation time, empty, oversized or not UTF-8.
# Self-heal only once the file is clearly not a half-written fresh lock (a peer between O_EXCL and
# _write_lock_info), so the brief create-then-write window is never mistaken for a stale lock.
if time.time() - mtime >= _MALFORMED_LOCK_AGE_THRESHOLD:
break_lock_file(self.lock_file, mtime)
return
pid, hostname, creation_time = holder
if hostname != socket.gethostname():
return
if self._is_process_alive(pid):
if sys.platform != "win32" or creation_time is None: # pragma: win32 no cover
return # same process, or no creation time to disambiguate a recycled PID — don't evict
actual = self._get_process_creation_time(pid) # pragma: win32 cover
if actual is None or actual == creation_time: # pragma: win32 cover
return # same process or can't verify — don't evict
# else: PID alive but creation time differs — the PID was recycled, so the lock is stale.
break_lock_file(self.lock_file, mtime)
@staticmethod
def _is_process_alive(pid: int) -> bool:
if sys.platform == "win32": # pragma: win32 cover
import ctypes # noqa: PLC0415
kernel32 = ctypes.windll.kernel32
handle = kernel32.OpenProcess(_WIN_SYNCHRONIZE, 0, pid)
if handle:
kernel32.CloseHandle(handle)
return True
return kernel32.GetLastError() != _WIN_ERROR_INVALID_PARAMETER
try:
os.kill(pid, 0)
except OSError as exc:
if exc.errno == ESRCH:
return False
if exc.errno == EPERM:
return True
raise
return True
@staticmethod
def _get_process_creation_time(pid: int) -> int | None:
"""Return the process creation FILETIME as an integer on Windows, ``None`` otherwise."""
if sys.platform != "win32": # pragma: win32 no cover
return None
import ctypes # pragma: win32 cover # noqa: PLC0415
from ctypes import wintypes # noqa: PLC0415
kernel32 = ctypes.windll.kernel32
handle = kernel32.OpenProcess(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
if not handle:
return None
try:
creation = wintypes.FILETIME()
exit_time = wintypes.FILETIME()
kernel_time = wintypes.FILETIME()
user_time = wintypes.FILETIME()
if not kernel32.GetProcessTimes(
handle,
ctypes.byref(creation),
ctypes.byref(exit_time),
ctypes.byref(kernel_time),
ctypes.byref(user_time),
):
return None
finally:
kernel32.CloseHandle(handle)
return (creation.dwHighDateTime << 32) | creation.dwLowDateTime
@staticmethod
def _write_lock_info(fd: int) -> None:
with suppress(OSError):
info = f"{os.getpid()}\n{socket.gethostname()}\n"
if sys.platform == "win32" and (ct := SoftFileLock._get_process_creation_time(os.getpid())) is not None:
info += f"{ct}\n"
os.write(fd, info.encode())
@property
def pid(self) -> int | None:
"""
The PID of the process holding this lock, read from the lock file.
:returns: the PID as an integer, or ``None`` if the lock file does not exist or cannot be parsed
"""
with suppress(OSError, ValueError):
holder = _parse_lock_holder(_read_lock_file(self.lock_file)[0])
if holder is not None:
return holder[0]
return None
@property
def is_lock_held_by_us(self) -> bool:
"""
Whether this lock is held by the current process.
:returns: ``True`` if the lock file exists and names the current process's PID and hostname
"""
with suppress(OSError, ValueError):
holder = _parse_lock_holder(_read_lock_file(self.lock_file)[0])
if holder is not None:
pid, hostname, _ = holder
return pid == os.getpid() and hostname == socket.gethostname()
return False
def break_lock(self) -> None:
"""Forcibly break the lock by removing the lock file, regardless of who holds it."""
with suppress(OSError):
Path(self.lock_file).unlink()
def _release(self) -> None:
assert self._context.lock_file_fd is not None # noqa: S101
os.close(self._context.lock_file_fd) # the lock file is definitely not None
os.close(self._context.lock_file_fd)
self._context.lock_file_fd = None
with suppress(OSError): # the file is already deleted and that's what we want
Path(self.lock_file).unlink()
if sys.platform == "win32":
self._windows_unlink_with_retry()
else:
with suppress(OSError):
Path(self.lock_file).unlink()
def _windows_unlink_with_retry(self) -> None:
max_retries = 10
retry_delay = 0.001
for attempt in range(max_retries):
# Windows doesn't immediately release file handles after close, causing EACCES/EPERM on unlink
try:
Path(self.lock_file).unlink()
except OSError as exc: # noqa: PERF203
if exc.errno not in {EACCES, EPERM}:
return
if attempt < max_retries - 1:
time.sleep(retry_delay)
retry_delay *= 2
else:
return
def _read_lock_file(path: str) -> tuple[str | None, float]:
# The lock file is created with O_EXCL | O_NOFOLLOW, so a symlink here is a hostile replacement and must
# not be followed. O_NONBLOCK keeps an attacker-placed FIFO from stalling the open (O_NOFOLLOW alone only
# rejects a symlink, not a real FIFO at the path), and the capped read stops a huge file (e.g. /dev/zero)
# from exhausting memory. Content is None when the file is too large or not UTF-8, but the mtime still
# flows back so the caller can evict it as a stale, malformed lock.
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
try:
mtime, data = os.fstat(fd).st_mtime, os.read(fd, _MAX_LOCK_FILE_SIZE + 1)
finally:
os.close(fd)
if len(data) <= _MAX_LOCK_FILE_SIZE:
with suppress(UnicodeDecodeError):
return data.decode("utf-8"), mtime
return None, mtime
def _parse_lock_holder(content: str | None) -> tuple[int, str, int | None] | None:
# A well-formed lock file is "<pid>\n<hostname>\n" with an optional "<creation_time>\n" third line on Windows.
# Anything else — wrong line count, a non-integer PID or creation time, empty or unreadable content — is
# unparsable; returning None lets the caller treat it as a malformed lock to self-heal rather than a holder.
if not content or len(lines := content.strip().splitlines()) not in {2, 3}:
return None
try:
pid = int(lines[0])
creation_time = int(lines[2]) if len(lines) == 3 else None # noqa: PLR2004
except ValueError:
return None
# A pid outside the valid range is a malformed lock, not a holder. Without this, a non-positive pid
# reaches os.kill() where 0 / -1 mean "the caller's own process group / every process" so a dead
# holder reads as alive and the lock is never reclaimed, while an oversized pid raises OverflowError
# (not OSError/ValueError) out of the self-heal path. _parse_marker_bytes already enforces this range.
if not 1 <= pid <= 2**31 - 1:
return None
return pid, lines[1], creation_time
__all__ = [

View file

@ -0,0 +1,12 @@
"""Cross-process and cross-host reader/writer lock on :class:`~filelock.SoftFileLock` primitives."""
from __future__ import annotations
from ._async import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock
from ._sync import SoftReadWriteLock
__all__ = [
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncSoftReadWriteLock",
"SoftReadWriteLock",
]

View file

@ -0,0 +1,213 @@
"""Async wrapper around :class:`SoftReadWriteLock` for use with ``asyncio``."""
from __future__ import annotations
import asyncio
import functools
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING
from ._sync import SoftReadWriteLock
if TYPE_CHECKING:
import os
from collections.abc import AsyncGenerator, Callable
from concurrent import futures
from types import TracebackType
class AsyncAcquireSoftReadWriteReturnProxy:
"""Async context-aware object that releases an :class:`AsyncSoftReadWriteLock` on exit."""
def __init__(self, lock: AsyncSoftReadWriteLock) -> None:
self.lock = lock
async def __aenter__(self) -> AsyncSoftReadWriteLock:
return self.lock
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock.release()
class AsyncSoftReadWriteLock:
"""
Async wrapper around :class:`SoftReadWriteLock` for ``asyncio`` applications.
The sync class's blocking filesystem operations run on a thread pool via ``loop.run_in_executor()``.
Reentrancy, upgrade/downgrade rules, fork handling, heartbeat and TTL stale detection, and singleton
behavior are delegated to the underlying :class:`SoftReadWriteLock`.
:param lock_file: path to the lock file; sidecar state/write/readers live next to it
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
:param is_singleton: if ``True``, reuse existing :class:`SoftReadWriteLock` instances per resolved path
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
:param stale_threshold: seconds of mtime inactivity before a marker is stale; defaults to ``3 * heartbeat_interval``
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
:param executor: executor for ``run_in_executor``; ``None`` uses the default executor
.. versionadded:: 3.27.0
"""
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
loop: asyncio.AbstractEventLoop | None = None,
executor: futures.Executor | None = None,
) -> None:
self._lock = SoftReadWriteLock(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
)
self._loop = loop
self._executor = executor
@property
def lock_file(self) -> str:
""":returns: the path to the lock file passed to the constructor."""
return self._lock.lock_file
@property
def timeout(self) -> float:
""":returns: the default timeout applied when ``acquire_read`` / ``acquire_write`` is called without one."""
return self._lock.timeout
@property
def blocking(self) -> bool:
""":returns: whether ``acquire_*`` defaults to blocking; ``False`` makes contention raise immediately."""
return self._lock.blocking
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
""":returns: the event loop used for ``run_in_executor``, or ``None`` for the running loop."""
return self._loop
@property
def executor(self) -> futures.Executor | None:
""":returns: the executor used for ``run_in_executor``, or ``None`` for the default executor."""
return self._executor
async def acquire_read(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireSoftReadWriteReturnProxy:
"""
Acquire a shared read lock.
See :meth:`SoftReadWriteLock.acquire_read` for the full reentrancy / upgrade / fork semantics. The blocking
work runs inside ``run_in_executor`` so other coroutines on the same loop continue to progress while this
call waits.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:returns: a proxy usable as an async context manager to release the lock
:raises RuntimeError: if a write lock is already held, if this instance was invalidated by
:func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_read, timeout, blocking=blocking)
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
async def acquire_write(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireSoftReadWriteReturnProxy:
"""
Acquire an exclusive write lock.
See :meth:`SoftReadWriteLock.acquire_write` for the two-phase writer-preferring semantics. The blocking
work runs inside ``run_in_executor``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:returns: a proxy usable as an async context manager to release the lock
:raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if
this instance was invalidated by :func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_write, timeout, blocking=blocking)
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
async def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
await self._run(self._lock.release, force=force)
@asynccontextmanager
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases a shared read lock.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
@asynccontextmanager
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases an exclusive write lock.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
async def close(self) -> None:
"""Release any held lock and release the underlying filesystem resources. Idempotent."""
await self._run(self._lock.close)
async def _run(self, func: Callable[..., object], *args: object, **kwargs: object) -> object:
loop = self._loop or asyncio.get_running_loop()
return await loop.run_in_executor(self._executor, functools.partial(func, *args, **kwargs))
__all__ = [
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncSoftReadWriteLock",
]

View file

@ -0,0 +1,851 @@
"""Cross-process and cross-host reader/writer lock built on :class:`SoftFileLock` primitives."""
from __future__ import annotations
import atexit
import hmac
import os
import re
import secrets
import socket
import stat
import sys
import threading
import time
import uuid
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Literal
from weakref import WeakValueDictionary
from filelock._api import AcquireReturnProxy
from filelock._error import Timeout
from filelock._soft import SoftFileLock
from filelock._util import ensure_directory_exists
if TYPE_CHECKING:
from collections.abc import Callable, Generator
_Mode = Literal["read", "write"]
_BREAK_SUFFIX = ".break"
_MAX_MARKER_SIZE = 1024
_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
# dirfd-relative I/O is a Unix-only optimization; Windows cannot ``os.open()`` a directory at all, and
# its ``os`` module skips dir_fd support entirely. When disabled, callers fall back to full-path ops.
_SUPPORTS_DIR_FD = sys.platform != "win32" and os.open in os.supports_dir_fd
_all_instances: WeakValueDictionary[Path, SoftReadWriteLock] = WeakValueDictionary()
_all_instances_lock = threading.Lock()
_atexit_registered = False
_fork_registered = False
@dataclass(frozen=True)
class _Paths:
state: str
write: str
readers: str
@dataclass
class _Locks:
internal: threading.Lock
transaction: threading.Lock
state: SoftFileLock
@dataclass(frozen=True)
class _MarkerInfo:
token: str
pid: int
hostname: str
@dataclass
class _Hold:
"""Everything that exists only while a lock is held; ``None`` when the instance has no lock."""
level: int
mode: _Mode
write_thread_id: int | None
marker_name: str
is_reader: bool
token: str
heartbeat_thread: _HeartbeatThread
heartbeat_stop: threading.Event
class _SoftRWMeta(type):
_instances: WeakValueDictionary[Path, SoftReadWriteLock]
_instances_lock: threading.Lock
def __call__( # noqa: PLR0913
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
) -> SoftReadWriteLock:
if not is_singleton:
return super().__call__(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
)
normalized = Path(lock_file).resolve()
with cls._instances_lock:
instance = cls._instances.get(normalized)
if instance is None:
instance = super().__call__(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
)
cls._instances[normalized] = instance
elif instance.timeout != timeout or instance.blocking != blocking:
msg = (
f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking},"
f" cannot be changed to timeout={timeout}, blocking={blocking}"
)
raise ValueError(msg)
return instance
class SoftReadWriteLock(metaclass=_SoftRWMeta):
"""
Cross-process and cross-host reader/writer lock built on :class:`SoftFileLock` primitives.
Use this class instead of :class:`~filelock.ReadWriteLock` when the lock file lives on a network
filesystem (NFS, Lustre with ``-o flock``, HPC cluster shared storage). ``ReadWriteLock`` is backed
by SQLite and cannot run on NFS because SQLite's ``fcntl`` locking is unreliable there.
Layout on disk for a lock at ``foo.lock``:
- ``foo.lock.state`` a :class:`SoftFileLock` taken only during state transitions (microseconds).
- ``foo.lock.write`` writer marker; its presence means a writer is claiming or holding the lock.
- ``foo.lock.readers/<host>.<pid>.<uuid>`` one file per reader.
Each marker stores a random token (``secrets.token_hex(16)``), the holder's pid, and the holder's
hostname. A daemon heartbeat thread refreshes ``mtime`` on every held marker. A marker whose mtime
has not advanced in ``stale_threshold`` seconds may be evicted by any process on any host, giving
correct behavior when a compute node crashes with a lock held.
Writer acquire is two-phase and writer-preferring: phase 1 claims ``.write`` (blocking any new
reader), phase 2 waits for existing readers to drain. Writer starvation is impossible.
Reentrancy, upgrade/downgrade rules, thread pinning, and singleton caching by resolved path match
:class:`~filelock.ReadWriteLock`.
Forking while holding a lock invalidates the inherited instance in the child so the child cannot
double-own the lock with its parent; ``release()`` on a fork-invalidated instance is a no-op, and
the child must re-acquire if it needs a lock.
Trust boundary: protects against same-UID non-cooperating processes (one host or cross-host) and
same-host different-UID users via ``0o600`` / ``0o700`` permissions. Does not protect against root
compromise, NTP tampering on same-UID cross-host nodes, or multi-tenant mounts where hostile
co-tenants share the UID.
:param lock_file: path to the lock file; sidecar state/write/readers live next to it
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
:param is_singleton: if ``True``, reuse existing instances for the same resolved path
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
:param stale_threshold: seconds of ``mtime`` inactivity before a marker is stale; defaults to
``3 * heartbeat_interval``, matching etcd's ``LeaseKeepAlive`` convention
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
.. versionadded:: 3.27.0
"""
_instances: WeakValueDictionary[Path, SoftReadWriteLock] = WeakValueDictionary()
_instances_lock = threading.Lock()
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True, # noqa: ARG002
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
) -> None:
if heartbeat_interval <= 0:
msg = f"heartbeat_interval must be positive, got {heartbeat_interval}"
raise ValueError(msg)
if stale_threshold is None:
stale_threshold = heartbeat_interval * 3
if stale_threshold <= heartbeat_interval:
msg = f"stale_threshold must exceed heartbeat_interval ({stale_threshold} <= {heartbeat_interval})"
raise ValueError(msg)
if poll_interval <= 0:
msg = f"poll_interval must be positive, got {poll_interval}"
raise ValueError(msg)
self.lock_file: str = os.fspath(lock_file)
self.timeout: float = timeout
self.blocking: bool = blocking
self.heartbeat_interval: float = heartbeat_interval
self.stale_threshold: float = stale_threshold
self.poll_interval: float = poll_interval
self._paths = _Paths(
state=f"{self.lock_file}.state",
write=f"{self.lock_file}.write",
readers=f"{self.lock_file}.readers",
)
ensure_directory_exists(self.lock_file)
self._locks = _Locks(
internal=threading.Lock(),
transaction=threading.Lock(),
state=SoftFileLock(self._paths.state, timeout=-1),
)
self._readers_dir_fd: int | None = None
self._hold: _Hold | None = None
self._fork_invalidated: bool = False
self._closed: bool = False
with _all_instances_lock:
_all_instances[Path(self.lock_file).resolve()] = self
_register_hooks()
@contextmanager
def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
self.release()
@contextmanager
def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
self.release()
def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
"""
Acquire a shared read lock.
If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a
read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). On the 01
transition a daemon heartbeat thread is started that refreshes the reader marker's ``mtime`` every
``heartbeat_interval`` seconds so peers on other hosts do not evict the marker as stale.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
``None`` uses the instance default
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a write lock is already held on this instance, if this instance was invalidated by
:func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("read", timeout, blocking=blocking)
def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
"""
Acquire an exclusive write lock.
If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant).
Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not
allowed). Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises
:class:`RuntimeError`.
Writer acquisition runs in two phases. Phase 1 atomically claims ``<path>.write`` via ``O_CREAT | O_EXCL``,
which immediately blocks any new reader on any host. Phase 2 waits for existing readers to drain. Writer
starvation is impossible: new readers see ``<path>.write`` during phase 2 and wait behind the pending writer.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
``None`` uses the instance default
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if this
instance was invalidated by :func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("write", timeout, blocking=blocking)
def close(self) -> None:
"""
Release any held lock and release internal filesystem resources.
Idempotent. After calling this method the instance can no longer acquire locks subsequent acquires raise
:class:`RuntimeError`. A fork-invalidated instance is closed without raising.
"""
self.release(force=True)
with self._locks.internal:
if self._closed:
return
self._closed = True
if self._readers_dir_fd is not None:
with suppress(OSError):
os.close(self._readers_dir_fd)
self._readers_dir_fd = None
def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
When the lock level reaches zero the heartbeat thread is stopped and the held marker file is unlinked. On a
fork-invalidated instance (that is, the child of a :func:`os.fork` call made while the parent held a lock)
this method is a no-op so inherited ``with`` blocks can unwind cleanly in the child.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
with self._locks.internal:
if self._fork_invalidated:
# Inherited state from the parent is meaningless in the child; clear any counters and return.
self._hold = None
return
hold = self._hold
if hold is None:
if force:
return
msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held"
raise RuntimeError(msg)
if force:
hold.level = 0
else:
hold.level -= 1
if hold.level > 0:
return
self._hold = None
# Order matters: signal → join → unlink. A late tick on a deleted marker is harmless, and the
# token check in the heartbeat callback would catch any re-acquisition race, but joining first
# removes even that theoretical race.
hold.heartbeat_stop.set()
hold.heartbeat_thread.join(timeout=self.heartbeat_interval + 1.0)
if hold.is_reader:
_unlink(hold.marker_name, dir_fd=self._readers_dir_fd)
else:
_unlink(hold.marker_name)
@classmethod
def get_lock(
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
) -> SoftReadWriteLock:
"""
Return the singleton :class:`SoftReadWriteLock` for *lock_file*.
:param lock_file: path to the lock file; sidecar state/write/readers live next to it
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: the singleton lock instance
:raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values
"""
return cls(lock_file, timeout, blocking=blocking)
def _acquire(
self,
mode: _Mode,
timeout: float | None,
*,
blocking: bool | None,
) -> AcquireReturnProxy:
timeout = self.timeout if timeout is None else timeout
blocking = self.blocking if blocking is None else blocking
with self._locks.internal:
if self._fork_invalidated:
msg = f"SoftReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance"
raise RuntimeError(msg)
if self._closed:
msg = f"SoftReadWriteLock on {self.lock_file} has been closed"
raise RuntimeError(msg)
if self._hold is not None:
return self._validate_reentrant(mode)
start = time.perf_counter()
if not blocking:
acquired = self._locks.transaction.acquire(blocking=False)
elif timeout == -1:
acquired = self._locks.transaction.acquire(blocking=True)
else:
acquired = self._locks.transaction.acquire(blocking=True, timeout=timeout)
if not acquired:
raise Timeout(self.lock_file) from None
try:
return self._do_acquire_inner(mode, timeout, start, blocking=blocking)
finally:
self._locks.transaction.release()
def _do_acquire_inner(
self,
mode: _Mode,
effective_timeout: float,
start: float,
*,
blocking: bool,
) -> AcquireReturnProxy:
with self._locks.internal:
if self._hold is not None:
return self._validate_reentrant(mode)
deadline = None if effective_timeout == -1 else start + effective_timeout
token = secrets.token_hex(16)
if mode == "write":
marker_name, is_reader = self._acquire_writer_slot(token, deadline=deadline, blocking=blocking)
else:
marker_name, is_reader = self._acquire_reader_slot(token, deadline=deadline, blocking=blocking)
stop_event = threading.Event()
heartbeat = _HeartbeatThread(
refresh=self._refresh_marker,
interval=self.heartbeat_interval,
stop_event=stop_event,
name=f"filelock-heartbeat-{id(self):x}",
)
with self._locks.internal:
self._hold = _Hold(
level=1,
mode=mode,
write_thread_id=threading.get_ident() if mode == "write" else None,
marker_name=marker_name,
is_reader=is_reader,
token=token,
heartbeat_thread=heartbeat,
heartbeat_stop=stop_event,
)
heartbeat.start()
return AcquireReturnProxy(lock=self)
def _validate_reentrant(self, mode: _Mode) -> AcquireReturnProxy:
hold = self._hold
assert hold is not None # noqa: S101
if hold.mode != mode:
opposite = "write" if mode == "read" else "read"
direction = "downgrade" if mode == "read" else "upgrade"
msg = (
f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): "
f"already holding a {opposite} lock ({direction} not allowed)"
)
raise RuntimeError(msg)
if mode == "write" and (cur := threading.get_ident()) != hold.write_thread_id:
msg = (
f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) "
f"from thread {cur} while it is held by thread {hold.write_thread_id}"
)
raise RuntimeError(msg)
hold.level += 1
return AcquireReturnProxy(lock=self)
def _acquire_writer_slot(
self,
token: str,
*,
deadline: float | None,
blocking: bool,
) -> tuple[str, bool]:
# Phase 2 scans readers/ via dirfd (where supported), so we need it open even though writers never
# create files inside.
self._open_readers_dir()
def try_claim_writer() -> bool:
with self._locks.state:
_break_stale_marker(self._paths.write, stale_threshold=self.stale_threshold, now=time.time())
if _file_exists(self._paths.write):
return False
try:
_atomic_create_marker(self._paths.write, token)
except FileExistsError:
return False
return True
def readers_drained_touching() -> bool:
with self._locks.state:
# Refresh our writer marker on every scan iteration. Otherwise phase 2 can exceed
# ``stale_threshold`` under contention and a peer would treat us as stale and evict us.
with suppress(OSError):
_touch(self._paths.write)
self._break_stale_readers(time.time())
return not self._any_readers()
self._wait_for(try_claim_writer, deadline=deadline, blocking=blocking)
try:
self._wait_for(readers_drained_touching, deadline=deadline, blocking=blocking)
except Timeout:
# Give up our writer claim so readers can make progress again.
_unlink(self._paths.write)
raise
return self._paths.write, False
def _acquire_reader_slot(
self,
token: str,
*,
deadline: float | None,
blocking: bool,
) -> tuple[str, bool]:
self._open_readers_dir()
reader_name = f"{uuid.uuid4().hex}.{os.getpid()}"
dir_fd = self._readers_dir_fd
full_reader_path = str(Path(self._paths.readers) / reader_name)
def try_claim_reader() -> bool:
with self._locks.state:
_break_stale_marker(self._paths.write, stale_threshold=self.stale_threshold, now=time.time())
if _file_exists(self._paths.write):
return False
if dir_fd is not None:
_atomic_create_marker(reader_name, token, dir_fd=dir_fd)
else: # pragma: win32 cover
_atomic_create_marker(full_reader_path, token)
return True
self._wait_for(try_claim_reader, deadline=deadline, blocking=blocking)
return (reader_name if dir_fd is not None else full_reader_path), True
def _wait_for(
self,
predicate: Callable[[], bool],
*,
deadline: float | None,
blocking: bool,
) -> None:
while True:
if predicate():
return
now = time.perf_counter()
if not blocking:
raise Timeout(self.lock_file)
if deadline is not None and now >= deadline:
raise Timeout(self.lock_file)
sleep_for = self.poll_interval
if deadline is not None:
sleep_for = min(sleep_for, max(deadline - now, 0.0))
time.sleep(sleep_for)
def _open_readers_dir(self) -> None:
readers_path = Path(self._paths.readers)
with suppress(FileExistsError):
readers_path.mkdir(mode=0o700)
# mkdir has no O_NOFOLLOW, so verify via lstat that we did not land on an attacker-placed symlink
# or a regular file before we open or scan inside.
st = os.lstat(self._paths.readers)
if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
msg = f"{self._paths.readers} exists but is not a directory or is a symlink; refusing to use it"
raise RuntimeError(msg)
if self._readers_dir_fd is None and _SUPPORTS_DIR_FD:
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | _O_NOFOLLOW
self._readers_dir_fd = os.open(self._paths.readers, flags)
def _any_readers(self) -> bool:
for _ in self._iter_reader_entries():
return True
return False
def _iter_reader_entries(self) -> Generator[tuple[str, bool]]:
"""
Yield ``(name, dirfd_relative)`` pairs for every live reader marker.
``dirfd_relative`` is ``True`` when *name* should be passed to ``dir_fd=``-aware syscalls; ``False``
when *name* is a full path because dirfd-relative I/O is unavailable on this platform.
"""
if self._readers_dir_fd is not None:
with os.scandir(self._readers_dir_fd) as it:
for entry in it:
if not _is_housekeeping_name(entry.name):
yield entry.name, True
return
readers_path = Path(self._paths.readers) # pragma: win32 cover
with os.scandir(readers_path) as it: # pragma: win32 cover
for entry in it: # pragma: win32 cover
if not _is_housekeeping_name(entry.name): # pragma: win32 cover
yield str(readers_path / entry.name), False # pragma: win32 cover
def _break_stale_readers(self, now: float) -> None:
names: list[tuple[str, int | None]] = []
try:
for name, dirfd_relative in self._iter_reader_entries():
names.append((name, self._readers_dir_fd if dirfd_relative else None))
except OSError: # pragma: no cover - transient NFS scandir hiccup
return
for name, fd in names:
_break_stale_marker(name, stale_threshold=self.stale_threshold, now=now, dir_fd=fd)
def _refresh_marker(self) -> bool:
with self._locks.internal:
hold = self._hold
if hold is None: # pragma: no cover - race between stop_event.set and join
return False
marker_name = hold.marker_name
token = hold.token
dir_fd = self._readers_dir_fd if hold.is_reader else None
read_result = _read_marker(marker_name, dir_fd=dir_fd)
if read_result is None:
return False
info, _mtime = read_result
# Token mismatch means another process already evicted our marker and created its own; stop the
# thread so it does not touch a stranger's file.
if info is None or not hmac.compare_digest(info.token, token):
return False
try:
_touch(marker_name, dir_fd=dir_fd)
except FileNotFoundError: # pragma: no cover - race between successful read and touch
return False
return True
def _reset_after_fork_in_child(self) -> None: # pragma: no cover - fork child not tracked
# Replace every lock this instance owns with a fresh one; the inherited locks may still be held
# by threads that no longer exist in the child. The readers dirfd and the SoftFileLock state
# mutex both get dropped for the same reason — the child re-creates them on its next acquire.
self._locks = _Locks(
internal=threading.Lock(),
transaction=threading.Lock(),
state=SoftFileLock(self._paths.state, timeout=-1),
)
self._hold = None
self._readers_dir_fd = None
self._fork_invalidated = True
class _HeartbeatThread(threading.Thread):
def __init__(
self,
refresh: Callable[[], bool],
interval: float,
stop_event: threading.Event,
name: str,
) -> None:
super().__init__(name=name, daemon=True)
self._refresh = refresh
self._interval = interval
self._stop_event = stop_event
def run(self) -> None:
while not self._stop_event.wait(self._interval):
if not self._refresh():
self._stop_event.set()
return
def _atomic_create_marker(name: str, token: str, *, dir_fd: int | None = None) -> None:
# O_NOFOLLOW blocks the symlink-overwrite attack where an attacker pre-creates the marker path as a
# symlink pointing at a victim file. Mode 0o600 keeps the token unreadable to other users.
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | _O_NOFOLLOW
if _SUPPORTS_DIR_FD and dir_fd is not None:
fd = os.open(name, flags, 0o600, dir_fd=dir_fd)
else:
fd = os.open(name, flags, 0o600)
try:
content = f"{token}\n{os.getpid()}\n{socket.gethostname()}\n".encode("ascii")
os.write(fd, content)
finally:
os.close(fd)
def _read_marker(name: str, *, dir_fd: int | None = None) -> tuple[_MarkerInfo | None, float] | None:
# The file is ours; these guard a hostile mid-flight swap. O_NOFOLLOW rejects a symlink; O_NONBLOCK keeps
# a real FIFO from blocking the open forever, so it reads as a malformed marker instead of wedging a peer
# that holds the state lock.
flags = os.O_RDONLY | _O_NOFOLLOW | _O_NONBLOCK
try:
fd = os.open(name, flags, dir_fd=dir_fd) if _SUPPORTS_DIR_FD and dir_fd is not None else os.open(name, flags)
except OSError:
return None
try:
try:
st = os.fstat(fd)
data = os.read(fd, _MAX_MARKER_SIZE + 1)
except OSError: # pragma: no cover - e.g. EAGAIN from a hostile FIFO that has a writer attached
return None
finally:
os.close(fd)
return _parse_marker_bytes(data), st.st_mtime
def _parse_marker_bytes(data: bytes) -> _MarkerInfo | None:
# Trust nothing about attacker-controlled markers; any deviation returns None so callers fall through
# to stale cleanup. ``re.match`` caches compiled patterns internally, so the regex is built only once
# despite being defined inline.
if not data or len(data) > _MAX_MARKER_SIZE:
return None
try:
text = data.decode("ascii")
except UnicodeDecodeError:
return None
match = re.match(
r"""
\A # start of string
(?P<token> [0-9a-f]{32} ) \n # 128-bit hex token
(?P<pid> [1-9][0-9]{0,9} ) \n # decimal pid: no leading zero, ≤ 10 digits
(?P<hostname> [\x21-\x7e]{1,253}) # printable non-whitespace ASCII (RFC 1123 hostname limit)
\n* # tolerate sloppy writers that append extra newlines
\Z # end of string
""",
text,
re.VERBOSE,
)
if match is None:
return None
pid = int(match["pid"], 10)
if pid > 2**31 - 1:
return None
return _MarkerInfo(token=match["token"], pid=pid, hostname=match["hostname"])
def _break_stale_marker( # noqa: PLR0911
name: str,
*,
stale_threshold: float,
now: float,
dir_fd: int | None = None,
) -> bool:
# Atomic break pattern: read → rename to unique break-name → re-verify → unlink. The rename gives us a
# private name nobody else can touch; if the re-verify sees a newer mtime or a different token, the
# legitimate holder's heartbeat fired between read and rename and we must abort (leaving the .break.*
# file behind rather than rollback-renaming, because rollback is itself racy).
read_result = _read_marker(name, dir_fd=dir_fd)
if read_result is None:
return False
info_before, mtime_before = read_result
if now - mtime_before <= stale_threshold:
return False
if info_before is None:
_unlink(name, dir_fd=dir_fd)
return True
break_name = f"{name}{_BREAK_SUFFIX}.{os.getpid()}.{secrets.token_hex(16)}"
try:
if _SUPPORTS_DIR_FD and dir_fd is not None:
os.rename(name, break_name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd)
else:
Path(name).rename(break_name)
except OSError: # pragma: no cover - race where the marker vanishes between read and rename
return False
read_after = _read_marker(break_name, dir_fd=dir_fd)
if read_after is None: # pragma: no cover - race where a peer unlinks the break-name file
return False
info_after, mtime_after = read_after
if info_after is None: # pragma: no cover - content replaced post-rename by a racing peer
_unlink(break_name, dir_fd=dir_fd)
return True
if not hmac.compare_digest(info_before.token, info_after.token): # pragma: no cover - race only
return False
if mtime_after > mtime_before: # pragma: no cover - heartbeat raced our rename
return False
_unlink(break_name, dir_fd=dir_fd)
return True
def _unlink(name: str, *, dir_fd: int | None = None) -> None:
with suppress(FileNotFoundError):
if _SUPPORTS_DIR_FD and dir_fd is not None:
# Path.unlink has no dir_fd support, so we stay on os.unlink for the dirfd path.
os.unlink(name, dir_fd=dir_fd)
else:
Path(name).unlink()
def _touch(name: str, *, dir_fd: int | None = None) -> None:
if _SUPPORTS_DIR_FD and dir_fd is not None:
os.utime(name, None, dir_fd=dir_fd)
else:
os.utime(name, None)
def _file_exists(path: str) -> bool:
try:
st = os.lstat(path)
except FileNotFoundError:
return False
return stat.S_ISREG(st.st_mode)
def _is_housekeeping_name(name: str) -> bool:
return name.startswith(".") or _BREAK_SUFFIX in name
def _reset_all_after_fork() -> None: # pragma: no cover - fork child, not tracked by coverage
global _all_instances_lock # noqa: PLW0603
# User-created threading locks do not auto-reset across fork: any lock held by a parent thread stays
# locked in the child with no owner to release it. Replace the module-level lock and every instance's
# locks with fresh ones; the child is single-threaded at this point so no synchronization is needed.
_all_instances_lock = threading.Lock()
for instance in list(_all_instances.values()):
instance._reset_after_fork_in_child() # noqa: SLF001
def _cleanup_all_instances() -> None: # pragma: no cover - runs from atexit at interpreter shutdown
for instance in list(_all_instances.values()):
with suppress(Exception):
instance.release(force=True)
def _register_hooks() -> None:
global _atexit_registered, _fork_registered # noqa: PLW0603
if not _atexit_registered:
atexit.register(_cleanup_all_instances)
_atexit_registered = True
# after_in_child replaces inherited state so the child cannot double-own any lock the parent held.
if not _fork_registered and hasattr(os, "register_at_fork"):
os.register_at_fork(after_in_child=_reset_all_after_fork)
_fork_registered = True
__all__ = [
"SoftReadWriteLock",
]

View file

@ -2,8 +2,9 @@ from __future__ import annotations
import os
import sys
import warnings
from contextlib import suppress
from errno import ENOSYS
from errno import EAGAIN, ENOSYS, EWOULDBLOCK
from pathlib import Path
from typing import cast
@ -34,37 +35,79 @@ else: # pragma: win32 no cover
has_fcntl = True
class UnixFileLock(BaseFileLock):
"""Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems."""
"""
Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.
def _acquire(self) -> None:
Lock file cleanup: Unix and macOS delete the lock file reliably after release, even in
multi-threaded scenarios. Unlike Windows, Unix allows unlinking files that other processes
have open.
"""
def _acquire(self) -> None: # noqa: C901, PLR0912
ensure_directory_exists(self.lock_file)
open_flags = os.O_RDWR | os.O_TRUNC
o_nofollow = getattr(os, "O_NOFOLLOW", None)
if o_nofollow is not None:
open_flags |= o_nofollow
if not Path(self.lock_file).exists():
open_flags |= os.O_CREAT
fd = os.open(self.lock_file, open_flags, self._context.mode)
with suppress(PermissionError): # This locked is not owned by this UID
os.fchmod(fd, self._context.mode)
open_flags |= os.O_CREAT
open_mode = self._open_mode()
try:
fd = os.open(self.lock_file, open_flags, open_mode)
except FileNotFoundError:
# On FUSE/NFS, os.open(O_CREAT) is not atomic: LOOKUP + CREATE can be split, allowing a concurrent
# unlink() to delete the file between them. For valid paths, treat ENOENT as transient contention.
# For invalid paths (e.g., empty string), re-raise to avoid infinite retry loops.
if self.lock_file and Path(self.lock_file).parent.exists():
return
raise
except PermissionError:
# Sticky-bit dirs (e.g. /tmp): O_CREAT fails if the file is owned by another user (#317).
# Fall back to opening the existing file without O_CREAT.
if not Path(self.lock_file).exists():
raise
try:
fd = os.open(self.lock_file, open_flags & ~os.O_CREAT, open_mode)
except FileNotFoundError:
return
if self.has_explicit_mode:
with suppress(PermissionError):
os.fchmod(fd, self._context.mode)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as exception:
os.close(fd)
if exception.errno == ENOSYS: # NotImplemented error
msg = "FileSystem does not appear to support flock; use SoftFileLock instead"
raise NotImplementedError(msg) from exception
if exception.errno == ENOSYS:
with suppress(OSError):
Path(self.lock_file).unlink()
self._fallback_to_soft_lock()
self._acquire()
return
if exception.errno not in {EAGAIN, EWOULDBLOCK}:
raise
else:
self._context.lock_file_fd = fd
# The file may have been unlinked by a concurrent _release() between our open() and flock().
# A lock on an unlinked inode is useless — discard and let the retry loop start fresh.
if os.fstat(fd).st_nlink == 0:
os.close(fd)
else:
self._context.lock_file_fd = fd
def _fallback_to_soft_lock(self) -> None:
from ._soft import SoftFileLock # noqa: PLC0415
warnings.warn("flock not supported on this filesystem, falling back to SoftFileLock", stacklevel=2)
from .asyncio import AsyncSoftFileLock, BaseAsyncFileLock # noqa: PLC0415
self.__class__ = AsyncSoftFileLock if isinstance(self, BaseAsyncFileLock) else SoftFileLock
def _release(self) -> None:
# Do not remove the lockfile:
# https://github.com/tox-dev/py-filelock/issues/31
# https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
fd = cast("int", self._context.lock_file_fd)
self._context.lock_file_fd = None
with suppress(OSError):
Path(self.lock_file).unlink()
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
with suppress(OSError): # close can raise EIO on FUSE/Docker bind-mount filesystems after unlink
os.close(fd)
__all__ = [

View file

@ -15,6 +15,7 @@ def raise_on_not_writable_file(filename: str) -> None:
locked.
:param filename: file to check
:raises OSError: as if the file was opened for writing.
"""
@ -46,7 +47,36 @@ def ensure_directory_exists(filename: Path | str) -> None:
Path(filename).parent.mkdir(parents=True, exist_ok=True)
def break_lock_file(lock_file: str, mtime_before: float) -> None:
"""
Atomically break a stale lock file that was judged stale at modification time *mtime_before*.
The file is renamed to a process-private name before being unlinked, so two processes breaking the same lock
cannot delete each other's work (only one rename of a given inode succeeds; the loser gets ``OSError``). After the
rename the modification time is re-checked: a value newer than *mtime_before* means a peer recreated the lock
between the stale decision and the rename, so we grabbed a live file and must abort, leaving the renamed file in
place rather than rolling back (a rollback rename is itself racy same trade-off as the soft read/write marker
break). ``lstat`` is used so a hostile symlink swapped in after the decision is not followed.
:param lock_file: path to the lock file to break.
:param mtime_before: modification time observed when the lock was judged stale.
:raises OSError: if the rename fails (e.g. the file vanished or is not owned in a sticky directory).
"""
break_path = f"{lock_file}.break.{os.getpid()}"
Path(lock_file).rename(break_path)
try:
mtime_after = os.lstat(break_path).st_mtime
except OSError:
return
if mtime_after > mtime_before:
return
Path(break_path).unlink()
__all__ = [
"break_lock_file",
"ensure_directory_exists",
"raise_on_not_writable_file",
]

View file

@ -29,8 +29,11 @@ if sys.platform == "win32": # pragma: win32 cover
Check if a path is a reparse point (symlink, junction, etc.) on Windows.
:param path: Path to check
:return: True if path is a reparse point, False otherwise
:returns: True if path is a reparse point, False otherwise
:raises OSError: If GetFileAttributesW fails for reasons other than file-not-found
"""
attrs = _kernel32.GetFileAttributesW(path)
if attrs == INVALID_FILE_ATTRIBUTES:
@ -45,7 +48,13 @@ if sys.platform == "win32": # pragma: win32 cover
return bool(attrs & FILE_ATTRIBUTE_REPARSE_POINT)
class WindowsFileLock(BaseFileLock):
"""Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems."""
"""
Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems.
Lock file cleanup: Windows attempts to delete the lock file after release, but deletion is
not guaranteed in multi-threaded scenarios where another thread holds an open handle. The lock
file may persist on disk, which does not affect lock correctness.
"""
def _acquire(self) -> None:
raise_on_not_writable_file(self.lock_file)
@ -60,10 +69,9 @@ if sys.platform == "win32": # pragma: win32 cover
flags = (
os.O_RDWR # open for read and write
| os.O_CREAT # create file if not exists
| os.O_TRUNC # truncate file if not empty
)
try:
fd = os.open(self.lock_file, flags, self._context.mode)
fd = os.open(self.lock_file, flags, self._open_mode())
except OSError as exception:
if exception.errno != EACCES: # has no access to this lock
raise
@ -83,7 +91,7 @@ if sys.platform == "win32": # pragma: win32 cover
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
os.close(fd)
with suppress(OSError): # Probably another instance of the application hat acquired the file lock.
with suppress(OSError):
Path(self.lock_file).unlink()
else: # pragma: win32 no cover

View file

@ -10,9 +10,9 @@ import time
from dataclasses import dataclass
from inspect import iscoroutinefunction
from threading import local
from typing import TYPE_CHECKING, Any, NoReturn, cast
from typing import TYPE_CHECKING, Any, NoReturn, TypeVar
from ._api import BaseFileLock, FileLockContext, FileLockMeta
from ._api import _UNSET_FILE_MODE, BaseFileLock, FileLockContext, FileLockMeta
from ._error import Timeout
from ._soft import SoftFileLock
from ._unix import UnixFileLock
@ -69,49 +69,62 @@ class AsyncAcquireReturnProxy:
await self.lock.release()
_AT = TypeVar("_AT", bound="BaseAsyncFileLock")
class AsyncFileLockMeta(FileLockMeta):
def __call__( # type: ignore[override] # noqa: PLR0913
cls, # noqa: N805
def __call__( # ty: ignore[invalid-method-override] # noqa: PLR0913
cls: type[_AT], # noqa: N805
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = 0o644,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = False, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
loop: asyncio.AbstractEventLoop | None = None,
run_in_executor: bool = True,
executor: futures.Executor | None = None,
) -> BaseAsyncFileLock:
) -> _AT:
if thread_local and run_in_executor:
msg = "run_in_executor is not supported when thread_local is True"
raise ValueError(msg)
instance = super().__call__(
return super().__call__(
lock_file=lock_file,
timeout=timeout,
mode=mode,
thread_local=thread_local,
blocking=blocking,
is_singleton=is_singleton,
poll_interval=poll_interval,
lifetime=lifetime,
loop=loop,
run_in_executor=run_in_executor,
executor=executor,
)
return cast("BaseAsyncFileLock", instance)
class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
"""Base class for asynchronous file locks."""
"""
Base class for asynchronous file locks.
.. versionadded:: 3.15.0
"""
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = 0o644,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = False, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
loop: asyncio.AbstractEventLoop | None = None,
run_in_executor: bool = True,
executor: futures.Executor | None = None,
@ -120,16 +133,27 @@ class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
Create a new lock object.
:param lock_file: path to the file
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in \
the acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it \
to a negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
:param mode: file permissions for the lockfile
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to \
``False`` then the lock will be reentrant across threads.
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the
acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a
negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
:param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and
default ACLs, preserving POSIX default ACL inheritance in shared directories.
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to
``False`` then the lock will be reentrant across threads. When ``True`` (the default), **all fields of the
lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``,
``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change
the value seen by another thread; threads that did not perform the write continue to see the value supplied
at construction time. If you need configuration values to be visible across threads, construct the lock
with ``thread_local=False``.
:param blocking: whether the lock should be blocking or not
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created \
per lock file. This is useful if you want to use the lock object for reentrant locking without needing \
to pass the same object around.
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock
file. This is useful if you want to use the lock object for reentrant locking without needing to pass the
same object around.
:param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value
in the acquire method, if no poll_interval value (``None``) is given.
:param lifetime: maximum time in seconds a lock can be held before it is considered expired. When set, a waiting
process will break a lock whose file modification time is older than ``lifetime`` seconds. ``None`` (the
default) means locks never expire.
:param loop: The event loop to use. If not specified, the running event loop will be used.
:param run_in_executor: If this is set to ``True`` then the lock will be acquired in an executor.
:param executor: The executor to use. If not specified, the default executor will be used.
@ -145,6 +169,8 @@ class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
"timeout": timeout,
"mode": mode,
"blocking": blocking,
"poll_interval": poll_interval,
"lifetime": lifetime,
"loop": loop,
"run_in_executor": run_in_executor,
"executor": executor,
@ -155,12 +181,12 @@ class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
@property
def run_in_executor(self) -> bool:
"""::return: whether run in executor."""
""":returns: whether run in executor."""
return self._context.run_in_executor
@property
def executor(self) -> futures.Executor | None:
"""::return: the executor."""
""":returns: the executor."""
return self._context.executor
@executor.setter
@ -168,35 +194,40 @@ class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
"""
Change the executor.
:param value: the new executor or ``None``
:type value: futures.Executor | None
:param futures.Executor | None value: the new executor or ``None``
"""
self._context.executor = value
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
"""::return: the event loop."""
""":returns: the event loop."""
return self._context.loop
async def acquire( # type: ignore[override]
async def acquire( # ty: ignore[invalid-method-override]
self,
timeout: float | None = None,
poll_interval: float = 0.05,
poll_interval: float | None = None,
*,
blocking: bool | None = None,
cancel_check: Callable[[], bool] | None = None,
) -> AsyncAcquireReturnProxy:
"""
Try to acquire the file lock.
:param timeout: maximum wait time for acquiring the lock, ``None`` means use the default
:attr:`~BaseFileLock.timeout` is and if ``timeout < 0``, there is no timeout and
this method will block until the lock could be acquired
:param poll_interval: interval of trying to acquire the lock file
:attr:`~BaseFileLock.timeout` is and if ``timeout < 0``, there is no timeout and this method will block
until the lock could be acquired
:param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default
:attr:`~BaseFileLock.poll_interval`
:param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the
first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.
first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.
:param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll
iteration. When triggered, raises :class:`~Timeout` just like an expired timeout.
:returns: a context object that will unlock the file when the context is exited
:raises Timeout: if fails to acquire lock within the timeout period
:return: a context object that will unlock the file when the context is exited
.. code-block:: python
@ -219,40 +250,64 @@ class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
if blocking is None:
blocking = self._context.blocking
if poll_interval is None:
poll_interval = self._context.poll_interval
# Increment the number right at the beginning. We can still undo it, if something fails.
self._context.lock_counter += 1
lock_id = id(self)
lock_filename = self.lock_file
start_time = time.perf_counter()
try:
while True:
if not self.is_locked:
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
await self._run_internal_method(self._acquire)
if self.is_locked:
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
break
if blocking is False:
_LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename)
raise Timeout(lock_filename) # noqa: TRY301
if 0 <= timeout < time.perf_counter() - start_time:
_LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename)
raise Timeout(lock_filename) # noqa: TRY301
msg = "Lock %s not acquired on %s, waiting %s seconds ..."
_LOGGER.debug(msg, lock_id, lock_filename, poll_interval)
await asyncio.sleep(poll_interval)
await self._async_poll_until_acquired(
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
poll_interval=poll_interval,
start_time=start_time,
)
except BaseException: # Something did go wrong, so decrement the counter.
self._context.lock_counter = max(0, self._context.lock_counter - 1)
raise
return AsyncAcquireReturnProxy(lock=self)
async def release(self, force: bool = False) -> None: # type: ignore[override] # noqa: FBT001, FBT002
"""
Releases the file lock. Please note, that the lock is only completely released, if the lock counter is 0.
Also note, that the lock file itself is not automatically deleted.
async def _async_poll_until_acquired(
self,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
poll_interval: float,
start_time: float,
) -> None:
lock_id = id(self)
lock_filename = self.lock_file
while True:
if not self.is_locked:
self._try_break_expired_lock()
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
await self._run_internal_method(self._acquire)
if self.is_locked:
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
return
if self._check_give_up(
lock_id,
lock_filename,
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
start_time=start_time,
):
raise Timeout(lock_filename)
msg = "Lock %s not acquired on %s, waiting %s seconds ..."
_LOGGER.debug(msg, lock_id, lock_filename, poll_interval)
await asyncio.sleep(poll_interval)
:param force: If true, the lock counter is ignored and the lock is released in every case/
async def release(self, force: bool = False) -> None: # ty: ignore[invalid-method-override] # noqa: FBT001, FBT002
"""
Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file
itself may be deleted automatically, the behavior is platform-specific.
:param force: If true, the lock counter is ignored and the lock is released in every case.
"""
if self.is_locked:
@ -270,28 +325,30 @@ class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
if iscoroutinefunction(method):
await method()
elif self.run_in_executor:
loop = self.loop or asyncio.get_running_loop()
await loop.run_in_executor(self.executor, method)
await asyncio.get_running_loop().run_in_executor(self.executor, method)
else:
method()
def __enter__(self) -> NoReturn:
"""
Replace old __enter__ method to avoid using it.
"""Sync context manager entry is not supported because lock acquisition is a coroutine."""
msg = "Use `async with` — acquire/release are coroutines and cannot be awaited in a sync context manager."
raise NotImplementedError(msg)
NOTE: DO NOT USE `with` FOR ASYNCIO LOCKS, USE `async with` INSTEAD.
:return: none
:rtype: NoReturn
"""
msg = "Do not use `with` for asyncio locks, use `async with` instead."
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: object,
) -> None:
"""Sync context manager exit is not supported because lock release is a coroutine."""
msg = "Use `async with` — acquire/release are coroutines and cannot be awaited in a sync context manager."
raise NotImplementedError(msg)
async def __aenter__(self) -> Self:
"""
Acquire the lock.
:return: the lock object
:returns: the lock object
"""
await self.acquire()
@ -314,9 +371,15 @@ class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
await self.release()
def __del__(self) -> None:
"""Called when the lock object is deleted."""
with contextlib.suppress(RuntimeError):
loop = self.loop or asyncio.get_running_loop()
"""Release on deletion — safe to call during GC even when no event loop is running."""
with contextlib.suppress(Exception):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
# No running loop — try stored loop or create one
loop = self._context.loop if self._context.loop and not self._context.loop.is_closed() else None
if loop is None:
return
if not loop.is_running(): # pragma: no cover
loop.run_until_complete(self.release(force=True))
else:

View file

@ -1,5 +1,6 @@
# file generated by setuptools-scm
# file generated by vcs-versioning
# don't change, don't track in version control
from __future__ import annotations
__all__ = [
"__version__",
@ -10,25 +11,14 @@ __all__ = [
"commit_id",
]
TYPE_CHECKING = False
if TYPE_CHECKING:
from typing import Tuple
from typing import Union
VERSION_TUPLE = Tuple[Union[int, str], ...]
COMMIT_ID = Union[str, None]
else:
VERSION_TUPLE = object
COMMIT_ID = object
version: str
__version__: str
__version_tuple__: VERSION_TUPLE
version_tuple: VERSION_TUPLE
commit_id: COMMIT_ID
__commit_id__: COMMIT_ID
__version_tuple__: tuple[int | str, ...]
version_tuple: tuple[int | str, ...]
commit_id: str | None
__commit_id__: str | None
__version__ = version = '3.20.3'
__version_tuple__ = version_tuple = (3, 20, 3)
__version__ = version = '3.29.3'
__version_tuple__ = version_tuple = (3, 29, 3)
__commit_id__ = commit_id = None