Python: JSONL store writes through commit_store_file (jsonl-tmp-leak-on-replace-failure) (#154)
This commit is contained in:
parent
558526c125
commit
adbbb780fd
3 changed files with 15 additions and 3 deletions
|
|
@ -9,6 +9,7 @@
|
|||
### Changed
|
||||
|
||||
- `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import.
|
||||
- `JsonLinesBackingStore` writes through the same `commit_store_file` as the single-file store: a unique staged file, fsynced before the replace and removed when the replace fails, so a refused replace no longer leaves `<path>.tmp` behind. It staged a fixed `<path>.tmp` without an fsync.
|
||||
- The single-file store's `push`, `push_all` and `delete` take the cross-process lock on `<path>.lock` (flock; LockFileEx over the whole file on Windows) across the read and the replace, waiting up to `lock_timeout` seconds (30 by default, `TimeoutError` after), and exclude the C#, Rust and TypeScript runtimes' writers. The staged file is `<path>.tmp-<pid>-<uuid>`, fsynced before the replace, with a directory fsync after it off Windows. A fixed `<path>.tmp` is no longer used.
|
||||
|
||||
### Breaking
|
||||
|
|
|
|||
|
|
@ -130,7 +130,6 @@ class JsonLinesBackingStore:
|
|||
|
||||
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp = self.path.with_suffix(self.path.suffix + ".tmp")
|
||||
lines = []
|
||||
for envelope in sorted(envelopes, key=lambda item: (item.type, item.key)):
|
||||
lines.append(json.dumps({
|
||||
|
|
@ -139,8 +138,7 @@ class JsonLinesBackingStore:
|
|||
"payload": base64.b64encode(envelope.payload).decode("ascii"),
|
||||
"stored_at": envelope.stored_at,
|
||||
}, ensure_ascii=False, sort_keys=True))
|
||||
temp.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
|
||||
_replace_store(temp, self.path)
|
||||
commit_store_file(self.path, ("\n".join(lines) + ("\n" if lines else "")).encode("utf-8"))
|
||||
|
||||
|
||||
class SingleFileMessagePackBackingStore:
|
||||
|
|
|
|||
|
|
@ -107,6 +107,19 @@ class StoreWritesRouteThroughTheReplace(unittest.TestCase):
|
|||
self.assertEqual([destination for _, destination in seen], [binary, lines])
|
||||
self.assertTrue(binary.read_bytes() and lines.read_text())
|
||||
|
||||
def test_a_failed_jsonl_replace_leaves_no_staged_file_and_the_old_store(self):
|
||||
envelope = CultCacheEnvelope(key="k", type="t", payload=b"\x01", stored_at="2026-11-06T00:00:00Z")
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
lines = Path(directory) / "s.jsonl"
|
||||
store = stores.JsonLinesBackingStore(lines)
|
||||
store.push_all([envelope])
|
||||
before = lines.read_bytes()
|
||||
with mock.patch.object(stores, "_replace_store", side_effect=OSError("replace refused")):
|
||||
with self.assertRaises(OSError):
|
||||
store.push_all([CultCacheEnvelope(key="k2", type="t", payload=b"\x02", stored_at="2026-11-06T00:00:00Z")])
|
||||
self.assertEqual(lines.read_bytes(), before)
|
||||
self.assertEqual([p.name for p in Path(directory).iterdir() if ".tmp" in p.name], [])
|
||||
|
||||
@unittest.skipUnless(sys.platform == "win32", "needs a real Windows")
|
||||
def test_real_kernel_replaces_through_both_primitives(self):
|
||||
kernel = stores.WindowsReplaceKernel()
|
||||
|
|
|
|||
Loading…
Reference in a new issue