Files
routine/routine/storage.py
T

297 lines
11 KiB
Python

"""Where the routine and the history are kept, and moving them (no Qt: tested in tests/test_storage.py).
First run and the routine (load_schedule): the user's own schedule.json always wins when it exists.
Without one:
- a first run (no history yet) starts from the template, shared/schedule.json, copied into place;
- an existing user (a routine.db already there, from before the template replaced the routine
that used to ship in the repo) is never switched to the template: a schedule.from-repo*.json
beside it (a saved copy of that old routine) is adopted, else the routine is rebuilt from the
most recent day recorded in the history (days of the week can't be told from one day: whatever
was on that day is kept, on every day).
A schedule.json that exists but can't be read is set aside (renamed, never deleted) first.
Moving (move_file): copy to a temporary name beside the target, check the copy (same bytes, and
for a database, SQLite's integrity check), put it in place, check again, and only then remove the
original. Any failure leaves the original where it was. A file already at the target is never
overwritten silently: the caller picks "use" (switch to it; ours stays where it is, untouched) or
"replace" (it's kept beside as <name>.replaced-<time><ext>, then ours goes in).
"""
import datetime as dt
import glob
import hashlib
import json
import os
import shutil
import sqlite3
import sys
import urllib.request
from routine import paths
FROM_REPO = "schedule.from-repo" # schedule.from-repo.json, schedule.from-repo.1.json, ...
def _ro(path):
"""A read-only SQLite URI for path (opening it never changes or creates the file)."""
return "file:" + urllib.request.pathname2url(os.path.abspath(path)) + "?mode=ro"
def _log(msg):
print("routine: " + msg, file=sys.stderr)
# ── reading and writing a routine ──
def valid_schedule(s):
return isinstance(s, dict) and isinstance(s.get("blocks"), list)
def read_schedule(path):
with open(path, encoding="utf-8") as f:
s = json.load(f)
if not valid_schedule(s):
raise ValueError("not a routine")
return s
def write_schedule(path, sch):
os.makedirs(os.path.dirname(path), exist_ok=True)
tmp = path + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(sch, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
def _stamp():
return dt.datetime.now().strftime("%Y%m%d-%H%M%S")
def _free_name(path, tag):
"""path with ".<tag>" before its extension, made unique with -2, -3, ..."""
stem, ext = os.path.splitext(path)
out, n = "%s.%s%s" % (stem, tag, ext), 2
while os.path.exists(out):
out, n = "%s.%s-%d%s" % (stem, tag, n, ext), n + 1
return out
# ── the first run, and existing users ──
def schedule_from_day(day):
"""A routine rebuilt from one day's snapshot (its blocks, tasks, sub-tasks, hints and calm
marks, with their ids, so the history carries on). Days of the week aren't in a snapshot."""
blocks = []
for b in day.get("blocks", []):
top = []
stack = [] # (depth, full id, item dict)
for it in b.get("items", []):
depth = int(it.get("depth", 0))
while stack and stack[-1][0] >= depth:
stack.pop()
parent_id = stack[-1][1] if stack else b["id"]
full = it["id"]
own = full[len(parent_id) + 1:] if full.startswith(parent_id + "/") else full.rsplit("/", 1)[-1]
x = {"id": own, "label": it["label"]}
if it.get("hint"):
x["hint"] = True
if it.get("calm"):
x["calm"] = True
(stack[-1][2].setdefault("items", []) if stack else top).append(x)
stack.append((depth, full, x))
blocks.append({"id": b["id"], "label": b["label"], "start": b["start"], "end": b["end"],
"color": b["color"], "items": top})
return {"rev": day.get("rev", 1), "dayStartsAt": 4, "blocks": blocks}
def latest_day(db):
"""The most recent day in a routine.db that has any blocks, read without changing the file; None if none."""
try:
con = sqlite3.connect(_ro(db), uri=True)
except sqlite3.Error:
return None
try:
for (doc,) in con.execute("SELECT doc FROM days ORDER BY date DESC"):
day = json.loads(doc)
if day.get("blocks"):
return day
except (sqlite3.Error, ValueError):
return None
finally:
con.close()
return None
def from_repo_copies(*dirs):
"""Saved copies of the routine that used to ship in the repo, newest first."""
found = set()
for d in dirs:
found.update(glob.glob(os.path.join(d, FROM_REPO + "*.json")))
return sorted(found, key=lambda p: (os.path.getmtime(p), p), reverse=True)
def load_schedule(cfg, template=None):
"""The routine to use (see the module docstring), made into the user's own schedule.json when
there wasn't one. Returns (schedule, how): how is "yours", "template", "adopted" or "rebuilt"."""
template = template or paths.SCHEDULE
path, db = paths.schedule_path(cfg), paths.db_path(cfg)
if os.path.exists(path):
try:
return read_schedule(path), "yours"
except (OSError, ValueError):
aside = _free_name(path, "unreadable-" + _stamp())
try:
os.replace(path, aside)
_log("couldn't read the routine; set it aside as " + os.path.basename(aside))
except OSError:
_log("couldn't read the routine (left as it is); using the template for now")
return read_schedule(template), "template"
if os.path.exists(db): # an existing user, from before the template
for p in from_repo_copies(os.path.dirname(path), paths.DATA):
try:
sch = read_schedule(p)
except (OSError, ValueError):
continue
write_schedule(path, sch)
_log("routine taken from " + os.path.basename(p))
return sch, "adopted"
day = latest_day(db)
if day is not None:
sch = schedule_from_day(day)
write_schedule(path, sch)
_log("routine rebuilt from the history's most recent day")
return sch, "rebuilt"
sch = read_schedule(template)
write_schedule(path, sch)
return sch, "template"
# ── moving a file to another folder ──
class MoveError(Exception):
pass
class TargetTaken(MoveError):
"""There's already a file at the target: the caller has to pick "use" or "replace"."""
def _sha(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def check_db(path):
"""Raises MoveError unless path is a sound SQLite database."""
try:
con = sqlite3.connect(_ro(path), uri=True)
try:
ok = con.execute("PRAGMA integrity_check").fetchone()[0]
con.execute("SELECT count(*) FROM days").fetchone()
finally:
con.close()
except sqlite3.Error as e:
raise MoveError("not a routine database (%s)" % e)
if ok != "ok":
raise MoveError("the database failed its check (%s)" % ok)
def check_schedule(path):
try:
read_schedule(path)
except (OSError, ValueError) as e:
raise MoveError("not a routine (%s)" % e)
def _fsync_dir(d):
try:
fd = os.open(d, os.O_RDONLY)
except OSError:
return
try:
os.fsync(fd)
except OSError:
pass
finally:
os.close(fd)
def move_file(src, dst, taken=None, check=None, copy=shutil.copyfile):
"""Move src to dst safely (the module docstring). taken: None (raise TargetTaken if dst exists),
"use" (keep dst, leave src alone) or "replace". check(path) raises MoveError for a bad file.
Returns what happened: "same", "nothing" (no src: only the location changes), "used", "moved".
On any error src is untouched, and a file that was at dst is back there."""
if os.path.abspath(src) == os.path.abspath(dst) or \
(os.path.exists(src) and os.path.exists(dst) and os.path.samefile(src, dst)):
return "same"
if not os.path.exists(src): # nothing of ours to move
if os.path.exists(dst):
if check:
check(dst)
return "used"
try:
os.makedirs(os.path.dirname(dst), exist_ok=True)
except OSError as e:
raise MoveError("can't make the folder (%s)" % e)
return "nothing"
for side in ("-wal", "-journal"):
if os.path.exists(src + side):
raise MoveError("%s is still in use (%s)" % (os.path.basename(src), os.path.basename(src + side)))
if os.path.exists(dst):
if taken is None:
raise TargetTaken(dst)
if taken == "use":
if check:
check(dst)
return "used"
if taken != "replace":
raise ValueError(taken)
folder = os.path.dirname(dst)
partial = dst + ".partial"
backup, placed = None, False
try:
os.makedirs(folder, exist_ok=True)
want = _sha(src)
copy(src, partial)
with open(partial, "rb") as f:
os.fsync(f.fileno())
if _sha(partial) != want:
raise MoveError("the copy didn't match the original")
if check:
check(partial)
if os.path.exists(dst): # taken == "replace": keep theirs beside
backup = _free_name(dst, "replaced-" + _stamp())
os.replace(dst, backup)
os.replace(partial, dst)
placed = True
_fsync_dir(folder)
if _sha(dst) != want:
raise MoveError("the file in its new place didn't match the original")
except (OSError, MoveError) as e:
try:
os.remove(partial)
except OSError:
pass
try:
if backup: # put theirs back where it was
os.replace(backup, dst)
elif placed: # a bad copy of ours: take it away again
os.remove(dst)
except OSError:
pass
if isinstance(e, MoveError):
raise
raise MoveError(str(e))
os.remove(src)
_fsync_dir(os.path.dirname(src))
return "moved"
def replaced_name(dst):
"""What a "replace" will rename the file at dst to (for the question asked first)."""
return os.path.basename(_free_name(dst, "replaced-" + _stamp()))