feat: 故障推送 / 公需课自动重学 / 插件解耦，发版 v0.0.8
体检修复：启动日志不再打印 AUTH_TOKEN、密钥可清空、推送失败不占整段冷却、目录名与文案统一成 plugins/。
This commit is contained in:
study-tool
@@ -0,0 +1,387 @@
|
||||
"""SQLite 存储层（SQLAlchemy ORM）。
|
||||
|
||||
表
|
||||
settings 系统配置（name -> JSON），目前只有 llm
|
||||
kv 通用缓存（任意 key -> JSON），例如 StudyClient 试出来的接口前缀
|
||||
accounts 账号 / 密码 / 会话 cookie / 课程缓存 / 勾选 / 开关
|
||||
login_log 每次登录尝试（平台原话）
|
||||
run_log 每个账号（key）的运行日志
|
||||
run_state 运行状态快照，进程重启后 /api/runs 还能看到上次跑到哪
|
||||
snapshots 课程页原始 HTML（站点改版时定位选择器用）
|
||||
|
||||
并发模型
|
||||
* WAL + busy_timeout=15s：读写互不阻塞，写锁竞争自动重试
|
||||
* scoped_session：每个线程一条 Session，ThreadPoolExecutor 里多线程写日志不会串
|
||||
* 本进程内的写操作再用一把 RLock 串起来，避免多线程 commit 抖动（读不加锁）
|
||||
"""
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
from sqlalchemy import create_engine, delete, event, func, select
|
||||
from sqlalchemy.orm import scoped_session, sessionmaker
|
||||
|
||||
from study_tool.config import DATA
|
||||
from study_tool.database.models import (Account, Base, Kv, LoginLog, RunLog,
|
||||
RunState, Setting, Snapshot)
|
||||
|
||||
DATA_DIR = os.environ.get("DATA_DIR") or DATA # 仓库根目录的 data/，路径只在 config 里算一次
|
||||
DB_PATH = os.environ.get("DB_PATH") or os.path.join(DATA_DIR, "study.db")
|
||||
|
||||
# 每个账号保留的日志行数，超出就丢最旧的（远程长期挂机，不能无限涨）
|
||||
RUN_LOG_KEEP = int(os.environ.get("RUN_LOG_KEEP", "2000") or 2000)
|
||||
# 页面快照保留条数
|
||||
SNAPSHOT_KEEP = int(os.environ.get("SNAPSHOT_KEEP", "200") or 200)
|
||||
|
||||
os.makedirs(os.path.dirname(os.path.abspath(DB_PATH)), exist_ok=True)
|
||||
|
||||
engine = create_engine("sqlite:///" + os.path.abspath(DB_PATH),
|
||||
connect_args={"check_same_thread": False, "timeout": 15})
|
||||
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def _sqlite_pragmas(dbapi_conn, _record):
|
||||
cur = dbapi_conn.cursor()
|
||||
cur.execute("PRAGMA journal_mode=WAL")
|
||||
cur.execute("PRAGMA synchronous=NORMAL")
|
||||
cur.execute("PRAGMA busy_timeout=15000")
|
||||
cur.close()
|
||||
|
||||
|
||||
Session = scoped_session(sessionmaker(bind=engine, expire_on_commit=False))
|
||||
_lock = threading.RLock() # 只保护“写”
|
||||
_write_count = {}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 基础
|
||||
|
||||
def init():
|
||||
"""建库建表（幂等，启动时调用一次）。"""
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
|
||||
def db_info():
|
||||
try:
|
||||
size = os.path.getsize(DB_PATH) if os.path.exists(DB_PATH) else 0
|
||||
except OSError:
|
||||
size = 0
|
||||
models = {"accounts": Account, "run_log": RunLog, "login_log": LoginLog,
|
||||
"snapshots": Snapshot, "run_state": RunState, "kv": Kv, "settings": Setting}
|
||||
counts = {}
|
||||
with Session() as s:
|
||||
for name, model in models.items():
|
||||
counts[name] = s.scalar(select(func.count()).select_from(model))
|
||||
return {"path": DB_PATH, "size": size, "counts": counts}
|
||||
|
||||
|
||||
init()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ settings
|
||||
|
||||
def get_setting(name, default=None):
|
||||
with Session() as s:
|
||||
row = s.get(Setting, name)
|
||||
return row.value if row is not None else default
|
||||
|
||||
|
||||
def set_setting(name, value):
|
||||
with _lock, Session() as s:
|
||||
row = s.get(Setting, name)
|
||||
if row is None:
|
||||
row = Setting(name=name)
|
||||
s.add(row)
|
||||
row.value = value
|
||||
row.updated = time.time()
|
||||
s.commit()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ kv 缓存
|
||||
|
||||
def kv_get(key, default=None):
|
||||
with Session() as s:
|
||||
row = s.get(Kv, key)
|
||||
return row.value if row is not None else default
|
||||
|
||||
|
||||
def kv_set(key, value):
|
||||
with _lock, Session() as s:
|
||||
row = s.get(Kv, key)
|
||||
if row is None:
|
||||
row = Kv(key=key)
|
||||
s.add(row)
|
||||
row.value = value
|
||||
row.updated = time.time()
|
||||
s.commit()
|
||||
|
||||
|
||||
def kv_del(key):
|
||||
with _lock, Session() as s:
|
||||
s.execute(delete(Kv).where(Kv.key == key))
|
||||
s.commit()
|
||||
|
||||
|
||||
def kv_all():
|
||||
with Session() as s:
|
||||
return {row.key: row.value for row in s.scalars(select(Kv))}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ accounts
|
||||
|
||||
def _acc_row(row):
|
||||
"""ORM 对象 -> 前端用的记录（不含密码）。"""
|
||||
return {
|
||||
"id": row.id,
|
||||
"username": row.username or "",
|
||||
"platform": row.platform or "",
|
||||
"note": row.note or "",
|
||||
"user": row.user or "",
|
||||
"cookies": row.cookies or [],
|
||||
"courses": row.courses or [],
|
||||
"selected": row.selected or [],
|
||||
"status": row.status or "idle",
|
||||
"lastError": row.last_error or "",
|
||||
"lastErrorAt": row.last_error_at or 0,
|
||||
"optQuiz": bool(row.opt_quiz),
|
||||
"optFeedback": bool(row.opt_feedback),
|
||||
"loginAt": row.login_at or 0,
|
||||
"created": row.created or 0,
|
||||
"updated": row.updated or 0,
|
||||
"hasPassword": bool(row.password),
|
||||
}
|
||||
|
||||
|
||||
def account_list():
|
||||
with Session() as s:
|
||||
rows = s.scalars(select(Account).order_by(Account.created, Account.id)).all()
|
||||
return [_acc_row(row) for row in rows]
|
||||
|
||||
|
||||
def account_get(account_id):
|
||||
if not account_id:
|
||||
return None
|
||||
with Session() as s:
|
||||
row = s.get(Account, str(account_id))
|
||||
return _acc_row(row) if row is not None else None
|
||||
|
||||
|
||||
def account_password(account_id):
|
||||
if not account_id:
|
||||
return ""
|
||||
with Session() as s:
|
||||
row = s.get(Account, str(account_id))
|
||||
return (row.password or "") if row is not None else ""
|
||||
|
||||
|
||||
def account_exists(account_id):
|
||||
if not account_id:
|
||||
return False
|
||||
with Session() as s:
|
||||
return s.get(Account, str(account_id)) is not None
|
||||
|
||||
|
||||
def account_upsert(rec):
|
||||
"""新增/更新账号。rec['password'] 为空(None)时保留库里原有密码。
|
||||
|
||||
明文密码要留在库里：会话失效时后台要能自己重登，不能等用户回来输密码。
|
||||
"""
|
||||
now = time.time()
|
||||
aid = str(rec.get("id") or "").strip()
|
||||
if not aid:
|
||||
aid = "a%s%04d" % (int(now * 1000), int(time.time() * 10000) % 10000)
|
||||
with _lock, Session() as s:
|
||||
row = s.get(Account, aid)
|
||||
if row is None:
|
||||
row = Account(id=aid, created=float(rec.get("created") or now))
|
||||
s.add(row)
|
||||
row.username = rec.get("username") or ""
|
||||
platform = (rec.get("platform") or "").strip()
|
||||
if platform:
|
||||
row.platform = platform
|
||||
pwd = rec.get("password")
|
||||
if pwd is not None:
|
||||
row.password = str(pwd)
|
||||
row.note = rec.get("note") or ""
|
||||
row.user = rec.get("user") or ""
|
||||
row.cookies = rec.get("cookies") or []
|
||||
row.courses = rec.get("courses") or []
|
||||
row.selected = rec.get("selected") or []
|
||||
row.status = rec.get("status") or "idle"
|
||||
row.last_error = rec.get("lastError") or ""
|
||||
row.last_error_at = float(rec.get("lastErrorAt") or 0)
|
||||
row.opt_quiz = 1 if rec.get("optQuiz") is True else 0
|
||||
row.opt_feedback = 0 if rec.get("optFeedback") is False else 1
|
||||
row.login_at = float(rec.get("loginAt") or 0)
|
||||
row.created = float(rec.get("created") or row.created or now)
|
||||
row.updated = now
|
||||
s.commit()
|
||||
return aid
|
||||
|
||||
|
||||
def account_save_login(account_id, user, cookies, courses):
|
||||
"""登录成功后把新的会话写回账号（服务端自己记，不依赖前端回传）。"""
|
||||
rec = account_get(account_id)
|
||||
if not rec:
|
||||
return
|
||||
courses = courses or []
|
||||
plans = {c.get("plan_id") for c in courses}
|
||||
rec.update({
|
||||
"user": user or rec.get("user") or "",
|
||||
"cookies": cookies or [],
|
||||
"courses": courses,
|
||||
"selected": [p for p in (rec.get("selected") or []) if p in plans],
|
||||
"status": "logged_in",
|
||||
"lastError": "",
|
||||
"lastErrorAt": 0,
|
||||
"loginAt": time.time(),
|
||||
})
|
||||
rec.pop("hasPassword", None)
|
||||
account_upsert(rec)
|
||||
|
||||
|
||||
def account_save_cookies(account_id, cookies):
|
||||
"""只更新会话：学习中掉线重登后写回新会话，不动课程缓存和勾选。"""
|
||||
rec = account_get(account_id)
|
||||
if not rec:
|
||||
return
|
||||
rec.update({"cookies": cookies or [], "status": "logged_in", "lastError": "", "lastErrorAt": 0})
|
||||
rec.pop("hasPassword", None)
|
||||
account_upsert(rec)
|
||||
|
||||
|
||||
def account_delete(account_id):
|
||||
"""删账号，连同它的运行状态、课程日志、登录记录一起删。
|
||||
|
||||
登录日志的 key 和 login_log.username 记的是用户名（手机号），不是账号 id，
|
||||
两边都要清，否则删完账号旧日志还留在页面上。
|
||||
"""
|
||||
aid = str(account_id or "")
|
||||
acc = account_get(aid) or {}
|
||||
uname = str(acc.get("username") or "")
|
||||
with _lock, Session() as s:
|
||||
s.execute(delete(Account).where(Account.id == aid))
|
||||
s.execute(delete(RunState).where(RunState.key == aid))
|
||||
for key in (aid, uname):
|
||||
if key:
|
||||
s.execute(delete(RunLog).where(RunLog.key == key))
|
||||
s.execute(delete(LoginLog).where(LoginLog.username == key))
|
||||
s.commit()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 登录记录
|
||||
|
||||
def login_log_add(username, ok, code, elapsed, stage="", error="", ts=None):
|
||||
with _lock, Session() as s:
|
||||
s.add(LoginLog(ts=ts or time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
username=username, ok=1 if ok else 0, code=str(code or ""),
|
||||
elapsed=float(elapsed or 0), stage=str(stage or ""),
|
||||
error=str(error or "").replace("\n", " ")[:500]))
|
||||
s.commit()
|
||||
|
||||
|
||||
def login_log_tail(n=40):
|
||||
"""返回旧版文本行格式（ts\\t用户名\\tOK\\tcode\\t耗时\\tstage\\t错误），前端不用改。"""
|
||||
n = max(1, min(int(n or 40), 500))
|
||||
with Session() as s:
|
||||
rows = s.scalars(select(LoginLog).order_by(LoginLog.id.desc()).limit(n)).all()
|
||||
return ["%s\t%s\t%s\t%s\t%.1fs\t%s\t%s" % (
|
||||
r.ts, r.username, "OK" if r.ok else "FAIL", r.code,
|
||||
r.elapsed or 0, r.stage or "", r.error or "") for r in reversed(rows)]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 运行日志
|
||||
|
||||
def run_log_add(key, line, ts=None):
|
||||
key = str(key or "")
|
||||
with _lock, Session() as s:
|
||||
s.add(RunLog(key=key, ts=ts or time.time(), line=line))
|
||||
s.commit()
|
||||
n = _write_count.get("run_log", 0) + 1
|
||||
_write_count["run_log"] = n
|
||||
if n % 200 == 0: # 每 200 行清一次，避免每行都跑一遍 DELETE
|
||||
keep = (select(RunLog.id).where(RunLog.key == key)
|
||||
.order_by(RunLog.id.desc()).limit(RUN_LOG_KEEP))
|
||||
s.execute(delete(RunLog).where(RunLog.key == key, RunLog.id.not_in(keep)))
|
||||
s.commit()
|
||||
|
||||
|
||||
def run_log_tail(key, n=200):
|
||||
n = max(1, min(int(n or 200), RUN_LOG_KEEP))
|
||||
with Session() as s:
|
||||
rows = s.scalars(select(RunLog.line).where(RunLog.key == str(key))
|
||||
.order_by(RunLog.id.desc()).limit(n)).all()
|
||||
return list(reversed(rows))
|
||||
|
||||
|
||||
def run_log_keys():
|
||||
with Session() as s:
|
||||
rows = s.execute(select(RunLog.key, func.count(RunLog.id), func.max(RunLog.ts))
|
||||
.group_by(RunLog.key)).all()
|
||||
return [{"key": key, "lines": count, "updated": updated} for key, count, updated in rows]
|
||||
|
||||
|
||||
def run_log_clear(key):
|
||||
with _lock, Session() as s:
|
||||
s.execute(delete(RunLog).where(RunLog.key == str(key)))
|
||||
s.commit()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 运行状态
|
||||
|
||||
def run_state_set(key, state):
|
||||
st = dict(state or {})
|
||||
st.pop("log", None) # 日志单独进 run_log 表，不在这里重复一份
|
||||
with _lock, Session() as s:
|
||||
row = s.get(RunState, str(key))
|
||||
if row is None:
|
||||
row = RunState(key=str(key))
|
||||
s.add(row)
|
||||
row.state = st
|
||||
row.updated = time.time()
|
||||
s.commit()
|
||||
|
||||
|
||||
def run_state_all():
|
||||
with Session() as s:
|
||||
return {row.key: row.state for row in s.scalars(select(RunState))
|
||||
if isinstance(row.state, dict)}
|
||||
|
||||
|
||||
def run_state_delete(key):
|
||||
with _lock, Session() as s:
|
||||
s.execute(delete(RunState).where(RunState.key == str(key or "")))
|
||||
s.commit()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 页面快照
|
||||
|
||||
def snapshot_add(plan_id, project_id, url, html):
|
||||
with _lock, Session() as s:
|
||||
row = Snapshot(plan_id=str(plan_id or ""), project_id=str(project_id or ""),
|
||||
url=url or "", html=html or "", created=time.time())
|
||||
s.add(row)
|
||||
s.flush()
|
||||
sid = row.id
|
||||
keep = select(Snapshot.id).order_by(Snapshot.id.desc()).limit(SNAPSHOT_KEEP)
|
||||
s.execute(delete(Snapshot).where(Snapshot.id.not_in(keep)))
|
||||
s.commit()
|
||||
return sid
|
||||
|
||||
|
||||
def snapshot_get(sid):
|
||||
with Session() as s:
|
||||
row = s.get(Snapshot, int(sid))
|
||||
if row is None:
|
||||
return None
|
||||
return {c.name: getattr(row, c.name) for c in row.__table__.columns}
|
||||
|
||||
|
||||
def snapshot_list(limit=50):
|
||||
limit = max(1, min(int(limit or 50), 500))
|
||||
with Session() as s:
|
||||
rows = s.execute(select(Snapshot.id, Snapshot.plan_id, Snapshot.project_id,
|
||||
Snapshot.url, Snapshot.created,
|
||||
func.length(Snapshot.html).label("size"))
|
||||
.order_by(Snapshot.id.desc()).limit(limit)).all()
|
||||
return [{"id": r.id, "plan_id": r.plan_id, "project_id": r.project_id,
|
||||
"url": r.url, "created": r.created, "size": r.size} for r in rows]
|
||||
@@ -0,0 +1,102 @@
|
||||
"""SQLAlchemy ORM 模型：账号 / 设置 / 登录记录 / 运行日志 / 运行状态 / 快照。"""
|
||||
from sqlalchemy import Float, Index, Integer, String
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
from sqlalchemy.types import JSON
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class Setting(Base):
|
||||
"""系统配置（name -> JSON），目前只有 llm。"""
|
||||
|
||||
__tablename__ = "settings"
|
||||
|
||||
name: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
value: Mapped[dict] = mapped_column(JSON, nullable=False)
|
||||
updated: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
|
||||
|
||||
class Kv(Base):
|
||||
"""通用缓存（任意 key -> JSON），例如 StudyClient 试出来的接口前缀。"""
|
||||
|
||||
__tablename__ = "kv"
|
||||
|
||||
key: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
value: Mapped[dict] = mapped_column(JSON)
|
||||
updated: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
|
||||
|
||||
class Account(Base):
|
||||
"""账号 / 密码 / 会话 cookie / 课程缓存 / 勾选 / 开关。"""
|
||||
|
||||
__tablename__ = "accounts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
username: Mapped[str] = mapped_column(String, default="")
|
||||
platform: Mapped[str] = mapped_column(String, default="")
|
||||
password: Mapped[str] = mapped_column(String, default="")
|
||||
note: Mapped[str] = mapped_column(String, default="")
|
||||
user: Mapped[str] = mapped_column(String, default="")
|
||||
cookies: Mapped[list] = mapped_column(JSON, default=list)
|
||||
courses: Mapped[list] = mapped_column(JSON, default=list)
|
||||
selected: Mapped[list] = mapped_column(JSON, default=list)
|
||||
status: Mapped[str] = mapped_column(String, default="idle")
|
||||
last_error: Mapped[str] = mapped_column(String, default="")
|
||||
last_error_at: Mapped[float] = mapped_column(Float, default=0)
|
||||
opt_quiz: Mapped[int] = mapped_column(Integer, default=0)
|
||||
opt_feedback: Mapped[int] = mapped_column(Integer, default=1)
|
||||
login_at: Mapped[float] = mapped_column(Float, default=0)
|
||||
created: Mapped[float] = mapped_column(Float, default=0)
|
||||
updated: Mapped[float] = mapped_column(Float, default=0)
|
||||
|
||||
|
||||
class LoginLog(Base):
|
||||
"""每次登录尝试（平台原话）。"""
|
||||
|
||||
__tablename__ = "login_log"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
ts: Mapped[str] = mapped_column(String)
|
||||
username: Mapped[str] = mapped_column(String)
|
||||
ok: Mapped[int] = mapped_column(Integer)
|
||||
code: Mapped[str] = mapped_column(String)
|
||||
elapsed: Mapped[float] = mapped_column(Float)
|
||||
stage: Mapped[str] = mapped_column(String)
|
||||
error: Mapped[str] = mapped_column(String)
|
||||
|
||||
|
||||
class RunLog(Base):
|
||||
"""每个账号（key）的运行日志。"""
|
||||
|
||||
__tablename__ = "run_log"
|
||||
__table_args__ = (Index("idx_run_log_key", "key", "id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
key: Mapped[str] = mapped_column(String, nullable=False)
|
||||
ts: Mapped[float] = mapped_column(Float)
|
||||
line: Mapped[str] = mapped_column(String)
|
||||
|
||||
|
||||
class RunState(Base):
|
||||
"""运行状态快照，进程重启后 /api/runs 还能看到上次跑到哪。"""
|
||||
|
||||
__tablename__ = "run_state"
|
||||
|
||||
key: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
state: Mapped[dict] = mapped_column(JSON)
|
||||
updated: Mapped[float] = mapped_column(Float)
|
||||
|
||||
|
||||
class Snapshot(Base):
|
||||
"""课程页原始 HTML（站点改版时定位选择器用）。"""
|
||||
|
||||
__tablename__ = "snapshots"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
plan_id: Mapped[str] = mapped_column(String)
|
||||
project_id: Mapped[str] = mapped_column(String)
|
||||
url: Mapped[str] = mapped_column(String)
|
||||
html: Mapped[str] = mapped_column(String)
|
||||
created: Mapped[float] = mapped_column(Float)
|
||||
Reference in New Issue
Block a user