fix(env): 本机启动读取根目录 .env,监听地址兼容 BIND_ADDR

- study_tool/__init__.py 用 python-dotenv 加载根目录 .env,位置早于
  store / config 读环境变量;已有环境变量优先,Docker 注入的不受影响
- app.py 监听地址改为 HOST -> BIND_ADDR -> 127.0.0.1 依次回退
- 新增依赖 python-dotenv>=1.0(pyproject.toml / requirements.txt / uv.lock)
- README 环境变量表补充 .env 说明与 BIND_ADDR / TZ 两行
- CHANGELOG / docs/OPTIMIZATION.md 记录本次修复,版本号 0.0.6 -> 0.0.7
This commit is contained in:
study-tool
2026-10-31 16:12:52 +08:00
parent cfa960221e
commit 8e8d6b83f1
14 changed files with 100 additions and 50 deletions
+825
View File
@@ -0,0 +1,825 @@
"""旧版平台(jspx.dgjy.net):会话 / cookie / 登录 / 课程抓取。
对外:new_session / login / scrape_courses / discover。
"""
import json
import re
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor
from html import unescape as html_unescape
from urllib.parse import unquote, urljoin, urlsplit
import requests
from bs4 import BeautifulSoup
from study_tool.config import *
from study_tool.platforms import registry
from study_tool.platforms.detect import detect_generation, has_study_marker
from study_tool.runtime import RUNS, RUNS_LOCK, log, set_state
# ------------------------------------------------------------------------ crypto
def str_enc(data: str) -> str:
r = subprocess.run([NODE, DESENC, data, "1", "2", "3"],
capture_output=True, text=True, timeout=30)
if r.returncode != 0:
raise RuntimeError("strEnc failed: %s" % (r.stderr or "")[:200])
return r.stdout.strip()
# ------------------------------------------------------------------- http client
def new_session(cookies=None):
s = requests.Session()
s.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
for c in (cookies or []):
if "dgjy" not in (c.get("domain") or ""):
continue
s.cookies.set(c["name"], c.get("value") or "",
domain=(c.get("domain") or "").lstrip("."),
path=c.get("path") or "/")
return s
def dump_cookies(s):
return [{"name": c.name, "value": c.value, "domain": c.domain, "path": c.path}
for c in s.cookies]
def whoami(html):
soup = BeautifulSoup(html, "lxml")
hdr = soup.select_one("#header")
if hdr:
m = re.search(r"你好[,,]?\s*(\S+)\s*老师", re.sub(r"\s+", " ", hdr.get_text(" ", strip=True)))
if m:
return m.group(1)
return ""
def parse_ids(text):
plan = project = ""
m = re.search(r"planId=([0-9A-Fa-f]{16,})", text or "")
if m:
plan = m.group(1)
m = re.search(r"projectId=([0-9A-Fa-f]{16,})", text or "")
if m:
project = m.group(1)
return plan, project
def _follow_js_login(s, page):
"""补上浏览器才会执行的两跳,否则只拿得到 CASTGC、拿不到 jspx 的会话。 `oauth2.0/authorize` 链的最后一跳**不是 302**,而是 dj.dgjy.net 上一个
1486 字节的 JS 中转页,里面动态插 <script>:
scriptLogin.src = 'https://passport.dgjy.net/scriptLogin?action=simpleLogin&d1=...&d2=...';
scriptLogin 返回的是 JS,不是跳转:
var loginError=null;
var verifyURL="https://dj.dgjy.net/fpf/login/verify.action?ticket=...&auth=...&sid=...";
location.href=verifyURL;
verify.action 之后才会 302 → loginCall → jspx/tstudy/passport/login.action,
到这一步 jspx.dgjy.net 的 JSESSIONID 才真正下发。
requests 不执行 JS,所以这两跳必须手工走一遍。
"""
m = re.search(r"scriptLogin\.src\s*=\s*'([^']+)'", page or "")
if not m:
return False
r = s.get(html_unescape(m.group(1)), timeout=40)
body = r.text or ""
if re.search(r"loginError\s*=\s*(?!null)", body):
raise RuntimeError("SSO 校验未通过:%s" % body.strip()[:200])
m2 = re.search(r'verifyURL\s*=\s*"([^"]+)"', body)
if not m2:
return False
s.get(html_unescape(m2.group(1)).replace("\\/", "/"), timeout=40, allow_redirects=True)
return True
def _js_redirect_candidates(page):
"""从一个不执行 JS 的响应体里,按出现顺序抽出下一跳 URL。"""
out = []
for pat in (r"scriptLogin\.src\s*=\s*'([^']+)'",
r'scriptLogin\.src\s*=\s*"([^"]+)"',
r'verifyURL\s*=\s*"([^"]+)"',
r"verifyURL\s*=\s*'([^']+)'",
r"location\.href\s*=\s*['\"]([^'\"]+)['\"]",
r"window\.location\.href\s*=\s*['\"]([^'\"]+)['\"]"):
out.extend(re.findall(pat, page or ""))
return [html_unescape(u).replace("\\/", "/") for u in out if u]
def follow_js_hops(s, start_url, stop=None, max_hops=6):
"""通用 SSO 补跳:页面给的不是 302 而是 JS/scriptLogin 时,手工走完。
study.dgjy.net 的授权链和 jspx 不是同一套页面模板,所以这里按「下一跳 URL」
逐个 GET,直到 stop() 为真(例如 Middle-Token 已经下发)或没有下一跳。
"""
cur, seen = start_url, set()
for _ in range(max_hops):
if cur in seen:
break
seen.add(cur)
try:
r = s.get(cur, timeout=40, allow_redirects=True)
except Exception:
break
if stop and stop():
return True
nxt = _js_redirect_candidates(r.text or "")
if not nxt:
break
cur = urljoin(r.url, nxt[0])
return bool(stop and stop())
def _login_error(html, status=200):
"""从 /tpass/login 的响应里取出服务端提示。
提示藏在 <script type="text/html" id="login_template"> 里的
<span id="errormsghide" style="display:none">,BeautifulSoup 的 get_text()
看不见 script 内容,所以只能按原文正则取;锁号是另一张 403 提示页。
"""
text = html or ""
m = re.search(r"验证失败次数过多[^<]*", text)
if m:
return m.group(0).strip()
m = re.search(r'<span[^>]*id="errormsghide"[^>]*>(.*?)</span>', text, re.S)
if m:
msg = re.sub(r"<[^>]+>", " ", m.group(1))
msg = re.sub(r"\s+", " ", html_unescape(msg).replace("\xa0", " ")).strip()
if msg:
return msg
m = re.search(r"(用户名|密码|账号|账户)[^<>]{0,40}(错误|不存在|不正确|锁定)", text)
if m:
return m.group(0).strip()
if status == 403:
return "账号已被锁定(403),约 10 分钟后解锁"
return ""
def parse_accountlist(html):
"""多身份账号:登录响应里内联 `var accountlist = '<json>';`。
每个元素带 USERNAME(身份证)、CODENAME(教师/家长/学生)、UNIT_NAME,
以及该身份自己的 LOGINUSERNAME / LOGINPASSWORD —— 选身份那一步是服务端
把可登录的账号密码回给前端,再带 choosenumber 提交一次。
"""
m = re.search(r"var\s+accountlist\s*=\s*'(.*?)';", html or "", re.S)
if not m:
return []
try:
data = json.loads(html_unescape(m.group(1)))
except Exception:
return []
return data if isinstance(data, list) else []
def _login_as_identity(s, un, lt, ex, it):
"""按 login_dgjy.js 的选身份分支提交:rsa 用该身份的 LOGINUSERNAME+LOGINPASSWORD。"""
name = it.get("LOGINUSERNAME") or ""
pwd = it.get("LOGINPASSWORD") or ""
payload = {"un": un, "ul": str(len(name)), "pl": str(len(pwd)),
"rsa": str_enc(name + pwd + lt), "lt": lt,
"choosenumber": it.get("USERNAME") or "", "_eventId": "submit"}
if ex:
payload["execution"] = ex
return s.post(TPASS + "/login", data=payload, timeout=30, allow_redirects=True,
headers=LOGIN_HEADERS)
def ensure_jspx_session(s):
"""用 CASTGC 走老平台(jspx)的授权链,建立 jspx.dgjy.net 的会话;成功返回 True。
必须先把 jspx 的 JSESSIONID / route 清掉:这两个是粘性会话 cookie,
浏览器里带过来的那份常常指向已经下线的后端节点(route 对不上号),
留着就永远跳 /admin/login.action,看起来像「登录失效」。
"""
try:
r = s.get(JSPX + "/admin/main.action", timeout=30)
if "table-results" in (r.text or ""):
return True
except Exception:
pass
for c in list(s.cookies):
if "jspx" in (c.domain or ""):
s.cookies.clear(c.domain, c.path, c.name)
try:
r = s.get(AUTHORIZE, timeout=40, allow_redirects=True)
_follow_js_login(s, r.text or "")
r2 = s.get(JSPX + "/admin/main.action", timeout=30)
return "table-results" in (r2.text or "")
except Exception:
return False
def _jspx_call(s, fn):
"""跑一个 jspx 请求;会话失效且手上还有 CASTGC 时,清掉粘性 cookie 重新授权后重试一次。
浏览器 / IndexedDB 里存着的 jspx JSESSIONID+route 经常指向已经下线的后端节点,
fetch_admin 看到的就是「跳回 /admin/login.action」,看着像登录过期,
实际 CASTGC 还有效 —— 这种不该让用户重新输密码。
"""
try:
return fn()
except SessionExpired:
if not ensure_jspx_session(s):
raise
return fn()
def login(username: str, password: str):
"""SSO login -> (session, user)。"""
s = new_session()
r = s.get(TPASS + "/login", timeout=30)
lt = re.search(r'id="lt"[^>]*value="([^"]*)"', r.text)
lt = lt.group(1) if lt else ""
ex = re.search(r'name="execution"[^>]*value="([^"]*)"', r.text)
payload = {
"un": username, "pd": password,
"ul": str(len(username)), "pl": str(len(password)),
"rsa": str_enc(username + password + lt),
"lt": lt, "choosenumber": "", "_eventId": "submit",
}
if ex:
payload["execution"] = ex.group(1)
r2 = s.post(TPASS + "/login", data=payload, timeout=30, allow_redirects=True,
headers=LOGIN_HEADERS)
if not any(c.name == "CASTGC" for c in s.cookies):
accts = parse_accountlist(r2.text)
if accts:
# 一个身份证下挂多个身份时,第一次提交只回账号选择页,不会给票据。
# 这里取教师身份(本平台主要场景),没有就取第一个,然后用该身份重提。
pick = next((x for x in accts if "教师" in (x.get("CODENAME") or "")), accts[0])
m = re.search(r'id="lt"[^>]*value="([^"]*)"', r2.text)
ex2 = re.search(r'name="execution"[^>]*value="([^"]*)"', r2.text)
r2 = _login_as_identity(s, username, (m.group(1) if m else lt),
(ex2.group(1) if ex2 else (ex.group(1) if ex else "")), pick)
if not any(c.name == "CASTGC" for c in s.cookies):
msg = _login_error(r2.text, r2.status_code)
raise RuntimeError((msg or "登录失败:未获得登录票据(账号或密码错误?)")[:200])
# 老平台:authorize 链的终点是 JS 中转页,必须手工补两跳才建得起 jspx 会话
if not ensure_jspx_session(s):
raise RuntimeError("SSO 登录未完成:jspx.dgjy.net 没有建立会话(登录链路可能已改版)")
html = fetch_admin(s)
# 同一张 CASTGC 再授权一次给新版平台(client_id=gyx),拿 Middle-Token
ensure_study_session(s)
return s, whoami(html)
def _next_hop_urls(page):
"""不执行 JS 时,从响应体里找出所有可能的下一跳(SSO 中转页/脚本/表单)。"""
out = _js_redirect_candidates(page)
for m in re.finditer(r'(?:src|href)\s*=\s*["\']([^"\']*(?:login|callback|authorize|verify|code)[^"\']*)["\']',
page or "", re.I):
out.append(html_unescape(m.group(1)))
for m in re.finditer(r'<form[^>]+action=["\']([^"\']+)["\']', page or "", re.I):
out.append(html_unescape(m.group(1)))
seen, uniq = set(), []
for u in out:
u = u.replace("\\/", "/")
if u.startswith("http") or u.startswith("/") or "?" in u:
if u not in seen:
seen.add(u)
uniq.append(u)
return uniq
def ensure_study_session(s, force=False):
"""用现有 SSO 票把 study.dgjy.net 的会话补上。
新版平台的登录是 OAuth 授权码换 token(前端 SPA 源码里写死了这几个地址):
GET tyyh.dgjy.net/tpass/oauth2.0/authorize?client_id=gyx
&redirect_uri=https://study.dgjy.net/#/login/back&response_type=code
... -> study.dgjy.net/#/login/back?code=ST-xxxx-tpass
POST study.dgjy.net/mod/gjt/login/<CODE> # 换出前端会话(JSESSIONID)
GET study.dgjy.net/mod/education/gyx/train/front/login
-> {"result":{"url":".../training","token":"<jwt>"}} # 这才是 Middle-Token
成功标准只有一个:cookie 里出现 study.dgjy.net 的 Middle-Token。
"""
from study_tool.platforms.study import _study_token
if force:
for c in list(s.cookies):
if c.name == "Middle-Token":
try:
s.cookies.clear(c.domain, c.path, c.name)
except Exception:
pass
if _study_token(s):
return True
try:
r = s.get(TPASS + TRAIN_AUTHORIZE_PATH, timeout=40, allow_redirects=True)
except Exception:
return False
cur_url, body = r.url or "", r.text or ""
tried = set()
for _ in range(6):
m = CODE_RE.search(body) or CODE_RE.search(cur_url)
if m and m.group(1) not in tried:
tried.add(m.group(1))
if _exchange_code(s, m.group(1)):
return True
hops = _next_hop_urls(body)
if not hops:
break
nxt = urljoin(cur_url or STUDY, hops[0])
if "code=" in nxt:
code = (CODE_RE.search(nxt) or [None, ""])[1]
if code and code not in tried and _exchange_code(s, code):
return True
try:
r = s.get(nxt, timeout=40, allow_redirects=True)
except Exception:
break
cur_url, body = r.url or "", r.text or ""
return bool(_study_token(s))
def _fetch_train_token(s):
"""SSO 登录后,再用 gyx 会话换培训系统的 token(前端 `_51` 那个接口)。
返回体形如:
{"result":{"url":"https://study.dgjy.net/training","token":"<jwt>"},"resultCode":200}
这个 token 就是培训系统的 `Middle-Token`(前端把它写进 cookie 再发 Bearer)。
"""
try:
r = s.get(STUDY + "/mod/education/gyx/train/front/login", timeout=40,
headers={"Accept": "application/json, text/plain, */*"})
j = r.json()
except Exception:
return False
data = j.get("result") if isinstance(j, dict) else None
if not isinstance(data, dict):
data = j.get("data") if isinstance(j, dict) else None
tok = ""
if isinstance(data, dict):
for k in ("token", "accessToken", "middleToken"):
if data.get(k):
tok = str(data[k])
break
if not tok:
return False
s.cookies.set("Middle-Token", tok, domain=urlsplit(STUDY).hostname, path="/")
return True
def _exchange_code(s, code):
"""code -> Middle-Token。前端是 POST /mod/gjt/login/<code>,再取 gyx 培训 token。"""
from study_tool.platforms.study import _study_token
paths = ("/mod/gjt/login/%s", "/train/mod/gjt/login/%s")
for path in paths:
try:
rr = s.post(STUDY + (path % code), timeout=40,
headers={"Accept": "application/json, text/plain, */*"})
except Exception:
continue
if _study_token(s):
return True
try:
j = rr.json()
except Exception:
j = {}
ok = isinstance(j, dict) and str(j.get("resultCode")) == "200"
if ok:
# SSO 换票成功后,再换培训系统的 Bearer token(Middle-Token)
if _fetch_train_token(s):
return True
data = j.get("result") or {}
if isinstance(data, dict):
for k in ("token", "accessToken", "middleToken"):
if data.get(k):
s.cookies.set("Middle-Token", str(data[k]),
domain=urlsplit(STUDY).hostname, path="/")
return True
return False
def fetch_admin(s):
"""GET /admin/main.action。
站点会话失效时**不返回 302**,而是 200 + 55 字节的 JS 跳转存根:
<script>location.href="/admin/login.action";</script>
旧代码直接把它当正常页面解析 -> 一个表格都没有 -> 静默返回空列表,
于是「会话失效」「站点改版」「账号确实没有课」被压成同一条提示。
"""
r = s.get(JSPX + "/admin/main.action", timeout=40)
html = r.text or ""
if "/admin/login.action" in html and "table-results" not in html:
# CASTGC 还有效的话,这里能无密码把 jspx 会话补回来
if ensure_jspx_session(s):
r = s.get(JSPX + "/admin/main.action", timeout=40)
html = r.text or ""
if "/admin/login.action" in html and "table-results" not in html:
raise SessionExpired("登录已失效:站点把请求跳回了 /admin/login.action(请重新登录)")
if "table-results" not in html:
raise RuntimeError("课程页结构异常:未找到 table-results 表格(站点可能已改版,页面长度 %d)"
% len(html))
return html
def _parse_course_rows(html, section=""):
"""从「我的课程」列表片段里抽出课程行。
行结构(各板块一致):
<table class="table-results"><tr><th>在学课程</th></tr>
<tr><td><p class="course-tt"><a href="courseStudyMain.action?planId=..&projectId=..">名字</a></p>
<p class="course-dd"><span>班期:..</span><span>分数:82</span><span>成绩:合格</span></p>
</td><td><a class="abtn-orange-big">查看</a></td></tr>
</table>
注意 href 是**不带前导斜杠的相对路径**(`courseStudyMain.action?...`),
相对 `/train/trainee/` 解析,不能直接 `JSPX + href`。
"""
soup = BeautifulSoup(html, "lxml")
out = []
for table in soup.select("table.table-results"):
th = table.select_one("th")
sec = section or (re.sub(r"\s+", " ", th.get_text(" ", strip=True)) if th else "")
# 只按「有没有课程行」判断,不能用 "暂无" in table.get_text():
# 表格里任何一个单元格出现「暂无」都会把整张表(含全部课程)丢掉。
rows = [tr for tr in table.select("tr") if tr.select_one("p.course-tt a")]
for tr in rows:
a = tr.select_one("p.course-tt a")
href = unquote(a.get("href") or "")
plan, project = parse_ids(href)
if not plan or not project:
continue
dd = tr.select_one("p.course-dd")
out.append({
"section": sec,
"name": a.get_text(" ", strip=True),
"info": re.sub(r"\s+", " ", dd.get_text(" ", strip=True)) if dd else "",
"plan_id": plan,
"project_id": project,
"url": urljoin(JSPX + "/train/trainee/", href),
"progress": "",
})
return out
def scrape_courses(s, with_progress=True):
"""把「我的课程」下三个板块 × 三种状态**全部**拉出来,含已学完的。
站点把列表拆成了 AJAX 接口,每个组合一次请求:
/train/trainee/studyingCourse.action?courseType=studying&planStatus=TAKING_PLAN
/train/assist/assistanceCourse.action?courseType=assistance&planStatus=FINISHED_PLAN
/train/inspect/inspectingCourse.action?courseType=inspecting&planStatus=NO_BEGAIN_PLAN
...
只看 /admin/main.action 是看不到「已结束」课程的——那个首页只渲染在学/助学/督学
三张「进行中」的概览表,课一学完就从首页消失了。
返回空列表只代表「三个板块九种状态都没课」;
会话失效或页面改版一律抛异常,让调用方能给出准确提示。
"""
# 先用首页校验会话,避免九个接口各自失败还看不出来
_jspx_call(s, lambda: fetch_admin(s))
out, seen = [], set()
for section, path, ctype in COURSE_SECTIONS:
for label, status in PLAN_STATUS:
url = "%s%s?courseType=%s&planStatus=%s" % (JSPX, path, ctype, status)
try:
html = s.get(url, timeout=40).text
except Exception:
continue
for c in _parse_course_rows(html, section):
key = (c["plan_id"], c["project_id"])
if key in seen:
continue
seen.add(key)
c["course_type"] = ctype
c["status"] = status
c["status_label"] = label
c["in_study"] = status in STUDYABLE_STATUS
out.append(c)
# 进度只在「进行中/未开始」的课上查,已结束的动辄几十门,逐个请求太慢
if with_progress:
for c in out:
if not c["in_study"]:
continue
try:
rr = s.get(c["url"], timeout=30)
p = BeautifulSoup(rr.text, "lxml").select_one("p.loading font")
c["progress"] = p.get_text(strip=True) if p else ""
except Exception:
pass
return out
def parse_jspx_course(html):
"""老版 jspx 课程页(tier-tree)解析。"""
soup = BeautifulSoup(html, "lxml")
course = soup.select_one("h1")
prog = soup.select_one("p.loading font")
info = soup.select_one("div.results-info p.info")
cw, fb, ts = [], [], []
for one in soup.select("div.tier-tree-one"):
ns = one.select_one("p.tier1.has-children span.name")
chapter = re.sub(r"\s+", " ", ns.get_text(" ", strip=True)) if ns else "?"
for p in one.select("p.tier3"):
a = p.select_one("a")
if not a:
continue
href = a.get("href") or ""
txt = re.sub(r"\s+", " ", p.get_text(" ", strip=True))
name = p.select_one("span.name")
title = re.sub(r"\s+", " ", name.get_text(" ", strip=True)) if name else txt[:40]
m = re.search(r"(\d+)分钟", txt)
done = "已完成" in txt
if "courseware.action" in href:
cw.append({"chapter": chapter, "title": title, "href": href, "done": done,
"duration": int(m.group(1)) if m else None})
elif "doFeedback.action" in href:
fb.append({"title": title, "href": href, "done": done})
elif "doTest.action" in href:
ts.append({"title": title, "href": href, "done": done})
return {
"user": whoami(html),
"course": course.get_text(" ", strip=True) if course else "?",
"progress": prog.get_text(strip=True) if prog else "?",
"info": re.sub(r"\s+", " ", info.get_text(" ", strip=True)) if info else "?",
"courseware": cw, "feedback": fb, "test": ts,
}
def discover(s, plan, project, name=""):
"""打开课程页并解析;会话失效时先用 CASTGC 补一次 jspx 授权再重试。"""
return _jspx_call(s, lambda: _discover_once(s, plan, project, name))
def _discover_once(s, plan, project, name=""):
# study.py 反向 import 本模块,模块级 import 会成环,用到时再取
from study_tool.platforms.study import (StudyClient, discover_study, extract_study_token,
extract_train_id, save_snapshot,
study_resolve_train_id)
url = "%s/train/trainee/courseStudyMain.action?planId=%s&projectId=%s&trainRole=0" % (JSPX, plan, project)
r = s.get(url, timeout=40)
html = r.text or ""
if "/admin/login.action" in html and len(html) < 2000:
# 课程页也走同一个判断:CASTGC 有效就无密码续期
if ensure_jspx_session(s):
r = s.get(url, timeout=40)
html = r.text or ""
if "没有权限" in html:
raise RuntimeError("该账号对此课程没有权限")
if "/admin/login.action" in html and len(html) < 2000:
raise SessionExpired("登录已失效:课程页把请求跳回了 /admin/login.action(请重新登录)")
# 课程页 302 到 study 时,token 在跳转 URL 里;哪怕 SPA 外壳返回空也要当成学过的新平台
study_url = "study.dgjy.net" in (r.url or "")
tok = extract_study_token(r.url, r.history)
if tok:
host = urlsplit(r.url).hostname or urlsplit(STUDY).hostname
try:
s.cookies.set("Middle-Token", tok, domain=host, path="/")
except Exception:
pass
if len(html) < 3000 and not (study_url or tok or has_study_marker(html, r.url)):
raise RuntimeError("课程页返回异常(页面长度仅 %d,登录可能已失效)" % len(html))
generation = detect_generation(html, r.url)
# 按登录 / 学习入口识别平台;以后新增平台时这里不用改,靠 Platform.entries 命中
pf = registry.detect(r.url, html)
platform_id = pf.id if pf else "guanyixue"
if generation == "legacy":
d = parse_jspx_course(html)
if d["courseware"] or d["feedback"] or d["test"]:
d["platform"] = platform_id
d["generation"] = "legacy"
return d
# 老结构但一个活动都没有:多半是「慕课」外链页(页面还在 jspx,课时在 study 平台)
if not has_study_marker(html, r.url):
snap = save_snapshot(plan, project, r.url, html)
raise RuntimeError("课程页能打开,但没有任何课件/反馈表/测验(页面已存 %s)。"
"若是慕课课程,请确认新平台登录态是否有效" % snap)
elif generation != "spa":
snap = save_snapshot(plan, project, r.url, html)
raise RuntimeError("无法识别课程页结构(站点可能又改版):页面里既没有 jspx 的 "
"tier-tree,也没有新版 SPA 标记。页面已存 %s,请把它发我" % snap)
# ---- 新版平台(Vue SPA + /train/course/front/* JSON 接口)----
tid = extract_train_id(r.url, html)
user = whoami(html)
client = StudyClient(s, r.url)
def _load():
nonlocal tid
if not tid:
tid = study_resolve_train_id(client, name)
if not tid:
snap = save_snapshot(plan, project, r.url, html)
raise RuntimeError("新版课程页未带 trainCourseId,且按课程名也匹配不到(课程名:%s)。"
"页面已存 %s" % (name or "?", snap))
return discover_study(client, tid, name, user)
try:
d = _load()
except SessionExpired:
client.bootstrap()
tid = extract_train_id(r.url, html)
d = _load()
d["platform"] = platform_id
d["generation"] = "spa"
# URL 里可能带 token(凭据),日志/状态里一律打码;鉴权已经写进 cookie
d["page_url"] = re.sub(r"([?&])(token|access_token|securityKey|sign)=[^&]*", r"\1\2=***", r.url or "")
d["plan_id"] = plan
d["project_id"] = project
return d
# ------------------------------------------------------------ 旧版学习流程
class CwItem:
def __init__(self, d, cookies):
self.title = d["title"]
href = d["href"]
self.url = href if href.startswith("http") else JSPX + href
self.s = new_session(cookies)
self.rec = None
self.status = "done" if d.get("done") else "pending"
self.last = None
def resolve(self):
try:
r = self.s.get(self.url, timeout=40)
except Exception as e:
return "exc:%s" % e
m = re.search(r"'id':'([0-9A-F]{32})'", r.text)
if not m:
return "no-id"
self.rec = m.group(1)
return "ok"
def start(self):
try:
r = self.s.post(EP_STUDY, data={"id": self.rec, "order": "server_start",
"updatePassPerc": "true"}, timeout=40)
return r.text.strip()[:40]
except Exception as e:
return "EXC:%s" % e
def beat(self):
try:
r = self.s.post(EP_STUDY, data={"id": self.rec, "order": "server_time",
"updatePassPerc": "true",
"videoSchedule": 30}, timeout=40)
return r.text.strip()[:60]
except Exception as e:
return "EXC:%s" % e
def submit_feedback(s, href, key):
url = href if href.startswith("http") else JSPX + href
r = s.get(url, timeout=40)
soup = BeautifulSoup(r.text, "lxml")
fields = {i.get("name"): i.get("value") for i in soup.select("form input[type=hidden]") if i.get("name")}
if fields.get("allowStudy") == "false":
log(key, "反馈表:allowStudy=false,跳过")
return
r2 = s.post(JSPX + "/train/trainee/doFeedbackList.action", data=fields, timeout=40)
qs = BeautifulSoup(r2.text, "lxml")
answers = []
for it in qs.select("div.job-item"):
qtype = it.get("questype")
opts = [sp.get("data-name") for sp in it.select(".job-item-a span")]
ans = ("课程内容充实,讲师讲解清晰,对我的教学实践很有帮助,希望今后能增加更多一线教学案例。"
if qtype == "04" else (opts[0] if opts else "A"))
answers.append({"id": it.get("traineeid"), "quesId": it.get("qumid"),
"quesType": qtype, "quesNo": it.get("quesno"), "answer": ans})
payload = {"planId": fields.get("planId"), "objectId": fields.get("objectId"),
"answers": json.dumps(answers, ensure_ascii=False, separators=(",", ":")),
"submitStatus": "finish"}
rr = s.post(JSPX + "/train/trainee/saveFeedback.action", data=payload, timeout=40)
log(key, "反馈表:%d 题,%s" % (len(answers), "提交成功" if "MSG" not in rr.text else "返回 " + rr.text[:80]))
def submit_quiz(s, href, key, answer_key=None):
url = href if href.startswith("http") else JSPX + href
plan, _ = parse_ids(url)
m = re.search(r"objectId=([0-9A-Fa-f]+)", url)
obj = m.group(1) if m else ""
rr = s.get("%s/train/trainee/showTestQums.action?planId=%s&objectId=%s" % (JSPX, plan, obj), timeout=40)
qs = BeautifulSoup(rr.text, "lxml")
singles = (answer_key or {}).get("singles", {})
multis = (answer_key or {}).get("multis", {})
answers = []
for it in qs.select("div.job-item"):
t, qno = it.get("questype"), it.get("quesno")
opts = [sp.get("data-name") for sp in it.select(".job-item-a span")]
if t == "02":
ans = (multis.get(qno) or ("#".join(opts) if opts else "A"))
else:
ans = singles.get(qno) or (opts[0] if opts else "A")
answers.append({"id": it.get("traineeid"), "quesId": it.get("qumid"),
"quesType": t, "quesNo": qno, "answer": ans})
if not answers:
log(key, "测验:未取到题目,跳过")
return
payload = {"planId": plan, "objectId": obj,
"answers": json.dumps(answers, ensure_ascii=False, separators=(",", ":"))}
r3 = s.post(JSPX + "/train/trainee/endTest.action", data=payload, timeout=40)
r4 = s.get("%s/train/trainee/doTest.action?planId=%s&objectId=%s" % (JSPX, plan, obj), timeout=40)
sc = re.search(r"得分:\s*(\d+)\s*分",
re.sub(r"\s+", " ", BeautifulSoup(r4.text, "lxml").get_text(" ", strip=True)))
log(key, "测验:%d 题,%s,得分 %s" %
(len(answers), "提交成功" if "MSG" not in r3.text else r3.text[:60],
sc.group(1) if sc else "?"))
def _study_builtin(key, s, d, workers=1, cap=30, enable_quiz=False, enable_feedback=False,
mode="fast", parallel=1, quiz_key="", cookies=None, plan="", project="",
name=""):
"""旧版 jspx 的内置流程:反馈表 + 测验 + 刷课件。"""
if enable_feedback:
for fb in d["feedback"]:
if fb["done"]:
log(key, "反馈表已完成,跳过")
continue
try:
submit_feedback(s, fb["href"], key)
except Exception as e:
log(key, "反馈表异常:%s" % e)
else:
for fb in d["feedback"]:
log(key, "反馈表:已按设置跳过(%s)" % ("已完成" if fb["done"] else "未完成"))
if enable_quiz:
for t in d["test"]:
if t["done"]:
log(key, "测验已完成,跳过")
continue
try:
submit_quiz(s, t["href"], key, quiz_key)
except Exception as e:
log(key, "测验异常:%s" % e)
else:
for t in d["test"]:
if t["done"]:
log(key, "测验:已完成,跳过")
else:
log(key, "测验:⚠ 已按设置跳过答题(避免因正确率不足卡住「完成」),"
"需要时请在页面上手动作答,或打开「自动交测验」开关")
objs = [CwItem(c, cookies) for c in d["courseware"]]
todo = [o for o in objs if o.status != "done"]
done0 = len(objs) - len(todo)
set_state(key, status="studying", total=len(objs), done=done0)
if todo:
log(key, "解析 %d 个学习记录…" % len(todo))
with ThreadPoolExecutor(max_workers=min(workers, len(todo))) as ex:
list(ex.map(lambda x: x.resolve(), todo))
live = [o for o in todo if o.rec]
log(key, "已解析 %d/%d,并发启动会话" % (len(live), len(todo)))
with ThreadPoolExecutor(max_workers=min(workers, len(live))) as ex:
list(ex.map(lambda x: x.start(), live))
t0 = time.time()
while time.time() - t0 < cap * 60:
with RUNS_LOCK:
if (RUNS.get(key) or {}).get("status") == "stopped":
log(key, "已按要求停止")
return
pending = [o for o in live if o.status != "done"]
if not pending:
break
with ThreadPoolExecutor(max_workers=min(workers, len(pending))) as ex:
resps = list(ex.map(lambda x: x.beat(), pending))
newly = []
for o, resp in zip(pending, resps):
o.last = resp
if resp == "study_over":
o.status = "done"
newly.append(o)
el = int(time.time() - t0)
dc = len([o for o in live if o.status == "done"]) + done0
set_state(key, done=dc, elapsed=el)
log(key, "第 %d 轮 %dm%02ds 完成 %d/%d" % (el // 60 + 1, el // 60, el % 60, dc, len(objs)))
for o in newly:
log(key, " ✓ %s" % o.title[:40])
time.sleep(60)
else:
log(key, "所有课件均已完成")
try:
d2 = discover(s, plan, project, name)
set_state(key, progress=d2["progress"], info=d2["info"], cookies=dump_cookies(s))
log(key, "课程结束:%s | %s" % (d2["progress"], d2["info"]))
except Exception as e:
log(key, "复查失败:%s" % e)
# ---------------------------------------------------------------- 平台注册
def _study(key, s, d, **opts):
"""旧版页面:反馈表 + 测验 + 刷课件。"""
return _study_builtin(key, s, d, **opts)