study-tool
8e8d6b83f1
- study_tool/__init__.py 用 python-dotenv 加载根目录 .env，位置早于 store / config 读环境变量；已有环境变量优先，Docker 注入的不受影响 - app.py 监听地址改为 HOST -> BIND_ADDR -> 127.0.0.1 依次回退 - 新增依赖 python-dotenv>=1.0（pyproject.toml / requirements.txt / uv.lock） - README 环境变量表补充 .env 说明与 BIND_ADDR / TZ 两行 - CHANGELOG / docs/OPTIMIZATION.md 记录本次修复，版本号 0.0.6 -> 0.0.7
826 lines
35 KiB
Python
826 lines
35 KiB
Python
"""旧版平台（jspx.dgjy.net）：会话 / cookie / 登录 / 课程抓取。
|
||
|
||
对外：new_session / login / scrape_courses / discover。
|
||
"""
|
||
import json
|
||
import re
|
||
import subprocess
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from html import unescape as html_unescape
|
||
from urllib.parse import unquote, urljoin, urlsplit
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
|
||
from study_tool.config import *
|
||
from study_tool.platforms import registry
|
||
from study_tool.platforms.detect import detect_generation, has_study_marker
|
||
from study_tool.runtime import RUNS, RUNS_LOCK, log, set_state
|
||
|
||
# ------------------------------------------------------------------------ crypto
|
||
|
||
def str_enc(data: str) -> str:
|
||
r = subprocess.run([NODE, DESENC, data, "1", "2", "3"],
|
||
capture_output=True, text=True, timeout=30)
|
||
if r.returncode != 0:
|
||
raise RuntimeError("strEnc failed: %s" % (r.stderr or "")[:200])
|
||
return r.stdout.strip()
|
||
|
||
|
||
# ------------------------------------------------------------------- http client
|
||
|
||
def new_session(cookies=None):
|
||
s = requests.Session()
|
||
s.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
|
||
for c in (cookies or []):
|
||
if "dgjy" not in (c.get("domain") or ""):
|
||
continue
|
||
s.cookies.set(c["name"], c.get("value") or "",
|
||
domain=(c.get("domain") or "").lstrip("."),
|
||
path=c.get("path") or "/")
|
||
return s
|
||
|
||
|
||
def dump_cookies(s):
|
||
return [{"name": c.name, "value": c.value, "domain": c.domain, "path": c.path}
|
||
for c in s.cookies]
|
||
|
||
|
||
def whoami(html):
|
||
soup = BeautifulSoup(html, "lxml")
|
||
hdr = soup.select_one("#header")
|
||
if hdr:
|
||
m = re.search(r"你好[，,]?\s*(\S+)\s*老师", re.sub(r"\s+", " ", hdr.get_text(" ", strip=True)))
|
||
if m:
|
||
return m.group(1)
|
||
return ""
|
||
|
||
|
||
def parse_ids(text):
|
||
plan = project = ""
|
||
m = re.search(r"planId=([0-9A-Fa-f]{16,})", text or "")
|
||
if m:
|
||
plan = m.group(1)
|
||
m = re.search(r"projectId=([0-9A-Fa-f]{16,})", text or "")
|
||
if m:
|
||
project = m.group(1)
|
||
return plan, project
|
||
|
||
|
||
def _follow_js_login(s, page):
|
||
"""补上浏览器才会执行的两跳，否则只拿得到 CASTGC、拿不到 jspx 的会话。 `oauth2.0/authorize` 链的最后一跳**不是 302**，而是 dj.dgjy.net 上一个
|
||
1486 字节的 JS 中转页，里面动态插 <script>：
|
||
|
||
scriptLogin.src = 'https://passport.dgjy.net/scriptLogin?action=simpleLogin&d1=...&d2=...';
|
||
|
||
scriptLogin 返回的是 JS，不是跳转：
|
||
|
||
var loginError=null;
|
||
var verifyURL="https://dj.dgjy.net/fpf/login/verify.action?ticket=...&auth=...&sid=...";
|
||
location.href=verifyURL;
|
||
|
||
verify.action 之后才会 302 → loginCall → jspx/tstudy/passport/login.action，
|
||
到这一步 jspx.dgjy.net 的 JSESSIONID 才真正下发。
|
||
requests 不执行 JS，所以这两跳必须手工走一遍。
|
||
"""
|
||
m = re.search(r"scriptLogin\.src\s*=\s*'([^']+)'", page or "")
|
||
if not m:
|
||
return False
|
||
r = s.get(html_unescape(m.group(1)), timeout=40)
|
||
body = r.text or ""
|
||
if re.search(r"loginError\s*=\s*(?!null)", body):
|
||
raise RuntimeError("SSO 校验未通过：%s" % body.strip()[:200])
|
||
m2 = re.search(r'verifyURL\s*=\s*"([^"]+)"', body)
|
||
if not m2:
|
||
return False
|
||
s.get(html_unescape(m2.group(1)).replace("\\/", "/"), timeout=40, allow_redirects=True)
|
||
return True
|
||
|
||
|
||
def _js_redirect_candidates(page):
|
||
"""从一个不执行 JS 的响应体里，按出现顺序抽出下一跳 URL。"""
|
||
out = []
|
||
for pat in (r"scriptLogin\.src\s*=\s*'([^']+)'",
|
||
r'scriptLogin\.src\s*=\s*"([^"]+)"',
|
||
r'verifyURL\s*=\s*"([^"]+)"',
|
||
r"verifyURL\s*=\s*'([^']+)'",
|
||
r"location\.href\s*=\s*['\"]([^'\"]+)['\"]",
|
||
r"window\.location\.href\s*=\s*['\"]([^'\"]+)['\"]"):
|
||
out.extend(re.findall(pat, page or ""))
|
||
return [html_unescape(u).replace("\\/", "/") for u in out if u]
|
||
|
||
|
||
def follow_js_hops(s, start_url, stop=None, max_hops=6):
|
||
"""通用 SSO 补跳：页面给的不是 302 而是 JS/scriptLogin 时，手工走完。
|
||
|
||
study.dgjy.net 的授权链和 jspx 不是同一套页面模板，所以这里按「下一跳 URL」
|
||
逐个 GET，直到 stop() 为真（例如 Middle-Token 已经下发）或没有下一跳。
|
||
"""
|
||
cur, seen = start_url, set()
|
||
for _ in range(max_hops):
|
||
if cur in seen:
|
||
break
|
||
seen.add(cur)
|
||
try:
|
||
r = s.get(cur, timeout=40, allow_redirects=True)
|
||
except Exception:
|
||
break
|
||
if stop and stop():
|
||
return True
|
||
nxt = _js_redirect_candidates(r.text or "")
|
||
if not nxt:
|
||
break
|
||
cur = urljoin(r.url, nxt[0])
|
||
return bool(stop and stop())
|
||
|
||
|
||
def _login_error(html, status=200):
|
||
"""从 /tpass/login 的响应里取出服务端提示。
|
||
|
||
提示藏在 <script type="text/html" id="login_template"> 里的
|
||
<span id="errormsghide" style="display:none">，BeautifulSoup 的 get_text()
|
||
看不见 script 内容，所以只能按原文正则取；锁号是另一张 403 提示页。
|
||
"""
|
||
text = html or ""
|
||
m = re.search(r"验证失败次数过多[^<]*", text)
|
||
if m:
|
||
return m.group(0).strip()
|
||
m = re.search(r'<span[^>]*id="errormsghide"[^>]*>(.*?)</span>', text, re.S)
|
||
if m:
|
||
msg = re.sub(r"<[^>]+>", " ", m.group(1))
|
||
msg = re.sub(r"\s+", " ", html_unescape(msg).replace("\xa0", " ")).strip()
|
||
if msg:
|
||
return msg
|
||
m = re.search(r"(用户名|密码|账号|账户)[^<>]{0,40}(错误|不存在|不正确|锁定)", text)
|
||
if m:
|
||
return m.group(0).strip()
|
||
if status == 403:
|
||
return "账号已被锁定（403），约 10 分钟后解锁"
|
||
return ""
|
||
|
||
|
||
def parse_accountlist(html):
|
||
"""多身份账号：登录响应里内联 `var accountlist = '<json>';`。
|
||
|
||
每个元素带 USERNAME(身份证)、CODENAME(教师/家长/学生)、UNIT_NAME，
|
||
以及该身份自己的 LOGINUSERNAME / LOGINPASSWORD —— 选身份那一步是服务端
|
||
把可登录的账号密码回给前端，再带 choosenumber 提交一次。
|
||
"""
|
||
m = re.search(r"var\s+accountlist\s*=\s*'(.*?)';", html or "", re.S)
|
||
if not m:
|
||
return []
|
||
try:
|
||
data = json.loads(html_unescape(m.group(1)))
|
||
except Exception:
|
||
return []
|
||
return data if isinstance(data, list) else []
|
||
|
||
|
||
def _login_as_identity(s, un, lt, ex, it):
|
||
"""按 login_dgjy.js 的选身份分支提交：rsa 用该身份的 LOGINUSERNAME+LOGINPASSWORD。"""
|
||
name = it.get("LOGINUSERNAME") or ""
|
||
pwd = it.get("LOGINPASSWORD") or ""
|
||
payload = {"un": un, "ul": str(len(name)), "pl": str(len(pwd)),
|
||
"rsa": str_enc(name + pwd + lt), "lt": lt,
|
||
"choosenumber": it.get("USERNAME") or "", "_eventId": "submit"}
|
||
if ex:
|
||
payload["execution"] = ex
|
||
return s.post(TPASS + "/login", data=payload, timeout=30, allow_redirects=True,
|
||
headers=LOGIN_HEADERS)
|
||
|
||
|
||
def ensure_jspx_session(s):
|
||
"""用 CASTGC 走老平台(jspx)的授权链，建立 jspx.dgjy.net 的会话；成功返回 True。
|
||
|
||
必须先把 jspx 的 JSESSIONID / route 清掉：这两个是粘性会话 cookie，
|
||
浏览器里带过来的那份常常指向已经下线的后端节点（route 对不上号），
|
||
留着就永远跳 /admin/login.action，看起来像「登录失效」。
|
||
"""
|
||
try:
|
||
r = s.get(JSPX + "/admin/main.action", timeout=30)
|
||
if "table-results" in (r.text or ""):
|
||
return True
|
||
except Exception:
|
||
pass
|
||
for c in list(s.cookies):
|
||
if "jspx" in (c.domain or ""):
|
||
s.cookies.clear(c.domain, c.path, c.name)
|
||
try:
|
||
r = s.get(AUTHORIZE, timeout=40, allow_redirects=True)
|
||
_follow_js_login(s, r.text or "")
|
||
r2 = s.get(JSPX + "/admin/main.action", timeout=30)
|
||
return "table-results" in (r2.text or "")
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _jspx_call(s, fn):
|
||
"""跑一个 jspx 请求；会话失效且手上还有 CASTGC 时，清掉粘性 cookie 重新授权后重试一次。
|
||
|
||
浏览器 / IndexedDB 里存着的 jspx JSESSIONID+route 经常指向已经下线的后端节点，
|
||
fetch_admin 看到的就是「跳回 /admin/login.action」，看着像登录过期，
|
||
实际 CASTGC 还有效 —— 这种不该让用户重新输密码。
|
||
"""
|
||
try:
|
||
return fn()
|
||
except SessionExpired:
|
||
if not ensure_jspx_session(s):
|
||
raise
|
||
return fn()
|
||
|
||
|
||
def login(username: str, password: str):
|
||
"""SSO login -> (session, user)。"""
|
||
s = new_session()
|
||
r = s.get(TPASS + "/login", timeout=30)
|
||
lt = re.search(r'id="lt"[^>]*value="([^"]*)"', r.text)
|
||
lt = lt.group(1) if lt else ""
|
||
ex = re.search(r'name="execution"[^>]*value="([^"]*)"', r.text)
|
||
payload = {
|
||
"un": username, "pd": password,
|
||
"ul": str(len(username)), "pl": str(len(password)),
|
||
"rsa": str_enc(username + password + lt),
|
||
"lt": lt, "choosenumber": "", "_eventId": "submit",
|
||
}
|
||
if ex:
|
||
payload["execution"] = ex.group(1)
|
||
r2 = s.post(TPASS + "/login", data=payload, timeout=30, allow_redirects=True,
|
||
headers=LOGIN_HEADERS)
|
||
|
||
if not any(c.name == "CASTGC" for c in s.cookies):
|
||
accts = parse_accountlist(r2.text)
|
||
if accts:
|
||
# 一个身份证下挂多个身份时，第一次提交只回账号选择页，不会给票据。
|
||
# 这里取教师身份（本平台主要场景），没有就取第一个，然后用该身份重提。
|
||
pick = next((x for x in accts if "教师" in (x.get("CODENAME") or "")), accts[0])
|
||
m = re.search(r'id="lt"[^>]*value="([^"]*)"', r2.text)
|
||
ex2 = re.search(r'name="execution"[^>]*value="([^"]*)"', r2.text)
|
||
r2 = _login_as_identity(s, username, (m.group(1) if m else lt),
|
||
(ex2.group(1) if ex2 else (ex.group(1) if ex else "")), pick)
|
||
if not any(c.name == "CASTGC" for c in s.cookies):
|
||
msg = _login_error(r2.text, r2.status_code)
|
||
raise RuntimeError((msg or "登录失败：未获得登录票据（账号或密码错误？）")[:200])
|
||
|
||
# 老平台：authorize 链的终点是 JS 中转页，必须手工补两跳才建得起 jspx 会话
|
||
if not ensure_jspx_session(s):
|
||
raise RuntimeError("SSO 登录未完成：jspx.dgjy.net 没有建立会话（登录链路可能已改版）")
|
||
html = fetch_admin(s)
|
||
# 同一张 CASTGC 再授权一次给新版平台（client_id=gyx），拿 Middle-Token
|
||
ensure_study_session(s)
|
||
return s, whoami(html)
|
||
|
||
|
||
def _next_hop_urls(page):
|
||
"""不执行 JS 时，从响应体里找出所有可能的下一跳（SSO 中转页/脚本/表单）。"""
|
||
out = _js_redirect_candidates(page)
|
||
for m in re.finditer(r'(?:src|href)\s*=\s*["\']([^"\']*(?:login|callback|authorize|verify|code)[^"\']*)["\']',
|
||
page or "", re.I):
|
||
out.append(html_unescape(m.group(1)))
|
||
for m in re.finditer(r'<form[^>]+action=["\']([^"\']+)["\']', page or "", re.I):
|
||
out.append(html_unescape(m.group(1)))
|
||
seen, uniq = set(), []
|
||
for u in out:
|
||
u = u.replace("\\/", "/")
|
||
if u.startswith("http") or u.startswith("/") or "?" in u:
|
||
if u not in seen:
|
||
seen.add(u)
|
||
uniq.append(u)
|
||
return uniq
|
||
|
||
|
||
def ensure_study_session(s, force=False):
|
||
"""用现有 SSO 票把 study.dgjy.net 的会话补上。
|
||
|
||
新版平台的登录是 OAuth 授权码换 token（前端 SPA 源码里写死了这几个地址）：
|
||
GET tyyh.dgjy.net/tpass/oauth2.0/authorize?client_id=gyx
|
||
&redirect_uri=https://study.dgjy.net/#/login/back&response_type=code
|
||
... -> study.dgjy.net/#/login/back?code=ST-xxxx-tpass
|
||
POST study.dgjy.net/mod/gjt/login/<CODE> # 换出前端会话（JSESSIONID）
|
||
GET study.dgjy.net/mod/education/gyx/train/front/login
|
||
-> {"result":{"url":".../training","token":"<jwt>"}} # 这才是 Middle-Token
|
||
成功标准只有一个：cookie 里出现 study.dgjy.net 的 Middle-Token。
|
||
"""
|
||
from study_tool.platforms.study import _study_token
|
||
if force:
|
||
for c in list(s.cookies):
|
||
if c.name == "Middle-Token":
|
||
try:
|
||
s.cookies.clear(c.domain, c.path, c.name)
|
||
except Exception:
|
||
pass
|
||
if _study_token(s):
|
||
return True
|
||
try:
|
||
r = s.get(TPASS + TRAIN_AUTHORIZE_PATH, timeout=40, allow_redirects=True)
|
||
except Exception:
|
||
return False
|
||
cur_url, body = r.url or "", r.text or ""
|
||
tried = set()
|
||
for _ in range(6):
|
||
m = CODE_RE.search(body) or CODE_RE.search(cur_url)
|
||
if m and m.group(1) not in tried:
|
||
tried.add(m.group(1))
|
||
if _exchange_code(s, m.group(1)):
|
||
return True
|
||
hops = _next_hop_urls(body)
|
||
if not hops:
|
||
break
|
||
nxt = urljoin(cur_url or STUDY, hops[0])
|
||
if "code=" in nxt:
|
||
code = (CODE_RE.search(nxt) or [None, ""])[1]
|
||
if code and code not in tried and _exchange_code(s, code):
|
||
return True
|
||
try:
|
||
r = s.get(nxt, timeout=40, allow_redirects=True)
|
||
except Exception:
|
||
break
|
||
cur_url, body = r.url or "", r.text or ""
|
||
return bool(_study_token(s))
|
||
|
||
|
||
def _fetch_train_token(s):
|
||
"""SSO 登录后，再用 gyx 会话换培训系统的 token（前端 `_51` 那个接口）。
|
||
|
||
返回体形如：
|
||
{"result":{"url":"https://study.dgjy.net/training","token":"<jwt>"},"resultCode":200}
|
||
这个 token 就是培训系统的 `Middle-Token`（前端把它写进 cookie 再发 Bearer）。
|
||
"""
|
||
try:
|
||
r = s.get(STUDY + "/mod/education/gyx/train/front/login", timeout=40,
|
||
headers={"Accept": "application/json, text/plain, */*"})
|
||
j = r.json()
|
||
except Exception:
|
||
return False
|
||
data = j.get("result") if isinstance(j, dict) else None
|
||
if not isinstance(data, dict):
|
||
data = j.get("data") if isinstance(j, dict) else None
|
||
tok = ""
|
||
if isinstance(data, dict):
|
||
for k in ("token", "accessToken", "middleToken"):
|
||
if data.get(k):
|
||
tok = str(data[k])
|
||
break
|
||
if not tok:
|
||
return False
|
||
s.cookies.set("Middle-Token", tok, domain=urlsplit(STUDY).hostname, path="/")
|
||
return True
|
||
|
||
|
||
def _exchange_code(s, code):
|
||
"""code -> Middle-Token。前端是 POST /mod/gjt/login/<code>，再取 gyx 培训 token。"""
|
||
from study_tool.platforms.study import _study_token
|
||
paths = ("/mod/gjt/login/%s", "/train/mod/gjt/login/%s")
|
||
for path in paths:
|
||
try:
|
||
rr = s.post(STUDY + (path % code), timeout=40,
|
||
headers={"Accept": "application/json, text/plain, */*"})
|
||
except Exception:
|
||
continue
|
||
if _study_token(s):
|
||
return True
|
||
try:
|
||
j = rr.json()
|
||
except Exception:
|
||
j = {}
|
||
ok = isinstance(j, dict) and str(j.get("resultCode")) == "200"
|
||
if ok:
|
||
# SSO 换票成功后，再换培训系统的 Bearer token（Middle-Token）
|
||
if _fetch_train_token(s):
|
||
return True
|
||
data = j.get("result") or {}
|
||
if isinstance(data, dict):
|
||
for k in ("token", "accessToken", "middleToken"):
|
||
if data.get(k):
|
||
s.cookies.set("Middle-Token", str(data[k]),
|
||
domain=urlsplit(STUDY).hostname, path="/")
|
||
return True
|
||
return False
|
||
|
||
|
||
def fetch_admin(s):
|
||
"""GET /admin/main.action。
|
||
|
||
站点会话失效时**不返回 302**，而是 200 + 55 字节的 JS 跳转存根：
|
||
<script>location.href="/admin/login.action";</script>
|
||
旧代码直接把它当正常页面解析 -> 一个表格都没有 -> 静默返回空列表，
|
||
于是「会话失效」「站点改版」「账号确实没有课」被压成同一条提示。
|
||
"""
|
||
r = s.get(JSPX + "/admin/main.action", timeout=40)
|
||
html = r.text or ""
|
||
if "/admin/login.action" in html and "table-results" not in html:
|
||
# CASTGC 还有效的话，这里能无密码把 jspx 会话补回来
|
||
if ensure_jspx_session(s):
|
||
r = s.get(JSPX + "/admin/main.action", timeout=40)
|
||
html = r.text or ""
|
||
if "/admin/login.action" in html and "table-results" not in html:
|
||
raise SessionExpired("登录已失效：站点把请求跳回了 /admin/login.action（请重新登录）")
|
||
if "table-results" not in html:
|
||
raise RuntimeError("课程页结构异常：未找到 table-results 表格（站点可能已改版，页面长度 %d）"
|
||
% len(html))
|
||
return html
|
||
|
||
|
||
def _parse_course_rows(html, section=""):
|
||
"""从「我的课程」列表片段里抽出课程行。
|
||
|
||
行结构（各板块一致）：
|
||
<table class="table-results"><tr><th>在学课程</th></tr>
|
||
<tr><td><p class="course-tt"><a href="courseStudyMain.action?planId=..&projectId=..">名字</a></p>
|
||
<p class="course-dd"><span>班期：..</span><span>分数：82</span><span>成绩：合格</span></p>
|
||
</td><td><a class="abtn-orange-big">查看</a></td></tr>
|
||
</table>
|
||
|
||
注意 href 是**不带前导斜杠的相对路径**（`courseStudyMain.action?...`），
|
||
相对 `/train/trainee/` 解析，不能直接 `JSPX + href`。
|
||
"""
|
||
soup = BeautifulSoup(html, "lxml")
|
||
out = []
|
||
for table in soup.select("table.table-results"):
|
||
th = table.select_one("th")
|
||
sec = section or (re.sub(r"\s+", " ", th.get_text(" ", strip=True)) if th else "")
|
||
# 只按「有没有课程行」判断，不能用 "暂无" in table.get_text()：
|
||
# 表格里任何一个单元格出现「暂无」都会把整张表（含全部课程）丢掉。
|
||
rows = [tr for tr in table.select("tr") if tr.select_one("p.course-tt a")]
|
||
for tr in rows:
|
||
a = tr.select_one("p.course-tt a")
|
||
href = unquote(a.get("href") or "")
|
||
plan, project = parse_ids(href)
|
||
if not plan or not project:
|
||
continue
|
||
dd = tr.select_one("p.course-dd")
|
||
out.append({
|
||
"section": sec,
|
||
"name": a.get_text(" ", strip=True),
|
||
"info": re.sub(r"\s+", " ", dd.get_text(" ", strip=True)) if dd else "",
|
||
"plan_id": plan,
|
||
"project_id": project,
|
||
"url": urljoin(JSPX + "/train/trainee/", href),
|
||
"progress": "",
|
||
})
|
||
return out
|
||
|
||
|
||
def scrape_courses(s, with_progress=True):
|
||
"""把「我的课程」下三个板块 × 三种状态**全部**拉出来，含已学完的。
|
||
|
||
站点把列表拆成了 AJAX 接口，每个组合一次请求：
|
||
|
||
/train/trainee/studyingCourse.action?courseType=studying&planStatus=TAKING_PLAN
|
||
/train/assist/assistanceCourse.action?courseType=assistance&planStatus=FINISHED_PLAN
|
||
/train/inspect/inspectingCourse.action?courseType=inspecting&planStatus=NO_BEGAIN_PLAN
|
||
...
|
||
|
||
只看 /admin/main.action 是看不到「已结束」课程的——那个首页只渲染在学/助学/督学
|
||
三张「进行中」的概览表，课一学完就从首页消失了。
|
||
|
||
返回空列表只代表「三个板块九种状态都没课」；
|
||
会话失效或页面改版一律抛异常，让调用方能给出准确提示。
|
||
"""
|
||
# 先用首页校验会话，避免九个接口各自失败还看不出来
|
||
_jspx_call(s, lambda: fetch_admin(s))
|
||
|
||
out, seen = [], set()
|
||
for section, path, ctype in COURSE_SECTIONS:
|
||
for label, status in PLAN_STATUS:
|
||
url = "%s%s?courseType=%s&planStatus=%s" % (JSPX, path, ctype, status)
|
||
try:
|
||
html = s.get(url, timeout=40).text
|
||
except Exception:
|
||
continue
|
||
for c in _parse_course_rows(html, section):
|
||
key = (c["plan_id"], c["project_id"])
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
c["course_type"] = ctype
|
||
c["status"] = status
|
||
c["status_label"] = label
|
||
c["in_study"] = status in STUDYABLE_STATUS
|
||
out.append(c)
|
||
|
||
# 进度只在「进行中/未开始」的课上查，已结束的动辄几十门，逐个请求太慢
|
||
if with_progress:
|
||
for c in out:
|
||
if not c["in_study"]:
|
||
continue
|
||
try:
|
||
rr = s.get(c["url"], timeout=30)
|
||
p = BeautifulSoup(rr.text, "lxml").select_one("p.loading font")
|
||
c["progress"] = p.get_text(strip=True) if p else ""
|
||
except Exception:
|
||
pass
|
||
return out
|
||
|
||
|
||
def parse_jspx_course(html):
|
||
"""老版 jspx 课程页（tier-tree）解析。"""
|
||
soup = BeautifulSoup(html, "lxml")
|
||
course = soup.select_one("h1")
|
||
prog = soup.select_one("p.loading font")
|
||
info = soup.select_one("div.results-info p.info")
|
||
|
||
cw, fb, ts = [], [], []
|
||
for one in soup.select("div.tier-tree-one"):
|
||
ns = one.select_one("p.tier1.has-children span.name")
|
||
chapter = re.sub(r"\s+", " ", ns.get_text(" ", strip=True)) if ns else "?"
|
||
for p in one.select("p.tier3"):
|
||
a = p.select_one("a")
|
||
if not a:
|
||
continue
|
||
href = a.get("href") or ""
|
||
txt = re.sub(r"\s+", " ", p.get_text(" ", strip=True))
|
||
name = p.select_one("span.name")
|
||
title = re.sub(r"\s+", " ", name.get_text(" ", strip=True)) if name else txt[:40]
|
||
m = re.search(r"(\d+)分钟", txt)
|
||
done = "已完成" in txt
|
||
if "courseware.action" in href:
|
||
cw.append({"chapter": chapter, "title": title, "href": href, "done": done,
|
||
"duration": int(m.group(1)) if m else None})
|
||
elif "doFeedback.action" in href:
|
||
fb.append({"title": title, "href": href, "done": done})
|
||
elif "doTest.action" in href:
|
||
ts.append({"title": title, "href": href, "done": done})
|
||
return {
|
||
"user": whoami(html),
|
||
"course": course.get_text(" ", strip=True) if course else "?",
|
||
"progress": prog.get_text(strip=True) if prog else "?",
|
||
"info": re.sub(r"\s+", " ", info.get_text(" ", strip=True)) if info else "?",
|
||
"courseware": cw, "feedback": fb, "test": ts,
|
||
}
|
||
|
||
|
||
def discover(s, plan, project, name=""):
|
||
"""打开课程页并解析；会话失效时先用 CASTGC 补一次 jspx 授权再重试。"""
|
||
return _jspx_call(s, lambda: _discover_once(s, plan, project, name))
|
||
|
||
|
||
def _discover_once(s, plan, project, name=""):
|
||
# study.py 反向 import 本模块，模块级 import 会成环，用到时再取
|
||
from study_tool.platforms.study import (StudyClient, discover_study, extract_study_token,
|
||
extract_train_id, save_snapshot,
|
||
study_resolve_train_id)
|
||
url = "%s/train/trainee/courseStudyMain.action?planId=%s&projectId=%s&trainRole=0" % (JSPX, plan, project)
|
||
r = s.get(url, timeout=40)
|
||
html = r.text or ""
|
||
if "/admin/login.action" in html and len(html) < 2000:
|
||
# 课程页也走同一个判断：CASTGC 有效就无密码续期
|
||
if ensure_jspx_session(s):
|
||
r = s.get(url, timeout=40)
|
||
html = r.text or ""
|
||
if "没有权限" in html:
|
||
raise RuntimeError("该账号对此课程没有权限")
|
||
if "/admin/login.action" in html and len(html) < 2000:
|
||
raise SessionExpired("登录已失效：课程页把请求跳回了 /admin/login.action（请重新登录）")
|
||
|
||
# 课程页 302 到 study 时，token 在跳转 URL 里；哪怕 SPA 外壳返回空也要当成学过的新平台
|
||
study_url = "study.dgjy.net" in (r.url or "")
|
||
tok = extract_study_token(r.url, r.history)
|
||
if tok:
|
||
host = urlsplit(r.url).hostname or urlsplit(STUDY).hostname
|
||
try:
|
||
s.cookies.set("Middle-Token", tok, domain=host, path="/")
|
||
except Exception:
|
||
pass
|
||
|
||
if len(html) < 3000 and not (study_url or tok or has_study_marker(html, r.url)):
|
||
raise RuntimeError("课程页返回异常（页面长度仅 %d，登录可能已失效）" % len(html))
|
||
|
||
generation = detect_generation(html, r.url)
|
||
# 按登录 / 学习入口识别平台；以后新增平台时这里不用改，靠 Platform.entries 命中
|
||
pf = registry.detect(r.url, html)
|
||
platform_id = pf.id if pf else "guanyixue"
|
||
if generation == "legacy":
|
||
d = parse_jspx_course(html)
|
||
if d["courseware"] or d["feedback"] or d["test"]:
|
||
d["platform"] = platform_id
|
||
d["generation"] = "legacy"
|
||
return d
|
||
# 老结构但一个活动都没有：多半是「慕课」外链页（页面还在 jspx，课时在 study 平台）
|
||
if not has_study_marker(html, r.url):
|
||
snap = save_snapshot(plan, project, r.url, html)
|
||
raise RuntimeError("课程页能打开，但没有任何课件/反馈表/测验（页面已存 %s）。"
|
||
"若是慕课课程，请确认新平台登录态是否有效" % snap)
|
||
elif generation != "spa":
|
||
snap = save_snapshot(plan, project, r.url, html)
|
||
raise RuntimeError("无法识别课程页结构（站点可能又改版）：页面里既没有 jspx 的 "
|
||
"tier-tree，也没有新版 SPA 标记。页面已存 %s，请把它发我" % snap)
|
||
|
||
# ---- 新版平台（Vue SPA + /train/course/front/* JSON 接口）----
|
||
tid = extract_train_id(r.url, html)
|
||
user = whoami(html)
|
||
client = StudyClient(s, r.url)
|
||
|
||
def _load():
|
||
nonlocal tid
|
||
if not tid:
|
||
tid = study_resolve_train_id(client, name)
|
||
if not tid:
|
||
snap = save_snapshot(plan, project, r.url, html)
|
||
raise RuntimeError("新版课程页未带 trainCourseId，且按课程名也匹配不到（课程名：%s）。"
|
||
"页面已存 %s" % (name or "?", snap))
|
||
return discover_study(client, tid, name, user)
|
||
|
||
try:
|
||
d = _load()
|
||
except SessionExpired:
|
||
client.bootstrap()
|
||
tid = extract_train_id(r.url, html)
|
||
d = _load()
|
||
d["platform"] = platform_id
|
||
d["generation"] = "spa"
|
||
# URL 里可能带 token（凭据），日志/状态里一律打码；鉴权已经写进 cookie
|
||
d["page_url"] = re.sub(r"([?&])(token|access_token|securityKey|sign)=[^&]*", r"\1\2=***", r.url or "")
|
||
d["plan_id"] = plan
|
||
d["project_id"] = project
|
||
return d
|
||
|
||
|
||
# ------------------------------------------------------------ 旧版学习流程
|
||
|
||
class CwItem:
|
||
def __init__(self, d, cookies):
|
||
self.title = d["title"]
|
||
href = d["href"]
|
||
self.url = href if href.startswith("http") else JSPX + href
|
||
self.s = new_session(cookies)
|
||
self.rec = None
|
||
self.status = "done" if d.get("done") else "pending"
|
||
self.last = None
|
||
|
||
def resolve(self):
|
||
try:
|
||
r = self.s.get(self.url, timeout=40)
|
||
except Exception as e:
|
||
return "exc:%s" % e
|
||
m = re.search(r"'id':'([0-9A-F]{32})'", r.text)
|
||
if not m:
|
||
return "no-id"
|
||
self.rec = m.group(1)
|
||
return "ok"
|
||
|
||
def start(self):
|
||
try:
|
||
r = self.s.post(EP_STUDY, data={"id": self.rec, "order": "server_start",
|
||
"updatePassPerc": "true"}, timeout=40)
|
||
return r.text.strip()[:40]
|
||
except Exception as e:
|
||
return "EXC:%s" % e
|
||
|
||
def beat(self):
|
||
try:
|
||
r = self.s.post(EP_STUDY, data={"id": self.rec, "order": "server_time",
|
||
"updatePassPerc": "true",
|
||
"videoSchedule": 30}, timeout=40)
|
||
return r.text.strip()[:60]
|
||
except Exception as e:
|
||
return "EXC:%s" % e
|
||
|
||
|
||
def submit_feedback(s, href, key):
|
||
url = href if href.startswith("http") else JSPX + href
|
||
r = s.get(url, timeout=40)
|
||
soup = BeautifulSoup(r.text, "lxml")
|
||
fields = {i.get("name"): i.get("value") for i in soup.select("form input[type=hidden]") if i.get("name")}
|
||
if fields.get("allowStudy") == "false":
|
||
log(key, "反馈表：allowStudy=false，跳过")
|
||
return
|
||
r2 = s.post(JSPX + "/train/trainee/doFeedbackList.action", data=fields, timeout=40)
|
||
qs = BeautifulSoup(r2.text, "lxml")
|
||
answers = []
|
||
for it in qs.select("div.job-item"):
|
||
qtype = it.get("questype")
|
||
opts = [sp.get("data-name") for sp in it.select(".job-item-a span")]
|
||
ans = ("课程内容充实，讲师讲解清晰，对我的教学实践很有帮助，希望今后能增加更多一线教学案例。"
|
||
if qtype == "04" else (opts[0] if opts else "A"))
|
||
answers.append({"id": it.get("traineeid"), "quesId": it.get("qumid"),
|
||
"quesType": qtype, "quesNo": it.get("quesno"), "answer": ans})
|
||
payload = {"planId": fields.get("planId"), "objectId": fields.get("objectId"),
|
||
"answers": json.dumps(answers, ensure_ascii=False, separators=(",", ":")),
|
||
"submitStatus": "finish"}
|
||
rr = s.post(JSPX + "/train/trainee/saveFeedback.action", data=payload, timeout=40)
|
||
log(key, "反馈表：%d 题，%s" % (len(answers), "提交成功" if "MSG" not in rr.text else "返回 " + rr.text[:80]))
|
||
|
||
|
||
def submit_quiz(s, href, key, answer_key=None):
|
||
url = href if href.startswith("http") else JSPX + href
|
||
plan, _ = parse_ids(url)
|
||
m = re.search(r"objectId=([0-9A-Fa-f]+)", url)
|
||
obj = m.group(1) if m else ""
|
||
rr = s.get("%s/train/trainee/showTestQums.action?planId=%s&objectId=%s" % (JSPX, plan, obj), timeout=40)
|
||
qs = BeautifulSoup(rr.text, "lxml")
|
||
singles = (answer_key or {}).get("singles", {})
|
||
multis = (answer_key or {}).get("multis", {})
|
||
answers = []
|
||
for it in qs.select("div.job-item"):
|
||
t, qno = it.get("questype"), it.get("quesno")
|
||
opts = [sp.get("data-name") for sp in it.select(".job-item-a span")]
|
||
if t == "02":
|
||
ans = (multis.get(qno) or ("#".join(opts) if opts else "A"))
|
||
else:
|
||
ans = singles.get(qno) or (opts[0] if opts else "A")
|
||
answers.append({"id": it.get("traineeid"), "quesId": it.get("qumid"),
|
||
"quesType": t, "quesNo": qno, "answer": ans})
|
||
if not answers:
|
||
log(key, "测验：未取到题目，跳过")
|
||
return
|
||
payload = {"planId": plan, "objectId": obj,
|
||
"answers": json.dumps(answers, ensure_ascii=False, separators=(",", ":"))}
|
||
r3 = s.post(JSPX + "/train/trainee/endTest.action", data=payload, timeout=40)
|
||
r4 = s.get("%s/train/trainee/doTest.action?planId=%s&objectId=%s" % (JSPX, plan, obj), timeout=40)
|
||
sc = re.search(r"得分：\s*(\d+)\s*分",
|
||
re.sub(r"\s+", " ", BeautifulSoup(r4.text, "lxml").get_text(" ", strip=True)))
|
||
log(key, "测验：%d 题，%s，得分 %s" %
|
||
(len(answers), "提交成功" if "MSG" not in r3.text else r3.text[:60],
|
||
sc.group(1) if sc else "?"))
|
||
|
||
|
||
def _study_builtin(key, s, d, workers=1, cap=30, enable_quiz=False, enable_feedback=False,
|
||
mode="fast", parallel=1, quiz_key="", cookies=None, plan="", project="",
|
||
name=""):
|
||
"""旧版 jspx 的内置流程：反馈表 + 测验 + 刷课件。"""
|
||
if enable_feedback:
|
||
for fb in d["feedback"]:
|
||
if fb["done"]:
|
||
log(key, "反馈表已完成，跳过")
|
||
continue
|
||
try:
|
||
submit_feedback(s, fb["href"], key)
|
||
except Exception as e:
|
||
log(key, "反馈表异常：%s" % e)
|
||
else:
|
||
for fb in d["feedback"]:
|
||
log(key, "反馈表：已按设置跳过（%s）" % ("已完成" if fb["done"] else "未完成"))
|
||
|
||
if enable_quiz:
|
||
for t in d["test"]:
|
||
if t["done"]:
|
||
log(key, "测验已完成，跳过")
|
||
continue
|
||
try:
|
||
submit_quiz(s, t["href"], key, quiz_key)
|
||
except Exception as e:
|
||
log(key, "测验异常：%s" % e)
|
||
else:
|
||
for t in d["test"]:
|
||
if t["done"]:
|
||
log(key, "测验：已完成，跳过")
|
||
else:
|
||
log(key, "测验：⚠ 已按设置跳过答题（避免因正确率不足卡住「完成」），"
|
||
"需要时请在页面上手动作答，或打开「自动交测验」开关")
|
||
|
||
objs = [CwItem(c, cookies) for c in d["courseware"]]
|
||
todo = [o for o in objs if o.status != "done"]
|
||
done0 = len(objs) - len(todo)
|
||
set_state(key, status="studying", total=len(objs), done=done0)
|
||
if todo:
|
||
log(key, "解析 %d 个学习记录…" % len(todo))
|
||
with ThreadPoolExecutor(max_workers=min(workers, len(todo))) as ex:
|
||
list(ex.map(lambda x: x.resolve(), todo))
|
||
live = [o for o in todo if o.rec]
|
||
log(key, "已解析 %d/%d，并发启动会话" % (len(live), len(todo)))
|
||
with ThreadPoolExecutor(max_workers=min(workers, len(live))) as ex:
|
||
list(ex.map(lambda x: x.start(), live))
|
||
|
||
t0 = time.time()
|
||
while time.time() - t0 < cap * 60:
|
||
with RUNS_LOCK:
|
||
if (RUNS.get(key) or {}).get("status") == "stopped":
|
||
log(key, "已按要求停止")
|
||
return
|
||
pending = [o for o in live if o.status != "done"]
|
||
if not pending:
|
||
break
|
||
with ThreadPoolExecutor(max_workers=min(workers, len(pending))) as ex:
|
||
resps = list(ex.map(lambda x: x.beat(), pending))
|
||
newly = []
|
||
for o, resp in zip(pending, resps):
|
||
o.last = resp
|
||
if resp == "study_over":
|
||
o.status = "done"
|
||
newly.append(o)
|
||
el = int(time.time() - t0)
|
||
dc = len([o for o in live if o.status == "done"]) + done0
|
||
set_state(key, done=dc, elapsed=el)
|
||
log(key, "第 %d 轮 %dm%02ds 完成 %d/%d" % (el // 60 + 1, el // 60, el % 60, dc, len(objs)))
|
||
for o in newly:
|
||
log(key, " ✓ %s" % o.title[:40])
|
||
time.sleep(60)
|
||
else:
|
||
log(key, "所有课件均已完成")
|
||
|
||
try:
|
||
d2 = discover(s, plan, project, name)
|
||
set_state(key, progress=d2["progress"], info=d2["info"], cookies=dump_cookies(s))
|
||
log(key, "课程结束：%s | %s" % (d2["progress"], d2["info"]))
|
||
except Exception as e:
|
||
log(key, "复查失败：%s" % e)
|
||
|
||
|
||
|
||
# ---------------------------------------------------------------- 平台注册
|
||
|
||
def _study(key, s, d, **opts):
|
||
"""旧版页面：反馈表 + 测验 + 刷课件。"""
|
||
return _study_builtin(key, s, d, **opts)
|