refactor: React 前端重构,新旧平台合并为莞易学
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
"""平台插件包。
|
||||
|
||||
内置平台在 `load_builtin()` 里导入(导入即注册)。新增平台:
|
||||
1. 新建 `platforms/<名字>.py`,实现 `Platform.study`、声明 `entries` 登录 / 学习入口,
|
||||
并 `registry.register(...)`;
|
||||
2. 在下面的 `load_builtin()` 里加一行 import;
|
||||
3. 路由 / 引擎不用改 —— 它们只认 `course["platform"]`。
|
||||
"""
|
||||
from study_tool.platforms import registry # noqa: F401 供 engine、路由使用
|
||||
from study_tool.platforms.base import Platform # noqa: F401
|
||||
|
||||
|
||||
def load_builtin():
|
||||
"""导入内置平台,触发注册。幂等,app 启动时调用一次。"""
|
||||
from study_tool.platforms import guanyixue # noqa: F401
|
||||
@@ -0,0 +1,33 @@
|
||||
"""平台插件契约。
|
||||
|
||||
新增一个平台 = 在 `platforms/` 下加一个模块,构造 `Platform` 并 `registry.register()`,
|
||||
然后把它加进 `platforms/__init__.py` 的 `load_builtin()`。引擎只按
|
||||
`course["platform"]` 去注册表取实现,不再 import 具体平台。
|
||||
|
||||
`entries` 声明登录入口 / 学习入口的 URL 片段,`registry.detect(url, html)` 用它识别
|
||||
页面属于哪个平台;同一平台的不同代页面(如莞易学的旧版 / 新版)用 `course["generation"]`
|
||||
区分,由平台内部自行分发。
|
||||
|
||||
Platform.study 的签名约定:
|
||||
study(key, s, d, **opts)
|
||||
key 运行 key(账号 id,日志也用它)
|
||||
s requests.Session(带该账号 cookie)
|
||||
d discover() 返回的课程描述(course/courseware/feedback/test/platform/generation/...)
|
||||
opts workers / cap / enable_quiz / enable_feedback / mode / parallel /
|
||||
quiz_key / cookies / plan / project / name
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class Platform:
|
||||
id: str # course["platform"] 的取值
|
||||
label: str # 日志 / 界面里的展示名
|
||||
study: Optional[Callable] = None # 学一门课;None = 这个平台还没实现
|
||||
entries: tuple = () # 登录 / 学习入口 URL 片段,用于识别平台
|
||||
|
||||
def matches(self, url="", html=""):
|
||||
"""登录入口或学习入口命中任意片段,就认为页面属于本平台。"""
|
||||
text = "%s\n%s" % (url or "", html or "")
|
||||
return any(e and e in text for e in self.entries)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""页面代际识别。
|
||||
|
||||
莞易学的登录入口(jspx.dgjy.net)和学习入口(study.dgjy.net)指向同一套课程,
|
||||
但课程页有两代前端:旧版 jspx 页面和新版 Vue SPA。这里只根据页面 / URL 判断代际,
|
||||
平台归属由 `Platform.entries`(登录、学习入口)负责。
|
||||
"""
|
||||
|
||||
# 旧版页面骨架
|
||||
LEGACY_MARKERS = ("tier-tree-one", "tier3")
|
||||
|
||||
# 新版 SPA 标记:Vue 入口、培训课程接口路由、study 域名
|
||||
SPA_MARKERS = ("/training/curriculum", "/training/courseVideo", "/training/comment",
|
||||
"static/js/app.", "study.dgjy.net/training", "study.dgjy.net/#/")
|
||||
SPA_HOST = "study.dgjy.net"
|
||||
|
||||
|
||||
def has_study_marker(html, page_url=""):
|
||||
"""页面是不是新版莞易学(study.dgjy.net / Vue SPA)。"""
|
||||
if page_url and SPA_HOST in page_url:
|
||||
return True
|
||||
if any(k in html for k in SPA_MARKERS):
|
||||
return True
|
||||
flat = html.replace('"', "").replace("'", "")
|
||||
return "<div id=app>" in flat
|
||||
|
||||
|
||||
def detect_generation(html, page_url=""):
|
||||
"""返回课程页代际:legacy(旧版)/ spa(新版)/ ""(无法识别)。"""
|
||||
if any(k in html for k in LEGACY_MARKERS):
|
||||
return "legacy"
|
||||
if has_study_marker(html, page_url):
|
||||
return "spa"
|
||||
return ""
|
||||
@@ -0,0 +1,22 @@
|
||||
"""莞易学平台:jspx.dgjy.net(登录入口)与 study.dgjy.net(学习入口)是同一个平台。
|
||||
|
||||
课程页有两代前端,`discover` 会把代际写进 `course["generation"]`(legacy / spa),
|
||||
这里按代际分发到对应的学习实现。新增其他平台时,照本文件写一个模块:
|
||||
在 `Platform.entries` 声明登录 / 学习入口,注册表就能按入口识别平台。
|
||||
"""
|
||||
from study_tool.platforms import jspx, registry, study
|
||||
from study_tool.platforms.base import Platform
|
||||
|
||||
ID = "guanyixue"
|
||||
LABEL = "莞易学"
|
||||
# 登录入口 / 学习入口;命中任意一个就归到莞易学
|
||||
ENTRIES = ("jspx.dgjy.net", "study.dgjy.net", "dgjy.net")
|
||||
|
||||
|
||||
def _study(key, s, d, **opts):
|
||||
if d.get("generation") == "spa":
|
||||
return study._study(key, s, d, **opts)
|
||||
return jspx._study(key, s, d, **opts)
|
||||
|
||||
|
||||
PLATFORM = registry.register(Platform(id=ID, label=LABEL, study=_study, entries=ENTRIES))
|
||||
@@ -0,0 +1,635 @@
|
||||
"""旧版平台(jspx.dgjy.net):会话 / cookie / 登录 / 课程抓取。
|
||||
|
||||
对外:new_session / login / scrape_courses / discover。
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from html import unescape as html_unescape
|
||||
from urllib.parse import unquote, urljoin, urlsplit
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from study_tool.config import *
|
||||
from study_tool.platforms import registry
|
||||
from study_tool.platforms.detect import detect_generation, has_study_marker
|
||||
|
||||
# ------------------------------------------------------------------------ crypto
|
||||
|
||||
def str_enc(data: str) -> str:
|
||||
r = subprocess.run([NODE, DESENC, data, "1", "2", "3"],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError("strEnc failed: %s" % (r.stderr or "")[:200])
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- http client
|
||||
|
||||
def new_session(cookies=None):
|
||||
s = requests.Session()
|
||||
s.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
|
||||
for c in (cookies or []):
|
||||
if "dgjy" not in (c.get("domain") or ""):
|
||||
continue
|
||||
s.cookies.set(c["name"], c.get("value") or "",
|
||||
domain=(c.get("domain") or "").lstrip("."),
|
||||
path=c.get("path") or "/")
|
||||
return s
|
||||
|
||||
|
||||
def dump_cookies(s):
|
||||
return [{"name": c.name, "value": c.value, "domain": c.domain, "path": c.path}
|
||||
for c in s.cookies]
|
||||
|
||||
|
||||
def whoami(html):
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
hdr = soup.select_one("#header")
|
||||
if hdr:
|
||||
m = re.search(r"你好[,,]?\s*(\S+)\s*老师", re.sub(r"\s+", " ", hdr.get_text(" ", strip=True)))
|
||||
if m:
|
||||
return m.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def parse_ids(text):
|
||||
plan = project = ""
|
||||
m = re.search(r"planId=([0-9A-Fa-f]{16,})", text or "")
|
||||
if m:
|
||||
plan = m.group(1)
|
||||
m = re.search(r"projectId=([0-9A-Fa-f]{16,})", text or "")
|
||||
if m:
|
||||
project = m.group(1)
|
||||
return plan, project
|
||||
|
||||
|
||||
def _follow_js_login(s, page):
|
||||
"""补上浏览器才会执行的两跳,否则只拿得到 CASTGC、拿不到 jspx 的会话。 `oauth2.0/authorize` 链的最后一跳**不是 302**,而是 dj.dgjy.net 上一个
|
||||
1486 字节的 JS 中转页,里面动态插 <script>:
|
||||
|
||||
scriptLogin.src = 'https://passport.dgjy.net/scriptLogin?action=simpleLogin&d1=...&d2=...';
|
||||
|
||||
scriptLogin 返回的是 JS,不是跳转:
|
||||
|
||||
var loginError=null;
|
||||
var verifyURL="https://dj.dgjy.net/fpf/login/verify.action?ticket=...&auth=...&sid=...";
|
||||
location.href=verifyURL;
|
||||
|
||||
verify.action 之后才会 302 → loginCall → jspx/tstudy/passport/login.action,
|
||||
到这一步 jspx.dgjy.net 的 JSESSIONID 才真正下发。
|
||||
requests 不执行 JS,所以这两跳必须手工走一遍。
|
||||
"""
|
||||
m = re.search(r"scriptLogin\.src\s*=\s*'([^']+)'", page or "")
|
||||
if not m:
|
||||
return False
|
||||
r = s.get(html_unescape(m.group(1)), timeout=40)
|
||||
body = r.text or ""
|
||||
if re.search(r"loginError\s*=\s*(?!null)", body):
|
||||
raise RuntimeError("SSO 校验未通过:%s" % body.strip()[:200])
|
||||
m2 = re.search(r'verifyURL\s*=\s*"([^"]+)"', body)
|
||||
if not m2:
|
||||
return False
|
||||
s.get(html_unescape(m2.group(1)).replace("\\/", "/"), timeout=40, allow_redirects=True)
|
||||
return True
|
||||
|
||||
|
||||
def _js_redirect_candidates(page):
|
||||
"""从一个不执行 JS 的响应体里,按出现顺序抽出下一跳 URL。"""
|
||||
out = []
|
||||
for pat in (r"scriptLogin\.src\s*=\s*'([^']+)'",
|
||||
r'scriptLogin\.src\s*=\s*"([^"]+)"',
|
||||
r'verifyURL\s*=\s*"([^"]+)"',
|
||||
r"verifyURL\s*=\s*'([^']+)'",
|
||||
r"location\.href\s*=\s*['\"]([^'\"]+)['\"]",
|
||||
r"window\.location\.href\s*=\s*['\"]([^'\"]+)['\"]"):
|
||||
out.extend(re.findall(pat, page or ""))
|
||||
return [html_unescape(u).replace("\\/", "/") for u in out if u]
|
||||
|
||||
|
||||
def follow_js_hops(s, start_url, stop=None, max_hops=6):
|
||||
"""通用 SSO 补跳:页面给的不是 302 而是 JS/scriptLogin 时,手工走完。
|
||||
|
||||
study.dgjy.net 的授权链和 jspx 不是同一套页面模板,所以这里按「下一跳 URL」
|
||||
逐个 GET,直到 stop() 为真(例如 Middle-Token 已经下发)或没有下一跳。
|
||||
"""
|
||||
cur, seen = start_url, set()
|
||||
for _ in range(max_hops):
|
||||
if cur in seen:
|
||||
break
|
||||
seen.add(cur)
|
||||
try:
|
||||
r = s.get(cur, timeout=40, allow_redirects=True)
|
||||
except Exception:
|
||||
break
|
||||
if stop and stop():
|
||||
return True
|
||||
nxt = _js_redirect_candidates(r.text or "")
|
||||
if not nxt:
|
||||
break
|
||||
cur = urljoin(r.url, nxt[0])
|
||||
return bool(stop and stop())
|
||||
|
||||
|
||||
def _login_error(html, status=200):
|
||||
"""从 /tpass/login 的响应里取出服务端提示。
|
||||
|
||||
提示藏在 <script type="text/html" id="login_template"> 里的
|
||||
<span id="errormsghide" style="display:none">,BeautifulSoup 的 get_text()
|
||||
看不见 script 内容,所以只能按原文正则取;锁号是另一张 403 提示页。
|
||||
"""
|
||||
text = html or ""
|
||||
m = re.search(r"验证失败次数过多[^<]*", text)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
m = re.search(r'<span[^>]*id="errormsghide"[^>]*>(.*?)</span>', text, re.S)
|
||||
if m:
|
||||
msg = re.sub(r"<[^>]+>", " ", m.group(1))
|
||||
msg = re.sub(r"\s+", " ", html_unescape(msg).replace("\xa0", " ")).strip()
|
||||
if msg:
|
||||
return msg
|
||||
m = re.search(r"(用户名|密码|账号|账户)[^<>]{0,40}(错误|不存在|不正确|锁定)", text)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
if status == 403:
|
||||
return "账号已被锁定(403),约 10 分钟后解锁"
|
||||
return ""
|
||||
|
||||
|
||||
def parse_accountlist(html):
|
||||
"""多身份账号:登录响应里内联 `var accountlist = '<json>';`。
|
||||
|
||||
每个元素带 USERNAME(身份证)、CODENAME(教师/家长/学生)、UNIT_NAME,
|
||||
以及该身份自己的 LOGINUSERNAME / LOGINPASSWORD —— 选身份那一步是服务端
|
||||
把可登录的账号密码回给前端,再带 choosenumber 提交一次。
|
||||
"""
|
||||
m = re.search(r"var\s+accountlist\s*=\s*'(.*?)';", html or "", re.S)
|
||||
if not m:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(html_unescape(m.group(1)))
|
||||
except Exception:
|
||||
return []
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
|
||||
def _login_as_identity(s, un, lt, ex, it):
|
||||
"""按 login_dgjy.js 的选身份分支提交:rsa 用该身份的 LOGINUSERNAME+LOGINPASSWORD。"""
|
||||
name = it.get("LOGINUSERNAME") or ""
|
||||
pwd = it.get("LOGINPASSWORD") or ""
|
||||
payload = {"un": un, "ul": str(len(name)), "pl": str(len(pwd)),
|
||||
"rsa": str_enc(name + pwd + lt), "lt": lt,
|
||||
"choosenumber": it.get("USERNAME") or "", "_eventId": "submit"}
|
||||
if ex:
|
||||
payload["execution"] = ex
|
||||
return s.post(TPASS + "/login", data=payload, timeout=30, allow_redirects=True,
|
||||
headers=LOGIN_HEADERS)
|
||||
|
||||
|
||||
def ensure_jspx_session(s):
|
||||
"""用 CASTGC 走老平台(jspx)的授权链,建立 jspx.dgjy.net 的会话;成功返回 True。
|
||||
|
||||
必须先把 jspx 的 JSESSIONID / route 清掉:这两个是粘性会话 cookie,
|
||||
浏览器里带过来的那份常常指向已经下线的后端节点(route 对不上号),
|
||||
留着就永远跳 /admin/login.action,看起来像「登录失效」。
|
||||
"""
|
||||
try:
|
||||
r = s.get(JSPX + "/admin/main.action", timeout=30)
|
||||
if "table-results" in (r.text or ""):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
for c in list(s.cookies):
|
||||
if "jspx" in (c.domain or ""):
|
||||
s.cookies.clear(c.domain, c.path, c.name)
|
||||
try:
|
||||
r = s.get(AUTHORIZE, timeout=40, allow_redirects=True)
|
||||
_follow_js_login(s, r.text or "")
|
||||
r2 = s.get(JSPX + "/admin/main.action", timeout=30)
|
||||
return "table-results" in (r2.text or "")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _jspx_call(s, fn):
|
||||
"""跑一个 jspx 请求;会话失效且手上还有 CASTGC 时,清掉粘性 cookie 重新授权后重试一次。
|
||||
|
||||
浏览器 / IndexedDB 里存着的 jspx JSESSIONID+route 经常指向已经下线的后端节点,
|
||||
fetch_admin 看到的就是「跳回 /admin/login.action」,看着像登录过期,
|
||||
实际 CASTGC 还有效 —— 这种不该让用户重新输密码。
|
||||
"""
|
||||
try:
|
||||
return fn()
|
||||
except SessionExpired:
|
||||
if not ensure_jspx_session(s):
|
||||
raise
|
||||
return fn()
|
||||
|
||||
|
||||
def login(username: str, password: str):
|
||||
"""SSO login -> (session, user)。"""
|
||||
s = new_session()
|
||||
r = s.get(TPASS + "/login", timeout=30)
|
||||
lt = re.search(r'id="lt"[^>]*value="([^"]*)"', r.text)
|
||||
lt = lt.group(1) if lt else ""
|
||||
ex = re.search(r'name="execution"[^>]*value="([^"]*)"', r.text)
|
||||
payload = {
|
||||
"un": username, "pd": password,
|
||||
"ul": str(len(username)), "pl": str(len(password)),
|
||||
"rsa": str_enc(username + password + lt),
|
||||
"lt": lt, "choosenumber": "", "_eventId": "submit",
|
||||
}
|
||||
if ex:
|
||||
payload["execution"] = ex.group(1)
|
||||
r2 = s.post(TPASS + "/login", data=payload, timeout=30, allow_redirects=True,
|
||||
headers=LOGIN_HEADERS)
|
||||
|
||||
if not any(c.name == "CASTGC" for c in s.cookies):
|
||||
accts = parse_accountlist(r2.text)
|
||||
if accts:
|
||||
# 一个身份证下挂多个身份时,第一次提交只回账号选择页,不会给票据。
|
||||
# 这里取教师身份(本平台主要场景),没有就取第一个,然后用该身份重提。
|
||||
pick = next((x for x in accts if "教师" in (x.get("CODENAME") or "")), accts[0])
|
||||
m = re.search(r'id="lt"[^>]*value="([^"]*)"', r2.text)
|
||||
ex2 = re.search(r'name="execution"[^>]*value="([^"]*)"', r2.text)
|
||||
r2 = _login_as_identity(s, username, (m.group(1) if m else lt),
|
||||
(ex2.group(1) if ex2 else (ex.group(1) if ex else "")), pick)
|
||||
if not any(c.name == "CASTGC" for c in s.cookies):
|
||||
msg = _login_error(r2.text, r2.status_code)
|
||||
raise RuntimeError((msg or "登录失败:未获得登录票据(账号或密码错误?)")[:200])
|
||||
|
||||
# 老平台:authorize 链的终点是 JS 中转页,必须手工补两跳才建得起 jspx 会话
|
||||
if not ensure_jspx_session(s):
|
||||
raise RuntimeError("SSO 登录未完成:jspx.dgjy.net 没有建立会话(登录链路可能已改版)")
|
||||
html = fetch_admin(s)
|
||||
# 同一张 CASTGC 再授权一次给新版平台(client_id=gyx),拿 Middle-Token
|
||||
ensure_study_session(s)
|
||||
return s, whoami(html)
|
||||
|
||||
|
||||
def _next_hop_urls(page):
|
||||
"""不执行 JS 时,从响应体里找出所有可能的下一跳(SSO 中转页/脚本/表单)。"""
|
||||
out = _js_redirect_candidates(page)
|
||||
for m in re.finditer(r'(?:src|href)\s*=\s*["\']([^"\']*(?:login|callback|authorize|verify|code)[^"\']*)["\']',
|
||||
page or "", re.I):
|
||||
out.append(html_unescape(m.group(1)))
|
||||
for m in re.finditer(r'<form[^>]+action=["\']([^"\']+)["\']', page or "", re.I):
|
||||
out.append(html_unescape(m.group(1)))
|
||||
seen, uniq = set(), []
|
||||
for u in out:
|
||||
u = u.replace("\\/", "/")
|
||||
if u.startswith("http") or u.startswith("/") or "?" in u:
|
||||
if u not in seen:
|
||||
seen.add(u)
|
||||
uniq.append(u)
|
||||
return uniq
|
||||
|
||||
|
||||
def ensure_study_session(s, force=False):
|
||||
"""用现有 SSO 票把 study.dgjy.net 的会话补上。
|
||||
|
||||
新版平台的登录是 OAuth 授权码换 token(前端 SPA 源码里写死了这几个地址):
|
||||
GET tyyh.dgjy.net/tpass/oauth2.0/authorize?client_id=gyx
|
||||
&redirect_uri=https://study.dgjy.net/#/login/back&response_type=code
|
||||
... -> study.dgjy.net/#/login/back?code=ST-xxxx-tpass
|
||||
POST study.dgjy.net/mod/gjt/login/<CODE> # 换出前端会话(JSESSIONID)
|
||||
GET study.dgjy.net/mod/education/gyx/train/front/login
|
||||
-> {"result":{"url":".../training","token":"<jwt>"}} # 这才是 Middle-Token
|
||||
成功标准只有一个:cookie 里出现 study.dgjy.net 的 Middle-Token。
|
||||
"""
|
||||
if force:
|
||||
for c in list(s.cookies):
|
||||
if c.name == "Middle-Token":
|
||||
try:
|
||||
s.cookies.clear(c.domain, c.path, c.name)
|
||||
except Exception:
|
||||
pass
|
||||
if _study_token(s):
|
||||
return True
|
||||
try:
|
||||
r = s.get(TPASS + TRAIN_AUTHORIZE_PATH, timeout=40, allow_redirects=True)
|
||||
except Exception:
|
||||
return False
|
||||
cur_url, body = r.url or "", r.text or ""
|
||||
tried = set()
|
||||
for _ in range(6):
|
||||
m = CODE_RE.search(body) or CODE_RE.search(cur_url)
|
||||
if m and m.group(1) not in tried:
|
||||
tried.add(m.group(1))
|
||||
if _exchange_code(s, m.group(1)):
|
||||
return True
|
||||
hops = _next_hop_urls(body)
|
||||
if not hops:
|
||||
break
|
||||
nxt = urljoin(cur_url or STUDY, hops[0])
|
||||
if "code=" in nxt:
|
||||
code = (CODE_RE.search(nxt) or [None, ""])[1]
|
||||
if code and code not in tried and _exchange_code(s, code):
|
||||
return True
|
||||
try:
|
||||
r = s.get(nxt, timeout=40, allow_redirects=True)
|
||||
except Exception:
|
||||
break
|
||||
cur_url, body = r.url or "", r.text or ""
|
||||
return bool(_study_token(s))
|
||||
|
||||
|
||||
def _fetch_train_token(s):
|
||||
"""SSO 登录后,再用 gyx 会话换培训系统的 token(前端 `_51` 那个接口)。
|
||||
|
||||
返回体形如:
|
||||
{"result":{"url":"https://study.dgjy.net/training","token":"<jwt>"},"resultCode":200}
|
||||
这个 token 就是培训系统的 `Middle-Token`(前端把它写进 cookie 再发 Bearer)。
|
||||
"""
|
||||
try:
|
||||
r = s.get(STUDY + "/mod/education/gyx/train/front/login", timeout=40,
|
||||
headers={"Accept": "application/json, text/plain, */*"})
|
||||
j = r.json()
|
||||
except Exception:
|
||||
return False
|
||||
data = j.get("result") if isinstance(j, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
data = j.get("data") if isinstance(j, dict) else None
|
||||
tok = ""
|
||||
if isinstance(data, dict):
|
||||
for k in ("token", "accessToken", "middleToken"):
|
||||
if data.get(k):
|
||||
tok = str(data[k])
|
||||
break
|
||||
if not tok:
|
||||
return False
|
||||
s.cookies.set("Middle-Token", tok, domain=urlsplit(STUDY).hostname, path="/")
|
||||
return True
|
||||
|
||||
|
||||
def _exchange_code(s, code):
|
||||
"""code -> Middle-Token。前端是 POST /mod/gjt/login/<code>,再取 gyx 培训 token。"""
|
||||
paths = ("/mod/gjt/login/%s", "/train/mod/gjt/login/%s")
|
||||
for path in paths:
|
||||
try:
|
||||
rr = s.post(STUDY + (path % code), timeout=40,
|
||||
headers={"Accept": "application/json, text/plain, */*"})
|
||||
except Exception:
|
||||
continue
|
||||
if _study_token(s):
|
||||
return True
|
||||
try:
|
||||
j = rr.json()
|
||||
except Exception:
|
||||
j = {}
|
||||
ok = isinstance(j, dict) and str(j.get("resultCode")) == "200"
|
||||
if ok:
|
||||
# SSO 换票成功后,再换培训系统的 Bearer token(Middle-Token)
|
||||
if _fetch_train_token(s):
|
||||
return True
|
||||
data = j.get("result") or {}
|
||||
if isinstance(data, dict):
|
||||
for k in ("token", "accessToken", "middleToken"):
|
||||
if data.get(k):
|
||||
s.cookies.set("Middle-Token", str(data[k]),
|
||||
domain=urlsplit(STUDY).hostname, path="/")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def fetch_admin(s):
|
||||
"""GET /admin/main.action。
|
||||
|
||||
站点会话失效时**不返回 302**,而是 200 + 55 字节的 JS 跳转存根:
|
||||
<script>location.href="/admin/login.action";</script>
|
||||
旧代码直接把它当正常页面解析 -> 一个表格都没有 -> 静默返回空列表,
|
||||
于是「会话失效」「站点改版」「账号确实没有课」被压成同一条提示。
|
||||
"""
|
||||
r = s.get(JSPX + "/admin/main.action", timeout=40)
|
||||
html = r.text or ""
|
||||
if "/admin/login.action" in html and "table-results" not in html:
|
||||
# CASTGC 还有效的话,这里能无密码把 jspx 会话补回来
|
||||
if ensure_jspx_session(s):
|
||||
r = s.get(JSPX + "/admin/main.action", timeout=40)
|
||||
html = r.text or ""
|
||||
if "/admin/login.action" in html and "table-results" not in html:
|
||||
raise SessionExpired("登录已失效:站点把请求跳回了 /admin/login.action(请重新登录)")
|
||||
if "table-results" not in html:
|
||||
raise RuntimeError("课程页结构异常:未找到 table-results 表格(站点可能已改版,页面长度 %d)"
|
||||
% len(html))
|
||||
return html
|
||||
|
||||
|
||||
def _parse_course_rows(html, section=""):
|
||||
"""从「我的课程」列表片段里抽出课程行。
|
||||
|
||||
行结构(各板块一致):
|
||||
<table class="table-results"><tr><th>在学课程</th></tr>
|
||||
<tr><td><p class="course-tt"><a href="courseStudyMain.action?planId=..&projectId=..">名字</a></p>
|
||||
<p class="course-dd"><span>班期:..</span><span>分数:82</span><span>成绩:合格</span></p>
|
||||
</td><td><a class="abtn-orange-big">查看</a></td></tr>
|
||||
</table>
|
||||
|
||||
注意 href 是**不带前导斜杠的相对路径**(`courseStudyMain.action?...`),
|
||||
相对 `/train/trainee/` 解析,不能直接 `JSPX + href`。
|
||||
"""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
out = []
|
||||
for table in soup.select("table.table-results"):
|
||||
th = table.select_one("th")
|
||||
sec = section or (re.sub(r"\s+", " ", th.get_text(" ", strip=True)) if th else "")
|
||||
# 只按「有没有课程行」判断,不能用 "暂无" in table.get_text():
|
||||
# 表格里任何一个单元格出现「暂无」都会把整张表(含全部课程)丢掉。
|
||||
rows = [tr for tr in table.select("tr") if tr.select_one("p.course-tt a")]
|
||||
for tr in rows:
|
||||
a = tr.select_one("p.course-tt a")
|
||||
href = unquote(a.get("href") or "")
|
||||
plan, project = parse_ids(href)
|
||||
if not plan or not project:
|
||||
continue
|
||||
dd = tr.select_one("p.course-dd")
|
||||
out.append({
|
||||
"section": sec,
|
||||
"name": a.get_text(" ", strip=True),
|
||||
"info": re.sub(r"\s+", " ", dd.get_text(" ", strip=True)) if dd else "",
|
||||
"plan_id": plan,
|
||||
"project_id": project,
|
||||
"url": urljoin(JSPX + "/train/trainee/", href),
|
||||
"progress": "",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def scrape_courses(s, with_progress=True):
|
||||
"""把「我的课程」下三个板块 × 三种状态**全部**拉出来,含已学完的。
|
||||
|
||||
站点把列表拆成了 AJAX 接口,每个组合一次请求:
|
||||
|
||||
/train/trainee/studyingCourse.action?courseType=studying&planStatus=TAKING_PLAN
|
||||
/train/assist/assistanceCourse.action?courseType=assistance&planStatus=FINISHED_PLAN
|
||||
/train/inspect/inspectingCourse.action?courseType=inspecting&planStatus=NO_BEGAIN_PLAN
|
||||
...
|
||||
|
||||
只看 /admin/main.action 是看不到「已结束」课程的——那个首页只渲染在学/助学/督学
|
||||
三张「进行中」的概览表,课一学完就从首页消失了。
|
||||
|
||||
返回空列表只代表「三个板块九种状态都没课」;
|
||||
会话失效或页面改版一律抛异常,让调用方能给出准确提示。
|
||||
"""
|
||||
# 先用首页校验会话,避免九个接口各自失败还看不出来
|
||||
_jspx_call(s, lambda: fetch_admin(s))
|
||||
|
||||
out, seen = [], set()
|
||||
for section, path, ctype in COURSE_SECTIONS:
|
||||
for label, status in PLAN_STATUS:
|
||||
url = "%s%s?courseType=%s&planStatus=%s" % (JSPX, path, ctype, status)
|
||||
try:
|
||||
html = s.get(url, timeout=40).text
|
||||
except Exception:
|
||||
continue
|
||||
for c in _parse_course_rows(html, section):
|
||||
key = (c["plan_id"], c["project_id"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
c["course_type"] = ctype
|
||||
c["status"] = status
|
||||
c["status_label"] = label
|
||||
c["in_study"] = status in STUDYABLE_STATUS
|
||||
out.append(c)
|
||||
|
||||
# 进度只在「进行中/未开始」的课上查,已结束的动辄几十门,逐个请求太慢
|
||||
if with_progress:
|
||||
for c in out:
|
||||
if not c["in_study"]:
|
||||
continue
|
||||
try:
|
||||
rr = s.get(c["url"], timeout=30)
|
||||
p = BeautifulSoup(rr.text, "lxml").select_one("p.loading font")
|
||||
c["progress"] = p.get_text(strip=True) if p else ""
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def parse_jspx_course(html):
|
||||
"""老版 jspx 课程页(tier-tree)解析。"""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
course = soup.select_one("h1")
|
||||
prog = soup.select_one("p.loading font")
|
||||
info = soup.select_one("div.results-info p.info")
|
||||
|
||||
cw, fb, ts = [], [], []
|
||||
for one in soup.select("div.tier-tree-one"):
|
||||
ns = one.select_one("p.tier1.has-children span.name")
|
||||
chapter = re.sub(r"\s+", " ", ns.get_text(" ", strip=True)) if ns else "?"
|
||||
for p in one.select("p.tier3"):
|
||||
a = p.select_one("a")
|
||||
if not a:
|
||||
continue
|
||||
href = a.get("href") or ""
|
||||
txt = re.sub(r"\s+", " ", p.get_text(" ", strip=True))
|
||||
name = p.select_one("span.name")
|
||||
title = re.sub(r"\s+", " ", name.get_text(" ", strip=True)) if name else txt[:40]
|
||||
m = re.search(r"(\d+)分钟", txt)
|
||||
done = "已完成" in txt
|
||||
if "courseware.action" in href:
|
||||
cw.append({"chapter": chapter, "title": title, "href": href, "done": done,
|
||||
"duration": int(m.group(1)) if m else None})
|
||||
elif "doFeedback.action" in href:
|
||||
fb.append({"title": title, "href": href, "done": done})
|
||||
elif "doTest.action" in href:
|
||||
ts.append({"title": title, "href": href, "done": done})
|
||||
return {
|
||||
"user": whoami(html),
|
||||
"course": course.get_text(" ", strip=True) if course else "?",
|
||||
"progress": prog.get_text(strip=True) if prog else "?",
|
||||
"info": re.sub(r"\s+", " ", info.get_text(" ", strip=True)) if info else "?",
|
||||
"courseware": cw, "feedback": fb, "test": ts,
|
||||
}
|
||||
|
||||
|
||||
def discover(s, plan, project, name=""):
|
||||
"""打开课程页并解析;会话失效时先用 CASTGC 补一次 jspx 授权再重试。"""
|
||||
return _jspx_call(s, lambda: _discover_once(s, plan, project, name))
|
||||
|
||||
|
||||
def _discover_once(s, plan, project, name=""):
|
||||
url = "%s/train/trainee/courseStudyMain.action?planId=%s&projectId=%s&trainRole=0" % (JSPX, plan, project)
|
||||
r = s.get(url, timeout=40)
|
||||
html = r.text or ""
|
||||
if "/admin/login.action" in html and len(html) < 2000:
|
||||
# 课程页也走同一个判断:CASTGC 有效就无密码续期
|
||||
if ensure_jspx_session(s):
|
||||
r = s.get(url, timeout=40)
|
||||
html = r.text or ""
|
||||
if "没有权限" in html:
|
||||
raise RuntimeError("该账号对此课程没有权限")
|
||||
if "/admin/login.action" in html and len(html) < 2000:
|
||||
raise SessionExpired("登录已失效:课程页把请求跳回了 /admin/login.action(请重新登录)")
|
||||
|
||||
# 课程页 302 到 study 时,token 在跳转 URL 里;哪怕 SPA 外壳返回空也要当成学过的新平台
|
||||
study_url = "study.dgjy.net" in (r.url or "")
|
||||
tok = extract_study_token(r.url, r.history)
|
||||
if tok:
|
||||
host = urlsplit(r.url).hostname or urlsplit(STUDY).hostname
|
||||
try:
|
||||
s.cookies.set("Middle-Token", tok, domain=host, path="/")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if len(html) < 3000 and not (study_url or tok or has_study_marker(html, r.url)):
|
||||
raise RuntimeError("课程页返回异常(页面长度仅 %d,登录可能已失效)" % len(html))
|
||||
|
||||
generation = detect_generation(html, r.url)
|
||||
# 按登录 / 学习入口识别平台;以后新增平台时这里不用改,靠 Platform.entries 命中
|
||||
pf = registry.detect(r.url, html)
|
||||
platform_id = pf.id if pf else "guanyixue"
|
||||
if generation == "legacy":
|
||||
d = parse_jspx_course(html)
|
||||
if d["courseware"] or d["feedback"] or d["test"]:
|
||||
d["platform"] = platform_id
|
||||
d["generation"] = "legacy"
|
||||
return d
|
||||
# 老结构但一个活动都没有:多半是「慕课」外链页(页面还在 jspx,课时在 study 平台)
|
||||
if not has_study_marker(html, r.url):
|
||||
snap = save_snapshot(plan, project, r.url, html)
|
||||
raise RuntimeError("课程页能打开,但没有任何课件/反馈表/测验(页面已存 %s)。"
|
||||
"若是慕课课程,请确认新平台登录态是否有效" % snap)
|
||||
elif generation != "spa":
|
||||
snap = save_snapshot(plan, project, r.url, html)
|
||||
raise RuntimeError("无法识别课程页结构(站点可能又改版):页面里既没有 jspx 的 "
|
||||
"tier-tree,也没有新版 SPA 标记。页面已存 %s,请把它发我" % snap)
|
||||
|
||||
# ---- 新版平台(Vue SPA + /train/course/front/* JSON 接口)----
|
||||
tid = extract_train_id(r.url, html)
|
||||
user = whoami(html)
|
||||
client = StudyClient(s, r.url)
|
||||
|
||||
def _load():
|
||||
nonlocal tid
|
||||
if not tid:
|
||||
tid = study_resolve_train_id(client, name)
|
||||
if not tid:
|
||||
snap = save_snapshot(plan, project, r.url, html)
|
||||
raise RuntimeError("新版课程页未带 trainCourseId,且按课程名也匹配不到(课程名:%s)。"
|
||||
"页面已存 %s" % (name or "?", snap))
|
||||
return discover_study(client, tid, name, user)
|
||||
|
||||
try:
|
||||
d = _load()
|
||||
except SessionExpired:
|
||||
client.bootstrap()
|
||||
tid = extract_train_id(r.url, html)
|
||||
d = _load()
|
||||
d["platform"] = platform_id
|
||||
d["generation"] = "spa"
|
||||
# URL 里可能带 token(凭据),日志/状态里一律打码;鉴权已经写进 cookie
|
||||
d["page_url"] = re.sub(r"([?&])(token|access_token|securityKey|sign)=[^&]*", r"\1\2=***", r.url or "")
|
||||
d["plan_id"] = plan
|
||||
d["project_id"] = project
|
||||
return d
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 平台注册
|
||||
|
||||
def _study(key, s, d, **opts):
|
||||
"""旧版页面的学习流程仍在 engine._study_builtin(反馈表 / 测验 / 刷课件)。"""
|
||||
from study_tool.engine import _study_builtin
|
||||
return _study_builtin(key, s, d, **opts)
|
||||
@@ -0,0 +1,26 @@
|
||||
"""平台注册表:引擎、路由都只通过这里拿平台实现。"""
|
||||
from study_tool.platforms.base import Platform
|
||||
|
||||
_PLATFORMS = {}
|
||||
|
||||
|
||||
def register(platform: Platform) -> Platform:
|
||||
_PLATFORMS[platform.id] = platform
|
||||
return platform
|
||||
|
||||
|
||||
def get(pid):
|
||||
"""按 course["platform"] 取平台实现。"""
|
||||
return _PLATFORMS.get(pid or "")
|
||||
|
||||
|
||||
def detect(url="", html=""):
|
||||
"""按登录入口 / 学习入口识别平台;识别不到返回 None。"""
|
||||
for p in all():
|
||||
if p.matches(url, html):
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def all():
|
||||
return sorted(_PLATFORMS.values(), key=lambda p: p.id)
|
||||
@@ -0,0 +1,688 @@
|
||||
"""新版莞易学课程页(study.dgjy.net):StudyClient、课程结构、学习动作。"""
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from study_tool import store
|
||||
from study_tool.config import *
|
||||
from study_tool.platforms.jspx import *
|
||||
from study_tool.runtime import log
|
||||
|
||||
# ------------------------------------------------------- 新版平台:页面识别/提取
|
||||
|
||||
|
||||
def extract_train_id(page_url, html):
|
||||
"""从跳转后的 URL / 页面内链接 / 隐藏域里找新版培训课程 ID(32 位 hex)。"""
|
||||
q = parse_qs(urlsplit(page_url or "").query)
|
||||
for k in ("trainCourseId", "courseId", "id"):
|
||||
for v in q.get(k, []):
|
||||
if HEX32.match(v or ""):
|
||||
return v
|
||||
|
||||
# 页面里指向 study 的链接 / iframe / JS 跳转
|
||||
for m in re.finditer(r"(?:https?:)?//[^\s\"'<>]*study\.dgjy\.net/[^\s\"'<>]+", html or ""):
|
||||
u = m.group(0)
|
||||
tail = u.split("#", 1)[1] if "#" in u else u
|
||||
qq = parse_qs(tail.split("?", 1)[1]) if "?" in tail else {}
|
||||
for k in ("trainCourseId", "courseId", "id"):
|
||||
for v in qq.get(k, []):
|
||||
if HEX32.match(v or ""):
|
||||
return v
|
||||
# 相对路径 /training/curriculum?id=...
|
||||
for m in re.finditer(r"/training/(?:curriculum|courseVideo|comment)\?([^\s\"'<>]+)", html or ""):
|
||||
qq = parse_qs(m.group(1).replace("&", "&"))
|
||||
for k in ("trainCourseId", "courseId", "id"):
|
||||
for v in qq.get(k, []):
|
||||
if HEX32.match(v or ""):
|
||||
return v
|
||||
# 隐藏域
|
||||
try:
|
||||
soup = BeautifulSoup(html or "", "lxml")
|
||||
for i in soup.select("input[type=hidden]"):
|
||||
nm = (i.get("name") or "").lower()
|
||||
val = (i.get("value") or "").strip()
|
||||
if HEX32.match(val) and ("course" in nm or nm in ("id", "trainid")):
|
||||
return val
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
def extract_study_token(page_url, history=None):
|
||||
"""课程页跳转到 study 时,token 直接挂在 URL 上(前端读它写 Middle-Token cookie)。
|
||||
|
||||
courseStudyMain.action -> 302 study/openapi/train/front/getById?...
|
||||
-> 302 study/training/trainDetails?id=..&token=<jwt>
|
||||
"""
|
||||
urls = [page_url or ""]
|
||||
for h in history or []:
|
||||
try:
|
||||
urls.append(h.headers.get("Location") or "")
|
||||
except Exception:
|
||||
pass
|
||||
for u in urls:
|
||||
if not u:
|
||||
continue
|
||||
tail = u.split("#", 1)[1] if "#" in u else u
|
||||
qq = parse_qs(tail.split("?", 1)[1]) if "?" in tail else {}
|
||||
for k in ("token", "access_token", "Middle-Token"):
|
||||
v = qq.get(k)
|
||||
if v and v[0]:
|
||||
return v[0]
|
||||
return ""
|
||||
|
||||
|
||||
def save_snapshot(plan, project, url, html):
|
||||
"""存下原始页面(进 SQLite),方便站点再次改版时定位选择器。
|
||||
|
||||
返回给用户的是取回地址 /api/snapshot/<id>;写库失败才退回文件。
|
||||
"""
|
||||
safe_url = re.sub(r"([?&])(token|access_token|securityKey|sign)=[^&]*", r"\1\2=***", url or "")
|
||||
try:
|
||||
sid = store.snapshot_add(plan, project, safe_url, html or "")
|
||||
return "/api/snapshot/%d" % sid
|
||||
except Exception:
|
||||
pass
|
||||
name = "course_%s_%s_%s.html" % (re.sub(r"[^\w.-]", "_", str(plan))[:20],
|
||||
re.sub(r"[^\w.-]", "_", str(project))[:20],
|
||||
time.strftime("%H%M%S"))
|
||||
path = os.path.join(DEBUG_DIR, name)
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("<!-- final url: %s -->\n" % safe_url)
|
||||
f.write(html or "")
|
||||
return path
|
||||
except Exception:
|
||||
return "(保存失败)"
|
||||
|
||||
|
||||
# ------------------------------------------------------- 新版平台:HTTP 客户端
|
||||
|
||||
def _study_token(s):
|
||||
for c in s.cookies:
|
||||
if c.name == "Middle-Token" and "dgjy" in (c.domain or ""):
|
||||
return c.value
|
||||
return ""
|
||||
|
||||
|
||||
class StudyClient:
|
||||
"""新版平台接口客户端。
|
||||
|
||||
API 前缀不写死:课程页可能由 jspx 代理、也可能跳到 study.dgjy.net,
|
||||
所以按「页面 origin/train」→「study.dgjy.net/train」→「gjt/tp_edu_biz/train」
|
||||
依次试,哪个能返回 200 就用哪个(结果缓存)。
|
||||
"""
|
||||
|
||||
def __init__(self, session, page_url=""):
|
||||
self.s = session
|
||||
self.token = _study_token(session)
|
||||
self.bases = []
|
||||
origin = ""
|
||||
if page_url:
|
||||
u = urlsplit(page_url)
|
||||
if u.scheme and u.netloc:
|
||||
origin = "%s://%s" % (u.scheme, u.netloc)
|
||||
for b in ([origin + "/train"] if origin else []) + [STUDY + "/train"]:
|
||||
if b not in self.bases:
|
||||
self.bases.append(b)
|
||||
if origin and "gjt.dgjy.net" in origin:
|
||||
for b in (origin + "/tp_edu_biz/train",):
|
||||
if b not in self.bases:
|
||||
self.bases.append(b)
|
||||
self.base = None
|
||||
self.last_errors = []
|
||||
|
||||
def _raw(self, method, base, path, params=None, json_body=None):
|
||||
h = {"Accept": "application/json, text/plain, */*",
|
||||
"Referer": STUDY + "/training/curriculum"}
|
||||
if self.token:
|
||||
h["Authorization"] = "Bearer " + self.token
|
||||
r = self.s.request(method, base + path, params=params, json=json_body,
|
||||
headers=h, timeout=40)
|
||||
try:
|
||||
return r.json()
|
||||
except Exception:
|
||||
# 不是 JSON:多半是打到了错误前缀(返回登录页/HTML),标 -1 让 call() 换下一个 base
|
||||
return {"code": -1 if r.status_code < 400 else r.status_code,
|
||||
"msg": "非 JSON 响应:%s" % (r.text or "")[:120]}
|
||||
|
||||
def call(self, method, path, params=None, json_body=None):
|
||||
bases = ([self.base] if self.base else []) + [b for b in self.bases if b != self.base]
|
||||
self.last_errors = []
|
||||
saw_auth = False
|
||||
for b in bases:
|
||||
try:
|
||||
j = self._raw(method, b, path, params, json_body)
|
||||
except Exception as e:
|
||||
self.last_errors.append("%s -> %s" % (b, e))
|
||||
continue
|
||||
code = j.get("code")
|
||||
if code in (401, 403) or code == 404 or code == -1:
|
||||
saw_auth = saw_auth or code in (401, 403)
|
||||
self.last_errors.append("%s -> %s" % (b, j.get("msg") or ("业务码 %s" % code)))
|
||||
if self.base == b:
|
||||
self.base = None
|
||||
continue
|
||||
self.base = b
|
||||
return j
|
||||
if saw_auth and not self.base:
|
||||
raise SessionExpired("新版平台没有登录态(Middle-Token 缺失或已过期):请点「重新登录」。%s"
|
||||
% "; ".join(self.last_errors)[:180])
|
||||
raise RuntimeError("新版平台接口不可用:%s" % ("; ".join(self.last_errors)[:300] or "无可用地址"))
|
||||
|
||||
def bootstrap(self):
|
||||
"""用现有 SSO 票据强制换一张新的 study.dgjy.net token(旧的可能已过期)。"""
|
||||
try:
|
||||
ensure_study_session(self.s, force=True)
|
||||
except Exception:
|
||||
pass
|
||||
self.token = _study_token(self.s)
|
||||
|
||||
|
||||
# ------------------------------------------------------- 新版平台:课程结构
|
||||
|
||||
def _flatten_periods(nodes, chapter="", out=None):
|
||||
out = [] if out is None else out
|
||||
for n in nodes or []:
|
||||
if not isinstance(n, dict):
|
||||
continue
|
||||
nm = re.sub(r"\s+", " ", n.get("name") or "").strip()
|
||||
row = dict(n)
|
||||
row["_chapter"] = chapter
|
||||
out.append(row)
|
||||
if isinstance(n.get("children"), list) and n["children"]:
|
||||
_flatten_periods(n["children"], nm or chapter, out)
|
||||
return out
|
||||
|
||||
|
||||
def _video_duration(n):
|
||||
f = n.get("file") or {}
|
||||
for v in (f.get("videoDuration"), f.get("duration"), f.get("fileDuration"),
|
||||
n.get("duration"), n.get("videoDuration")):
|
||||
try:
|
||||
iv = int(float(v))
|
||||
if iv > 0:
|
||||
return iv
|
||||
except Exception:
|
||||
continue
|
||||
return 0
|
||||
|
||||
|
||||
def _period_done(n):
|
||||
sl = n.get("studyLog") or {}
|
||||
if sl.get("completeTime"):
|
||||
return True
|
||||
try:
|
||||
return float(sl.get("progress") or 0) >= 1
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _int_or_none(v):
|
||||
"""平台把次数偶尔给成字符串;取不到就返回 None(页面按「未知」处理)。"""
|
||||
try:
|
||||
return int(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# 视频文件类型 -> 播放器通道(来自课程播放页 chunk-3d0ebfea 的 judgeVideoType)
|
||||
CLOUD_TYPES = {"cloud_video", "middle"} # polyv 云视频:/period/middle/play
|
||||
MOOC_TYPES = {"mooc_video"} # 网易云课堂:走签名/moc 播放
|
||||
OTHER_TYPES = {"other"} # 普通 mp4:原生 <video>,ended 即完成
|
||||
|
||||
|
||||
def _norm_name(s):
|
||||
return re.sub(r"[\s()()【】\[\]·、,,::\-—_/]+", "", s or "").lower()
|
||||
|
||||
|
||||
def _walk_json(o):
|
||||
if isinstance(o, dict):
|
||||
yield o
|
||||
for v in o.values():
|
||||
yield from _walk_json(v)
|
||||
elif isinstance(o, list):
|
||||
for v in o:
|
||||
yield from _walk_json(v)
|
||||
|
||||
|
||||
def study_resolve_train_id(client, name):
|
||||
"""课程页没带 ID 时,用「我的课程」/课程列表按名字匹配。"""
|
||||
target = _norm_name(name)
|
||||
if not target:
|
||||
return ""
|
||||
cands = []
|
||||
sources = [
|
||||
("GET", "/course/front/study/studyCenter/myCourse/list/0",
|
||||
{"pageNum": 1, "pageSize": 200}),
|
||||
("GET", "/course/front/study/studyCenter/myCourse/list/1",
|
||||
{"pageNum": 1, "pageSize": 200}),
|
||||
("GET", "/course/front/trainCourse/list",
|
||||
{"pageNum": 1, "pageSize": 100, "courseName": name}),
|
||||
]
|
||||
for method, path, params in sources:
|
||||
try:
|
||||
j = client.call(method, path, params=params)
|
||||
except SessionExpired:
|
||||
raise
|
||||
except Exception:
|
||||
continue
|
||||
for rec in _walk_json(j.get("data")):
|
||||
nm = ""
|
||||
for k in ("courseName", "trainCourseName", "name", "title"):
|
||||
if isinstance(rec.get(k), str) and rec.get(k).strip():
|
||||
nm = rec[k]
|
||||
break
|
||||
if not nm:
|
||||
for k, v in rec.items():
|
||||
if k.lower().endswith("name") and isinstance(v, str) and v.strip():
|
||||
nm = v
|
||||
break
|
||||
cid = ""
|
||||
for k in ("trainCourseId", "courseId", "id"):
|
||||
v = rec.get(k)
|
||||
if isinstance(v, str) and HEX32.match(v):
|
||||
cid = v
|
||||
break
|
||||
if not cid:
|
||||
for k, v in rec.items():
|
||||
if k.lower().endswith("id") and isinstance(v, str) and HEX32.match(v):
|
||||
cid = v
|
||||
break
|
||||
if not nm or not cid:
|
||||
continue
|
||||
a = _norm_name(nm)
|
||||
if a == target:
|
||||
return cid
|
||||
if target in a or a in target:
|
||||
cands.append((cid, nm))
|
||||
if cands:
|
||||
return cands[0][0]
|
||||
return ""
|
||||
|
||||
|
||||
def discover_study(client, train_id, name="", user=""):
|
||||
info = client.call("GET", "/course/front/trainCourse/getById",
|
||||
params={"id": train_id}).get("data") or {}
|
||||
tree = client.call("GET", "/course/front/period/list/tree/studyLog",
|
||||
params={"courseId": train_id, "isFront": 1}).get("data") or []
|
||||
if not info and not tree:
|
||||
raise RuntimeError("新版平台读不到课程结构(trainCourseId=%s)" % train_id)
|
||||
|
||||
courseware, feedback, test, containers = [], [], [], []
|
||||
mooc_nodes, seen = [], set()
|
||||
|
||||
def add_video(n, chapter):
|
||||
pid = str(n.get("id") or "")
|
||||
if not pid or pid in seen:
|
||||
return
|
||||
seen.add(pid)
|
||||
f = n.get("file") or {}
|
||||
ftype = str(f.get("fileType") or "")
|
||||
if ftype in CLOUD_TYPES:
|
||||
kind = "polyv"
|
||||
elif ftype in MOOC_TYPES:
|
||||
kind = "mooc"
|
||||
else:
|
||||
kind = "direct"
|
||||
courseware.append({
|
||||
"chapter": chapter or "课程",
|
||||
"title": re.sub(r"\s+", " ", n.get("name") or "") or ("课时 %s" % pid),
|
||||
"href": "",
|
||||
"done": _period_done(n),
|
||||
"duration": _video_duration(n),
|
||||
"period_id": pid,
|
||||
"source": "study",
|
||||
"file_type": ftype,
|
||||
"kind": kind,
|
||||
"from_type": str(f.get("fromType") or ""),
|
||||
"allow_drag": str(n.get("allowVideoDrag") or ""),
|
||||
})
|
||||
|
||||
for n in _flatten_periods(tree):
|
||||
pt = str(n.get("periodType") or "")
|
||||
nm = re.sub(r"\s+", " ", n.get("name") or "").strip()
|
||||
chapter = n.get("_chapter") or ""
|
||||
if pt == "2":
|
||||
add_video(n, chapter)
|
||||
elif pt == "17":
|
||||
feedback.append({
|
||||
"title": nm or "培训反馈",
|
||||
"href": "",
|
||||
"done": _period_done(n),
|
||||
"period_id": str(n.get("id") or ""),
|
||||
"evaluation_id": n.get("evaluationId"),
|
||||
})
|
||||
elif pt in ("5", "6"):
|
||||
tp = n.get("testpaper") or {}
|
||||
status = str(tp.get("resultStatus") or "")
|
||||
done = status in ("FINISHED", "REVIEWING", "REVIEWED", "PASS", "UNPASS") \
|
||||
or _period_done(n)
|
||||
test.append({
|
||||
"title": nm or "测验",
|
||||
"href": "",
|
||||
"done": done,
|
||||
"period_id": str(n.get("id") or ""),
|
||||
"testpaper_id": tp.get("testpaperId"),
|
||||
"testpaper_result_id": tp.get("testpaperResultId"),
|
||||
"score": (n.get("studyLog") or {}).get("logPeriodScore"),
|
||||
"passed_score": tp.get("passedScore"),
|
||||
"result_status": status,
|
||||
# 平台的考试次数:allow=总次数,surplus=还剩几次;重考要先看它
|
||||
"allow_exam_count": _int_or_none(n.get("allowExamCount")),
|
||||
"surplus_exam_count": _int_or_none(n.get("surplusExamCount")),
|
||||
# 交卷后是否允许看正确答案/解析(每个测验单独配置)
|
||||
"exam_after_watch_answer": str(n.get("examAfterWatchAnswer") or ""),
|
||||
"exam_after_watch_question": str(n.get("examAfterWatchQuestion") or ""),
|
||||
"complete_limit": str(n.get("completeLimit") or ""),
|
||||
})
|
||||
elif pt in ("10", "13") and n.get("kcdm"):
|
||||
# 10/13 都是「莞易学课程」容器节点,真正的视频课时挂在 kcdm 指向的 MOOC 课程下
|
||||
mooc_nodes.append((nm or chapter, str(n["kcdm"])))
|
||||
containers.append({"period_id": str(n.get("id") or ""), "name": nm or chapter,
|
||||
"kcdm": str(n["kcdm"]), "done": _period_done(n)})
|
||||
|
||||
# 莞易学课程节点(periodType=10):真正的视频课时挂在 kcdm 指向的 MOOC 课程下
|
||||
for mooc_name, mooc_id in mooc_nodes:
|
||||
try:
|
||||
sub = client.call("GET", "/course/front/period/list/tree/studyLog",
|
||||
params={"courseId": mooc_id, "isFront": 1}).get("data") or []
|
||||
except Exception:
|
||||
continue
|
||||
for v in _flatten_periods(sub, mooc_name):
|
||||
if str(v.get("periodType") or "") == "2":
|
||||
add_video(v, mooc_name)
|
||||
|
||||
st = info.get("study") or {}
|
||||
if st.get("periodTotal"):
|
||||
progress = "%s/%s 课时" % (st.get("periodStudy") or 0, st.get("periodTotal"))
|
||||
else:
|
||||
progress = "%d/%d 课时" % (len([c for c in courseware if c["done"]]), len(courseware))
|
||||
|
||||
bits = []
|
||||
if info.get("id"):
|
||||
bits.append("trainCourseId=%s" % info["id"])
|
||||
if info.get("isJoin") is not None:
|
||||
bits.append("已加入=%s" % ("是" if str(info.get("isJoin")) == "1" else "否"))
|
||||
if info.get("needJoin") is not None:
|
||||
bits.append("需加入=%s" % ("是" if str(info.get("needJoin")) == "1" else "否"))
|
||||
if info.get("completeRule") is not None:
|
||||
bits.append("完成规则=%s" % info.get("completeRule"))
|
||||
|
||||
return {
|
||||
"user": user,
|
||||
"course": info.get("courseName") or name or "?",
|
||||
"progress": progress,
|
||||
"info": " | ".join(bits) or "?",
|
||||
"courseware": courseware,
|
||||
"feedback": feedback,
|
||||
"test": test,
|
||||
"containers": containers,
|
||||
"train_id": train_id,
|
||||
# 未加入培训时 study/log 会被静默丢弃,调用方要先补 join
|
||||
"is_join": str(info.get("isJoin")) == "1",
|
||||
"need_join": str(info.get("needJoin")) == "1",
|
||||
# 浏览器「刷新课程页就能加入」靠的就是它:/mod/school/wanpeng/syncSelf?courseCode=<courseCode>
|
||||
"course_code": str(info.get("courseCode") or ""),
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------- 新版平台:动作
|
||||
|
||||
def study_period_log(client, period_id, done, seconds=0):
|
||||
"""播放器两个包:播放中 {isComplete:0, lastProgressTime},播完 {isComplete:1}(不带时长)。"""
|
||||
body = {"periodId": int(period_id), "isComplete": 1 if done else 0}
|
||||
if not done:
|
||||
body["lastProgressTime"] = str(int(seconds or 0))
|
||||
return client.call("POST", "/course/front/period/study/log", json_body=body)
|
||||
|
||||
|
||||
def study_log_detail(client, period_id, stime, etime):
|
||||
"""学时台账(播放器暂停/离开时发):stime/etime 是这次学习的起止时间。"""
|
||||
fmt = "%Y-%m-%d %H:%M:%S"
|
||||
return client.call("POST", "/course/front/study/log/detail/save",
|
||||
json_body={"stime": time.strftime(fmt, time.localtime(stime)),
|
||||
"etime": time.strftime(fmt, time.localtime(etime)),
|
||||
"periodId": int(period_id)})
|
||||
|
||||
|
||||
def study_is_done(client, period_id):
|
||||
j = client.call("GET", "/course/front/period/getById", params={"periodId": int(period_id)})
|
||||
sl = (j.get("data") or {}).get("studyLog") or {}
|
||||
if sl.get("completeTime"):
|
||||
return True
|
||||
try:
|
||||
return float(sl.get("progress") or 0) >= 1
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def study_cal_final_score(client, course_id):
|
||||
"""按学习日志重算课程的完成计数(PUT)。
|
||||
|
||||
并发上报 period/study/log 会丢增量:实测 14 个课时 completeTime 全有,但课程上的
|
||||
periodStudy 只记到 11,培训那边的「课程」节点就永远不完成。这个接口能把计数补齐。
|
||||
"""
|
||||
return client.call("PUT", "/course/front/calFinalScore?courseId=%s" % course_id)
|
||||
|
||||
|
||||
def study_complete_container(client, ct):
|
||||
"""培训里的「课程」是 periodType 10/13 的容器节点:子课时学完它不会自动完成。
|
||||
|
||||
前端从不给这种节点发 study/log(播视频只动子课时),但接口本身没按节点类型拦,
|
||||
所以这里补一发 isComplete=1,写完回读 completeTime 确认真生效。
|
||||
"""
|
||||
pid = ct["period_id"]
|
||||
try:
|
||||
client.call("POST", "/course/front/period/study/log",
|
||||
json_body={"periodId": int(pid), "isComplete": 1})
|
||||
except Exception as e:
|
||||
return False, str(e)[:100]
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
j = client.call("GET", "/course/front/period/getById", params={"periodId": int(pid)})
|
||||
sl = (j.get("data") or {}).get("studyLog") or {}
|
||||
if sl.get("completeTime"):
|
||||
return True, "已标记完成(%s)" % sl["completeTime"]
|
||||
return False, "接口回 200 但 completeTime 仍为空"
|
||||
except Exception as e:
|
||||
return False, "回读失败:%s" % str(e)[:80]
|
||||
|
||||
|
||||
def study_refresh_page(s, d):
|
||||
"""照浏览器的做法把课程页重新走一遍(jspx 课程页 → 带新 token 的 trainDetails)。
|
||||
|
||||
新版平台的 isJoin 有时候要等页面真正被打开一次才落库:直接调 student/join 被拒
|
||||
(「该课程无法直接加入,请联系管理员」),但刷新一次页面它就变 1 了。
|
||||
所以加入失败后补一次页面访问,再复查。
|
||||
"""
|
||||
plan, project = d.get("plan_id") or "", d.get("project_id") or ""
|
||||
if not (plan and project):
|
||||
return ""
|
||||
url = "%s/train/trainee/courseStudyMain.action?planId=%s&projectId=%s&trainRole=0" % (
|
||||
JSPX, plan, project)
|
||||
try:
|
||||
r = s.get(url, timeout=40, allow_redirects=True)
|
||||
return r.url or ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def study_mod_get(client, path, params=None):
|
||||
"""调 /mod/ 这组接口(不在 /train 下,也不带 Bearer,只认 cookie 会话)。
|
||||
|
||||
浏览器课程页刷新时发的就是这组:/mod/student/currentInfo、/mod/school/wanpeng/syncSelf、
|
||||
/mod/gjt/roleList。用 StudyClient 的会话(cookie)直接 GET 就能通,
|
||||
而 /train/* 必须额外带 Authorization: Bearer <Middle-Token>。
|
||||
"""
|
||||
h = {"Accept": "application/json, text/plain, */*",
|
||||
"Referer": STUDY + "/training/trainDetails"}
|
||||
r = client.s.get(STUDY + path, params=params, headers=h, timeout=40)
|
||||
try:
|
||||
return r.json()
|
||||
except Exception:
|
||||
return {"resultCode": r.status_code, "resultMsg": (r.text or "")[:200]}
|
||||
|
||||
|
||||
def study_sync_self(client, course_code):
|
||||
"""把当前账号同步进课程 —— 浏览器刷新课程页时真正「加入培训」的那一步。
|
||||
|
||||
抓包实录(常用公文写作):
|
||||
getById -> isJoin=0 needJoin=1
|
||||
GET /mod/school/wanpeng/syncSelf?courseCode=<code> (耗时约 2s)
|
||||
getById -> isJoin=1
|
||||
前端在 isJoin != 1 时就是发这一发把学员同步进课程;
|
||||
isFreeJoin=0 的慕课课不走 student/join,只能靠它落库。
|
||||
返回 (ok, msg)。
|
||||
"""
|
||||
j = study_mod_get(client, "/mod/school/wanpeng/syncSelf", {"courseCode": course_code})
|
||||
res = j.get("result") or {}
|
||||
code = str(j.get("resultCode") or res.get("code") or "")
|
||||
return code == "200", str(res.get("msg") or j.get("resultMsg") or code or "?")
|
||||
|
||||
|
||||
def study_sync_identity(client):
|
||||
"""页面上紧随 syncSelf 的身份拉取(/mod/gjt/roleList),个别课程同步后要靠它才认人。
|
||||
|
||||
返回 True/False,失败无所谓,只当补一刀。
|
||||
"""
|
||||
try:
|
||||
j = study_mod_get(client, "/mod/gjt/roleList")
|
||||
return bool(j.get("result")) or str(j.get("resultCode") or "") == "200"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def ensure_study_join(s, client, d):
|
||||
"""确保这门培训已加入;返回 (ok, 说明)。
|
||||
|
||||
顺序(照浏览器课程页的真实行为):
|
||||
当前状态 → student/join → 被拒就 syncSelf(courseCode) 轮询复查
|
||||
→ 再不行就刷新一次 jspx 课程页复查。
|
||||
"""
|
||||
tid = d.get("train_id") or ""
|
||||
code = d.get("course_code") or ""
|
||||
|
||||
def joined():
|
||||
j = client.call("GET", "/course/front/trainCourse/getById", params={"id": tid})
|
||||
return str((j.get("data") or {}).get("isJoin")) == "1"
|
||||
|
||||
if d.get("is_join") or joined():
|
||||
return True, "已加入"
|
||||
|
||||
ok, msg = study_join_course(client, tid)
|
||||
if ok:
|
||||
return True, "加入成功"
|
||||
|
||||
# 浏览器刷新课程页走的就是这条:syncSelf 落库,再重新拉 getById
|
||||
if code:
|
||||
sync_ok, sync_msg = study_sync_self(client, code)
|
||||
for i in range(8):
|
||||
time.sleep(0.6 if i == 0 else 1.5)
|
||||
try:
|
||||
if joined():
|
||||
return True, "同步课程页后已加入(student/join 被拒:%s;syncSelf=%s)" % (
|
||||
msg or "-", sync_msg or "-")
|
||||
except SessionExpired:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
if i == 2:
|
||||
# 页面在 syncSelf 之后还会拉一次身份,个别课程靠它才认人
|
||||
study_sync_identity(client)
|
||||
note = "syncSelf:%s" % ("成功" if sync_ok else "失败(%s)" % (sync_msg or "-"))
|
||||
else:
|
||||
note = "课程没有 courseCode,无法走同步接口"
|
||||
|
||||
study_refresh_page(s, d)
|
||||
try:
|
||||
if joined():
|
||||
return True, "刷新课程页后已加入(student/join 被拒:%s;%s)" % (msg or "-", note)
|
||||
except SessionExpired:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
return False, "%s;%s" % (msg or "未加入", note)
|
||||
|
||||
|
||||
def study_join_course(client, train_id):
|
||||
"""加入培训(POST /course/front/student/join?courseId=...)。
|
||||
|
||||
课程 needJoin=1 且 isJoin=0 时必须先加入,否则 study/log 照样回 200,
|
||||
但学习记录不会计数 —— 现象就是「每个课时都报成功、复查全是未完成」。
|
||||
isFreeJoin=0 的课程服务端会直接拒:「该课程无法直接加入,请联系管理员」。
|
||||
慕课课(isFreeJoin=0)别指望这个接口,要改用 study_sync_self —— 见 ensure_study_join。
|
||||
返回 (ok, msg)。
|
||||
"""
|
||||
j = client.call("POST", "/course/front/student/join", params={"courseId": train_id})
|
||||
return str(j.get("code")) == "200", str(j.get("msg") or j.get("message") or "")
|
||||
|
||||
|
||||
def study_complete_period(cookies, page_url, item):
|
||||
"""把一个视频课时报到 complete=1,并以服务端 completeTime 为准。"""
|
||||
s = new_session(cookies)
|
||||
c = StudyClient(s, page_url)
|
||||
pid, dur = item["period_id"], int(item.get("duration") or 0)
|
||||
last = ""
|
||||
for attempt in range(1, 4):
|
||||
try:
|
||||
study_period_log(c, pid, False, 0)
|
||||
time.sleep(0.35)
|
||||
now = time.time()
|
||||
try: # 学时台账:播放器暂停/离开时都会发,缺它学时可能不记
|
||||
study_log_detail(c, pid, now - max(60, dur), now)
|
||||
except Exception:
|
||||
pass
|
||||
j = study_period_log(c, pid, True, dur)
|
||||
last = "code=%s %s" % (j.get("code"), str(j.get("msg") or "")[:60])
|
||||
except SessionExpired:
|
||||
raise
|
||||
except Exception as e:
|
||||
last = str(e)[:100]
|
||||
time.sleep(1.2)
|
||||
try:
|
||||
if study_is_done(c, pid):
|
||||
return True, "ok"
|
||||
except Exception as e:
|
||||
last = str(e)[:100]
|
||||
time.sleep(1.0)
|
||||
return False, last
|
||||
|
||||
|
||||
def study_submit_feedback(client, fb, key, train_id):
|
||||
eid = fb.get("evaluation_id")
|
||||
if not eid:
|
||||
log(key, "反馈表:该节点没有 evaluationId,跳过")
|
||||
return
|
||||
j = client.call("GET", "/course/front/evaluation/result/getDetail",
|
||||
params={"evaluationId": eid})
|
||||
data = j.get("data") or {}
|
||||
if data.get("resultId"):
|
||||
log(key, "反馈表:已提交过(resultId=%s),跳过" % data["resultId"])
|
||||
return
|
||||
items = data.get("items") or []
|
||||
if not items:
|
||||
log(key, "反馈表:没有题目(evaluationId=%s),跳过" % eid)
|
||||
return
|
||||
score_flag = str(data.get("scoreFlag")) != "0"
|
||||
stars = data.get("starCount") or 10
|
||||
answers = [{"itemId": it.get("itemId"),
|
||||
"answer": stars if score_flag else ""} for it in items]
|
||||
payload = {"periodId": int(fb["period_id"]), "evaluationId": int(eid),
|
||||
"items": answers, "trainCourseId": train_id}
|
||||
r = client.call("POST", "/course/front/evaluation/result", json_body=payload)
|
||||
if r.get("code") == 200:
|
||||
log(key, "反馈表:%d 题,提交成功(%s分)" % (len(answers), stars))
|
||||
else:
|
||||
log(key, "反馈表:提交失败 code=%s %s" % (r.get("code"), str(r.get("msg") or "")[:80]))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 学习实现
|
||||
|
||||
def _study(key, s, d, **opts):
|
||||
from study_tool.engine import run_study_course
|
||||
return run_study_course(key, s, d, opts.get("workers", 1), opts.get("cap", 30),
|
||||
opts.get("enable_quiz", False), opts.get("enable_feedback", False),
|
||||
opts.get("mode", "fast"), opts.get("parallel", 2))
|
||||
Reference in New Issue
Block a user