feat(wiki): 历史上的今天数据源切换为百度百科
Build and Publish Server / build (push) Successful in 2m13s

维基百科在生产服务器被阻断,改用百度百科开放接口:
- 端点 /cms/home/eventsOnHistory/{MM}.json,按月返回结构化 JSON
- 12 次请求替代原 366 次,大幅减负
- type 直接映射 event/birth/death,festival 字段提取节假日
- 清洗 title/desc 中的 HTML 标签
- 2 月无 29 日,全年 365 天
- 本地验证:365 天 4805 条,12/12 月全成功
This commit is contained in:
gouki
2026-08-12 22:52:21 +00:00
parent cc7f30191e
commit 0ce1a1ef97
2 changed files with 247 additions and 311 deletions
+175 -240
View File
@@ -6,7 +6,6 @@ import (
"io"
"log"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
@@ -18,8 +17,9 @@ import (
"gorm.io/gorm"
)
// WikiSyncService 维基百科「X月X日」页面同步服务
// 全量共 366 页(含 2 月 29 日),首次抓取后按配置间隔定期增量刷新
// WikiSyncService 「历史上的今天」同步服务
// 数据源:百度百科「历史上的今天」开放接口(按月返回结构化 JSON,共 12 次请求)
// 首次抓取后按配置间隔定期增量刷新
type WikiSyncService struct {
db *gorm.DB
@@ -39,32 +39,21 @@ func NewWikiSyncService(db *gorm.DB) *WikiSyncService {
}
var (
reComment = regexp.MustCompile(`(?s)<!--.*?-->`)
reRef = regexp.MustCompile(`(?is)<ref[^>]*/>|<ref[^>]*>.*?</ref>`)
reFlag = regexp.MustCompile(`\{\{(?:flag|flagcountry|flagicon)\|([^}|]+)(?:\|[^{}]*)?\}\}`)
reTemplate = regexp.MustCompile(`\{\{[^{}]*\}\}`)
reLink = regexp.MustCompile(`\[\[(?:[^|\[\]]*\|)?([^|\[\]]+)\]\]`)
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
reHeading = regexp.MustCompile(`^==+\s*(.*?)\s*==+\s*$`)
// 语言转换标记 -{zh-cn:A;zh-tw:B}- 或 -{A}-
reConv = regexp.MustCompile(`-\{([^{}]*)\}-`)
// 条目开头的年份:前612年: / 1912年: / 1980年代:
reYearPrefix = regexp.MustCompile(`^(前)?(\d{1,4})\s*年代?\s*[:]\s*`)
reNoYear = regexp.MustCompile(`^(年份不详|年份不詳|不详|不詳|生年不详|生年不詳)\s*[::]\s*`)
reLeadPunct = regexp.MustCompile(`^[:,、;\.\s]+`)
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
reSpace = regexp.MustCompile(`\s+`)
)
// daysInMonth 每月天数(含 2 月 29 日,闰日页面维基也有
var daysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
// daysInMonth 每月天数(百度百科数据源不含 2 月 29 日,共 365 天
var daysInMonth = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
type dayKey struct {
Month int
Day int
}
// allDays 全年 366
// allDays 全年 365
func allDays() []dayKey {
days := make([]dayKey, 0, 366)
days := make([]dayKey, 0, 365)
for m := 1; m <= 12; m++ {
for d := 1; d <= daysInMonth[m-1]; d++ {
days = append(days, dayKey{Month: m, Day: d})
@@ -107,7 +96,7 @@ func (s *WikiSyncService) missingDays() []dayKey {
}
// TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false
// full=true 全量刷新 366 页full=false 仅补齐缺失的天
// full=true 全量刷新 365 天full=false 仅补齐缺失的天
func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool {
days := allDays()
if !full {
@@ -133,7 +122,7 @@ func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) {
go func() {
if missing := s.missingDays(); len(missing) > 0 {
// 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐
log.Printf("wiki sync: 缺失 %d/366 天,10秒后自动补齐", len(missing))
log.Printf("wiki sync: 缺失 %d/365 天,10秒后自动补齐", len(missing))
time.Sleep(10 * time.Second)
s.TriggerSync("startup", false)
}
@@ -171,7 +160,8 @@ func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) {
return *syncLog.FinishedAt, true
}
// runSync 同步指定日期集合(全量 366 页或缺失补齐)
// runSync 同步指定日期集合(全量 365 天或缺失补齐)
// 百度百科按月返回数据,故先归并出涉及的月份,每月一次请求,再按天写入
func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
syncLog := model.WikiSyncLog{
Trigger: trigger,
@@ -181,27 +171,37 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
}
s.db.Create(&syncLog)
jobs := make(chan dayKey, len(days))
// 归并涉及的月份
monthSet := make(map[int]bool)
for _, d := range days {
jobs <- d
monthSet[d.Month] = true
}
months := make([]int, 0, len(monthSet))
for m := range monthSet {
months = append(months, m)
}
jobs := make(chan int, len(months))
for _, m := range months {
jobs <- m
}
close(jobs)
var success, failed int32
var wg sync.WaitGroup
workerCount := 3
workerCount := 2 // 百度百科反爬较敏感,低并发 + 间隔
for i := 0; i < workerCount; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for d := range jobs {
if err := s.syncOneDay(d.Month, d.Day); err != nil {
for m := range jobs {
if err := s.syncOneMonth(m); err != nil {
atomic.AddInt32(&failed, 1)
log.Printf("wiki sync: %d月%d日 失败: %v", d.Month, d.Day, err)
log.Printf("wiki sync: %d月 失败: %v", m, err)
} else {
atomic.AddInt32(&success, 1)
}
time.Sleep(800 * time.Millisecond) // 控制请求频率,避免被限流
time.Sleep(1200 * time.Millisecond) // 控制请求频率,避免触发安全验证
}
}()
}
@@ -221,266 +221,201 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
"failed": int(failed),
"finished_at": &now,
})
log.Printf("wiki sync: 同步完成 trigger=%s pages=%d success=%d failed=%d", trigger, len(days), success, failed)
log.Printf("wiki sync: 同步完成 trigger=%s days=%d months=%d success=%d failed=%d", trigger, len(days), len(months), success, failed)
}
// syncOneDay 抓取并更新某一天的数据(失败不覆盖旧数据;死锁自动重试)
func (s *WikiSyncService) syncOneDay(month, day int) error {
wikitext, err := s.fetchDayPage(month, day)
// syncOneMonth 抓取某月数据并按天整体替换(死锁自动重试)
func (s *WikiSyncService) syncOneMonth(month int) error {
items, err := s.fetchMonth(month)
if err != nil {
return err
}
items := parseDayPage(wikitext)
if len(items) == 0 {
return fmt.Errorf("页面解析结果为空")
return fmt.Errorf("%d月解析结果为空", month)
}
for i := range items {
items[i].Month = month
items[i].Day = day
items[i].Status = 1
// 按天分组
byDay := make(map[int][]model.WikiOnThisDay)
for _, it := range items {
byDay[it.Day] = append(byDay[it.Day], it)
}
// 按天整体替换:先删后插;并发写入同表可能死锁,重试 3 次
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
time.Sleep(time.Duration(attempt) * time.Second)
}
lastErr = s.db.Transaction(func(tx *gorm.DB) error {
if err := tx.Where("month = ? AND day = ?", month, day).
Delete(&model.WikiOnThisDay{}).Error; err != nil {
return err
// 逐天事务替换
for day, dayItems := range byDay {
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
time.Sleep(time.Duration(attempt) * time.Second)
}
lastErr = s.db.Transaction(func(tx *gorm.DB) error {
if err := tx.Where("month = ? AND day = ?", month, day).
Delete(&model.WikiOnThisDay{}).Error; err != nil {
return err
}
return tx.Create(&dayItems).Error
})
if lastErr == nil {
break
}
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
return lastErr
}
return tx.Create(&items).Error
})
if lastErr == nil {
return nil
}
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
break
if lastErr != nil {
return fmt.Errorf("%d月%d日写入失败: %w", month, day, lastErr)
}
}
return lastErr
return nil
}
// fetchDayPage 抓取「X月X日」页面的 wikitext(失败重试;429 限流时加大退避间隔)
func (s *WikiSyncService) fetchDayPage(month, day int) (string, error) {
title := fmt.Sprintf("%d月%d日", month, day)
apiURL := "https://zh.wikipedia.org/w/api.php?action=parse&prop=wikitext&format=json&formatversion=2&page=" +
url.QueryEscape(title)
// ===== 百度百科数据抓取与解析 =====
// baikeEvent 百度百科「历史上的今天」单条数据
type baikeEvent struct {
Year string `json:"year"` // 年份,负数表示公元前(如 "-30"
Title string `json:"title"` // 标题(含 HTML 链接标签)
Festival string `json:"festival"` // 节假日名(当天所有条目共享,可能为空)
Link string `json:"link"` // 百科词条链接
Type string `json:"type"` // event / birth / death
Desc string `json:"desc"` // 描述(含 HTML 标签)
}
// fetchMonth 抓取某月「历史上的今天」数据并解析为按天的条目(失败重试)
func (s *WikiSyncService) fetchMonth(month int) ([]model.WikiOnThisDay, error) {
apiURL := fmt.Sprintf("https://baike.baidu.com/cms/home/eventsOnHistory/%02d.json", month)
var lastErr error
for attempt := 0; attempt < 4; attempt++ {
if attempt > 0 {
backoff := time.Duration(attempt*3) * time.Second
if lastErr != nil && strings.Contains(lastErr.Error(), "429") {
backoff = time.Duration(attempt*30) * time.Second // 限流时退避 30s/60s/90s
}
time.Sleep(backoff)
time.Sleep(time.Duration(attempt*3) * time.Second)
}
text, err := s.doFetch(apiURL)
if err == nil {
return text, nil
body, err := s.doFetch(apiURL)
if err != nil {
lastErr = err
continue
}
lastErr = err
items, err := parseBaikeMonth(body, month)
if err != nil {
lastErr = err
continue
}
return items, nil
}
return "", lastErr
return nil, lastErr
}
func (s *WikiSyncService) doFetch(apiURL string) (string, error) {
// doFetch 发起 HTTP 请求并返回响应体(带浏览器 UA 与 Referer,规避安全验证)
func (s *WikiSyncService) doFetch(apiURL string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
if err != nil {
return "", err
return nil, err
}
req.Header.Set("User-Agent", "LunarServer/1.0 (wiki on-this-day sync; contact: admin@neatcn.com)")
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
req.Header.Set("Referer", "https://baike.baidu.com/calendar")
req.Header.Set("Accept", "application/json, text/plain, */*")
resp, err := s.httpClient.Do(req)
if err != nil {
return "", err
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB
if err != nil {
return "", err
}
var result struct {
Parse struct {
Wikitext string `json:"wikitext"`
} `json:"parse"`
Error struct {
Info string `json:"info"`
} `json:"error"`
}
if err := json.Unmarshal(body, &result); err != nil {
return "", fmt.Errorf("JSON 解析失败: %w", err)
}
if result.Error.Info != "" {
return "", fmt.Errorf("API 错误: %s", result.Error.Info)
}
if result.Parse.Wikitext == "" {
return "", fmt.Errorf("页面内容为空")
}
return result.Parse.Wikitext, nil
return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) // 上限 4MB
}
// ===== wikitext 解析 =====
// parseBaikeMonth 解析百度百科某月 JSON,输出该月全部条目
func parseBaikeMonth(body []byte, month int) ([]model.WikiOnThisDay, error) {
// 顶层结构:{"08": {"0801": [...], "0802": [...]}}
var root map[string]map[string][]baikeEvent
if err := json.Unmarshal(body, &root); err != nil {
return nil, fmt.Errorf("JSON 解析失败: %w", err)
}
monthKey := fmt.Sprintf("%02d", month)
dayMap, ok := root[monthKey]
if !ok || len(dayMap) == 0 {
return nil, fmt.Errorf("%s月无数据", monthKey)
}
// classifySection 根据二级标题识别板块类型
func classifySection(title string) string {
switch {
case strings.Contains(title, "大事") || strings.Contains(title, "事件"):
items := make([]model.WikiOnThisDay, 0, 256)
for dayKeyStr, events := range dayMap {
day, err := strconv.Atoi(dayKeyStr[2:]) // "0812" -> 12
if err != nil {
continue
}
seenFestival := make(map[string]bool)
seenContent := make(map[string]bool)
for _, ev := range events {
// 节假日:从 festival 字段提取(挂在每条上,去重后单独成 festival 条目)
if fest := cleanHTMLText(ev.Festival); fest != "" && !seenFestival[fest] {
seenFestival[fest] = true
items = append(items, model.WikiOnThisDay{
Month: month, Day: day, Kind: "festival", Year: 0, Content: fest, Status: 1,
})
}
kind := normalizeKind(ev.Type)
if kind == "" {
continue
}
text := cleanHTMLText(ev.Title)
if len([]rune(text)) < 4 {
continue
}
if len([]rune(text)) > 500 {
text = string([]rune(text)[:500])
}
if seenContent[kind+"|"+text] {
continue
}
seenContent[kind+"|"+text] = true
items = append(items, model.WikiOnThisDay{
Month: month,
Day: day,
Kind: kind,
Year: parseBaikeYear(ev.Year),
Content: text,
Status: 1,
})
}
}
return items, nil
}
// normalizeKind 映射百度百科 type 到内部 kind
func normalizeKind(t string) string {
switch strings.ToLower(strings.TrimSpace(t)) {
case "event":
return "event"
case strings.Contains(title, "出生"):
case "birth":
return "birth"
case strings.Contains(title, "逝世") || strings.Contains(title, "去世"):
case "death":
return "death"
case strings.Contains(title, "节假日") || strings.Contains(title, "节日") ||
strings.Contains(title, "節日") || strings.Contains(title, "假日") ||
strings.Contains(title, "习俗") || strings.Contains(title, "風俗") ||
strings.Contains(title, "风俗"):
return "festival"
}
return ""
}
// parseDayPage 解析页面 wikitext,提取大事记/出生/逝世/节假日条目
func parseDayPage(wikitext string) []model.WikiOnThisDay {
items := make([]model.WikiOnThisDay, 0, 128)
seen := make(map[string]bool) // 同板块去重(维基页面偶有重复条目)
curKind := ""
inTable := false
for _, rawLine := range strings.Split(wikitext, "\n") {
line := strings.TrimSpace(rawLine)
// 跳过表格区块
if strings.HasPrefix(line, "{|") {
inTable = true
continue
}
if inTable {
if strings.HasPrefix(line, "|}") {
inTable = false
}
continue
}
// 二级/三级标题切换板块(三级标题归属当前二级板块)
if m := reHeading.FindStringSubmatch(line); m != nil {
if strings.HasPrefix(line, "=== ") || strings.HasPrefix(line, "===") {
continue // 三级标题:大事记内的世纪分组,不改变板块
}
curKind = classifySection(m[1])
continue
}
if curKind == "" || !strings.HasPrefix(line, "*") || strings.HasPrefix(line, "*>") {
continue
}
content := strings.TrimSpace(strings.TrimLeft(line, "*"))
if content == "" {
continue
}
year, text := extractYear(cleanWikitext(content))
if len([]rune(text)) < 4 { // 过短条目无展示价值
continue
}
if len([]rune(text)) > 500 {
text = string([]rune(text)[:500])
}
dedupeKey := curKind + "|" + text
if seen[dedupeKey] {
continue
}
seen[dedupeKey] = true
items = append(items, model.WikiOnThisDay{
Kind: curKind,
Year: year,
Content: text,
})
// parseBaikeYear 解析年份字符串:"1912"→1912"-30"→-30(公元前),非法→0
func parseBaikeYear(s string) int {
s = strings.TrimSpace(s)
if s == "" {
return 0
}
return items
y, err := strconv.Atoi(s)
if err != nil {
return 0
}
return y
}
// extractYear 从条目文本提取年份:前612年→-612,1912年→1912,无年份→0
func extractYear(text string) (int, string) {
text = strings.TrimSpace(text)
if m := reYearPrefix.FindStringSubmatch(text); m != nil {
year, err := strconv.Atoi(m[2])
if err == nil {
if m[1] == "前" {
year = -year
}
return year, strings.TrimSpace(text[len(m[0]):])
}
}
if m := reNoYear.FindStringSubmatch(text); m != nil {
return 0, strings.TrimSpace(text[len(m[0]):])
}
return 0, text
}
// convertConvMarkup 处理语言转换标记 -{zh-cn:A;zh-tw:B}-,优先取简体变体
func convertConvMarkup(s string) string {
return reConv.ReplaceAllStringFunc(s, func(m string) string {
inner := m[2 : len(m)-2] // 去掉 -{ 和 }-
parts := strings.Split(inner, ";")
// 无冒号:-{A}- 直接取内容
if !strings.Contains(inner, ":") {
return strings.TrimSpace(inner)
}
fallback := ""
for _, p := range parts {
p = strings.TrimSpace(p)
if p == "" {
continue
}
kv := strings.SplitN(p, ":", 2)
if len(kv) != 2 {
continue
}
key := strings.ToLower(strings.TrimSpace(kv[0]))
val := strings.TrimSpace(kv[1])
if key == "zh-cn" || key == "zh-hans" {
return val
}
fallback = val
}
return fallback
})
}
// cleanWikitext 去除 wiki 语法,输出纯文本
func cleanWikitext(s string) string {
s = reComment.ReplaceAllString(s, "")
s = reRef.ReplaceAllString(s, "")
s = convertConvMarkup(s)
// 国旗模板保留国家名
s = reFlag.ReplaceAllString(s, "$1")
// 模板可能嵌套,循环剥离直至稳定
for i := 0; i < 6; i++ {
next := reTemplate.ReplaceAllString(s, "")
if next == s {
break
}
s = next
}
s = reLink.ReplaceAllString(s, "$1")
s = strings.ReplaceAll(s, "'''", "")
s = strings.ReplaceAll(s, "''", "")
// cleanHTMLText 去除 HTML 标签与多余空白,输出纯文本
func cleanHTMLText(s string) string {
s = reHTMLTag.ReplaceAllString(s, "")
s = strings.ReplaceAll(s, "&nbsp;", " ")
s = strings.TrimSpace(s)
s = reLeadPunct.ReplaceAllString(s, "") // 模板被剥离后可能残留开头标点
s = reSpace.ReplaceAllString(s, " ")
return strings.TrimSpace(s)
}
+72 -71
View File
@@ -1,43 +1,31 @@
package service
import (
"encoding/json"
"os"
"strings"
"testing"
)
// 取自 zh.wikipedia.org「8月10日」页面的真实片段
const sampleWikitext = `'''8月10日'''是[[阳历]]年的第222天。
// 取自百度百科 /cms/home/eventsOnHistory/08.json 的真实片段(8月12日/13日)
const sampleBaikeJSON = `{"08":{
"0812":[
{"year":"-30","title":"埃及艳后<a target=\"_blank\" href=\"https://baike.baidu.com/item/x\">克利奥帕特拉七世</a>逝世","festival":"","link":"https://baike.baidu.com/item/x","type":"death","desc":"克利奥帕特拉七世"},
{"year":"1848","title":"英国著名小说家<a target=\"_blank\" href=\"x\">乔治·艾略特</a>出生","festival":"","link":"x","type":"birth","desc":"x"},
{"year":"1848","title":"英国著名小说家乔治·艾略特出生","festival":"","link":"x","type":"birth","desc":"x"},
{"year":"1905","title":"挪威音乐家<a href=\"x\">某</a>指挥首演","festival":"","link":"x","type":"event","desc":"x"}
],
"0813":[
{"year":"604","title":"隋文帝<a href=\"x\">杨坚</a>逝世","festival":"国际左撇子日","link":"x","type":"death","desc":"x"},
{"year":"1926","title":"古巴革命领导人<a href=\"x\">卡斯特罗</a>出生","festival":"国际左撇子日","link":"x","type":"birth","desc":"x"},
{"year":"1910","title":"某事件<a href=\"x\">发生</a>","festival":"国际左撇子日","link":"x","type":"event","desc":"x"},
{"year":"abc","title":"年份非法的<a href=\"x\">事件</a>条目","festival":"国际左撇子日","link":"x","type":"event","desc":"x"}
]
}}`
== 大事记 ==
=== 19世紀以前 ===
* [[前612年]]:[[亚述]]最后一位君主[[辛·沙里施昆]]死于[[尼尼微]],王国灭亡。
* [[955年]]:[[鄂圖一世 (神聖羅馬帝國)|鄂圖一世]]麾下的軍隊在[[第二次萊希菲爾德之戰 (955)|萊希菲爾德之戰]]中取得勝利。<ref>注释</ref>
=== 20世紀 ===
* [[1912年]]:[[威廉·布利斯]]出版了第一本关于[[童子军]]的书。
* [[1945年]]{{le|蒙古}}对日宣战。
* [[2002年]]-{zh-hans:凯尔·斯科特;zh-hk:卡爾·史葛;zh-tw:凱爾·斯科特;}-,英國職業足球運動員。
* 年份不详:某件无法确定年份的事情发生了。
== 出生 ==
* [[前156年]][[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
* [[前156年]][[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
* [[1874年]][[赫伯特·胡佛]],美国第31任[[总统]]([[1964年]]逝世)
== 逝世 ==
* [[258年]]:[[羅馬的聖老楞佐]],基督教殉道者
== 节假日和习俗 ==
* {{flag|厄瓜多尔}}:独立日
* 世界[[狮子]]日
==参考资料==
{{Commonscat}}
`
func TestParseDayPage(t *testing.T) {
items := parseDayPage(sampleWikitext)
func TestParseBaikeMonth(t *testing.T) {
items, err := parseBaikeMonth([]byte(sampleBaikeJSON), 8)
if err != nil {
t.Fatalf("parseBaikeMonth 失败: %v", err)
}
count := func(kind string) int {
n := 0
@@ -49,65 +37,78 @@ func TestParseDayPage(t *testing.T) {
return n
}
if got := count("event"); got != 6 {
t.Errorf("event 条目数 = %d, 期望 6", got)
// 0812: 1 death + 2 birth(重复去重后1) + 1 event = 30813: 1 death + 1 birth + 2 event = 4
if got := count("death"); got != 2 {
t.Errorf("death 条目数 = %d, 期望 2", got)
}
if got := count("birth"); got != 2 { // 重复条目应去重
if got := count("birth"); got != 2 { // 0812 重复出生条目应去重
t.Errorf("birth 条目数 = %d, 期望 2", got)
}
if got := count("death"); got != 1 {
t.Errorf("death 条目数 = %d, 期望 1", got)
if got := count("event"); got != 3 {
t.Errorf("event 条目数 = %d, 期望 3", got)
}
if got := count("festival"); got != 2 {
t.Errorf("festival 条目数 = %d, 期望 2", got)
// 国际左撇子日只在 0813,去重后 1 条 festival
if got := count("festival"); got != 1 {
t.Errorf("festival 条目数 = %d, 期望 1", got)
}
// 校验具体条目的年份与文本清洗
assertItem := func(kind string, wantYear int, wantContent string) {
// 校验具体条目
assertItem := func(kind string, day, wantYear int, wantContent string) {
for _, it := range items {
if it.Kind == kind && it.Content == wantContent {
if it.Kind == kind && it.Day == day && it.Content == wantContent {
if it.Year != wantYear {
t.Errorf("%q 年份 = %d, 期望 %d", wantContent, it.Year, wantYear)
}
if it.Month != 8 {
t.Errorf("%q 月份 = %d, 期望 8", wantContent, it.Month)
}
return
}
}
t.Errorf("未找到条目: kind=%s content=%q", kind, wantContent)
t.Errorf("未找到条目: kind=%s day=%d content=%q", kind, day, wantContent)
}
assertItem("event", -612, "亚述最后一位君主辛·沙里施昆死于尼尼微,王国灭亡。")
assertItem("event", 955, "鄂圖一世麾下的軍隊在萊希菲爾德之戰中取得勝利。")
assertItem("event", 0, "某件无法确定年份的事情发生了。")
assertItem("event", 2002, "凯尔·斯科特,英國職業足球運動員。")
assertItem("festival", 0, "厄瓜多尔:独立日")
assertItem("birth", -156, "汉武帝刘彻,西汉皇帝(前87年逝世)")
assertItem("festival", 0, "世界狮子日")
assertItem("death", 12, -30, "埃及艳后克利奥帕特拉七世逝世")
assertItem("birth", 12, 1848, "英国著名小说家乔治·艾略特出生")
assertItem("event", 13, 0, "年份非法的事件条目") // 非法年份应为 0
assertItem("festival", 13, 0, "国际左撇子日")
// 不应包含 wiki 语法残留
// 不应残留 HTML 标签
for _, it := range items {
if strings.Contains(it.Content, "[[") || strings.Contains(it.Content, "{{") ||
strings.Contains(it.Content, "<ref") {
t.Errorf("条目残留 wiki 语法: %q", it.Content)
if strings.Contains(it.Content, "<a") || strings.Contains(it.Content, "href") ||
strings.Contains(it.Content, "</a>") {
t.Errorf("条目残留 HTML 标签: %q", it.Content)
}
}
}
// TestParseRealPage 若本地存在真实抓取的样本文件则验证(可选)
func TestParseRealPage(t *testing.T) {
data, err := os.ReadFile("/tmp/wiki_810.json")
if err != nil {
t.Skip("无真实样本文件,跳过")
func TestParseBaikeYear(t *testing.T) {
cases := map[string]int{
"1912": 1912,
"-30": -30,
"0": 0,
"abc": 0,
"": 0,
" 77 ": 77,
}
var d struct {
Parse struct {
Wikitext string `json:"wikitext"`
} `json:"parse"`
for in, want := range cases {
if got := parseBaikeYear(in); got != want {
t.Errorf("parseBaikeYear(%q) = %d, 期望 %d", in, got, want)
}
}
}
func TestNormalizeKind(t *testing.T) {
if normalizeKind("event") != "event" || normalizeKind("birth") != "birth" ||
normalizeKind("death") != "death" || normalizeKind("other") != "" ||
normalizeKind(" EVENT ") != "event" {
t.Error("normalizeKind 映射异常")
}
}
func TestCleanHTMLText(t *testing.T) {
in := `埃及艳后<a target="_blank" href="x">克利奥帕特拉七世</a> 逝世&nbsp;test`
want := "埃及艳后克利奥帕特拉七世 逝世 test"
if got := cleanHTMLText(in); got != want {
t.Errorf("cleanHTMLText = %q, 期望 %q", got, want)
}
if err := json.Unmarshal(data, &d); err != nil {
t.Fatalf("样本解析失败: %v", err)
}
items := parseDayPage(d.Parse.Wikitext)
if len(items) < 20 {
t.Errorf("真实页面解析条目过少: %d", len(items))
}
t.Logf("真实页面共解析 %d 条", len(items))
}