diff --git a/server/internal/service/wiki_sync.go b/server/internal/service/wiki_sync.go index a282fb0..4ed8402 100644 --- a/server/internal/service/wiki_sync.go +++ b/server/internal/service/wiki_sync.go @@ -6,7 +6,6 @@ import ( "io" "log" "net/http" - "net/url" "regexp" "strconv" "strings" @@ -18,8 +17,9 @@ import ( "gorm.io/gorm" ) -// WikiSyncService 维基百科「X月X日」页面同步服务 -// 全量共 366 页(含 2 月 29 日),首次抓取后按配置间隔定期增量刷新 +// WikiSyncService 「历史上的今天」同步服务 +// 数据源:百度百科「历史上的今天」开放接口(按月返回结构化 JSON,共 12 次请求) +// 首次抓取后按配置间隔定期增量刷新 type WikiSyncService struct { db *gorm.DB @@ -39,32 +39,21 @@ func NewWikiSyncService(db *gorm.DB) *WikiSyncService { } var ( - reComment = regexp.MustCompile(`(?s)`) - reRef = regexp.MustCompile(`(?is)]*/>|]*>.*?`) - reFlag = regexp.MustCompile(`\{\{(?:flag|flagcountry|flagicon)\|([^}|]+)(?:\|[^{}]*)?\}\}`) - reTemplate = regexp.MustCompile(`\{\{[^{}]*\}\}`) - reLink = regexp.MustCompile(`\[\[(?:[^|\[\]]*\|)?([^|\[\]]+)\]\]`) - reHTMLTag = regexp.MustCompile(`]*>`) - reHeading = regexp.MustCompile(`^==+\s*(.*?)\s*==+\s*$`) - // 语言转换标记 -{zh-cn:A;zh-tw:B}- 或 -{A}- - reConv = regexp.MustCompile(`-\{([^{}]*)\}-`) - // 条目开头的年份:前612年: / 1912年: / 1980年代: - reYearPrefix = regexp.MustCompile(`^(前)?(\d{1,4})\s*年代?\s*[::]\s*`) - reNoYear = regexp.MustCompile(`^(年份不详|年份不詳|不详|不詳|生年不详|生年不詳)\s*[::]\s*`) - reLeadPunct = regexp.MustCompile(`^[::,,、;;\.\s]+`) + reHTMLTag = regexp.MustCompile(`]*>`) + reSpace = regexp.MustCompile(`\s+`) ) -// daysInMonth 每月天数(含 2 月 29 日,闰日页面维基也有) -var daysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} +// daysInMonth 每月天数(百度百科数据源不含 2 月 29 日,共 365 天) +var daysInMonth = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} type dayKey struct { Month int Day int } -// allDays 全年 366 天 +// allDays 全年 365 天 func allDays() []dayKey { - days := make([]dayKey, 0, 366) + days := make([]dayKey, 0, 365) for m := 1; m <= 12; m++ { for d := 1; d <= daysInMonth[m-1]; d++ { days = append(days, dayKey{Month: m, Day: d}) @@ -107,7 +96,7 @@ func (s *WikiSyncService) missingDays() []dayKey { } // TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false -// full=true 全量刷新 366 页;full=false 仅补齐缺失的天 +// full=true 全量刷新 365 天;full=false 仅补齐缺失的天 func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool { days := allDays() if !full { @@ -133,7 +122,7 @@ func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) { go func() { if missing := s.missingDays(); len(missing) > 0 { // 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐 - log.Printf("wiki sync: 缺失 %d/366 天,10秒后自动补齐", len(missing)) + log.Printf("wiki sync: 缺失 %d/365 天,10秒后自动补齐", len(missing)) time.Sleep(10 * time.Second) s.TriggerSync("startup", false) } @@ -171,7 +160,8 @@ func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) { return *syncLog.FinishedAt, true } -// runSync 同步指定日期集合(全量 366 页或缺失补齐) +// runSync 同步指定日期集合(全量 365 天或缺失补齐) +// 百度百科按月返回数据,故先归并出涉及的月份,每月一次请求,再按天写入 func (s *WikiSyncService) runSync(trigger string, days []dayKey) { syncLog := model.WikiSyncLog{ Trigger: trigger, @@ -181,27 +171,37 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) { } s.db.Create(&syncLog) - jobs := make(chan dayKey, len(days)) + // 归并涉及的月份 + monthSet := make(map[int]bool) for _, d := range days { - jobs <- d + monthSet[d.Month] = true + } + months := make([]int, 0, len(monthSet)) + for m := range monthSet { + months = append(months, m) + } + + jobs := make(chan int, len(months)) + for _, m := range months { + jobs <- m } close(jobs) var success, failed int32 var wg sync.WaitGroup - workerCount := 3 + workerCount := 2 // 百度百科反爬较敏感,低并发 + 间隔 for i := 0; i < workerCount; i++ { wg.Add(1) go func() { defer wg.Done() - for d := range jobs { - if err := s.syncOneDay(d.Month, d.Day); err != nil { + for m := range jobs { + if err := s.syncOneMonth(m); err != nil { atomic.AddInt32(&failed, 1) - log.Printf("wiki sync: %d月%d日 失败: %v", d.Month, d.Day, err) + log.Printf("wiki sync: %d月 失败: %v", m, err) } else { atomic.AddInt32(&success, 1) } - time.Sleep(800 * time.Millisecond) // 控制请求频率,避免被限流 + time.Sleep(1200 * time.Millisecond) // 控制请求频率,避免触发安全验证 } }() } @@ -221,266 +221,201 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) { "failed": int(failed), "finished_at": &now, }) - log.Printf("wiki sync: 同步完成 trigger=%s pages=%d success=%d failed=%d", trigger, len(days), success, failed) + log.Printf("wiki sync: 同步完成 trigger=%s days=%d months=%d success=%d failed=%d", trigger, len(days), len(months), success, failed) } -// syncOneDay 抓取并更新某一天的数据(失败不覆盖旧数据;死锁自动重试) -func (s *WikiSyncService) syncOneDay(month, day int) error { - wikitext, err := s.fetchDayPage(month, day) +// syncOneMonth 抓取某月数据并按天整体替换(死锁自动重试) +func (s *WikiSyncService) syncOneMonth(month int) error { + items, err := s.fetchMonth(month) if err != nil { return err } - items := parseDayPage(wikitext) if len(items) == 0 { - return fmt.Errorf("页面解析结果为空") + return fmt.Errorf("%d月解析结果为空", month) } - for i := range items { - items[i].Month = month - items[i].Day = day - items[i].Status = 1 + // 按天分组 + byDay := make(map[int][]model.WikiOnThisDay) + for _, it := range items { + byDay[it.Day] = append(byDay[it.Day], it) } - // 按天整体替换:先删后插;并发写入同表可能死锁,重试 3 次 - var lastErr error - for attempt := 0; attempt < 3; attempt++ { - if attempt > 0 { - time.Sleep(time.Duration(attempt) * time.Second) - } - lastErr = s.db.Transaction(func(tx *gorm.DB) error { - if err := tx.Where("month = ? AND day = ?", month, day). - Delete(&model.WikiOnThisDay{}).Error; err != nil { - return err + // 逐天事务替换 + for day, dayItems := range byDay { + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * time.Second) + } + lastErr = s.db.Transaction(func(tx *gorm.DB) error { + if err := tx.Where("month = ? AND day = ?", month, day). + Delete(&model.WikiOnThisDay{}).Error; err != nil { + return err + } + return tx.Create(&dayItems).Error + }) + if lastErr == nil { + break + } + if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") { + return lastErr } - return tx.Create(&items).Error - }) - if lastErr == nil { - return nil } - if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") { - break + if lastErr != nil { + return fmt.Errorf("%d月%d日写入失败: %w", month, day, lastErr) } } - return lastErr + return nil } -// fetchDayPage 抓取「X月X日」页面的 wikitext(失败重试;429 限流时加大退避间隔) -func (s *WikiSyncService) fetchDayPage(month, day int) (string, error) { - title := fmt.Sprintf("%d月%d日", month, day) - apiURL := "https://zh.wikipedia.org/w/api.php?action=parse&prop=wikitext&format=json&formatversion=2&page=" + - url.QueryEscape(title) +// ===== 百度百科数据抓取与解析 ===== + +// baikeEvent 百度百科「历史上的今天」单条数据 +type baikeEvent struct { + Year string `json:"year"` // 年份,负数表示公元前(如 "-30") + Title string `json:"title"` // 标题(含 HTML 链接标签) + Festival string `json:"festival"` // 节假日名(当天所有条目共享,可能为空) + Link string `json:"link"` // 百科词条链接 + Type string `json:"type"` // event / birth / death + Desc string `json:"desc"` // 描述(含 HTML 标签) +} + +// fetchMonth 抓取某月「历史上的今天」数据并解析为按天的条目(失败重试) +func (s *WikiSyncService) fetchMonth(month int) ([]model.WikiOnThisDay, error) { + apiURL := fmt.Sprintf("https://baike.baidu.com/cms/home/eventsOnHistory/%02d.json", month) var lastErr error for attempt := 0; attempt < 4; attempt++ { if attempt > 0 { - backoff := time.Duration(attempt*3) * time.Second - if lastErr != nil && strings.Contains(lastErr.Error(), "429") { - backoff = time.Duration(attempt*30) * time.Second // 限流时退避 30s/60s/90s - } - time.Sleep(backoff) + time.Sleep(time.Duration(attempt*3) * time.Second) } - text, err := s.doFetch(apiURL) - if err == nil { - return text, nil + body, err := s.doFetch(apiURL) + if err != nil { + lastErr = err + continue } - lastErr = err + items, err := parseBaikeMonth(body, month) + if err != nil { + lastErr = err + continue + } + return items, nil } - return "", lastErr + return nil, lastErr } -func (s *WikiSyncService) doFetch(apiURL string) (string, error) { +// doFetch 发起 HTTP 请求并返回响应体(带浏览器 UA 与 Referer,规避安全验证) +func (s *WikiSyncService) doFetch(apiURL string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, apiURL, nil) if err != nil { - return "", err + return nil, err } - req.Header.Set("User-Agent", "LunarServer/1.0 (wiki on-this-day sync; contact: admin@neatcn.com)") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36") + req.Header.Set("Referer", "https://baike.baidu.com/calendar") + req.Header.Set("Accept", "application/json, text/plain, */*") resp, err := s.httpClient.Do(req) if err != nil { - return "", err + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("HTTP %d", resp.StatusCode) + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } - - body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB - if err != nil { - return "", err - } - - var result struct { - Parse struct { - Wikitext string `json:"wikitext"` - } `json:"parse"` - Error struct { - Info string `json:"info"` - } `json:"error"` - } - if err := json.Unmarshal(body, &result); err != nil { - return "", fmt.Errorf("JSON 解析失败: %w", err) - } - if result.Error.Info != "" { - return "", fmt.Errorf("API 错误: %s", result.Error.Info) - } - if result.Parse.Wikitext == "" { - return "", fmt.Errorf("页面内容为空") - } - return result.Parse.Wikitext, nil + return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) // 上限 4MB } -// ===== wikitext 解析 ===== +// parseBaikeMonth 解析百度百科某月 JSON,输出该月全部条目 +func parseBaikeMonth(body []byte, month int) ([]model.WikiOnThisDay, error) { + // 顶层结构:{"08": {"0801": [...], "0802": [...]}} + var root map[string]map[string][]baikeEvent + if err := json.Unmarshal(body, &root); err != nil { + return nil, fmt.Errorf("JSON 解析失败: %w", err) + } + monthKey := fmt.Sprintf("%02d", month) + dayMap, ok := root[monthKey] + if !ok || len(dayMap) == 0 { + return nil, fmt.Errorf("%s月无数据", monthKey) + } -// classifySection 根据二级标题识别板块类型 -func classifySection(title string) string { - switch { - case strings.Contains(title, "大事") || strings.Contains(title, "事件"): + items := make([]model.WikiOnThisDay, 0, 256) + for dayKeyStr, events := range dayMap { + day, err := strconv.Atoi(dayKeyStr[2:]) // "0812" -> 12 + if err != nil { + continue + } + seenFestival := make(map[string]bool) + seenContent := make(map[string]bool) + + for _, ev := range events { + // 节假日:从 festival 字段提取(挂在每条上,去重后单独成 festival 条目) + if fest := cleanHTMLText(ev.Festival); fest != "" && !seenFestival[fest] { + seenFestival[fest] = true + items = append(items, model.WikiOnThisDay{ + Month: month, Day: day, Kind: "festival", Year: 0, Content: fest, Status: 1, + }) + } + + kind := normalizeKind(ev.Type) + if kind == "" { + continue + } + text := cleanHTMLText(ev.Title) + if len([]rune(text)) < 4 { + continue + } + if len([]rune(text)) > 500 { + text = string([]rune(text)[:500]) + } + if seenContent[kind+"|"+text] { + continue + } + seenContent[kind+"|"+text] = true + + items = append(items, model.WikiOnThisDay{ + Month: month, + Day: day, + Kind: kind, + Year: parseBaikeYear(ev.Year), + Content: text, + Status: 1, + }) + } + } + return items, nil +} + +// normalizeKind 映射百度百科 type 到内部 kind +func normalizeKind(t string) string { + switch strings.ToLower(strings.TrimSpace(t)) { + case "event": return "event" - case strings.Contains(title, "出生"): + case "birth": return "birth" - case strings.Contains(title, "逝世") || strings.Contains(title, "去世"): + case "death": return "death" - case strings.Contains(title, "节假日") || strings.Contains(title, "节日") || - strings.Contains(title, "節日") || strings.Contains(title, "假日") || - strings.Contains(title, "习俗") || strings.Contains(title, "風俗") || - strings.Contains(title, "风俗"): - return "festival" } return "" } -// parseDayPage 解析页面 wikitext,提取大事记/出生/逝世/节假日条目 -func parseDayPage(wikitext string) []model.WikiOnThisDay { - items := make([]model.WikiOnThisDay, 0, 128) - seen := make(map[string]bool) // 同板块去重(维基页面偶有重复条目) - - curKind := "" - inTable := false - for _, rawLine := range strings.Split(wikitext, "\n") { - line := strings.TrimSpace(rawLine) - - // 跳过表格区块 - if strings.HasPrefix(line, "{|") { - inTable = true - continue - } - if inTable { - if strings.HasPrefix(line, "|}") { - inTable = false - } - continue - } - - // 二级/三级标题切换板块(三级标题归属当前二级板块) - if m := reHeading.FindStringSubmatch(line); m != nil { - if strings.HasPrefix(line, "=== ") || strings.HasPrefix(line, "===") { - continue // 三级标题:大事记内的世纪分组,不改变板块 - } - curKind = classifySection(m[1]) - continue - } - - if curKind == "" || !strings.HasPrefix(line, "*") || strings.HasPrefix(line, "*>") { - continue - } - - content := strings.TrimSpace(strings.TrimLeft(line, "*")) - if content == "" { - continue - } - year, text := extractYear(cleanWikitext(content)) - if len([]rune(text)) < 4 { // 过短条目无展示价值 - continue - } - if len([]rune(text)) > 500 { - text = string([]rune(text)[:500]) - } - - dedupeKey := curKind + "|" + text - if seen[dedupeKey] { - continue - } - seen[dedupeKey] = true - - items = append(items, model.WikiOnThisDay{ - Kind: curKind, - Year: year, - Content: text, - }) +// parseBaikeYear 解析年份字符串:"1912"→1912,"-30"→-30(公元前),非法→0 +func parseBaikeYear(s string) int { + s = strings.TrimSpace(s) + if s == "" { + return 0 } - return items + y, err := strconv.Atoi(s) + if err != nil { + return 0 + } + return y } -// extractYear 从条目文本提取年份:前612年→-612,1912年→1912,无年份→0 -func extractYear(text string) (int, string) { - text = strings.TrimSpace(text) - if m := reYearPrefix.FindStringSubmatch(text); m != nil { - year, err := strconv.Atoi(m[2]) - if err == nil { - if m[1] == "前" { - year = -year - } - return year, strings.TrimSpace(text[len(m[0]):]) - } - } - if m := reNoYear.FindStringSubmatch(text); m != nil { - return 0, strings.TrimSpace(text[len(m[0]):]) - } - return 0, text -} - -// convertConvMarkup 处理语言转换标记 -{zh-cn:A;zh-tw:B}-,优先取简体变体 -func convertConvMarkup(s string) string { - return reConv.ReplaceAllStringFunc(s, func(m string) string { - inner := m[2 : len(m)-2] // 去掉 -{ 和 }- - parts := strings.Split(inner, ";") - // 无冒号:-{A}- 直接取内容 - if !strings.Contains(inner, ":") { - return strings.TrimSpace(inner) - } - fallback := "" - for _, p := range parts { - p = strings.TrimSpace(p) - if p == "" { - continue - } - kv := strings.SplitN(p, ":", 2) - if len(kv) != 2 { - continue - } - key := strings.ToLower(strings.TrimSpace(kv[0])) - val := strings.TrimSpace(kv[1]) - if key == "zh-cn" || key == "zh-hans" { - return val - } - fallback = val - } - return fallback - }) -} - -// cleanWikitext 去除 wiki 语法,输出纯文本 -func cleanWikitext(s string) string { - s = reComment.ReplaceAllString(s, "") - s = reRef.ReplaceAllString(s, "") - s = convertConvMarkup(s) - // 国旗模板保留国家名 - s = reFlag.ReplaceAllString(s, "$1") - // 模板可能嵌套,循环剥离直至稳定 - for i := 0; i < 6; i++ { - next := reTemplate.ReplaceAllString(s, "") - if next == s { - break - } - s = next - } - s = reLink.ReplaceAllString(s, "$1") - s = strings.ReplaceAll(s, "'''", "") - s = strings.ReplaceAll(s, "''", "") +// cleanHTMLText 去除 HTML 标签与多余空白,输出纯文本 +func cleanHTMLText(s string) string { s = reHTMLTag.ReplaceAllString(s, "") s = strings.ReplaceAll(s, " ", " ") - s = strings.TrimSpace(s) - s = reLeadPunct.ReplaceAllString(s, "") // 模板被剥离后可能残留开头标点 + s = reSpace.ReplaceAllString(s, " ") return strings.TrimSpace(s) } diff --git a/server/internal/service/wiki_sync_test.go b/server/internal/service/wiki_sync_test.go index 8304839..b114fa4 100644 --- a/server/internal/service/wiki_sync_test.go +++ b/server/internal/service/wiki_sync_test.go @@ -1,43 +1,31 @@ package service import ( - "encoding/json" - "os" "strings" "testing" ) -// 取自 zh.wikipedia.org「8月10日」页面的真实片段 -const sampleWikitext = `'''8月10日'''是[[阳历]]年的第222天。 +// 取自百度百科 /cms/home/eventsOnHistory/08.json 的真实片段(8月12日/13日) +const sampleBaikeJSON = `{"08":{ + "0812":[ + {"year":"-30","title":"埃及艳后克利奥帕特拉七世逝世","festival":"","link":"https://baike.baidu.com/item/x","type":"death","desc":"克利奥帕特拉七世"}, + {"year":"1848","title":"英国著名小说家乔治·艾略特出生","festival":"","link":"x","type":"birth","desc":"x"}, + {"year":"1848","title":"英国著名小说家乔治·艾略特出生","festival":"","link":"x","type":"birth","desc":"x"}, + {"year":"1905","title":"挪威音乐家指挥首演","festival":"","link":"x","type":"event","desc":"x"} + ], + "0813":[ + {"year":"604","title":"隋文帝杨坚逝世","festival":"国际左撇子日","link":"x","type":"death","desc":"x"}, + {"year":"1926","title":"古巴革命领导人卡斯特罗出生","festival":"国际左撇子日","link":"x","type":"birth","desc":"x"}, + {"year":"1910","title":"某事件发生","festival":"国际左撇子日","link":"x","type":"event","desc":"x"}, + {"year":"abc","title":"年份非法的事件条目","festival":"国际左撇子日","link":"x","type":"event","desc":"x"} + ] +}}` -== 大事记 == -=== 19世紀以前 === -* [[前612年]]:[[亚述]]最后一位君主[[辛·沙里施昆]]死于[[尼尼微]],王国灭亡。 -* [[955年]]:[[鄂圖一世 (神聖羅馬帝國)|鄂圖一世]]麾下的軍隊在[[第二次萊希菲爾德之戰 (955)|萊希菲爾德之戰]]中取得勝利。注释 -=== 20世紀 === -* [[1912年]]:[[威廉·布利斯]]出版了第一本关于[[童子军]]的书。 -* [[1945年]]:{{le|蒙古}}对日宣战。 -* [[2002年]]:-{zh-hans:凯尔·斯科特;zh-hk:卡爾·史葛;zh-tw:凱爾·斯科特;}-,英國職業足球運動員。 -* 年份不详:某件无法确定年份的事情发生了。 - -== 出生 == -* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世) -* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世) -* [[1874年]]:[[赫伯特·胡佛]],美国第31任[[总统]]([[1964年]]逝世) - -== 逝世 == -* [[258年]]:[[羅馬的聖老楞佐]],基督教殉道者 - -== 节假日和习俗 == -* {{flag|厄瓜多尔}}:独立日 -* 世界[[狮子]]日 - -==参考资料== -{{Commonscat}} -` - -func TestParseDayPage(t *testing.T) { - items := parseDayPage(sampleWikitext) +func TestParseBaikeMonth(t *testing.T) { + items, err := parseBaikeMonth([]byte(sampleBaikeJSON), 8) + if err != nil { + t.Fatalf("parseBaikeMonth 失败: %v", err) + } count := func(kind string) int { n := 0 @@ -49,65 +37,78 @@ func TestParseDayPage(t *testing.T) { return n } - if got := count("event"); got != 6 { - t.Errorf("event 条目数 = %d, 期望 6", got) + // 0812: 1 death + 2 birth(重复去重后1) + 1 event = 3;0813: 1 death + 1 birth + 2 event = 4 + if got := count("death"); got != 2 { + t.Errorf("death 条目数 = %d, 期望 2", got) } - if got := count("birth"); got != 2 { // 重复条目应去重 + if got := count("birth"); got != 2 { // 0812 重复出生条目应去重 t.Errorf("birth 条目数 = %d, 期望 2", got) } - if got := count("death"); got != 1 { - t.Errorf("death 条目数 = %d, 期望 1", got) + if got := count("event"); got != 3 { + t.Errorf("event 条目数 = %d, 期望 3", got) } - if got := count("festival"); got != 2 { - t.Errorf("festival 条目数 = %d, 期望 2", got) + // 国际左撇子日只在 0813,去重后 1 条 festival + if got := count("festival"); got != 1 { + t.Errorf("festival 条目数 = %d, 期望 1", got) } - // 校验具体条目的年份与文本清洗 - assertItem := func(kind string, wantYear int, wantContent string) { + // 校验具体条目 + assertItem := func(kind string, day, wantYear int, wantContent string) { for _, it := range items { - if it.Kind == kind && it.Content == wantContent { + if it.Kind == kind && it.Day == day && it.Content == wantContent { if it.Year != wantYear { t.Errorf("%q 年份 = %d, 期望 %d", wantContent, it.Year, wantYear) } + if it.Month != 8 { + t.Errorf("%q 月份 = %d, 期望 8", wantContent, it.Month) + } return } } - t.Errorf("未找到条目: kind=%s content=%q", kind, wantContent) + t.Errorf("未找到条目: kind=%s day=%d content=%q", kind, day, wantContent) } - assertItem("event", -612, "亚述最后一位君主辛·沙里施昆死于尼尼微,王国灭亡。") - assertItem("event", 955, "鄂圖一世麾下的軍隊在萊希菲爾德之戰中取得勝利。") - assertItem("event", 0, "某件无法确定年份的事情发生了。") - assertItem("event", 2002, "凯尔·斯科特,英國職業足球運動員。") - assertItem("festival", 0, "厄瓜多尔:独立日") - assertItem("birth", -156, "汉武帝刘彻,西汉皇帝(前87年逝世)") - assertItem("festival", 0, "世界狮子日") + assertItem("death", 12, -30, "埃及艳后克利奥帕特拉七世逝世") + assertItem("birth", 12, 1848, "英国著名小说家乔治·艾略特出生") + assertItem("event", 13, 0, "年份非法的事件条目") // 非法年份应为 0 + assertItem("festival", 13, 0, "国际左撇子日") - // 不应包含 wiki 语法残留 + // 不应残留 HTML 标签 for _, it := range items { - if strings.Contains(it.Content, "[[") || strings.Contains(it.Content, "{{") || - strings.Contains(it.Content, "") { + t.Errorf("条目残留 HTML 标签: %q", it.Content) } } } -// TestParseRealPage 若本地存在真实抓取的样本文件则验证(可选) -func TestParseRealPage(t *testing.T) { - data, err := os.ReadFile("/tmp/wiki_810.json") - if err != nil { - t.Skip("无真实样本文件,跳过") +func TestParseBaikeYear(t *testing.T) { + cases := map[string]int{ + "1912": 1912, + "-30": -30, + "0": 0, + "abc": 0, + "": 0, + " 77 ": 77, } - var d struct { - Parse struct { - Wikitext string `json:"wikitext"` - } `json:"parse"` + for in, want := range cases { + if got := parseBaikeYear(in); got != want { + t.Errorf("parseBaikeYear(%q) = %d, 期望 %d", in, got, want) + } + } +} + +func TestNormalizeKind(t *testing.T) { + if normalizeKind("event") != "event" || normalizeKind("birth") != "birth" || + normalizeKind("death") != "death" || normalizeKind("other") != "" || + normalizeKind(" EVENT ") != "event" { + t.Error("normalizeKind 映射异常") + } +} + +func TestCleanHTMLText(t *testing.T) { + in := `埃及艳后克利奥帕特拉七世 逝世 test` + want := "埃及艳后克利奥帕特拉七世 逝世 test" + if got := cleanHTMLText(in); got != want { + t.Errorf("cleanHTMLText = %q, 期望 %q", got, want) } - if err := json.Unmarshal(data, &d); err != nil { - t.Fatalf("样本解析失败: %v", err) - } - items := parseDayPage(d.Parse.Wikitext) - if len(items) < 20 { - t.Errorf("真实页面解析条目过少: %d", len(items)) - } - t.Logf("真实页面共解析 %d 条", len(items)) }