Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0ce1a1ef97 | ||
|
|
cc7f30191e |
@@ -6,7 +6,6 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -18,8 +17,9 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// WikiSyncService 维基百科「X月X日」页面同步服务
|
||||
// 全量共 366 页(含 2 月 29 日),首次抓取后按配置间隔定期增量刷新
|
||||
// WikiSyncService 「历史上的今天」同步服务
|
||||
// 数据源:百度百科「历史上的今天」开放接口(按月返回结构化 JSON,共 12 次请求)
|
||||
// 首次抓取后按配置间隔定期增量刷新
|
||||
type WikiSyncService struct {
|
||||
db *gorm.DB
|
||||
|
||||
@@ -39,32 +39,21 @@ func NewWikiSyncService(db *gorm.DB) *WikiSyncService {
|
||||
}
|
||||
|
||||
var (
|
||||
reComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
reRef = regexp.MustCompile(`(?is)<ref[^>]*/>|<ref[^>]*>.*?</ref>`)
|
||||
reFlag = regexp.MustCompile(`\{\{(?:flag|flagcountry|flagicon)\|([^}|]+)(?:\|[^{}]*)?\}\}`)
|
||||
reTemplate = regexp.MustCompile(`\{\{[^{}]*\}\}`)
|
||||
reLink = regexp.MustCompile(`\[\[(?:[^|\[\]]*\|)?([^|\[\]]+)\]\]`)
|
||||
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||||
reHeading = regexp.MustCompile(`^==+\s*(.*?)\s*==+\s*$`)
|
||||
// 语言转换标记 -{zh-cn:A;zh-tw:B}- 或 -{A}-
|
||||
reConv = regexp.MustCompile(`-\{([^{}]*)\}-`)
|
||||
// 条目开头的年份:前612年: / 1912年: / 1980年代:
|
||||
reYearPrefix = regexp.MustCompile(`^(前)?(\d{1,4})\s*年代?\s*[::]\s*`)
|
||||
reNoYear = regexp.MustCompile(`^(年份不详|年份不詳|不详|不詳|生年不详|生年不詳)\s*[::]\s*`)
|
||||
reLeadPunct = regexp.MustCompile(`^[::,,、;;\.\s]+`)
|
||||
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||||
reSpace = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
// daysInMonth 每月天数(含 2 月 29 日,闰日页面维基也有)
|
||||
var daysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
|
||||
// daysInMonth 每月天数(百度百科数据源不含 2 月 29 日,共 365 天)
|
||||
var daysInMonth = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
|
||||
|
||||
type dayKey struct {
|
||||
Month int
|
||||
Day int
|
||||
}
|
||||
|
||||
// allDays 全年 366 天
|
||||
// allDays 全年 365 天
|
||||
func allDays() []dayKey {
|
||||
days := make([]dayKey, 0, 366)
|
||||
days := make([]dayKey, 0, 365)
|
||||
for m := 1; m <= 12; m++ {
|
||||
for d := 1; d <= daysInMonth[m-1]; d++ {
|
||||
days = append(days, dayKey{Month: m, Day: d})
|
||||
@@ -107,7 +96,7 @@ func (s *WikiSyncService) missingDays() []dayKey {
|
||||
}
|
||||
|
||||
// TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false
|
||||
// full=true 全量刷新 366 页;full=false 仅补齐缺失的天
|
||||
// full=true 全量刷新 365 天;full=false 仅补齐缺失的天
|
||||
func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool {
|
||||
days := allDays()
|
||||
if !full {
|
||||
@@ -133,7 +122,7 @@ func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) {
|
||||
go func() {
|
||||
if missing := s.missingDays(); len(missing) > 0 {
|
||||
// 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐
|
||||
log.Printf("wiki sync: 缺失 %d/366 天,10秒后自动补齐", len(missing))
|
||||
log.Printf("wiki sync: 缺失 %d/365 天,10秒后自动补齐", len(missing))
|
||||
time.Sleep(10 * time.Second)
|
||||
s.TriggerSync("startup", false)
|
||||
}
|
||||
@@ -171,7 +160,8 @@ func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) {
|
||||
return *syncLog.FinishedAt, true
|
||||
}
|
||||
|
||||
// runSync 同步指定日期集合(全量 366 页或缺失补齐)
|
||||
// runSync 同步指定日期集合(全量 365 天或缺失补齐)
|
||||
// 百度百科按月返回数据,故先归并出涉及的月份,每月一次请求,再按天写入
|
||||
func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
||||
syncLog := model.WikiSyncLog{
|
||||
Trigger: trigger,
|
||||
@@ -181,27 +171,37 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
||||
}
|
||||
s.db.Create(&syncLog)
|
||||
|
||||
jobs := make(chan dayKey, len(days))
|
||||
// 归并涉及的月份
|
||||
monthSet := make(map[int]bool)
|
||||
for _, d := range days {
|
||||
jobs <- d
|
||||
monthSet[d.Month] = true
|
||||
}
|
||||
months := make([]int, 0, len(monthSet))
|
||||
for m := range monthSet {
|
||||
months = append(months, m)
|
||||
}
|
||||
|
||||
jobs := make(chan int, len(months))
|
||||
for _, m := range months {
|
||||
jobs <- m
|
||||
}
|
||||
close(jobs)
|
||||
|
||||
var success, failed int32
|
||||
var wg sync.WaitGroup
|
||||
workerCount := 3
|
||||
workerCount := 2 // 百度百科反爬较敏感,低并发 + 间隔
|
||||
for i := 0; i < workerCount; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for d := range jobs {
|
||||
if err := s.syncOneDay(d.Month, d.Day); err != nil {
|
||||
for m := range jobs {
|
||||
if err := s.syncOneMonth(m); err != nil {
|
||||
atomic.AddInt32(&failed, 1)
|
||||
log.Printf("wiki sync: %d月%d日 失败: %v", d.Month, d.Day, err)
|
||||
log.Printf("wiki sync: %d月 失败: %v", m, err)
|
||||
} else {
|
||||
atomic.AddInt32(&success, 1)
|
||||
}
|
||||
time.Sleep(800 * time.Millisecond) // 控制请求频率,避免被限流
|
||||
time.Sleep(1200 * time.Millisecond) // 控制请求频率,避免触发安全验证
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -221,266 +221,201 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
||||
"failed": int(failed),
|
||||
"finished_at": &now,
|
||||
})
|
||||
log.Printf("wiki sync: 同步完成 trigger=%s pages=%d success=%d failed=%d", trigger, len(days), success, failed)
|
||||
log.Printf("wiki sync: 同步完成 trigger=%s days=%d months=%d success=%d failed=%d", trigger, len(days), len(months), success, failed)
|
||||
}
|
||||
|
||||
// syncOneDay 抓取并更新某一天的数据(失败不覆盖旧数据;死锁自动重试)
|
||||
func (s *WikiSyncService) syncOneDay(month, day int) error {
|
||||
wikitext, err := s.fetchDayPage(month, day)
|
||||
// syncOneMonth 抓取某月数据并按天整体替换(死锁自动重试)
|
||||
func (s *WikiSyncService) syncOneMonth(month int) error {
|
||||
items, err := s.fetchMonth(month)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items := parseDayPage(wikitext)
|
||||
if len(items) == 0 {
|
||||
return fmt.Errorf("页面解析结果为空")
|
||||
return fmt.Errorf("%d月解析结果为空", month)
|
||||
}
|
||||
|
||||
for i := range items {
|
||||
items[i].Month = month
|
||||
items[i].Day = day
|
||||
items[i].Status = 1
|
||||
// 按天分组
|
||||
byDay := make(map[int][]model.WikiOnThisDay)
|
||||
for _, it := range items {
|
||||
byDay[it.Day] = append(byDay[it.Day], it)
|
||||
}
|
||||
|
||||
// 按天整体替换:先删后插;并发写入同表可能死锁,重试 3 次
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(time.Duration(attempt) * time.Second)
|
||||
}
|
||||
lastErr = s.db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Where("month = ? AND day = ?", month, day).
|
||||
Delete(&model.WikiOnThisDay{}).Error; err != nil {
|
||||
return err
|
||||
// 逐天事务替换
|
||||
for day, dayItems := range byDay {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(time.Duration(attempt) * time.Second)
|
||||
}
|
||||
lastErr = s.db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Where("month = ? AND day = ?", month, day).
|
||||
Delete(&model.WikiOnThisDay{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Create(&dayItems).Error
|
||||
})
|
||||
if lastErr == nil {
|
||||
break
|
||||
}
|
||||
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
|
||||
return lastErr
|
||||
}
|
||||
return tx.Create(&items).Error
|
||||
})
|
||||
if lastErr == nil {
|
||||
return nil
|
||||
}
|
||||
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
|
||||
break
|
||||
if lastErr != nil {
|
||||
return fmt.Errorf("%d月%d日写入失败: %w", month, day, lastErr)
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchDayPage 抓取「X月X日」页面的 wikitext(失败重试;429 限流时加大退避间隔)
|
||||
func (s *WikiSyncService) fetchDayPage(month, day int) (string, error) {
|
||||
title := fmt.Sprintf("%d月%d日", month, day)
|
||||
apiURL := "https://zh.wikipedia.org/w/api.php?action=parse&prop=wikitext&format=json&formatversion=2&page=" +
|
||||
url.QueryEscape(title)
|
||||
// ===== 百度百科数据抓取与解析 =====
|
||||
|
||||
// baikeEvent 百度百科「历史上的今天」单条数据
|
||||
type baikeEvent struct {
|
||||
Year string `json:"year"` // 年份,负数表示公元前(如 "-30")
|
||||
Title string `json:"title"` // 标题(含 HTML 链接标签)
|
||||
Festival string `json:"festival"` // 节假日名(当天所有条目共享,可能为空)
|
||||
Link string `json:"link"` // 百科词条链接
|
||||
Type string `json:"type"` // event / birth / death
|
||||
Desc string `json:"desc"` // 描述(含 HTML 标签)
|
||||
}
|
||||
|
||||
// fetchMonth 抓取某月「历史上的今天」数据并解析为按天的条目(失败重试)
|
||||
func (s *WikiSyncService) fetchMonth(month int) ([]model.WikiOnThisDay, error) {
|
||||
apiURL := fmt.Sprintf("https://baike.baidu.com/cms/home/eventsOnHistory/%02d.json", month)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 4; attempt++ {
|
||||
if attempt > 0 {
|
||||
backoff := time.Duration(attempt*3) * time.Second
|
||||
if lastErr != nil && strings.Contains(lastErr.Error(), "429") {
|
||||
backoff = time.Duration(attempt*30) * time.Second // 限流时退避 30s/60s/90s
|
||||
}
|
||||
time.Sleep(backoff)
|
||||
time.Sleep(time.Duration(attempt*3) * time.Second)
|
||||
}
|
||||
text, err := s.doFetch(apiURL)
|
||||
if err == nil {
|
||||
return text, nil
|
||||
body, err := s.doFetch(apiURL)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
lastErr = err
|
||||
items, err := parseBaikeMonth(body, month)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
return "", lastErr
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func (s *WikiSyncService) doFetch(apiURL string) (string, error) {
|
||||
// doFetch 发起 HTTP 请求并返回响应体(带浏览器 UA 与 Referer,规避安全验证)
|
||||
func (s *WikiSyncService) doFetch(apiURL string) ([]byte, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "LunarServer/1.0 (wiki on-this-day sync; contact: admin@neatcn.com)")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
||||
req.Header.Set("Referer", "https://baike.baidu.com/calendar")
|
||||
req.Header.Set("Accept", "application/json, text/plain, */*")
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Parse struct {
|
||||
Wikitext string `json:"wikitext"`
|
||||
} `json:"parse"`
|
||||
Error struct {
|
||||
Info string `json:"info"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return "", fmt.Errorf("JSON 解析失败: %w", err)
|
||||
}
|
||||
if result.Error.Info != "" {
|
||||
return "", fmt.Errorf("API 错误: %s", result.Error.Info)
|
||||
}
|
||||
if result.Parse.Wikitext == "" {
|
||||
return "", fmt.Errorf("页面内容为空")
|
||||
}
|
||||
return result.Parse.Wikitext, nil
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) // 上限 4MB
|
||||
}
|
||||
|
||||
// ===== wikitext 解析 =====
|
||||
// parseBaikeMonth 解析百度百科某月 JSON,输出该月全部条目
|
||||
func parseBaikeMonth(body []byte, month int) ([]model.WikiOnThisDay, error) {
|
||||
// 顶层结构:{"08": {"0801": [...], "0802": [...]}}
|
||||
var root map[string]map[string][]baikeEvent
|
||||
if err := json.Unmarshal(body, &root); err != nil {
|
||||
return nil, fmt.Errorf("JSON 解析失败: %w", err)
|
||||
}
|
||||
monthKey := fmt.Sprintf("%02d", month)
|
||||
dayMap, ok := root[monthKey]
|
||||
if !ok || len(dayMap) == 0 {
|
||||
return nil, fmt.Errorf("%s月无数据", monthKey)
|
||||
}
|
||||
|
||||
// classifySection 根据二级标题识别板块类型
|
||||
func classifySection(title string) string {
|
||||
switch {
|
||||
case strings.Contains(title, "大事") || strings.Contains(title, "事件"):
|
||||
items := make([]model.WikiOnThisDay, 0, 256)
|
||||
for dayKeyStr, events := range dayMap {
|
||||
day, err := strconv.Atoi(dayKeyStr[2:]) // "0812" -> 12
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
seenFestival := make(map[string]bool)
|
||||
seenContent := make(map[string]bool)
|
||||
|
||||
for _, ev := range events {
|
||||
// 节假日:从 festival 字段提取(挂在每条上,去重后单独成 festival 条目)
|
||||
if fest := cleanHTMLText(ev.Festival); fest != "" && !seenFestival[fest] {
|
||||
seenFestival[fest] = true
|
||||
items = append(items, model.WikiOnThisDay{
|
||||
Month: month, Day: day, Kind: "festival", Year: 0, Content: fest, Status: 1,
|
||||
})
|
||||
}
|
||||
|
||||
kind := normalizeKind(ev.Type)
|
||||
if kind == "" {
|
||||
continue
|
||||
}
|
||||
text := cleanHTMLText(ev.Title)
|
||||
if len([]rune(text)) < 4 {
|
||||
continue
|
||||
}
|
||||
if len([]rune(text)) > 500 {
|
||||
text = string([]rune(text)[:500])
|
||||
}
|
||||
if seenContent[kind+"|"+text] {
|
||||
continue
|
||||
}
|
||||
seenContent[kind+"|"+text] = true
|
||||
|
||||
items = append(items, model.WikiOnThisDay{
|
||||
Month: month,
|
||||
Day: day,
|
||||
Kind: kind,
|
||||
Year: parseBaikeYear(ev.Year),
|
||||
Content: text,
|
||||
Status: 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// normalizeKind 映射百度百科 type 到内部 kind
|
||||
func normalizeKind(t string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(t)) {
|
||||
case "event":
|
||||
return "event"
|
||||
case strings.Contains(title, "出生"):
|
||||
case "birth":
|
||||
return "birth"
|
||||
case strings.Contains(title, "逝世") || strings.Contains(title, "去世"):
|
||||
case "death":
|
||||
return "death"
|
||||
case strings.Contains(title, "节假日") || strings.Contains(title, "节日") ||
|
||||
strings.Contains(title, "節日") || strings.Contains(title, "假日") ||
|
||||
strings.Contains(title, "习俗") || strings.Contains(title, "風俗") ||
|
||||
strings.Contains(title, "风俗"):
|
||||
return "festival"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseDayPage 解析页面 wikitext,提取大事记/出生/逝世/节假日条目
|
||||
func parseDayPage(wikitext string) []model.WikiOnThisDay {
|
||||
items := make([]model.WikiOnThisDay, 0, 128)
|
||||
seen := make(map[string]bool) // 同板块去重(维基页面偶有重复条目)
|
||||
|
||||
curKind := ""
|
||||
inTable := false
|
||||
for _, rawLine := range strings.Split(wikitext, "\n") {
|
||||
line := strings.TrimSpace(rawLine)
|
||||
|
||||
// 跳过表格区块
|
||||
if strings.HasPrefix(line, "{|") {
|
||||
inTable = true
|
||||
continue
|
||||
}
|
||||
if inTable {
|
||||
if strings.HasPrefix(line, "|}") {
|
||||
inTable = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// 二级/三级标题切换板块(三级标题归属当前二级板块)
|
||||
if m := reHeading.FindStringSubmatch(line); m != nil {
|
||||
if strings.HasPrefix(line, "=== ") || strings.HasPrefix(line, "===") {
|
||||
continue // 三级标题:大事记内的世纪分组,不改变板块
|
||||
}
|
||||
curKind = classifySection(m[1])
|
||||
continue
|
||||
}
|
||||
|
||||
if curKind == "" || !strings.HasPrefix(line, "*") || strings.HasPrefix(line, "*>") {
|
||||
continue
|
||||
}
|
||||
|
||||
content := strings.TrimSpace(strings.TrimLeft(line, "*"))
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
year, text := extractYear(cleanWikitext(content))
|
||||
if len([]rune(text)) < 4 { // 过短条目无展示价值
|
||||
continue
|
||||
}
|
||||
if len([]rune(text)) > 500 {
|
||||
text = string([]rune(text)[:500])
|
||||
}
|
||||
|
||||
dedupeKey := curKind + "|" + text
|
||||
if seen[dedupeKey] {
|
||||
continue
|
||||
}
|
||||
seen[dedupeKey] = true
|
||||
|
||||
items = append(items, model.WikiOnThisDay{
|
||||
Kind: curKind,
|
||||
Year: year,
|
||||
Content: text,
|
||||
})
|
||||
// parseBaikeYear 解析年份字符串:"1912"→1912,"-30"→-30(公元前),非法→0
|
||||
func parseBaikeYear(s string) int {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
return items
|
||||
y, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
||||
// extractYear 从条目文本提取年份:前612年→-612,1912年→1912,无年份→0
|
||||
func extractYear(text string) (int, string) {
|
||||
text = strings.TrimSpace(text)
|
||||
if m := reYearPrefix.FindStringSubmatch(text); m != nil {
|
||||
year, err := strconv.Atoi(m[2])
|
||||
if err == nil {
|
||||
if m[1] == "前" {
|
||||
year = -year
|
||||
}
|
||||
return year, strings.TrimSpace(text[len(m[0]):])
|
||||
}
|
||||
}
|
||||
if m := reNoYear.FindStringSubmatch(text); m != nil {
|
||||
return 0, strings.TrimSpace(text[len(m[0]):])
|
||||
}
|
||||
return 0, text
|
||||
}
|
||||
|
||||
// convertConvMarkup 处理语言转换标记 -{zh-cn:A;zh-tw:B}-,优先取简体变体
|
||||
func convertConvMarkup(s string) string {
|
||||
return reConv.ReplaceAllStringFunc(s, func(m string) string {
|
||||
inner := m[2 : len(m)-2] // 去掉 -{ 和 }-
|
||||
parts := strings.Split(inner, ";")
|
||||
// 无冒号:-{A}- 直接取内容
|
||||
if !strings.Contains(inner, ":") {
|
||||
return strings.TrimSpace(inner)
|
||||
}
|
||||
fallback := ""
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
kv := strings.SplitN(p, ":", 2)
|
||||
if len(kv) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(strings.TrimSpace(kv[0]))
|
||||
val := strings.TrimSpace(kv[1])
|
||||
if key == "zh-cn" || key == "zh-hans" {
|
||||
return val
|
||||
}
|
||||
fallback = val
|
||||
}
|
||||
return fallback
|
||||
})
|
||||
}
|
||||
|
||||
// cleanWikitext 去除 wiki 语法,输出纯文本
|
||||
func cleanWikitext(s string) string {
|
||||
s = reComment.ReplaceAllString(s, "")
|
||||
s = reRef.ReplaceAllString(s, "")
|
||||
s = convertConvMarkup(s)
|
||||
// 国旗模板保留国家名
|
||||
s = reFlag.ReplaceAllString(s, "$1")
|
||||
// 模板可能嵌套,循环剥离直至稳定
|
||||
for i := 0; i < 6; i++ {
|
||||
next := reTemplate.ReplaceAllString(s, "")
|
||||
if next == s {
|
||||
break
|
||||
}
|
||||
s = next
|
||||
}
|
||||
s = reLink.ReplaceAllString(s, "$1")
|
||||
s = strings.ReplaceAll(s, "'''", "")
|
||||
s = strings.ReplaceAll(s, "''", "")
|
||||
// cleanHTMLText 去除 HTML 标签与多余空白,输出纯文本
|
||||
func cleanHTMLText(s string) string {
|
||||
s = reHTMLTag.ReplaceAllString(s, "")
|
||||
s = strings.ReplaceAll(s, " ", " ")
|
||||
s = strings.TrimSpace(s)
|
||||
s = reLeadPunct.ReplaceAllString(s, "") // 模板被剥离后可能残留开头标点
|
||||
s = reSpace.ReplaceAllString(s, " ")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
@@ -1,43 +1,31 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 取自 zh.wikipedia.org「8月10日」页面的真实片段
|
||||
const sampleWikitext = `'''8月10日'''是[[阳历]]年的第222天。
|
||||
// 取自百度百科 /cms/home/eventsOnHistory/08.json 的真实片段(8月12日/13日)
|
||||
const sampleBaikeJSON = `{"08":{
|
||||
"0812":[
|
||||
{"year":"-30","title":"埃及艳后<a target=\"_blank\" href=\"https://baike.baidu.com/item/x\">克利奥帕特拉七世</a>逝世","festival":"","link":"https://baike.baidu.com/item/x","type":"death","desc":"克利奥帕特拉七世"},
|
||||
{"year":"1848","title":"英国著名小说家<a target=\"_blank\" href=\"x\">乔治·艾略特</a>出生","festival":"","link":"x","type":"birth","desc":"x"},
|
||||
{"year":"1848","title":"英国著名小说家乔治·艾略特出生","festival":"","link":"x","type":"birth","desc":"x"},
|
||||
{"year":"1905","title":"挪威音乐家<a href=\"x\">某</a>指挥首演","festival":"","link":"x","type":"event","desc":"x"}
|
||||
],
|
||||
"0813":[
|
||||
{"year":"604","title":"隋文帝<a href=\"x\">杨坚</a>逝世","festival":"国际左撇子日","link":"x","type":"death","desc":"x"},
|
||||
{"year":"1926","title":"古巴革命领导人<a href=\"x\">卡斯特罗</a>出生","festival":"国际左撇子日","link":"x","type":"birth","desc":"x"},
|
||||
{"year":"1910","title":"某事件<a href=\"x\">发生</a>","festival":"国际左撇子日","link":"x","type":"event","desc":"x"},
|
||||
{"year":"abc","title":"年份非法的<a href=\"x\">事件</a>条目","festival":"国际左撇子日","link":"x","type":"event","desc":"x"}
|
||||
]
|
||||
}}`
|
||||
|
||||
== 大事记 ==
|
||||
=== 19世紀以前 ===
|
||||
* [[前612年]]:[[亚述]]最后一位君主[[辛·沙里施昆]]死于[[尼尼微]],王国灭亡。
|
||||
* [[955年]]:[[鄂圖一世 (神聖羅馬帝國)|鄂圖一世]]麾下的軍隊在[[第二次萊希菲爾德之戰 (955)|萊希菲爾德之戰]]中取得勝利。<ref>注释</ref>
|
||||
=== 20世紀 ===
|
||||
* [[1912年]]:[[威廉·布利斯]]出版了第一本关于[[童子军]]的书。
|
||||
* [[1945年]]:{{le|蒙古}}对日宣战。
|
||||
* [[2002年]]:-{zh-hans:凯尔·斯科特;zh-hk:卡爾·史葛;zh-tw:凱爾·斯科特;}-,英國職業足球運動員。
|
||||
* 年份不详:某件无法确定年份的事情发生了。
|
||||
|
||||
== 出生 ==
|
||||
* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
|
||||
* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
|
||||
* [[1874年]]:[[赫伯特·胡佛]],美国第31任[[总统]]([[1964年]]逝世)
|
||||
|
||||
== 逝世 ==
|
||||
* [[258年]]:[[羅馬的聖老楞佐]],基督教殉道者
|
||||
|
||||
== 节假日和习俗 ==
|
||||
* {{flag|厄瓜多尔}}:独立日
|
||||
* 世界[[狮子]]日
|
||||
|
||||
==参考资料==
|
||||
{{Commonscat}}
|
||||
`
|
||||
|
||||
func TestParseDayPage(t *testing.T) {
|
||||
items := parseDayPage(sampleWikitext)
|
||||
func TestParseBaikeMonth(t *testing.T) {
|
||||
items, err := parseBaikeMonth([]byte(sampleBaikeJSON), 8)
|
||||
if err != nil {
|
||||
t.Fatalf("parseBaikeMonth 失败: %v", err)
|
||||
}
|
||||
|
||||
count := func(kind string) int {
|
||||
n := 0
|
||||
@@ -49,65 +37,78 @@ func TestParseDayPage(t *testing.T) {
|
||||
return n
|
||||
}
|
||||
|
||||
if got := count("event"); got != 6 {
|
||||
t.Errorf("event 条目数 = %d, 期望 6", got)
|
||||
// 0812: 1 death + 2 birth(重复去重后1) + 1 event = 3;0813: 1 death + 1 birth + 2 event = 4
|
||||
if got := count("death"); got != 2 {
|
||||
t.Errorf("death 条目数 = %d, 期望 2", got)
|
||||
}
|
||||
if got := count("birth"); got != 2 { // 重复条目应去重
|
||||
if got := count("birth"); got != 2 { // 0812 重复出生条目应去重
|
||||
t.Errorf("birth 条目数 = %d, 期望 2", got)
|
||||
}
|
||||
if got := count("death"); got != 1 {
|
||||
t.Errorf("death 条目数 = %d, 期望 1", got)
|
||||
if got := count("event"); got != 3 {
|
||||
t.Errorf("event 条目数 = %d, 期望 3", got)
|
||||
}
|
||||
if got := count("festival"); got != 2 {
|
||||
t.Errorf("festival 条目数 = %d, 期望 2", got)
|
||||
// 国际左撇子日只在 0813,去重后 1 条 festival
|
||||
if got := count("festival"); got != 1 {
|
||||
t.Errorf("festival 条目数 = %d, 期望 1", got)
|
||||
}
|
||||
|
||||
// 校验具体条目的年份与文本清洗
|
||||
assertItem := func(kind string, wantYear int, wantContent string) {
|
||||
// 校验具体条目
|
||||
assertItem := func(kind string, day, wantYear int, wantContent string) {
|
||||
for _, it := range items {
|
||||
if it.Kind == kind && it.Content == wantContent {
|
||||
if it.Kind == kind && it.Day == day && it.Content == wantContent {
|
||||
if it.Year != wantYear {
|
||||
t.Errorf("%q 年份 = %d, 期望 %d", wantContent, it.Year, wantYear)
|
||||
}
|
||||
if it.Month != 8 {
|
||||
t.Errorf("%q 月份 = %d, 期望 8", wantContent, it.Month)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("未找到条目: kind=%s content=%q", kind, wantContent)
|
||||
t.Errorf("未找到条目: kind=%s day=%d content=%q", kind, day, wantContent)
|
||||
}
|
||||
assertItem("event", -612, "亚述最后一位君主辛·沙里施昆死于尼尼微,王国灭亡。")
|
||||
assertItem("event", 955, "鄂圖一世麾下的軍隊在萊希菲爾德之戰中取得勝利。")
|
||||
assertItem("event", 0, "某件无法确定年份的事情发生了。")
|
||||
assertItem("event", 2002, "凯尔·斯科特,英國職業足球運動員。")
|
||||
assertItem("festival", 0, "厄瓜多尔:独立日")
|
||||
assertItem("birth", -156, "汉武帝刘彻,西汉皇帝(前87年逝世)")
|
||||
assertItem("festival", 0, "世界狮子日")
|
||||
assertItem("death", 12, -30, "埃及艳后克利奥帕特拉七世逝世")
|
||||
assertItem("birth", 12, 1848, "英国著名小说家乔治·艾略特出生")
|
||||
assertItem("event", 13, 0, "年份非法的事件条目") // 非法年份应为 0
|
||||
assertItem("festival", 13, 0, "国际左撇子日")
|
||||
|
||||
// 不应包含 wiki 语法残留
|
||||
// 不应残留 HTML 标签
|
||||
for _, it := range items {
|
||||
if strings.Contains(it.Content, "[[") || strings.Contains(it.Content, "{{") ||
|
||||
strings.Contains(it.Content, "<ref") {
|
||||
t.Errorf("条目残留 wiki 语法: %q", it.Content)
|
||||
if strings.Contains(it.Content, "<a") || strings.Contains(it.Content, "href") ||
|
||||
strings.Contains(it.Content, "</a>") {
|
||||
t.Errorf("条目残留 HTML 标签: %q", it.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseRealPage 若本地存在真实抓取的样本文件则验证(可选)
|
||||
func TestParseRealPage(t *testing.T) {
|
||||
data, err := os.ReadFile("/tmp/wiki_810.json")
|
||||
if err != nil {
|
||||
t.Skip("无真实样本文件,跳过")
|
||||
func TestParseBaikeYear(t *testing.T) {
|
||||
cases := map[string]int{
|
||||
"1912": 1912,
|
||||
"-30": -30,
|
||||
"0": 0,
|
||||
"abc": 0,
|
||||
"": 0,
|
||||
" 77 ": 77,
|
||||
}
|
||||
var d struct {
|
||||
Parse struct {
|
||||
Wikitext string `json:"wikitext"`
|
||||
} `json:"parse"`
|
||||
for in, want := range cases {
|
||||
if got := parseBaikeYear(in); got != want {
|
||||
t.Errorf("parseBaikeYear(%q) = %d, 期望 %d", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeKind(t *testing.T) {
|
||||
if normalizeKind("event") != "event" || normalizeKind("birth") != "birth" ||
|
||||
normalizeKind("death") != "death" || normalizeKind("other") != "" ||
|
||||
normalizeKind(" EVENT ") != "event" {
|
||||
t.Error("normalizeKind 映射异常")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanHTMLText(t *testing.T) {
|
||||
in := `埃及艳后<a target="_blank" href="x">克利奥帕特拉七世</a> 逝世 test`
|
||||
want := "埃及艳后克利奥帕特拉七世 逝世 test"
|
||||
if got := cleanHTMLText(in); got != want {
|
||||
t.Errorf("cleanHTMLText = %q, 期望 %q", got, want)
|
||||
}
|
||||
if err := json.Unmarshal(data, &d); err != nil {
|
||||
t.Fatalf("样本解析失败: %v", err)
|
||||
}
|
||||
items := parseDayPage(d.Parse.Wikitext)
|
||||
if len(items) < 20 {
|
||||
t.Errorf("真实页面解析条目过少: %d", len(items))
|
||||
}
|
||||
t.Logf("真实页面共解析 %d 条", len(items))
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ export default {
|
||||
<div class="text-center mb-6">
|
||||
<h1 class="text-2xl font-bold text-red-600">祈福小助手</h1>
|
||||
<p class="text-gray-500 text-sm mt-1">管理后台登录</p>
|
||||
<p class="text-gray-300 text-xs mt-2">v2026.08.12-login</p>
|
||||
</div>
|
||||
<form @submit.prevent="submit">
|
||||
<div class="mb-4">
|
||||
|
||||
Reference in New Issue
Block a user