package service import ( "encoding/json" "fmt" "io" "log" "net/http" "net/url" "regexp" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/gouki/lunar-server/internal/model" "gorm.io/gorm" ) // WikiSyncService 维基百科「X月X日」页面同步服务 // 全量共 366 页(含 2 月 29 日),首次抓取后按配置间隔定期增量刷新 type WikiSyncService struct { db *gorm.DB running int32 // 原子标记:是否有同步正在执行 httpClient *http.Client } // NewWikiSyncService 创建同步服务 func NewWikiSyncService(db *gorm.DB) *WikiSyncService { return &WikiSyncService{ db: db, httpClient: &http.Client{ Timeout: 30 * time.Second, }, } } var ( reComment = regexp.MustCompile(`(?s)`) reRef = regexp.MustCompile(`(?is)]*/>|]*>.*?`) reFlag = regexp.MustCompile(`\{\{(?:flag|flagcountry|flagicon)\|([^}|]+)(?:\|[^{}]*)?\}\}`) reTemplate = regexp.MustCompile(`\{\{[^{}]*\}\}`) reLink = regexp.MustCompile(`\[\[(?:[^|\[\]]*\|)?([^|\[\]]+)\]\]`) reHTMLTag = regexp.MustCompile(`]*>`) reHeading = regexp.MustCompile(`^==+\s*(.*?)\s*==+\s*$`) // 语言转换标记 -{zh-cn:A;zh-tw:B}- 或 -{A}- reConv = regexp.MustCompile(`-\{([^{}]*)\}-`) // 条目开头的年份:前612年: / 1912年: / 1980年代: reYearPrefix = regexp.MustCompile(`^(前)?(\d{1,4})\s*年代?\s*[::]\s*`) reNoYear = regexp.MustCompile(`^(年份不详|年份不詳|不详|不詳|生年不详|生年不詳)\s*[::]\s*`) reLeadPunct = regexp.MustCompile(`^[::,,、;;\.\s]+`) ) // daysInMonth 每月天数(含 2 月 29 日,闰日页面维基也有) var daysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} type dayKey struct { Month int Day int } // allDays 全年 366 天 func allDays() []dayKey { days := make([]dayKey, 0, 366) for m := 1; m <= 12; m++ { for d := 1; d <= daysInMonth[m-1]; d++ { days = append(days, dayKey{Month: m, Day: d}) } } return days } // IsRunning 是否有同步任务正在执行 func (s *WikiSyncService) IsRunning() bool { return atomic.LoadInt32(&s.running) == 1 } // coveredDaySet 已覆盖的月日集合 func (s *WikiSyncService) coveredDaySet() map[dayKey]bool { set := make(map[dayKey]bool, 400) type row struct { Month int Day int } var rows []row s.db.Model(&model.WikiOnThisDay{}).Where("status = 1"). Select("DISTINCT month, day").Scan(&rows) for _, r := range rows { set[dayKey{Month: r.Month, Day: r.Day}] = true } return set } // missingDays 尚未覆盖的天 func (s *WikiSyncService) missingDays() []dayKey { covered := s.coveredDaySet() missing := make([]dayKey, 0) for _, d := range allDays() { if !covered[d] { missing = append(missing, d) } } return missing } // TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false // full=true 全量刷新 366 页;full=false 仅补齐缺失的天 func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool { days := allDays() if !full { days = s.missingDays() if len(days) == 0 { return false // 无缺失,无需同步 } } if !atomic.CompareAndSwapInt32(&s.running, 0, 1) { return false } go func() { defer atomic.StoreInt32(&s.running, 0) s.runSync(trigger, days) }() return true } // StartScheduler 启动定时同步任务: // - 启动时若数据库为空,延迟 10 秒后自动执行首次全量抓取 // - 之后每天在 syncHour 小时点检查,距上次成功超过 intervalDays 天才执行 func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) { go func() { if missing := s.missingDays(); len(missing) > 0 { // 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐 log.Printf("wiki sync: 缺失 %d/366 天,10秒后自动补齐", len(missing)) time.Sleep(10 * time.Second) s.TriggerSync("startup", false) } if intervalDays <= 0 { intervalDays = 7 } if syncHour < 0 || syncHour > 23 { syncHour = 4 } ticker := time.NewTicker(time.Hour) defer ticker.Stop() for range ticker.C { now := time.Now() if now.Hour() != syncHour { continue } if last, ok := s.lastSuccessAt(); ok && time.Since(last) < time.Duration(intervalDays)*24*time.Hour { continue // 未到刷新周期 } log.Println("wiki sync: 定时任务触发全量同步") s.TriggerSync("cron", true) } }() } // lastSuccessAt 最近一次成功(含部分成功)的同步完成时间 func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) { var syncLog model.WikiSyncLog err := s.db.Where("status IN ?", []string{"success", "partial"}). Order("finished_at DESC").First(&syncLog).Error if err != nil || syncLog.FinishedAt == nil { return time.Time{}, false } return *syncLog.FinishedAt, true } // runSync 同步指定日期集合(全量 366 页或缺失补齐) func (s *WikiSyncService) runSync(trigger string, days []dayKey) { syncLog := model.WikiSyncLog{ Trigger: trigger, Status: "running", Pages: len(days), StartedAt: time.Now(), } s.db.Create(&syncLog) jobs := make(chan dayKey, len(days)) for _, d := range days { jobs <- d } close(jobs) var success, failed int32 var wg sync.WaitGroup workerCount := 3 for i := 0; i < workerCount; i++ { wg.Add(1) go func() { defer wg.Done() for d := range jobs { if err := s.syncOneDay(d.Month, d.Day); err != nil { atomic.AddInt32(&failed, 1) log.Printf("wiki sync: %d月%d日 失败: %v", d.Month, d.Day, err) } else { atomic.AddInt32(&success, 1) } time.Sleep(800 * time.Millisecond) // 控制请求频率,避免被限流 } }() } wg.Wait() status := "success" if failed > 0 { status = "partial" } if success == 0 { status = "failed" } now := time.Now() s.db.Model(&syncLog).Updates(map[string]interface{}{ "status": status, "success": int(success), "failed": int(failed), "finished_at": &now, }) log.Printf("wiki sync: 同步完成 trigger=%s pages=%d success=%d failed=%d", trigger, len(days), success, failed) } // syncOneDay 抓取并更新某一天的数据(失败不覆盖旧数据;死锁自动重试) func (s *WikiSyncService) syncOneDay(month, day int) error { wikitext, err := s.fetchDayPage(month, day) if err != nil { return err } items := parseDayPage(wikitext) if len(items) == 0 { return fmt.Errorf("页面解析结果为空") } for i := range items { items[i].Month = month items[i].Day = day items[i].Status = 1 } // 按天整体替换:先删后插;并发写入同表可能死锁,重试 3 次 var lastErr error for attempt := 0; attempt < 3; attempt++ { if attempt > 0 { time.Sleep(time.Duration(attempt) * time.Second) } lastErr = s.db.Transaction(func(tx *gorm.DB) error { if err := tx.Where("month = ? AND day = ?", month, day). Delete(&model.WikiOnThisDay{}).Error; err != nil { return err } return tx.Create(&items).Error }) if lastErr == nil { return nil } if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") { break } } return lastErr } // fetchDayPage 抓取「X月X日」页面的 wikitext(失败重试;429 限流时加大退避间隔) func (s *WikiSyncService) fetchDayPage(month, day int) (string, error) { title := fmt.Sprintf("%d月%d日", month, day) apiURL := "https://zh.wikipedia.org/w/api.php?action=parse&prop=wikitext&format=json&formatversion=2&page=" + url.QueryEscape(title) var lastErr error for attempt := 0; attempt < 4; attempt++ { if attempt > 0 { backoff := time.Duration(attempt*3) * time.Second if lastErr != nil && strings.Contains(lastErr.Error(), "429") { backoff = time.Duration(attempt*30) * time.Second // 限流时退避 30s/60s/90s } time.Sleep(backoff) } text, err := s.doFetch(apiURL) if err == nil { return text, nil } lastErr = err } return "", lastErr } func (s *WikiSyncService) doFetch(apiURL string) (string, error) { req, err := http.NewRequest(http.MethodGet, apiURL, nil) if err != nil { return "", err } req.Header.Set("User-Agent", "LunarServer/1.0 (wiki on-this-day sync; contact: admin@neatcn.com)") resp, err := s.httpClient.Do(req) if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP %d", resp.StatusCode) } body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB if err != nil { return "", err } var result struct { Parse struct { Wikitext string `json:"wikitext"` } `json:"parse"` Error struct { Info string `json:"info"` } `json:"error"` } if err := json.Unmarshal(body, &result); err != nil { return "", fmt.Errorf("JSON 解析失败: %w", err) } if result.Error.Info != "" { return "", fmt.Errorf("API 错误: %s", result.Error.Info) } if result.Parse.Wikitext == "" { return "", fmt.Errorf("页面内容为空") } return result.Parse.Wikitext, nil } // ===== wikitext 解析 ===== // classifySection 根据二级标题识别板块类型 func classifySection(title string) string { switch { case strings.Contains(title, "大事") || strings.Contains(title, "事件"): return "event" case strings.Contains(title, "出生"): return "birth" case strings.Contains(title, "逝世") || strings.Contains(title, "去世"): return "death" case strings.Contains(title, "节假日") || strings.Contains(title, "节日") || strings.Contains(title, "節日") || strings.Contains(title, "假日") || strings.Contains(title, "习俗") || strings.Contains(title, "風俗") || strings.Contains(title, "风俗"): return "festival" } return "" } // parseDayPage 解析页面 wikitext,提取大事记/出生/逝世/节假日条目 func parseDayPage(wikitext string) []model.WikiOnThisDay { items := make([]model.WikiOnThisDay, 0, 128) seen := make(map[string]bool) // 同板块去重(维基页面偶有重复条目) curKind := "" inTable := false for _, rawLine := range strings.Split(wikitext, "\n") { line := strings.TrimSpace(rawLine) // 跳过表格区块 if strings.HasPrefix(line, "{|") { inTable = true continue } if inTable { if strings.HasPrefix(line, "|}") { inTable = false } continue } // 二级/三级标题切换板块(三级标题归属当前二级板块) if m := reHeading.FindStringSubmatch(line); m != nil { if strings.HasPrefix(line, "=== ") || strings.HasPrefix(line, "===") { continue // 三级标题:大事记内的世纪分组,不改变板块 } curKind = classifySection(m[1]) continue } if curKind == "" || !strings.HasPrefix(line, "*") || strings.HasPrefix(line, "*>") { continue } content := strings.TrimSpace(strings.TrimLeft(line, "*")) if content == "" { continue } year, text := extractYear(cleanWikitext(content)) if len([]rune(text)) < 4 { // 过短条目无展示价值 continue } if len([]rune(text)) > 500 { text = string([]rune(text)[:500]) } dedupeKey := curKind + "|" + text if seen[dedupeKey] { continue } seen[dedupeKey] = true items = append(items, model.WikiOnThisDay{ Kind: curKind, Year: year, Content: text, }) } return items } // extractYear 从条目文本提取年份:前612年→-612,1912年→1912,无年份→0 func extractYear(text string) (int, string) { text = strings.TrimSpace(text) if m := reYearPrefix.FindStringSubmatch(text); m != nil { year, err := strconv.Atoi(m[2]) if err == nil { if m[1] == "前" { year = -year } return year, strings.TrimSpace(text[len(m[0]):]) } } if m := reNoYear.FindStringSubmatch(text); m != nil { return 0, strings.TrimSpace(text[len(m[0]):]) } return 0, text } // convertConvMarkup 处理语言转换标记 -{zh-cn:A;zh-tw:B}-,优先取简体变体 func convertConvMarkup(s string) string { return reConv.ReplaceAllStringFunc(s, func(m string) string { inner := m[2 : len(m)-2] // 去掉 -{ 和 }- parts := strings.Split(inner, ";") // 无冒号:-{A}- 直接取内容 if !strings.Contains(inner, ":") { return strings.TrimSpace(inner) } fallback := "" for _, p := range parts { p = strings.TrimSpace(p) if p == "" { continue } kv := strings.SplitN(p, ":", 2) if len(kv) != 2 { continue } key := strings.ToLower(strings.TrimSpace(kv[0])) val := strings.TrimSpace(kv[1]) if key == "zh-cn" || key == "zh-hans" { return val } fallback = val } return fallback }) } // cleanWikitext 去除 wiki 语法,输出纯文本 func cleanWikitext(s string) string { s = reComment.ReplaceAllString(s, "") s = reRef.ReplaceAllString(s, "") s = convertConvMarkup(s) // 国旗模板保留国家名 s = reFlag.ReplaceAllString(s, "$1") // 模板可能嵌套,循环剥离直至稳定 for i := 0; i < 6; i++ { next := reTemplate.ReplaceAllString(s, "") if next == s { break } s = next } s = reLink.ReplaceAllString(s, "$1") s = strings.ReplaceAll(s, "'''", "") s = strings.ReplaceAll(s, "''", "") s = reHTMLTag.ReplaceAllString(s, "") s = strings.ReplaceAll(s, " ", " ") s = strings.TrimSpace(s) s = reLeadPunct.ReplaceAllString(s, "") // 模板被剥离后可能残留开头标点 return strings.TrimSpace(s) }