package service import ( "encoding/json" "fmt" "io" "log" "net/http" "regexp" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/gouki/lunar-server/internal/model" "gorm.io/gorm" ) // WikiSyncService 「历史上的今天」同步服务 // 数据源:百度百科「历史上的今天」开放接口(按月返回结构化 JSON,共 12 次请求) // 首次抓取后按配置间隔定期增量刷新 type WikiSyncService struct { db *gorm.DB running int32 // 原子标记:是否有同步正在执行 httpClient *http.Client } // NewWikiSyncService 创建同步服务 func NewWikiSyncService(db *gorm.DB) *WikiSyncService { return &WikiSyncService{ db: db, httpClient: &http.Client{ Timeout: 30 * time.Second, }, } } var ( reHTMLTag = regexp.MustCompile(`]*>`) reSpace = regexp.MustCompile(`\s+`) ) // daysInMonth 每月天数(百度百科数据源不含 2 月 29 日,共 365 天) var daysInMonth = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} type dayKey struct { Month int Day int } // allDays 全年 365 天 func allDays() []dayKey { days := make([]dayKey, 0, 365) for m := 1; m <= 12; m++ { for d := 1; d <= daysInMonth[m-1]; d++ { days = append(days, dayKey{Month: m, Day: d}) } } return days } // IsRunning 是否有同步任务正在执行 func (s *WikiSyncService) IsRunning() bool { return atomic.LoadInt32(&s.running) == 1 } // coveredDaySet 已覆盖的月日集合 func (s *WikiSyncService) coveredDaySet() map[dayKey]bool { set := make(map[dayKey]bool, 400) type row struct { Month int Day int } var rows []row s.db.Model(&model.WikiOnThisDay{}).Where("status = 1"). Select("DISTINCT month, day").Scan(&rows) for _, r := range rows { set[dayKey{Month: r.Month, Day: r.Day}] = true } return set } // missingDays 尚未覆盖的天 func (s *WikiSyncService) missingDays() []dayKey { covered := s.coveredDaySet() missing := make([]dayKey, 0) for _, d := range allDays() { if !covered[d] { missing = append(missing, d) } } return missing } // TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false // full=true 全量刷新 365 天;full=false 仅补齐缺失的天 func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool { days := allDays() if !full { days = s.missingDays() if len(days) == 0 { return false // 无缺失,无需同步 } } if !atomic.CompareAndSwapInt32(&s.running, 0, 1) { return false } go func() { defer atomic.StoreInt32(&s.running, 0) s.runSync(trigger, days) }() return true } // StartScheduler 启动定时同步任务: // - 启动时若数据库为空,延迟 10 秒后自动执行首次全量抓取 // - 之后每天在 syncHour 小时点检查,距上次成功超过 intervalDays 天才执行 func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) { go func() { if missing := s.missingDays(); len(missing) > 0 { // 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐 log.Printf("wiki sync: 缺失 %d/365 天,10秒后自动补齐", len(missing)) time.Sleep(10 * time.Second) s.TriggerSync("startup", false) } if intervalDays <= 0 { intervalDays = 7 } if syncHour < 0 || syncHour > 23 { syncHour = 4 } ticker := time.NewTicker(time.Hour) defer ticker.Stop() for range ticker.C { now := time.Now() if now.Hour() != syncHour { continue } if last, ok := s.lastSuccessAt(); ok && time.Since(last) < time.Duration(intervalDays)*24*time.Hour { continue // 未到刷新周期 } log.Println("wiki sync: 定时任务触发全量同步") s.TriggerSync("cron", true) } }() } // lastSuccessAt 最近一次成功(含部分成功)的同步完成时间 func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) { var syncLog model.WikiSyncLog err := s.db.Where("status IN ?", []string{"success", "partial"}). Order("finished_at DESC").First(&syncLog).Error if err != nil || syncLog.FinishedAt == nil { return time.Time{}, false } return *syncLog.FinishedAt, true } // runSync 同步指定日期集合(全量 365 天或缺失补齐) // 百度百科按月返回数据,故先归并出涉及的月份,每月一次请求,再按天写入 func (s *WikiSyncService) runSync(trigger string, days []dayKey) { syncLog := model.WikiSyncLog{ Trigger: trigger, Status: "running", Pages: len(days), StartedAt: time.Now(), } s.db.Create(&syncLog) // 归并涉及的月份 monthSet := make(map[int]bool) for _, d := range days { monthSet[d.Month] = true } months := make([]int, 0, len(monthSet)) for m := range monthSet { months = append(months, m) } jobs := make(chan int, len(months)) for _, m := range months { jobs <- m } close(jobs) var success, failed int32 var wg sync.WaitGroup workerCount := 2 // 百度百科反爬较敏感,低并发 + 间隔 for i := 0; i < workerCount; i++ { wg.Add(1) go func() { defer wg.Done() for m := range jobs { if err := s.syncOneMonth(m); err != nil { atomic.AddInt32(&failed, 1) log.Printf("wiki sync: %d月 失败: %v", m, err) } else { atomic.AddInt32(&success, 1) } time.Sleep(1200 * time.Millisecond) // 控制请求频率,避免触发安全验证 } }() } wg.Wait() status := "success" if failed > 0 { status = "partial" } if success == 0 { status = "failed" } now := time.Now() s.db.Model(&syncLog).Updates(map[string]interface{}{ "status": status, "success": int(success), "failed": int(failed), "finished_at": &now, }) log.Printf("wiki sync: 同步完成 trigger=%s days=%d months=%d success=%d failed=%d", trigger, len(days), len(months), success, failed) } // syncOneMonth 抓取某月数据并按天整体替换(死锁自动重试) func (s *WikiSyncService) syncOneMonth(month int) error { items, err := s.fetchMonth(month) if err != nil { return err } if len(items) == 0 { return fmt.Errorf("%d月解析结果为空", month) } // 按天分组 byDay := make(map[int][]model.WikiOnThisDay) for _, it := range items { byDay[it.Day] = append(byDay[it.Day], it) } // 逐天事务替换 for day, dayItems := range byDay { var lastErr error for attempt := 0; attempt < 3; attempt++ { if attempt > 0 { time.Sleep(time.Duration(attempt) * time.Second) } lastErr = s.db.Transaction(func(tx *gorm.DB) error { if err := tx.Where("month = ? AND day = ?", month, day). Delete(&model.WikiOnThisDay{}).Error; err != nil { return err } return tx.Create(&dayItems).Error }) if lastErr == nil { break } if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") { return lastErr } } if lastErr != nil { return fmt.Errorf("%d月%d日写入失败: %w", month, day, lastErr) } } return nil } // ===== 百度百科数据抓取与解析 ===== // baikeEvent 百度百科「历史上的今天」单条数据 type baikeEvent struct { Year string `json:"year"` // 年份,负数表示公元前(如 "-30") Title string `json:"title"` // 标题(含 HTML 链接标签) Festival string `json:"festival"` // 节假日名(当天所有条目共享,可能为空) Link string `json:"link"` // 百科词条链接 Type string `json:"type"` // event / birth / death Desc string `json:"desc"` // 描述(含 HTML 标签) } // fetchMonth 抓取某月「历史上的今天」数据并解析为按天的条目(失败重试) func (s *WikiSyncService) fetchMonth(month int) ([]model.WikiOnThisDay, error) { apiURL := fmt.Sprintf("https://baike.baidu.com/cms/home/eventsOnHistory/%02d.json", month) var lastErr error for attempt := 0; attempt < 4; attempt++ { if attempt > 0 { time.Sleep(time.Duration(attempt*3) * time.Second) } body, err := s.doFetch(apiURL) if err != nil { lastErr = err continue } items, err := parseBaikeMonth(body, month) if err != nil { lastErr = err continue } return items, nil } return nil, lastErr } // doFetch 发起 HTTP 请求并返回响应体(带浏览器 UA 与 Referer,规避安全验证) func (s *WikiSyncService) doFetch(apiURL string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, apiURL, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36") req.Header.Set("Referer", "https://baike.baidu.com/calendar") req.Header.Set("Accept", "application/json, text/plain, */*") resp, err := s.httpClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) // 上限 4MB } // parseBaikeMonth 解析百度百科某月 JSON,输出该月全部条目 func parseBaikeMonth(body []byte, month int) ([]model.WikiOnThisDay, error) { // 顶层结构:{"08": {"0801": [...], "0802": [...]}} var root map[string]map[string][]baikeEvent if err := json.Unmarshal(body, &root); err != nil { return nil, fmt.Errorf("JSON 解析失败: %w", err) } monthKey := fmt.Sprintf("%02d", month) dayMap, ok := root[monthKey] if !ok || len(dayMap) == 0 { return nil, fmt.Errorf("%s月无数据", monthKey) } items := make([]model.WikiOnThisDay, 0, 256) for dayKeyStr, events := range dayMap { day, err := strconv.Atoi(dayKeyStr[2:]) // "0812" -> 12 if err != nil { continue } seenFestival := make(map[string]bool) seenContent := make(map[string]bool) for _, ev := range events { // 节假日:从 festival 字段提取(挂在每条上,去重后单独成 festival 条目) if fest := cleanHTMLText(ev.Festival); fest != "" && !seenFestival[fest] { seenFestival[fest] = true items = append(items, model.WikiOnThisDay{ Month: month, Day: day, Kind: "festival", Year: 0, Content: fest, Status: 1, }) } kind := normalizeKind(ev.Type) if kind == "" { continue } text := cleanHTMLText(ev.Title) if len([]rune(text)) < 4 { continue } if len([]rune(text)) > 500 { text = string([]rune(text)[:500]) } if seenContent[kind+"|"+text] { continue } seenContent[kind+"|"+text] = true items = append(items, model.WikiOnThisDay{ Month: month, Day: day, Kind: kind, Year: parseBaikeYear(ev.Year), Content: text, Status: 1, }) } } return items, nil } // normalizeKind 映射百度百科 type 到内部 kind func normalizeKind(t string) string { switch strings.ToLower(strings.TrimSpace(t)) { case "event": return "event" case "birth": return "birth" case "death": return "death" } return "" } // parseBaikeYear 解析年份字符串:"1912"→1912,"-30"→-30(公元前),非法→0 func parseBaikeYear(s string) int { s = strings.TrimSpace(s) if s == "" { return 0 } y, err := strconv.Atoi(s) if err != nil { return 0 } return y } // cleanHTMLText 去除 HTML 标签与多余空白,输出纯文本 func cleanHTMLText(s string) string { s = reHTMLTag.ReplaceAllString(s, "") s = strings.ReplaceAll(s, " ", " ") s = reSpace.ReplaceAllString(s, " ") return strings.TrimSpace(s) }