gouki
0ce1a1ef97
Build and Publish Server / build (push) Successful in 2m13s
维基百科在生产服务器被阻断，改用百度百科开放接口：
- 端点 /cms/home/eventsOnHistory/{MM}.json，按月返回结构化 JSON
- 12 次请求替代原 366 次，大幅减负
- type 直接映射 event/birth/death，festival 字段提取节假日
- 清洗 title/desc 中的 HTML 标签
- 2 月无 29 日，全年 365 天
- 本地验证：365 天 4805 条，12/12 月全成功
422 lines
11 KiB
Go
422 lines
11 KiB
Go
package service
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/gouki/lunar-server/internal/model"
|
||
"gorm.io/gorm"
|
||
)
|
||
|
||
// WikiSyncService 「历史上的今天」同步服务
|
||
// 数据源：百度百科「历史上的今天」开放接口（按月返回结构化 JSON，共 12 次请求）
|
||
// 首次抓取后按配置间隔定期增量刷新
|
||
type WikiSyncService struct {
|
||
db *gorm.DB
|
||
|
||
running int32 // 原子标记：是否有同步正在执行
|
||
|
||
httpClient *http.Client
|
||
}
|
||
|
||
// NewWikiSyncService 创建同步服务
|
||
func NewWikiSyncService(db *gorm.DB) *WikiSyncService {
|
||
return &WikiSyncService{
|
||
db: db,
|
||
httpClient: &http.Client{
|
||
Timeout: 30 * time.Second,
|
||
},
|
||
}
|
||
}
|
||
|
||
var (
|
||
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||
reSpace = regexp.MustCompile(`\s+`)
|
||
)
|
||
|
||
// daysInMonth 每月天数（百度百科数据源不含 2 月 29 日，共 365 天）
|
||
var daysInMonth = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
|
||
|
||
type dayKey struct {
|
||
Month int
|
||
Day int
|
||
}
|
||
|
||
// allDays 全年 365 天
|
||
func allDays() []dayKey {
|
||
days := make([]dayKey, 0, 365)
|
||
for m := 1; m <= 12; m++ {
|
||
for d := 1; d <= daysInMonth[m-1]; d++ {
|
||
days = append(days, dayKey{Month: m, Day: d})
|
||
}
|
||
}
|
||
return days
|
||
}
|
||
|
||
// IsRunning 是否有同步任务正在执行
|
||
func (s *WikiSyncService) IsRunning() bool {
|
||
return atomic.LoadInt32(&s.running) == 1
|
||
}
|
||
|
||
// coveredDaySet 已覆盖的月日集合
|
||
func (s *WikiSyncService) coveredDaySet() map[dayKey]bool {
|
||
set := make(map[dayKey]bool, 400)
|
||
type row struct {
|
||
Month int
|
||
Day int
|
||
}
|
||
var rows []row
|
||
s.db.Model(&model.WikiOnThisDay{}).Where("status = 1").
|
||
Select("DISTINCT month, day").Scan(&rows)
|
||
for _, r := range rows {
|
||
set[dayKey{Month: r.Month, Day: r.Day}] = true
|
||
}
|
||
return set
|
||
}
|
||
|
||
// missingDays 尚未覆盖的天
|
||
func (s *WikiSyncService) missingDays() []dayKey {
|
||
covered := s.coveredDaySet()
|
||
missing := make([]dayKey, 0)
|
||
for _, d := range allDays() {
|
||
if !covered[d] {
|
||
missing = append(missing, d)
|
||
}
|
||
}
|
||
return missing
|
||
}
|
||
|
||
// TriggerSync 触发一次同步（异步执行）；已有任务执行中时返回 false
|
||
// full=true 全量刷新 365 天；full=false 仅补齐缺失的天
|
||
func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool {
|
||
days := allDays()
|
||
if !full {
|
||
days = s.missingDays()
|
||
if len(days) == 0 {
|
||
return false // 无缺失，无需同步
|
||
}
|
||
}
|
||
if !atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
||
return false
|
||
}
|
||
go func() {
|
||
defer atomic.StoreInt32(&s.running, 0)
|
||
s.runSync(trigger, days)
|
||
}()
|
||
return true
|
||
}
|
||
|
||
// StartScheduler 启动定时同步任务：
|
||
// - 启动时若数据库为空，延迟 10 秒后自动执行首次全量抓取
|
||
// - 之后每天在 syncHour 小时点检查，距上次成功超过 intervalDays 天才执行
|
||
func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) {
|
||
go func() {
|
||
if missing := s.missingDays(); len(missing) > 0 {
|
||
// 数据缺失（首次部署或上次抓取中断），延迟 10 秒等服务稳定后补齐
|
||
log.Printf("wiki sync: 缺失 %d/365 天，10秒后自动补齐", len(missing))
|
||
time.Sleep(10 * time.Second)
|
||
s.TriggerSync("startup", false)
|
||
}
|
||
|
||
if intervalDays <= 0 {
|
||
intervalDays = 7
|
||
}
|
||
if syncHour < 0 || syncHour > 23 {
|
||
syncHour = 4
|
||
}
|
||
ticker := time.NewTicker(time.Hour)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
now := time.Now()
|
||
if now.Hour() != syncHour {
|
||
continue
|
||
}
|
||
if last, ok := s.lastSuccessAt(); ok && time.Since(last) < time.Duration(intervalDays)*24*time.Hour {
|
||
continue // 未到刷新周期
|
||
}
|
||
log.Println("wiki sync: 定时任务触发全量同步")
|
||
s.TriggerSync("cron", true)
|
||
}
|
||
}()
|
||
}
|
||
|
||
// lastSuccessAt 最近一次成功（含部分成功）的同步完成时间
|
||
func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) {
|
||
var syncLog model.WikiSyncLog
|
||
err := s.db.Where("status IN ?", []string{"success", "partial"}).
|
||
Order("finished_at DESC").First(&syncLog).Error
|
||
if err != nil || syncLog.FinishedAt == nil {
|
||
return time.Time{}, false
|
||
}
|
||
return *syncLog.FinishedAt, true
|
||
}
|
||
|
||
// runSync 同步指定日期集合（全量 365 天或缺失补齐）
|
||
// 百度百科按月返回数据，故先归并出涉及的月份，每月一次请求，再按天写入
|
||
func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
||
syncLog := model.WikiSyncLog{
|
||
Trigger: trigger,
|
||
Status: "running",
|
||
Pages: len(days),
|
||
StartedAt: time.Now(),
|
||
}
|
||
s.db.Create(&syncLog)
|
||
|
||
// 归并涉及的月份
|
||
monthSet := make(map[int]bool)
|
||
for _, d := range days {
|
||
monthSet[d.Month] = true
|
||
}
|
||
months := make([]int, 0, len(monthSet))
|
||
for m := range monthSet {
|
||
months = append(months, m)
|
||
}
|
||
|
||
jobs := make(chan int, len(months))
|
||
for _, m := range months {
|
||
jobs <- m
|
||
}
|
||
close(jobs)
|
||
|
||
var success, failed int32
|
||
var wg sync.WaitGroup
|
||
workerCount := 2 // 百度百科反爬较敏感，低并发 + 间隔
|
||
for i := 0; i < workerCount; i++ {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for m := range jobs {
|
||
if err := s.syncOneMonth(m); err != nil {
|
||
atomic.AddInt32(&failed, 1)
|
||
log.Printf("wiki sync: %d月 失败: %v", m, err)
|
||
} else {
|
||
atomic.AddInt32(&success, 1)
|
||
}
|
||
time.Sleep(1200 * time.Millisecond) // 控制请求频率，避免触发安全验证
|
||
}
|
||
}()
|
||
}
|
||
wg.Wait()
|
||
|
||
status := "success"
|
||
if failed > 0 {
|
||
status = "partial"
|
||
}
|
||
if success == 0 {
|
||
status = "failed"
|
||
}
|
||
now := time.Now()
|
||
s.db.Model(&syncLog).Updates(map[string]interface{}{
|
||
"status": status,
|
||
"success": int(success),
|
||
"failed": int(failed),
|
||
"finished_at": &now,
|
||
})
|
||
log.Printf("wiki sync: 同步完成 trigger=%s days=%d months=%d success=%d failed=%d", trigger, len(days), len(months), success, failed)
|
||
}
|
||
|
||
// syncOneMonth 抓取某月数据并按天整体替换（死锁自动重试）
|
||
func (s *WikiSyncService) syncOneMonth(month int) error {
|
||
items, err := s.fetchMonth(month)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if len(items) == 0 {
|
||
return fmt.Errorf("%d月解析结果为空", month)
|
||
}
|
||
|
||
// 按天分组
|
||
byDay := make(map[int][]model.WikiOnThisDay)
|
||
for _, it := range items {
|
||
byDay[it.Day] = append(byDay[it.Day], it)
|
||
}
|
||
|
||
// 逐天事务替换
|
||
for day, dayItems := range byDay {
|
||
var lastErr error
|
||
for attempt := 0; attempt < 3; attempt++ {
|
||
if attempt > 0 {
|
||
time.Sleep(time.Duration(attempt) * time.Second)
|
||
}
|
||
lastErr = s.db.Transaction(func(tx *gorm.DB) error {
|
||
if err := tx.Where("month = ? AND day = ?", month, day).
|
||
Delete(&model.WikiOnThisDay{}).Error; err != nil {
|
||
return err
|
||
}
|
||
return tx.Create(&dayItems).Error
|
||
})
|
||
if lastErr == nil {
|
||
break
|
||
}
|
||
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
|
||
return lastErr
|
||
}
|
||
}
|
||
if lastErr != nil {
|
||
return fmt.Errorf("%d月%d日写入失败: %w", month, day, lastErr)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ===== 百度百科数据抓取与解析 =====
|
||
|
||
// baikeEvent 百度百科「历史上的今天」单条数据
|
||
type baikeEvent struct {
|
||
Year string `json:"year"` // 年份，负数表示公元前（如 "-30"）
|
||
Title string `json:"title"` // 标题（含 HTML 链接标签）
|
||
Festival string `json:"festival"` // 节假日名（当天所有条目共享，可能为空）
|
||
Link string `json:"link"` // 百科词条链接
|
||
Type string `json:"type"` // event / birth / death
|
||
Desc string `json:"desc"` // 描述（含 HTML 标签）
|
||
}
|
||
|
||
// fetchMonth 抓取某月「历史上的今天」数据并解析为按天的条目（失败重试）
|
||
func (s *WikiSyncService) fetchMonth(month int) ([]model.WikiOnThisDay, error) {
|
||
apiURL := fmt.Sprintf("https://baike.baidu.com/cms/home/eventsOnHistory/%02d.json", month)
|
||
|
||
var lastErr error
|
||
for attempt := 0; attempt < 4; attempt++ {
|
||
if attempt > 0 {
|
||
time.Sleep(time.Duration(attempt*3) * time.Second)
|
||
}
|
||
body, err := s.doFetch(apiURL)
|
||
if err != nil {
|
||
lastErr = err
|
||
continue
|
||
}
|
||
items, err := parseBaikeMonth(body, month)
|
||
if err != nil {
|
||
lastErr = err
|
||
continue
|
||
}
|
||
return items, nil
|
||
}
|
||
return nil, lastErr
|
||
}
|
||
|
||
// doFetch 发起 HTTP 请求并返回响应体（带浏览器 UA 与 Referer，规避安全验证）
|
||
func (s *WikiSyncService) doFetch(apiURL string) ([]byte, error) {
|
||
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
||
req.Header.Set("Referer", "https://baike.baidu.com/calendar")
|
||
req.Header.Set("Accept", "application/json, text/plain, */*")
|
||
|
||
resp, err := s.httpClient.Do(req)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode != http.StatusOK {
|
||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||
}
|
||
return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) // 上限 4MB
|
||
}
|
||
|
||
// parseBaikeMonth 解析百度百科某月 JSON，输出该月全部条目
|
||
func parseBaikeMonth(body []byte, month int) ([]model.WikiOnThisDay, error) {
|
||
// 顶层结构：{"08": {"0801": [...], "0802": [...]}}
|
||
var root map[string]map[string][]baikeEvent
|
||
if err := json.Unmarshal(body, &root); err != nil {
|
||
return nil, fmt.Errorf("JSON 解析失败: %w", err)
|
||
}
|
||
monthKey := fmt.Sprintf("%02d", month)
|
||
dayMap, ok := root[monthKey]
|
||
if !ok || len(dayMap) == 0 {
|
||
return nil, fmt.Errorf("%s月无数据", monthKey)
|
||
}
|
||
|
||
items := make([]model.WikiOnThisDay, 0, 256)
|
||
for dayKeyStr, events := range dayMap {
|
||
day, err := strconv.Atoi(dayKeyStr[2:]) // "0812" -> 12
|
||
if err != nil {
|
||
continue
|
||
}
|
||
seenFestival := make(map[string]bool)
|
||
seenContent := make(map[string]bool)
|
||
|
||
for _, ev := range events {
|
||
// 节假日：从 festival 字段提取（挂在每条上，去重后单独成 festival 条目）
|
||
if fest := cleanHTMLText(ev.Festival); fest != "" && !seenFestival[fest] {
|
||
seenFestival[fest] = true
|
||
items = append(items, model.WikiOnThisDay{
|
||
Month: month, Day: day, Kind: "festival", Year: 0, Content: fest, Status: 1,
|
||
})
|
||
}
|
||
|
||
kind := normalizeKind(ev.Type)
|
||
if kind == "" {
|
||
continue
|
||
}
|
||
text := cleanHTMLText(ev.Title)
|
||
if len([]rune(text)) < 4 {
|
||
continue
|
||
}
|
||
if len([]rune(text)) > 500 {
|
||
text = string([]rune(text)[:500])
|
||
}
|
||
if seenContent[kind+"|"+text] {
|
||
continue
|
||
}
|
||
seenContent[kind+"|"+text] = true
|
||
|
||
items = append(items, model.WikiOnThisDay{
|
||
Month: month,
|
||
Day: day,
|
||
Kind: kind,
|
||
Year: parseBaikeYear(ev.Year),
|
||
Content: text,
|
||
Status: 1,
|
||
})
|
||
}
|
||
}
|
||
return items, nil
|
||
}
|
||
|
||
// normalizeKind 映射百度百科 type 到内部 kind
|
||
func normalizeKind(t string) string {
|
||
switch strings.ToLower(strings.TrimSpace(t)) {
|
||
case "event":
|
||
return "event"
|
||
case "birth":
|
||
return "birth"
|
||
case "death":
|
||
return "death"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// parseBaikeYear 解析年份字符串："1912"→1912，"-30"→-30（公元前），非法→0
|
||
func parseBaikeYear(s string) int {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" {
|
||
return 0
|
||
}
|
||
y, err := strconv.Atoi(s)
|
||
if err != nil {
|
||
return 0
|
||
}
|
||
return y
|
||
}
|
||
|
||
// cleanHTMLText 去除 HTML 标签与多余空白，输出纯文本
|
||
func cleanHTMLText(s string) string {
|
||
s = reHTMLTag.ReplaceAllString(s, "")
|
||
s = strings.ReplaceAll(s, " ", " ")
|
||
s = reSpace.ReplaceAllString(s, " ")
|
||
return strings.TrimSpace(s)
|
||
}
|