package service import ( "encoding/json" "fmt" "io" "log" "net/http" "net/url" "regexp" "strings" "sync/atomic" "time" "github.com/gouki/lunar-server/internal/model" "gorm.io/gorm" ) // BaikeTermSyncService 二十四节气详情同步服务 // 数据源:百度百科开放接口 BaikeLemmaCardApi(词条卡片,含简介/配图/信息卡) // 与「历史上的今天」(WikiSyncService)完全独立,互不影响 type BaikeTermSyncService struct { db *gorm.DB running int32 // 原子标记:是否有同步正在执行 httpClient *http.Client } // NewBaikeTermSyncService 创建节气同步服务 func NewBaikeTermSyncService(db *gorm.DB) *BaikeTermSyncService { return &BaikeTermSyncService{ db: db, httpClient: &http.Client{ Timeout: 30 * time.Second, }, } } // solarTermNames 二十四节气(顺序即 Sort 排序) var solarTermNames = []string{ "立春", "雨水", "惊蛰", "春分", "清明", "谷雨", "立夏", "小满", "芒种", "夏至", "小暑", "大暑", "立秋", "处暑", "白露", "秋分", "寒露", "霜降", "立冬", "小雪", "大雪", "冬至", "小寒", "大寒", } var ( // reTermSup 匹配 ... 引用标记(含内容一并去除,避免残留引用编号) reTermSup = regexp.MustCompile(`(?s)]*>.*?`) reTermHTMLTag = regexp.MustCompile(`]*>`) reTermSpace = regexp.MustCompile(`\s+`) ) // IsRunning 是否有同步任务正在执行 func (s *BaikeTermSyncService) IsRunning() bool { return atomic.LoadInt32(&s.running) == 1 } // detailedTermCount 已含详情(image 非空)的节气条目数 func (s *BaikeTermSyncService) detailedTermCount() int64 { var count int64 s.db.Model(&model.WikiEntry{}). Where("category = ? AND image != ''", "term"). Count(&count) return count } // TriggerSync 触发一次节气同步(异步执行);已有任务执行中时返回 false // full=true 全量刷新 24 节气;full=false 仅补齐缺详情的节气 func (s *BaikeTermSyncService) TriggerSync(trigger string, full bool) bool { names := solarTermNames if !full { names = s.missingTerms() if len(names) == 0 { return false // 无缺失,无需同步 } } if !atomic.CompareAndSwapInt32(&s.running, 0, 1) { return false } go func() { defer atomic.StoreInt32(&s.running, 0) s.runSync(trigger, names) }() return true } // missingTerms 尚未抓取详情的节气(image 为空) func (s *BaikeTermSyncService) missingTerms() []string { var existing []string s.db.Model(&model.WikiEntry{}). Where("category = ? AND image != ''", "term"). Pluck("title", &existing) set := make(map[string]bool, len(existing)) for _, t := range existing { set[t] = true } missing := make([]string, 0, len(solarTermNames)) for _, name := range solarTermNames { if !set[name] { missing = append(missing, name) } } return missing } // StartScheduler 启动定时同步任务: // - 启动时若节气详情缺失,延迟 15 秒后自动补齐 // - 之后每天在 syncHour 小时点检查,距上次成功超过 intervalDays 天才执行 func (s *BaikeTermSyncService) StartScheduler(intervalDays, syncHour int) { go func() { if missing := s.missingTerms(); len(missing) > 0 { log.Printf("term sync: 缺失 %d/24 节气详情,15秒后自动补齐", len(missing)) time.Sleep(15 * time.Second) s.TriggerSync("startup", false) } if intervalDays <= 0 { intervalDays = 7 } if syncHour < 0 || syncHour > 23 { syncHour = 4 } ticker := time.NewTicker(time.Hour) defer ticker.Stop() for range ticker.C { now := time.Now() if now.Hour() != syncHour { continue } if last, ok := s.lastSuccessAt(); ok && time.Since(last) < time.Duration(intervalDays)*24*time.Hour { continue // 未到刷新周期 } log.Println("term sync: 定时任务触发全量同步") s.TriggerSync("cron", true) } }() } // lastSuccessAt 最近一次成功(含部分成功)的同步完成时间 func (s *BaikeTermSyncService) lastSuccessAt() (time.Time, bool) { var syncLog model.BaikeTermSyncLog err := s.db.Where("status IN ?", []string{"success", "partial"}). Order("finished_at DESC").First(&syncLog).Error if err != nil || syncLog.FinishedAt == nil { return time.Time{}, false } return *syncLog.FinishedAt, true } // runSync 同步指定节气集合(全量 24 或缺失补齐) func (s *BaikeTermSyncService) runSync(trigger string, names []string) { syncLog := model.BaikeTermSyncLog{ Trigger: trigger, Status: "running", Pages: len(names), StartedAt: time.Now(), } s.db.Create(&syncLog) var success, failed int32 for _, name := range names { if err := s.syncOneTerm(name); err != nil { atomic.AddInt32(&failed, 1) log.Printf("term sync: %s 失败: %v", name, err) } else { atomic.AddInt32(&success, 1) } time.Sleep(1200 * time.Millisecond) // 控制请求频率,避免触发限流 } status := "success" if failed > 0 { status = "partial" } if success == 0 { status = "failed" } now := time.Now() s.db.Model(&syncLog).Updates(map[string]interface{}{ "status": status, "success": int(success), "failed": int(failed), "finished_at": &now, }) log.Printf("term sync: 同步完成 trigger=%s terms=%d success=%d failed=%d", trigger, len(names), success, failed) } // ===== 百度百科节气抓取与解析 ===== // baikeCardItem 信息卡单项 type baikeCardItem struct { Key string `json:"key"` Name string `json:"name"` Value []string `json:"value"` } // baikeLemmaCard 百度百科词条卡片返回结构(只取需要的字段) type baikeLemmaCard struct { Key string `json:"key"` // 词条名 Desc string `json:"desc"` // 一句话描述 Title string `json:"title"` // 标题 Abstract string `json:"abstract"`// 简介(含 HTML) Image string `json:"image"` // 配图 URL Card []baikeCardItem `json:"card"` // 信息卡 } // syncOneTerm 抓取单个节气详情并写入/更新 WikiEntry func (s *BaikeTermSyncService) syncOneTerm(name string) error { lemma, err := s.fetchLemma(name) if err != nil { return err } // 解析信息卡为扩展字段 extra := map[string]string{} for _, c := range lemma.Card { text := cleanTermHTML(firstValue(c.Value)) if text == "" { continue } switch c.Name { case "别名": extra["alias"] = text case "外文名": extra["english"] = text case "涵义": extra["meaning"] = text case "公历时间": extra["solarDate"] = text case "黄道位置": extra["ecliptic"] = text case "气候特点": extra["climate"] = text case "物候现象": extra["phenology"] = text case "农事活动": extra["farming"] = text case "传统习俗": extra["customs"] = text case "起居养生": extra["health"] = text case "花信": extra["flower"] = text } } extraJSON, _ := json.Marshal(extra) content := trimTruncatedTail(cleanTermHTML(lemma.Abstract)) brief := extra["meaning"] if brief == "" { brief = firstSentence(content) } if len([]rune(brief)) > 120 { brief = string([]rune(brief)[:120]) } // 按 category=term AND title=name 查找,存在则更新,不存在则新建 var entry model.WikiEntry err = s.db.Where("category = ? AND title = ?", "term", name).First(&entry).Error if err == gorm.ErrRecordNotFound { entry = model.WikiEntry{ Category: "term", Title: name, Brief: brief, Content: content, Image: lemma.Image, Extra: string(extraJSON), Sort: termSort(name), Status: 1, } return s.db.Create(&entry).Error } else if err != nil { return err } return s.db.Model(&entry).Updates(map[string]interface{}{ "brief": brief, "content": content, "image": lemma.Image, "extra": string(extraJSON), }).Error } // fetchLemma 调用百度百科 BaikeLemmaCardApi 抓取词条卡片(失败重试) func (s *BaikeTermSyncService) fetchLemma(name string) (*baikeLemmaCard, error) { apiURL := "https://baike.baidu.com/api/openapi/BaikeLemmaCardApi?scope=103&format=json&appid=379020&bk_length=600&bk_key=" + url.QueryEscape(name) var lastErr error for attempt := 0; attempt < 4; attempt++ { if attempt > 0 { time.Sleep(time.Duration(attempt*3) * time.Second) } req, err := http.NewRequest(http.MethodGet, apiURL, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36") req.Header.Set("Referer", "https://baike.baidu.com/") req.Header.Set("Accept", "application/json, text/plain, */*") resp, err := s.httpClient.Do(req) if err != nil { lastErr = err continue } body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB resp.Body.Close() if resp.StatusCode != http.StatusOK { lastErr = fmt.Errorf("HTTP %d", resp.StatusCode) continue } if err != nil { lastErr = err continue } var lemma baikeLemmaCard if err := json.Unmarshal(body, &lemma); err != nil { lastErr = fmt.Errorf("JSON 解析失败: %w", err) continue } if lemma.Abstract == "" && len(lemma.Card) == 0 { lastErr = fmt.Errorf("%s 返回内容为空", name) continue } return &lemma, nil } return nil, lastErr } // termSort 节气在列表中的排序(按节气顺序) func termSort(name string) int { for i, n := range solarTermNames { if n == name { return i + 1 } } return 0 } // firstValue 取信息卡 value 的第一个值 func firstValue(values []string) string { if len(values) == 0 { return "" } return values[0] } // firstSentence 取文本首句(按句号/感叹号/问号切分) func firstSentence(s string) string { for _, sep := range []string{"。", "!", "?"} { if idx := strings.Index(s, sep); idx > 0 { return s[:idx+len(sep)] } } return s } // cleanTermHTML 去除 HTML 标签(含 sup 引用标记整体)与多余空白,输出纯文本(本地实现,与维基同步解耦) func cleanTermHTML(s string) string { s = reTermSup.ReplaceAllString(s, "") // 先去 引用编号(含内容) s = reTermHTMLTag.ReplaceAllString(s, "") s = strings.ReplaceAll(s, " ", " ") s = reTermSpace.ReplaceAllString(s, " ") return strings.TrimSpace(s) } // trimTruncatedTail 清理百度百科摘要的截断结尾: // 摘要以「...」「…」等结尾说明被接口截断,先去掉省略号; // 若去掉后末句不是完整句(不以 。!? 结尾),回退到最后一个完整句标点,避免半截话 func trimTruncatedTail(s string) string { // 去掉结尾的省略号(英文 ... / 中文 … / 多个点) s = strings.TrimRight(s, ".… ") if s == "" { return s } // 末字符已是完整句标点则无需回退 last := []rune(s) if len(last) == 0 { return s } if strings.ContainsRune("。!?", last[len(last)-1]) { return s } // 回退到最后一个完整句标点(LastIndex 未找到返回 -1,需先判断 j >= 0) idx := -1 for _, sep := range []string{"。", "!", "?"} { if j := strings.LastIndex(s, sep); j >= 0 && j+len(sep) > idx { idx = j + len(sep) } } if idx > 0 { return s[:idx] } // 没有完整句标点则保留原文(避免清空) return s }