gouki
241a04496b
百度百科 BaikeLemmaCardApi 摘要以 ... 硬截断，出现半截话。 新增 trimTruncatedTail：去掉结尾省略号后，若末句不完整则 回退到最后一个完整句标点（。！？），避免「唐宋时期，冬至...」 这类戛然而止的结尾。仅依赖百度百科，不引入维基，线上可用。
399 lines
11 KiB
Go
399 lines
11 KiB
Go
package service
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"net/url"
|
||
"regexp"
|
||
"strings"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/gouki/lunar-server/internal/model"
|
||
"gorm.io/gorm"
|
||
)
|
||
|
||
// BaikeTermSyncService 二十四节气详情同步服务
|
||
// 数据源：百度百科开放接口 BaikeLemmaCardApi（词条卡片，含简介/配图/信息卡）
|
||
// 与「历史上的今天」（WikiSyncService）完全独立，互不影响
|
||
type BaikeTermSyncService struct {
|
||
db *gorm.DB
|
||
|
||
running int32 // 原子标记：是否有同步正在执行
|
||
|
||
httpClient *http.Client
|
||
}
|
||
|
||
// NewBaikeTermSyncService 创建节气同步服务
|
||
func NewBaikeTermSyncService(db *gorm.DB) *BaikeTermSyncService {
|
||
return &BaikeTermSyncService{
|
||
db: db,
|
||
httpClient: &http.Client{
|
||
Timeout: 30 * time.Second,
|
||
},
|
||
}
|
||
}
|
||
|
||
// solarTermNames 二十四节气（顺序即 Sort 排序）
|
||
var solarTermNames = []string{
|
||
"立春", "雨水", "惊蛰", "春分", "清明", "谷雨",
|
||
"立夏", "小满", "芒种", "夏至", "小暑", "大暑",
|
||
"立秋", "处暑", "白露", "秋分", "寒露", "霜降",
|
||
"立冬", "小雪", "大雪", "冬至", "小寒", "大寒",
|
||
}
|
||
|
||
var (
|
||
// reTermSup 匹配 <sup>...</sup> 引用标记（含内容一并去除，避免残留引用编号）
|
||
reTermSup = regexp.MustCompile(`(?s)<sup[^>]*>.*?</sup>`)
|
||
reTermHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||
reTermSpace = regexp.MustCompile(`\s+`)
|
||
)
|
||
|
||
// IsRunning 是否有同步任务正在执行
|
||
func (s *BaikeTermSyncService) IsRunning() bool {
|
||
return atomic.LoadInt32(&s.running) == 1
|
||
}
|
||
|
||
// detailedTermCount 已含详情（image 非空）的节气条目数
|
||
func (s *BaikeTermSyncService) detailedTermCount() int64 {
|
||
var count int64
|
||
s.db.Model(&model.WikiEntry{}).
|
||
Where("category = ? AND image != ''", "term").
|
||
Count(&count)
|
||
return count
|
||
}
|
||
|
||
// TriggerSync 触发一次节气同步（异步执行）；已有任务执行中时返回 false
|
||
// full=true 全量刷新 24 节气；full=false 仅补齐缺详情的节气
|
||
func (s *BaikeTermSyncService) TriggerSync(trigger string, full bool) bool {
|
||
names := solarTermNames
|
||
if !full {
|
||
names = s.missingTerms()
|
||
if len(names) == 0 {
|
||
return false // 无缺失，无需同步
|
||
}
|
||
}
|
||
if !atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
||
return false
|
||
}
|
||
go func() {
|
||
defer atomic.StoreInt32(&s.running, 0)
|
||
s.runSync(trigger, names)
|
||
}()
|
||
return true
|
||
}
|
||
|
||
// missingTerms 尚未抓取详情的节气（image 为空）
|
||
func (s *BaikeTermSyncService) missingTerms() []string {
|
||
var existing []string
|
||
s.db.Model(&model.WikiEntry{}).
|
||
Where("category = ? AND image != ''", "term").
|
||
Pluck("title", &existing)
|
||
set := make(map[string]bool, len(existing))
|
||
for _, t := range existing {
|
||
set[t] = true
|
||
}
|
||
missing := make([]string, 0, len(solarTermNames))
|
||
for _, name := range solarTermNames {
|
||
if !set[name] {
|
||
missing = append(missing, name)
|
||
}
|
||
}
|
||
return missing
|
||
}
|
||
|
||
// StartScheduler 启动定时同步任务：
|
||
// - 启动时若节气详情缺失，延迟 15 秒后自动补齐
|
||
// - 之后每天在 syncHour 小时点检查，距上次成功超过 intervalDays 天才执行
|
||
func (s *BaikeTermSyncService) StartScheduler(intervalDays, syncHour int) {
|
||
go func() {
|
||
if missing := s.missingTerms(); len(missing) > 0 {
|
||
log.Printf("term sync: 缺失 %d/24 节气详情，15秒后自动补齐", len(missing))
|
||
time.Sleep(15 * time.Second)
|
||
s.TriggerSync("startup", false)
|
||
}
|
||
|
||
if intervalDays <= 0 {
|
||
intervalDays = 7
|
||
}
|
||
if syncHour < 0 || syncHour > 23 {
|
||
syncHour = 4
|
||
}
|
||
ticker := time.NewTicker(time.Hour)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
now := time.Now()
|
||
if now.Hour() != syncHour {
|
||
continue
|
||
}
|
||
if last, ok := s.lastSuccessAt(); ok && time.Since(last) < time.Duration(intervalDays)*24*time.Hour {
|
||
continue // 未到刷新周期
|
||
}
|
||
log.Println("term sync: 定时任务触发全量同步")
|
||
s.TriggerSync("cron", true)
|
||
}
|
||
}()
|
||
}
|
||
|
||
// lastSuccessAt 最近一次成功（含部分成功）的同步完成时间
|
||
func (s *BaikeTermSyncService) lastSuccessAt() (time.Time, bool) {
|
||
var syncLog model.BaikeTermSyncLog
|
||
err := s.db.Where("status IN ?", []string{"success", "partial"}).
|
||
Order("finished_at DESC").First(&syncLog).Error
|
||
if err != nil || syncLog.FinishedAt == nil {
|
||
return time.Time{}, false
|
||
}
|
||
return *syncLog.FinishedAt, true
|
||
}
|
||
|
||
// runSync 同步指定节气集合（全量 24 或缺失补齐）
|
||
func (s *BaikeTermSyncService) runSync(trigger string, names []string) {
|
||
syncLog := model.BaikeTermSyncLog{
|
||
Trigger: trigger,
|
||
Status: "running",
|
||
Pages: len(names),
|
||
StartedAt: time.Now(),
|
||
}
|
||
s.db.Create(&syncLog)
|
||
|
||
var success, failed int32
|
||
for _, name := range names {
|
||
if err := s.syncOneTerm(name); err != nil {
|
||
atomic.AddInt32(&failed, 1)
|
||
log.Printf("term sync: %s 失败: %v", name, err)
|
||
} else {
|
||
atomic.AddInt32(&success, 1)
|
||
}
|
||
time.Sleep(1200 * time.Millisecond) // 控制请求频率，避免触发限流
|
||
}
|
||
|
||
status := "success"
|
||
if failed > 0 {
|
||
status = "partial"
|
||
}
|
||
if success == 0 {
|
||
status = "failed"
|
||
}
|
||
now := time.Now()
|
||
s.db.Model(&syncLog).Updates(map[string]interface{}{
|
||
"status": status,
|
||
"success": int(success),
|
||
"failed": int(failed),
|
||
"finished_at": &now,
|
||
})
|
||
log.Printf("term sync: 同步完成 trigger=%s terms=%d success=%d failed=%d", trigger, len(names), success, failed)
|
||
}
|
||
|
||
// ===== 百度百科节气抓取与解析 =====
|
||
|
||
// baikeCardItem 信息卡单项
|
||
type baikeCardItem struct {
|
||
Key string `json:"key"`
|
||
Name string `json:"name"`
|
||
Value []string `json:"value"`
|
||
}
|
||
|
||
// baikeLemmaCard 百度百科词条卡片返回结构（只取需要的字段）
|
||
type baikeLemmaCard struct {
|
||
Key string `json:"key"` // 词条名
|
||
Desc string `json:"desc"` // 一句话描述
|
||
Title string `json:"title"` // 标题
|
||
Abstract string `json:"abstract"`// 简介（含 HTML）
|
||
Image string `json:"image"` // 配图 URL
|
||
Card []baikeCardItem `json:"card"` // 信息卡
|
||
}
|
||
|
||
// syncOneTerm 抓取单个节气详情并写入/更新 WikiEntry
|
||
func (s *BaikeTermSyncService) syncOneTerm(name string) error {
|
||
lemma, err := s.fetchLemma(name)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
// 解析信息卡为扩展字段
|
||
extra := map[string]string{}
|
||
for _, c := range lemma.Card {
|
||
text := cleanTermHTML(firstValue(c.Value))
|
||
if text == "" {
|
||
continue
|
||
}
|
||
switch c.Name {
|
||
case "别名":
|
||
extra["alias"] = text
|
||
case "外文名":
|
||
extra["english"] = text
|
||
case "涵义":
|
||
extra["meaning"] = text
|
||
case "公历时间":
|
||
extra["solarDate"] = text
|
||
case "黄道位置":
|
||
extra["ecliptic"] = text
|
||
case "气候特点":
|
||
extra["climate"] = text
|
||
case "物候现象":
|
||
extra["phenology"] = text
|
||
case "农事活动":
|
||
extra["farming"] = text
|
||
case "传统习俗":
|
||
extra["customs"] = text
|
||
case "起居养生":
|
||
extra["health"] = text
|
||
case "花信":
|
||
extra["flower"] = text
|
||
}
|
||
}
|
||
extraJSON, _ := json.Marshal(extra)
|
||
|
||
content := trimTruncatedTail(cleanTermHTML(lemma.Abstract))
|
||
brief := extra["meaning"]
|
||
if brief == "" {
|
||
brief = firstSentence(content)
|
||
}
|
||
if len([]rune(brief)) > 120 {
|
||
brief = string([]rune(brief)[:120])
|
||
}
|
||
|
||
// 按 category=term AND title=name 查找，存在则更新，不存在则新建
|
||
var entry model.WikiEntry
|
||
err = s.db.Where("category = ? AND title = ?", "term", name).First(&entry).Error
|
||
if err == gorm.ErrRecordNotFound {
|
||
entry = model.WikiEntry{
|
||
Category: "term",
|
||
Title: name,
|
||
Brief: brief,
|
||
Content: content,
|
||
Image: lemma.Image,
|
||
Extra: string(extraJSON),
|
||
Sort: termSort(name),
|
||
Status: 1,
|
||
}
|
||
return s.db.Create(&entry).Error
|
||
} else if err != nil {
|
||
return err
|
||
}
|
||
return s.db.Model(&entry).Updates(map[string]interface{}{
|
||
"brief": brief,
|
||
"content": content,
|
||
"image": lemma.Image,
|
||
"extra": string(extraJSON),
|
||
}).Error
|
||
}
|
||
|
||
// fetchLemma 调用百度百科 BaikeLemmaCardApi 抓取词条卡片（失败重试）
|
||
func (s *BaikeTermSyncService) fetchLemma(name string) (*baikeLemmaCard, error) {
|
||
apiURL := "https://baike.baidu.com/api/openapi/BaikeLemmaCardApi?scope=103&format=json&appid=379020&bk_length=600&bk_key=" + url.QueryEscape(name)
|
||
|
||
var lastErr error
|
||
for attempt := 0; attempt < 4; attempt++ {
|
||
if attempt > 0 {
|
||
time.Sleep(time.Duration(attempt*3) * time.Second)
|
||
}
|
||
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
||
req.Header.Set("Referer", "https://baike.baidu.com/")
|
||
req.Header.Set("Accept", "application/json, text/plain, */*")
|
||
|
||
resp, err := s.httpClient.Do(req)
|
||
if err != nil {
|
||
lastErr = err
|
||
continue
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB
|
||
resp.Body.Close()
|
||
if resp.StatusCode != http.StatusOK {
|
||
lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)
|
||
continue
|
||
}
|
||
if err != nil {
|
||
lastErr = err
|
||
continue
|
||
}
|
||
|
||
var lemma baikeLemmaCard
|
||
if err := json.Unmarshal(body, &lemma); err != nil {
|
||
lastErr = fmt.Errorf("JSON 解析失败: %w", err)
|
||
continue
|
||
}
|
||
if lemma.Abstract == "" && len(lemma.Card) == 0 {
|
||
lastErr = fmt.Errorf("%s 返回内容为空", name)
|
||
continue
|
||
}
|
||
return &lemma, nil
|
||
}
|
||
return nil, lastErr
|
||
}
|
||
|
||
// termSort 节气在列表中的排序（按节气顺序）
|
||
func termSort(name string) int {
|
||
for i, n := range solarTermNames {
|
||
if n == name {
|
||
return i + 1
|
||
}
|
||
}
|
||
return 0
|
||
}
|
||
|
||
// firstValue 取信息卡 value 的第一个值
|
||
func firstValue(values []string) string {
|
||
if len(values) == 0 {
|
||
return ""
|
||
}
|
||
return values[0]
|
||
}
|
||
|
||
// firstSentence 取文本首句（按句号/感叹号/问号切分）
|
||
func firstSentence(s string) string {
|
||
for _, sep := range []string{"。", "！", "？"} {
|
||
if idx := strings.Index(s, sep); idx > 0 {
|
||
return s[:idx+len(sep)]
|
||
}
|
||
}
|
||
return s
|
||
}
|
||
|
||
// cleanTermHTML 去除 HTML 标签（含 sup 引用标记整体）与多余空白，输出纯文本（本地实现，与维基同步解耦）
|
||
func cleanTermHTML(s string) string {
|
||
s = reTermSup.ReplaceAllString(s, "") // 先去 <sup>引用编号</sup>（含内容）
|
||
s = reTermHTMLTag.ReplaceAllString(s, "")
|
||
s = strings.ReplaceAll(s, " ", " ")
|
||
s = reTermSpace.ReplaceAllString(s, " ")
|
||
return strings.TrimSpace(s)
|
||
}
|
||
|
||
// trimTruncatedTail 清理百度百科摘要的截断结尾：
|
||
// 摘要以「...」「…」等结尾说明被接口截断，先去掉省略号；
|
||
// 若去掉后末句不是完整句（不以 。！？ 结尾），回退到最后一个完整句标点，避免半截话
|
||
func trimTruncatedTail(s string) string {
|
||
// 去掉结尾的省略号（英文 ... / 中文 … / 多个点）
|
||
s = strings.TrimRight(s, ".… ")
|
||
if s == "" {
|
||
return s
|
||
}
|
||
// 末字符已是完整句标点则无需回退
|
||
last := []rune(s)
|
||
if len(last) == 0 {
|
||
return s
|
||
}
|
||
if strings.ContainsRune("。！？", last[len(last)-1]) {
|
||
return s
|
||
}
|
||
// 回退到最后一个完整句标点（LastIndex 未找到返回 -1，需先判断 j >= 0）
|
||
idx := -1
|
||
for _, sep := range []string{"。", "！", "？"} {
|
||
if j := strings.LastIndex(s, sep); j >= 0 && j+len(sep) > idx {
|
||
idx = j + len(sep)
|
||
}
|
||
}
|
||
if idx > 0 {
|
||
return s[:idx]
|
||
}
|
||
// 没有完整句标点则保留原文（避免清空）
|
||
return s
|
||
}
|