gouki
2a68eae999
1. 修复swiper箭头点击动画丢失问题 - 改用固定7天窗口，滑动到边缘时整体滚动数据 - 保持dayIndex连续，确保swiper平滑动画 2. 修复宜忌两行显示压在一起 - 设置item固定高度32rpx和行距12rpx - 超出2行显示...省略号 3. 修复月历数字竖向排列问题 - 修正wxml嵌套结构，让42个格子正确排成6行7列 - 压缩格子高度，放大数字字体 4. 节日速查功能优化 - 默认显示3条，右侧添加更多
487 lines
13 KiB
Go
487 lines
13 KiB
Go
package service
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"net/url"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/gouki/lunar-server/internal/model"
|
||
"gorm.io/gorm"
|
||
)
|
||
|
||
// WikiSyncService 维基百科「X月X日」页面同步服务
|
||
// 全量共 366 页（含 2 月 29 日），首次抓取后按配置间隔定期增量刷新
|
||
type WikiSyncService struct {
|
||
db *gorm.DB
|
||
|
||
running int32 // 原子标记：是否有同步正在执行
|
||
|
||
httpClient *http.Client
|
||
}
|
||
|
||
// NewWikiSyncService 创建同步服务
|
||
func NewWikiSyncService(db *gorm.DB) *WikiSyncService {
|
||
return &WikiSyncService{
|
||
db: db,
|
||
httpClient: &http.Client{
|
||
Timeout: 30 * time.Second,
|
||
},
|
||
}
|
||
}
|
||
|
||
var (
|
||
reComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||
reRef = regexp.MustCompile(`(?is)<ref[^>]*/>|<ref[^>]*>.*?</ref>`)
|
||
reFlag = regexp.MustCompile(`\{\{(?:flag|flagcountry|flagicon)\|([^}|]+)(?:\|[^{}]*)?\}\}`)
|
||
reTemplate = regexp.MustCompile(`\{\{[^{}]*\}\}`)
|
||
reLink = regexp.MustCompile(`\[\[(?:[^|\[\]]*\|)?([^|\[\]]+)\]\]`)
|
||
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||
reHeading = regexp.MustCompile(`^==+\s*(.*?)\s*==+\s*$`)
|
||
// 语言转换标记 -{zh-cn:A;zh-tw:B}- 或 -{A}-
|
||
reConv = regexp.MustCompile(`-\{([^{}]*)\}-`)
|
||
// 条目开头的年份：前612年： / 1912年： / 1980年代：
|
||
reYearPrefix = regexp.MustCompile(`^(前)?(\d{1,4})\s*年代?\s*[：:]\s*`)
|
||
reNoYear = regexp.MustCompile(`^(年份不详|年份不詳|不详|不詳|生年不详|生年不詳)\s*[：:]\s*`)
|
||
reLeadPunct = regexp.MustCompile(`^[：:，,、;；\.\s]+`)
|
||
)
|
||
|
||
// daysInMonth 每月天数（含 2 月 29 日，闰日页面维基也有）
|
||
var daysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
|
||
|
||
type dayKey struct {
|
||
Month int
|
||
Day int
|
||
}
|
||
|
||
// allDays 全年 366 天
|
||
func allDays() []dayKey {
|
||
days := make([]dayKey, 0, 366)
|
||
for m := 1; m <= 12; m++ {
|
||
for d := 1; d <= daysInMonth[m-1]; d++ {
|
||
days = append(days, dayKey{Month: m, Day: d})
|
||
}
|
||
}
|
||
return days
|
||
}
|
||
|
||
// IsRunning 是否有同步任务正在执行
|
||
func (s *WikiSyncService) IsRunning() bool {
|
||
return atomic.LoadInt32(&s.running) == 1
|
||
}
|
||
|
||
// coveredDaySet 已覆盖的月日集合
|
||
func (s *WikiSyncService) coveredDaySet() map[dayKey]bool {
|
||
set := make(map[dayKey]bool, 400)
|
||
type row struct {
|
||
Month int
|
||
Day int
|
||
}
|
||
var rows []row
|
||
s.db.Model(&model.WikiOnThisDay{}).Where("status = 1").
|
||
Select("DISTINCT month, day").Scan(&rows)
|
||
for _, r := range rows {
|
||
set[dayKey{Month: r.Month, Day: r.Day}] = true
|
||
}
|
||
return set
|
||
}
|
||
|
||
// missingDays 尚未覆盖的天
|
||
func (s *WikiSyncService) missingDays() []dayKey {
|
||
covered := s.coveredDaySet()
|
||
missing := make([]dayKey, 0)
|
||
for _, d := range allDays() {
|
||
if !covered[d] {
|
||
missing = append(missing, d)
|
||
}
|
||
}
|
||
return missing
|
||
}
|
||
|
||
// TriggerSync 触发一次同步（异步执行）；已有任务执行中时返回 false
|
||
// full=true 全量刷新 366 页；full=false 仅补齐缺失的天
|
||
func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool {
|
||
days := allDays()
|
||
if !full {
|
||
days = s.missingDays()
|
||
if len(days) == 0 {
|
||
return false // 无缺失，无需同步
|
||
}
|
||
}
|
||
if !atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
||
return false
|
||
}
|
||
go func() {
|
||
defer atomic.StoreInt32(&s.running, 0)
|
||
s.runSync(trigger, days)
|
||
}()
|
||
return true
|
||
}
|
||
|
||
// StartScheduler 启动定时同步任务：
|
||
// - 启动时若数据库为空，延迟 10 秒后自动执行首次全量抓取
|
||
// - 之后每天在 syncHour 小时点检查，距上次成功超过 intervalDays 天才执行
|
||
func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) {
|
||
go func() {
|
||
if missing := s.missingDays(); len(missing) > 0 {
|
||
// 数据缺失（首次部署或上次抓取中断），延迟 10 秒等服务稳定后补齐
|
||
log.Printf("wiki sync: 缺失 %d/366 天，10秒后自动补齐", len(missing))
|
||
time.Sleep(10 * time.Second)
|
||
s.TriggerSync("startup", false)
|
||
}
|
||
|
||
if intervalDays <= 0 {
|
||
intervalDays = 7
|
||
}
|
||
if syncHour < 0 || syncHour > 23 {
|
||
syncHour = 4
|
||
}
|
||
ticker := time.NewTicker(time.Hour)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
now := time.Now()
|
||
if now.Hour() != syncHour {
|
||
continue
|
||
}
|
||
if last, ok := s.lastSuccessAt(); ok && time.Since(last) < time.Duration(intervalDays)*24*time.Hour {
|
||
continue // 未到刷新周期
|
||
}
|
||
log.Println("wiki sync: 定时任务触发全量同步")
|
||
s.TriggerSync("cron", true)
|
||
}
|
||
}()
|
||
}
|
||
|
||
// lastSuccessAt 最近一次成功（含部分成功）的同步完成时间
|
||
func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) {
|
||
var syncLog model.WikiSyncLog
|
||
err := s.db.Where("status IN ?", []string{"success", "partial"}).
|
||
Order("finished_at DESC").First(&syncLog).Error
|
||
if err != nil || syncLog.FinishedAt == nil {
|
||
return time.Time{}, false
|
||
}
|
||
return *syncLog.FinishedAt, true
|
||
}
|
||
|
||
// runSync 同步指定日期集合（全量 366 页或缺失补齐）
|
||
func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
||
syncLog := model.WikiSyncLog{
|
||
Trigger: trigger,
|
||
Status: "running",
|
||
Pages: len(days),
|
||
StartedAt: time.Now(),
|
||
}
|
||
s.db.Create(&syncLog)
|
||
|
||
jobs := make(chan dayKey, len(days))
|
||
for _, d := range days {
|
||
jobs <- d
|
||
}
|
||
close(jobs)
|
||
|
||
var success, failed int32
|
||
var wg sync.WaitGroup
|
||
workerCount := 3
|
||
for i := 0; i < workerCount; i++ {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for d := range jobs {
|
||
if err := s.syncOneDay(d.Month, d.Day); err != nil {
|
||
atomic.AddInt32(&failed, 1)
|
||
log.Printf("wiki sync: %d月%d日 失败: %v", d.Month, d.Day, err)
|
||
} else {
|
||
atomic.AddInt32(&success, 1)
|
||
}
|
||
time.Sleep(800 * time.Millisecond) // 控制请求频率，避免被限流
|
||
}
|
||
}()
|
||
}
|
||
wg.Wait()
|
||
|
||
status := "success"
|
||
if failed > 0 {
|
||
status = "partial"
|
||
}
|
||
if success == 0 {
|
||
status = "failed"
|
||
}
|
||
now := time.Now()
|
||
s.db.Model(&syncLog).Updates(map[string]interface{}{
|
||
"status": status,
|
||
"success": int(success),
|
||
"failed": int(failed),
|
||
"finished_at": &now,
|
||
})
|
||
log.Printf("wiki sync: 同步完成 trigger=%s pages=%d success=%d failed=%d", trigger, len(days), success, failed)
|
||
}
|
||
|
||
// syncOneDay 抓取并更新某一天的数据（失败不覆盖旧数据；死锁自动重试）
|
||
func (s *WikiSyncService) syncOneDay(month, day int) error {
|
||
wikitext, err := s.fetchDayPage(month, day)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
items := parseDayPage(wikitext)
|
||
if len(items) == 0 {
|
||
return fmt.Errorf("页面解析结果为空")
|
||
}
|
||
|
||
for i := range items {
|
||
items[i].Month = month
|
||
items[i].Day = day
|
||
items[i].Status = 1
|
||
}
|
||
|
||
// 按天整体替换：先删后插；并发写入同表可能死锁，重试 3 次
|
||
var lastErr error
|
||
for attempt := 0; attempt < 3; attempt++ {
|
||
if attempt > 0 {
|
||
time.Sleep(time.Duration(attempt) * time.Second)
|
||
}
|
||
lastErr = s.db.Transaction(func(tx *gorm.DB) error {
|
||
if err := tx.Where("month = ? AND day = ?", month, day).
|
||
Delete(&model.WikiOnThisDay{}).Error; err != nil {
|
||
return err
|
||
}
|
||
return tx.Create(&items).Error
|
||
})
|
||
if lastErr == nil {
|
||
return nil
|
||
}
|
||
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
|
||
break
|
||
}
|
||
}
|
||
return lastErr
|
||
}
|
||
|
||
// fetchDayPage 抓取「X月X日」页面的 wikitext（失败重试；429 限流时加大退避间隔）
|
||
func (s *WikiSyncService) fetchDayPage(month, day int) (string, error) {
|
||
title := fmt.Sprintf("%d月%d日", month, day)
|
||
apiURL := "https://zh.wikipedia.org/w/api.php?action=parse&prop=wikitext&format=json&formatversion=2&page=" +
|
||
url.QueryEscape(title)
|
||
|
||
var lastErr error
|
||
for attempt := 0; attempt < 4; attempt++ {
|
||
if attempt > 0 {
|
||
backoff := time.Duration(attempt*3) * time.Second
|
||
if lastErr != nil && strings.Contains(lastErr.Error(), "429") {
|
||
backoff = time.Duration(attempt*30) * time.Second // 限流时退避 30s/60s/90s
|
||
}
|
||
time.Sleep(backoff)
|
||
}
|
||
text, err := s.doFetch(apiURL)
|
||
if err == nil {
|
||
return text, nil
|
||
}
|
||
lastErr = err
|
||
}
|
||
return "", lastErr
|
||
}
|
||
|
||
func (s *WikiSyncService) doFetch(apiURL string) (string, error) {
|
||
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
req.Header.Set("User-Agent", "LunarServer/1.0 (wiki on-this-day sync; contact: admin@neatcn.com)")
|
||
|
||
resp, err := s.httpClient.Do(req)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode != http.StatusOK {
|
||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||
}
|
||
|
||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
var result struct {
|
||
Parse struct {
|
||
Wikitext string `json:"wikitext"`
|
||
} `json:"parse"`
|
||
Error struct {
|
||
Info string `json:"info"`
|
||
} `json:"error"`
|
||
}
|
||
if err := json.Unmarshal(body, &result); err != nil {
|
||
return "", fmt.Errorf("JSON 解析失败: %w", err)
|
||
}
|
||
if result.Error.Info != "" {
|
||
return "", fmt.Errorf("API 错误: %s", result.Error.Info)
|
||
}
|
||
if result.Parse.Wikitext == "" {
|
||
return "", fmt.Errorf("页面内容为空")
|
||
}
|
||
return result.Parse.Wikitext, nil
|
||
}
|
||
|
||
// ===== wikitext 解析 =====
|
||
|
||
// classifySection 根据二级标题识别板块类型
|
||
func classifySection(title string) string {
|
||
switch {
|
||
case strings.Contains(title, "大事") || strings.Contains(title, "事件"):
|
||
return "event"
|
||
case strings.Contains(title, "出生"):
|
||
return "birth"
|
||
case strings.Contains(title, "逝世") || strings.Contains(title, "去世"):
|
||
return "death"
|
||
case strings.Contains(title, "节假日") || strings.Contains(title, "节日") ||
|
||
strings.Contains(title, "節日") || strings.Contains(title, "假日") ||
|
||
strings.Contains(title, "习俗") || strings.Contains(title, "風俗") ||
|
||
strings.Contains(title, "风俗"):
|
||
return "festival"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// parseDayPage 解析页面 wikitext，提取大事记/出生/逝世/节假日条目
|
||
func parseDayPage(wikitext string) []model.WikiOnThisDay {
|
||
items := make([]model.WikiOnThisDay, 0, 128)
|
||
seen := make(map[string]bool) // 同板块去重（维基页面偶有重复条目）
|
||
|
||
curKind := ""
|
||
inTable := false
|
||
for _, rawLine := range strings.Split(wikitext, "\n") {
|
||
line := strings.TrimSpace(rawLine)
|
||
|
||
// 跳过表格区块
|
||
if strings.HasPrefix(line, "{|") {
|
||
inTable = true
|
||
continue
|
||
}
|
||
if inTable {
|
||
if strings.HasPrefix(line, "|}") {
|
||
inTable = false
|
||
}
|
||
continue
|
||
}
|
||
|
||
// 二级/三级标题切换板块（三级标题归属当前二级板块）
|
||
if m := reHeading.FindStringSubmatch(line); m != nil {
|
||
if strings.HasPrefix(line, "=== ") || strings.HasPrefix(line, "===") {
|
||
continue // 三级标题：大事记内的世纪分组，不改变板块
|
||
}
|
||
curKind = classifySection(m[1])
|
||
continue
|
||
}
|
||
|
||
if curKind == "" || !strings.HasPrefix(line, "*") || strings.HasPrefix(line, "*>") {
|
||
continue
|
||
}
|
||
|
||
content := strings.TrimSpace(strings.TrimLeft(line, "*"))
|
||
if content == "" {
|
||
continue
|
||
}
|
||
year, text := extractYear(cleanWikitext(content))
|
||
if len([]rune(text)) < 4 { // 过短条目无展示价值
|
||
continue
|
||
}
|
||
if len([]rune(text)) > 500 {
|
||
text = string([]rune(text)[:500])
|
||
}
|
||
|
||
dedupeKey := curKind + "|" + text
|
||
if seen[dedupeKey] {
|
||
continue
|
||
}
|
||
seen[dedupeKey] = true
|
||
|
||
items = append(items, model.WikiOnThisDay{
|
||
Kind: curKind,
|
||
Year: year,
|
||
Content: text,
|
||
})
|
||
}
|
||
return items
|
||
}
|
||
|
||
// extractYear 从条目文本提取年份：前612年→-612，1912年→1912，无年份→0
|
||
func extractYear(text string) (int, string) {
|
||
text = strings.TrimSpace(text)
|
||
if m := reYearPrefix.FindStringSubmatch(text); m != nil {
|
||
year, err := strconv.Atoi(m[2])
|
||
if err == nil {
|
||
if m[1] == "前" {
|
||
year = -year
|
||
}
|
||
return year, strings.TrimSpace(text[len(m[0]):])
|
||
}
|
||
}
|
||
if m := reNoYear.FindStringSubmatch(text); m != nil {
|
||
return 0, strings.TrimSpace(text[len(m[0]):])
|
||
}
|
||
return 0, text
|
||
}
|
||
|
||
// convertConvMarkup 处理语言转换标记 -{zh-cn:A;zh-tw:B}-，优先取简体变体
|
||
func convertConvMarkup(s string) string {
|
||
return reConv.ReplaceAllStringFunc(s, func(m string) string {
|
||
inner := m[2 : len(m)-2] // 去掉 -{ 和 }-
|
||
parts := strings.Split(inner, ";")
|
||
// 无冒号：-{A}- 直接取内容
|
||
if !strings.Contains(inner, ":") {
|
||
return strings.TrimSpace(inner)
|
||
}
|
||
fallback := ""
|
||
for _, p := range parts {
|
||
p = strings.TrimSpace(p)
|
||
if p == "" {
|
||
continue
|
||
}
|
||
kv := strings.SplitN(p, ":", 2)
|
||
if len(kv) != 2 {
|
||
continue
|
||
}
|
||
key := strings.ToLower(strings.TrimSpace(kv[0]))
|
||
val := strings.TrimSpace(kv[1])
|
||
if key == "zh-cn" || key == "zh-hans" {
|
||
return val
|
||
}
|
||
fallback = val
|
||
}
|
||
return fallback
|
||
})
|
||
}
|
||
|
||
// cleanWikitext 去除 wiki 语法，输出纯文本
|
||
func cleanWikitext(s string) string {
|
||
s = reComment.ReplaceAllString(s, "")
|
||
s = reRef.ReplaceAllString(s, "")
|
||
s = convertConvMarkup(s)
|
||
// 国旗模板保留国家名
|
||
s = reFlag.ReplaceAllString(s, "$1")
|
||
// 模板可能嵌套，循环剥离直至稳定
|
||
for i := 0; i < 6; i++ {
|
||
next := reTemplate.ReplaceAllString(s, "")
|
||
if next == s {
|
||
break
|
||
}
|
||
s = next
|
||
}
|
||
s = reLink.ReplaceAllString(s, "$1")
|
||
s = strings.ReplaceAll(s, "'''", "")
|
||
s = strings.ReplaceAll(s, "''", "")
|
||
s = reHTMLTag.ReplaceAllString(s, "")
|
||
s = strings.ReplaceAll(s, " ", " ")
|
||
s = strings.TrimSpace(s)
|
||
s = reLeadPunct.ReplaceAllString(s, "") // 模板被剥离后可能残留开头标点
|
||
return strings.TrimSpace(s)
|
||
}
|