feat(home): 修复swiper动画、宜忌布局、月历显示、节日速查
1. 修复swiper箭头点击动画丢失问题 - 改用固定7天窗口,滑动到边缘时整体滚动数据 - 保持dayIndex连续,确保swiper平滑动画 2. 修复宜忌两行显示压在一起 - 设置item固定高度32rpx和行距12rpx - 超出2行显示...省略号 3. 修复月历数字竖向排列问题 - 修正wxml嵌套结构,让42个格子正确排成6行7列 - 压缩格子高度,放大数字字体 4. 节日速查功能优化 - 默认显示3条,右侧添加更多
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/gouki/lunar-server/internal/config"
|
||||
"github.com/gouki/lunar-server/internal/model"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// WikiService 百科服务
|
||||
type WikiService struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewWikiService 创建百科服务
|
||||
func NewWikiService() *WikiService {
|
||||
return &WikiService{db: config.GetDB()}
|
||||
}
|
||||
|
||||
// ListEntries 百科条目列表(status<0 表示不过滤状态,供管理后台使用)
|
||||
func (s *WikiService) ListEntries(category, keyword string, status int) ([]*model.WikiEntry, error) {
|
||||
query := s.db.Model(&model.WikiEntry{})
|
||||
if status >= 0 {
|
||||
query = query.Where("status = ?", status)
|
||||
}
|
||||
if category != "" {
|
||||
query = query.Where("category = ?", category)
|
||||
}
|
||||
if keyword != "" {
|
||||
like := "%" + keyword + "%"
|
||||
query = query.Where("title LIKE ? OR brief LIKE ? OR content LIKE ?", like, like, like)
|
||||
}
|
||||
|
||||
var entries []*model.WikiEntry
|
||||
if err := query.Order("sort ASC, id ASC").Find(&entries).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// CreateEntry 新增百科条目
|
||||
func (s *WikiService) CreateEntry(entry *model.WikiEntry) error {
|
||||
if entry.Category == "" || entry.Title == "" {
|
||||
return errors.New("分类和标题不能为空")
|
||||
}
|
||||
if entry.Status == 0 {
|
||||
entry.Status = 1
|
||||
}
|
||||
return s.db.Create(entry).Error
|
||||
}
|
||||
|
||||
// UpdateEntry 更新百科条目
|
||||
func (s *WikiService) UpdateEntry(id uint, entry *model.WikiEntry) error {
|
||||
var existing model.WikiEntry
|
||||
if err := s.db.First(&existing, id).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return s.db.Model(&existing).Updates(map[string]interface{}{
|
||||
"category": entry.Category,
|
||||
"title": entry.Title,
|
||||
"brief": entry.Brief,
|
||||
"content": entry.Content,
|
||||
"sort": entry.Sort,
|
||||
"status": entry.Status,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// DeleteEntry 删除百科条目
|
||||
func (s *WikiService) DeleteEntry(id uint) error {
|
||||
return s.db.Delete(&model.WikiEntry{}, id).Error
|
||||
}
|
||||
|
||||
// OnThisDayResult 某一天的历史数据
|
||||
type OnThisDayResult struct {
|
||||
Events []model.WikiOnThisDay `json:"events"`
|
||||
Births []model.WikiOnThisDay `json:"births"`
|
||||
Deaths []model.WikiOnThisDay `json:"deaths"`
|
||||
Festivals []model.WikiOnThisDay `json:"festivals"`
|
||||
}
|
||||
|
||||
// GetOnThisDay 查询某一天的历史上的今天数据(各板块限量返回)
|
||||
func (s *WikiService) GetOnThisDay(month, day int) (*OnThisDayResult, error) {
|
||||
if month < 1 || month > 12 || day < 1 || day > 31 {
|
||||
return nil, errors.New("日期参数无效")
|
||||
}
|
||||
|
||||
var all []model.WikiOnThisDay
|
||||
if err := s.db.Where("month = ? AND day = ? AND status = 1", month, day).
|
||||
Order("year ASC, id ASC").
|
||||
Find(&all).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &OnThisDayResult{
|
||||
Events: make([]model.WikiOnThisDay, 0),
|
||||
Births: make([]model.WikiOnThisDay, 0),
|
||||
Deaths: make([]model.WikiOnThisDay, 0),
|
||||
Festivals: make([]model.WikiOnThisDay, 0),
|
||||
}
|
||||
limits := map[string]int{"event": 20, "birth": 15, "death": 10, "festival": 10}
|
||||
for _, item := range all {
|
||||
switch item.Kind {
|
||||
case "event":
|
||||
if len(result.Events) < limits["event"] {
|
||||
result.Events = append(result.Events, item)
|
||||
}
|
||||
case "birth":
|
||||
if len(result.Births) < limits["birth"] {
|
||||
result.Births = append(result.Births, item)
|
||||
}
|
||||
case "death":
|
||||
if len(result.Deaths) < limits["death"] {
|
||||
result.Deaths = append(result.Deaths, item)
|
||||
}
|
||||
case "festival":
|
||||
if len(result.Festivals) < limits["festival"] {
|
||||
result.Festivals = append(result.Festivals, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// SyncStatus 同步状态(管理后台展示)
|
||||
type SyncStatus struct {
|
||||
Running bool `json:"running"`
|
||||
TotalDays int64 `json:"totalDays"` // 已覆盖的天数
|
||||
TotalItems int64 `json:"totalItems"` // 总条目数
|
||||
LastSync *model.WikiSyncLog `json:"lastSync"`
|
||||
}
|
||||
|
||||
// GetSyncStatus 获取同步状态
|
||||
func (s *WikiService) GetSyncStatus(running bool) (*SyncStatus, error) {
|
||||
status := &SyncStatus{Running: running}
|
||||
|
||||
var totalDays int64
|
||||
s.db.Model(&model.WikiOnThisDay{}).Where("status = 1").
|
||||
Select("COUNT(DISTINCT month, day) AS c").Scan(&totalDays)
|
||||
status.TotalDays = totalDays
|
||||
s.db.Model(&model.WikiOnThisDay{}).Where("status = 1").Count(&status.TotalItems)
|
||||
|
||||
var lastSync model.WikiSyncLog
|
||||
if err := s.db.Order("id DESC").First(&lastSync).Error; err == nil {
|
||||
status.LastSync = &lastSync
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gouki/lunar-server/internal/model"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// WikiSyncService 维基百科「X月X日」页面同步服务
|
||||
// 全量共 366 页(含 2 月 29 日),首次抓取后按配置间隔定期增量刷新
|
||||
type WikiSyncService struct {
|
||||
db *gorm.DB
|
||||
|
||||
running int32 // 原子标记:是否有同步正在执行
|
||||
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewWikiSyncService 创建同步服务
|
||||
func NewWikiSyncService(db *gorm.DB) *WikiSyncService {
|
||||
return &WikiSyncService{
|
||||
db: db,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
reComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
reRef = regexp.MustCompile(`(?is)<ref[^>]*/>|<ref[^>]*>.*?</ref>`)
|
||||
reFlag = regexp.MustCompile(`\{\{(?:flag|flagcountry|flagicon)\|([^}|]+)(?:\|[^{}]*)?\}\}`)
|
||||
reTemplate = regexp.MustCompile(`\{\{[^{}]*\}\}`)
|
||||
reLink = regexp.MustCompile(`\[\[(?:[^|\[\]]*\|)?([^|\[\]]+)\]\]`)
|
||||
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||||
reHeading = regexp.MustCompile(`^==+\s*(.*?)\s*==+\s*$`)
|
||||
// 语言转换标记 -{zh-cn:A;zh-tw:B}- 或 -{A}-
|
||||
reConv = regexp.MustCompile(`-\{([^{}]*)\}-`)
|
||||
// 条目开头的年份:前612年: / 1912年: / 1980年代:
|
||||
reYearPrefix = regexp.MustCompile(`^(前)?(\d{1,4})\s*年代?\s*[::]\s*`)
|
||||
reNoYear = regexp.MustCompile(`^(年份不详|年份不詳|不详|不詳|生年不详|生年不詳)\s*[::]\s*`)
|
||||
reLeadPunct = regexp.MustCompile(`^[::,,、;;\.\s]+`)
|
||||
)
|
||||
|
||||
// daysInMonth 每月天数(含 2 月 29 日,闰日页面维基也有)
|
||||
var daysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
|
||||
|
||||
type dayKey struct {
|
||||
Month int
|
||||
Day int
|
||||
}
|
||||
|
||||
// allDays 全年 366 天
|
||||
func allDays() []dayKey {
|
||||
days := make([]dayKey, 0, 366)
|
||||
for m := 1; m <= 12; m++ {
|
||||
for d := 1; d <= daysInMonth[m-1]; d++ {
|
||||
days = append(days, dayKey{Month: m, Day: d})
|
||||
}
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
// IsRunning 是否有同步任务正在执行
|
||||
func (s *WikiSyncService) IsRunning() bool {
|
||||
return atomic.LoadInt32(&s.running) == 1
|
||||
}
|
||||
|
||||
// coveredDaySet 已覆盖的月日集合
|
||||
func (s *WikiSyncService) coveredDaySet() map[dayKey]bool {
|
||||
set := make(map[dayKey]bool, 400)
|
||||
type row struct {
|
||||
Month int
|
||||
Day int
|
||||
}
|
||||
var rows []row
|
||||
s.db.Model(&model.WikiOnThisDay{}).Where("status = 1").
|
||||
Select("DISTINCT month, day").Scan(&rows)
|
||||
for _, r := range rows {
|
||||
set[dayKey{Month: r.Month, Day: r.Day}] = true
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
// missingDays 尚未覆盖的天
|
||||
func (s *WikiSyncService) missingDays() []dayKey {
|
||||
covered := s.coveredDaySet()
|
||||
missing := make([]dayKey, 0)
|
||||
for _, d := range allDays() {
|
||||
if !covered[d] {
|
||||
missing = append(missing, d)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
// TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false
|
||||
// full=true 全量刷新 366 页;full=false 仅补齐缺失的天
|
||||
func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool {
|
||||
days := allDays()
|
||||
if !full {
|
||||
days = s.missingDays()
|
||||
if len(days) == 0 {
|
||||
return false // 无缺失,无需同步
|
||||
}
|
||||
}
|
||||
if !atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
||||
return false
|
||||
}
|
||||
go func() {
|
||||
defer atomic.StoreInt32(&s.running, 0)
|
||||
s.runSync(trigger, days)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// StartScheduler 启动定时同步任务:
|
||||
// - 启动时若数据库为空,延迟 10 秒后自动执行首次全量抓取
|
||||
// - 之后每天在 syncHour 小时点检查,距上次成功超过 intervalDays 天才执行
|
||||
func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) {
|
||||
go func() {
|
||||
if missing := s.missingDays(); len(missing) > 0 {
|
||||
// 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐
|
||||
log.Printf("wiki sync: 缺失 %d/366 天,10秒后自动补齐", len(missing))
|
||||
time.Sleep(10 * time.Second)
|
||||
s.TriggerSync("startup", false)
|
||||
}
|
||||
|
||||
if intervalDays <= 0 {
|
||||
intervalDays = 7
|
||||
}
|
||||
if syncHour < 0 || syncHour > 23 {
|
||||
syncHour = 4
|
||||
}
|
||||
ticker := time.NewTicker(time.Hour)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
now := time.Now()
|
||||
if now.Hour() != syncHour {
|
||||
continue
|
||||
}
|
||||
if last, ok := s.lastSuccessAt(); ok && time.Since(last) < time.Duration(intervalDays)*24*time.Hour {
|
||||
continue // 未到刷新周期
|
||||
}
|
||||
log.Println("wiki sync: 定时任务触发全量同步")
|
||||
s.TriggerSync("cron", true)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// lastSuccessAt 最近一次成功(含部分成功)的同步完成时间
|
||||
func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) {
|
||||
var syncLog model.WikiSyncLog
|
||||
err := s.db.Where("status IN ?", []string{"success", "partial"}).
|
||||
Order("finished_at DESC").First(&syncLog).Error
|
||||
if err != nil || syncLog.FinishedAt == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return *syncLog.FinishedAt, true
|
||||
}
|
||||
|
||||
// runSync 同步指定日期集合(全量 366 页或缺失补齐)
|
||||
func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
||||
syncLog := model.WikiSyncLog{
|
||||
Trigger: trigger,
|
||||
Status: "running",
|
||||
Pages: len(days),
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
s.db.Create(&syncLog)
|
||||
|
||||
jobs := make(chan dayKey, len(days))
|
||||
for _, d := range days {
|
||||
jobs <- d
|
||||
}
|
||||
close(jobs)
|
||||
|
||||
var success, failed int32
|
||||
var wg sync.WaitGroup
|
||||
workerCount := 3
|
||||
for i := 0; i < workerCount; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for d := range jobs {
|
||||
if err := s.syncOneDay(d.Month, d.Day); err != nil {
|
||||
atomic.AddInt32(&failed, 1)
|
||||
log.Printf("wiki sync: %d月%d日 失败: %v", d.Month, d.Day, err)
|
||||
} else {
|
||||
atomic.AddInt32(&success, 1)
|
||||
}
|
||||
time.Sleep(800 * time.Millisecond) // 控制请求频率,避免被限流
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
status := "success"
|
||||
if failed > 0 {
|
||||
status = "partial"
|
||||
}
|
||||
if success == 0 {
|
||||
status = "failed"
|
||||
}
|
||||
now := time.Now()
|
||||
s.db.Model(&syncLog).Updates(map[string]interface{}{
|
||||
"status": status,
|
||||
"success": int(success),
|
||||
"failed": int(failed),
|
||||
"finished_at": &now,
|
||||
})
|
||||
log.Printf("wiki sync: 同步完成 trigger=%s pages=%d success=%d failed=%d", trigger, len(days), success, failed)
|
||||
}
|
||||
|
||||
// syncOneDay 抓取并更新某一天的数据(失败不覆盖旧数据;死锁自动重试)
|
||||
func (s *WikiSyncService) syncOneDay(month, day int) error {
|
||||
wikitext, err := s.fetchDayPage(month, day)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items := parseDayPage(wikitext)
|
||||
if len(items) == 0 {
|
||||
return fmt.Errorf("页面解析结果为空")
|
||||
}
|
||||
|
||||
for i := range items {
|
||||
items[i].Month = month
|
||||
items[i].Day = day
|
||||
items[i].Status = 1
|
||||
}
|
||||
|
||||
// 按天整体替换:先删后插;并发写入同表可能死锁,重试 3 次
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(time.Duration(attempt) * time.Second)
|
||||
}
|
||||
lastErr = s.db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Where("month = ? AND day = ?", month, day).
|
||||
Delete(&model.WikiOnThisDay{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Create(&items).Error
|
||||
})
|
||||
if lastErr == nil {
|
||||
return nil
|
||||
}
|
||||
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
|
||||
break
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// fetchDayPage 抓取「X月X日」页面的 wikitext(失败重试;429 限流时加大退避间隔)
|
||||
func (s *WikiSyncService) fetchDayPage(month, day int) (string, error) {
|
||||
title := fmt.Sprintf("%d月%d日", month, day)
|
||||
apiURL := "https://zh.wikipedia.org/w/api.php?action=parse&prop=wikitext&format=json&formatversion=2&page=" +
|
||||
url.QueryEscape(title)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 4; attempt++ {
|
||||
if attempt > 0 {
|
||||
backoff := time.Duration(attempt*3) * time.Second
|
||||
if lastErr != nil && strings.Contains(lastErr.Error(), "429") {
|
||||
backoff = time.Duration(attempt*30) * time.Second // 限流时退避 30s/60s/90s
|
||||
}
|
||||
time.Sleep(backoff)
|
||||
}
|
||||
text, err := s.doFetch(apiURL)
|
||||
if err == nil {
|
||||
return text, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return "", lastErr
|
||||
}
|
||||
|
||||
func (s *WikiSyncService) doFetch(apiURL string) (string, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "LunarServer/1.0 (wiki on-this-day sync; contact: admin@neatcn.com)")
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Parse struct {
|
||||
Wikitext string `json:"wikitext"`
|
||||
} `json:"parse"`
|
||||
Error struct {
|
||||
Info string `json:"info"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return "", fmt.Errorf("JSON 解析失败: %w", err)
|
||||
}
|
||||
if result.Error.Info != "" {
|
||||
return "", fmt.Errorf("API 错误: %s", result.Error.Info)
|
||||
}
|
||||
if result.Parse.Wikitext == "" {
|
||||
return "", fmt.Errorf("页面内容为空")
|
||||
}
|
||||
return result.Parse.Wikitext, nil
|
||||
}
|
||||
|
||||
// ===== wikitext 解析 =====
|
||||
|
||||
// classifySection 根据二级标题识别板块类型
|
||||
func classifySection(title string) string {
|
||||
switch {
|
||||
case strings.Contains(title, "大事") || strings.Contains(title, "事件"):
|
||||
return "event"
|
||||
case strings.Contains(title, "出生"):
|
||||
return "birth"
|
||||
case strings.Contains(title, "逝世") || strings.Contains(title, "去世"):
|
||||
return "death"
|
||||
case strings.Contains(title, "节假日") || strings.Contains(title, "节日") ||
|
||||
strings.Contains(title, "節日") || strings.Contains(title, "假日") ||
|
||||
strings.Contains(title, "习俗") || strings.Contains(title, "風俗") ||
|
||||
strings.Contains(title, "风俗"):
|
||||
return "festival"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseDayPage 解析页面 wikitext,提取大事记/出生/逝世/节假日条目
|
||||
func parseDayPage(wikitext string) []model.WikiOnThisDay {
|
||||
items := make([]model.WikiOnThisDay, 0, 128)
|
||||
seen := make(map[string]bool) // 同板块去重(维基页面偶有重复条目)
|
||||
|
||||
curKind := ""
|
||||
inTable := false
|
||||
for _, rawLine := range strings.Split(wikitext, "\n") {
|
||||
line := strings.TrimSpace(rawLine)
|
||||
|
||||
// 跳过表格区块
|
||||
if strings.HasPrefix(line, "{|") {
|
||||
inTable = true
|
||||
continue
|
||||
}
|
||||
if inTable {
|
||||
if strings.HasPrefix(line, "|}") {
|
||||
inTable = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// 二级/三级标题切换板块(三级标题归属当前二级板块)
|
||||
if m := reHeading.FindStringSubmatch(line); m != nil {
|
||||
if strings.HasPrefix(line, "=== ") || strings.HasPrefix(line, "===") {
|
||||
continue // 三级标题:大事记内的世纪分组,不改变板块
|
||||
}
|
||||
curKind = classifySection(m[1])
|
||||
continue
|
||||
}
|
||||
|
||||
if curKind == "" || !strings.HasPrefix(line, "*") || strings.HasPrefix(line, "*>") {
|
||||
continue
|
||||
}
|
||||
|
||||
content := strings.TrimSpace(strings.TrimLeft(line, "*"))
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
year, text := extractYear(cleanWikitext(content))
|
||||
if len([]rune(text)) < 4 { // 过短条目无展示价值
|
||||
continue
|
||||
}
|
||||
if len([]rune(text)) > 500 {
|
||||
text = string([]rune(text)[:500])
|
||||
}
|
||||
|
||||
dedupeKey := curKind + "|" + text
|
||||
if seen[dedupeKey] {
|
||||
continue
|
||||
}
|
||||
seen[dedupeKey] = true
|
||||
|
||||
items = append(items, model.WikiOnThisDay{
|
||||
Kind: curKind,
|
||||
Year: year,
|
||||
Content: text,
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// extractYear 从条目文本提取年份:前612年→-612,1912年→1912,无年份→0
|
||||
func extractYear(text string) (int, string) {
|
||||
text = strings.TrimSpace(text)
|
||||
if m := reYearPrefix.FindStringSubmatch(text); m != nil {
|
||||
year, err := strconv.Atoi(m[2])
|
||||
if err == nil {
|
||||
if m[1] == "前" {
|
||||
year = -year
|
||||
}
|
||||
return year, strings.TrimSpace(text[len(m[0]):])
|
||||
}
|
||||
}
|
||||
if m := reNoYear.FindStringSubmatch(text); m != nil {
|
||||
return 0, strings.TrimSpace(text[len(m[0]):])
|
||||
}
|
||||
return 0, text
|
||||
}
|
||||
|
||||
// convertConvMarkup 处理语言转换标记 -{zh-cn:A;zh-tw:B}-,优先取简体变体
|
||||
func convertConvMarkup(s string) string {
|
||||
return reConv.ReplaceAllStringFunc(s, func(m string) string {
|
||||
inner := m[2 : len(m)-2] // 去掉 -{ 和 }-
|
||||
parts := strings.Split(inner, ";")
|
||||
// 无冒号:-{A}- 直接取内容
|
||||
if !strings.Contains(inner, ":") {
|
||||
return strings.TrimSpace(inner)
|
||||
}
|
||||
fallback := ""
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
kv := strings.SplitN(p, ":", 2)
|
||||
if len(kv) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(strings.TrimSpace(kv[0]))
|
||||
val := strings.TrimSpace(kv[1])
|
||||
if key == "zh-cn" || key == "zh-hans" {
|
||||
return val
|
||||
}
|
||||
fallback = val
|
||||
}
|
||||
return fallback
|
||||
})
|
||||
}
|
||||
|
||||
// cleanWikitext 去除 wiki 语法,输出纯文本
|
||||
func cleanWikitext(s string) string {
|
||||
s = reComment.ReplaceAllString(s, "")
|
||||
s = reRef.ReplaceAllString(s, "")
|
||||
s = convertConvMarkup(s)
|
||||
// 国旗模板保留国家名
|
||||
s = reFlag.ReplaceAllString(s, "$1")
|
||||
// 模板可能嵌套,循环剥离直至稳定
|
||||
for i := 0; i < 6; i++ {
|
||||
next := reTemplate.ReplaceAllString(s, "")
|
||||
if next == s {
|
||||
break
|
||||
}
|
||||
s = next
|
||||
}
|
||||
s = reLink.ReplaceAllString(s, "$1")
|
||||
s = strings.ReplaceAll(s, "'''", "")
|
||||
s = strings.ReplaceAll(s, "''", "")
|
||||
s = reHTMLTag.ReplaceAllString(s, "")
|
||||
s = strings.ReplaceAll(s, " ", " ")
|
||||
s = strings.TrimSpace(s)
|
||||
s = reLeadPunct.ReplaceAllString(s, "") // 模板被剥离后可能残留开头标点
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 取自 zh.wikipedia.org「8月10日」页面的真实片段
|
||||
const sampleWikitext = `'''8月10日'''是[[阳历]]年的第222天。
|
||||
|
||||
== 大事记 ==
|
||||
=== 19世紀以前 ===
|
||||
* [[前612年]]:[[亚述]]最后一位君主[[辛·沙里施昆]]死于[[尼尼微]],王国灭亡。
|
||||
* [[955年]]:[[鄂圖一世 (神聖羅馬帝國)|鄂圖一世]]麾下的軍隊在[[第二次萊希菲爾德之戰 (955)|萊希菲爾德之戰]]中取得勝利。<ref>注释</ref>
|
||||
=== 20世紀 ===
|
||||
* [[1912年]]:[[威廉·布利斯]]出版了第一本关于[[童子军]]的书。
|
||||
* [[1945年]]:{{le|蒙古}}对日宣战。
|
||||
* [[2002年]]:-{zh-hans:凯尔·斯科特;zh-hk:卡爾·史葛;zh-tw:凱爾·斯科特;}-,英國職業足球運動員。
|
||||
* 年份不详:某件无法确定年份的事情发生了。
|
||||
|
||||
== 出生 ==
|
||||
* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
|
||||
* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
|
||||
* [[1874年]]:[[赫伯特·胡佛]],美国第31任[[总统]]([[1964年]]逝世)
|
||||
|
||||
== 逝世 ==
|
||||
* [[258年]]:[[羅馬的聖老楞佐]],基督教殉道者
|
||||
|
||||
== 节假日和习俗 ==
|
||||
* {{flag|厄瓜多尔}}:独立日
|
||||
* 世界[[狮子]]日
|
||||
|
||||
==参考资料==
|
||||
{{Commonscat}}
|
||||
`
|
||||
|
||||
func TestParseDayPage(t *testing.T) {
|
||||
items := parseDayPage(sampleWikitext)
|
||||
|
||||
count := func(kind string) int {
|
||||
n := 0
|
||||
for _, it := range items {
|
||||
if it.Kind == kind {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
if got := count("event"); got != 6 {
|
||||
t.Errorf("event 条目数 = %d, 期望 6", got)
|
||||
}
|
||||
if got := count("birth"); got != 2 { // 重复条目应去重
|
||||
t.Errorf("birth 条目数 = %d, 期望 2", got)
|
||||
}
|
||||
if got := count("death"); got != 1 {
|
||||
t.Errorf("death 条目数 = %d, 期望 1", got)
|
||||
}
|
||||
if got := count("festival"); got != 2 {
|
||||
t.Errorf("festival 条目数 = %d, 期望 2", got)
|
||||
}
|
||||
|
||||
// 校验具体条目的年份与文本清洗
|
||||
assertItem := func(kind string, wantYear int, wantContent string) {
|
||||
for _, it := range items {
|
||||
if it.Kind == kind && it.Content == wantContent {
|
||||
if it.Year != wantYear {
|
||||
t.Errorf("%q 年份 = %d, 期望 %d", wantContent, it.Year, wantYear)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("未找到条目: kind=%s content=%q", kind, wantContent)
|
||||
}
|
||||
assertItem("event", -612, "亚述最后一位君主辛·沙里施昆死于尼尼微,王国灭亡。")
|
||||
assertItem("event", 955, "鄂圖一世麾下的軍隊在萊希菲爾德之戰中取得勝利。")
|
||||
assertItem("event", 0, "某件无法确定年份的事情发生了。")
|
||||
assertItem("event", 2002, "凯尔·斯科特,英國職業足球運動員。")
|
||||
assertItem("festival", 0, "厄瓜多尔:独立日")
|
||||
assertItem("birth", -156, "汉武帝刘彻,西汉皇帝(前87年逝世)")
|
||||
assertItem("festival", 0, "世界狮子日")
|
||||
|
||||
// 不应包含 wiki 语法残留
|
||||
for _, it := range items {
|
||||
if strings.Contains(it.Content, "[[") || strings.Contains(it.Content, "{{") ||
|
||||
strings.Contains(it.Content, "<ref") {
|
||||
t.Errorf("条目残留 wiki 语法: %q", it.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseRealPage 若本地存在真实抓取的样本文件则验证(可选)
|
||||
func TestParseRealPage(t *testing.T) {
|
||||
data, err := os.ReadFile("/tmp/wiki_810.json")
|
||||
if err != nil {
|
||||
t.Skip("无真实样本文件,跳过")
|
||||
}
|
||||
var d struct {
|
||||
Parse struct {
|
||||
Wikitext string `json:"wikitext"`
|
||||
} `json:"parse"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &d); err != nil {
|
||||
t.Fatalf("样本解析失败: %v", err)
|
||||
}
|
||||
items := parseDayPage(d.Parse.Wikitext)
|
||||
if len(items) < 20 {
|
||||
t.Errorf("真实页面解析条目过少: %d", len(items))
|
||||
}
|
||||
t.Logf("真实页面共解析 %d 条", len(items))
|
||||
}
|
||||
Reference in New Issue
Block a user