Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0ce1a1ef97 | ||
|
|
cc7f30191e | ||
|
|
e0b447ddb1 |
@@ -6,7 +6,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -18,8 +17,9 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WikiSyncService 维基百科「X月X日」页面同步服务
|
// WikiSyncService 「历史上的今天」同步服务
|
||||||
// 全量共 366 页(含 2 月 29 日),首次抓取后按配置间隔定期增量刷新
|
// 数据源:百度百科「历史上的今天」开放接口(按月返回结构化 JSON,共 12 次请求)
|
||||||
|
// 首次抓取后按配置间隔定期增量刷新
|
||||||
type WikiSyncService struct {
|
type WikiSyncService struct {
|
||||||
db *gorm.DB
|
db *gorm.DB
|
||||||
|
|
||||||
@@ -39,32 +39,21 @@ func NewWikiSyncService(db *gorm.DB) *WikiSyncService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
reComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
|
||||||
reRef = regexp.MustCompile(`(?is)<ref[^>]*/>|<ref[^>]*>.*?</ref>`)
|
|
||||||
reFlag = regexp.MustCompile(`\{\{(?:flag|flagcountry|flagicon)\|([^}|]+)(?:\|[^{}]*)?\}\}`)
|
|
||||||
reTemplate = regexp.MustCompile(`\{\{[^{}]*\}\}`)
|
|
||||||
reLink = regexp.MustCompile(`\[\[(?:[^|\[\]]*\|)?([^|\[\]]+)\]\]`)
|
|
||||||
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||||||
reHeading = regexp.MustCompile(`^==+\s*(.*?)\s*==+\s*$`)
|
reSpace = regexp.MustCompile(`\s+`)
|
||||||
// 语言转换标记 -{zh-cn:A;zh-tw:B}- 或 -{A}-
|
|
||||||
reConv = regexp.MustCompile(`-\{([^{}]*)\}-`)
|
|
||||||
// 条目开头的年份:前612年: / 1912年: / 1980年代:
|
|
||||||
reYearPrefix = regexp.MustCompile(`^(前)?(\d{1,4})\s*年代?\s*[::]\s*`)
|
|
||||||
reNoYear = regexp.MustCompile(`^(年份不详|年份不詳|不详|不詳|生年不详|生年不詳)\s*[::]\s*`)
|
|
||||||
reLeadPunct = regexp.MustCompile(`^[::,,、;;\.\s]+`)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// daysInMonth 每月天数(含 2 月 29 日,闰日页面维基也有)
|
// daysInMonth 每月天数(百度百科数据源不含 2 月 29 日,共 365 天)
|
||||||
var daysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
|
var daysInMonth = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
|
||||||
|
|
||||||
type dayKey struct {
|
type dayKey struct {
|
||||||
Month int
|
Month int
|
||||||
Day int
|
Day int
|
||||||
}
|
}
|
||||||
|
|
||||||
// allDays 全年 366 天
|
// allDays 全年 365 天
|
||||||
func allDays() []dayKey {
|
func allDays() []dayKey {
|
||||||
days := make([]dayKey, 0, 366)
|
days := make([]dayKey, 0, 365)
|
||||||
for m := 1; m <= 12; m++ {
|
for m := 1; m <= 12; m++ {
|
||||||
for d := 1; d <= daysInMonth[m-1]; d++ {
|
for d := 1; d <= daysInMonth[m-1]; d++ {
|
||||||
days = append(days, dayKey{Month: m, Day: d})
|
days = append(days, dayKey{Month: m, Day: d})
|
||||||
@@ -107,7 +96,7 @@ func (s *WikiSyncService) missingDays() []dayKey {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false
|
// TriggerSync 触发一次同步(异步执行);已有任务执行中时返回 false
|
||||||
// full=true 全量刷新 366 页;full=false 仅补齐缺失的天
|
// full=true 全量刷新 365 天;full=false 仅补齐缺失的天
|
||||||
func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool {
|
func (s *WikiSyncService) TriggerSync(trigger string, full bool) bool {
|
||||||
days := allDays()
|
days := allDays()
|
||||||
if !full {
|
if !full {
|
||||||
@@ -133,7 +122,7 @@ func (s *WikiSyncService) StartScheduler(intervalDays, syncHour int) {
|
|||||||
go func() {
|
go func() {
|
||||||
if missing := s.missingDays(); len(missing) > 0 {
|
if missing := s.missingDays(); len(missing) > 0 {
|
||||||
// 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐
|
// 数据缺失(首次部署或上次抓取中断),延迟 10 秒等服务稳定后补齐
|
||||||
log.Printf("wiki sync: 缺失 %d/366 天,10秒后自动补齐", len(missing))
|
log.Printf("wiki sync: 缺失 %d/365 天,10秒后自动补齐", len(missing))
|
||||||
time.Sleep(10 * time.Second)
|
time.Sleep(10 * time.Second)
|
||||||
s.TriggerSync("startup", false)
|
s.TriggerSync("startup", false)
|
||||||
}
|
}
|
||||||
@@ -171,7 +160,8 @@ func (s *WikiSyncService) lastSuccessAt() (time.Time, bool) {
|
|||||||
return *syncLog.FinishedAt, true
|
return *syncLog.FinishedAt, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// runSync 同步指定日期集合(全量 366 页或缺失补齐)
|
// runSync 同步指定日期集合(全量 365 天或缺失补齐)
|
||||||
|
// 百度百科按月返回数据,故先归并出涉及的月份,每月一次请求,再按天写入
|
||||||
func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
||||||
syncLog := model.WikiSyncLog{
|
syncLog := model.WikiSyncLog{
|
||||||
Trigger: trigger,
|
Trigger: trigger,
|
||||||
@@ -181,27 +171,37 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
|||||||
}
|
}
|
||||||
s.db.Create(&syncLog)
|
s.db.Create(&syncLog)
|
||||||
|
|
||||||
jobs := make(chan dayKey, len(days))
|
// 归并涉及的月份
|
||||||
|
monthSet := make(map[int]bool)
|
||||||
for _, d := range days {
|
for _, d := range days {
|
||||||
jobs <- d
|
monthSet[d.Month] = true
|
||||||
|
}
|
||||||
|
months := make([]int, 0, len(monthSet))
|
||||||
|
for m := range monthSet {
|
||||||
|
months = append(months, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobs := make(chan int, len(months))
|
||||||
|
for _, m := range months {
|
||||||
|
jobs <- m
|
||||||
}
|
}
|
||||||
close(jobs)
|
close(jobs)
|
||||||
|
|
||||||
var success, failed int32
|
var success, failed int32
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
workerCount := 3
|
workerCount := 2 // 百度百科反爬较敏感,低并发 + 间隔
|
||||||
for i := 0; i < workerCount; i++ {
|
for i := 0; i < workerCount; i++ {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
for d := range jobs {
|
for m := range jobs {
|
||||||
if err := s.syncOneDay(d.Month, d.Day); err != nil {
|
if err := s.syncOneMonth(m); err != nil {
|
||||||
atomic.AddInt32(&failed, 1)
|
atomic.AddInt32(&failed, 1)
|
||||||
log.Printf("wiki sync: %d月%d日 失败: %v", d.Month, d.Day, err)
|
log.Printf("wiki sync: %d月 失败: %v", m, err)
|
||||||
} else {
|
} else {
|
||||||
atomic.AddInt32(&success, 1)
|
atomic.AddInt32(&success, 1)
|
||||||
}
|
}
|
||||||
time.Sleep(800 * time.Millisecond) // 控制请求频率,避免被限流
|
time.Sleep(1200 * time.Millisecond) // 控制请求频率,避免触发安全验证
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
@@ -221,27 +221,27 @@ func (s *WikiSyncService) runSync(trigger string, days []dayKey) {
|
|||||||
"failed": int(failed),
|
"failed": int(failed),
|
||||||
"finished_at": &now,
|
"finished_at": &now,
|
||||||
})
|
})
|
||||||
log.Printf("wiki sync: 同步完成 trigger=%s pages=%d success=%d failed=%d", trigger, len(days), success, failed)
|
log.Printf("wiki sync: 同步完成 trigger=%s days=%d months=%d success=%d failed=%d", trigger, len(days), len(months), success, failed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// syncOneDay 抓取并更新某一天的数据(失败不覆盖旧数据;死锁自动重试)
|
// syncOneMonth 抓取某月数据并按天整体替换(死锁自动重试)
|
||||||
func (s *WikiSyncService) syncOneDay(month, day int) error {
|
func (s *WikiSyncService) syncOneMonth(month int) error {
|
||||||
wikitext, err := s.fetchDayPage(month, day)
|
items, err := s.fetchMonth(month)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
items := parseDayPage(wikitext)
|
|
||||||
if len(items) == 0 {
|
if len(items) == 0 {
|
||||||
return fmt.Errorf("页面解析结果为空")
|
return fmt.Errorf("%d月解析结果为空", month)
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range items {
|
// 按天分组
|
||||||
items[i].Month = month
|
byDay := make(map[int][]model.WikiOnThisDay)
|
||||||
items[i].Day = day
|
for _, it := range items {
|
||||||
items[i].Status = 1
|
byDay[it.Day] = append(byDay[it.Day], it)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 按天整体替换:先删后插;并发写入同表可能死锁,重试 3 次
|
// 逐天事务替换
|
||||||
|
for day, dayItems := range byDay {
|
||||||
var lastErr error
|
var lastErr error
|
||||||
for attempt := 0; attempt < 3; attempt++ {
|
for attempt := 0; attempt < 3; attempt++ {
|
||||||
if attempt > 0 {
|
if attempt > 0 {
|
||||||
@@ -252,235 +252,170 @@ func (s *WikiSyncService) syncOneDay(month, day int) error {
|
|||||||
Delete(&model.WikiOnThisDay{}).Error; err != nil {
|
Delete(&model.WikiOnThisDay{}).Error; err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return tx.Create(&items).Error
|
return tx.Create(&dayItems).Error
|
||||||
})
|
})
|
||||||
if lastErr == nil {
|
if lastErr == nil {
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
if !strings.Contains(lastErr.Error(), "Deadlock") && !strings.Contains(lastErr.Error(), "1213") {
|
||||||
return lastErr
|
return lastErr
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if lastErr != nil {
|
||||||
|
return fmt.Errorf("%d月%d日写入失败: %w", month, day, lastErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// fetchDayPage 抓取「X月X日」页面的 wikitext(失败重试;429 限流时加大退避间隔)
|
// ===== 百度百科数据抓取与解析 =====
|
||||||
func (s *WikiSyncService) fetchDayPage(month, day int) (string, error) {
|
|
||||||
title := fmt.Sprintf("%d月%d日", month, day)
|
// baikeEvent 百度百科「历史上的今天」单条数据
|
||||||
apiURL := "https://zh.wikipedia.org/w/api.php?action=parse&prop=wikitext&format=json&formatversion=2&page=" +
|
type baikeEvent struct {
|
||||||
url.QueryEscape(title)
|
Year string `json:"year"` // 年份,负数表示公元前(如 "-30")
|
||||||
|
Title string `json:"title"` // 标题(含 HTML 链接标签)
|
||||||
|
Festival string `json:"festival"` // 节假日名(当天所有条目共享,可能为空)
|
||||||
|
Link string `json:"link"` // 百科词条链接
|
||||||
|
Type string `json:"type"` // event / birth / death
|
||||||
|
Desc string `json:"desc"` // 描述(含 HTML 标签)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchMonth 抓取某月「历史上的今天」数据并解析为按天的条目(失败重试)
|
||||||
|
func (s *WikiSyncService) fetchMonth(month int) ([]model.WikiOnThisDay, error) {
|
||||||
|
apiURL := fmt.Sprintf("https://baike.baidu.com/cms/home/eventsOnHistory/%02d.json", month)
|
||||||
|
|
||||||
var lastErr error
|
var lastErr error
|
||||||
for attempt := 0; attempt < 4; attempt++ {
|
for attempt := 0; attempt < 4; attempt++ {
|
||||||
if attempt > 0 {
|
if attempt > 0 {
|
||||||
backoff := time.Duration(attempt*3) * time.Second
|
time.Sleep(time.Duration(attempt*3) * time.Second)
|
||||||
if lastErr != nil && strings.Contains(lastErr.Error(), "429") {
|
|
||||||
backoff = time.Duration(attempt*30) * time.Second // 限流时退避 30s/60s/90s
|
|
||||||
}
|
|
||||||
time.Sleep(backoff)
|
|
||||||
}
|
|
||||||
text, err := s.doFetch(apiURL)
|
|
||||||
if err == nil {
|
|
||||||
return text, nil
|
|
||||||
}
|
}
|
||||||
|
body, err := s.doFetch(apiURL)
|
||||||
|
if err != nil {
|
||||||
lastErr = err
|
lastErr = err
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
return "", lastErr
|
items, err := parseBaikeMonth(body, month)
|
||||||
|
if err != nil {
|
||||||
|
lastErr = err
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
return nil, lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WikiSyncService) doFetch(apiURL string) (string, error) {
|
// doFetch 发起 HTTP 请求并返回响应体(带浏览器 UA 与 Referer,规避安全验证)
|
||||||
|
func (s *WikiSyncService) doFetch(apiURL string) ([]byte, error) {
|
||||||
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
|
req, err := http.NewRequest(http.MethodGet, apiURL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return nil, err
|
||||||
}
|
}
|
||||||
req.Header.Set("User-Agent", "LunarServer/1.0 (wiki on-this-day sync; contact: admin@neatcn.com)")
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
||||||
|
req.Header.Set("Referer", "https://baike.baidu.com/calendar")
|
||||||
|
req.Header.Set("Accept", "application/json, text/plain, */*")
|
||||||
|
|
||||||
resp, err := s.httpClient.Do(req)
|
resp, err := s.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) // 上限 4MB
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) // 上限 2MB
|
// parseBaikeMonth 解析百度百科某月 JSON,输出该月全部条目
|
||||||
|
func parseBaikeMonth(body []byte, month int) ([]model.WikiOnThisDay, error) {
|
||||||
|
// 顶层结构:{"08": {"0801": [...], "0802": [...]}}
|
||||||
|
var root map[string]map[string][]baikeEvent
|
||||||
|
if err := json.Unmarshal(body, &root); err != nil {
|
||||||
|
return nil, fmt.Errorf("JSON 解析失败: %w", err)
|
||||||
|
}
|
||||||
|
monthKey := fmt.Sprintf("%02d", month)
|
||||||
|
dayMap, ok := root[monthKey]
|
||||||
|
if !ok || len(dayMap) == 0 {
|
||||||
|
return nil, fmt.Errorf("%s月无数据", monthKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
items := make([]model.WikiOnThisDay, 0, 256)
|
||||||
|
for dayKeyStr, events := range dayMap {
|
||||||
|
day, err := strconv.Atoi(dayKeyStr[2:]) // "0812" -> 12
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
var result struct {
|
|
||||||
Parse struct {
|
|
||||||
Wikitext string `json:"wikitext"`
|
|
||||||
} `json:"parse"`
|
|
||||||
Error struct {
|
|
||||||
Info string `json:"info"`
|
|
||||||
} `json:"error"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &result); err != nil {
|
|
||||||
return "", fmt.Errorf("JSON 解析失败: %w", err)
|
|
||||||
}
|
|
||||||
if result.Error.Info != "" {
|
|
||||||
return "", fmt.Errorf("API 错误: %s", result.Error.Info)
|
|
||||||
}
|
|
||||||
if result.Parse.Wikitext == "" {
|
|
||||||
return "", fmt.Errorf("页面内容为空")
|
|
||||||
}
|
|
||||||
return result.Parse.Wikitext, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ===== wikitext 解析 =====
|
|
||||||
|
|
||||||
// classifySection 根据二级标题识别板块类型
|
|
||||||
func classifySection(title string) string {
|
|
||||||
switch {
|
|
||||||
case strings.Contains(title, "大事") || strings.Contains(title, "事件"):
|
|
||||||
return "event"
|
|
||||||
case strings.Contains(title, "出生"):
|
|
||||||
return "birth"
|
|
||||||
case strings.Contains(title, "逝世") || strings.Contains(title, "去世"):
|
|
||||||
return "death"
|
|
||||||
case strings.Contains(title, "节假日") || strings.Contains(title, "节日") ||
|
|
||||||
strings.Contains(title, "節日") || strings.Contains(title, "假日") ||
|
|
||||||
strings.Contains(title, "习俗") || strings.Contains(title, "風俗") ||
|
|
||||||
strings.Contains(title, "风俗"):
|
|
||||||
return "festival"
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseDayPage 解析页面 wikitext,提取大事记/出生/逝世/节假日条目
|
|
||||||
func parseDayPage(wikitext string) []model.WikiOnThisDay {
|
|
||||||
items := make([]model.WikiOnThisDay, 0, 128)
|
|
||||||
seen := make(map[string]bool) // 同板块去重(维基页面偶有重复条目)
|
|
||||||
|
|
||||||
curKind := ""
|
|
||||||
inTable := false
|
|
||||||
for _, rawLine := range strings.Split(wikitext, "\n") {
|
|
||||||
line := strings.TrimSpace(rawLine)
|
|
||||||
|
|
||||||
// 跳过表格区块
|
|
||||||
if strings.HasPrefix(line, "{|") {
|
|
||||||
inTable = true
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if inTable {
|
seenFestival := make(map[string]bool)
|
||||||
if strings.HasPrefix(line, "|}") {
|
seenContent := make(map[string]bool)
|
||||||
inTable = false
|
|
||||||
}
|
for _, ev := range events {
|
||||||
continue
|
// 节假日:从 festival 字段提取(挂在每条上,去重后单独成 festival 条目)
|
||||||
|
if fest := cleanHTMLText(ev.Festival); fest != "" && !seenFestival[fest] {
|
||||||
|
seenFestival[fest] = true
|
||||||
|
items = append(items, model.WikiOnThisDay{
|
||||||
|
Month: month, Day: day, Kind: "festival", Year: 0, Content: fest, Status: 1,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// 二级/三级标题切换板块(三级标题归属当前二级板块)
|
kind := normalizeKind(ev.Type)
|
||||||
if m := reHeading.FindStringSubmatch(line); m != nil {
|
if kind == "" {
|
||||||
if strings.HasPrefix(line, "=== ") || strings.HasPrefix(line, "===") {
|
|
||||||
continue // 三级标题:大事记内的世纪分组,不改变板块
|
|
||||||
}
|
|
||||||
curKind = classifySection(m[1])
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
text := cleanHTMLText(ev.Title)
|
||||||
if curKind == "" || !strings.HasPrefix(line, "*") || strings.HasPrefix(line, "*>") {
|
if len([]rune(text)) < 4 {
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
content := strings.TrimSpace(strings.TrimLeft(line, "*"))
|
|
||||||
if content == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
year, text := extractYear(cleanWikitext(content))
|
|
||||||
if len([]rune(text)) < 4 { // 过短条目无展示价值
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len([]rune(text)) > 500 {
|
if len([]rune(text)) > 500 {
|
||||||
text = string([]rune(text)[:500])
|
text = string([]rune(text)[:500])
|
||||||
}
|
}
|
||||||
|
if seenContent[kind+"|"+text] {
|
||||||
dedupeKey := curKind + "|" + text
|
|
||||||
if seen[dedupeKey] {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[dedupeKey] = true
|
seenContent[kind+"|"+text] = true
|
||||||
|
|
||||||
items = append(items, model.WikiOnThisDay{
|
items = append(items, model.WikiOnThisDay{
|
||||||
Kind: curKind,
|
Month: month,
|
||||||
Year: year,
|
Day: day,
|
||||||
|
Kind: kind,
|
||||||
|
Year: parseBaikeYear(ev.Year),
|
||||||
Content: text,
|
Content: text,
|
||||||
|
Status: 1,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return items
|
}
|
||||||
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractYear 从条目文本提取年份:前612年→-612,1912年→1912,无年份→0
|
// normalizeKind 映射百度百科 type 到内部 kind
|
||||||
func extractYear(text string) (int, string) {
|
func normalizeKind(t string) string {
|
||||||
text = strings.TrimSpace(text)
|
switch strings.ToLower(strings.TrimSpace(t)) {
|
||||||
if m := reYearPrefix.FindStringSubmatch(text); m != nil {
|
case "event":
|
||||||
year, err := strconv.Atoi(m[2])
|
return "event"
|
||||||
if err == nil {
|
case "birth":
|
||||||
if m[1] == "前" {
|
return "birth"
|
||||||
year = -year
|
case "death":
|
||||||
|
return "death"
|
||||||
}
|
}
|
||||||
return year, strings.TrimSpace(text[len(m[0]):])
|
return ""
|
||||||
}
|
|
||||||
}
|
|
||||||
if m := reNoYear.FindStringSubmatch(text); m != nil {
|
|
||||||
return 0, strings.TrimSpace(text[len(m[0]):])
|
|
||||||
}
|
|
||||||
return 0, text
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// convertConvMarkup 处理语言转换标记 -{zh-cn:A;zh-tw:B}-,优先取简体变体
|
// parseBaikeYear 解析年份字符串:"1912"→1912,"-30"→-30(公元前),非法→0
|
||||||
func convertConvMarkup(s string) string {
|
func parseBaikeYear(s string) int {
|
||||||
return reConv.ReplaceAllStringFunc(s, func(m string) string {
|
s = strings.TrimSpace(s)
|
||||||
inner := m[2 : len(m)-2] // 去掉 -{ 和 }-
|
if s == "" {
|
||||||
parts := strings.Split(inner, ";")
|
return 0
|
||||||
// 无冒号:-{A}- 直接取内容
|
|
||||||
if !strings.Contains(inner, ":") {
|
|
||||||
return strings.TrimSpace(inner)
|
|
||||||
}
|
}
|
||||||
fallback := ""
|
y, err := strconv.Atoi(s)
|
||||||
for _, p := range parts {
|
if err != nil {
|
||||||
p = strings.TrimSpace(p)
|
return 0
|
||||||
if p == "" {
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
kv := strings.SplitN(p, ":", 2)
|
return y
|
||||||
if len(kv) != 2 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := strings.ToLower(strings.TrimSpace(kv[0]))
|
|
||||||
val := strings.TrimSpace(kv[1])
|
|
||||||
if key == "zh-cn" || key == "zh-hans" {
|
|
||||||
return val
|
|
||||||
}
|
|
||||||
fallback = val
|
|
||||||
}
|
|
||||||
return fallback
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// cleanWikitext 去除 wiki 语法,输出纯文本
|
// cleanHTMLText 去除 HTML 标签与多余空白,输出纯文本
|
||||||
func cleanWikitext(s string) string {
|
func cleanHTMLText(s string) string {
|
||||||
s = reComment.ReplaceAllString(s, "")
|
|
||||||
s = reRef.ReplaceAllString(s, "")
|
|
||||||
s = convertConvMarkup(s)
|
|
||||||
// 国旗模板保留国家名
|
|
||||||
s = reFlag.ReplaceAllString(s, "$1")
|
|
||||||
// 模板可能嵌套,循环剥离直至稳定
|
|
||||||
for i := 0; i < 6; i++ {
|
|
||||||
next := reTemplate.ReplaceAllString(s, "")
|
|
||||||
if next == s {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
s = next
|
|
||||||
}
|
|
||||||
s = reLink.ReplaceAllString(s, "$1")
|
|
||||||
s = strings.ReplaceAll(s, "'''", "")
|
|
||||||
s = strings.ReplaceAll(s, "''", "")
|
|
||||||
s = reHTMLTag.ReplaceAllString(s, "")
|
s = reHTMLTag.ReplaceAllString(s, "")
|
||||||
s = strings.ReplaceAll(s, " ", " ")
|
s = strings.ReplaceAll(s, " ", " ")
|
||||||
s = strings.TrimSpace(s)
|
s = reSpace.ReplaceAllString(s, " ")
|
||||||
s = reLeadPunct.ReplaceAllString(s, "") // 模板被剥离后可能残留开头标点
|
|
||||||
return strings.TrimSpace(s)
|
return strings.TrimSpace(s)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,43 +1,31 @@
|
|||||||
package service
|
package service
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
// 取自 zh.wikipedia.org「8月10日」页面的真实片段
|
// 取自百度百科 /cms/home/eventsOnHistory/08.json 的真实片段(8月12日/13日)
|
||||||
const sampleWikitext = `'''8月10日'''是[[阳历]]年的第222天。
|
const sampleBaikeJSON = `{"08":{
|
||||||
|
"0812":[
|
||||||
|
{"year":"-30","title":"埃及艳后<a target=\"_blank\" href=\"https://baike.baidu.com/item/x\">克利奥帕特拉七世</a>逝世","festival":"","link":"https://baike.baidu.com/item/x","type":"death","desc":"克利奥帕特拉七世"},
|
||||||
|
{"year":"1848","title":"英国著名小说家<a target=\"_blank\" href=\"x\">乔治·艾略特</a>出生","festival":"","link":"x","type":"birth","desc":"x"},
|
||||||
|
{"year":"1848","title":"英国著名小说家乔治·艾略特出生","festival":"","link":"x","type":"birth","desc":"x"},
|
||||||
|
{"year":"1905","title":"挪威音乐家<a href=\"x\">某</a>指挥首演","festival":"","link":"x","type":"event","desc":"x"}
|
||||||
|
],
|
||||||
|
"0813":[
|
||||||
|
{"year":"604","title":"隋文帝<a href=\"x\">杨坚</a>逝世","festival":"国际左撇子日","link":"x","type":"death","desc":"x"},
|
||||||
|
{"year":"1926","title":"古巴革命领导人<a href=\"x\">卡斯特罗</a>出生","festival":"国际左撇子日","link":"x","type":"birth","desc":"x"},
|
||||||
|
{"year":"1910","title":"某事件<a href=\"x\">发生</a>","festival":"国际左撇子日","link":"x","type":"event","desc":"x"},
|
||||||
|
{"year":"abc","title":"年份非法的<a href=\"x\">事件</a>条目","festival":"国际左撇子日","link":"x","type":"event","desc":"x"}
|
||||||
|
]
|
||||||
|
}}`
|
||||||
|
|
||||||
== 大事记 ==
|
func TestParseBaikeMonth(t *testing.T) {
|
||||||
=== 19世紀以前 ===
|
items, err := parseBaikeMonth([]byte(sampleBaikeJSON), 8)
|
||||||
* [[前612年]]:[[亚述]]最后一位君主[[辛·沙里施昆]]死于[[尼尼微]],王国灭亡。
|
if err != nil {
|
||||||
* [[955年]]:[[鄂圖一世 (神聖羅馬帝國)|鄂圖一世]]麾下的軍隊在[[第二次萊希菲爾德之戰 (955)|萊希菲爾德之戰]]中取得勝利。<ref>注释</ref>
|
t.Fatalf("parseBaikeMonth 失败: %v", err)
|
||||||
=== 20世紀 ===
|
}
|
||||||
* [[1912年]]:[[威廉·布利斯]]出版了第一本关于[[童子军]]的书。
|
|
||||||
* [[1945年]]:{{le|蒙古}}对日宣战。
|
|
||||||
* [[2002年]]:-{zh-hans:凯尔·斯科特;zh-hk:卡爾·史葛;zh-tw:凱爾·斯科特;}-,英國職業足球運動員。
|
|
||||||
* 年份不详:某件无法确定年份的事情发生了。
|
|
||||||
|
|
||||||
== 出生 ==
|
|
||||||
* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
|
|
||||||
* [[前156年]]:[[汉武帝]]刘彻,[[西汉]]皇帝([[前87年]]逝世)
|
|
||||||
* [[1874年]]:[[赫伯特·胡佛]],美国第31任[[总统]]([[1964年]]逝世)
|
|
||||||
|
|
||||||
== 逝世 ==
|
|
||||||
* [[258年]]:[[羅馬的聖老楞佐]],基督教殉道者
|
|
||||||
|
|
||||||
== 节假日和习俗 ==
|
|
||||||
* {{flag|厄瓜多尔}}:独立日
|
|
||||||
* 世界[[狮子]]日
|
|
||||||
|
|
||||||
==参考资料==
|
|
||||||
{{Commonscat}}
|
|
||||||
`
|
|
||||||
|
|
||||||
func TestParseDayPage(t *testing.T) {
|
|
||||||
items := parseDayPage(sampleWikitext)
|
|
||||||
|
|
||||||
count := func(kind string) int {
|
count := func(kind string) int {
|
||||||
n := 0
|
n := 0
|
||||||
@@ -49,65 +37,78 @@ func TestParseDayPage(t *testing.T) {
|
|||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := count("event"); got != 6 {
|
// 0812: 1 death + 2 birth(重复去重后1) + 1 event = 3;0813: 1 death + 1 birth + 2 event = 4
|
||||||
t.Errorf("event 条目数 = %d, 期望 6", got)
|
if got := count("death"); got != 2 {
|
||||||
|
t.Errorf("death 条目数 = %d, 期望 2", got)
|
||||||
}
|
}
|
||||||
if got := count("birth"); got != 2 { // 重复条目应去重
|
if got := count("birth"); got != 2 { // 0812 重复出生条目应去重
|
||||||
t.Errorf("birth 条目数 = %d, 期望 2", got)
|
t.Errorf("birth 条目数 = %d, 期望 2", got)
|
||||||
}
|
}
|
||||||
if got := count("death"); got != 1 {
|
if got := count("event"); got != 3 {
|
||||||
t.Errorf("death 条目数 = %d, 期望 1", got)
|
t.Errorf("event 条目数 = %d, 期望 3", got)
|
||||||
}
|
}
|
||||||
if got := count("festival"); got != 2 {
|
// 国际左撇子日只在 0813,去重后 1 条 festival
|
||||||
t.Errorf("festival 条目数 = %d, 期望 2", got)
|
if got := count("festival"); got != 1 {
|
||||||
|
t.Errorf("festival 条目数 = %d, 期望 1", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 校验具体条目的年份与文本清洗
|
// 校验具体条目
|
||||||
assertItem := func(kind string, wantYear int, wantContent string) {
|
assertItem := func(kind string, day, wantYear int, wantContent string) {
|
||||||
for _, it := range items {
|
for _, it := range items {
|
||||||
if it.Kind == kind && it.Content == wantContent {
|
if it.Kind == kind && it.Day == day && it.Content == wantContent {
|
||||||
if it.Year != wantYear {
|
if it.Year != wantYear {
|
||||||
t.Errorf("%q 年份 = %d, 期望 %d", wantContent, it.Year, wantYear)
|
t.Errorf("%q 年份 = %d, 期望 %d", wantContent, it.Year, wantYear)
|
||||||
}
|
}
|
||||||
|
if it.Month != 8 {
|
||||||
|
t.Errorf("%q 月份 = %d, 期望 8", wantContent, it.Month)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
t.Errorf("未找到条目: kind=%s content=%q", kind, wantContent)
|
t.Errorf("未找到条目: kind=%s day=%d content=%q", kind, day, wantContent)
|
||||||
}
|
}
|
||||||
assertItem("event", -612, "亚述最后一位君主辛·沙里施昆死于尼尼微,王国灭亡。")
|
assertItem("death", 12, -30, "埃及艳后克利奥帕特拉七世逝世")
|
||||||
assertItem("event", 955, "鄂圖一世麾下的軍隊在萊希菲爾德之戰中取得勝利。")
|
assertItem("birth", 12, 1848, "英国著名小说家乔治·艾略特出生")
|
||||||
assertItem("event", 0, "某件无法确定年份的事情发生了。")
|
assertItem("event", 13, 0, "年份非法的事件条目") // 非法年份应为 0
|
||||||
assertItem("event", 2002, "凯尔·斯科特,英國職業足球運動員。")
|
assertItem("festival", 13, 0, "国际左撇子日")
|
||||||
assertItem("festival", 0, "厄瓜多尔:独立日")
|
|
||||||
assertItem("birth", -156, "汉武帝刘彻,西汉皇帝(前87年逝世)")
|
|
||||||
assertItem("festival", 0, "世界狮子日")
|
|
||||||
|
|
||||||
// 不应包含 wiki 语法残留
|
// 不应残留 HTML 标签
|
||||||
for _, it := range items {
|
for _, it := range items {
|
||||||
if strings.Contains(it.Content, "[[") || strings.Contains(it.Content, "{{") ||
|
if strings.Contains(it.Content, "<a") || strings.Contains(it.Content, "href") ||
|
||||||
strings.Contains(it.Content, "<ref") {
|
strings.Contains(it.Content, "</a>") {
|
||||||
t.Errorf("条目残留 wiki 语法: %q", it.Content)
|
t.Errorf("条目残留 HTML 标签: %q", it.Content)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseRealPage 若本地存在真实抓取的样本文件则验证(可选)
|
func TestParseBaikeYear(t *testing.T) {
|
||||||
func TestParseRealPage(t *testing.T) {
|
cases := map[string]int{
|
||||||
data, err := os.ReadFile("/tmp/wiki_810.json")
|
"1912": 1912,
|
||||||
if err != nil {
|
"-30": -30,
|
||||||
t.Skip("无真实样本文件,跳过")
|
"0": 0,
|
||||||
|
"abc": 0,
|
||||||
|
"": 0,
|
||||||
|
" 77 ": 77,
|
||||||
}
|
}
|
||||||
var d struct {
|
for in, want := range cases {
|
||||||
Parse struct {
|
if got := parseBaikeYear(in); got != want {
|
||||||
Wikitext string `json:"wikitext"`
|
t.Errorf("parseBaikeYear(%q) = %d, 期望 %d", in, got, want)
|
||||||
} `json:"parse"`
|
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(data, &d); err != nil {
|
|
||||||
t.Fatalf("样本解析失败: %v", err)
|
|
||||||
}
|
}
|
||||||
items := parseDayPage(d.Parse.Wikitext)
|
|
||||||
if len(items) < 20 {
|
|
||||||
t.Errorf("真实页面解析条目过少: %d", len(items))
|
|
||||||
}
|
}
|
||||||
t.Logf("真实页面共解析 %d 条", len(items))
|
|
||||||
|
func TestNormalizeKind(t *testing.T) {
|
||||||
|
if normalizeKind("event") != "event" || normalizeKind("birth") != "birth" ||
|
||||||
|
normalizeKind("death") != "death" || normalizeKind("other") != "" ||
|
||||||
|
normalizeKind(" EVENT ") != "event" {
|
||||||
|
t.Error("normalizeKind 映射异常")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanHTMLText(t *testing.T) {
|
||||||
|
in := `埃及艳后<a target="_blank" href="x">克利奥帕特拉七世</a> 逝世 test`
|
||||||
|
want := "埃及艳后克利奥帕特拉七世 逝世 test"
|
||||||
|
if got := cleanHTMLText(in); got != want {
|
||||||
|
t.Errorf("cleanHTMLText = %q, 期望 %q", got, want)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-5
@@ -5,16 +5,23 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>{{ .title }} - 祈福小助手管理后台</title>
|
<title>{{ .title }} - 祈福小助手管理后台</title>
|
||||||
<script src="https://cdn.tailwindcss.com"></script>
|
<script src="https://cdn.tailwindcss.com"></script>
|
||||||
<script src="https://unpkg.com/vue@3/dist/vue.global.js"></script>
|
|
||||||
<script src="https://unpkg.com/@inertiajs/vue3@1.0.0/dist/index.umd.js"></script>
|
|
||||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css">
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css">
|
||||||
|
<!-- Vue + Inertia 通过 import map 从 esm.sh 加载(ESM,自动解析依赖树,国内可访问) -->
|
||||||
|
<script type="importmap">
|
||||||
|
{
|
||||||
|
"imports": {
|
||||||
|
"vue": "https://esm.sh/vue@3.4.21/dist/vue.esm-browser.prod.js",
|
||||||
|
"@inertiajs/vue3": "https://esm.sh/@inertiajs/vue3@1.0.0?bundle&external=vue"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
</head>
|
</head>
|
||||||
<body class="bg-gray-100">
|
<body class="bg-gray-100">
|
||||||
<div id="app" data-page='{"component":"{{ .page }}","props":{},"url":"{{ .url }}","version":""}'></div>
|
<div id="app" data-page='{"component":"{{ .page }}","props":{},"url":"{{ .url }}","version":""}'></div>
|
||||||
|
|
||||||
<script>
|
<script type="module">
|
||||||
const { createApp, h } = Vue;
|
import { createApp, h } from 'vue';
|
||||||
const { createInertiaApp } = Inertia;
|
import { createInertiaApp } from '@inertiajs/vue3';
|
||||||
|
|
||||||
createInertiaApp({
|
createInertiaApp({
|
||||||
resolve: name => {
|
resolve: name => {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ export default {
|
|||||||
<div class="text-center mb-6">
|
<div class="text-center mb-6">
|
||||||
<h1 class="text-2xl font-bold text-red-600">祈福小助手</h1>
|
<h1 class="text-2xl font-bold text-red-600">祈福小助手</h1>
|
||||||
<p class="text-gray-500 text-sm mt-1">管理后台登录</p>
|
<p class="text-gray-500 text-sm mt-1">管理后台登录</p>
|
||||||
|
<p class="text-gray-300 text-xs mt-2">v2026.08.12-login</p>
|
||||||
</div>
|
</div>
|
||||||
<form @submit.prevent="submit">
|
<form @submit.prevent="submit">
|
||||||
<div class="mb-4">
|
<div class="mb-4">
|
||||||
|
|||||||
Reference in New Issue
Block a user