chore: sync local updates

This commit is contained in:
2026-05-16 19:03:32 +08:00
parent 360a5f15b9
commit aff88ba97d
47 changed files with 3651 additions and 12175 deletions

View File

@@ -0,0 +1,161 @@
package storage
import (
"encoding/json"
"fmt"
"log"
"time"
"mengyaping-backend/config"
)
const (
kvKeyServer = "cfg_server"
kvKeyMonitor = "cfg_monitor"
kvKeyDataPath = "cfg_data_path"
kvKeyDatabase = "cfg_database"
)
type monitorKVPayload struct {
IntervalMinutes int `json:"interval_minutes"`
TimeoutSeconds int `json:"timeout_seconds"`
RetryCount int `json:"retry_count"`
HistoryDays int `json:"history_days"`
}
func (s *Storage) getKVRaw(key string) string {
var row MonitorKV
s.db.Where("cfg_key = ?", key).Limit(1).Find(&row)
if row.CfgKey == "" {
return ""
}
return row.CfgValue
}
func (s *Storage) setKVRaw(key, val string) error {
var row MonitorKV
s.db.Where("cfg_key = ?", key).Limit(1).Find(&row)
if row.CfgKey == "" {
return s.db.Create(&MonitorKV{CfgKey: key, CfgValue: val}).Error
}
row.CfgValue = val
return s.db.Save(&row).Error
}
// loadAndSyncAppConfig 从 monitor_kv 覆盖内存配置,再写回全量(补全缺省键、统一格式)
func (s *Storage) loadAndSyncAppConfig() {
cfg := config.GetConfig()
if v := s.getKVRaw(kvKeyServer); v != "" {
var sc config.ServerConfig
if err := json.Unmarshal([]byte(v), &sc); err != nil {
log.Printf("解析 %s: %v", kvKeyServer, err)
} else {
if sc.Port != "" {
cfg.Server.Port = sc.Port
}
if sc.Host != "" {
cfg.Server.Host = sc.Host
}
}
}
if v := s.getKVRaw(kvKeyMonitor); v != "" {
var m monitorKVPayload
if err := json.Unmarshal([]byte(v), &m); err != nil {
log.Printf("解析 %s: %v", kvKeyMonitor, err)
} else {
if m.IntervalMinutes > 0 {
snapped := config.SnapMonitorIntervalMinutes(m.IntervalMinutes)
cfg.Monitor.Interval = time.Duration(snapped) * time.Minute
}
if m.TimeoutSeconds > 0 {
cfg.Monitor.Timeout = time.Duration(m.TimeoutSeconds) * time.Second
}
if m.RetryCount > 0 {
cfg.Monitor.RetryCount = m.RetryCount
}
if m.HistoryDays > 0 {
cfg.Monitor.HistoryDays = m.HistoryDays
}
}
}
if v := s.getKVRaw(kvKeyDataPath); v != "" {
cfg.DataPath = v
}
if v := s.getKVRaw(kvKeyDatabase); v != "" {
var dc config.DatabaseConfig
if err := json.Unmarshal([]byte(v), &dc); err != nil {
log.Printf("解析 %s: %v", kvKeyDatabase, err)
} else {
if dc.Host != "" {
cfg.Database.Host = dc.Host
}
if dc.Port != "" {
cfg.Database.Port = dc.Port
}
if dc.User != "" {
cfg.Database.User = dc.User
}
if dc.Password != "" {
cfg.Database.Password = dc.Password
}
if dc.Database != "" {
cfg.Database.Database = dc.Database
}
}
}
if err := s.PersistAppConfig(); err != nil {
log.Printf("同步应用配置到 monitor_kv: %v", err)
}
}
// PersistAppConfig 将当前内存中的 server/monitor/data_path/database 写入 monitor_kv
func (s *Storage) PersistAppConfig() error {
cfg := config.GetConfig()
b, err := json.Marshal(cfg.Server)
if err != nil {
return err
}
if err := s.setKVRaw(kvKeyServer, string(b)); err != nil {
return err
}
m := monitorKVPayload{
IntervalMinutes: config.SnapMonitorIntervalMinutes(int(cfg.Monitor.Interval / time.Minute)),
TimeoutSeconds: int(cfg.Monitor.Timeout.Seconds()),
RetryCount: cfg.Monitor.RetryCount,
HistoryDays: cfg.Monitor.HistoryDays,
}
b, err = json.Marshal(m)
if err != nil {
return err
}
if err := s.setKVRaw(kvKeyMonitor, string(b)); err != nil {
return err
}
if err := s.setKVRaw(kvKeyDataPath, cfg.DataPath); err != nil {
return err
}
b, err = json.Marshal(cfg.Database)
if err != nil {
return err
}
return s.setKVRaw(kvKeyDatabase, string(b))
}
// SetMonitorIntervalMinutes 更新检测周期并写入 monitor_kv
func (s *Storage) SetMonitorIntervalMinutes(minutes int) error {
if !config.IsAllowedMonitorInterval(minutes) {
return fmt.Errorf("interval_minutes 必须是预设值之一: %v", config.AllowedMonitorIntervalMinutes)
}
cfg := config.GetConfig()
cfg.Monitor.Interval = time.Duration(minutes) * time.Minute
return s.PersistAppConfig()
}

View File

@@ -0,0 +1,14 @@
package storage
import "time"
// MonitorKV 键值配置(管理员令牌等),列名避开 MySQL 保留字 key/value
type MonitorKV struct {
CfgKey string `gorm:"primaryKey;size:128;column:cfg_key"`
CfgValue string `gorm:"type:text;column:cfg_value"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
}
func (MonitorKV) TableName() string { return "monitor_kv" }
const kvKeyAdminToken = "admin_token"

View File

@@ -0,0 +1,93 @@
package storage
import (
"time"
"gorm.io/datatypes"
)
// MonitorGroup 网站分类(避免使用 MySQL 保留字 group
type MonitorGroup struct {
ID string `gorm:"primaryKey;size:64"`
Name string `gorm:"size:255;not null"`
SortOrder int `gorm:"default:0;index"`
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
}
func (MonitorGroup) TableName() string { return "monitor_groups" }
// MonitorWebsite 监控站点主表
type MonitorWebsite struct {
ID string `gorm:"primaryKey;size:128"`
Name string `gorm:"size:255;not null;index"`
LegacyGroup string `gorm:"size:64;column:legacy_group"` // 旧版单分组,仅兼容
Favicon string `gorm:"type:text"`
Title string `gorm:"size:512"`
IPAddresses datatypes.JSON `gorm:"type:json"` // []string
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
URLs []MonitorWebsiteURL `gorm:"foreignKey:WebsiteID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
GroupLinks []MonitorWebsiteGroup `gorm:"foreignKey:WebsiteID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
}
func (MonitorWebsite) TableName() string { return "monitor_websites" }
// MonitorWebsiteURL 站点监控 URL一对多
type MonitorWebsiteURL struct {
ID uint64 `gorm:"primaryKey;autoIncrement"`
WebsiteID string `gorm:"size:128;not null;index:idx_url_site,priority:1"`
URLID string `gorm:"size:64;not null;index:idx_url_site,priority:2"`
URL string `gorm:"type:text;not null"`
Remark string `gorm:"size:512"`
SortOrder int `gorm:"default:0"`
}
func (MonitorWebsiteURL) TableName() string { return "monitor_website_urls" }
// MonitorWebsiteGroup 站点与分类多对多
type MonitorWebsiteGroup struct {
WebsiteID string `gorm:"primaryKey;size:128"`
GroupID string `gorm:"primaryKey;size:64"`
}
func (MonitorWebsiteGroup) TableName() string { return "monitor_website_groups" }
// MonitorProbeLatest 每个 URL 最新一次探测结果(不逐条存历史)
type MonitorProbeLatest struct {
WebsiteID string `gorm:"primaryKey;size:128"`
URLID string `gorm:"primaryKey;size:64"`
URL string `gorm:"type:text"`
StatusCode int `gorm:"column:status_code"`
LatencyMs int64 `gorm:"column:latency_ms"`
IsUp bool `gorm:"column:is_up"`
ErrorText string `gorm:"type:text;column:error_text"`
CheckedAt time.Time `gorm:"index"`
}
func (MonitorProbeLatest) TableName() string { return "monitor_probe_latest" }
// MonitorProbeHour 按小时汇总(用于 24h / 7d 统计与图表),行数 ≈ URL 数 × 小时数
type MonitorProbeHour struct {
WebsiteID string `gorm:"primaryKey;size:128"`
URLID string `gorm:"primaryKey;size:64"`
HourAt time.Time `gorm:"primaryKey;type:datetime"` // 整点
ProbeCount int `gorm:"default:0"`
UpCount int `gorm:"default:0"`
LatencySum int64 `gorm:"default:0"` // 总延迟 ms均值为 sum/count
}
func (MonitorProbeHour) TableName() string { return "monitor_probe_hour" }
// MonitorProbeDay 按自然日汇总(用于 90 天柱状图),行数 ≈ URL 数 × 天数
type MonitorProbeDay struct {
WebsiteID string `gorm:"primaryKey;size:128"`
URLID string `gorm:"primaryKey;size:64"`
StatDate time.Time `gorm:"primaryKey;type:date"`
ProbeCount int `gorm:"default:0"`
UpCount int `gorm:"default:0"`
LatencySum int64 `gorm:"default:0"`
}
func (MonitorProbeDay) TableName() string { return "monitor_probe_day" }

View File

@@ -0,0 +1,62 @@
package storage
import (
"errors"
"log"
"os"
"strings"
)
var errEmptyAdminToken = errors.New("admin_token 不能为空")
// EffectiveAdminToken 管理员令牌:环境变量 ADMIN_TOKEN 优先,否则读 MySQL monitor_kv
func (s *Storage) EffectiveAdminToken() string {
if v := strings.TrimSpace(os.Getenv("ADMIN_TOKEN")); v != "" {
return v
}
return strings.TrimSpace(s.adminTokenFromDB())
}
func (s *Storage) adminTokenFromDB() string {
var row MonitorKV
if err := s.db.Where("cfg_key = ?", kvKeyAdminToken).Limit(1).Find(&row).Error; err != nil {
return ""
}
if row.CfgKey == "" {
return ""
}
return row.CfgValue
}
// SetAdminToken 更新数据库中的管理员令牌(需已能通过 AdminAuth
func (s *Storage) SetAdminToken(token string) error {
token = strings.TrimSpace(token)
if token == "" {
return errEmptyAdminToken
}
var row MonitorKV
s.db.Where("cfg_key = ?", kvKeyAdminToken).Limit(1).Find(&row)
if row.CfgKey == "" {
return s.db.Create(&MonitorKV{CfgKey: kvKeyAdminToken, CfgValue: token}).Error
}
row.CfgValue = token
return s.db.Save(&row).Error
}
func (s *Storage) seedAdminTokenKV() {
var n int64
s.db.Model(&MonitorKV{}).Where("cfg_key = ?", kvKeyAdminToken).Count(&n)
if n > 0 {
return
}
val := strings.TrimSpace(os.Getenv("ADMIN_TOKEN"))
if val == "" {
log.Println("提示: 数据库 monitor_kv 中尚无 admin_token。可设置环境变量 ADMIN_TOKEN 后重启以自动写入,或对 monitor_kv 执行 INSERT。")
return
}
if err := s.db.Create(&MonitorKV{CfgKey: kvKeyAdminToken, CfgValue: val}).Error; err != nil {
log.Printf("写入初始 admin_token 失败: %v", err)
return
}
log.Println("已从环境变量 ADMIN_TOKEN 写入数据库 monitor_kv后续可去掉该环境变量改用库中令牌。")
}

View File

@@ -0,0 +1,295 @@
package storage
import (
"log"
"time"
"gorm.io/gorm"
"mengyaping-backend/config"
"mengyaping-backend/models"
)
func latestToRecord(p *MonitorProbeLatest) models.MonitorRecord {
return models.MonitorRecord{
WebsiteID: p.WebsiteID,
URLID: p.URLID,
URL: p.URL,
StatusCode: p.StatusCode,
Latency: p.LatencyMs,
IsUp: p.IsUp,
Error: p.ErrorText,
CheckedAt: p.CheckedAt,
}
}
// AddRecord 写入最新探测 + 按小时/按天汇总(不插逐条历史)
func (s *Storage) AddRecord(record models.MonitorRecord) error {
hourAt := record.CheckedAt.Truncate(time.Hour)
statDate := time.Date(record.CheckedAt.Year(), record.CheckedAt.Month(), record.CheckedAt.Day(),
0, 0, 0, 0, record.CheckedAt.Location())
upDelta := 0
if record.IsUp {
upDelta = 1
}
return s.db.Transaction(func(tx *gorm.DB) error {
lp := MonitorProbeLatest{
WebsiteID: record.WebsiteID,
URLID: record.URLID,
URL: record.URL,
StatusCode: record.StatusCode,
LatencyMs: record.Latency,
IsUp: record.IsUp,
ErrorText: record.Error,
CheckedAt: record.CheckedAt,
}
if err := tx.Save(&lp).Error; err != nil {
return err
}
if err := tx.Exec(`
INSERT INTO monitor_probe_hour (website_id, url_id, hour_at, probe_count, up_count, latency_sum)
VALUES (?, ?, ?, 1, ?, ?)
ON DUPLICATE KEY UPDATE
probe_count = probe_count + 1,
up_count = up_count + VALUES(up_count),
latency_sum = latency_sum + VALUES(latency_sum)
`, record.WebsiteID, record.URLID, hourAt, upDelta, record.Latency).Error; err != nil {
return err
}
return tx.Exec(`
INSERT INTO monitor_probe_day (website_id, url_id, stat_date, probe_count, up_count, latency_sum)
VALUES (?, ?, ?, 1, ?, ?)
ON DUPLICATE KEY UPDATE
probe_count = probe_count + 1,
up_count = up_count + VALUES(up_count),
latency_sum = latency_sum + VALUES(latency_sum)
`, record.WebsiteID, record.URLID, statDate.Format("2006-01-02"), upDelta, record.Latency).Error
})
}
// GetLatestRecord 最新一条(无记录时返回 nil用 Find 而非 First避免 GORM 将「未找到」打成 record not found 日志)
func (s *Storage) GetLatestRecord(websiteID, urlID string) *models.MonitorRecord {
var p MonitorProbeLatest
if err := s.db.Where("website_id = ? AND url_id = ?", websiteID, urlID).Limit(1).Find(&p).Error; err != nil {
return nil
}
if p.WebsiteID == "" {
return nil
}
m := latestToRecord(&p)
return &m
}
// GetRecords 兼容旧接口:按小时聚合展开为「代表点」(每探测窗口 1 条),仅用于仍依赖 slice 的逻辑;新逻辑请用汇总 API
func (s *Storage) GetRecords(websiteID, urlID string, since time.Time) []models.MonitorRecord {
rows := s.getProbeHourRows(websiteID, urlID, since)
var out []models.MonitorRecord
for _, h := range rows {
if h.ProbeCount <= 0 {
continue
}
avgLat := h.LatencySum / int64(h.ProbeCount)
isUp := h.UpCount*2 >= h.ProbeCount
out = append(out, models.MonitorRecord{
WebsiteID: websiteID,
URLID: urlID,
StatusCode: 0,
Latency: avgLat,
IsUp: isUp,
CheckedAt: h.HourAt,
})
}
return out
}
func (s *Storage) getProbeHourRows(websiteID, urlID string, since time.Time) []MonitorProbeHour {
var rows []MonitorProbeHour
s.db.Where("website_id = ? AND url_id = ? AND hour_at >= ?", websiteID, urlID, since).
Order("hour_at ASC").Find(&rows)
return rows
}
// GetProbeHourRowsForURL 某 URL 自 since 起的小时汇总行(升序)
func (s *Storage) GetProbeHourRowsForURL(websiteID, urlID string, since time.Time) []MonitorProbeHour {
return s.getProbeHourRows(websiteID, urlID, since)
}
// GetProbeLatestKeyMap 全表 latest每 URL 一行key = website_id + "\x00" + url_id
func (s *Storage) GetProbeLatestKeyMap() map[string]models.MonitorRecord {
var rows []MonitorProbeLatest
if err := s.db.Find(&rows).Error; err != nil {
log.Printf("GetProbeLatestKeyMap: %v", err)
return map[string]models.MonitorRecord{}
}
m := make(map[string]models.MonitorRecord, len(rows))
for i := range rows {
k := rows[i].WebsiteID + "\x00" + rows[i].URLID
m[k] = latestToRecord(&rows[i])
}
return m
}
// GroupProbeHoursSince 近窗口内全部小时汇总按站点、URL 分组(列表接口一次查出)
func (s *Storage) GroupProbeHoursSince(since time.Time) map[string]map[string][]MonitorProbeHour {
var rows []MonitorProbeHour
if err := s.db.Where("hour_at >= ?", since).Order("website_id, url_id, hour_at").Find(&rows).Error; err != nil {
log.Printf("GroupProbeHoursSince: %v", err)
return map[string]map[string][]MonitorProbeHour{}
}
out := make(map[string]map[string][]MonitorProbeHour)
for i := range rows {
h := rows[i]
if out[h.WebsiteID] == nil {
out[h.WebsiteID] = make(map[string][]MonitorProbeHour)
}
out[h.WebsiteID][h.URLID] = append(out[h.WebsiteID][h.URLID], h)
}
return out
}
// LoadAllWebsiteDayRollups 自 since 日起各站每日汇总 + 90d 整体可用率(单条 SQL 聚合)
func (s *Storage) LoadAllWebsiteDayRollups(since time.Time) (map[string][]models.DailyStats, map[string]float64) {
sinceDay := time.Date(since.Year(), since.Month(), since.Day(), 0, 0, 0, 0, since.Location())
sinceStr := sinceDay.Format("2006-01-02")
type aggRow struct {
Wid string `gorm:"column:wid"`
D time.Time `gorm:"column:d"`
Pc int64 `gorm:"column:pc"`
Uc int64 `gorm:"column:uc"`
Ls int64 `gorm:"column:ls"`
}
var agg []aggRow
if err := s.db.Raw(`
SELECT website_id AS wid, stat_date AS d, SUM(probe_count) AS pc, SUM(up_count) AS uc, SUM(latency_sum) AS ls
FROM monitor_probe_day
WHERE stat_date >= ?
GROUP BY website_id, stat_date
ORDER BY website_id, stat_date
`, sinceStr).Scan(&agg).Error; err != nil {
log.Printf("LoadAllWebsiteDayRollups: %v", err)
return map[string][]models.DailyStats{}, map[string]float64{}
}
daily := make(map[string][]models.DailyStats)
totP := make(map[string]int64)
totU := make(map[string]int64)
for _, r := range agg {
st := models.DailyStats{
Date: r.D,
TotalCount: int(r.Pc),
UpCount: int(r.Uc),
}
if r.Pc > 0 {
st.AvgLatency = r.Ls / r.Pc
st.Uptime = float64(r.Uc) / float64(r.Pc) * 100
}
daily[r.Wid] = append(daily[r.Wid], st)
totP[r.Wid] += r.Pc
totU[r.Wid] += r.Uc
}
uptime := make(map[string]float64)
for wid, p := range totP {
if p > 0 {
uptime[wid] = float64(totU[wid]) / float64(p) * 100
}
}
return daily, uptime
}
// GetProbeHourTotals 某 URL 在时间窗口内汇总(来自小时表)
func (s *Storage) GetProbeHourTotals(websiteID, urlID string, since time.Time) (probeTotal int64, upTotal int64, latencySum int64) {
type row struct {
P int64
U int64
L int64
}
var r row
s.db.Model(&MonitorProbeHour{}).
Select("COALESCE(SUM(probe_count),0), COALESCE(SUM(up_count),0), COALESCE(SUM(latency_sum),0)").
Where("website_id = ? AND url_id = ? AND hour_at >= ?", websiteID, urlID, since).
Scan(&r)
return r.P, r.U, r.L
}
// GetHourlyStatsForURL 7 天折线用(已聚合)
func (s *Storage) GetHourlyStatsForURL(websiteID, urlID string, since time.Time) []models.HourlyStats {
rows := s.getProbeHourRows(websiteID, urlID, since)
out := make([]models.HourlyStats, 0, len(rows))
for _, h := range rows {
st := models.HourlyStats{
Hour: h.HourAt,
TotalCount: h.ProbeCount,
UpCount: h.UpCount,
}
if h.ProbeCount > 0 {
st.AvgLatency = h.LatencySum / int64(h.ProbeCount)
st.Uptime = float64(h.UpCount) / float64(h.ProbeCount) * 100
}
out = append(out, st)
}
return out
}
// GetWebsiteDailyAggregates 站点维度按日合并(多 URL 汇总到同一天)
func (s *Storage) GetWebsiteDailyAggregates(websiteID string, since time.Time) []models.DailyStats {
type aggRow struct {
D time.Time `gorm:"column:d"`
Pc int64 `gorm:"column:pc"`
Uc int64 `gorm:"column:uc"`
Ls int64 `gorm:"column:ls"`
}
var rows []aggRow
sinceDay := time.Date(since.Year(), since.Month(), since.Day(), 0, 0, 0, 0, since.Location())
s.db.Raw(`
SELECT stat_date AS d, SUM(probe_count) AS pc, SUM(up_count) AS uc, SUM(latency_sum) AS ls
FROM monitor_probe_day
WHERE website_id = ? AND stat_date >= ?
GROUP BY stat_date
ORDER BY stat_date ASC
`, websiteID, sinceDay.Format("2006-01-02")).Scan(&rows)
out := make([]models.DailyStats, 0, len(rows))
for _, r := range rows {
st := models.DailyStats{
Date: r.D,
TotalCount: int(r.Pc),
UpCount: int(r.Uc),
}
if r.Pc > 0 {
st.AvgLatency = r.Ls / r.Pc
st.Uptime = float64(r.Uc) / float64(r.Pc) * 100
}
out = append(out, st)
}
return out
}
// GetWebsiteDayProbeTotals 整站在日期范围内探测次数汇总(用于 90 天可用率)
func (s *Storage) GetWebsiteDayProbeTotals(websiteID string, since time.Time) (probeTotal int64, upTotal int64) {
sinceDay := time.Date(since.Year(), since.Month(), since.Day(), 0, 0, 0, 0, since.Location())
type row struct {
P int64
U int64
}
var r row
s.db.Model(&MonitorProbeDay{}).
Select("COALESCE(SUM(probe_count),0), COALESCE(SUM(up_count),0)").
Where("website_id = ? AND stat_date >= ?", websiteID, sinceDay.Format("2006-01-02")).
Scan(&r)
return r.P, r.U
}
func (s *Storage) purgeProbeRollups() {
cfg := config.GetConfig()
now := time.Now()
dayCutoff := now.AddDate(0, 0, -cfg.Monitor.HistoryDays).Truncate(24 * time.Hour)
if err := s.db.Where("stat_date < ?", dayCutoff.Format("2006-01-02")).Delete(&MonitorProbeDay{}).Error; err != nil {
log.Printf("清理 monitor_probe_day: %v", err)
}
// 小时图保留约 10 天(覆盖 7 天曲线 + 余量)
hourCutoff := now.Add(-10 * 24 * time.Hour).Truncate(time.Hour)
if err := s.db.Where("hour_at < ?", hourCutoff).Delete(&MonitorProbeHour{}).Error; err != nil {
log.Printf("清理 monitor_probe_hour: %v", err)
}
}

View File

@@ -1,302 +1,365 @@
package storage
import (
"encoding/json"
"os"
"path/filepath"
"sync"
"time"
"mengyaping-backend/config"
"mengyaping-backend/models"
)
// Storage 数据存储
type Storage struct {
dataPath string
mu sync.RWMutex
websites []models.Website
records map[string][]models.MonitorRecord // key: websiteID_urlID
groups []models.Group
}
var (
store *Storage
once sync.Once
)
// GetStorage 获取存储单例
func GetStorage() *Storage {
once.Do(func() {
cfg := config.GetConfig()
store = &Storage{
dataPath: cfg.DataPath,
websites: []models.Website{},
records: make(map[string][]models.MonitorRecord),
groups: models.DefaultGroups,
}
store.ensureDataDir()
store.load()
})
return store
}
// ensureDataDir 确保数据目录存在
func (s *Storage) ensureDataDir() {
os.MkdirAll(s.dataPath, 0755)
}
// load 加载数据
func (s *Storage) load() {
s.loadWebsites()
s.loadRecords()
s.loadGroups()
}
// loadWebsites 加载网站数据
func (s *Storage) loadWebsites() {
filePath := filepath.Join(s.dataPath, "websites.json")
data, err := os.ReadFile(filePath)
if err != nil {
return
}
json.Unmarshal(data, &s.websites)
s.migrateWebsiteGroups()
}
// migrateWebsiteGroups 将旧的单分组字段迁移到多分组数组
func (s *Storage) migrateWebsiteGroups() {
migrated := false
for i := range s.websites {
w := &s.websites[i]
if len(w.Groups) == 0 && w.Group != "" {
w.Groups = []string{w.Group}
w.Group = ""
migrated = true
}
}
if migrated {
s.saveWebsites()
}
}
// loadRecords 加载监控记录
func (s *Storage) loadRecords() {
filePath := filepath.Join(s.dataPath, "records.json")
data, err := os.ReadFile(filePath)
if err != nil {
return
}
json.Unmarshal(data, &s.records)
// 清理过期记录
s.cleanOldRecords()
}
// loadGroups 加载分组
func (s *Storage) loadGroups() {
filePath := filepath.Join(s.dataPath, "groups.json")
data, err := os.ReadFile(filePath)
if err != nil {
// 使用默认分组
s.groups = models.DefaultGroups
s.saveGroups()
return
}
json.Unmarshal(data, &s.groups)
}
// saveWebsites 保存网站数据
func (s *Storage) saveWebsites() error {
filePath := filepath.Join(s.dataPath, "websites.json")
data, err := json.MarshalIndent(s.websites, "", " ")
if err != nil {
return err
}
return os.WriteFile(filePath, data, 0644)
}
// saveRecords 保存监控记录
func (s *Storage) saveRecords() error {
filePath := filepath.Join(s.dataPath, "records.json")
data, err := json.MarshalIndent(s.records, "", " ")
if err != nil {
return err
}
return os.WriteFile(filePath, data, 0644)
}
// saveGroups 保存分组
func (s *Storage) saveGroups() error {
filePath := filepath.Join(s.dataPath, "groups.json")
data, err := json.MarshalIndent(s.groups, "", " ")
if err != nil {
return err
}
return os.WriteFile(filePath, data, 0644)
}
// cleanOldRecords 清理过期记录
func (s *Storage) cleanOldRecords() {
cfg := config.GetConfig()
cutoff := time.Now().AddDate(0, 0, -cfg.Monitor.HistoryDays)
for key, records := range s.records {
var newRecords []models.MonitorRecord
for _, r := range records {
if r.CheckedAt.After(cutoff) {
newRecords = append(newRecords, r)
}
}
s.records[key] = newRecords
}
}
// GetWebsites 获取所有网站
func (s *Storage) GetWebsites() []models.Website {
s.mu.RLock()
defer s.mu.RUnlock()
result := make([]models.Website, len(s.websites))
copy(result, s.websites)
return result
}
// GetWebsite 获取单个网站
func (s *Storage) GetWebsite(id string) *models.Website {
s.mu.RLock()
defer s.mu.RUnlock()
for _, w := range s.websites {
if w.ID == id {
website := w
return &website
}
}
return nil
}
// AddWebsite 添加网站
func (s *Storage) AddWebsite(website models.Website) error {
s.mu.Lock()
defer s.mu.Unlock()
s.websites = append(s.websites, website)
return s.saveWebsites()
}
// UpdateWebsite 更新网站
func (s *Storage) UpdateWebsite(website models.Website) error {
s.mu.Lock()
defer s.mu.Unlock()
for i, w := range s.websites {
if w.ID == website.ID {
s.websites[i] = website
return s.saveWebsites()
}
}
return nil
}
// DeleteWebsite 删除网站
func (s *Storage) DeleteWebsite(id string) error {
s.mu.Lock()
defer s.mu.Unlock()
for i, w := range s.websites {
if w.ID == id {
s.websites = append(s.websites[:i], s.websites[i+1:]...)
// 删除相关记录
for key := range s.records {
if len(key) > len(id) && key[:len(id)] == id {
delete(s.records, key)
}
}
s.saveRecords()
return s.saveWebsites()
}
}
return nil
}
// AddRecord 添加监控记录
func (s *Storage) AddRecord(record models.MonitorRecord) error {
s.mu.Lock()
defer s.mu.Unlock()
key := record.WebsiteID + "_" + record.URLID
s.records[key] = append(s.records[key], record)
// 每100条记录保存一次
if len(s.records[key])%100 == 0 {
return s.saveRecords()
}
return nil
}
// GetRecords 获取监控记录
func (s *Storage) GetRecords(websiteID, urlID string, since time.Time) []models.MonitorRecord {
s.mu.RLock()
defer s.mu.RUnlock()
key := websiteID + "_" + urlID
records := s.records[key]
var result []models.MonitorRecord
for _, r := range records {
if r.CheckedAt.After(since) {
result = append(result, r)
}
}
return result
}
// GetLatestRecord 获取最新记录
func (s *Storage) GetLatestRecord(websiteID, urlID string) *models.MonitorRecord {
s.mu.RLock()
defer s.mu.RUnlock()
key := websiteID + "_" + urlID
records := s.records[key]
if len(records) == 0 {
return nil
}
latest := records[len(records)-1]
return &latest
}
// GetGroups 获取所有分组
func (s *Storage) GetGroups() []models.Group {
s.mu.RLock()
defer s.mu.RUnlock()
result := make([]models.Group, len(s.groups))
copy(result, s.groups)
return result
}
// AddGroup 添加分组
func (s *Storage) AddGroup(group models.Group) error {
s.mu.Lock()
defer s.mu.Unlock()
s.groups = append(s.groups, group)
return s.saveGroups()
}
// SaveAll 保存所有数据
func (s *Storage) SaveAll() error {
s.mu.Lock()
defer s.mu.Unlock()
if err := s.saveWebsites(); err != nil {
return err
}
if err := s.saveRecords(); err != nil {
return err
}
return s.saveGroups()
}
package storage
import (
"encoding/json"
"fmt"
"log"
"os"
"strings"
"sync"
"time"
"gorm.io/datatypes"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"mengyaping-backend/config"
"mengyaping-backend/models"
)
// Storage 基于 MySQL 的持久化(站点 / 分类 / 检测记录)
type Storage struct {
db *gorm.DB
}
var (
store *Storage
once sync.Once
)
// GetStorage 初始化数据库连接并迁移表结构
func GetStorage() *Storage {
once.Do(func() {
cfg := config.GetConfig()
dsn := cfg.DatabaseDSN()
gormLog := logger.Default.LogMode(logger.Warn)
if os.Getenv("DB_DEBUG") == "1" {
gormLog = logger.Default.LogMode(logger.Info)
}
db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{
Logger: gormLog,
})
if err != nil {
log.Fatalf("MySQL 连接失败: %v请检查 DB_* / DB_DSN 环境变量)", err)
}
sqlDB, err := db.DB()
if err != nil {
log.Fatalf("MySQL sqlDB: %v", err)
}
sqlDB.SetMaxIdleConns(8)
sqlDB.SetMaxOpenConns(40)
sqlDB.SetConnMaxLifetime(time.Hour)
store = &Storage{db: db}
if err := store.migrate(); err != nil {
log.Fatalf("数据库迁移失败: %v", err)
}
store.seedAdminTokenKV()
store.loadAndSyncAppConfig()
store.seedDefaultGroups()
log.Println("MySQL 存储已就绪: " + cfg.Database.Database + "@" + cfg.Database.Host)
})
return store
}
func (s *Storage) migrate() error {
err := s.db.AutoMigrate(
&MonitorGroup{},
&MonitorWebsite{},
&MonitorWebsiteURL{},
&MonitorWebsiteGroup{},
&MonitorProbeLatest{},
&MonitorProbeHour{},
&MonitorProbeDay{},
&MonitorKV{},
)
if err != nil {
return err
}
// 旧版逐条 monitor_checks 表体积极大,已弃用
if s.db.Migrator().HasTable("monitor_checks") {
if err := s.db.Migrator().DropTable("monitor_checks"); err != nil {
log.Printf("删除旧表 monitor_checks: %v可手工 DROP", err)
} else {
log.Println("已删除旧表 monitor_checks")
}
}
return nil
}
func (s *Storage) seedDefaultGroups() {
var n int64
s.db.Model(&MonitorGroup{}).Count(&n)
if n > 0 {
return
}
order := 0
for _, g := range models.DefaultGroups {
order++
s.db.Create(&MonitorGroup{
ID: g.ID,
Name: g.Name,
SortOrder: order,
})
}
log.Println("已写入默认分类 monitor_groups")
}
func ipsToJSON(ips []string) []byte {
if ips == nil {
ips = []string{}
}
b, err := json.Marshal(ips)
if err != nil {
return []byte("[]")
}
return b
}
func jsonToIPs(raw []byte) []string {
if len(raw) == 0 {
return nil
}
var ips []string
if err := json.Unmarshal(raw, &ips); err != nil {
return nil
}
return ips
}
func rowToWebsite(w *MonitorWebsite) models.Website {
groups := make([]string, 0, len(w.GroupLinks))
for _, l := range w.GroupLinks {
groups = append(groups, l.GroupID)
}
if len(groups) == 0 && w.LegacyGroup != "" {
groups = []string{w.LegacyGroup}
}
urls := make([]models.URLInfo, 0, len(w.URLs))
for _, u := range w.URLs {
urls = append(urls, models.URLInfo{
ID: u.URLID,
URL: u.URL,
Remark: u.Remark,
})
}
return models.Website{
ID: w.ID,
Name: w.Name,
Groups: groups,
Group: "",
URLs: urls,
IPAddresses: jsonToIPs(w.IPAddresses),
Favicon: w.Favicon,
Title: w.Title,
CreatedAt: w.CreatedAt,
UpdatedAt: w.UpdatedAt,
}
}
func websiteToRow(w *models.Website) *MonitorWebsite {
return &MonitorWebsite{
ID: w.ID,
Name: w.Name,
LegacyGroup: "",
Favicon: w.Favicon,
Title: w.Title,
IPAddresses: datatypes.JSON(ipsToJSON(w.IPAddresses)),
CreatedAt: w.CreatedAt,
UpdatedAt: w.UpdatedAt,
}
}
func (s *Storage) listWebsiteModels() ([]models.Website, error) {
var rows []MonitorWebsite
if err := s.db.Preload("URLs", func(db *gorm.DB) *gorm.DB {
return db.Order("sort_order ASC, id ASC")
}).Preload("GroupLinks").Order("created_at ASC").Find(&rows).Error; err != nil {
return nil, err
}
out := make([]models.Website, len(rows))
for i := range rows {
out[i] = rowToWebsite(&rows[i])
}
return out, nil
}
// GetWebsites 获取所有网站
func (s *Storage) GetWebsites() []models.Website {
list, err := s.listWebsiteModels()
if err != nil {
log.Printf("GetWebsites: %v", err)
return nil
}
return list
}
// GetWebsite 获取单个网站
func (s *Storage) GetWebsite(id string) *models.Website {
var w MonitorWebsite
if err := s.db.Preload("URLs", func(db *gorm.DB) *gorm.DB {
return db.Order("sort_order ASC, id ASC")
}).Preload("GroupLinks").First(&w, "id = ?", id).Error; err != nil {
return nil
}
m := rowToWebsite(&w)
return &m
}
func (s *Storage) persistWebsiteFull(w models.Website) error {
return s.db.Transaction(func(tx *gorm.DB) error {
row := websiteToRow(&w)
if err := tx.Save(row).Error; err != nil {
return err
}
if err := tx.Where("website_id = ?", w.ID).Delete(&MonitorWebsiteURL{}).Error; err != nil {
return err
}
if err := tx.Where("website_id = ?", w.ID).Delete(&MonitorWebsiteGroup{}).Error; err != nil {
return err
}
for i, u := range w.URLs {
if err := tx.Create(&MonitorWebsiteURL{
WebsiteID: w.ID,
URLID: u.ID,
URL: u.URL,
Remark: u.Remark,
SortOrder: i,
}).Error; err != nil {
return err
}
}
for _, gid := range w.Groups {
if err := tx.Create(&MonitorWebsiteGroup{WebsiteID: w.ID, GroupID: gid}).Error; err != nil {
return err
}
}
return nil
})
}
// AddWebsite 添加网站
func (s *Storage) AddWebsite(website models.Website) error {
return s.persistWebsiteFull(website)
}
// UpdateWebsite 更新网站
func (s *Storage) UpdateWebsite(website models.Website) error {
return s.persistWebsiteFull(website)
}
// DeleteWebsite 删除网站
func (s *Storage) DeleteWebsite(id string) error {
return s.db.Transaction(func(tx *gorm.DB) error {
_ = tx.Where("website_id = ?", id).Delete(&MonitorProbeLatest{})
_ = tx.Where("website_id = ?", id).Delete(&MonitorProbeHour{})
_ = tx.Where("website_id = ?", id).Delete(&MonitorProbeDay{})
_ = tx.Where("website_id = ?", id).Delete(&MonitorWebsiteURL{})
_ = tx.Where("website_id = ?", id).Delete(&MonitorWebsiteGroup{})
return tx.Where("id = ?", id).Delete(&MonitorWebsite{}).Error
})
}
// GetGroups 获取所有分组
func (s *Storage) GetGroups() []models.Group {
var rows []MonitorGroup
s.db.Order("sort_order ASC, id ASC").Find(&rows)
out := make([]models.Group, len(rows))
for i, r := range rows {
out[i] = models.Group{ID: r.ID, Name: r.Name}
}
return out
}
// AddGroup 添加分组
func (s *Storage) AddGroup(group models.Group) error {
var n int64
s.db.Model(&MonitorGroup{}).Count(&n)
return s.db.Create(&MonitorGroup{
ID: group.ID,
Name: group.Name,
SortOrder: int(n) + 1,
}).Error
}
// GetGroupByID 按 ID 获取分组(无则 nil
func (s *Storage) GetGroupByID(id string) *models.Group {
var r MonitorGroup
if err := s.db.First(&r, "id = ?", id).Error; err != nil {
return nil
}
g := models.Group{ID: r.ID, Name: r.Name}
return &g
}
// UpdateGroup 更新分组显示名称ID 不变)
func (s *Storage) UpdateGroup(id, name string) error {
name = strings.TrimSpace(name)
if name == "" {
return fmt.Errorf("分类名称不能为空")
}
res := s.db.Model(&MonitorGroup{}).Where("id = ?", id).Update("name", name)
if res.Error != nil {
return res.Error
}
if res.RowsAffected == 0 {
return fmt.Errorf("分类不存在")
}
return nil
}
// DeleteGroup 删除分组;从所有网站中移除该分类;若网站无剩余分类则归入删除后列表中的第一个分类。至少保留一个分类。
func (s *Storage) DeleteGroup(id string) error {
var total int64
s.db.Model(&MonitorGroup{}).Count(&total)
if total <= 1 {
return fmt.Errorf("至少保留一个分类")
}
var remain []MonitorGroup
s.db.Where("id != ?", id).Order("sort_order ASC, id ASC").Find(&remain)
if len(remain) == 0 {
return fmt.Errorf("至少保留一个分类")
}
fallbackID := remain[0].ID
var sites []MonitorWebsite
if err := s.db.Preload("GroupLinks").Find(&sites).Error; err != nil {
return err
}
return s.db.Transaction(func(tx *gorm.DB) error {
for i := range sites {
next := make([]string, 0)
for _, l := range sites[i].GroupLinks {
if l.GroupID != id {
next = append(next, l.GroupID)
}
}
if len(next) == 0 {
next = []string{fallbackID}
}
if err := tx.Where("website_id = ?", sites[i].ID).Delete(&MonitorWebsiteGroup{}).Error; err != nil {
return err
}
for _, gid := range next {
if err := tx.Create(&MonitorWebsiteGroup{WebsiteID: sites[i].ID, GroupID: gid}).Error; err != nil {
return err
}
}
}
if err := tx.Delete(&MonitorGroup{}, "id = ?", id).Error; err != nil {
return err
}
return nil
})
}
// SaveAll 周期结束时清理过期汇总分区(日表按 history_days小时表保留约 10 天)
func (s *Storage) SaveAll() error {
s.purgeProbeRollups()
return nil
}