Add per-sensor temperature alert thresholds

internal/alerts/alerts.go

@@ -74,6 +74,7 @@ type SystemAlertData struct {
 	count        uint8
 	min          uint8
 	mapSums      map[string]float32
+	mapCounts    map[string]uint8
 	descriptor   string // override descriptor in notification body (for temp sensor, disk partition, etc)
 }
 

internal/alerts/alerts_api.go

@@ -19,11 +19,12 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
 	userID := e.Auth.Id
 
 	reqData := struct {
-		Min       uint8    `json:"min"`
-		Value     float64  `json:"value"`
-		Name      string   `json:"name"`
-		Systems   []string `json:"systems"`
-		Overwrite bool     `json:"overwrite"`
+		Min        uint8              `json:"min"`
+		Value      float64            `json:"value"`
+		Thresholds map[string]float64 `json:"thresholds"`
+		Name       string             `json:"name"`
+		Systems    []string           `json:"systems"`
+		Overwrite  bool               `json:"overwrite"`
 	}{}
 	err := e.BindBody(&reqData)
 	if err != nil || userID == "" || reqData.Name == "" || len(reqData.Systems) == 0 {
@@ -61,6 +62,9 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
 
 			alertRecord.Set("value", reqData.Value)
 			alertRecord.Set("min", reqData.Min)
+			if reqData.Name == "Temperature" {
+				alertRecord.Set("thresholds", reqData.Thresholds)
+			}
 
 			if err := txApp.SaveNoValidate(alertRecord); err != nil {
 				return err

internal/alerts/alerts_api_test.go

@@ -190,6 +190,31 @@ func TestUserAlertsApi(t *testing.T) {
 				assert.EqualValues(t, 3, user1Alerts, "should have 3 alerts")
 			},
 		},
+		{
+			Name:   "POST temperature alert with per-sensor thresholds",
+			Method: http.MethodPost,
+			URL:    "/api/beszel/user-alerts",
+			Headers: map[string]string{
+				"Authorization": user1Token,
+			},
+			ExpectedStatus:  200,
+			ExpectedContent: []string{"\"success\":true"},
+			TestAppFactory:  testAppFactory,
+			Body: jsonReader(map[string]any{
+				"name":       "Temperature",
+				"systems":    []string{system1.Id},
+				"value":      80,
+				"thresholds": map[string]float64{"CPU Package": 85, "NVMe": 70},
+				"min":        10,
+			}),
+			AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
+				alert, err := app.FindFirstRecordByFilter("alerts", "name = 'Temperature' && user = {:user}", dbx.Params{"user": user1.Id})
+				assert.NoError(t, err)
+				var thresholds map[string]float64
+				assert.NoError(t, alert.UnmarshalJSONField("thresholds", &thresholds))
+				assert.Equal(t, map[string]float64{"CPU Package": 85, "NVMe": 70}, thresholds)
+			},
+		},
 		{
 			Name:   "Overwrite: false, should not overwrite existing alert",
 			Method: http.MethodPost,

internal/alerts/alerts_cache.go

@@ -8,13 +8,14 @@ import (
 
 // CachedAlertData represents the relevant fields of an alert record for status checking and updates.
 type CachedAlertData struct {
-	Id        string
-	SystemID  string
-	UserID    string
-	Name      string
-	Value     float64
-	Triggered bool
-	Min       uint8
+	Id         string
+	SystemID   string
+	UserID     string
+	Name       string
+	Value      float64
+	Thresholds map[string]float64
+	Triggered  bool
+	Min        uint8
 	// Created   types.DateTime
 }
 
@@ -24,6 +25,7 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
 	a.UserID = record.GetString("user")
 	a.Name = record.GetString("name")
 	a.Value = record.GetFloat("value")
+	_ = record.UnmarshalJSONField("thresholds", &a.Thresholds)
 	a.Triggered = record.GetBool("triggered")
 	a.Min = uint8(record.GetInt("min"))
 	// a.Created = record.GetDateTime("created")

internal/alerts/alerts_system.go

@@ -27,6 +27,8 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 		name := alertData.Name
 		var val float64
 		unit := "%"
+		threshold := alertData.Value
+		descriptor := ""
 
 		switch name {
 		case "CPU":
@@ -46,10 +48,16 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 			}
 			val = maxUsedPct
 		case "Temperature":
-			if data.Info.DashboardTemp < 1 {
+			var ok bool
+			val, threshold, descriptor, _, ok = selectTemperatureValue(
+				alertData.Thresholds,
+				data.Stats.Temperatures,
+				data.Info.DashboardTemp,
+				alertData.Value,
+			)
+			if !ok {
 				continue
 			}
-			val = data.Info.DashboardTemp
 			unit = "°C"
 		case "LoadAvg1":
 			val = data.Info.LoadAvg[0]
@@ -70,7 +78,6 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 		}
 
 		triggered := alertData.Triggered
-		threshold := alertData.Value
 
 		// Battery alert has inverted logic: trigger when value is BELOW threshold
 		lowAlert := isLowAlert(name)
@@ -99,6 +106,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 			threshold:    threshold,
 			triggered:    triggered,
 			min:          min,
+			descriptor:   descriptor,
 		}
 
 		// send alert immediately if min is 1 - no need to sum up values.
@@ -210,12 +218,19 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 			case "Temperature":
 				if alert.mapSums == nil {
 					alert.mapSums = make(map[string]float32, len(stats.Temperatures))
+					alert.mapCounts = make(map[string]uint8, len(stats.Temperatures))
 				}
 				for key, temp := range stats.Temperatures {
+					if len(alert.alertData.Thresholds) > 0 {
+						if _, selected := alert.alertData.Thresholds[key]; !selected {
+							continue
+						}
+					}
 					if _, ok := alert.mapSums[key]; !ok {
 						alert.mapSums[key] = float32(0)
 					}
 					alert.mapSums[key] += temp
+					alert.mapCounts[key]++
 				}
 			case "LoadAvg1":
 				alert.val += stats.LoadAvg[0]
@@ -244,6 +259,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 	}
 	// sum up vals for each alert
 	for _, alert := range validAlerts {
+		minCount := float32(alert.min) / 1.2
 		switch alert.name {
 		case "Disk":
 			maxPct := float32(0)
@@ -256,19 +272,30 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 			}
 			alert.val = float64(maxPct / float32(alert.count))
 		case "Temperature":
-			maxTemp := float32(0)
-			for key, value := range alert.mapSums {
-				sumTemp := float32(value) / float32(alert.count)
-				if sumTemp > maxTemp {
-					maxTemp = sumTemp
-					alert.descriptor = fmt.Sprintf("Highest sensor %s", key)
+			averages := make(map[string]float64, len(alert.mapSums))
+			for key, sum := range alert.mapSums {
+				count := alert.mapCounts[key]
+				if count == 0 || float32(count) < minCount {
+					continue
 				}
+				averages[key] = float64(sum) / float64(count)
+			}
+			value, threshold, descriptor, sensor, ok := selectTemperatureValue(
+				alert.alertData.Thresholds,
+				averages,
+				0,
+				alert.alertData.Value,
+			)
+			if !ok {
+				continue
 			}
-			alert.val = float64(maxTemp)
+			alert.val = value
+			alert.threshold = threshold
+			alert.descriptor = descriptor
+			alert.count = alert.mapCounts[sensor]
 		default:
 			alert.val = alert.val / float64(alert.count)
 		}
-		minCount := float32(alert.min) / 1.2
 		// log.Println("alert", alert.name, "val", alert.val, "threshold", alert.threshold, "triggered", alert.triggered)
 		// log.Printf("%s: val %f | count %d | min-count %f | threshold %f\n", alert.name, alert.val, alert.count, minCount, alert.threshold)
 		// pass through alert if count is greater than or equal to minCount
@@ -297,6 +324,51 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 	return nil
 }
 
+// selectTemperatureValue returns the temperature reading that determines the
+// aggregate alert state. With per-sensor thresholds, this is the sensor furthest
+// above its own threshold. Otherwise it is the hottest available sensor.
+func selectTemperatureValue(
+	thresholds map[string]float64,
+	temperatures map[string]float64,
+	fallback float64,
+	defaultThreshold float64,
+) (value float64, threshold float64, descriptor string, sensor string, ok bool) {
+	bestMargin := 0.0
+	for key, temp := range temperatures {
+		sensorThreshold := defaultThreshold
+		if len(thresholds) > 0 {
+			var selected bool
+			sensorThreshold, selected = thresholds[key]
+			if !selected {
+				continue
+			}
+		}
+
+		margin := temp - sensorThreshold
+		if !ok || margin > bestMargin || (margin == bestMargin && key < sensor) {
+			value = temp
+			threshold = sensorThreshold
+			sensor = key
+			bestMargin = margin
+			ok = true
+		}
+	}
+
+	if ok {
+		if len(thresholds) > 0 {
+			descriptor = fmt.Sprintf("Temperature sensor %s", sensor)
+		} else {
+			descriptor = fmt.Sprintf("Highest sensor %s", sensor)
+		}
+		return value, threshold, descriptor, sensor, true
+	}
+
+	if len(thresholds) == 0 && fallback >= 1 {
+		return fallback, defaultThreshold, "", "", true
+	}
+	return 0, 0, "", "", false
+}
+
 func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
 	// log.Printf("Sending alert %s: val %f | count %d | threshold %f\n", alert.name, alert.val, alert.count, alert.threshold)
 	systemName := alert.systemRecord.GetString("name")

internal/alerts/alerts_system_test.go

@@ -28,6 +28,16 @@ func createCombinedData[T any](value T, setValue systemAlertValueSetter[T]) *sys
 }
 
 func newSystemAlertTestFixture(t *testing.T, alertName string, min int, threshold float64) *systemAlertTestFixture {
+	return newSystemAlertTestFixtureWithThresholds(t, alertName, min, threshold, nil)
+}
+
+func newSystemAlertTestFixtureWithThresholds(
+	t *testing.T,
+	alertName string,
+	min int,
+	threshold float64,
+	thresholds map[string]float64,
+) *systemAlertTestFixture {
 	t.Helper()
 
 	hub, user := beszelTests.GetHubWithUser(t)
@@ -46,13 +56,17 @@ func newSystemAlertTestFixture(t *testing.T, alertName string, min int, threshol
 	userSettings.Set("settings", `{"emails":["test@example.com"],"webhooks":[]}`)
 	require.NoError(t, hub.Save(userSettings))
 
-	alertRecord, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
+	alertFields := map[string]any{
 		"name":   alertName,
 		"system": systemRecord.Id,
 		"user":   user.Id,
 		"min":    min,
 		"value":  threshold,
-	})
+	}
+	if thresholds != nil {
+		alertFields["thresholds"] = thresholds
+	}
+	alertRecord, err := beszelTests.CreateRecord(hub, "alerts", alertFields)
 	require.NoError(t, err)
 
 	assert.False(t, alertRecord.GetBool("triggered"), "Alert should not be triggered initially")
@@ -181,6 +195,15 @@ func setTemperatureAlertValue(info *system.Info, stats *system.Stats, value floa
 	}
 }
 
+func setTemperatureAlertValues(info *system.Info, stats *system.Stats, values map[string]float64) {
+	stats.Temperatures = values
+	for _, value := range values {
+		if value > info.DashboardTemp {
+			info.DashboardTemp = value
+		}
+	}
+}
+
 func setLoadAvgAlertValue(info *system.Info, stats *system.Stats, value [3]float64) {
 	info.LoadAvg = value
 	stats.LoadAvg = value
@@ -216,3 +239,60 @@ func TestSystemAlertsTwoMin(t *testing.T) {
 	testMultiMinuteSystemAlert(t, "LoadAvg15", 4, 2, setLoadAvgAlertValue, [3]float64{0, 0, 2}, [3]float64{0, 0, 4.1}, [3]float64{0, 0, 3.5})
 	testMultiMinuteSystemAlert(t, "Battery", 20, 2, setBatteryAlertValue, [2]uint8{21, 0}, [2]uint8{19, 0}, [2]uint8{25, 1})
 }
+
+func TestTemperatureAlertPerSensorThresholdsOneMin(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		fixture := newSystemAlertTestFixtureWithThresholds(t, "Temperature", 1, 70, map[string]float64{
+			"CPU Package": 80,
+			"NVMe":        60,
+		})
+		defer fixture.cleanup()
+
+		// A hot unselected sensor and selected sensors below their own thresholds do not trigger.
+		submitValue(fixture, t, map[string]float64{"CPU Package": 79, "NVMe": 59, "GPU": 99}, setTemperatureAlertValues)
+		waitForSystemAlert(time.Second)
+		fixture.assertTriggered(t, false, "Alert should ignore unselected sensors")
+		assert.Zero(t, fixture.hub.TestMailer.TotalSend())
+
+		// Either selected sensor can trigger using its individual threshold.
+		submitValue(fixture, t, map[string]float64{"CPU Package": 79, "NVMe": 61, "GPU": 99}, setTemperatureAlertValues)
+		waitForSystemAlert(time.Second)
+		fixture.assertTriggered(t, true, "Alert should trigger when a selected sensor exceeds its threshold")
+		assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
+
+		submitValue(fixture, t, map[string]float64{"CPU Package": 79, "NVMe": 59, "GPU": 99}, setTemperatureAlertValues)
+		waitForSystemAlert(time.Second)
+		fixture.assertTriggered(t, false, "Alert should resolve when selected sensors fall below their thresholds")
+		assert.Equal(t, 2, fixture.hub.TestMailer.TotalSend())
+
+		waitForSystemAlert(time.Minute)
+	})
+}
+
+func TestTemperatureAlertPerSensorThresholdsTwoMin(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		fixture := newSystemAlertTestFixtureWithThresholds(t, "Temperature", 2, 70, map[string]float64{
+			"CPU Package": 80,
+			"NVMe":        60,
+		})
+		defer fixture.cleanup()
+
+		submitValue(fixture, t, map[string]float64{"CPU Package": 50, "NVMe": 40, "GPU": 99}, setTemperatureAlertValues)
+		waitForSystemAlert(time.Minute + time.Second)
+		fixture.assertTriggered(t, false, "Alert should not trigger on the baseline reading")
+
+		submitValue(fixture, t, map[string]float64{"CPU Package": 82, "NVMe": 62, "GPU": 99}, setTemperatureAlertValues)
+		waitForSystemAlert(time.Minute)
+		fixture.assertTriggered(t, false, "Alert should wait for a complete history window")
+
+		submitValue(fixture, t, map[string]float64{"CPU Package": 82, "NVMe": 62, "GPU": 99}, setTemperatureAlertValues)
+		waitForSystemAlert(time.Second)
+		fixture.assertTriggered(t, true, "Alert should use per-sensor averages and thresholds")
+		assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
+
+		submitValue(fixture, t, map[string]float64{"CPU Package": 70, "NVMe": 50, "GPU": 99}, setTemperatureAlertValues)
+		waitForSystemAlert(time.Second)
+		fixture.assertTriggered(t, false, "Alert should resolve when per-sensor averages recover")
+		assert.Equal(t, 2, fixture.hub.TestMailer.TotalSend())
+	})
+}

internal/hub/collections_test.go

@@ -50,6 +50,7 @@ func TestCollectionRulesDefault(t *testing.T) {
 	assert.Equal(t, isUserMatchesUser, *alertsCollection.CreateRule)
 	assert.Equal(t, isUserMatchesUser, *alertsCollection.UpdateRule)
 	assert.Equal(t, isUserMatchesUser, *alertsCollection.DeleteRule)
+	assert.IsType(t, &core.JSONField{}, alertsCollection.Fields.GetByName("thresholds"))
 
 	// alerts_history collection
 	alertsHistoryCollection, err := hub.FindCollectionByNameOrId("alerts_history")