diff --git a/internal/model/device_templates.http.go b/internal/model/device_templates.http.go index 82f0f8197..799c1bbf2 100644 --- a/internal/model/device_templates.http.go +++ b/internal/model/device_templates.http.go @@ -106,8 +106,8 @@ type PublishToMarketReq struct { MarketName string `json:"market_name"` // 市场展示名称 Brand string `json:"brand"` // 品牌 Model string `json:"model"` // 型号 - Category string `json:"category"` // 分类 - Version string `json:"version"` // 版本号 + Category string `json:"category"` // 分类 + Version string `json:"version"` // 版本号 Author string `json:"author"` // 作者 Description string `json:"description"` // 描述 } @@ -128,16 +128,17 @@ type DeviceConfigPayload struct { // PublishTemplateReq 发布模板到市场的业务契约对象（发送给 market-service） type PublishTemplateReq struct { - CoverURL string `json:"cover_url,omitempty"` - Name string `json:"name"` - Brand string `json:"brand"` - Model string `json:"model"` - Category string `json:"category"` - Author string `json:"author"` - Version string `json:"version"` - Description string `json:"description"` + CoverURL string `json:"cover_url,omitempty"` + SourceTemplateID string `json:"source_template_id"` + Name string `json:"name"` + Brand string `json:"brand"` + Model string `json:"model"` + Category string `json:"category"` + Author string `json:"author"` + Version string `json:"version"` + Description string `json:"description"` // DeviceConfig 凭证协议配置 - DeviceConfig *DeviceConfigPayload `json:"device_config"` + DeviceConfig *DeviceConfigPayload `json:"device_config"` // TemplateDefinition 模板定义（面板+物模型） TemplateDefinition map[string]interface{} `json:"template_definition"` PluginDependencies []PluginDependency `json:"plugin_dependencies"` @@ -176,20 +177,20 @@ type MarketTemplateListReq struct { // MarketTemplateFullData 市场模板完整数据（从市场下载的完整定义） type MarketTemplateFullData struct { - Name string `json:"name"` - Brand string `json:"brand"` - ModelNumber string `json:"model_number"` - Category string `json:"category"` - Author string `json:"author"` - VersionID string `json:"version_id"` - Version string `json:"version"` - Description string `json:"description"` - CoverURL string `json:"cover_url"` + Name string `json:"name"` + Brand string `json:"brand"` + ModelNumber string `json:"model_number"` + Category string `json:"category"` + Author string `json:"author"` + VersionID string `json:"version_id"` + Version string `json:"version"` + Description string `json:"description"` + CoverURL string `json:"cover_url"` // 设备配置（凭证协议配置） DeviceConfig *DeviceConfigPayload `json:"device_config"` // 模板定义（物模型 + 面板配置） TemplateDefinition *TemplateDefinitionPayload `json:"template_definition"` - PluginDependencies []PluginDependency `json:"plugin_dependencies"` + PluginDependencies []PluginDependency `json:"plugin_dependencies"` } // TemplateDefinitionPayload 模板定义（面板配置） diff --git a/internal/service/device_template_install.go b/internal/service/device_template_install.go index 3e878da44..ceeba8641 100644 --- a/internal/service/device_template_install.go +++ b/internal/service/device_template_install.go @@ -3,7 +3,7 @@ package service import ( "context" "encoding/json" - "strings" + "os" "time" "project/internal/dal" @@ -30,6 +30,23 @@ func (*DeviceTemplate) InstallFromMarket(req model.InstallFromMarketReq, claims }) } + // Copy the resource-center cover locally before creating database records. + // The local file is removed if any later installation step fails. + localImageURL, localImagePath, err := localizeMarketTemplateImage(context.Background(), client, fullData) + if err != nil { + return nil, errcode.WithData(errcode.CodeSystemError, map[string]interface{}{ + "error": "Failed to localize market template cover: " + err.Error(), + }) + } + installCommitted := false + defer func() { + if !installCommitted && localImagePath != "" { + if removeErr := os.Remove(localImagePath); removeErr != nil && !os.IsNotExist(removeErr) { + logrus.Errorf("Failed to clean up localized market template cover %s: %v", localImagePath, removeErr) + } + } + }() + // 2. Check plugin dependencies (before any DB writes) missingPlugins := checkMissingPlugins(fullData.PluginDependencies) @@ -260,7 +277,7 @@ func (*DeviceTemplate) InstallFromMarket(req model.InstallFromMarketReq, claims DeviceTemplateID: &templateID, // 引用新创建的 DeviceTemplate DeviceType: "1", // 默认直连设备 TenantID: claims.TenantID, - ImageURL: resolveMarketTemplateImageURL(fullData), + ImageURL: localImageURL, CreatedAt: now, UpdatedAt: now, } @@ -313,6 +330,7 @@ func (*DeviceTemplate) InstallFromMarket(req model.InstallFromMarketReq, claims "error": "Failed to commit transaction: " + err.Error(), }) } + installCommitted = true // 4. Report successful local use to the resource center (async, non-blocking). // dcID is stable and acts as the idempotency key for this real local import. @@ -343,24 +361,6 @@ func (*DeviceTemplate) InstallFromMarket(req model.InstallFromMarketReq, claims }, nil } -// resolveMarketTemplateImageURL prefers the resource-center-owned cover and -// only falls back to the publisher's original device-config image for legacy -// download payloads. -func resolveMarketTemplateImageURL(fullData *model.MarketTemplateFullData) *string { - if fullData == nil { - return nil - } - if coverURL := strings.TrimSpace(fullData.CoverURL); coverURL != "" { - return &coverURL - } - if fullData.DeviceConfig != nil { - if imageURL := strings.TrimSpace(fullData.DeviceConfig.ImageURL); imageURL != "" { - return &imageURL - } - } - return nil -} - // checkMissingPlugins checks which plugin dependencies are not installed locally func checkMissingPlugins(deps []model.PluginDependency) []model.PluginDependency { if len(deps) == 0 { diff --git a/internal/service/device_template_install_test.go b/internal/service/device_template_install_test.go index 5b40a30a6..bf56f5270 100644 --- a/internal/service/device_template_install_test.go +++ b/internal/service/device_template_install_test.go @@ -1,61 +1,164 @@ package service import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" + "time" "project/internal/model" ) -func TestResolveMarketTemplateImageURL(t *testing.T) { +var testPNG = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 0x00, 0x00, 0x00, 0x0d, 'I', 'H', 'D', 'R'} + +func TestDownloadMarketTemplateImage(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/market/templates/assets/covers/template/cover.png" { + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write(testPNG) + })) + defer server.Close() + + storageDir := t.TempDir() + client := &MarketClient{baseURL: server.URL, httpClient: &http.Client{Timeout: time.Second}} + localURL, diskPath, err := downloadMarketTemplateImage( + context.Background(), client, + server.URL+"/api/market/templates/assets/covers/template/cover.png", + storageDir, + ) + if err != nil { + t.Fatalf("downloadMarketTemplateImage() error = %v", err) + } + if !strings.HasPrefix(localURL, "./files/deviceConfig/") || !strings.HasSuffix(localURL, ".png") { + t.Fatalf("unexpected local URL %q", localURL) + } + if filepath.Ext(diskPath) != ".png" { + t.Fatalf("unexpected disk path %q", diskPath) + } + data, err := os.ReadFile(diskPath) + if err != nil { + t.Fatalf("read localized image: %v", err) + } + if string(data) != string(testPNG) { + t.Fatalf("localized image content differs") + } + entries, err := os.ReadDir(filepath.Join(storageDir, ".staging")) + if err != nil || len(entries) != 0 { + t.Fatalf("staging directory was not cleaned: entries=%d err=%v", len(entries), err) + } +} + +func TestLocalizeMarketTemplateImageWithoutCover(t *testing.T) { + client := &MarketClient{baseURL: "https://resources.example.com", httpClient: &http.Client{}} tests := []struct { - name string - data *model.MarketTemplateFullData - expected string - wantNil bool + name string + data *model.MarketTemplateFullData }{ + {name: "nil payload", data: nil}, + {name: "empty cover", data: &model.MarketTemplateFullData{}}, { - name: "resource center cover takes precedence", - data: &model.MarketTemplateFullData{ - CoverURL: " https://r.thingspanel.cn/api/market/templates/assets/covers/template/1.0.0.png ", - DeviceConfig: &model.DeviceConfigPayload{ - ImageURL: "https://publisher.example/files/old.png", - }, - }, - expected: "https://r.thingspanel.cn/api/market/templates/assets/covers/template/1.0.0.png", - }, - { - name: "legacy device config image is used as fallback", - data: &model.MarketTemplateFullData{ - DeviceConfig: &model.DeviceConfigPayload{ - ImageURL: " https://publisher.example/files/legacy.png ", - }, - }, - expected: "https://publisher.example/files/legacy.png", - }, - { - name: "missing image remains empty", - data: &model.MarketTemplateFullData{DeviceConfig: &model.DeviceConfigPayload{}}, - wantNil: true, - }, - { - name: "nil payload remains empty", - data: nil, - wantNil: true, + name: "legacy publisher image is not downloaded", + data: &model.MarketTemplateFullData{DeviceConfig: &model.DeviceConfigPayload{ + ImageURL: "https://publisher.example/files/legacy.png", + }}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual := resolveMarketTemplateImageURL(tt.data) - if tt.wantNil { - if actual != nil { - t.Fatalf("expected nil, got %q", *actual) - } - return + localURL, diskPath, err := localizeMarketTemplateImage(context.Background(), client, tt.data) + if err != nil { + t.Fatalf("localizeMarketTemplateImage() error = %v", err) } - if actual == nil || *actual != tt.expected { - t.Fatalf("expected %q, got %v", tt.expected, actual) + if localURL != nil || diskPath != "" { + t.Fatalf("expected no localized image, got url=%v path=%q", localURL, diskPath) } }) } } + +func TestDownloadMarketTemplateImageRejectsUntrustedOrigin(t *testing.T) { + client := &MarketClient{baseURL: "https://resources.example.com", httpClient: &http.Client{}} + _, _, err := downloadMarketTemplateImage( + context.Background(), client, + "https://publisher.example/api/market/templates/assets/covers/cover.png", + t.TempDir(), + ) + if err == nil || !strings.Contains(err.Error(), "not hosted by the configured resource center") { + t.Fatalf("expected untrusted origin error, got %v", err) + } +} + +func TestValidateMarketTemplateAssetURLAllowsEquivalentLoopbackHost(t *testing.T) { + err := validateMarketTemplateAssetURL( + "http://127.0.0.1:18000", + "http://localhost:18000/api/market/templates/assets/covers/cover.png", + ) + if err != nil { + t.Fatalf("expected equivalent loopback hosts to be allowed, got %v", err) + } +} + +func TestDownloadMarketTemplateImageRejectsInvalidContent(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not an image")) + })) + defer server.Close() + + client := &MarketClient{baseURL: server.URL, httpClient: server.Client()} + _, _, err := downloadMarketTemplateImage( + context.Background(), client, + server.URL+"/api/market/templates/assets/covers/cover.png", + t.TempDir(), + ) + if err == nil || !strings.Contains(err.Error(), "unsupported market cover content type") { + t.Fatalf("expected invalid content error, got %v", err) + } +} + +func TestDownloadMarketTemplateImageRejectsOversizeBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "5242881") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := &MarketClient{baseURL: server.URL, httpClient: server.Client()} + _, _, err := downloadMarketTemplateImage( + context.Background(), client, + server.URL+"/api/market/templates/assets/covers/cover.png", + t.TempDir(), + ) + if err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("expected oversize error, got %v", err) + } +} + +func TestDownloadMarketTemplateImageRejectsCrossOriginRedirect(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(testPNG) + })) + defer target.Close() + + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL+"/api/market/templates/assets/covers/cover.png", http.StatusFound) + })) + defer source.Close() + + client := &MarketClient{baseURL: source.URL, httpClient: source.Client()} + _, _, err := downloadMarketTemplateImage( + context.Background(), client, + source.URL+"/api/market/templates/assets/covers/cover.png", + t.TempDir(), + ) + if err == nil || !strings.Contains(err.Error(), "not hosted by the configured resource center") { + t.Fatalf("expected cross-origin redirect error, got %v", err) + } +} diff --git a/internal/service/device_template_publish.go b/internal/service/device_template_publish.go index 4005d3c71..3a4bd96e0 100644 --- a/internal/service/device_template_publish.go +++ b/internal/service/device_template_publish.go @@ -157,15 +157,16 @@ func (*DeviceTemplate) PublishToMarket(req model.PublishToMarketReq, _ *utils.Us // 8. Build publish request to market marketReq := &model.PublishTemplateReq{ - Name: name, - Brand: brand, - Model: devModel, - Category: category, - Author: author, - Version: version, - Description: description, - CoverURL: coverURL, - DeviceConfig: deviceConfig, + SourceTemplateID: tplID, + Name: name, + Brand: brand, + Model: devModel, + Category: category, + Author: author, + Version: version, + Description: description, + CoverURL: coverURL, + DeviceConfig: deviceConfig, TemplateDefinition: map[string]interface{}{ "web_chart_config": tplDef["web_chart_config"], "app_chart_config": tplDef["app_chart_config"], diff --git a/internal/service/market_template_image.go b/internal/service/market_template_image.go new file mode 100644 index 000000000..7d057306c --- /dev/null +++ b/internal/service/market_template_image.go @@ -0,0 +1,183 @@ +package service + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "project/internal/model" + + "github.com/go-basic/uuid" +) + +const ( + marketTemplateImageMaxSize = 5 << 20 + marketTemplateAssetPrefix = "/api/market/templates/assets/" +) + +var marketTemplateImageDir = filepath.Join(".", "files", "deviceConfig") + +// localizeMarketTemplateImage copies the resource-center-owned cover to local +// storage. Legacy device_config.image_url values are intentionally not fetched: +// they may point to an arbitrary publisher-controlled host. +func localizeMarketTemplateImage(ctx context.Context, client *MarketClient, fullData *model.MarketTemplateFullData) (*string, string, error) { + if fullData == nil || strings.TrimSpace(fullData.CoverURL) == "" { + return nil, "", nil + } + + localURL, diskPath, err := downloadMarketTemplateImage(ctx, client, strings.TrimSpace(fullData.CoverURL), marketTemplateImageDir) + if err != nil { + return nil, "", err + } + return &localURL, diskPath, nil +} + +func downloadMarketTemplateImage(ctx context.Context, client *MarketClient, coverURL, storageDir string) (string, string, error) { + if client == nil || client.httpClient == nil { + return "", "", fmt.Errorf("market client is not configured") + } + if err := validateMarketTemplateAssetURL(client.baseURL, coverURL); err != nil { + return "", "", err + } + + httpClient := *client.httpClient + httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return fmt.Errorf("too many market cover redirects") + } + return validateMarketTemplateAssetURL(client.baseURL, req.URL.String()) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil) + if err != nil { + return "", "", fmt.Errorf("create market cover request: %w", err) + } + resp, err := httpClient.Do(req) + if err != nil { + return "", "", fmt.Errorf("download market cover: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("download market cover returned status %d", resp.StatusCode) + } + if resp.ContentLength > marketTemplateImageMaxSize { + return "", "", fmt.Errorf("market cover exceeds %d bytes", marketTemplateImageMaxSize) + } + + data, err := io.ReadAll(io.LimitReader(resp.Body, marketTemplateImageMaxSize+1)) + if err != nil { + return "", "", fmt.Errorf("read market cover: %w", err) + } + if len(data) == 0 { + return "", "", fmt.Errorf("market cover is empty") + } + if len(data) > marketTemplateImageMaxSize { + return "", "", fmt.Errorf("market cover exceeds %d bytes", marketTemplateImageMaxSize) + } + + extension, err := marketTemplateImageExtension(http.DetectContentType(data)) + if err != nil { + return "", "", err + } + + dateDir := time.Now().Format("2006-01-02") + finalDir := filepath.Join(storageDir, dateDir) + stagingDir := filepath.Join(storageDir, ".staging") + if err := os.MkdirAll(finalDir, 0755); err != nil { + return "", "", fmt.Errorf("create market cover directory: %w", err) + } + if err := os.MkdirAll(stagingDir, 0755); err != nil { + return "", "", fmt.Errorf("create market cover staging directory: %w", err) + } + + fileName := uuid.New() + extension + stagingPath := filepath.Join(stagingDir, fileName+".tmp") + finalPath := filepath.Join(finalDir, fileName) + if err := os.WriteFile(stagingPath, data, 0644); err != nil { + return "", "", fmt.Errorf("write market cover staging file: %w", err) + } + defer os.Remove(stagingPath) + + if err := os.Rename(stagingPath, finalPath); err != nil { + return "", "", fmt.Errorf("finalize market cover: %w", err) + } + + localURL := "./" + path.Join("files", "deviceConfig", dateDir, fileName) + return localURL, finalPath, nil +} + +func validateMarketTemplateAssetURL(baseURL, assetURL string) error { + base, err := url.Parse(strings.TrimSpace(baseURL)) + if err != nil || base.Scheme == "" || base.Host == "" { + return fmt.Errorf("invalid market base URL") + } + asset, err := url.Parse(strings.TrimSpace(assetURL)) + if err != nil || asset.Scheme == "" || asset.Host == "" { + return fmt.Errorf("invalid market cover URL") + } + if asset.User != nil || asset.Fragment != "" { + return fmt.Errorf("market cover URL contains unsupported components") + } + if !sameMarketOrigin(base, asset) { + return fmt.Errorf("market cover URL is not hosted by the configured resource center") + } + + cleanPath := path.Clean(asset.Path) + if cleanPath != asset.Path || !strings.HasPrefix(cleanPath, marketTemplateAssetPrefix) { + return fmt.Errorf("market cover URL is outside the template asset path") + } + return nil +} + +func sameMarketOrigin(base, asset *url.URL) bool { + if !strings.EqualFold(base.Scheme, asset.Scheme) || normalizedURLPort(base) != normalizedURLPort(asset) { + return false + } + if strings.EqualFold(base.Hostname(), asset.Hostname()) { + return true + } + return isLoopbackHost(base.Hostname()) && isLoopbackHost(asset.Hostname()) +} + +func normalizedURLPort(value *url.URL) string { + if port := value.Port(); port != "" { + return port + } + if strings.EqualFold(value.Scheme, "https") { + return "443" + } + if strings.EqualFold(value.Scheme, "http") { + return "80" + } + return "" +} + +func isLoopbackHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func marketTemplateImageExtension(contentType string) (string, error) { + switch strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) { + case "image/jpeg": + return ".jpg", nil + case "image/png": + return ".png", nil + case "image/webp": + return ".webp", nil + default: + return "", fmt.Errorf("unsupported market cover content type %q", contentType) + } +} diff --git a/internal/storage/telemetry_writer.go b/internal/storage/telemetry_writer.go index 2aa1295b2..5e1e5dbd7 100644 --- a/internal/storage/telemetry_writer.go +++ b/internal/storage/telemetry_writer.go @@ -309,7 +309,7 @@ func (w *telemetryWriter) batchInsert(historyData []TelemetryData, currentData [ // fallbackInsert 逐条插入兜底（批量失败时使用） func (w *telemetryWriter) fallbackInsert(historyData []TelemetryData, currentData []TelemetryCurrentData) (written, failed int) { for i := range historyData { - // 逐条使用事务插入 + // 逐条使用事务插入历史表 err := w.db.Transaction(func(tx *gorm.DB) error { // 插入历史表 if err := tx.Clauses(clause.OnConflict{ @@ -318,17 +318,6 @@ func (w *telemetryWriter) fallbackInsert(historyData []TelemetryData, currentDat }).Create(&historyData[i]).Error; err != nil { return err } - - // 插入最新值表 - if err := tx.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "device_id"}, {Name: "key"}}, - DoUpdates: clause.AssignmentColumns([]string{ - "ts", "bool_v", "number_v", "string_v", "tenant_id", - }), - }).Create(¤tData[i]).Error; err != nil { - return err - } - return nil }) @@ -354,6 +343,40 @@ func (w *telemetryWriter) fallbackInsert(historyData []TelemetryData, currentDat } } + // 单独循环插入最新值表 + for i := range currentData { + err := w.db.Transaction(func(tx *gorm.DB) error { + // 插入最新值表 + if err := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "device_id"}, {Name: "key"}}, + DoUpdates: clause.AssignmentColumns([]string{ + "ts", "bool_v", "number_v", "string_v", "tenant_id", + }), + }).Create(¤tData[i]).Error; err != nil { + return err + } + return nil + }) + + if err != nil { + sample := map[string]interface{}{ + "device_id": currentData[i].DeviceID, + "key": currentData[i].Key, + "ts": currentData[i].TS, + "tenant_id": currentData[i].TenantID, + } + if j, jerr := json.Marshal(sample); jerr == nil { + w.logger.Errorf("current data single insert failed: sample=%s, err=%v", string(j), err) + } else { + w.logger.Errorf("current data single insert failed: device_id=%s, key=%s, err=%v", currentData[i].DeviceID, currentData[i].Key, err) + } + diagnostics.GetInstance().RecordStorageFailed(currentData[i].DeviceID, fmt.Sprintf("最新值存储失败：%v", err)) + failed++ + } else { + written++ + } + } + return written, failed }