diff --git a/data/templates/report/report.css b/data/templates/report/report.css index 52a4b2f8..f29773f7 100644 --- a/data/templates/report/report.css +++ b/data/templates/report/report.css @@ -44,4 +44,84 @@ table { font-size: 8pt; } .closebtn:hover { color: black; +} + +/* Split-pane layout for etcd page */ +.etcd-panel-left { + width: 70%; + min-width: 70%; + padding-right: 12px; +} +.etcd-panel-divider { + width: 6px; + min-width: 6px; + cursor: col-resize; + background: #dee2e6; + transition: background 0.15s; +} +.etcd-panel-divider:hover { + background: #adb5bd; +} +.etcd-panel-right { + width: 30%; + min-width: 30%; + padding-left: 12px; +} +.etcd-chart-card { + background: #fff; + border: 1px solid #dee2e6; + border-radius: 6px; + padding: 16px; + margin-bottom: 16px; +} +.etcd-chart-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} +.etcd-chart-card h6 { + margin: 0; + color: #495057; + font-size: 0.85rem; + font-weight: 600; +} +.etcd-chart-toolbar { + display: flex; + gap: 4px; +} +.etcd-chart-btn { + background: #f8f9fa; + border: 1px solid #dee2e6; + border-radius: 4px; + padding: 2px 8px; + font-size: 0.75rem; + cursor: pointer; + color: #495057; + line-height: 1.4; +} +.etcd-chart-btn:hover { + background: #e9ecef; +} +.etcd-chart-overlay { + position: fixed; + top: 0; left: 0; right: 0; bottom: 0; + background: rgba(0,0,0,0.6); + z-index: 9999; + display: flex; + align-items: center; + justify-content: center; +} +.etcd-chart-overlay-content { + background: #fff; + border-radius: 8px; + padding: 20px; + width: 90vw; + max-height: 90vh; +} +.etcd-chart-overlay-header { + display: flex; + justify-content: flex-end; + gap: 4px; + margin-bottom: 8px; } \ No newline at end of file diff --git a/data/templates/report/report.html b/data/templates/report/report.html index 0bd93b9c..2730bc6e 100644 --- a/data/templates/report/report.html +++ b/data/templates/report/report.html @@ -10,6 +10,10 @@ + + + + @@ -21,12 +25,11 @@
- + - - +
@@ -98,8 +101,19 @@
Loading... ({{ loadingMessage }})
-
-
+
+
+
+ +
+
+
+ +
+
+
+ +
@@ -201,6 +215,13 @@ isLoading: true, loadingMessage: '', showMetrics: false, + menuBodyRight: '', + chartColors: ['#4e79a7', '#f28e2b', '#e15759', '#76b7b2', '#59a14f', '#edc948', '#b07aa1', '#ff9da7'], + chartInstances: {}, + etcdCharts: [ + { id: "etcd-chart-0", path: "./metrics/query_range-etcd-disk-fsync-wal-duration-p99.json.gz.json", title: "etcd fsync WAL p99", unit: "ms", convertToMs: true }, + { id: "etcd-chart-1", path: "./metrics/query_range-etcd-disk-fsync-db-duration-p99.json.gz.json", title: "etcd fsync DB p99", unit: "ms", convertToMs: true }, + ], }; }, methods: { @@ -288,6 +309,15 @@ changeMenuCleanup() { this.menuTitle = ''; this.menuBody = ''; + this.menuBodyRight = ''; + // Destroy existing Chart.js instances to prevent memory leaks + Object.values(this.chartInstances).forEach(chart => chart.destroy()); + this.chartInstances = {}; + this._chartRenderToken = (this._chartRenderToken || 0) + 1; + var left = document.getElementById('panel-left'); + if (left) { left.style.width = ''; left.style.minWidth = ''; } + var right = document.getElementById('panel-right'); + if (right) { right.style.width = ''; right.style.minWidth = ''; } }, createTableRevHTML(table=[]) { htmlTable = `` @@ -760,10 +790,8 @@ changeMenuETCD() { this.menuTitle = `

etcd information

` this.menuBody = this.pageHeadline - this.menuBody += "

Information extracted from etcd logs." + this.menuBody += "

Information extracted from etcd logs.

" - // TODO#1 create checks / summary in the top of page - // TODO#2 implement checks rules if (this.report.provider.mustGatherInfo.ErrorEtcdLogs.ErrorCounters !== undefined) { table = this.extractErrorCountersToTable(this.report.provider.mustGatherInfo.ErrorEtcdLogs.ErrorCounters) table.header = "Counters for error pattern in etcd logs" @@ -775,7 +803,6 @@ "Aggregated pod logs 'apply request took too long'", this.report.provider.mustGatherInfo.ErrorEtcdLogs) } - // by pod for (let i in this.report.provider.mustGatherInfo.NamespaceErrors) { if (this.report.provider.mustGatherInfo.NamespaceErrors[i].ErrorEtcdLogs !== undefined) { podName = this.report.provider.mustGatherInfo.NamespaceErrors[i].Pod @@ -785,6 +812,144 @@ this.report.provider.mustGatherInfo.NamespaceErrors[i].ErrorEtcdLogs) } } + + if (this.report.summary.features.hasMetricsData) { + this.menuBodyRight = '
' + + '📊 More Metrics: ' + + 'Open full dashboard ' + + 'to view all 6 charts in an interactive grid dashboard with click-to-expand functionality.' + + '
' + for (let chart of this.etcdCharts) { + this.menuBodyRight += `
` + + `
` + chart.title + `
` + + `
` + + `` + + `` + + `
` + + `
` + } + this.$nextTick(() => { this.renderEtcdCharts(); }); + } + }, + resetChartZoom(chartId) { + let instance = this.chartInstances[chartId]; + if (instance) instance.resetZoom(); + }, + expandChart(chartId) { + let instance = this.chartInstances[chartId]; + if (!instance) return; + let overlay = document.createElement('div'); + overlay.className = 'etcd-chart-overlay'; + overlay.innerHTML = `
` + + `
` + + `` + + `` + + `
`; + document.body.appendChild(overlay); + let canvas = document.getElementById('overlay-canvas'); + let expanded = new Chart(canvas, { + type: 'line', + data: JSON.parse(JSON.stringify(instance.data)), + options: Object.assign({}, JSON.parse(JSON.stringify(instance.options)), { + aspectRatio: 1.8, + plugins: { + legend: { position: 'bottom', labels: { boxWidth: 12, font: { size: 12 } } }, + tooltip: { mode: 'index', intersect: false }, + zoom: { + zoom: { drag: { enabled: true, backgroundColor: 'rgba(78,121,167,0.15)', borderColor: '#4e79a7', borderWidth: 1 }, mode: 'x' }, + pan: { enabled: true, mode: 'x' }, + }, + }, + }), + }); + // re-parse date objects lost in JSON serialization + expanded.data.datasets.forEach((ds, i) => { + ds.data = instance.data.datasets[i].data.map(p => ({ x: new Date(p.x), y: p.y })); + ds.borderColor = instance.data.datasets[i].borderColor; + ds.backgroundColor = instance.data.datasets[i].backgroundColor; + }); + expanded.update(); + document.getElementById('overlay-close').onclick = function() { expanded.destroy(); document.body.removeChild(overlay); }; + document.getElementById('overlay-reset').onclick = function() { expanded.resetZoom(); }; + overlay.onclick = function(e) { if (e.target === overlay) { expanded.destroy(); document.body.removeChild(overlay); } }; + }, + renderEtcdCharts() { + // Generate render token to ignore stale axios responses + const renderToken = (this._chartRenderToken || 0) + 1; + this._chartRenderToken = renderToken; + // Destroy existing Chart.js instances before creating new ones + Object.values(this.chartInstances).forEach(chart => chart.destroy()); + this.chartInstances = {}; + for (let chart of this.etcdCharts) { + axios.get(chart.path) + .then(resp => { + // Ignore stale responses from previous render calls + if (renderToken !== this._chartRenderToken) return; + // Parse Prometheus JSON format (query_range API response) + let promResponse = resp.data; + if (!promResponse.data || !promResponse.data.result) { + throw new Error('Invalid Prometheus response format'); + } + + let datasets = promResponse.data.result.map((result, i) => { + // Extract label from metric (first non-__name__ key) + let labelKey = Object.keys(result.metric).find(k => k !== '__name__') || 'instance'; + let labelValue = result.metric[labelKey] || 'unknown'; + + // Convert Prometheus values [timestamp, "value"] to Chart.js format + let dataPoints = result.values.map(v => ({ + x: new Date(v[0] * 1000), // Unix timestamp to JS Date + y: chart.convertToMs ? parseFloat(v[1]) * 1000 : parseFloat(v[1]) // Convert seconds to ms if needed + })); + + return { + label: labelValue, + data: dataPoints, + borderColor: this.chartColors[i % this.chartColors.length], + backgroundColor: 'transparent', + borderWidth: 1.5, + pointRadius: 0, + tension: 0.3, + }; + }); + let canvas = document.getElementById(chart.id); + if (!canvas) return; + this.chartInstances[chart.id] = new Chart(canvas, { + type: 'line', + data: { datasets: datasets }, + options: { + responsive: true, + maintainAspectRatio: true, + aspectRatio: 2.2, + interaction: { mode: 'index', intersect: false }, + plugins: { + legend: { position: 'bottom', labels: { boxWidth: 12, font: { size: 11 } } }, + tooltip: { mode: 'index', intersect: false }, + zoom: { + zoom: { drag: { enabled: true, backgroundColor: 'rgba(78,121,167,0.15)', borderColor: '#4e79a7', borderWidth: 1 }, mode: 'x' }, + pan: { enabled: true, mode: 'x' }, + }, + }, + scales: { + x: { type: 'time', time: { tooltipFormat: 'yyyy-MM-dd HH:mm' }, ticks: { maxTicksAllowed: 8, font: { size: 10 } } }, + y: { + beginAtZero: false, + ticks: { font: { size: 10 } }, + title: { + display: true, + text: chart.unit, + font: { size: 11, weight: 'bold' } + } + }, + }, + }, + }); + }) + .catch(() => { + let el = document.getElementById(chart.id); + if (el) el.parentElement.innerHTML = '

Chart not available

'; + }); + } }, changeMenuNetwork() { this.menuTitle = `

Network

` @@ -1165,6 +1330,36 @@ /* main */ document.getElementById("tabDefault").click(); app_opct.changeMenu('summary') + + /* Resizable split-pane divider */ + document.addEventListener('mousedown', function(e) { + if (e.target.id !== 'panel-divider') return; + e.preventDefault(); + var container = document.getElementById('split-container'); + var left = document.getElementById('panel-left'); + var right = document.getElementById('panel-right'); + var divider = document.getElementById('panel-divider'); + var startX = e.clientX; + var startLeftW = left.offsetWidth; + var totalW = container.offsetWidth - divider.offsetWidth; + + function onMove(e) { + var dx = e.clientX - startX; + var newLeftW = Math.max(200, Math.min(totalW - 200, startLeftW + dx)); + var leftPct = (newLeftW / container.offsetWidth) * 100; + var rightPct = ((totalW - newLeftW) / container.offsetWidth) * 100; + left.style.width = leftPct + '%'; + left.style.minWidth = leftPct + '%'; + right.style.width = rightPct + '%'; + right.style.minWidth = rightPct + '%'; + } + function onUp() { + document.removeEventListener('mousemove', onMove); + document.removeEventListener('mouseup', onUp); + } + document.addEventListener('mousemove', onMove); + document.addEventListener('mouseup', onUp); + }); diff --git a/go.mod b/go.mod index 444dbc33..cb6bd048 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( require ( github.com/aws/aws-sdk-go v1.55.6 github.com/evanphx/json-patch v4.12.0+incompatible - github.com/go-echarts/go-echarts/v2 v2.5.1 github.com/google/go-cmp v0.7.0 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jedib0t/go-pretty/v6 v6.6.7 diff --git a/go.sum b/go.sum index 72c611e1..bb22253c 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-echarts/go-echarts/v2 v2.5.1 h1:kFVNaS3IsszKOQmUyCi95D2IhipE5twfvaBhFLOfPrs= -github.com/go-echarts/go-echarts/v2 v2.5.1/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= diff --git a/internal/opct/summary/result.go b/internal/opct/summary/result.go index 58fc305c..6b23de52 100644 --- a/internal/opct/summary/result.go +++ b/internal/opct/summary/result.go @@ -512,15 +512,15 @@ func (rs *ResultSummary) extractAndLoadData() error { log.Error("Processing results/Populating/Populating Summary/Processing/CAMGI: Not Found") } if len(MetricsData.Bytes()) > 0 { - rs.Metrics, err = mustgathermetrics.NewMustGatherMetrics(rs.SavePath+"/metrics", pathMetrics, "/metrics", &MetricsData) + rs.Metrics, err = mustgathermetrics.NewMustGatherMetrics(rs.SavePath+"/metrics", &MetricsData) if err != nil { log.Errorf("Processing results/Populating/Populating Summary/Processing/MetricsData: %v", err) } else { - err := rs.Metrics.Process() - if err != nil { + if err := rs.Metrics.Process(); err != nil { log.Errorf("Processing MetricsData: %v", err) + } else { + rs.HasMetrics = true } - rs.HasMetrics = true } } else { log.Error("Processing results/Populating/Populating Summary/Processing/MetricsData: Not Found") diff --git a/internal/openshift/mustgathermetrics/charts-config.json b/internal/openshift/mustgathermetrics/charts-config.json new file mode 100644 index 00000000..fe882c7c --- /dev/null +++ b/internal/openshift/mustgathermetrics/charts-config.json @@ -0,0 +1,40 @@ +{ + "charts": [ + { + "file": "query_range-etcd-disk-fsync-db-duration-p99.json.gz", + "label": "instance", + "title": "etcd fsync DB p99", + "id": "id1" + }, + { + "file": "query_range-api-kas-request-duration-p99.json.gz", + "label": "verb", + "title": "Kube API request p99", + "id": "id2" + }, + { + "file": "query_range-etcd-disk-fsync-wal-duration-p99.json.gz", + "label": "instance", + "title": "etcd fsync WAL p99", + "id": "id0" + }, + { + "file": "query_range-etcd-peer-round-trip-time.json.gz", + "label": "instance", + "title": "etcd peer round trip", + "id": "id3" + }, + { + "file": "query_range-etcd-total-leader-elections-day.json.gz", + "label": "instance", + "title": "etcd peer total leader election", + "id": "id4" + }, + { + "file": "query_range-etcd-request-duration-p99.json.gz", + "label": "operation", + "title": "etcd req duration p99", + "id": "id5" + } + ] +} diff --git a/internal/openshift/mustgathermetrics/charts.go b/internal/openshift/mustgathermetrics/charts.go deleted file mode 100644 index 1711079a..00000000 --- a/internal/openshift/mustgathermetrics/charts.go +++ /dev/null @@ -1,198 +0,0 @@ -package mustgathermetrics - -import ( - "encoding/json" - "fmt" - "io" - "os" - "time" - - "github.com/go-echarts/go-echarts/v2/charts" - "github.com/go-echarts/go-echarts/v2/components" - "github.com/go-echarts/go-echarts/v2/opts" - log "github.com/sirupsen/logrus" - "k8s.io/utils/ptr" -) - -type MetricValue struct { - Timestap time.Time - Value string -} - -type PrometheusResultMetric struct { - Metric map[string]string `json:"metric"` - Values [][]interface{} `json:"values"` -} - -type PrometheusResponse struct { - Status string `json:"status"` - Data struct { - ResultType string `json:"resultType"` - Result []PrometheusResultMetric `json:"result"` - } `json:"data"` -} - -type readMetricInput struct { - filename string - label string - title string - subtitle string -} - -// newMetricsPage create the page object to genera the metric report. -func newMetricsPage() *components.Page { - page := components.NewPage() - page.PageTitle = "OPCT Report Metrics" - return page -} - -// SaveMetricsPageReport Create HTML metrics file in a given path. -func SaveMetricsPageReport(page *components.Page, path string) error { - - f, err := os.Create(path) - if err != nil { - return err - } - if err := page.Render(io.MultiWriter(f)); err != nil { - return err - } - return nil -} - -func (mmm *MustGatherChart) NewChart() *charts.Line { - return mmm.processMetric(&readMetricInput{ - filename: mmm.Path, - label: mmm.PlotLabel, - title: mmm.PlotTitle, - subtitle: mmm.PlotSubTitle, - }) -} - -func (mmm *MustGatherChart) NewCharts() []*charts.Line { - in := &readMetricInput{ - filename: mmm.Path, - label: mmm.PlotLabel, - title: mmm.PlotTitle, - subtitle: mmm.PlotSubTitle, - } - return mmm.processMetrics(in) -} - -// LoadData generates the metric widget (plot graph from data series). -func (mmm *MustGatherChart) LoadData(payload []byte) error { - mmm.MetricData = &PrometheusResponse{} - - err := json.Unmarshal(payload, &mmm.MetricData) - if err != nil { - log.Errorf("Metrics/Extractor/Processing/LoadMetric ERROR parsing metric data: %v", err) - return err - } - log.Debugf("Metrics/Extractor/Processing/LoadMetric Status: %s\n", mmm.MetricData.Status) - return nil -} - -// processMetric generates the metric widget (plot graph from data series). -func (mmm *MustGatherChart) processMetric(in *readMetricInput) *charts.Line { - - line := charts.NewLine() - line.SetGlobalOptions( - charts.WithTitleOpts(opts.Title{ - Title: in.title, - Subtitle: in.subtitle, - }), - charts.WithTooltipOpts(opts.Tooltip{Show: ptr.To(true), Trigger: "axis"}), - ) - - allTimestamps := []string{} - - type ChartData struct { - Label string - DataPoints []opts.LineData - } - - chartData := []ChartData{} - idx := 0 - for _, res := range mmm.MetricData.Data.Result { - chart := ChartData{ - Label: res.Metric[in.label], - DataPoints: make([]opts.LineData, 0), - } - for _, datapoints := range res.Values { - value := datapoints[1].(string) - if value == "" { - log.Debugf("Metrics/Extractor/Processing/GenChart: Empty value [%s], ignoring...", value) - continue - } - // Convert from Unix timestamp to string value - tm := time.Unix(int64(datapoints[0].(float64)), 0) - strTimestamp := fmt.Sprintf("%d-%d-%d %d:%d:%d", tm.Year(), tm.Month(), tm.Day(), tm.Hour(), tm.Minute(), tm.Second()) - - allTimestamps = append(allTimestamps, strTimestamp) - chart.DataPoints = append(chart.DataPoints, opts.LineData{ - Value: value, - XAxisIndex: idx, - }) - idx += 1 - } - chartData = append(chartData, chart) - } - - // sort.Strings(allTimestamps) - line.SetXAxis(allTimestamps). - SetSeriesOptions(charts.WithLineChartOpts( - opts.LineChart{Smooth: ptr.To(false), ShowSymbol: ptr.To(true), SymbolSize: 15, Symbol: "diamond"}, - )) - for _, chart := range chartData { - line.AddSeries(chart.Label, chart.DataPoints) - } - - return line -} - -// processMetric generates the metric widget (plot graph from data series). -func (mmm *MustGatherChart) processMetrics(in *readMetricInput) []*charts.Line { - - var lines []*charts.Line - idx := 0 - for _, res := range mmm.MetricData.Data.Result { - allTimestamps := []string{} - line := charts.NewLine() - line.SetGlobalOptions( - charts.WithTitleOpts(opts.Title{ - Title: in.title, - Subtitle: in.subtitle, - }), - charts.WithTooltipOpts(opts.Tooltip{Show: ptr.To(true), Trigger: "axis"}), - ) - dataPoints := make([]opts.LineData, 0) - for _, datapoints := range res.Values { - value := datapoints[1].(string) - if value == "" { - log.Debugf("Metrics/Extractor/Processing/GenChart: Empty value [%s], ignoring...", value) - continue - } - // Convert from Unix timestamp to string value - tm := time.Unix(int64(datapoints[0].(float64)), 0) - strTimestamp := fmt.Sprintf("%d-%d-%d %d:%d:%d", tm.Year(), tm.Month(), tm.Day(), tm.Hour(), tm.Minute(), tm.Second()) - - allTimestamps = append(allTimestamps, strTimestamp) - dataPoints = append(dataPoints, opts.LineData{ - Value: value, - XAxisIndex: idx, - }) - idx += 1 - } - line.SetXAxis(allTimestamps). - SetSeriesOptions(charts.WithLineChartOpts( - opts.LineChart{Smooth: ptr.To(false), ShowSymbol: ptr.To(true), SymbolSize: 15, Symbol: "diamond"}, - )) - line.AddSeries(res.Metric[in.label], dataPoints) - lines = append(lines, line) - } - - // sort.Strings(allTimestamps) - // line.SetSeriesOptions(charts.WithLineChartOpts( - // opts.LineChart{Smooth: false, ShowSymbol: true, SymbolSize: 15, Symbol: "diamond"}, - // )) - return lines -} diff --git a/internal/openshift/mustgathermetrics/index.html b/internal/openshift/mustgathermetrics/index.html new file mode 100644 index 00000000..d1ebe625 --- /dev/null +++ b/internal/openshift/mustgathermetrics/index.html @@ -0,0 +1,10 @@ + + + + + Redirecting to Metrics + + +

Redirecting to metrics dashboard...

+ + diff --git a/internal/openshift/mustgathermetrics/main.go b/internal/openshift/mustgathermetrics/main.go index 7d321bbf..6fb46c2f 100644 --- a/internal/openshift/mustgathermetrics/main.go +++ b/internal/openshift/mustgathermetrics/main.go @@ -4,238 +4,342 @@ import ( "archive/tar" "bytes" "compress/gzip" + _ "embed" + "encoding/json" "fmt" "io" + "math" + "os" "path/filepath" + "strconv" "strings" log "github.com/sirupsen/logrus" "github.com/ulikunitz/xz" ) -type MustGatherChart struct { - Path string - OriginalQuery string - PlotLabel string - PlotTitle string - PlotSubTitle string - CollectorAvailable bool - MetricData *PrometheusResponse - DivId string +//go:embed charts-config.json +var chartsConfigJSON []byte + +//go:embed metrics.html +var metricsHTML []byte + +//go:embed index.html +var indexHTML []byte + +// ChartConfig represents a single metric chart configuration +type ChartConfig struct { + File string `json:"file"` + Label string `json:"label"` + Title string `json:"title"` + ID string `json:"id"` } -type MustGatherCharts map[string]*MustGatherChart +// ChartsConfig represents the configuration file structure +type ChartsConfig struct { + Charts []ChartConfig `json:"charts"` +} +// PrometheusResultMetric represents a single result from Prometheus query_range API +type PrometheusResultMetric struct { + Metric map[string]string `json:"metric"` + Values [][]interface{} `json:"values"` +} + +// PrometheusResponse represents the response from Prometheus query_range API +type PrometheusResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []PrometheusResultMetric `json:"result"` + } `json:"data"` +} + +// Chart represents a single metric chart with its data +type Chart struct { + Config ChartConfig + Data *PrometheusResponse +} + +// MustGatherMetrics processes metrics from must-gather archive type MustGatherMetrics struct { - fileName string - data *bytes.Buffer - ReportPath string - ReportChartFile string - ServePath string - charts MustGatherCharts - page *ChartPagePlotly + reportPath string + data *bytes.Buffer + charts map[string]*Chart } -func NewMustGatherMetrics(report, file, uri string, data *bytes.Buffer) (*MustGatherMetrics, error) { - mgm := &MustGatherMetrics{ - fileName: filepath.Base(file), - data: data, - ReportPath: report, - ServePath: uri, - ReportChartFile: "/metrics.html", - } - - mgm.charts = make(map[string]*MustGatherChart, 0) - mgm.charts["query_range-etcd-disk-fsync-db-duration-p99.json.gz"] = &MustGatherChart{ - Path: "query_range-etcd-disk-fsync-db-duration-p99.json.gz", - OriginalQuery: "", - PlotLabel: "instance", - PlotTitle: "etcd fsync DB p99", - PlotSubTitle: "", - CollectorAvailable: true, - DivId: "id1", - } - mgm.charts["query_range-api-kas-request-duration-p99.json.gz"] = &MustGatherChart{ - Path: "query_range-api-kas-request-duration-p99.json.gz", - OriginalQuery: "", - PlotLabel: "verb", - PlotTitle: "Kube API request p99", - PlotSubTitle: "", - CollectorAvailable: true, - DivId: "id2", - } - mgm.charts["query_range-etcd-disk-fsync-wal-duration-p99.json.gz"] = &MustGatherChart{ - Path: "query_range-etcd-disk-fsync-wal-duration-p99.json.gz", - OriginalQuery: "", - PlotLabel: "instance", - PlotTitle: "etcd fsync WAL p99", - PlotSubTitle: "", - CollectorAvailable: true, - DivId: "id0", - } - mgm.charts["query_range-etcd-peer-round-trip-time.json.gz"] = &MustGatherChart{ - Path: "query_range-etcd-peer-round-trip-time.json.gz", - OriginalQuery: "", - PlotLabel: "instance", - PlotTitle: "etcd peer round trip", - PlotSubTitle: "", - CollectorAvailable: true, - DivId: "id3", - } - - mgm.charts["query_range-etcd-total-leader-elections-day.json.gz"] = &MustGatherChart{ - Path: "query_range-etcd-total-leader-elections-day.json.gz", - OriginalQuery: "", - PlotLabel: "instance", - PlotTitle: "etcd peer total leader election", - PlotSubTitle: "", - CollectorAvailable: true, - DivId: "id4", - } - mgm.charts["query_range-etcd-request-duration-p99.json.gz"] = &MustGatherChart{ - Path: "query_range-etcd-request-duration-p99.json.gz", - OriginalQuery: "", - PlotLabel: "operation", - PlotTitle: "etcd req duration p99", - PlotSubTitle: "", - CollectorAvailable: true, - DivId: "id5", - } - mgm.charts["query_range-cluster-storage-iops.json.gz"] = &MustGatherChart{ - Path: "query_range-cluster-storage-iops.json.gz", - OriginalQuery: "", - PlotLabel: "namespace", - PlotTitle: "Cluster storage IOPS", - PlotSubTitle: "", - CollectorAvailable: false, - DivId: "id6", - } - mgm.charts["query_range-cluster-storage-throughput.json.gz"] = &MustGatherChart{ - Path: "query_range-cluster-storage-throughput.json.gz", - OriginalQuery: "", - PlotLabel: "namespace", - PlotTitle: "Cluster storage throughput", - PlotSubTitle: "", - CollectorAvailable: false, - DivId: "id7", - } - mgm.charts["query_range-cluster-cpu-usage.json.gz"] = &MustGatherChart{ - Path: "query_range-cluster-cpu-usage.json.gz", - OriginalQuery: "", - PlotLabel: "namespace", - PlotTitle: "Cluster CPU", - PlotSubTitle: "", - CollectorAvailable: false, - DivId: "id8", - } - mgm.page = newMetricsPageWithPlotly(report, uri, mgm.charts) - return mgm, nil +// NewMustGatherMetrics creates a new metrics processor +func NewMustGatherMetrics(reportPath string, data *bytes.Buffer) (*MustGatherMetrics, error) { + // Load chart configurations + var config ChartsConfig + if err := json.Unmarshal(chartsConfigJSON, &config); err != nil { + return nil, fmt.Errorf("failed to load chart config: %w", err) + } + + // Initialize charts map + charts := make(map[string]*Chart) + for _, chartCfg := range config.Charts { + charts[chartCfg.File] = &Chart{ + Config: chartCfg, + } + } + + return &MustGatherMetrics{ + reportPath: reportPath, + data: data, + charts: charts, + }, nil } +// Process extracts and processes metrics from the must-gather archive func (mg *MustGatherMetrics) Process() error { - log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather/Reading") - tar, err := mg.read(mg.data) + log.Debugf("Processing must-gather metrics archive") + + // Read tar.xz archive + tarReader, err := mg.readArchive(mg.data) if err != nil { - return err + return fmt.Errorf("failed to read archive: %w", err) } - log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather/Processing") - err = mg.extract(tar) - if err != nil { - return err + + // Extract metrics + if err := mg.extractMetrics(tarReader); err != nil { + return fmt.Errorf("failed to extract metrics: %w", err) + } + + // Generate output files + if err := mg.generateOutputFiles(); err != nil { + return fmt.Errorf("failed to generate output files: %w", err) } + + log.Debugf("Metrics processing complete: %s", mg.reportPath) return nil } -func (mg *MustGatherMetrics) read(buf *bytes.Buffer) (*tar.Reader, error) { - file, err := xz.NewReader(buf) +// readArchive reads the tar.xz archive +func (mg *MustGatherMetrics) readArchive(buf *bytes.Buffer) (*tar.Reader, error) { + xzReader, err := xz.NewReader(buf) if err != nil { return nil, err } - return tar.NewReader(file), nil + return tar.NewReader(xzReader), nil } -// extract dispatch to process must-gather items. -func (mg *MustGatherMetrics) extract(tarball *tar.Reader) error { - - keepReading := true - metricsPage := newMetricsPage() - reportPath := mg.ReportPath + mg.ReportChartFile - - // Walk through files in tarball. - for keepReading { - header, err := tarball.Next() +// extractMetrics walks through the tar archive and extracts metric files +func (mg *MustGatherMetrics) extractMetrics(tarReader *tar.Reader) error { + for { + header, err := tarReader.Next() switch { - - // no more files case err == io.EOF: - - err := SaveMetricsPageReport(metricsPage, reportPath) - if err != nil { - log.Errorf("error saving metrics to: %s\n", reportPath) - return err - } - // Ploty Page - log.Debugf("Generating Charts with Plotly\n") - err = mg.page.RenderPage() - if err != nil { - log.Errorf("error rendering page: %v\n", err) - return err - } - - log.Debugf("metrics saved at: %s\n", reportPath) return nil - - // return on error case err != nil: - return fmt.Errorf("error reading tarball: %w", err) - - // skip it when the headr isn't set (not sure how this happens) + return fmt.Errorf("error reading tar: %w", err) case header == nil: continue } - // process only metris file. Example: monitoring/prometheus/metrics/metric.json.gz - if !(strings.HasPrefix(header.Name, "monitoring/prometheus/metrics") && strings.HasSuffix(header.Name, ".json.gz")) { + // Only process Prometheus metric files: monitoring/prometheus/metrics/*.json.gz + if !strings.HasPrefix(header.Name, "monitoring/prometheus/metrics") { continue } - - metricFileName := filepath.Base(header.Name) - - chart, ok := mg.charts[metricFileName] - if !ok { - log.Debugf("Metrics/Extractor/Unsupported metric, ignoring metric data %s\n", header.Name) + if !strings.HasSuffix(header.Name, ".json.gz") { continue } - if !chart.CollectorAvailable { - log.Debugf("Metrics/Extractor/No charts available for metric %s\n", header.Name) + + fileName := filepath.Base(header.Name) + chart, ok := mg.charts[fileName] + if !ok { + log.Debugf("Skipping unsupported metric: %s", fileName) continue } - log.Debugf("Metrics/Extractor/Processing: %s\n", header.Name) - gz, err := gzip.NewReader(tarball) + log.Debugf("Processing metric: %s", fileName) + + // Decompress gzip + gzReader, err := gzip.NewReader(tarReader) if err != nil { - log.Debugf("Metrics/Extractor/Processing/ERROR reading metric %v", err) + log.Warnf("Failed to decompress %s: %v", fileName, err) continue } - defer gz.Close() + + // Read metric data var metricPayload bytes.Buffer - if _, err := io.Copy(&metricPayload, gz); err != nil { - log.Debugf("Metrics/Extractor/Processing/ERROR copying metric data for %v", err) + if _, err := io.Copy(&metricPayload, gzReader); err != nil { + gzReader.Close() + log.Warnf("Failed to read %s: %v", fileName, err) continue } + gzReader.Close() - err = chart.LoadData(metricPayload.Bytes()) - if err != nil { - log.Debugf("Metrics/Extractor/Processing/ERROR loading metric for %v", err) + // Parse Prometheus JSON + var promResponse PrometheusResponse + if err := json.Unmarshal(metricPayload.Bytes(), &promResponse); err != nil { + log.Warnf("Failed to parse JSON for %s: %v", fileName, err) + continue + } + + chart.Data = &promResponse + log.Debugf("Loaded metric: %s (status=%s)", fileName, promResponse.Status) + } +} + +// generateOutputFiles creates index.json and individual chart JSON files +func (mg *MustGatherMetrics) generateOutputFiles() error { + // Create output directory + if err := os.MkdirAll(mg.reportPath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", mg.reportPath, err) + } + + // Build index + type IndexEntry struct { + ID string `json:"id"` + Path string `json:"path"` + } + var index []IndexEntry + + // Process each chart + for fileName, chart := range mg.charts { + if chart.Data == nil { + log.Debugf("Skipping chart %s: no data loaded", fileName) + continue + } + + // Filter NaN/Inf values (JSON doesn't support them) + filteredData := mg.filterInvalidValues(chart.Data) + + // Check if any valid data remains + hasValidData := false + for _, result := range filteredData.Data.Result { + if len(result.Values) > 0 { + hasValidData = true + break + } + } + + if !hasValidData { + log.Warnf("Skipping chart %s: no valid data after filtering", fileName) continue } - // charts with - for _, line := range chart.NewCharts() { - metricsPage.AddCharts(line) + // Save chart JSON (Prometheus format) + chartPath := filepath.Join(mg.reportPath, fileName+".json") + if err := mg.saveChartJSON(chartPath, filteredData); err != nil { + return fmt.Errorf("failed to save chart %s: %w", fileName, err) } - log.Debugf("Metrics/Extractor/Processing/Done %v", header.Name) + + // Add to index + index = append(index, IndexEntry{ + ID: chart.Config.ID, + Path: fmt.Sprintf("./%s.json", fileName), + }) + + log.Debugf("Saved chart: %s", chartPath) + } + + // Save index.json + indexPath := filepath.Join(mg.reportPath, "index.json") + if len(index) == 0 { + return fmt.Errorf("no chart JSON files were generated") + } + + indexJSON, err := json.MarshalIndent(index, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal index: %w", err) + } + + if err := os.WriteFile(indexPath, indexJSON, 0644); err != nil { + return fmt.Errorf("failed to write index.json: %w", err) + } + + log.Debugf("Saved index: %s (%d charts)", indexPath, len(index)) + + // Save metrics.html (interactive dashboard) + metricsHTMLPath := filepath.Join(mg.reportPath, "metrics.html") + if err := os.WriteFile(metricsHTMLPath, metricsHTML, 0644); err != nil { + return fmt.Errorf("failed to write metrics.html: %w", err) + } + log.Debugf("Saved metrics dashboard: %s", metricsHTMLPath) + + // Save index.html (redirect to metrics.html) + indexHTMLPath := filepath.Join(mg.reportPath, "index.html") + if err := os.WriteFile(indexHTMLPath, indexHTML, 0644); err != nil { + return fmt.Errorf("failed to write index.html: %w", err) + } + log.Debugf("Saved index redirect: %s", indexHTMLPath) + + return nil +} + +// filterInvalidValues removes NaN and Inf values from Prometheus response +// JSON doesn't support these values, so they must be filtered out +func (mg *MustGatherMetrics) filterInvalidValues(data *PrometheusResponse) *PrometheusResponse { + filtered := &PrometheusResponse{ + Status: data.Status, + } + filtered.Data.ResultType = data.Data.ResultType + + totalDatapoints := 0 + filteredDatapoints := 0 + + for _, result := range data.Data.Result { + validValues := [][]interface{}{} + originalCount := len(result.Values) + totalDatapoints += originalCount + + for _, v := range result.Values { + // v[0] = timestamp (float64), v[1] = value (string) + if len(v) < 2 { + continue + } + + valueStr, ok := v[1].(string) + if !ok { + continue + } + + // Parse and check for NaN/Inf + valueFloat, err := strconv.ParseFloat(valueStr, 64) + if err != nil { + continue + } + + if math.IsNaN(valueFloat) || math.IsInf(valueFloat, 0) { + filteredDatapoints++ + continue + } + + // Valid value, keep it + validValues = append(validValues, v) + } + + // Only include results with valid data + if len(validValues) > 0 { + filteredResult := PrometheusResultMetric{ + Metric: result.Metric, + Values: validValues, + } + filtered.Data.Result = append(filtered.Data.Result, filteredResult) + } + } + + // Log summary if data was filtered + if filteredDatapoints > 0 { + log.Debugf("Filtered %d NaN/Inf datapoints from %d total (%.1f%% invalid)", + filteredDatapoints, totalDatapoints, float64(filteredDatapoints)/float64(totalDatapoints)*100) + } + + return filtered +} + +// saveChartJSON saves a chart's Prometheus data as JSON +func (mg *MustGatherMetrics) saveChartJSON(path string, data *PrometheusResponse) error { + jsonData, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON: %w", err) + } + + if err := os.WriteFile(path, jsonData, 0644); err != nil { + return fmt.Errorf("failed to write file: %w", err) } return nil diff --git a/internal/openshift/mustgathermetrics/metrics.html b/internal/openshift/mustgathermetrics/metrics.html new file mode 100644 index 00000000..9abcaa6b --- /dev/null +++ b/internal/openshift/mustgathermetrics/metrics.html @@ -0,0 +1,397 @@ + + + + + + OPCT Metrics + + + + + + + + + +
+
+

OPCT Metrics Dashboard

+

Prometheus metrics collected during conformance testing

+
+ +
+ +
+
+ + +
+
+
+ + +
+
+ +
+
+
+ + + + + diff --git a/internal/openshift/mustgathermetrics/plotly.go b/internal/openshift/mustgathermetrics/plotly.go deleted file mode 100644 index 627e7375..00000000 --- a/internal/openshift/mustgathermetrics/plotly.go +++ /dev/null @@ -1,246 +0,0 @@ -package mustgathermetrics - -import ( - "bytes" - "encoding/json" - "fmt" - "math" - "os" - "sort" - "strconv" - "text/template" - "time" - - log "github.com/sirupsen/logrus" -) - -type ChartPagePlotly struct { - PageTitle string - Charts MustGatherCharts - RootPath string - UriPath string -} - -const indexHTML = ` - - - OPCT Charts - - - - - -
-{{.Table}} - -` - -// inspired by https://github.com/353words/stocks/tree/main -const indexJS = ` -async function updateCharts() { - let chartsResp = await fetch('./index.json'); - let charts = await chartsResp.json(); - for (idx in charts) { - let resp = await fetch(charts[idx].path); - let reply = await resp.json(); - Plotly.newPlot(charts[idx].id, reply.data, reply.layout); - } -}` - -func newMetricsPageWithPlotly(path, uri string, charts MustGatherCharts) *ChartPagePlotly { - - page := &ChartPagePlotly{ - PageTitle: "OPCT Report Metrics", - Charts: charts, - RootPath: path, - UriPath: uri, - } - - // create base dir - err := os.Mkdir(page.RootPath, 0755) - if err != nil { - log.Errorf("Unable to create directory %s: %v", page.RootPath, err) - } - log.Debugf("Chart/Directory created %s", page.RootPath) - return page -} - -func roundFloat(val float64, precision uint) float64 { - ratio := math.Pow(10, float64(precision)) - return math.Round(val*ratio) / ratio -} - -func (cpp *ChartPagePlotly) RenderPage() error { - - // - index.js - indexJsFilePath := fmt.Sprintf("%s/index.js", cpp.RootPath) - err := os.WriteFile(indexJsFilePath, []byte(indexJS), 0644) - if err != nil { - log.Errorf("Unable to save file %s: %v", indexJsFilePath, err) - } - log.Debugf("Chart/file saved %s", indexJsFilePath) - - // render metrics data - indexChartsMap := []map[string]string{} - validDivIds := []string{} - for k := range cpp.Charts { - if err := cpp.processMetricV2(k); err != nil { - log.Debug(err) - continue - } - if cpp.Charts[k].DivId != "" { - indexChartsMap = append(indexChartsMap, map[string]string{ - "id": cpp.Charts[k].DivId, - "path": fmt.Sprintf("./%s.json", cpp.Charts[k].Path), - }) - validDivIds = append(validDivIds, cpp.Charts[k].DivId) - } - } - // create table with charts - sort.Strings(validDivIds) - type TemplateData struct { - Table string - } - table := TemplateData{"\t\t
"} - for idx, div := range validDivIds { - if idx%2 == 0 { - table.Table += fmt.Sprintf("\n\t\t\t", div) - } else { - table.Table += fmt.Sprintf("", div) - } - } - table.Table += "\n\t\t
" - - // - index.html - indexHTMLFilePath := fmt.Sprintf("%s/index.html", cpp.RootPath) - tmplS, err := template.New("report").Parse(indexHTML) - if err != nil { - log.Errorf("Unable to create template for %s: %v", indexHTMLFilePath, err) - } - var fileBufferS bytes.Buffer - err = tmplS.Execute(&fileBufferS, table) - if err != nil { - log.Errorf("Unable to render template for %s: %v", indexHTMLFilePath, err) - } - - err = os.WriteFile(indexHTMLFilePath, fileBufferS.Bytes(), 0644) - if err != nil { - log.Errorf("Unable to save file %s: %v", indexHTMLFilePath, err) - } - log.Debugf("Chart/file saved %s", indexHTMLFilePath) - - // - index.json - indexJsonFileData, _ := json.MarshalIndent(indexChartsMap, "", " ") - indexJsonFilePath := fmt.Sprintf("%s/index.json", cpp.RootPath) - err = os.WriteFile(indexJsonFilePath, indexJsonFileData, 0644) - if err != nil { - log.Errorf("Unable to save file %s: %v", indexJsonFileData, err) - } - log.Debugf("Chart/file saved %s", indexJsonFilePath) - return nil -} - -// processMetric generates the metric widget (plot graph from data series). -func (cpp *ChartPagePlotly) processMetricV2(name string) error { - - chart := cpp.Charts[name] - type LabelData struct { - Name string - XAxis []string - YAxis []float64 - } - - type Labels map[string]LabelData - - // process query - labels := make(Labels, 0) - if chart.MetricData == nil { - return fmt.Errorf("empty metric data, ignoring metric %s", name) - } - log.Debugf("Processing metric %s", name) - for _, res := range chart.MetricData.Data.Result { - // process labels - for _, datapoints := range res.Values { - value := datapoints[1].(string) - if value == "" { - log.Debugf("Metrics/Extractor/Processing/GenChart: Empty value [%s], ignoring...", value) - continue - } - // Convert from Unix timestamp to string value - tm := time.Unix(int64(datapoints[0].(float64)), 0) - strTimestamp := fmt.Sprintf("%d-%d-%d %d:%d:%d", tm.Year(), tm.Month(), tm.Day(), tm.Hour(), tm.Minute(), tm.Second()) - - valF, err := strconv.ParseFloat(value, 64) - if err != nil { - log.Errorf("error metric %s: converting datapoint, ignoring", name) - continue - } - - labelValue := res.Metric[chart.PlotLabel] - if _, ok := labels[labelValue]; !ok { - labels[labelValue] = LabelData{Name: labelValue} - } - - label := labels[labelValue] - label.XAxis = append(label.XAxis, strTimestamp) - label.YAxis = append(label.YAxis, roundFloat(valF, 4)) - labels[labelValue] = label - } - } - - var data []map[string]interface{} - count := 1 - for _, label := range labels { - dataAxis := map[string]interface{}{ - "x": label.XAxis, - "y": label.YAxis, - "name": label.Name, - "type": "scatter", - "connectgaps": true, - "mode": "lines+markers", - } - if count != 1 { - dataAxis["yaxis"] = fmt.Sprintf("y%d", count) - } - data = append(data, dataAxis) - count += 1 - } - - if len(data) == 0 { - return fmt.Errorf("error processing metric: no valid data for %q", name) - } - - // create table with rows by label - reply := map[string]interface{}{ - "data": data, - "layout": map[string]interface{}{ - "title": chart.PlotTitle, - "grid": map[string]int{ - "rows": len(labels), - "columns": 1, - }, - "autosize": false, - "width": 1000, - "height": 1000, - }, - } - - indexJsonFileData, err := json.MarshalIndent(reply, "", " ") - if err != nil { - // log.Errorf("Unable to unmarshall metric file %v", err) - return fmt.Errorf("unable to unmarshall metric file %v", err) - } - - indexJsonFilePath := fmt.Sprintf("%s/%s.json", cpp.RootPath, chart.Path) - err = os.WriteFile(indexJsonFilePath, indexJsonFileData, 0644) - if err != nil { - // log.Errorf("Unable to save file %s: %v", indexJsonFilePath, err) - return fmt.Errorf("unable to save file %s: %v", indexJsonFilePath, err) - } - return nil -} diff --git a/pkg/cmd/adm/parsemetrics.go b/pkg/cmd/adm/parsemetrics.go index 80d1c365..45217431 100644 --- a/pkg/cmd/adm/parsemetrics.go +++ b/pkg/cmd/adm/parsemetrics.go @@ -62,8 +62,7 @@ func parseMetricsRun(cmd *cobra.Command, args []string) { panic(err) } - htmlFile := "/metrics.html" - mgm, err := mustgathermetrics.NewMustGatherMetrics(parseMetricsArgs.output, htmlFile, "/", buf) + mgm, err := mustgathermetrics.NewMustGatherMetrics(parseMetricsArgs.output, buf) if err != nil { log.Errorf("unable to read metric archive: %v", err) panic(err) @@ -73,7 +72,7 @@ func parseMetricsRun(cmd *cobra.Command, args []string) { log.Errorf("processing metric: %v", err) os.Exit(1) } - log.Infof("Success! HTML report created at %s/%s\n", parseMetricsArgs.output, htmlFile) - log.Infof("TIP: cd %s && python -m http.server", parseMetricsArgs.output) - log.Info("Open your browser and navigate the reports: http://localhost:8000/index.html http://localhost:8000/metrics.html") + log.Infof("Success! Chart JSON files created at %s/\n", parseMetricsArgs.output) + log.Infof("Generated files: index.json and individual chart JSON files") + log.Infof("TIP: These files are used by the OPCT web UI report") }