From 0375a5cca6339e7d8d7b0f838356c45189aff2f8 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Wed, 6 Nov 2024 10:16:08 +0000 Subject: [PATCH] WIP: zipped reports download --- deepfence_server/handler/export_reports.go | 1 + deepfence_server/model/reports.go | 1 + deepfence_utils/utils/constants.go | 1 + deepfence_utils/utils/structs.go | 1 + deepfence_utils/utils/utils.go | 40 ++ deepfence_worker/tasks/reports/pdf.go | 19 +- .../tasks/reports/pdf_cloud_compliance.go | 250 +++++++++---- .../tasks/reports/pdf_compliance.go | 349 ++++++++++++------ deepfence_worker/tasks/reports/pdf_malware.go | 207 ++++++++--- deepfence_worker/tasks/reports/pdf_secret.go | 200 +++++++--- .../tasks/reports/pdf_vulnerability.go | 215 ++++++++--- deepfence_worker/tasks/reports/reports.go | 53 ++- 12 files changed, 948 insertions(+), 389 deletions(-) diff --git a/deepfence_server/handler/export_reports.go b/deepfence_server/handler/export_reports.go index 1232c64388..ce42f826ad 100644 --- a/deepfence_server/handler/export_reports.go +++ b/deepfence_server/handler/export_reports.go @@ -492,6 +492,7 @@ func (h *Handler) GenerateReport(w http.ResponseWriter, r *http.Request) { ToTimestamp: toTimestamp, Filters: req.Filters, Options: req.Options, + ZippedReport: req.ZippedReport, } // scan id can only be sent while downloading individual scans diff --git a/deepfence_server/model/reports.go b/deepfence_server/model/reports.go index 46b636059a..51fc98af5a 100644 --- a/deepfence_server/model/reports.go +++ b/deepfence_server/model/reports.go @@ -10,6 +10,7 @@ type GenerateReportReq struct { ToTimestamp int64 `json:"to_timestamp"` // timestamp in milliseconds Filters utils.ReportFilters `json:"filters"` Options utils.ReportOptions `json:"options" validate:"omitempty"` + ZippedReport bool `json:"zipped_report"` } type GenerateReportResp struct { diff --git a/deepfence_utils/utils/constants.go b/deepfence_utils/utils/constants.go index f109c5f875..d9fb5cc5fc 100644 --- a/deepfence_utils/utils/constants.go +++ b/deepfence_utils/utils/constants.go @@ -252,6 +252,7 @@ const ( ReportXLSX ReportType = "xlsx" ReportPDF ReportType = "pdf" ReportSBOM ReportType = "sbom" + ReportZIP ReportType = "zip" ) // mask_global : This is to mask gobally. (same as previous mask_across_hosts_and_images flag) diff --git a/deepfence_utils/utils/structs.go b/deepfence_utils/utils/structs.go index 6f7f9e2842..9fbf930bf6 100644 --- a/deepfence_utils/utils/structs.go +++ b/deepfence_utils/utils/structs.go @@ -83,6 +83,7 @@ type ReportParams struct { ToTimestamp time.Time `json:"to_timestamp"` Filters ReportFilters `json:"filters"` Options ReportOptions `json:"options,omitempty"` + ZippedReport bool `json:"zipped_report"` } type ReportOptions struct { diff --git a/deepfence_utils/utils/utils.go b/deepfence_utils/utils/utils.go index 0990e75d68..030dd56c53 100644 --- a/deepfence_utils/utils/utils.go +++ b/deepfence_utils/utils/utils.go @@ -45,6 +45,8 @@ var ( SBOMFormatReplacer = strings.NewReplacer("@", "_", ".", "_") + NodeNameReplacer = strings.NewReplacer("/", "_", " ", "") + matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") once1, once2 sync.Once @@ -782,3 +784,41 @@ func ComputeChecksumForFile(filePath string) (string, error) { cs := fmt.Sprintf("%x", h.Sum(nil)) return cs, nil } + +func ZipDir(sourceDir string, baseZipPath string, outputZip string) error { + archive, err := os.Create(outputZip) + if err != nil { + return err + } + defer archive.Close() + + zw := zip.NewWriter(archive) + defer zw.Close() + + return filepath.Walk(sourceDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + f, err := zw.Create(filepath.Join(baseZipPath, info.Name())) + if err != nil { + return err + } + + _, err = io.Copy(f, file) + if err != nil { + return err + } + + return nil + }) +} diff --git a/deepfence_worker/tasks/reports/pdf.go b/deepfence_worker/tasks/reports/pdf.go index 062af7a949..6d22dde0d9 100644 --- a/deepfence_worker/tasks/reports/pdf.go +++ b/deepfence_worker/tasks/reports/pdf.go @@ -3,7 +3,6 @@ package reports import ( "context" _ "embed" - "os" "strconv" "time" @@ -189,7 +188,7 @@ func generatePDF(ctx context.Context, params utils.ReportParams) (string, error) log := log.WithCtx(ctx) var ( - document core.Document + document string err error ) @@ -213,19 +212,5 @@ func generatePDF(ctx context.Context, params utils.ReportParams) (string, error) return "", err } - // create a temp file to hold pdf report - temp, err := os.CreateTemp("", "report-*-"+reportFileName(params)) - if err != nil { - return "", err - } - defer temp.Close() - - if _, err := temp.Write(document.GetBytes()); err != nil { - return "", err - } - - log.Info().Msgf("report id %s pdf generation metrics %s", - params.ReportID, document.GetReport()) - - return temp.Name(), nil + return document, nil } diff --git a/deepfence_worker/tasks/reports/pdf_cloud_compliance.go b/deepfence_worker/tasks/reports/pdf_cloud_compliance.go index 5fa04a6e12..b25cd3d65d 100644 --- a/deepfence_worker/tasks/reports/pdf_cloud_compliance.go +++ b/deepfence_worker/tasks/reports/pdf_cloud_compliance.go @@ -3,11 +3,15 @@ package reports import ( "context" "fmt" + "os" + "path/filepath" "strconv" "strings" + "time" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/log" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/johnfercher/maroto/v2/pkg/components/page" "github.com/johnfercher/maroto/v2/pkg/components/row" "github.com/johnfercher/maroto/v2/pkg/components/text" @@ -19,56 +23,74 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func getCloudComplianceSummaryPage(data map[string]map[string]int32) core.Page { - - cellStyle := &props.Cell{ +var ( + cloudCompResultCellStyle = &props.Cell{ BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, BorderType: border.Full, BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, BorderThickness: 0.1, } - summaryPage := page.New() - summaryPage.Add(text.NewRow(12, "Scans Summary", - props.Text{Size: 10, Align: align.Center, Style: fontstyle.Bold})) - - summaryTableHeaderProps := props.Text{ + cloudCompResultHeaderProps = props.Text{ Size: 10, + Left: 1, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: &props.Color{Red: 0, Green: 0, Blue: 200}, } - summaryPage.Add( - row.New(6).Add( - text.NewCol(7, "Node Name", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Alarm", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Ok", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Skip", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Info", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Total", summaryTableHeaderProps).WithStyle(cellStyle), - ), - ) + cloudCompSummaryCellStyle = &props.Cell{ + BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, + BorderType: border.Full, + BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, + BorderThickness: 0.1, + } + + cloudCompSummaryTableHeaderProps = props.Text{ + Size: 10, + Top: 1, + Align: align.Center, + Style: fontstyle.Bold, + Color: &props.Color{Red: 0, Green: 0, Blue: 200}, + } - summaryProps := props.Text{ + cloudCompSummaryRowsProps = props.Text{ Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Normal, } +) + +func getCloudComplianceSummaryPage(data map[string]map[string]int32) core.Page { + + summaryPage := page.New() + summaryPage.Add(text.NewRow(12, "Scans Summary", + props.Text{Size: 10, Align: align.Center, Style: fontstyle.Bold})) + + summaryPage.Add( + row.New(6).Add( + text.NewCol(7, "Node Name", cloudCompSummaryTableHeaderProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, "Alarm", cloudCompSummaryTableHeaderProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, "Ok", cloudCompSummaryTableHeaderProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, "Skip", cloudCompSummaryTableHeaderProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, "Info", cloudCompSummaryTableHeaderProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, "Total", cloudCompSummaryTableHeaderProps).WithStyle(cloudCompSummaryCellStyle), + ), + ) summaryRows := []core.Row{} for k, v := range data { summaryRows = append( summaryRows, row.New(6).Add( - text.NewCol(7, k, summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["alarm"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["ok"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["skip"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["info"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["alarm"]+v["ok"]+v["skip"]+v["info"])), summaryProps).WithStyle(cellStyle), + text.NewCol(7, k, cloudCompSummaryRowsProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["alarm"])), cloudCompSummaryRowsProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["ok"])), cloudCompSummaryRowsProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["skip"])), cloudCompSummaryRowsProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["info"])), cloudCompSummaryRowsProps).WithStyle(cloudCompSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["alarm"]+v["ok"]+v["skip"]+v["info"])), cloudCompSummaryRowsProps).WithStyle(cloudCompSummaryCellStyle), ), ) } @@ -78,19 +100,26 @@ func getCloudComplianceSummaryPage(data map[string]map[string]int32) core.Page { return summaryPage } -func cloudcompliancePDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +func cloudcompliancePDF(ctx context.Context, params sdkUtils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getCloudComplianceData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get compliance info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", params.ReportID, data.NodeWiseData.RecordCount) + if !params.ZippedReport { + return createCloudCompSingleFile(data, params) + } + return createCloudCompZippedFile(data, params) +} + +func createCloudCompSingleFile(data *Info[model.CloudCompliance], params sdkUtils.ReportParams) (string, error) { // get new instance of marato m := getMarato() @@ -107,22 +136,6 @@ func cloudcompliancePDF(ctx context.Context, params utils.ReportParams) (core.Do // summary table summaryPage := getCloudComplianceSummaryPage(data.NodeWiseData.SeverityCount) - cellStyle := &props.Cell{ - BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, - BorderType: border.Full, - BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, - BorderThickness: 0.1, - } - - resultHeaderProps := props.Text{ - Size: 10, - Left: 1, - Top: 1, - Align: align.Center, - Style: fontstyle.Bold, - Color: &props.Color{Red: 0, Green: 0, Blue: 200}, - } - // page per scan resultPages := []core.Page{} for i, d := range data.NodeWiseData.ScanData { @@ -133,38 +146,8 @@ func cloudcompliancePDF(ctx context.Context, params utils.ReportParams) (core.Do } p := page.New() - p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", i), resultHeaderProps)) - p.Add(row.New(10).Add( - text.NewCol(1, "No.", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(5, "Resource", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Check Type", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(4, "Title", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Status", resultHeaderProps).WithStyle(cellStyle), - )) - - resultRows := []core.Row{} - for k, v := range d.ScanResults { - resultRows = append( - resultRows, - row.New(18).Add( - text.NewCol(1, strconv.Itoa(k+1), - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(5, v.Resource, - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(1, v.ComplianceCheckType, - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(4, v.Title, - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(1, v.Status, - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Status]}).WithStyle(cellStyle), - ), - ) - } - p.Add(resultRows...) + addCloudCompResultHeaders(p, i) + p.Add(getCloudCompResultRows(d)...) resultPages = append(resultPages, p) } @@ -173,5 +156,118 @@ func cloudcompliancePDF(ctx context.Context, params utils.ReportParams) (core.Do m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile(os.TempDir(), tempReportFile(params), doc.GetBytes()) +} + +func createCloudCompZippedFile(data *Info[model.CloudCompliance], params sdkUtils.ReportParams) (string, error) { + + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + // get new instance of marato + m := getMarato() + + // applied filter page + filtersPage := getFiltersPage( + data.Title, + data.ScanType, + data.AppliedFilters.NodeType, + fmt.Sprintf("%s - %s", data.StartTime, data.EndTime), + strings.Join(data.AppliedFilters.SeverityOrCheckType, ","), + data.AppliedFilters.AdvancedReportFilters.String(), + ) + + // summary tableparams sdkUtils.ReportParams + singleSummary := map[string]map[string]int32{ + i: data.NodeWiseData.SeverityCount[i], + } + summaryPage := getCloudComplianceSummaryPage(singleSummary) + + // skip if there are no results + if len(d.ScanResults) == 0 { + continue + } + + resultPage := page.New() + addCloudCompResultHeaders(resultPage, i) + resultPage.Add(getCloudCompResultRows(d)...) + + // add all pages + m.AddPages(filtersPage) + m.AddPages(summaryPage) + m.AddPages(resultPage) + + doc, err := m.Generate() + if err != nil { + return "", err + } + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + log.Info().Msgf("report id %s %s pdf generation metrics %s", + params.ReportID, outputFile, doc.GetReport()) + + if _, err := writeReportToFile(tmpDir, outputFile, doc.GetBytes()); err != nil { + log.Error().Err(err).Msg("failed to write report to file") + } + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func addCloudCompResultHeaders(p core.Page, nodeName string) { + p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", nodeName), cloudCompResultHeaderProps)) + p.Add(row.New(10).Add( + text.NewCol(1, "No.", cloudCompResultHeaderProps).WithStyle(cloudCompResultCellStyle), + text.NewCol(5, "Resource", cloudCompResultHeaderProps).WithStyle(cloudCompResultCellStyle), + text.NewCol(1, "Check Type", cloudCompResultHeaderProps).WithStyle(cloudCompResultCellStyle), + text.NewCol(4, "Title", cloudCompResultHeaderProps).WithStyle(cloudCompResultCellStyle), + text.NewCol(1, "Status", cloudCompResultHeaderProps).WithStyle(cloudCompResultCellStyle), + )) +} + +func getCloudCompResultRows(d ScanData[model.CloudCompliance]) []core.Row { + resultRows := []core.Row{} + for k, v := range d.ScanResults { + resultRows = append( + resultRows, + row.New(18).Add( + text.NewCol(1, strconv.Itoa(k+1), + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(cloudCompResultCellStyle), + text.NewCol(5, v.Resource, + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(cloudCompResultCellStyle), + text.NewCol(1, v.ComplianceCheckType, + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(cloudCompResultCellStyle), + text.NewCol(4, v.Title, + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(cloudCompResultCellStyle), + text.NewCol(1, v.Status, + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Status]}).WithStyle(cloudCompResultCellStyle), + ), + ) + } + return resultRows } diff --git a/deepfence_worker/tasks/reports/pdf_compliance.go b/deepfence_worker/tasks/reports/pdf_compliance.go index 64ff06392b..dc5a4011a9 100644 --- a/deepfence_worker/tasks/reports/pdf_compliance.go +++ b/deepfence_worker/tasks/reports/pdf_compliance.go @@ -3,11 +3,15 @@ package reports import ( "context" "fmt" + "os" + "path/filepath" "strconv" "strings" + "time" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/log" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/johnfercher/maroto/v2/pkg/components/page" "github.com/johnfercher/maroto/v2/pkg/components/row" "github.com/johnfercher/maroto/v2/pkg/components/text" @@ -19,58 +23,76 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func getComplianceSummaryPage(data map[string]map[string]int32) core.Page { - - cellStyle := &props.Cell{ +var ( + compResultCellStyle = &props.Cell{ BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, BorderType: border.Full, BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, BorderThickness: 0.1, } - summaryPage := page.New() - summaryPage.Add(text.NewRow(12, "Scans Summary", - props.Text{Size: 10, Align: align.Center, Style: fontstyle.Bold})) - - summaryTableHeaderProps := props.Text{ + compResultHeaderProps = props.Text{ Size: 10, + Left: 1, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: &props.Color{Red: 0, Green: 0, Blue: 200}, } - summaryPage.Add( - row.New(6).Add( - text.NewCol(6, "Node Name", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Pass", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Fail", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Info", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Warn", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Note", summaryTableHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Total", summaryTableHeaderProps).WithStyle(cellStyle), - ), - ) + compSummaryCellStyle = &props.Cell{ + BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, + BorderType: border.Full, + BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, + BorderThickness: 0.1, + } - summaryProps := props.Text{ + compSummaryTableHeaderProps = props.Text{ + Size: 10, + Top: 1, + Align: align.Center, + Style: fontstyle.Bold, + Color: &props.Color{Red: 0, Green: 0, Blue: 200}, + } + + compSummaryProps = props.Text{ Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Normal, } +) + +func getComplianceSummaryPage(data map[string]map[string]int32) core.Page { + + summaryPage := page.New() + summaryPage.Add(text.NewRow(12, "Scans Summary", + props.Text{Size: 10, Align: align.Center, Style: fontstyle.Bold})) + + summaryPage.Add( + row.New(6).Add( + text.NewCol(6, "Node Name", compSummaryTableHeaderProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, "Pass", compSummaryTableHeaderProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, "Fail", compSummaryTableHeaderProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, "Info", compSummaryTableHeaderProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, "Warn", compSummaryTableHeaderProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, "Note", compSummaryTableHeaderProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, "Total", compSummaryTableHeaderProps).WithStyle(compSummaryCellStyle), + ), + ) summaryRows := []core.Row{} for k, v := range data { summaryRows = append( summaryRows, row.New(6).Add( - text.NewCol(6, k, summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["pass"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["fail"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["info"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["warn"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["note"])), summaryProps).WithStyle(cellStyle), - text.NewCol(1, strconv.Itoa(int(v["pass"]+v["fail"]+v["info"]+v["warn"]+v["note"])), summaryProps).WithStyle(cellStyle), + text.NewCol(6, k, compSummaryProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["pass"])), compSummaryProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["fail"])), compSummaryProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["info"])), compSummaryProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["warn"])), compSummaryProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["note"])), compSummaryProps).WithStyle(compSummaryCellStyle), + text.NewCol(1, strconv.Itoa(int(v["pass"]+v["fail"]+v["info"]+v["warn"]+v["note"])), compSummaryProps).WithStyle(compSummaryCellStyle), ), ) } @@ -80,19 +102,26 @@ func getComplianceSummaryPage(data map[string]map[string]int32) core.Page { return summaryPage } -func compliancePDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +func compliancePDF(ctx context.Context, params sdkUtils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getComplianceData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get compliance info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", params.ReportID, data.NodeWiseData.RecordCount) + if !params.ZippedReport { + return createCompSingleReport(data, params) + } + return createCompZippedReport(data, params) +} + +func createCompSingleReport(data *Info[model.Compliance], params sdkUtils.ReportParams) (string, error) { // get new instance of marato m := getMarato() @@ -114,22 +143,6 @@ func compliancePDF(ctx context.Context, params utils.ReportParams) (core.Documen summaryPage = getCloudComplianceSummaryPage(data.NodeWiseData.SeverityCount) } - cellStyle := &props.Cell{ - BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, - BorderType: border.Full, - BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, - BorderThickness: 0.1, - } - - resultHeaderProps := props.Text{ - Size: 10, - Left: 1, - Top: 1, - Align: align.Center, - Style: fontstyle.Bold, - Color: &props.Color{Red: 0, Green: 0, Blue: 200}, - } - // page per scan resultPages := []core.Page{} for i, d := range data.NodeWiseData.ScanData { @@ -139,83 +152,10 @@ func compliancePDF(ctx context.Context, params utils.ReportParams) (core.Documen continue } + // add result pages p := page.New() - p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", i), resultHeaderProps)) - if data.AppliedFilters.NodeType != "cluster" { - p.Add(row.New(10).Add( - text.NewCol(1, "No.", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Status", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Category", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Test Number", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(5, "Description", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Check Type", resultHeaderProps).WithStyle(cellStyle), - )) - } else { - p.Add(row.New(10).Add( - text.NewCol(1, "No.", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Status", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Check Type", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Category", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Test Number", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(3, "Description", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Resource", resultHeaderProps).WithStyle(cellStyle), - )) - } - - resultRows := []core.Row{} - for k, v := range d.ScanResults { - if data.AppliedFilters.NodeType != "cluster" { - resultRows = append( - resultRows, - row.New(15).Add( - text.NewCol(1, strconv.Itoa(k+1), - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(1, v.Status, - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Status]}).WithStyle(cellStyle), - text.NewCol(2, v.TestCategory, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(2, v.TestNumber, - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(5, truncateText(v.TestInfo, 100), - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). - WithStyle(cellStyle), - text.NewCol(1, v.ComplianceCheckType, - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - ), - ) - } else { - resultRows = append( - resultRows, - row.New(15).Add( - text.NewCol(1, strconv.Itoa(k+1), - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(1, v.Status, - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Status]}).WithStyle(cellStyle), - text.NewCol(1, v.ComplianceCheckType, - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(2, v.TestCategory, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(2, v.TestNumber, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(3, truncateText(v.TestInfo, 100), - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). - WithStyle(cellStyle), - text.NewCol(2, v.Resource, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - ), - ) - } - } - p.Add(resultRows...) + addCompResultHeaders(p, i, data.AppliedFilters.NodeType) + p.Add(getCompResultRows(d, data.AppliedFilters.NodeType)...) resultPages = append(resultPages, p) } @@ -224,5 +164,168 @@ func compliancePDF(ctx context.Context, params utils.ReportParams) (core.Documen m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile(os.TempDir(), tempReportFile(params), doc.GetBytes()) +} + +func createCompZippedReport(data *Info[model.Compliance], params sdkUtils.ReportParams) (string, error) { + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + // get new instance of marato + m := getMarato() + + // applied filter page + filtersPage := getFiltersPage( + data.Title, + data.ScanType, + data.AppliedFilters.NodeType, + fmt.Sprintf("%s - %s", data.StartTime, data.EndTime), + strings.Join(data.AppliedFilters.SeverityOrCheckType, ","), + data.AppliedFilters.AdvancedReportFilters.String(), + ) + + // summary tableparams sdkUtils.ReportParams + singleSummary := map[string]map[string]int32{ + i: data.NodeWiseData.SeverityCount[i], + } + var summaryPage core.Page + if data.AppliedFilters.NodeType != "cluster" { + summaryPage = getComplianceSummaryPage(singleSummary) + } else { + summaryPage = getCloudComplianceSummaryPage(singleSummary) + } + + // skip if there are no results + if len(d.ScanResults) == 0 { + continue + } + + // add result pages + resultPage := page.New() + addCompResultHeaders(resultPage, i, data.AppliedFilters.NodeType) + resultPage.Add(getCompResultRows(d, data.AppliedFilters.NodeType)...) + + // add all pages + m.AddPages(filtersPage) + m.AddPages(summaryPage) + m.AddPages(resultPage) + + doc, err := m.Generate() + if err != nil { + return "", err + } + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + log.Info().Msgf("report id %s %s pdf generation metrics %s", + params.ReportID, outputFile, doc.GetReport()) + + if _, err := writeReportToFile(tmpDir, outputFile, doc.GetBytes()); err != nil { + log.Error().Err(err).Msg("failed to write report to file") + } + + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func addCompResultHeaders(p core.Page, nodeName string, nodeType string) { + p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", nodeName), compResultHeaderProps)) + if nodeType != "cluster" { + p.Add(row.New(10).Add( + text.NewCol(1, "No.", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(1, "Status", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(2, "Category", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(2, "Test Number", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(5, "Description", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(1, "Check Type", compResultHeaderProps).WithStyle(compResultCellStyle), + )) + } else { + p.Add(row.New(10).Add( + text.NewCol(1, "No.", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(1, "Status", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(1, "Check Type", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(2, "Category", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(2, "Test Number", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(3, "Description", compResultHeaderProps).WithStyle(compResultCellStyle), + text.NewCol(2, "Resource", compResultHeaderProps).WithStyle(compResultCellStyle), + )) + } +} + +func getCompResultRows(d ScanData[model.Compliance], nodeType string) []core.Row { + resultRows := []core.Row{} + for k, v := range d.ScanResults { + if nodeType != "cluster" { + resultRows = append( + resultRows, + row.New(15).Add( + text.NewCol(1, strconv.Itoa(k+1), + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(compResultCellStyle), + text.NewCol(1, v.Status, + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Status]}).WithStyle(compResultCellStyle), + text.NewCol(2, v.TestCategory, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(compResultCellStyle), + text.NewCol(2, v.TestNumber, + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(compResultCellStyle), + text.NewCol(5, truncateText(v.TestInfo, 100), + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). + WithStyle(compResultCellStyle), + text.NewCol(1, v.ComplianceCheckType, + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(compResultCellStyle), + ), + ) + } else { + resultRows = append( + resultRows, + row.New(15).Add( + text.NewCol(1, strconv.Itoa(k+1), + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(compResultCellStyle), + text.NewCol(1, v.Status, + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Status]}).WithStyle(compResultCellStyle), + text.NewCol(1, v.ComplianceCheckType, + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(compResultCellStyle), + text.NewCol(2, v.TestCategory, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(compResultCellStyle), + text.NewCol(2, v.TestNumber, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(compResultCellStyle), + text.NewCol(3, truncateText(v.TestInfo, 100), + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). + WithStyle(compResultCellStyle), + text.NewCol(2, v.Resource, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(compResultCellStyle), + ), + ) + } + } + return resultRows } diff --git a/deepfence_worker/tasks/reports/pdf_malware.go b/deepfence_worker/tasks/reports/pdf_malware.go index a59704d536..cc4d2d1e93 100644 --- a/deepfence_worker/tasks/reports/pdf_malware.go +++ b/deepfence_worker/tasks/reports/pdf_malware.go @@ -3,11 +3,15 @@ package reports import ( "context" "fmt" + "os" + "path/filepath" "strconv" "strings" + "time" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/log" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/johnfercher/maroto/v2/pkg/components/page" "github.com/johnfercher/maroto/v2/pkg/components/row" "github.com/johnfercher/maroto/v2/pkg/components/text" @@ -19,19 +23,44 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func malwarePDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +var ( + malwareCellStyle = &props.Cell{ + BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, + BorderType: border.Full, + BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, + BorderThickness: 0.1, + } + + malwareResultHeaderProps = props.Text{ + Size: 10, + Left: 1, + Top: 1, + Align: align.Center, + Style: fontstyle.Bold, + Color: &props.Color{Red: 0, Green: 0, Blue: 200}, + } +) + +func malwarePDF(ctx context.Context, params sdkUtils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getMalwareData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get malware info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", params.ReportID, data.NodeWiseData.RecordCount) + if !params.ZippedReport { + return createMalwareSingleFile(data, params) + } + return createMalwareZippedFile(data, params) +} + +func createMalwareSingleFile(data *Info[model.Malware], params sdkUtils.ReportParams) (string, error) { // get new instance of marato m := getMarato() @@ -48,22 +77,6 @@ func malwarePDF(ctx context.Context, params utils.ReportParams) (core.Document, // summary table summaryPage := getSummaryPage(&data.NodeWiseData.SeverityCount) - cellStyle := &props.Cell{ - BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, - BorderType: border.Full, - BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, - BorderThickness: 0.1, - } - - resultHeaderProps := props.Text{ - Size: 10, - Left: 1, - Top: 1, - Align: align.Center, - Style: fontstyle.Bold, - Color: &props.Color{Red: 0, Green: 0, Blue: 200}, - } - // page per scan resultPages := []core.Page{} for i, d := range data.NodeWiseData.ScanData { @@ -74,42 +87,8 @@ func malwarePDF(ctx context.Context, params utils.ReportParams) (core.Document, } p := page.New() - p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", i), resultHeaderProps)) - p.Add(row.New(6).Add( - text.NewCol(1, "No.", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Rule Name", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(3, "File Name", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Severity", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(3, "Summary", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Class", resultHeaderProps).WithStyle(cellStyle), - )) - - resultRows := []core.Row{} - for k, v := range d.ScanResults { - resultRows = append( - resultRows, - row.New(15).Add( - text.NewCol(1, strconv.Itoa(k+1), - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(2, v.RuleName, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(cellStyle), - text.NewCol(3, v.CompleteFilename, - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(1, v.FileSeverity, - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.FileSeverity]}). - WithStyle(cellStyle), - text.NewCol(3, truncateText(v.Summary, 80), - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(2, v.Class, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - ), - ) - } - p.Add(resultRows...) + addMalwareResultHeaders(p, i) + p.Add(getMalwareResultRows(d)...) resultPages = append(resultPages, p) } @@ -118,5 +97,121 @@ func malwarePDF(ctx context.Context, params utils.ReportParams) (core.Document, m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile(os.TempDir(), tempReportFile(params), doc.GetBytes()) +} + +func createMalwareZippedFile(data *Info[model.Malware], params sdkUtils.ReportParams) (string, error) { + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + // get new instance of marato + m := getMarato() + + // applied filter page + filtersPage := getFiltersPage( + data.Title, + data.ScanType, + data.AppliedFilters.NodeType, + fmt.Sprintf("%s - %s", data.StartTime, data.EndTime), + strings.Join(data.AppliedFilters.SeverityOrCheckType, ","), + data.AppliedFilters.AdvancedReportFilters.String(), + ) + + // summary tableparams sdkUtils.ReportParams + singleSummary := map[string]map[string]int32{ + i: data.NodeWiseData.SeverityCount[i], + } + summaryPage := getSummaryPage(&singleSummary) + + // skip if there are no results + if len(d.ScanResults) == 0 { + continue + } + + resultPage := page.New() + addMalwareResultHeaders(resultPage, i) + resultPage.Add(getMalwareResultRows(d)...) + + // add all pages + m.AddPages(filtersPage) + m.AddPages(summaryPage) + m.AddPages(resultPage) + + doc, err := m.Generate() + if err != nil { + return "", err + } + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + log.Info().Msgf("report id %s %s pdf generation metrics %s", + params.ReportID, outputFile, doc.GetReport()) + + if _, err := writeReportToFile(tmpDir, outputFile, doc.GetBytes()); err != nil { + log.Error().Err(err).Msg("failed to write report to file") + } + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func addMalwareResultHeaders(p core.Page, nodeName string) { + p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", nodeName), malwareResultHeaderProps)) + p.Add(row.New(6).Add( + text.NewCol(1, "No.", malwareResultHeaderProps).WithStyle(malwareCellStyle), + text.NewCol(2, "Rule Name", malwareResultHeaderProps).WithStyle(malwareCellStyle), + text.NewCol(3, "File Name", malwareResultHeaderProps).WithStyle(malwareCellStyle), + text.NewCol(1, "Severity", malwareResultHeaderProps).WithStyle(malwareCellStyle), + text.NewCol(3, "Summary", malwareResultHeaderProps).WithStyle(malwareCellStyle), + text.NewCol(2, "Class", malwareResultHeaderProps).WithStyle(malwareCellStyle), + )) +} + +func getMalwareResultRows(d ScanData[model.Malware]) []core.Row { + resultRows := []core.Row{} + for k, v := range d.ScanResults { + resultRows = append( + resultRows, + row.New(15).Add( + text.NewCol(1, strconv.Itoa(k+1), + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(malwareCellStyle), + text.NewCol(2, v.RuleName, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(malwareCellStyle), + text.NewCol(3, v.CompleteFilename, + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(malwareCellStyle), + text.NewCol(1, v.FileSeverity, + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.FileSeverity]}). + WithStyle(malwareCellStyle), + text.NewCol(3, truncateText(v.Summary, 80), + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(malwareCellStyle), + text.NewCol(2, v.Class, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(malwareCellStyle), + ), + ) + } + return resultRows } diff --git a/deepfence_worker/tasks/reports/pdf_secret.go b/deepfence_worker/tasks/reports/pdf_secret.go index ae0f4b3f40..9953dca618 100644 --- a/deepfence_worker/tasks/reports/pdf_secret.go +++ b/deepfence_worker/tasks/reports/pdf_secret.go @@ -3,11 +3,15 @@ package reports import ( "context" "fmt" + "os" + "path/filepath" "strconv" "strings" + "time" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/log" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/johnfercher/maroto/v2/pkg/components/page" "github.com/johnfercher/maroto/v2/pkg/components/row" "github.com/johnfercher/maroto/v2/pkg/components/text" @@ -19,19 +23,44 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func secretPDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +var ( + secretCellStyle = &props.Cell{ + BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, + BorderType: border.Full, + BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, + BorderThickness: 0.1, + } + + secretResultHeaderProps = props.Text{ + Size: 10, + Left: 1, + Top: 1, + Align: align.Center, + Style: fontstyle.Bold, + Color: &props.Color{Red: 0, Green: 0, Blue: 200}, + } +) + +func secretPDF(ctx context.Context, params sdkUtils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getSecretData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get secrets info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", params.ReportID, data.NodeWiseData.RecordCount) + if !params.ZippedReport { + return createSecretSingleFile(data, params) + } + return createSecretZippedFile(data, params) +} + +func createSecretSingleFile(data *Info[model.Secret], params sdkUtils.ReportParams) (string, error) { // get new instance of marato m := getMarato() @@ -48,22 +77,6 @@ func secretPDF(ctx context.Context, params utils.ReportParams) (core.Document, e // summary table summaryPage := getSummaryPage(&data.NodeWiseData.SeverityCount) - cellStyle := &props.Cell{ - BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, - BorderType: border.Full, - BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, - BorderThickness: 0.1, - } - - resultHeaderProps := props.Text{ - Size: 10, - Left: 1, - Top: 1, - Align: align.Center, - Style: fontstyle.Bold, - Color: &props.Color{Red: 0, Green: 0, Blue: 200}, - } - // page per scan resultPages := []core.Page{} for i, d := range data.NodeWiseData.ScanData { @@ -74,38 +87,8 @@ func secretPDF(ctx context.Context, params utils.ReportParams) (core.Document, e } p := page.New() - p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", i), resultHeaderProps)) - p.Add(row.New(6).Add( - text.NewCol(1, "No.", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "Rule Name", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(3, "File Name", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Severity", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(4, "Matched Content", resultHeaderProps).WithStyle(cellStyle), - )) - - resultRows := []core.Row{} - for k, v := range d.ScanResults { - resultRows = append( - resultRows, - row.New(15).Add( - text.NewCol(1, strconv.Itoa(k+1), - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(2, v.RuleID, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(cellStyle), - text.NewCol(3, v.FullFilename, - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(1, v.Level, - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Level]}). - WithStyle(cellStyle), - text.NewCol(4, truncateText(v.MatchedContent, 80), - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - ), - ) - } - p.Add(resultRows...) + addSecretResultHeaders(p, i) + p.Add(getSecretResultRows(d)...) resultPages = append(resultPages, p) } @@ -114,5 +97,118 @@ func secretPDF(ctx context.Context, params utils.ReportParams) (core.Document, e m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile(os.TempDir(), tempReportFile(params), doc.GetBytes()) +} + +func createSecretZippedFile(data *Info[model.Secret], params sdkUtils.ReportParams) (string, error) { + + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + // get new instance of marato + m := getMarato() + + // applied filter page + filtersPage := getFiltersPage( + data.Title, + data.ScanType, + data.AppliedFilters.NodeType, + fmt.Sprintf("%s - %s", data.StartTime, data.EndTime), + strings.Join(data.AppliedFilters.SeverityOrCheckType, ","), + data.AppliedFilters.AdvancedReportFilters.String(), + ) + + // summary tableparams sdkUtils.ReportParams + singleSummary := map[string]map[string]int32{ + i: data.NodeWiseData.SeverityCount[i], + } + summaryPage := getSummaryPage(&singleSummary) + + // skip if there are no results + if len(d.ScanResults) == 0 { + continue + } + + resultPage := page.New() + addSecretResultHeaders(resultPage, i) + resultPage.Add(getSecretResultRows(d)...) + + // add all pages + m.AddPages(filtersPage) + m.AddPages(summaryPage) + m.AddPages(resultPage) + + doc, err := m.Generate() + if err != nil { + return "", err + } + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + log.Info().Msgf("report id %s %s pdf generation metrics %s", + params.ReportID, outputFile, doc.GetReport()) + + if _, err := writeReportToFile(tmpDir, outputFile, doc.GetBytes()); err != nil { + log.Error().Err(err).Msg("failed to write report to file") + } + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func addSecretResultHeaders(p core.Page, nodeName string) { + p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", nodeName), secretResultHeaderProps)) + p.Add(row.New(6).Add( + text.NewCol(1, "No.", secretResultHeaderProps).WithStyle(secretCellStyle), + text.NewCol(2, "Rule Name", secretResultHeaderProps).WithStyle(secretCellStyle), + text.NewCol(3, "File Name", secretResultHeaderProps).WithStyle(secretCellStyle), + text.NewCol(1, "Severity", secretResultHeaderProps).WithStyle(secretCellStyle), + text.NewCol(4, "Matched Content", secretResultHeaderProps).WithStyle(secretCellStyle), + )) +} + +func getSecretResultRows(d ScanData[model.Secret]) []core.Row { + resultRows := []core.Row{} + for k, v := range d.ScanResults { + resultRows = append( + resultRows, + row.New(15).Add( + text.NewCol(1, strconv.Itoa(k+1), + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(secretCellStyle), + text.NewCol(2, v.RuleID, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(secretCellStyle), + text.NewCol(3, v.FullFilename, + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(secretCellStyle), + text.NewCol(1, v.Level, + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.Level]}). + WithStyle(secretCellStyle), + text.NewCol(4, truncateText(v.MatchedContent, 80), + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(secretCellStyle), + ), + ) + } + return resultRows } diff --git a/deepfence_worker/tasks/reports/pdf_vulnerability.go b/deepfence_worker/tasks/reports/pdf_vulnerability.go index fe4b2b58ac..e082d66853 100644 --- a/deepfence_worker/tasks/reports/pdf_vulnerability.go +++ b/deepfence_worker/tasks/reports/pdf_vulnerability.go @@ -3,11 +3,15 @@ package reports import ( "context" "fmt" + "os" + "path/filepath" "strconv" "strings" + "time" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/log" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/johnfercher/maroto/v2/pkg/components/page" "github.com/johnfercher/maroto/v2/pkg/components/row" "github.com/johnfercher/maroto/v2/pkg/components/text" @@ -19,23 +23,48 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +var ( + vulnCellStyle = &props.Cell{ + BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, + BorderType: border.Full, + BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, + BorderThickness: 0.1, + } + + vulnResultHeaderProps = props.Text{ + Size: 10, + Left: 1, + Top: 1, + Align: align.Center, + Style: fontstyle.Bold, + Color: &props.Color{Red: 0, Green: 0, Blue: 200}, + } +) + +func vulnerabilityPDF(ctx context.Context, params sdkUtils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getVulnerabilityData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get vulnerabilities info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", params.ReportID, data.NodeWiseData.RecordCount) + if !params.ZippedReport { + return createVulnSingleReport(data, params) + } + return createVulnZippedReport(data, params) +} + +func createVulnSingleReport(data *Info[model.Vulnerability], params sdkUtils.ReportParams) (string, error) { // get new instance of marato m := getMarato() - // applied filter page + // applied filter pageparams sdkUtils.ReportParams filtersPage := getFiltersPage( data.Title, data.ScanType, @@ -45,71 +74,20 @@ func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (core.Docu data.AppliedFilters.AdvancedReportFilters.String(), ) - // summary table + // summary tableparams sdkUtils.ReportParams summaryPage := getSummaryPage(&data.NodeWiseData.SeverityCount) - cellStyle := &props.Cell{ - BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, - BorderType: border.Full, - BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, - BorderThickness: 0.1, - } - - resultHeaderProps := props.Text{ - Size: 10, - Left: 1, - Top: 1, - Align: align.Center, - Style: fontstyle.Bold, - Color: &props.Color{Red: 0, Green: 0, Blue: 200}, - } - // page per scan resultPages := []core.Page{} for i, d := range data.NodeWiseData.ScanData { - // skip if there are no results if len(d.ScanResults) == 0 { continue } - + // add result pages p := page.New() - p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", i), resultHeaderProps)) - p.Add(row.New(6).Add( - text.NewCol(1, "No.", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "CVE ID", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(3, "Package", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Severity", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(4, "Summary", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Link", resultHeaderProps).WithStyle(cellStyle), - )) - - resultRows := []core.Row{} - for k, v := range d.ScanResults { - resultRows = append( - resultRows, - row.New(15).Add( - text.NewCol(1, strconv.Itoa(k+1), - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(2, v.CveID, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(cellStyle), - text.NewCol(3, v.CveCausedByPackage, - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(1, v.GetCategory(), - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.GetCategory()]}). - WithStyle(cellStyle), - text.NewCol(4, truncateText(v.CveDescription, 80), - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). - WithStyle(cellStyle), - text.NewCol(1, "link", - props.Text{Size: 10, Top: 1, Align: align.Center, Hyperlink: &d.ScanResults[k].CveLink}). - WithStyle(cellStyle), - ), - ) - } - p.Add(resultRows...) + addVulnResultHeaders(p, i) + p.Add(getVulnResultRows(d)...) resultPages = append(resultPages, p) } @@ -118,5 +96,122 @@ func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (core.Docu m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile(os.TempDir(), tempReportFile(params), doc.GetBytes()) +} + +func createVulnZippedReport(data *Info[model.Vulnerability], params sdkUtils.ReportParams) (string, error) { + + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + // get new instance of marato + m := getMarato() + + // applied filter pageparams sdkUtils.ReportParams + filtersPage := getFiltersPage( + data.Title, + data.ScanType, + data.AppliedFilters.NodeType, + fmt.Sprintf("%s - %s", data.StartTime, data.EndTime), + strings.Join(data.AppliedFilters.SeverityOrCheckType, ","), + data.AppliedFilters.AdvancedReportFilters.String(), + ) + + // summary tableparams sdkUtils.ReportParams + singleSummary := map[string]map[string]int32{ + i: data.NodeWiseData.SeverityCount[i], + } + summaryPage := getSummaryPage(&singleSummary) + + // skip if there are no results + if len(d.ScanResults) == 0 { + continue + } + + resultPage := page.New() + addVulnResultHeaders(resultPage, i) + resultPage.Add(getVulnResultRows(d)...) + + // add all pages + m.AddPages(filtersPage) + m.AddPages(summaryPage) + m.AddPages(resultPage) + + doc, err := m.Generate() + if err != nil { + return "", err + } + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + log.Info().Msgf("report id %s %s pdf generation metrics %s", + params.ReportID, outputFile, doc.GetReport()) + + if _, err := writeReportToFile(tmpDir, outputFile, doc.GetBytes()); err != nil { + log.Error().Err(err).Msg("failed to write report to file") + } + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func addVulnResultHeaders(p core.Page, nodeName string) { + p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", nodeName), vulnResultHeaderProps)) + p.Add(row.New(6).Add( + text.NewCol(1, "No.", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(2, "CVE ID", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(3, "Package", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(1, "Severity", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(4, "Summary", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(1, "Link", vulnResultHeaderProps).WithStyle(vulnCellStyle), + )) +} + +func getVulnResultRows(d ScanData[model.Vulnerability]) []core.Row { + resultRows := []core.Row{} + for k, v := range d.ScanResults { + resultRows = append( + resultRows, + row.New(15).Add( + text.NewCol(1, strconv.Itoa(k+1), + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(vulnCellStyle), + text.NewCol(2, v.CveID, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(vulnCellStyle), + text.NewCol(3, v.CveCausedByPackage, + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(vulnCellStyle), + text.NewCol(1, v.GetCategory(), + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.GetCategory()]}). + WithStyle(vulnCellStyle), + text.NewCol(4, truncateText(v.CveDescription, 80), + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). + WithStyle(vulnCellStyle), + text.NewCol(1, "link", + props.Text{Size: 10, Top: 1, Align: align.Center, Hyperlink: &d.ScanResults[k].CveLink}). + WithStyle(vulnCellStyle), + ), + ) + } + return resultRows } diff --git a/deepfence_worker/tasks/reports/reports.go b/deepfence_worker/tasks/reports/reports.go index 5cdc88c497..e5fc4596f6 100644 --- a/deepfence_worker/tasks/reports/reports.go +++ b/deepfence_worker/tasks/reports/reports.go @@ -7,12 +7,14 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" "time" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" + "github.com/deepfence/ThreatMapper/deepfence_utils/utils" sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/hibiken/asynq" "github.com/minio/minio-go/v7" @@ -39,10 +41,43 @@ func reportFileName(params sdkUtils.ReportParams) string { if sdkUtils.ReportType(params.ReportType) == sdkUtils.ReportSBOM { return fmt.Sprintf("sbom_%s%s", params.ReportID, fileExt(sdkUtils.ReportSBOM)) } + list := []string{params.Filters.ScanType, params.Filters.NodeType, params.ReportID} + + if params.ZippedReport { + return strings.Join(list, "_") + ".zip" + } + return strings.Join(list, "_") + fileExt(sdkUtils.ReportType(params.ReportType)) } +func writeReportToFile(dir string, fileName string, data []byte) (string, error) { + + // make sure directory exists + os.MkdirAll(dir, os.ModePerm) + + out := filepath.Join(dir, fileName) + + log.Debug().Msgf("write report to path %s", out) + + err := os.WriteFile(out, data, os.ModePerm) + if err != nil { + return "", err + } + + return out, nil +} + +func tempReportFile(params utils.ReportParams) string { + return strings.Join( + []string{ + "report", + fmt.Sprintf("%d", time.Now().UnixMilli()), + reportFileName(params), + }, + "-") +} + func putOpts(reportType sdkUtils.ReportType) minio.PutObjectOptions { switch reportType { case sdkUtils.ReportXLSX: @@ -51,6 +86,8 @@ func putOpts(reportType sdkUtils.ReportType) minio.PutObjectOptions { return minio.PutObjectOptions{ContentType: "application/pdf"} case sdkUtils.ReportSBOM: return minio.PutObjectOptions{ContentType: "application/gzip"} + case sdkUtils.ReportZIP: + return minio.PutObjectOptions{ContentType: "application/zip"} } return minio.PutObjectOptions{} } @@ -107,7 +144,8 @@ func GenerateReport(ctx context.Context, task *asynq.Task) error { localReportPath, err := generateReport(ctx, params) if err != nil { log.Error().Err(err).Msgf("failed to generate report with params %+v", params) - updateReportState(ctx, session, params.ReportID, "", "", sdkUtils.ScanStatusFailed, err.Error()) + updateReportState(ctx, session, params.ReportID, + "", "", sdkUtils.ScanStatusFailed, err.Error()) return nil } log.Info().Msgf("report file path %s", localReportPath) @@ -123,14 +161,21 @@ func GenerateReport(ctx context.Context, task *asynq.Task) error { } reportName := path.Join("/report", reportFileName(params)) - res, err := mc.UploadLocalFile(ctx, reportName, - localReportPath, true, putOpts(sdkUtils.ReportType(params.ReportType))) + + putOptions := putOpts(sdkUtils.ReportType(params.ReportType)) + // specical case zip in not report type + if params.ZippedReport { + putOptions = putOpts(sdkUtils.ReportZIP) + } + + res, err := mc.UploadLocalFile(ctx, reportName, localReportPath, true, putOptions) if err != nil { log.Error().Err(err).Msg("failed to upload file to minio") return nil } - updateReportState(ctx, session, params.ReportID, reportName, res.Key, sdkUtils.ScanStatusSuccess, "") + updateReportState(ctx, session, params.ReportID, + reportName, res.Key, sdkUtils.ScanStatusSuccess, "") return nil }