From 932a53d908569cc1ccbf2001af159a183715803f Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Thu, 7 Nov 2024 11:19:16 +0000 Subject: [PATCH] add support for zipped xlsx report --- deepfence_utils/utils/utils.go | 5 + deepfence_worker/tasks/reports/pdf.go | 19 + deepfence_worker/tasks/reports/reports.go | 21 +- deepfence_worker/tasks/reports/xlsx.go | 513 ++++++++++++++-------- 4 files changed, 359 insertions(+), 199 deletions(-) diff --git a/deepfence_utils/utils/utils.go b/deepfence_utils/utils/utils.go index 030dd56c53..15bf0ba1e0 100644 --- a/deepfence_utils/utils/utils.go +++ b/deepfence_utils/utils/utils.go @@ -786,6 +786,9 @@ func ComputeChecksumForFile(filePath string) (string, error) { } func ZipDir(sourceDir string, baseZipPath string, outputZip string) error { + + log.Debug().Msgf("add files from directory %s to zip", sourceDir) + archive, err := os.Create(outputZip) if err != nil { return err @@ -809,6 +812,8 @@ func ZipDir(sourceDir string, baseZipPath string, outputZip string) error { } defer file.Close() + log.Debug().Msgf("adding file to zip %s", info.Name()) + f, err := zw.Create(filepath.Join(baseZipPath, info.Name())) if err != nil { return err diff --git a/deepfence_worker/tasks/reports/pdf.go b/deepfence_worker/tasks/reports/pdf.go index 6d22dde0d9..dc6cadfbcf 100644 --- a/deepfence_worker/tasks/reports/pdf.go +++ b/deepfence_worker/tasks/reports/pdf.go @@ -3,6 +3,8 @@ package reports import ( "context" _ "embed" + "os" + "path/filepath" "strconv" "time" @@ -214,3 +216,20 @@ func generatePDF(ctx context.Context, params utils.ReportParams) (string, error) return document, nil } + +func writeReportToFile(dir string, fileName string, data []byte) (string, error) { + + // make sure directory exists + os.MkdirAll(dir, os.ModePerm) + + out := filepath.Join(dir, fileName) + + log.Debug().Msgf("write report to path %s", out) + + err := os.WriteFile(out, data, os.ModePerm) + if err != nil { + return "", err + } + + return out, nil +} diff --git a/deepfence_worker/tasks/reports/reports.go b/deepfence_worker/tasks/reports/reports.go index e5fc4596f6..f7f0d08e92 100644 --- a/deepfence_worker/tasks/reports/reports.go +++ b/deepfence_worker/tasks/reports/reports.go @@ -7,14 +7,12 @@ import ( "fmt" "os" "path" - "path/filepath" "strings" "time" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/hibiken/asynq" "github.com/minio/minio-go/v7" @@ -51,24 +49,7 @@ func reportFileName(params sdkUtils.ReportParams) string { return strings.Join(list, "_") + fileExt(sdkUtils.ReportType(params.ReportType)) } -func writeReportToFile(dir string, fileName string, data []byte) (string, error) { - - // make sure directory exists - os.MkdirAll(dir, os.ModePerm) - - out := filepath.Join(dir, fileName) - - log.Debug().Msgf("write report to path %s", out) - - err := os.WriteFile(out, data, os.ModePerm) - if err != nil { - return "", err - } - - return out, nil -} - -func tempReportFile(params utils.ReportParams) string { +func tempReportFile(params sdkUtils.ReportParams) string { return strings.Join( []string{ "report", diff --git a/deepfence_worker/tasks/reports/xlsx.go b/deepfence_worker/tasks/reports/xlsx.go index fa39ad0394..bbb87a0268 100644 --- a/deepfence_worker/tasks/reports/xlsx.go +++ b/deepfence_worker/tasks/reports/xlsx.go @@ -2,11 +2,15 @@ package reports import ( "context" + "fmt" "os" + "path/filepath" + "time" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/xuri/excelize/v2" ) @@ -72,7 +76,7 @@ var ( } ) -func generateXLSX(ctx context.Context, params utils.ReportParams) (string, error) { +func generateXLSX(ctx context.Context, params sdkUtils.ReportParams) (string, error) { ctx, span := telemetry.NewSpan(ctx, "reports", "generate-xlsx-report") defer span.End() @@ -104,20 +108,29 @@ func generateXLSX(ctx context.Context, params utils.ReportParams) (string, error return xlsxFile, nil } -func xlsxSave(xlsx *excelize.File, params utils.ReportParams) (string, error) { - // create a temp file to hold xlsx report - temp, err := os.CreateTemp("", "report-*-"+reportFileName(params)) - if err != nil { - return "", err - } - defer temp.Close() +func xlsxSave(xlsx *excelize.File, dir, fileName string) (string, error) { + + // cleanup xlsx + defer func() { + if err := xlsx.Close(); err != nil { + log.Error().Err(err).Msg("failed to close file") + } + }() - // save spreadsheet by the given path. - if err := xlsx.SaveAs(temp.Name()); err != nil { + // make sure directory exists + os.MkdirAll(dir, os.ModePerm) + + out := filepath.Join(dir, fileName) + + log.Debug().Msgf("write xlsx report to path %s", out) + + // save spreadsheet to the given path. + if err := xlsx.SaveAs(out); err != nil { log.Error().Err(err).Msg("failed to save xlsx file") return "", err } - return temp.Name(), nil + + return out, nil } func xlsxSetHeader(xlsx *excelize.File, sheet string, headers map[string]string) { @@ -129,229 +142,371 @@ func xlsxSetHeader(xlsx *excelize.File, sheet string, headers map[string]string) } } -func vulnerabilityXLSX(ctx context.Context, params utils.ReportParams) (string, error) { +func vulnerabilityXLSX(ctx context.Context, params sdkUtils.ReportParams) (string, error) { data, err := getVulnerabilityData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get vulnerabilities info") return "", err } - xlsx := excelize.NewFile() - defer func() { - if err := xlsx.Close(); err != nil { - log.Error().Err(err).Msg("failed to close file") + // single file + if !params.ZippedReport { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", vulnerabilityHeader) + // add results + offset := 0 + for _, nodeScanData := range data.NodeWiseData.ScanData { + xlsxAddVulnResults(xlsx, "Sheet1", offset, nodeScanData) + offset = offset + len(nodeScanData.ScanResults) } - }() - xlsxSetHeader(xlsx, "Sheet1", vulnerabilityHeader) - - offset := 0 - for _, nodeScanData := range data.NodeWiseData.ScanData { - for i, v := range nodeScanData.ScanResults { - cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) - if err != nil { - log.Error().Err(err).Msg("error generating cell name") - } - value := []interface{}{ - v.CveID, - v.CveSeverity, - v.CveAttackVector, - v.CveCausedByPackage, - v.CveCausedByPackagePath, - v.CveCVSSScore, - v.CveDescription, - v.CveFixedIn, - v.CveLink, - v.CveOverallScore, - v.CveType, - nodeScanData.ScanInfo.NodeName, - nodeScanData.ScanInfo.NodeType, - nodeScanData.ScanInfo.KubernetesClusterName, - v.Masked, - } - err = xlsx.SetSheetRow("Sheet1", cellName, &value) - if err != nil { - log.Error().Msg(err.Error()) - } - } - offset = offset + len(nodeScanData.ScanResults) + return xlsxSave(xlsx, os.TempDir(), tempReportFile(params)) } - return xlsxSave(xlsx, params) + // zipped report + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", vulnerabilityHeader) + xlsxAddVulnResults(xlsx, "Sheet1", 0, d) + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + xlsxSave(xlsx, tmpDir, outputFile) + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil + +} + +func xlsxAddVulnResults(xlsx *excelize.File, sheet string, offset int, data ScanData[model.Vulnerability]) { + for i, v := range data.ScanResults { + cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) + if err != nil { + log.Error().Err(err).Msg("error generating cell name") + } + value := []interface{}{ + v.CveID, + v.CveSeverity, + v.CveAttackVector, + v.CveCausedByPackage, + v.CveCausedByPackagePath, + v.CveCVSSScore, + v.CveDescription, + v.CveFixedIn, + v.CveLink, + v.CveOverallScore, + v.CveType, + data.ScanInfo.NodeName, + data.ScanInfo.NodeType, + data.ScanInfo.KubernetesClusterName, + v.Masked, + } + err = xlsx.SetSheetRow(sheet, cellName, &value) + if err != nil { + log.Error().Msg(err.Error()) + } + } } -func secretXLSX(ctx context.Context, params utils.ReportParams) (string, error) { +func secretXLSX(ctx context.Context, params sdkUtils.ReportParams) (string, error) { data, err := getSecretData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get secrets info") return "", err } - xlsx := excelize.NewFile() - defer func() { - if err := xlsx.Close(); err != nil { - log.Error().Err(err).Msg("failed to close file") + // single file + if !params.ZippedReport { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", secretHeader) + // add results + offset := 0 + for _, nodeScanData := range data.NodeWiseData.ScanData { + xlsxAddSecretResults(xlsx, "Sheet1", offset, nodeScanData) + offset = offset + len(nodeScanData.ScanResults) } - }() - xlsxSetHeader(xlsx, "Sheet1", secretHeader) - - offset := 0 - for _, nodeScanData := range data.NodeWiseData.ScanData { - for i, s := range nodeScanData.ScanResults { - cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) - if err != nil { - log.Error().Err(err).Msg("error generating cell name") - } - value := []interface{}{ - s.FullFilename, - s.MatchedContent, - s.RuleID, - s.Level, - s.StartingIndex, - nodeScanData.ScanInfo.NodeName, - nodeScanData.ScanInfo.NodeType, - nodeScanData.ScanInfo.KubernetesClusterName, - s.Masked, - } - err = xlsx.SetSheetRow("Sheet1", cellName, &value) - if err != nil { - log.Error().Msg(err.Error()) - } - } - offset = offset + len(nodeScanData.ScanResults) + return xlsxSave(xlsx, os.TempDir(), tempReportFile(params)) + } + + // zipped report + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", vulnerabilityHeader) + xlsxAddSecretResults(xlsx, "Sheet1", 0, d) + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + xlsxSave(xlsx, tmpDir, outputFile) + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err } - return xlsxSave(xlsx, params) + return outputZip, nil + } -func malwareXLSX(ctx context.Context, params utils.ReportParams) (string, error) { +func xlsxAddSecretResults(xlsx *excelize.File, sheet string, offset int, data ScanData[model.Secret]) { + for i, s := range data.ScanResults { + cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) + if err != nil { + log.Error().Err(err).Msg("error generating cell name") + } + value := []interface{}{ + s.FullFilename, + s.MatchedContent, + s.RuleID, + s.Level, + s.StartingIndex, + data.ScanInfo.NodeName, + data.ScanInfo.NodeType, + data.ScanInfo.KubernetesClusterName, + s.Masked, + } + err = xlsx.SetSheetRow(sheet, cellName, &value) + if err != nil { + log.Error().Msg(err.Error()) + } + } +} + +func malwareXLSX(ctx context.Context, params sdkUtils.ReportParams) (string, error) { data, err := getMalwareData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get malwares info") return "", err } - xlsx := excelize.NewFile() - defer func() { - if err := xlsx.Close(); err != nil { - log.Error().Err(err).Msg("failed to close file") + // single file + if !params.ZippedReport { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", secretHeader) + // add results + offset := 0 + for _, nodeScanData := range data.NodeWiseData.ScanData { + xlsxAddMalwareResults(xlsx, "Sheet1", offset, nodeScanData) + offset = offset + len(nodeScanData.ScanResults) } - }() - xlsxSetHeader(xlsx, "Sheet1", malwareHeader) - - offset := 0 - for _, nodeScanData := range data.NodeWiseData.ScanData { - for i, m := range nodeScanData.ScanResults { - cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) - if err != nil { - log.Error().Err(err).Msg("error generating cell name") - } - value := []interface{}{ - m.RuleName, - m.CompleteFilename, - m.Summary, - m.FileSeverity, - nodeScanData.ScanInfo.NodeName, - nodeScanData.ScanInfo.NodeType, - nodeScanData.ScanInfo.KubernetesClusterName, - m.Masked, - } - err = xlsx.SetSheetRow("Sheet1", cellName, &value) - if err != nil { - log.Error().Msg(err.Error()) - } - } - offset = offset + len(nodeScanData.ScanResults) + return xlsxSave(xlsx, os.TempDir(), tempReportFile(params)) } - return xlsxSave(xlsx, params) + // zipped report + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", vulnerabilityHeader) + xlsxAddMalwareResults(xlsx, "Sheet1", 0, d) + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + xlsxSave(xlsx, tmpDir, outputFile) + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func xlsxAddMalwareResults(xlsx *excelize.File, sheet string, offset int, data ScanData[model.Malware]) { + for i, m := range data.ScanResults { + cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) + if err != nil { + log.Error().Err(err).Msg("error generating cell name") + } + value := []interface{}{ + m.RuleName, + m.CompleteFilename, + m.Summary, + m.FileSeverity, + data.ScanInfo.NodeName, + data.ScanInfo.NodeType, + data.ScanInfo.KubernetesClusterName, + m.Masked, + } + err = xlsx.SetSheetRow(sheet, cellName, &value) + if err != nil { + log.Error().Msg(err.Error()) + } + } } -func complianceXLSX(ctx context.Context, params utils.ReportParams) (string, error) { +func complianceXLSX(ctx context.Context, params sdkUtils.ReportParams) (string, error) { data, err := getComplianceData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get compliance info") return "", err } - xlsx := excelize.NewFile() - defer func() { - if err := xlsx.Close(); err != nil { - log.Error().Err(err).Msg("failed to close file") + // single file + if !params.ZippedReport { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", secretHeader) + // add results + offset := 0 + for _, nodeScanData := range data.NodeWiseData.ScanData { + xlsxAddCompResults(xlsx, "Sheet1", offset, nodeScanData) + offset = offset + len(nodeScanData.ScanResults) } - }() - xlsxSetHeader(xlsx, "Sheet1", complianceHeader) - - offset := 0 - for _, nodeScanData := range data.NodeWiseData.ScanData { - for i, c := range nodeScanData.ScanResults { - cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) - if err != nil { - log.Error().Err(err).Msg("error generating cell name") - } - value := []interface{}{ - c.ComplianceCheckType, - c.Status, - c.TestCategory, - c.TestDesc, - c.TestInfo, - c.TestNumber, - nodeScanData.ScanInfo.NodeName, - nodeScanData.ScanInfo.NodeType, - c.Masked, - } - err = xlsx.SetSheetRow("Sheet1", cellName, &value) - if err != nil { - log.Error().Msg(err.Error()) - } - } - offset = offset + len(nodeScanData.ScanResults) + return xlsxSave(xlsx, os.TempDir(), tempReportFile(params)) + } + + // zipped report + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", vulnerabilityHeader) + xlsxAddCompResults(xlsx, "Sheet1", 0, d) + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + xlsxSave(xlsx, tmpDir, outputFile) + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err } - return xlsxSave(xlsx, params) + return outputZip, nil } -func cloudComplianceXLSX(ctx context.Context, params utils.ReportParams) (string, error) { +func xlsxAddCompResults(xlsx *excelize.File, sheet string, offset int, data ScanData[model.Compliance]) { + for i, c := range data.ScanResults { + cellName, err := excelize.CoordinatesToCellName(1, offset+i+2) + if err != nil { + log.Error().Err(err).Msg("error generating cell name") + } + value := []interface{}{ + c.ComplianceCheckType, + c.Status, + c.TestCategory, + c.TestDesc, + c.TestInfo, + c.TestNumber, + data.ScanInfo.NodeName, + data.ScanInfo.NodeType, + c.Masked, + } + err = xlsx.SetSheetRow(sheet, cellName, &value) + if err != nil { + log.Error().Msg(err.Error()) + } + } +} + +func cloudComplianceXLSX(ctx context.Context, params sdkUtils.ReportParams) (string, error) { data, err := getCloudComplianceData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get cloud compliance info") return "", err } - xlsx := excelize.NewFile() - defer func() { - if err := xlsx.Close(); err != nil { - log.Error().Err(err).Msg("failed to close file") + // single file + if !params.ZippedReport { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", secretHeader) + // add results + for _, nodeScanData := range data.NodeWiseData.ScanData { + xlsxAddCloudCompResults(xlsx, "Sheet1", nodeScanData) } - }() - xlsxSetHeader(xlsx, "Sheet1", cloudComplianceHeader) - - for _, nodeScanData := range data.NodeWiseData.ScanData { - for i, c := range nodeScanData.ScanResults { - cellName, err := excelize.CoordinatesToCellName(1, i+2) - if err != nil { - log.Error().Err(err).Msg("error generating cell name") - } - value := []interface{}{ - c.ComplianceCheckType, - c.Status, - c.Title, - c.Description, - c.ControlID, - nodeScanData.ScanInfo.NodeName, - nodeScanData.ScanInfo.NodeType, - c.Masked, - } - err = xlsx.SetSheetRow("Sheet1", cellName, &value) - if err != nil { - log.Error().Msg(err.Error()) - } - } + return xlsxSave(xlsx, os.TempDir(), tempReportFile(params)) } - return xlsxSave(xlsx, params) + // zipped report + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + xlsx := excelize.NewFile() + xlsxSetHeader(xlsx, "Sheet1", vulnerabilityHeader) + xlsxAddCloudCompResults(xlsx, "Sheet1", d) + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + xlsxSave(xlsx, tmpDir, outputFile) + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func xlsxAddCloudCompResults(xlsx *excelize.File, sheet string, data ScanData[model.CloudCompliance]) { + for i, c := range data.ScanResults { + cellName, err := excelize.CoordinatesToCellName(1, i+2) + if err != nil { + log.Error().Err(err).Msg("error generating cell name") + } + value := []interface{}{ + c.ComplianceCheckType, + c.Status, + c.Title, + c.Description, + c.ControlID, + data.ScanInfo.NodeName, + data.ScanInfo.NodeType, + c.Masked, + } + err = xlsx.SetSheetRow(sheet, cellName, &value) + if err != nil { + log.Error().Msg(err.Error()) + } + } }