diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 362462a3c4..befb1f1d8a 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -352,6 +352,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"), flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"), flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"), + flagSet.DynamicVarP(&options.ErrorLabels, "error-label", "elabel", []string{"UNV"}, "enable logging for specific error labels (comma-separated) (labels: UNV"), ) flagSet.CreateGroup("update", "Update", diff --git a/internal/runner/options.go b/internal/runner/options.go index 828957c3cb..3abfe6feab 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -17,6 +17,7 @@ import ( "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" @@ -26,6 +27,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml" fileutil "github.com/projectdiscovery/utils/file" "github.com/projectdiscovery/utils/generic" @@ -87,6 +89,8 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) + options.ErrorLabels = parseErrorLables(options.ErrorLabels) + err := protocolinit.Init(options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) @@ -430,3 +434,21 @@ func getBoolEnvValue(key string) bool { value := os.Getenv(key) return strings.EqualFold(value, "true") } + +func parseErrorLables(errorLabels []string) []string { + // Make error label entries in stats + for _, v := range elabel.ErrorLableMap { + stats.NewEntry(v.Name, v.Description) + } + if len(errorLabels) == 0 { + return errorLabels + } + var errLabels []string + for _, label := range errorLabels { + label = strings.ToLower(label) + if v, ok := elabel.ErrorLableMap[label]; ok { + errLabels = append(errLabels, v.Name) + } + } + return errLabels +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 4256683eb1..b2e190f9ba 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -17,6 +17,7 @@ import ( uncoverlib "github.com/projectdiscovery/uncover" "github.com/projectdiscovery/utils/env" permissionutil "github.com/projectdiscovery/utils/permission" + stringsutil "github.com/projectdiscovery/utils/strings" updateutils "github.com/projectdiscovery/utils/update" "github.com/logrusorgru/aurora" @@ -40,6 +41,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" @@ -492,6 +494,7 @@ func (r *Runner) RunEnumeration() error { } } r.progress.Stop() + r.displayErrorInfo() if executorOpts.InputHelper != nil { _ = executorOpts.InputHelper.Close() @@ -615,6 +618,17 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { } } +func (r *Runner) displayErrorInfo() { + if !r.options.Verbose { + return + } + errLables := r.options.ErrorLabels + // if error label is not provided, display the stats for specific labels + if !stringsutil.ContainsAny(elabel.UnresolvedVariablesErrorLabel, errLables...) { + stats.Display(elabel.UnresolvedVariablesErrorLabel) + } +} + // SaveResumeConfig to file func (r *Runner) SaveResumeConfig(path string) error { resumeCfgClone := r.resumeCfg.Clone() diff --git a/pkg/protocols/common/errors/label/error_label.go b/pkg/protocols/common/errors/label/error_label.go new file mode 100644 index 0000000000..8e3912dee4 --- /dev/null +++ b/pkg/protocols/common/errors/label/error_label.go @@ -0,0 +1,35 @@ +package label + +import ( + "strings" +) + +const ( + UnresolvedVariablesErrorLabel = "unresolved-variables-error" +) + +var ( + ErrorLabels = []string{ + UnresolvedVariablesErrorLabel, + } + + ErrorLableMap = map[string]struct { + Name string + Description string + }{ + "unv": { + Name: UnresolvedVariablesErrorLabel, + Description: "Failed %v requests due to unresolved variables. Use -elabel=UNV to enable unresolved variables logs.", + }, + } +) + +// Contains checks if the error string contains any of the provided labels, if yes returns matched label and true +func Contains(errStr string, lables []string) (string, bool) { + for _, label := range lables { + if strings.Contains(errStr, label) { + return label, true + } + } + return "", false +} diff --git a/pkg/protocols/common/expressions/variables.go b/pkg/protocols/common/expressions/variables.go index f129eb9121..0289cc4b88 100644 --- a/pkg/protocols/common/expressions/variables.go +++ b/pkg/protocols/common/expressions/variables.go @@ -7,6 +7,8 @@ import ( "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" + errorutil "github.com/projectdiscovery/utils/errors" ) var ( @@ -38,7 +40,7 @@ func ContainsUnresolvedVariables(items ...string) error { unresolvedVariables = append(unresolvedVariables, match[1]) } if len(unresolvedVariables) > 0 { - return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ",")) + return errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: "+strings.Join(unresolvedVariables, ",")) } } diff --git a/pkg/protocols/common/expressions/variables_test.go b/pkg/protocols/common/expressions/variables_test.go index bda936bb6a..c614270a24 100644 --- a/pkg/protocols/common/expressions/variables_test.go +++ b/pkg/protocols/common/expressions/variables_test.go @@ -4,6 +4,8 @@ import ( "errors" "testing" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" + errorutil "github.com/projectdiscovery/utils/errors" "github.com/stretchr/testify/require" ) @@ -12,11 +14,11 @@ func TestUnresolvedVariablesCheck(t *testing.T) { data string err error }{ - {"{{test}}", errors.New("unresolved variables found: test")}, - {"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")}, + {"{{test}}", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test")}, + {"{{test}}/{{another}}", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test,another")}, {"test", nil}, - {"%7b%7btest%7d%7d", errors.New("unresolved variables found: test")}, - {"%7B%7Bfirst%2Asecond%7D%7D", errors.New("unresolved variables found: first%2Asecond")}, + {"%7b%7btest%7d%7d", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test")}, + {"%7B%7Bfirst%2Asecond%7D%7D", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, errors.New("unresolved variables found: first%2Asecond").Error())}, {"{{7*7}}", nil}, {"{{'a'+'b'}}", nil}, {"{{'a'}}", nil}, diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go index 7819ebcb85..abb0dc120b 100644 --- a/pkg/protocols/http/build_request.go +++ b/pkg/protocols/http/build_request.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" @@ -34,7 +35,7 @@ import ( // ErrEvalExpression var ( ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions") - ErrUnresolvedVars = errorutil.NewWithFmt("unresolved variables `%v` found in request") + UnresolvedErrFmt = "unresolved variables `%v` found in request" ) // generatedRequest is a single generated request wrapped for a template request @@ -208,7 +209,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st } if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil { - return nil, ErrUnresolvedVars.Msgf(parts[1]) + return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, parts[1]) } parsed, err := urlutil.ParseURL(parts[1], true) @@ -229,7 +230,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st if err := expressions.ContainsUnresolvedVariables(data); err != nil { // early exit: if there are any unresolved variables in `path` after evaluation // then return early since this will definitely fail - return nil, ErrUnresolvedVars.Msgf(data) + return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, data) } urlx, err := urlutil.ParseURL(data, true) @@ -289,6 +290,10 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st return unsafeReq, nil } + if err := expressions.ContainsUnresolvedVariables(rawRequestData.FullURL); err != nil { + return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, rawRequestData.FullURL) + } + urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw") diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 2328c8f30c..b5fd24d712 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -24,6 +24,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/fuzz" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" @@ -36,7 +37,9 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" "github.com/projectdiscovery/rawhttp" + errorutil "github.com/projectdiscovery/utils/errors" "github.com/projectdiscovery/utils/reader" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" @@ -368,6 +371,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa return true, nil } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + if lb, ok := elabel.Contains(err.Error(), elabel.ErrorLabels); ok { + stats.Increment(lb) + } return true, err } @@ -462,7 +468,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa const drainReqSize = int64(8 * 1024) -var errStopExecution = errors.New("stop execution due to unresolved variables") +var errStopExecution = errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "stop execution due to unresolved variables") // executeRequest executes the actual generated request and returns error if occurred func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error { @@ -538,7 +544,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr) + stats.Increment(elabel.UnresolvedVariablesErrorLabel) + if _, ok := elabel.Contains(varErr.Error(), request.options.Options.ErrorLabels); ok { + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr) + } return errStopExecution } } diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index e66524760e..65acb66b08 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" @@ -28,6 +29,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" errorutil "github.com/projectdiscovery/utils/errors" mapsutil "github.com/projectdiscovery/utils/maps" "github.com/projectdiscovery/utils/reader" @@ -245,7 +247,11 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac reqBuilder.Write(finalData) if err := expressions.ContainsUnresolvedVariables(string(finalData)); err != nil { - gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err) + request.options.Progress.IncrementFailedRequestsBy(1) + stats.Increment(elabel.UnresolvedVariablesErrorLabel) + if _, ok := elabel.Contains(err.Error(), request.options.Options.ErrorLabels); ok { + gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err) + } return nil } diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go index c5d25795cf..66e3c96689 100644 --- a/pkg/tmplexec/generic/exec.go +++ b/pkg/tmplexec/generic/exec.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" + elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label" "github.com/projectdiscovery/nuclei/v3/pkg/scan" ) @@ -79,7 +80,13 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error { if g.options.HostErrorsCache != nil { g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err) + if _, ok := elabel.Contains(err.Error(), g.options.Options.ErrorLabels); ok { + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err) + } + // TODO: remove logging for otherthan unresolved variables + if _, ok := elabel.Contains(err.Error(), []string{elabel.UnresolvedVariablesErrorLabel}); !ok { + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err) + } } // If a match was found and stop at first match is set, break out of the loop and return if g.results.Load() && (g.options.StopAtFirstMatch || g.options.Options.StopAtFirstMatch) { diff --git a/pkg/types/types.go b/pkg/types/types.go index 7dfa845652..3e00026d6d 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -96,6 +96,8 @@ type Options struct { TraceLogFile string // ErrorLogFile specifies a file to write with the errors of all requests ErrorLogFile string + // ErrorLabels is the list of error labels to match and enable logging + ErrorLabels []string // ReportingDB is the db for report storage as well as deduplication ReportingDB string // ReportingConfig is the config file for nuclei reporting module