From fb0d46b99c4ab9309dbff674f3d9008c771642ee Mon Sep 17 00:00:00 2001 From: ouvreboite Date: Thu, 13 Jun 2024 13:34:28 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=B8=20Simplify=20tags=20to=20declare?= =?UTF-8?q?=20rules/test=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/poltergust.yaml | 6 +- README.md | 49 ++--- example/invalid/README.md | 2 - example/valid/README.md | 5 - examples/invalid/README.md | 15 ++ {example => examples}/invalid/invalid.md | 5 +- .../invalid/spectral.base.yaml | 0 {example => examples}/invalid/spectral.yaml | 0 examples/valid/README.md | 28 +++ .../functions/isRequiredPropertyDefined.js | 0 {example => examples}/valid/rules1.md | 25 +-- {example => examples}/valid/rules2.md | 44 +--- .../valid/spectral.base.yaml | 0 {example => examples}/valid/spectral.yaml | 0 src/functions.js | 195 ++++++++---------- src/index.js | 8 - 16 files changed, 170 insertions(+), 212 deletions(-) delete mode 100644 example/invalid/README.md delete mode 100644 example/valid/README.md create mode 100644 examples/invalid/README.md rename {example => examples}/invalid/invalid.md (86%) rename {example => examples}/invalid/spectral.base.yaml (100%) rename {example => examples}/invalid/spectral.yaml (100%) create mode 100644 examples/valid/README.md rename {example => examples}/valid/functions/isRequiredPropertyDefined.js (100%) rename {example => examples}/valid/rules1.md (86%) rename {example => examples}/valid/rules2.md (66%) rename {example => examples}/valid/spectral.base.yaml (100%) rename {example => examples}/valid/spectral.yaml (100%) diff --git a/.github/workflows/poltergust.yaml b/.github/workflows/poltergust.yaml index bfb9d0a..f5ad2a3 100644 --- a/.github/workflows/poltergust.yaml +++ b/.github/workflows/poltergust.yaml @@ -20,8 +20,8 @@ jobs: - name: Install poltergust run: npm ci - - name: Test the example ruleset - run: npx poltergust test ./example/valid + - name: Test the valid example ruleset + run: npx poltergust test ./examples/valid build-and-test-invalid-example: needs: build-and-test-valid-example @@ -38,5 +38,5 @@ jobs: run: npm ci - name: Test the invalid example ruleset (should exit 1) - run: npx poltergust test ./example/invalid || (exitcode=$?; [ $exitcode -eq 1 ] || exit $exitcode) + run: npx poltergust test ./examples/invalid || (exitcode=$?; [ $exitcode -eq 1 ] || exit $exitcode) shell: bash \ No newline at end of file diff --git a/README.md b/README.md index ef84764..1e83d87 100644 --- a/README.md +++ b/README.md @@ -2,48 +2,23 @@ # 👻 poltergust -An npm CLI to extract Spectral rules from .md files, merge them and test them. +An npm CLI to extract, test and merge Spectral rules from .md files. -- Each yaml codeblock starting by `#spectral` is considered a spectral rule and will be aggregated in the spectral.yaml file -- Each yaml codeblock starting by `#✅-test-for: some-rule-name` is considered as an OpenAPI snippet that not should fail the corresponding spectral rule -- Each yaml codeblock starting by `#❌-test-for: some-rule-name` is considered as an OpenAPI snippet that should fail the corresponding spectral rule +- **Rules**: A YAML codeblock starting by `#👻-rule` is considered a spectral rule and will be aggregated in the spectral.yaml file +- **Test cases**: A YAML codeblock containing `#👻-failures:` or `#👻-fails-here:` is considered an OpenAPI test case + - `#👻-failures: X some-rule-name` expects the given rule to return only X failures for this test case. For example `#👻-failures: 0 some-rule-name` can be used to assert that a test case do not trigger a rule. + - `#👻-fails-here: some-rule-name` expects the given rule to be trigger at the very line where the comment is set +- **Base ruleset**: to merge the rules, a `spectral.base.yaml` is needed. It should be a standard Spectral ruleset files, with empty rules. But it can includes extends, aliases, functions... -## Install the poltergust CLI +## Setup and run ```sh npm install -npm link +npx poltergust test ./examples/valid +npx poltergust merge ./examples/valid ``` -## Test and merge the rules +## Examples -```sh -cd .. -poltergust test ./example -poltergust merge ./example -``` - -### Example of a test output - -``` -$ poltergust merge ./example -🔎 Testing the spectral rules from the .md files in the directory: ./rules -👻 base-path-must-start-with-slash (rules\api.md:42) - ✅ Test OK (rules\api.md:17) - ✅ Test OK (rules\api.md:28) -👻 operation-parameters-must-have-description (rules\api.md:94) - ✅ Test OK (rules\api.md:60) - ✅ Test OK (rules\api.md:75) -👻 operation-must-have-description (rules\api.md:110) -👻 operation-must-have-no-summary (rules\api.md:125) -👻 operation-must-have-at-least-one-response (rules\api.md:138) -👻 request-bodies-must-have-a-content (rules\api.md:155) -👻 path-parameters-must-be-kebab-case (rules\parameters.md:75) - ✅ Test OK (rules\parameters.md:9) - ❌ Was expecting to fail rule path-parameters-must-be-kebab-case at line 24 in test (rules\parameters.md:62) - But failed there instead: - { start: 8, end: 8 } (rules\parameters.md:46) - { start: 15, end: 15 } (rules\parameters.md:53) -👻 required-property-must-exist (rules\parameters.md:121) - ✅ Test OK (rules\parameters.md:92) -``` \ No newline at end of file +- [valid](examples/valid) +- [invalid](examples/invalid) \ No newline at end of file diff --git a/example/invalid/README.md b/example/invalid/README.md deleted file mode 100644 index d1c8b59..0000000 --- a/example/invalid/README.md +++ /dev/null @@ -1,2 +0,0 @@ -* This example contains a rule in the [invalid.md](invalid.md) file - * ❌ The test for this rule is expected to NOT pass \ No newline at end of file diff --git a/example/valid/README.md b/example/valid/README.md deleted file mode 100644 index 599125b..0000000 --- a/example/valid/README.md +++ /dev/null @@ -1,5 +0,0 @@ -* This example contains rules accross several markdown files: [rules1.md](rules1.md) and [rules2.md](rules2.md) - * ✅ All the test declared in those files are expected to pass -* Some of the rules use custom functions defined in the [functions](functions) directory -* [spectral.base.yaml](spectral.base.yaml) contains the base rule files that will be used to merge the rules into. For example, it contains the extends, aliases and functions declarations -* [spectral.yaml](spectral.yaml) is the output of the merge operation: [spectral.base.yaml](spectral.base.yaml) + all the rules found in [rules1.md](rules1.md) and [rules2.md](rules2.md) \ No newline at end of file diff --git a/examples/invalid/README.md b/examples/invalid/README.md new file mode 100644 index 0000000..09bd308 --- /dev/null +++ b/examples/invalid/README.md @@ -0,0 +1,15 @@ +# Invalid example + +* This example contains a rule in the [invalid.md](invalid.md) file + * ❌ The test for this rule is expected to NOT pass + +## Test ouput + +``` +npx poltergust test .\examples\invalid +🔎 Testing the spectral rules from the .md files in the directory: .\examples\invalid +👻 base-path-must-start-with-slash (examples\invalid\invalid.md:22) + ❌ Expected 0 failure(s) for rule base-path-must-start-with-slash in test (examples\invalid\invalid.md:9) + But got 1 instead: + { start: 6, end: 6 } (examples\invalid\invalid.md:15) +``` \ No newline at end of file diff --git a/example/invalid/invalid.md b/examples/invalid/invalid.md similarity index 86% rename from example/invalid/invalid.md rename to examples/invalid/invalid.md index 0cf7af1..1102164 100644 --- a/example/invalid/invalid.md +++ b/examples/invalid/invalid.md @@ -6,8 +6,7 @@ ❌ This test should not pass, as the server's url fails the rule ```yaml -#spectral-test -#spectral-should-not-fail-anywhere-✅: base-path-must-start-with-slash +#👻-failures: 0 base-path-must-start-with-slash openapi: 3.0.1 info: title: Test @@ -20,7 +19,7 @@ servers: Spectral rule 🤖 ```yaml -#spectral-rule +#👻-rule base-path-must-start-with-slash: description: Base path must start with /. message: "{{description}}. But was {{value}}." diff --git a/example/invalid/spectral.base.yaml b/examples/invalid/spectral.base.yaml similarity index 100% rename from example/invalid/spectral.base.yaml rename to examples/invalid/spectral.base.yaml diff --git a/example/invalid/spectral.yaml b/examples/invalid/spectral.yaml similarity index 100% rename from example/invalid/spectral.yaml rename to examples/invalid/spectral.yaml diff --git a/examples/valid/README.md b/examples/valid/README.md new file mode 100644 index 0000000..37ad7bf --- /dev/null +++ b/examples/valid/README.md @@ -0,0 +1,28 @@ +# Valid example + +* This example contains rules accross several markdown files: [rules1.md](rules1.md) and [rules2.md](rules2.md) + * ✅ All the test declared in those files are expected to pass +* Some of the rules use custom functions defined in the [functions](functions) directory +* [spectral.base.yaml](spectral.base.yaml) contains the base rule files that will be used to merge the rules into. For example, it contains the extends, aliases and functions declarations +* [spectral.yaml](spectral.yaml) is the output of the merge operation: [spectral.base.yaml](spectral.base.yaml) + all the rules found in [rules1.md](rules1.md) and [rules2.md](rules2.md) + +## Test ouput + +``` +npx poltergust test .\examples\valid +🔎 Testing the spectral rules from the .md files in the directory: .\examples\valid +👻 base-path-must-start-with-slash (examples\valid\rules1.md:40) + ✅ Test OK (examples\valid\rules1.md:17) + ✅ Test OK (examples\valid\rules1.md:27) +👻 operation-parameters-must-have-description (examples\valid\rules1.md:91) + ✅ Test OK (examples\valid\rules1.md:58) + ✅ Test OK (examples\valid\rules1.md:73) +👻 operation-must-have-description (examples\valid\rules1.md:107) +👻 operation-must-have-no-summary (examples\valid\rules1.md:122) +👻 operation-must-have-at-least-one-response (examples\valid\rules1.md:135) +👻 request-bodies-must-have-a-content (examples\valid\rules1.md:152) +👻 path-parameters-must-be-kebab-case (examples\valid\rules2.md:49) + ✅ Test OK (examples\valid\rules2.md:11) +👻 required-property-must-exist (examples\valid\rules2.md:96) + ✅ Test OK (examples\valid\rules2.md:66) +``` \ No newline at end of file diff --git a/example/valid/functions/isRequiredPropertyDefined.js b/examples/valid/functions/isRequiredPropertyDefined.js similarity index 100% rename from example/valid/functions/isRequiredPropertyDefined.js rename to examples/valid/functions/isRequiredPropertyDefined.js diff --git a/example/valid/rules1.md b/examples/valid/rules1.md similarity index 86% rename from example/valid/rules1.md rename to examples/valid/rules1.md index 9f85a99..6db0b41 100644 --- a/example/valid/rules1.md +++ b/examples/valid/rules1.md @@ -14,8 +14,7 @@ We don't want static base path, because the endpoints will be aggregated and a c OpenAPI examples: ```yaml -#spectral-test -#spectral-should-fail-anywhere-❌: base-path-must-start-with-slash +#👻-failures: 1 base-path-must-start-with-slash openapi: 3.0.1 info: title: Test @@ -25,8 +24,7 @@ servers: ``` ```yaml -#spectral-test -#spectral-should-not-fail-anywhere-✅: base-path-must-start-with-slash +#👻-failures: 0 base-path-must-start-with-slash openapi: 3.0.1 info: title: Test @@ -39,7 +37,7 @@ servers: Spectral rule 🤖 ```yaml -#spectral-rule +#👻-rule base-path-must-start-with-slash: description: Base path must start with /. message: "{{description}}. But was {{value}}." @@ -57,13 +55,13 @@ base-path-must-start-with-slash: ## operation-parameters-must-have-description ```yaml -#spectral-test +#👻-failures: 1 operation-parameters-must-have-description openapi: 3.0.1 paths: /test/{id}: get: parameters: - - name: id #spectral-should-fail-here-❌: operation-parameters-must-have-description + - name: id #👻-fails-here: operation-parameters-must-have-description in: path # need a description required: true @@ -72,8 +70,7 @@ paths: ``` ```yaml -#spectral-test -#spectral-should-not-fail-anywhere-✅: operation-parameters-must-have-description +#👻-failures: 0 operation-parameters-must-have-description openapi: 3.0.1 paths: /test/{id}: @@ -91,7 +88,7 @@ paths: Spectral rule 🤖 ```yaml -#spectral-rule +#👻-rule operation-parameters-must-have-description: description: Operation parameters must have a description given: $.paths[*][*].parameters[*] @@ -107,7 +104,7 @@ operation-parameters-must-have-description: ## operation-must-have-description ```yaml -#spectral-rule +#👻-rule operation-must-have-description: description: Operation must have a description given: $.paths[*][*] @@ -122,7 +119,7 @@ operation-must-have-description: Summary on operations mess up the documentation portal :( ```yaml -#spectral-rule +#👻-rule operation-must-have-no-summary: description: Operation must not have a summary given: $.paths[*][*] @@ -135,7 +132,7 @@ operation-must-have-no-summary: ## operation-must-have-at-least-one-response ```yaml -#spectral-rule +#👻-rule operation-must-have-at-least-one-response: description: Operation must have at least one response given: $.paths[*][*] @@ -152,7 +149,7 @@ operation-must-have-at-least-one-response: ## request-bodies-must-have-a-content ```yaml -#spectral-rule +#👻-rule request-bodies-must-have-a-content: description: Request bodies must have a content given: $.paths[*][*].requestBody diff --git a/example/valid/rules2.md b/examples/valid/rules2.md similarity index 66% rename from example/valid/rules2.md rename to examples/valid/rules2.md index abb56d6..05e78ec 100644 --- a/example/valid/rules2.md +++ b/examples/valid/rules2.md @@ -5,36 +5,10 @@ ## path-parameters-must-be-kebab-case An example of valid kebab case parameters, both defined at the path or operation level -```yaml -#spectral-test -#spectral-should-not-fail-anywhere-✅: path-parameters-must-be-kebab-case -openapi: 3.0.1 -info: - title: Test - version: 1.0.0 -paths: - /cats/{dog-id}: - get: - parameters: - - name: dog-id #good - in: path - required: true - schema: - type: number - /dogs/{cat-id}: - parameters: - - name: cat-id #good - in: path - required: true - schema: - type: number - get: {} -``` - Example of a wrongs parameters casing ```yaml -#spectral-test +#👻-failures: 2 path-parameters-must-be-kebab-case openapi: 3.0.1 info: title: Test @@ -43,14 +17,14 @@ paths: /cats/{dogId}: get: parameters: - - name: dogId #spectral-should-fail-here-❌: path-parameters-must-be-kebab-case + - name: dogId #👻-fails-here: path-parameters-must-be-kebab-case in: path required: true schema: type: number /dogs/{catId}: parameters: - - name: catId #spectral-should-fail-here-❌: path-parameters-must-be-kebab-case + - name: catId #👻-fails-here: path-parameters-must-be-kebab-case in: path required: true schema: @@ -59,7 +33,7 @@ paths: /mouses/{mice-id}: get: parameters: - - name: mice-id #spectral-should-not-fail-here-✅: path-parameters-must-be-kebab-case + - name: mice-id #this should not fail in: path required: true schema: @@ -72,7 +46,7 @@ paths: This use the **pathParameters** alias to target both the parameters in the "paths" and the "operations. ```yaml -#spectral-rule +#👻-rule path-parameters-must-be-kebab-case: description: Path parameters must be kebab case given: "#parameters[?(@.in==\"path\")]" @@ -89,7 +63,7 @@ path-parameters-must-be-kebab-case: ## required-property-must-exist ```yaml -#spectral-test +#👻-failures: 1 required-property-must-exist openapi: 3.0.1 info: title: Test @@ -106,8 +80,8 @@ components: nestedschema: type: object required: - - id #spectral-should-not-fail-here-✅: required-property-must-exist - - nonexistent #spectral-should-fail-here-❌: required-property-must-exist + - id + - nonexistent #👻-fails-here: required-property-must-exist properties: id: type: string @@ -119,7 +93,7 @@ components: This use the **isRequiredPropertyDefined** custom function. ```yaml -#spectral-rule +#👻-rule required-property-must-exist: description: Required property must exist message: "Required property must exist: {{error}}" diff --git a/example/valid/spectral.base.yaml b/examples/valid/spectral.base.yaml similarity index 100% rename from example/valid/spectral.base.yaml rename to examples/valid/spectral.base.yaml diff --git a/example/valid/spectral.yaml b/examples/valid/spectral.yaml similarity index 100% rename from example/valid/spectral.yaml rename to examples/valid/spectral.yaml diff --git a/src/functions.js b/src/functions.js index ee6fcd8..ebb0eba 100644 --- a/src/functions.js +++ b/src/functions.js @@ -7,70 +7,76 @@ import { bundleAndLoadRuleset } from "@stoplight/spectral-ruleset-bundler/with-l /** * @param {string} rulesDir - * @returns {Object.} the spectral rules and test cases by rule name + * @returns {Object.} the spectral rules and test cases by rule name */ export function extractAllRulesAndTestCases(rulesDir) { const markdownFiles = fs.readdirSync(rulesDir).filter(f => f.endsWith('.md')); - let rulesAndTestCases = {}; + /** @type {Object.} */ + let byRuleName = {}; for (let file of markdownFiles) { const filePath = path.join(rulesDir, file); - const rulesAndTestCasesForFile = extractRulesAndTestCases(filePath); - //merge or append rules and tests cases in - for (let ruleName in rulesAndTestCasesForFile) { - if(rulesAndTestCases[ruleName]){ - const rule = rulesAndTestCases[ruleName].rule || rulesAndTestCasesForFile[ruleName].rule; - //merge testcases - const testCases = (rulesAndTestCases[ruleName].testCases || []).concat(rulesAndTestCasesForFile[ruleName].testCases || []); - rulesAndTestCases[ruleName] = {rule, testCases}; - }else{ - rulesAndTestCases[ruleName] = rulesAndTestCasesForFile[ruleName]; - } + extractRulesAndTestCases(filePath, byRuleName); + } + + //check for missing rules + for (let ruleName in byRuleName) { + if (!byRuleName[ruleName].rule) { + const testcase = byRuleName[ruleName].testCases[0]; + console.error(`⚠️ Test case for unknown rule: ${ruleName} at (${testcase.filePath}:${testcase.line})`); + delete byRuleName[ruleName]; } } - return rulesAndTestCases + + return byRuleName } /** * @param {string} filePath - * @returns {Object.} rule and test cases by rule name + * @param {Object.} byRuleName */ -export function extractRulesAndTestCases(filePath) { +export function extractRulesAndTestCases(filePath, byRuleName) { const blocks = extractYamlBlocks(filePath); - // spectral rules starts by #spectral-rule - let byRuleName = {}; - blocks.filter(block => block.content.startsWith('#spectral-rule')).forEach(block => { + blocks.forEach(block => { const rule = extractRuleFromBlock(block, filePath); + const testCase = extractTestCaseBlock(block, filePath); - //handle duplicate rules - if(byRuleName[rule.name]) { - console.error(`❌ Rule name ${rule.name} is already defined`) + //invalid states + if(!rule && !testCase){ + console.log('Skipping block at line', block.line, 'in', block.filePath); + return; + } + if(rule && testCase){ + console.error(`⚠️ Rule and test case in the same block at (${filePath}:${block.line})`); + return; + } + if(rule && byRuleName[rule.name] && byRuleName[rule.name].rule){ + console.error(`⚠️ Rule ${rule.name} is defined in both ${byRuleName[rule.name].rule.filePath} and ${rule.filePath}`); + return; } - byRuleName[rule.name] = {rule}; - }); - - // spectral tests starts by #spectral-test - blocks.filter(block => block.content.startsWith('#spectral-test')).forEach(block => { - const testCase = extractTestCaseBlock(block, filePath); - //append the test case to each concerned rule - let disctinctRules = [...new Set(testCase.assertions.map(r => r.ruleName))]; - disctinctRules.forEach(ruleName => { - //if the rule name exist in byRuleName, add the test case to it - if(byRuleName[ruleName]){ - const testCases = byRuleName[ruleName].testCases || []; - testCases.push(testCase); - byRuleName[ruleName].testCases = testCases; - }else{ - byRuleName[ruleName] = {testCases: [testCase]}; + //add rule + if(rule){ + if(!byRuleName[rule.name]) + byRuleName[rule.name] = {rule, testCases: []}; + else + byRuleName[rule.name].rule = rule; + } + + //add test case to each rule impacted + if(testCase){ + //use a set of rule name to avoid duplicates when a test case has multiple assertions on the same rule + let ruleNames = new Set(testCase.assertions.map(a => a.ruleName)); + for (let ruleName of ruleNames) { + if(!byRuleName[ruleName]) + byRuleName[ruleName] = {rule: undefined, testCases: [testCase]}; + else + byRuleName[ruleName].testCases.push(testCase); } - - }); + } }); - - return byRuleName; } /** @@ -97,11 +103,13 @@ export function extractYamlBlocks(filePath) { /** * - * @param {*} block - * @param {*} filePath + * @param {{content: string, line: number, filePath: string}} block + * @param {string} filePath * @returns {{name: string, content: string, line: number, filePath: string}} the spectral rule */ function extractRuleFromBlock(block, filePath) { + if(!block.content.startsWith('#👻-rule')) + return null; const content = block.content.split('\n').slice(1).join('\n'); const name = content.split('\n')[0].trim().replace(':', ''); const line = block.line; @@ -110,35 +118,35 @@ function extractRuleFromBlock(block, filePath) { /** * - * @param {*} block - * @param {*} filePath - * @returns {{content: string, assertions: {ruleName: string, line: number, shouldFail: boolean}[], line: number, filePath: string}} the spectral rule + * @param {{content: string, line: number, filePath: string}} block + * @returns {{content: string, assertions: {ruleName: string, line: number, failuresCount: number}[], line: number, filePath: string}} the spectral rule */ -function extractTestCaseBlock(block, filePath) { - const content = block.content.split('\n').slice(1).join('\n'); +function extractTestCaseBlock(block) { + if(block.content.startsWith('#👻-rule')) + return null; + const content = block.content; const line = block.line; const assertions = []; content.split('\n').forEach((line, index) => { - if(line.includes('#spectral-should-not-fail-anywhere-✅:')){ - const ruleName = line.split('#spectral-should-not-fail-anywhere-✅:')[1].trim(); - assertions.push({ruleName, shouldFail: false}); - } - if(line.includes('#spectral-should-not-fail-here-✅:')){ - const ruleName = line.split('#spectral-should-not-fail-here-✅:')[1].trim(); - assertions.push({ruleName, shouldFail: false, line: index}); + if(line.includes('#👻-failures:')){ + const failureCountAndruleName = line.split('#👻-failures:')[1].trim(); + const failuresCount = failureCountAndruleName.split(' ')[0]; + const ruleName = failureCountAndruleName.split(' ')[1]; + assertions.push({ruleName, failuresCount: failuresCount}); } - if(line.includes('#spectral-should-fail-anywhere-❌:')){ - const ruleName = line.split('#spectral-should-fail-anywhere-❌:')[1].trim(); - assertions.push({ruleName, shouldFail: true}); - } - if(line.includes('#spectral-should-fail-here-❌:')){ - const ruleName = line.split('#spectral-should-fail-here-❌:')[1].trim(); - assertions.push({ruleName, shouldFail: true, line: index}); + else if(line.includes('#👻-fails-here:')){ + const ruleName = line.split('#👻-fails-here:')[1].trim(); + assertions.push({ruleName, failuresCount: 1, line: index}); + }else if(line.includes('#👻-')){ + const tag = line.split('#👻-')[1].trim(); + console.error(`⚠️ Unknown tag: "#👻-${tag}" at (${block.filePath}:${block.line+index})`); } }); - return {content, line, filePath, assertions}; + if(assertions.length === 0) + return null; + return {content, line, filePath: block.filePath, assertions}; } @@ -150,7 +158,7 @@ function extractTestCaseBlock(block, filePath) { export function generateRuleFileContent(spectralRulesContents, rulesDir){ const baseFile = path.join(rulesDir, 'spectral.base.yaml'); if (!fs.existsSync(baseFile)) { - console.error(`❌ The file ${baseFile} does not exist.`); + console.error(`⚠️ The file ${baseFile} does not exist.`); } let ruleFileContent = fs.readFileSync(baseFile, 'utf8'); @@ -165,7 +173,7 @@ export function generateRuleFileContent(spectralRulesContents, rulesDir){ /** * * @param {{name: string, content: string, line: number, filePath: string}} rule - * @param {{content: string, assertions: {ruleName: string, line: number, shouldFail: boolean}[], line: number, filePath: string}} testCase + * @param {{content: string, assertions: {ruleName: string, line: number, failuresCount: number}[], line: number, filePath: string}} testCase * @param {string} rulesDir * @return {Promise} success */ @@ -202,53 +210,30 @@ export async function runTestCase(rule, testCase, rulesDir){ const ruleAssertions = testCase.assertions.filter(a => a.ruleName === rule.name); for (let assertion of ruleAssertions) { - //should-not-fail-anywhere - if(!assertion.shouldFail && assertion.line === undefined){ - if(ruleErrors.length > 0){ - console.error(` ❌ Was not expecting to fail rule anywhere ${assertion.ruleName} in test (${testCase.filePath}:${testCase.line})`); - console.error(` But failed there instead:`); - ruleErrors.forEach(e => console.error(' ', {start: e.range.start.line, end: e.range.end.line}, `(${testCase.filePath}:${testCase.line+e.range.start.line+1})`)); - ok = false; - } - } - - //should-fail-anywhere - if(assertion.shouldFail && assertion.line === undefined){ - if(ruleErrors.length === 0){ - console.error(` ❌ Was expecting to fail rule anywhere ${assertion.ruleName} in test (${testCase.filePath}:${testCase.line})`); - ok = false; - } - } - - //should-not-fail-here - if(!assertion.shouldFail && assertion.line !== undefined){ - let matchingErrors = ruleErrors.filter(e => e.range.start.line <= assertion.line && e.range.end.line >= assertion.line); - if(matchingErrors.length > 0){ - console.error(` ❌ Was not expecting to fail rule ${assertion.ruleName} at line ${assertion.line} in test (${testCase.filePath}:${testCase.line+assertion.line+1})`); - ok = false; - } + //#👻-failures: + if(assertion.line === undefined && ruleErrors.length != assertion.failuresCount){ + console.error(` ❌ Expected ${assertion.failuresCount} failure(s) for rule ${assertion.ruleName} in test (${testCase.filePath}:${testCase.line})`); + console.error(` But got ${ruleErrors.length} instead:`); + ruleErrors.forEach(e => console.error(' ', {start: e.range.start.line, end: e.range.end.line}, `(${testCase.filePath}:${testCase.line+e.range.start.line})`)); + ok = false; } - - //should-fail-here - if(assertion.shouldFail && assertion.line !== undefined){ + //#👻-fails-here: + if(assertion.line !== undefined){ let matchingErrors = ruleErrors.filter(e => e.range.start.line <= assertion.line && e.range.end.line >= assertion.line); - if(matchingErrors.length === 0){ - console.error(` ❌ Was expecting to fail rule ${assertion.ruleName} at line ${assertion.line} in test (${testCase.filePath}:${testCase.line+assertion.line+1})`); - console.error(` But failed there instead:`); - ruleErrors.forEach(e => console.error(' ', {start: e.range.start.line, end: e.range.end.line}, `(${testCase.filePath}:${testCase.line+e.range.start.line+1})`)); + if(matchingErrors.length == 0){ + console.error(` ❌ Expected rule ${assertion.ruleName} to fail at line ${assertion.line} in test (${testCase.filePath}:${testCase.line+assertion.line})`); + console.error(` But got ${ruleErrors.length} instead:`); + ruleErrors.forEach(e => console.error(' ', {start: e.range.start.line, end: e.range.end.line}, `(${testCase.filePath}:${testCase.line+e.range.start.line})`)); ok = false; } } - } + + if(ok) + console.log(` ✅ Test OK (${testCase.filePath}:${testCase.line})`); }finally{ fs.rmSync(tempDir, {recursive: true}); } - if(ok){ - console.log(` ✅ Test OK (${testCase.filePath}:${testCase.line})`); - return true; - }else{ - return false; - } + return ok; } \ No newline at end of file diff --git a/src/index.js b/src/index.js index e249349..11a505e 100644 --- a/src/index.js +++ b/src/index.js @@ -25,14 +25,6 @@ if (args[0] === 'merge') { console.log("🔎 Testing the spectral rules from the .md files in the directory: " + rulesDir); let rulesAndTestCases = extractAllRulesAndTestCases(rulesDir); - //if an entry has no associated rule, throw an error - for (let ruleName in rulesAndTestCases) { - if (!rulesAndTestCases[ruleName].rule) { - console.error(`❌ Test cases exists for unknown rule: ${ruleName}`); - delete rulesAndTestCases[ruleName]; - } - } - //run the test cases for each rule for(let ruleName in rulesAndTestCases){ const rule = rulesAndTestCases[ruleName].rule;