From 1d3e21c1aac2614d39f5ba0fa0839106b2138e9a Mon Sep 17 00:00:00 2001 From: Daniel Kopp Date: Fri, 29 Nov 2024 10:19:37 +0100 Subject: [PATCH] feat(scaffolder-backend-module-utils): add support to handle multiple documents in yaml file in merge --- .changeset/mean-lions-rhyme.md | 5 + .../scaffolder-backend-module-utils/README.md | 1 + .../package.json | 4 +- .../src/actions/merge/merge.test.ts | 153 ++++++++++++++ .../src/actions/merge/merge.ts | 199 +++++++++++++++--- yarn.lock | 46 +++- 6 files changed, 370 insertions(+), 38 deletions(-) create mode 100644 .changeset/mean-lions-rhyme.md diff --git a/.changeset/mean-lions-rhyme.md b/.changeset/mean-lions-rhyme.md new file mode 100644 index 000000000..522ec3c0d --- /dev/null +++ b/.changeset/mean-lions-rhyme.md @@ -0,0 +1,5 @@ +--- +'@roadiehq/scaffolder-backend-module-utils': minor +--- + +add support to handle multiple documents in yaml file diff --git a/plugins/scaffolder-actions/scaffolder-backend-module-utils/README.md b/plugins/scaffolder-actions/scaffolder-backend-module-utils/README.md index 0b9e4b983..e3a9f57b2 100644 --- a/plugins/scaffolder-actions/scaffolder-backend-module-utils/README.md +++ b/plugins/scaffolder-actions/scaffolder-backend-module-utils/README.md @@ -583,6 +583,7 @@ spec: - mergeArrays: If `true` then where a value is an array the merge function will concatenate the provided array value with the target array. - preserveYamlComments: If `true`, it will preserve standalone and inline comments in YAML files. +- useDocumentIncludingField: If multiple documents are present in the YAML file, it will merge the content into the document that includes the specified field. - options: YAML stringify options to customize the output format. - indent: (default: 2) - indentation width to use (in spaces). - noArrayIndent: (default: false) - when true, will not add an indentation level to array elements. diff --git a/plugins/scaffolder-actions/scaffolder-backend-module-utils/package.json b/plugins/scaffolder-actions/scaffolder-backend-module-utils/package.json index 1d458db67..664d7bae2 100644 --- a/plugins/scaffolder-actions/scaffolder-backend-module-utils/package.json +++ b/plugins/scaffolder-actions/scaffolder-backend-module-utils/package.json @@ -57,8 +57,8 @@ "jsonata": "^2.0.4", "lodash": "^4.17.21", "winston": "^3.2.1", - "yaml": "^2.3.4", - "yawn-yaml": "^2.2.0" + "yaml": "^2.6.1", + "yawn-yaml": "^2.3.0" }, "devDependencies": { "@backstage/cli": "^0.28.0", diff --git a/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.test.ts b/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.test.ts index 8b96ecdf6..d9084d785 100644 --- a/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.test.ts +++ b/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.test.ts @@ -612,4 +612,157 @@ scripts: # Trailing comment scripts: { lsltr: 'ls -ltr', lsltrh: 'ls -ltrh' }, }); }); + + it('should merge content into the correct YAML document', async () => { + mock({ + 'fake-tmp-dir': { + 'fake-file.yaml': ` +--- +id: 1 +name: Document 1 +--- +id: 2 +scripts: + lsltr: ls -ltr +--- +id: 3 +name: Document 3 +`, + }, + }); + + await action.handler({ + ...mockContext, + workspacePath: 'fake-tmp-dir', + input: { + path: 'fake-file.yaml', + content: YAML.stringify({ + scripts: { + lsltrh: 'ls -ltrh', + }, + }), + useDocumentIncludingField: { + key: 'id', + value: '2', + }, + }, + }); + + expect(fs.existsSync('fake-tmp-dir/fake-file.yaml')).toBe(true); + const file = fs.readFileSync('fake-tmp-dir/fake-file.yaml', 'utf-8'); + const documents = YAML.parseAllDocuments(file); + expect(documents[0].toJSON()).toEqual({ + id: 1, + name: 'Document 1', + }); + expect(documents[1].toJSON()).toEqual({ + id: 2, + scripts: { + lsltr: 'ls -ltr', + lsltrh: 'ls -ltrh', + }, + }); + expect(documents[2].toJSON()).toEqual({ + id: 3, + name: 'Document 3', + }); + }); + + it('should merge content into the correct YAML document and preserve comments', async () => { + mock({ + 'fake-tmp-dir': { + 'fake-file.yaml': ` +--- +id: 1 +name: Document 1 +# Comment for document 1 +--- +id: 2 +scripts: + lsltr: ls -ltr # Inline comment +# Comment for document 2 +--- +id: 3 +name: Document 3 +# Comment for document 3 +`, + }, + }); + + await action.handler({ + ...mockContext, + workspacePath: 'fake-tmp-dir', + input: { + path: 'fake-file.yaml', + content: YAML.stringify({ + scripts: { + lsltrh: 'ls -ltrh', + }, + }), + useDocumentIncludingField: { + key: 'id', + value: '2', + }, + preserveYamlComments: true, + }, + }); + + expect(fs.existsSync('fake-tmp-dir/fake-file.yaml')).toBe(true); + const file = fs.readFileSync('fake-tmp-dir/fake-file.yaml', 'utf-8'); + const documents = YAML.parseAllDocuments(file); + expect(documents[0].toJSON()).toEqual({ + id: 1, + name: 'Document 1', + }); + expect(file).toContain('# Inline comment'); + expect(file).toContain('# Comment for document 2'); + expect(documents[1].toJSON()).toEqual({ + id: 2, + scripts: { + lsltr: 'ls -ltr', + lsltrh: 'ls -ltrh', + }, + }); + expect(file).toContain('# Comment for document 3'); + expect(documents[2].toJSON()).toEqual({ + id: 3, + name: 'Document 3', + }); + }); + + it('should throw error when multiple yaml documents exist, but useDocumentIncludingField is missing', async () => { + mock({ + 'fake-tmp-dir': { + 'fake-file.yaml': ` +--- +id: 1 +name: Document 1 +--- +id: 2 +scripts: + lsltr: ls -ltr +--- +id: 3 +name: Document 3 +`, + }, + }); + + await expect( + action.handler({ + ...mockContext, + workspacePath: 'fake-tmp-dir', + input: { + path: 'fake-file.yaml', + content: YAML.stringify({ + scripts: { + lsltrh: 'ls -ltrh', + }, + }), + }, + }), + ).rejects.toThrow( + 'Multiple documents found in the input content. Please provide a key and value to use to find the document to merge into.', + ); + }); }); diff --git a/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.ts b/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.ts index f0e0e3588..73baf92b7 100644 --- a/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.ts +++ b/plugins/scaffolder-actions/scaffolder-backend-module-utils/src/actions/merge/merge.ts @@ -19,7 +19,7 @@ import { resolveSafeChildPath } from '@backstage/backend-plugin-api'; import fs from 'fs-extra'; import { extname } from 'path'; import { isArray, isNull, mergeWith } from 'lodash'; -import YAML from 'yaml'; +import YAML, { Document } from 'yaml'; import YAWN from 'yawn-yaml'; import { stringifyOptions, yamlOptionsSchema } from '../../types'; import detectIndent from 'detect-indent'; @@ -31,6 +31,23 @@ function mergeArrayCustomiser(objValue: string | any[], srcValue: any) { return undefined; } +const existPathInObject = (object: any, path: string, value: string) => { + const keys = path.split('.'); + if (typeof object !== 'object') { + return false; + } + + let current = object; + for (const key of keys) { + if (current[key] === undefined) { + current = undefined; + break; + } + current = current[key]; + } + return current?.toString() === value; +}; + export function createMergeJSONAction({ actionId }: { actionId?: string }) { return createTemplateAction<{ path: string; @@ -139,6 +156,7 @@ export function createMergeAction() { return createTemplateAction<{ path: string; content: any; + useDocumentIncludingField?: { key: string; value: string }; mergeArrays?: boolean; preserveYamlComments?: boolean; options?: stringifyOptions; @@ -176,6 +194,27 @@ export function createMergeAction() { description: 'Will preserve standalone and inline comments in YAML files', }, + useDocumentIncludingField: { + type: 'object', + title: 'Use Document Including Field', + default: undefined, + description: + 'This option is only applicable to YAML files. It allows you to specify a field to use as a key to find the document to merge into.', + properties: { + key: { + title: 'Key', + description: + 'The key of the field to use to find the document to merge into.', + type: 'string', + }, + value: { + title: 'Value', + description: + 'The value of the field to use to find the document to merge into.', + type: 'string', + }, + }, + }, options: { ...yamlOptionsSchema, description: `${yamlOptionsSchema.description} (for YAML output only)`, @@ -223,36 +262,31 @@ export function createMergeAction() { break; } case '.yml': - case '.yaml': { - const newContent = - typeof ctx.input.content === 'string' - ? YAML.parse(ctx.input.content) - : ctx.input.content; // This supports the case where dynamic keys are required - if (ctx.input.preserveYamlComments) { - const yawn = new YAWN(originalContent); - const parsedOriginal = yawn.json; - const mergedJsonContent = mergeWith( - parsedOriginal, - newContent, - ctx.input.mergeArrays ? mergeArrayCustomiser : undefined, - ); - yawn.json = mergedJsonContent; - mergedContent = YAML.stringify( - YAML.parseDocument(yawn.yaml), - ctx.input.options, - ); - } else { - mergedContent = YAML.stringify( - mergeWith( - YAML.parse(originalContent), + case '.yaml': + { + const newContent = + typeof ctx.input.content === 'string' + ? YAML.parse(ctx.input.content) + : ctx.input.content; // This supports the case where dynamic keys are required + if (ctx.input.preserveYamlComments) { + mergedContent = mergeContentPreserveComments( + originalContent, newContent, - ctx.input.mergeArrays ? mergeArrayCustomiser : undefined, - ), - ctx.input.options, - ); + ctx, + ) + .map(doc => YAML.stringify(doc, ctx.input.options)) + .join('---\n'); + } else { + mergedContent = mergeDocumentsRemovingComments( + originalContent, + newContent, + ctx, + ) + .map(doc => YAML.stringify(doc, ctx.input.options)) + .join('---\n'); + } } break; - } default: break; } @@ -264,3 +298,112 @@ export function createMergeAction() { }, }); } + +function mergeDocumentsRemovingComments( + originalContent: string, + newContent: any, + ctx: any, +): Document[] { + const documents = YAML.parseAllDocuments(originalContent); + checkDocumentExists(ctx, documents); + if (documents.length === 1) { + return [ + mergeWith( + documents[0].toJSON(), + newContent, + ctx.input.mergeArrays ? mergeArrayCustomiser : undefined, + ), + ]; + } + checkUseDocumentIncludingFieldSet(ctx); + const { key, value } = ctx.input.useDocumentIncludingField; + return documents.map((document: Document) => { + let includingField = document.get(key); + if (typeof includingField === 'number') { + includingField = includingField.toString(); + } + if (typeof includingField !== 'string') { + ctx.logger.error( + `The value at "${key}" defined in useDocumentIncludingField must be a string or a number.`, + ); + throw new Error( + `The value at "${key}" defined in useDocumentIncludingField must be a string or a number.`, + ); + } + if ((includingField as string) === value) { + return mergeWith( + document.toJSON(), + newContent, + ctx.input.mergeArrays ? mergeArrayCustomiser : undefined, + ); + } + return document.toJSON(); + }); +} + +function mergeContentPreserveComments( + originalContent: string, + newContent: any, + ctx: any, +): Document[] { + const yawns = splitYaml(originalContent).map( + (document: string) => new YAWN(document), + ); + checkDocumentExists(ctx, yawns); + if (yawns.length === 1) { + return [ + YAML.parseDocument( + mergeYawn(yawns[0], newContent, ctx.input.mergeArrays).yaml, + ), + ]; + } + checkUseDocumentIncludingFieldSet(ctx); + const { key, value } = ctx.input.useDocumentIncludingField; + return yawns.map((yawn: YAWN) => + YAML.parseDocument( + existPathInObject(yawn.json, key, value) + ? mergeYawn(yawn, newContent, ctx.input.mergeArrays).yaml + : yawn.yaml, + ), + ); +} + +function checkDocumentExists(ctx: any, documents: any[]) { + if (documents.length === 0) { + ctx.logger.error( + `No documents found in the input content. Please provide a valid YAML file.`, + ); + throw new Error( + `No documents found in the input content. Please provide a valid YAML file.`, + ); + } +} + +function checkUseDocumentIncludingFieldSet(ctx: any) { + if (!ctx.input.useDocumentIncludingField) { + ctx.logger.error( + `Multiple documents found in the input content. Please provide a key and value to use to find the document to merge into.`, + ); + throw new Error( + `Multiple documents found in the input content. Please provide a key and value to use to find the document to merge into.`, + ); + } +} + +function mergeYawn( + yawn: YAWN, + newContent: any, + mergeArrays: boolean | undefined, +) { + const parsedOriginal = yawn.json; + yawn.json = mergeWith( + parsedOriginal, + newContent, + mergeArrays ? mergeArrayCustomiser : undefined, + ); + return yawn; +} + +function splitYaml(originalContent: string): string[] { + return originalContent.split(/^---\s*$/m).filter(doc => doc.trim() !== ''); +} diff --git a/yarn.lock b/yarn.lock index 238c7c605..94e2ac486 100644 --- a/yarn.lock +++ b/yarn.lock @@ -32726,7 +32726,16 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -32800,7 +32809,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -32814,6 +32823,13 @@ strip-ansi@5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -35018,7 +35034,7 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -35036,6 +35052,15 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.0.1, wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" @@ -35272,11 +35297,16 @@ yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2: resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== -yaml@^2.0.0, yaml@^2.2.1, yaml@^2.2.2, yaml@^2.3.4: +yaml@^2.0.0, yaml@^2.2.1, yaml@^2.2.2: version "2.4.0" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.4.0.tgz#2376db1083d157f4b3a452995803dbcf43b08140" integrity sha512-j9iR8g+/t0lArF4V6NE/QCfT+CO7iLqrXAHZbJdo+LfjqP1vR8Fg5bSiaq6Q2lOD1AUEVrEVIgABvBFYojJVYQ== +yaml@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.6.1.tgz#42f2b1ba89203f374609572d5349fb8686500773" + integrity sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg== + yargs-parser@20.2.4: version "20.2.4" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54" @@ -35359,10 +35389,10 @@ yauzl@^3.0.0: buffer-crc32 "~0.2.3" pend "~1.2.0" -yawn-yaml@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/yawn-yaml/-/yawn-yaml-2.2.0.tgz#aa9673725f7c277a8b97e8d12496d2333d701890" - integrity sha512-2SJvllxsNXwvbf6s8m7TNDtdn2zakLQpmrwpxIdCInLyEx+/WsZwSTScBOVk3zBbuFPeh7hc+yvUVtyKOgBeKQ== +yawn-yaml@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/yawn-yaml/-/yawn-yaml-2.3.0.tgz#79b2cdd58b7abe243c9b0418997795e5754b7135" + integrity sha512-hYS8kT4I2ftJyLw6LE/JG8rg+0NKdDmI+c2ncnJ9ZmUTsU0ggo3Zf8yjmhXxKlWINS/ZYkh3MvMIsQlIDtRL4A== dependencies: js-yaml "^4.1.0" lodash "^4.17.21"