add unit test for xml submission file validation

Kitware · Sep 10, 2024 · c5431c2 · c5431c2
1 parent ed8459c
commit c5431c2
Show file tree

Hide file tree

Showing 7 changed files with 263 additions and 0 deletions.
diff --git a/app/cdash/tests/CMakeLists.txt b/app/cdash/tests/CMakeLists.txt
@@ -596,5 +596,7 @@ set_tests_properties(misassignedconfigure PROPERTIES DEPENDS /Feature/SubProject
 add_laravel_test(/Feature/RemoteWorkers)
 set_tests_properties(/Feature/RemoteWorkers PROPERTIES DEPENDS install_3)
 
+add_laravel_test(/Unit/app/Console/Command/ValidateXmlCommandTest)
+
 add_subdirectory(ctest)
 add_subdirectory(autoremovebuilds)
diff --git a/tests/Unit/app/Console/Command/ValidateXmlCommandTest.php b/tests/Unit/app/Console/Command/ValidateXmlCommandTest.php
@@ -0,0 +1,145 @@
+<?php
+
+namespace Tests\Feature;
+
+use Tests\TestCase;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Console\Command;
+
+/**
+ * Tests the submission:validate artisan command.
+ */
+class ValidateXmlCommandTest extends TestCase
+{
+    /**
+     * Formats the arguments to pass to the artisan command
+     *
+     * @return array<string,array<string>>
+     */
+    private function formatCommandParams()
+    {
+        $data_dir = base_path()."/tests/data/XmlValidation";
+        $file_paths = [];
+        foreach (func_get_args() as $file) {
+            $file_paths[] = "{$data_dir}/{$file}";
+        }
+        return [
+            'xml_file' => $file_paths,
+        ];
+    }
+
+    /**
+     * Handle common assertions for a 'success' validation outcome
+     *
+     * @param string $output, int $rc, string $msg
+     * @return void
+     */
+    private function assertValidationSuccess(string $output, int $rc, string $msg): void
+    {
+        $this::assertMatchesRegularExpression("/All XML file checks passed./", $output, $msg);
+        $this::assertEquals(Command::SUCCESS, $rc, "Passing validation should return a SUCCESS return code");
+    }
+
+    /**
+     * Handle common assertions for a 'failure' validation outcome
+     *
+     * @param string $output, int $rc, string $msg, bool $skipped_files
+     * @return void
+     */
+    private function assertValidationFailure(string $output, int $rc, string $msg, bool $skipped_files=false): void
+    {
+        $output_regex = $skipped_files ? "/Some XML file checks were skipped!/" : "/Some XML file checks did not pass!/";
+        $this::assertMatchesRegularExpression($output_regex, $output, $msg);
+        $this::assertEquals(Command::FAILURE, $rc, "Failed validation should return a FAILURE return code");
+    }
+
+    /**
+     * Tests validating a single valid file
+     *
+     * @return void
+     */
+    public function testSingleValidXml(): void
+    {
+        $params = $this->formatCommandParams("valid_Build.xml");
+        $rc = Artisan::call('submission:validate', $params);
+        $output = trim(Artisan::output());
+        $this->assertValidationSuccess($output, $rc, "A single valid Build XML file should pass.");
+    }
+
+    /**
+     * Tests validating an invalid file path
+     *
+     * @return void
+     */
+    public function testInvalidFilePath(): void
+    {
+        $params = $this->formatCommandParams("no_such_file.xml");
+        $rc = Artisan::call('submission:validate', $params);
+        $output = trim(Artisan::output());
+        $this->assertValidationFailure($output, $rc, "An invalid file path should fail.", $skipped=true);
+    }
+
+    /**
+     * Tests validating a single file of unknown schema
+     *
+     * @return void
+     */
+    public function testUnknownSchema(): void
+    {
+        $params = $this->formatCommandParams("invalid_type.xml");
+        $rc = Artisan::call('submission:validate', $params);
+        $output = trim(Artisan::output());
+        $this->assertValidationFailure($output, $rc, "An XML file that doesn't match any supported schemas should fail.");
+    }
+
+    /**
+     * Tests validating a single file that does not adhere to its schema
+     *
+     * @return void
+     */
+    public function testSingleInvalidXml(): void
+    {
+        $params = $this->formatCommandParams("invalid_Configure.xml");
+        $rc = Artisan::call('submission:validate', $params);
+        $output = trim(Artisan::output());
+        $this->assertValidationFailure($output, $rc, "An XML file that doesn't adhere to its corresponding schema should fail.");
+    }
+
+    /**
+     * Tests validating a single file with invalid syntax
+     *
+     * @return void
+     */
+    public function testSingleInvalidSyntaxXml(): void
+    {
+        $params = $this->formatCommandParams("invalid_syntax_Build.xml");
+        $rc = Artisan::call('submission:validate', $params);
+        $output = trim(Artisan::output());
+        $this->assertValidationFailure($output, $rc, "An XML file with syntax errors should fail.");
+    }
+    /**
+     * Tests validating multiple valid files
+     *
+     * @return void
+     */
+    public function testMultipleValidXml(): void
+    {
+        $params = $this->formatCommandParams("valid_Build.xml", "valid_Test.xml");
+        $rc = Artisan::call('submission:validate', $params);
+        $output = trim(Artisan::output());
+        $this->assertValidationSuccess($output, $rc, "Multiple valid XML files should pass.");
+    }
+
+    /**
+     * Tests validating multiple files where some are invalid
+     *
+     * @return void
+     */
+    public function testMultipleFilesFailure(): void
+    {
+        $params = $this->formatCommandParams("valid_Build.xml", "invalid_Configure.xml", "valid_Test.xml");
+        $rc = Artisan::call('submission:validate', $params);
+        $output = trim(Artisan::output());
+        $this->assertValidationFailure($output, $rc, "Validation of a set of input files should fail when one or more errors occur.");
+    }
+}
diff --git a/tests/data/XmlValidation/invalid_Configure.xml b/tests/data/XmlValidation/invalid_Configure.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Site BuildName="dummy_build_name"
+  BuildStamp="dummy_stamp"
+  Name="dummy_name"
+>
+  <Configure>
+    <ConfigureCommand>"cmake" "/foo/bar/src"</ConfigureCommand>
+    <!-- missing Log element here -->
+    <ConfigureStatus>0</ConfigureStatus>
+    <ElapsedMinutes>0</ElapsedMinutes>
+  </Configure>
+</Site>
diff --git a/tests/data/XmlValidation/invalid_syntax_Build.xml b/tests/data/XmlValidation/invalid_syntax_Build.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Site BuildName="dummy_build_name"
+  BuildStamp="dummy_stamp"
+  Name="dummy_name"
+  >
+  <Build>
+    <StartDateTime>Aug 13 09:24 EDT</StartDateTime
+    <!-- intentional missing closing bracket here ^ -->
+    <StartBuildTime>1439472247</StartBuildTime>
+    <BuildCommand>/opt/cmake/bin/cmake --build . --config "Debug" -- -i</BuildCommand>
+    <Failure type="Error">
+      <Action>
+        <TargetName>foo</TargetName>
+        <Language>C++</Language>
+        <OutputFile>foo</OutputFile>
+        <OutputType>executable</OutputType>
+      </Action>
+      <Command>
+        <WorkingDirectory>/foo/bar/bin</WorkingDirectory>
+        <Argument>/usr/local/bin/c++</Argument>
+        <Argument>CMakeFiles/foo.dir/foo.cxx.o</Argument>
+        <Argument>-o</Argument>
+        <Argument>foo</Argument>
+      </Command>
+      <Result>
+        <StdOut/>
+                <StdErr>c++: error: CMakeFiles/foo.dir/foo.cxx.o: No such file or directory</StdErr>
+        <ExitCondition>1</ExitCondition>
+      </Result>
+    </Failure>
+    <Log Encoding="base64" Compression="bin/gzip"/>
+    <EndDateTime>Sep 1 09:24 EDT</EndDateTime>
+    <ElapsedMinutes>0</ElapsedMinutes>
+  </Build>
+</Site>
diff --git a/tests/data/XmlValidation/invalid_type.xml b/tests/data/XmlValidation/invalid_type.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Site BuildName="dummy_build_name"
+  BuildStamp=""
+  Name=""
+  >
+  <Foo baz="qux">
+    <Bar>Hello World</Bar>
+  </Foo>
+</Site>
diff --git a/tests/data/XmlValidation/valid_Build.xml b/tests/data/XmlValidation/valid_Build.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Site BuildName="dummy_build_name"
+  BuildStamp="dummy_stamp"
+  Name="dummy_name"
+  >
+  <Build>
+    <StartDateTime>Aug 13 09:24 EDT</StartDateTime>
+    <StartBuildTime>1439472247</StartBuildTime>
+    <BuildCommand>/opt/cmake/bin/cmake --build . --config "Debug" -- -i</BuildCommand>
+    <Failure type="Error">
+      <Action>
+        <TargetName>foo</TargetName>
+        <Language>C++</Language>
+        <OutputFile>foo</OutputFile>
+        <OutputType>executable</OutputType>
+      </Action>
+      <Command>
+        <WorkingDirectory>/foo/bar/bin</WorkingDirectory>
+        <Argument>/usr/local/bin/c++</Argument>
+        <Argument>CMakeFiles/foo.dir/foo.cxx.o</Argument>
+        <Argument>-o</Argument>
+        <Argument>foo</Argument>
+      </Command>
+      <Result>
+        <StdOut/>
+                <StdErr>c++: error: CMakeFiles/foo.dir/foo.cxx.o: No such file or directory</StdErr>
+        <ExitCondition>1</ExitCondition>
+      </Result>
+    </Failure>
+    <Log Encoding="base64" Compression="bin/gzip"/>
+    <EndDateTime>Sep 1 09:24 EDT</EndDateTime>
+    <ElapsedMinutes>0</ElapsedMinutes>
+  </Build>
+</Site>
diff --git a/tests/data/XmlValidation/valid_Test.xml b/tests/data/XmlValidation/valid_Test.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Site BuildName="dummy_test_name"
+  BuildStamp="dummy_stamp"
+  Name="dummy_name"
+  >
+  <Testing>
+    <TestList>
+      <Test>./foo</Test>
+    </TestList>
+    <Test Status="failed">
+      <Name>foo</Name>
+      <Path>.</Path>
+      <FullName>./foo</FullName>
+      <FullCommandLine>/tmp/bin/foo</FullCommandLine>
+      <Results>
+        <NamedMeasurement type="text/string" name="Exit Code">
+          <Value>Failed</Value>
+        </NamedMeasurement>
+        <NamedMeasurement type="text/string" name="Exit Value">
+          <Value>1</Value>
+        </NamedMeasurement>
+      </Results>
+    </Test>
+    <ElapsedMinutes>0</ElapsedMinutes>
+  </Testing>
+</Site>