Skip to content

Commit

Permalink
ui-test: isolate each tests
Browse files Browse the repository at this point in the history
Isolate each test with a different `extest` process.
  • Loading branch information
goneri committed Dec 2, 2024
1 parent 31e8834 commit 162cf58
Show file tree
Hide file tree
Showing 21 changed files with 1,746 additions and 1,704 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ jobs:
out/log
out/test-resources/settings/logs
out/userdata/logs
out/test-resources/screenshots
if-no-files-found: ignore
retention-days: 90

Expand Down
2 changes: 1 addition & 1 deletion .mocharc.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
extension: ["ts"]
require: ts-node/register
package: "./package.json"
timeout: 10001 # default is 2000
timeout: 30003 # default is 2000
# most UI tests are >22s due to our current wait times and we do not want
# red slow marker to distract us until we sort that part yet. Red is expected
# to appear on unexpected long tests, not on an expected duration.
Expand Down
4 changes: 1 addition & 3 deletions test/mockLightspeedServer/openUrl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ export function openUrl(url: string) {
errorText += data;
});
child.stderr.on("end", function () {
if (errorText) {
throw new Error(errorText);
}
console.log(`error: ${errorText}`);
});
}
10 changes: 5 additions & 5 deletions test/mockLightspeedServer/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,7 @@ export const logger = winston.createLogger({
],
});

let url = new URL("http://127.0.0.1:3000");
// Do not try to use envvars on macos -- ref: https://github.com/microsoft/vscode/issues/204005
if (process.platform !== "darwin" && process.env.TEST_LIGHTSPEED_URL) {
url = new URL(process.env.TEST_LIGHTSPEED_URL);
}
const url = new URL("http://127.0.0.1:3000");

export function permissionDeniedCanApplyForTrial(): {
code: string;
Expand Down Expand Up @@ -141,6 +137,10 @@ export default class Server {
res.status(200).send();
});

app.get("/__debug__/kill", () => {
process.exit(0);
});

app.listen(parseInt(url.port), url.hostname, () => {
logger.info(`Listening on port ${url.port} at ${url.hostname}`);
});
Expand Down
1 change: 1 addition & 0 deletions test/testFixtures/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
"ansibleServer.trace.server": "verbose",
"ansible.lightspeed.enabled": false,
"ansible.lightspeed.suggestions.enabled": false,
"ansible.lightspeed.URL": "https://c.ai.ansible.redhat.com",

"extensions.autoUpdate": false,
"extensions.autoCheckUpdates": false,
Expand Down
27 changes: 0 additions & 27 deletions test/ui-test/allTestsSuite.ts

This file was deleted.

153 changes: 77 additions & 76 deletions test/ui-test/contentCreatorUiTest.ts
Original file line number Diff line number Diff line change
@@ -1,92 +1,93 @@
import { By, EditorView, WebElement, Workbench } from "vscode-extension-tester";
import { getWebviewByLocator, sleep } from "./uiTestHelper";
import { By, EditorView, WebElement } from "vscode-extension-tester";
import {
getWebviewByLocator,
sleep,
workbenchExecuteCommand,
} from "./uiTestHelper";
import { config, expect } from "chai";

config.truncateThreshold = 0;
export function contentCreatorUiTest(): void {
describe("Test Ansible playbook project scaffolding", () => {
let workbench: Workbench;
let createButton: WebElement;
let editorView: EditorView;

before(async () => {
workbench = new Workbench();
editorView = new EditorView();
if (editorView) {
await editorView.closeAllEditors();
}
});
describe("Test Ansible playbook project scaffolding", () => {
let createButton: WebElement;
let editorView: EditorView;

async function testWebViewElements(
command: string,
editorTitle: string,
namespaceName: string,
collectionName: string,
) {
await workbench.executeCommand(command);
await sleep(4000);
before(async () => {
editorView = new EditorView();
if (editorView) {
await editorView.closeAllEditors();
}
});

await new EditorView().openEditor(editorTitle);
const webview = await getWebviewByLocator(
By.xpath("//vscode-text-field[@id='namespace-name']"),
);
async function testWebViewElements(
command: string,
editorTitle: string,
namespaceName: string,
collectionName: string,
) {
await workbenchExecuteCommand(command);
await sleep(4000);

const namespaceTextField = await webview.findWebElement(
By.xpath("//vscode-text-field[@id='namespace-name']"),
);
expect(namespaceTextField, "namespaceTextField should not be undefined")
.not.to.be.undefined;
await namespaceTextField.sendKeys(namespaceName);
await new EditorView().openEditor(editorTitle);
const webview = await getWebviewByLocator(
By.xpath("//vscode-text-field[@id='namespace-name']"),
);

const collectionTextField = await webview.findWebElement(
By.xpath("//vscode-text-field[@id='collection-name']"),
);
expect(collectionTextField, "collectionTextField should not be undefined")
.not.to.be.undefined;
await collectionTextField.sendKeys(collectionName);
const namespaceTextField = await webview.findWebElement(
By.xpath("//vscode-text-field[@id='namespace-name']"),
);
expect(namespaceTextField, "namespaceTextField should not be undefined").not
.to.be.undefined;
await namespaceTextField.sendKeys(namespaceName);

const overwriteCheckbox = await webview.findWebElement(
By.xpath("//vscode-checkbox[@id='overwrite-checkbox']"),
);
expect(overwriteCheckbox, "overwriteCheckbox should not be undefined").not
.to.be.undefined;
await overwriteCheckbox.click();
const collectionTextField = await webview.findWebElement(
By.xpath("//vscode-text-field[@id='collection-name']"),
);
expect(collectionTextField, "collectionTextField should not be undefined")
.not.to.be.undefined;
await collectionTextField.sendKeys(collectionName);

createButton = await webview.findWebElement(
By.xpath("//vscode-button[@id='create-button']"),
);
expect(createButton, "createButton should not be undefined").not.to.be
.undefined;
const overwriteCheckbox = await webview.findWebElement(
By.xpath("//vscode-checkbox[@id='overwrite-checkbox']"),
);
expect(overwriteCheckbox, "overwriteCheckbox should not be undefined").not
.to.be.undefined;
await overwriteCheckbox.click();

expect(
await createButton.isEnabled(),
"Create button should be enabled now",
).to.be.true;
createButton = await webview.findWebElement(
By.xpath("//vscode-button[@id='create-button']"),
);
expect(createButton, "createButton should not be undefined").not.to.be
.undefined;

await createButton.click();
await webview.switchBack();
editorView = new EditorView();
if (editorView) {
await editorView.closeAllEditors();
}
expect(
await createButton.isEnabled(),
"Create button should be enabled now",
).to.be.true;

await createButton.click();
await webview.switchBack();
editorView = new EditorView();
if (editorView) {
await editorView.closeAllEditors();
}
}

it("Check create-ansible-project webview elements", async () => {
await testWebViewElements(
"Ansible: Create New Playbook Project",
"Create Ansible project",
"test_namespace",
"test_collection",
);
});
it("Check create-ansible-project webview elements", async () => {
await testWebViewElements(
"Ansible: Create New Playbook Project",
"Create Ansible project",
"test_namespace",
"test_collection",
);
});

it("Check create-ansible-collection webview elements", async () => {
await testWebViewElements(
"Ansible: Create New Collection",
"Create Ansible collection",
"test_namespace",
"test_collection",
);
});
it("Check create-ansible-collection webview elements", async () => {
await testWebViewElements(
"Ansible: Create New Collection",
"Create Ansible collection",
"test_namespace",
"test_collection",
);
});
}
});
58 changes: 28 additions & 30 deletions test/ui-test/extensionUITest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,37 +9,35 @@ import {
const WAIT_TIME = 10000;

config.truncateThreshold = 0;
export function extensionUIAssetsTest(): void {
describe("Verify base assets are available after installation", () => {
let view: ViewControl;
let sideBar: SideBarView;
describe("Verify base assets are available after installation", () => {
let view: ViewControl;
let sideBar: SideBarView;

before(async function () {
this.timeout(WAIT_TIME);
view = (await new ActivityBar().getViewControl(
"Extensions",
)) as ViewControl;
sideBar = await view.openView();
});
before(async function () {
this.timeout(WAIT_TIME);
view = (await new ActivityBar().getViewControl(
"Extensions",
)) as ViewControl;
sideBar = await view.openView();
});

it("VSCode Ansible extension is installed", async function () {
// Execute only when code coverage is not enabled.
// When code coverage is enabled, the extension is not installed.
if (!process.env.COVERAGE) {
this.retries(3);
this.timeout(20000); // even 18s failed
const section = await sideBar.getContent().getSection("Installed");
const item = await section.findItem("@installed Ansible");
expect(item, "Failed to find Ansible extension").not.undefined;
expect(await item?.getText()).to.contain("Ansible language support");
} else {
this.skip();
}
});
it("VSCode Ansible extension is installed", async function () {
// Execute only when code coverage is not enabled.
// When code coverage is enabled, the extension is not installed.
if (!process.env.COVERAGE) {
this.retries(3);
this.timeout(20000); // even 18s failed
const section = await sideBar.getContent().getSection("Installed");
const item = await section.findItem("@installed Ansible");
expect(item, "Failed to find Ansible extension").not.undefined;
expect(await item?.getText()).to.contain("Ansible language support");
} else {
this.skip();
}
});

after(async function () {
this.timeout(8000);
await new Workbench().executeCommand("Clear Extensions Search Results");
});
after(async function () {
this.timeout(8000);
await new Workbench().executeCommand("Clear Extensions Search Results");
});
}
});
Loading

0 comments on commit 162cf58

Please sign in to comment.