-
Notifications
You must be signed in to change notification settings - Fork 8.2k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Getting flaky tests back in shape for reporting #46076
Changes from 12 commits
e8ccb8e
edb3b7a
b10f8a7
29c7e9b
b04d9e5
0d118e1
68cdaef
ee3d3f7
b9d08c7
841b790
347a814
c2d34a3
b24ab73
ac9dd96
cd1ce2b
382be06
45984d9
e27c04f
8d826d2
7180be9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -371,6 +371,12 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo | |
await browser.pressKeys(browser.keys.ENTER); | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is nice little helper for debugging tests when needed, left for future users |
||
// Pause the browser at a certain place for debugging | ||
// Not meant for usage in CI, only for dev-usage | ||
async pause() { | ||
return browser.pause(); | ||
} | ||
|
||
/** | ||
* Clicks cancel button on modal | ||
* @param overlayWillStay pass in true if your test will show multiple modals in succession | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -199,6 +199,14 @@ export async function BrowserProvider({ getService }: FtrProviderContext) { | |
return await driver.get(url); | ||
} | ||
|
||
/** | ||
* Pauses the execution in the browser, similar to setting a breakpoint for debugging. | ||
* @return {Promise<void>} | ||
*/ | ||
public async pause() { | ||
await driver.executeAsyncScript(`(async () => { debugger; return Promise.resolve(); })()`); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When/If this turns out to be useful, can you add a note in some developer docs about how to use this? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah! Great reminder, thanks! |
||
} | ||
|
||
/** | ||
* Moves the remote environment’s mouse cursor to the specified point {x, y} which is | ||
* offset to browser page top left corner. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -43,6 +43,7 @@ import { Browsers } from './browsers'; | |
|
||
const throttleOption: string = process.env.TEST_THROTTLE_NETWORK as string; | ||
const headlessBrowser: string = process.env.TEST_BROWSER_HEADLESS as string; | ||
const remoteDebug: string = process.env.TEST_REMOTE_DEBUG as string; | ||
const SECOND = 1000; | ||
const MINUTE = 60 * SECOND; | ||
const NO_QUEUE_COMMANDS = ['getLog', 'getStatus', 'newSession', 'quit']; | ||
|
@@ -97,6 +98,10 @@ async function attemptToCreateCommand( | |
// See: https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md | ||
chromeOptions.push('headless', 'disable-gpu'); | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've added this so you can remotely debug (view with devtools) the session as it's running headlessly |
||
if (remoteDebug === '1') { | ||
// Visit chrome://inspect in chrome to remotely view/debug | ||
chromeOptions.push('headless', 'disable-gpu', 'remote-debugging-port=9222'); | ||
} | ||
chromeCapabilities.set('goog:chromeOptions', { | ||
w3c: false, | ||
args: chromeOptions, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -169,10 +169,14 @@ export function TestSubjectsProvider({ getService }: FtrProviderContext) { | |
}); | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Underlying selenium method allows for a timeout, so I added it as an optional param here. |
||
public async getAttribute(selector: string, attribute: string): Promise<string> { | ||
public async getAttribute( | ||
selector: string, | ||
attribute: string, | ||
timeout?: number | ||
): Promise<string> { | ||
return await retry.try(async () => { | ||
log.debug(`TestSubjects.getAttribute(${selector}, ${attribute})`); | ||
const element = await this.find(selector); | ||
const element = await this.find(selector, timeout); | ||
return await element.getAttribute(attribute); | ||
}); | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -37,34 +37,6 @@ export function ReportingPageProvider({ getService, getPageObjects }) { | |
await browser.setWindowSize(1600, 850); | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No longer need theses now that we aren't tab-switching |
||
async getUrlOfTab(tabIndex) { | ||
return await retry.try(async () => { | ||
log.debug(`reportingPage.getUrlOfTab(${tabIndex}`); | ||
const handles = await browser.getAllWindowHandles(); | ||
log.debug(`Switching to window ${handles[tabIndex]}`); | ||
await browser.switchToWindow(handles[tabIndex]); | ||
|
||
const url = await browser.getCurrentUrl(); | ||
if (!url || url === 'about:blank') { | ||
throw new Error('url is blank'); | ||
} | ||
|
||
await browser.switchToWindow(handles[0]); | ||
return url; | ||
}); | ||
} | ||
|
||
async closeTab(tabIndex) { | ||
return await retry.try(async () => { | ||
log.debug(`reportingPage.closeTab(${tabIndex}`); | ||
const handles = await browser.getAllWindowHandles(); | ||
log.debug(`Switching to window ${handles[tabIndex]}`); | ||
await browser.switchToWindow(handles[tabIndex]); | ||
await browser.closeCurrentWindow(); | ||
await browser.switchToWindow(handles[0]); | ||
}); | ||
} | ||
|
||
async forceSharedItemsContainerSize({ width }) { | ||
await browser.execute(` | ||
var el = document.querySelector('[data-shared-items-container]'); | ||
|
@@ -73,6 +45,16 @@ export function ReportingPageProvider({ getService, getPageObjects }) { | |
`); | ||
} | ||
|
||
async getReportURL(timeout) { | ||
log.debug('getReportURL'); | ||
|
||
const url = await testSubjects.getAttribute('downloadCompletedReportButton', 'href', timeout); | ||
|
||
log.debug(`getReportURL got url: ${url}`); | ||
|
||
return url; | ||
} | ||
|
||
async removeForceSharedItemsContainerSize() { | ||
await browser.execute(` | ||
var el = document.querySelector('[data-shared-items-container]'); | ||
|
@@ -81,9 +63,8 @@ export function ReportingPageProvider({ getService, getPageObjects }) { | |
`); | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No longer use the response body, so we just return the raw response from here on out. |
||
getRawPdfReportData(url) { | ||
log.debug(`getRawPdfReportData for ${url}`); | ||
let data = []; // List of Buffer objects | ||
getResponse(url) { | ||
log.debug(`getResponse for ${url}`); | ||
const auth = config.get('servers.elasticsearch.auth'); | ||
const headers = { | ||
Authorization: `Basic ${Buffer.from(auth).toString('base64')}`, | ||
|
@@ -100,13 +81,7 @@ export function ReportingPageProvider({ getService, getPageObjects }) { | |
headers, | ||
}, | ||
res => { | ||
res.on('data', function(chunk) { | ||
data.push(chunk); | ||
}); | ||
res.on('end', function() { | ||
data = Buffer.concat(data); | ||
resolve(data); | ||
}); | ||
resolve(res); | ||
} | ||
) | ||
.on('error', e => { | ||
|
@@ -115,6 +90,18 @@ export function ReportingPageProvider({ getService, getPageObjects }) { | |
}); | ||
} | ||
|
||
async getRawPdfReportData(url) { | ||
const data = []; // List of Buffer objects | ||
log.debug(`getRawPdfReportData for ${url}`); | ||
|
||
return new Promise(async (resolve, reject) => { | ||
const response = await this.getResponse(url).catch(reject); | ||
|
||
response.on('data', chunk => data.push(chunk)); | ||
response.on('end', () => resolve(Buffer.concat(data))); | ||
}); | ||
} | ||
|
||
async openCsvReportingPanel() { | ||
log.debug('openCsvReportingPanel'); | ||
await PageObjects.share.openShareMenuItem('CSV Reports'); | ||
|
@@ -130,10 +117,6 @@ export function ReportingPageProvider({ getService, getPageObjects }) { | |
await PageObjects.share.openShareMenuItem('PNG Reports'); | ||
} | ||
|
||
async clickDownloadReportButton(timeout) { | ||
await testSubjects.click('downloadCompletedReportButton', timeout); | ||
} | ||
|
||
async clearToastNotifications() { | ||
const toasts = await testSubjects.findAll('toastCloseButton'); | ||
await Promise.all(toasts.map(async t => await t.click())); | ||
|
@@ -175,7 +158,9 @@ export function ReportingPageProvider({ getService, getPageObjects }) { | |
|
||
async setTimepickerInDataRange() { | ||
log.debug('Reporting:setTimepickerInDataRange'); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was causing Discover to return quite a few items, hitting our byte limit. I shortened the timeframe to hopefully improve test speed + get our tests running again. |
||
await PageObjects.timePicker.setDefaultAbsoluteRange(); | ||
const fromTime = 'Sep 19, 2015 @ 06:31:44.000'; | ||
const toTime = 'Sep 19, 2015 @ 18:01:44.000'; | ||
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); | ||
} | ||
|
||
async setTimepickerInNoDataRange() { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,8 +9,8 @@ Reporting tests have their own top level test folder because: | |
### Running the tests | ||
|
||
There is more information on running x-pack tests here: https://github.com/elastic/kibana/blob/master/x-pack/README.md#running-functional-tests. Similar to running the API tests, you need to specify a reporting configuration file. Reporting currently has two configuration files you can point to: | ||
- test/reporting/configs/chromium_api.js | ||
- test/reporting/configs/chromium_functional.js | ||
- test/reporting/configs/chromium_api.js | ||
- test/reporting/configs/chromium_functional.js | ||
|
||
The `api` versions hit the reporting api and ensure report generation completes successfully, but does not verify the output of the reports. This is done in the `functional` test versions, which does a snapshot comparison of the generated URL against a baseline to determine success. | ||
|
||
|
@@ -33,10 +33,6 @@ node scripts/functional_tests_server.js --config test/reporting/configs/[test_co | |
node ../scripts/functional_test_runner.js --config test/reporting/configs/[test_config_name_here].js | ||
``` | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No longer needed |
||
**Prerequisites** | ||
The reporting functional tests use [pdf-image](https://www.npmjs.com/package/pdf-image) to convert PDF's pages to png files for image comparisons between generated reports and baseline reports. | ||
pdf-image requires the system commands `convert`, `gs`, and `pdfinfo` to function. Those can be set up by running the following. | ||
|
||
```sh | ||
//OSX | ||
brew install imagemagick ghostscript poppler | ||
|
@@ -99,7 +95,7 @@ node ../scripts/es_archiver.js --es-url http://elastic:changeme@localhost:9200 l | |
7. Generate some reporting URLs | ||
- Use a mixture of Visualize, Discover (CSV), Dashboard | ||
- Can view the current test coverage by checkout out [api/generation_urls.js](https://github.com/elastic/kibana/blob/master/x-pack/test/reporting/api/generation_urls.js). You can use different ones for better test coverage (e.g. different dashboards, different visualizations). | ||
- Don’t generate urls from huge dashboards since this is time consuming. | ||
- Don’t generate urls from huge dashboards since this is time consuming. | ||
- Use dashboards that have time saved with them if you wish to have data included. | ||
8. Save these reporting urls. | ||
9. Navigate back to the main branch via `git checkout master`. Then create, or work off your branch as usual to add the extra test coverage. | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
dep no longer used