From 6799e2fa5a425d5c8e45e04664b09e6ec67b912f Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Thu, 9 May 2019 17:55:53 -0700 Subject: [PATCH] Try running screenshot tests on Travis (#782) * Try running screenshot tests on Travis * Standardize domain in integration tests * Allow images to not match that well * Turn back on all other tests --- js/__tests__/baseline.integration-test.js | 37 ++++++++++++++--------- package.json | 4 +-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/js/__tests__/baseline.integration-test.js b/js/__tests__/baseline.integration-test.js index 910d7d3a..38dafff7 100644 --- a/js/__tests__/baseline.integration-test.js +++ b/js/__tests__/baseline.integration-test.js @@ -2,60 +2,69 @@ const { toMatchImageSnapshot } = require("jest-image-snapshot"); expect.extend({ toMatchImageSnapshot }); +const DOMAIN = "http://localhost:8080"; +const snapshotOptions = { + // There are some font rendering issues which prevent us from pushing this lower right now. + // We could setup some tests which don't render text and set the threshold lower. + // Ideally we can resolve the font rendering issue. + failureThreshold: "0.01", + failureThresholdType: "percent", +}; + // Hack to ensure changing the hash causes a page reload beforeEach(async () => page.goto(`http://example.com`)); test("should render the default skin", async () => { - await page.goto(`http://localhost:8080/#{"disableMarquee":true}`); - expect(await page.screenshot()).toMatchImageSnapshot(); + await page.goto(`${DOMAIN}/#{"disableMarquee":true}`); + expect(await page.screenshot()).toMatchImageSnapshot(snapshotOptions); }); test("can 'pose' for a screenshot", async () => { - await page.goto(`http://localhost:8080/?screenshot=1`); - expect(await page.screenshot()).toMatchImageSnapshot(); + await page.goto(`${DOMAIN}/?screenshot=1`); + expect(await page.screenshot()).toMatchImageSnapshot(snapshotOptions); }); test("can load a skin via the query params", async () => { await page.goto( // If this test starts to fail, check that the cache-bust location of the skin has not changed. - `http://localhost:8080/?skinUrl=_/skins/MacOSXAqua1-5-88dbd4e043795c98625462a908a2d965.wsz#{"disableMarquee":true}` + `${DOMAIN}/?skinUrl=_/skins/MacOSXAqua1-5-88dbd4e043795c98625462a908a2d965.wsz#{"disableMarquee":true}` ); await page.evaluate(() => window.__webamp.skinIsLoaded()); - expect(await page.screenshot()).toMatchImageSnapshot(); + expect(await page.screenshot()).toMatchImageSnapshot(snapshotOptions); }); test("should render the Topaz skin", async () => { - await page.goto(`http://localhost:8080/#{"disableMarquee":true}`); + await page.goto(`${DOMAIN}/#{"disableMarquee":true}`); await expect(page).toUploadFile( "#webamp-file-input", "./skins/TopazAmp1-2.wsz" ); await page.evaluate(() => window.__webamp.skinIsLoaded()); - expect(await page.screenshot()).toMatchImageSnapshot(); + expect(await page.screenshot()).toMatchImageSnapshot(snapshotOptions); }); test("should render a skin that defines transparent regions", async () => { - await page.goto(`http://localhost:8080/#{"disableMarquee":true}`); + await page.goto(`${DOMAIN}/#{"disableMarquee":true}`); await expect(page).toUploadFile( "#webamp-file-input", "./skins/Green-Dimension-V2.wsz" ); await page.evaluate(() => window.__webamp.skinIsLoaded()); - expect(await page.screenshot()).toMatchImageSnapshot(); + expect(await page.screenshot()).toMatchImageSnapshot(snapshotOptions); }); test("uses the volume spirtes as a fallback when balance spirtes are missing", async () => { - await page.goto(`http://localhost:8080/#{"disableMarquee":true}`); + await page.goto(`${DOMAIN}/#{"disableMarquee":true}`); await expect(page).toUploadFile( "#webamp-file-input", "./skins/AmigaPPC-dark.wsz" ); await page.evaluate(() => window.__webamp.skinIsLoaded()); - expect(await page.screenshot()).toMatchImageSnapshot(); + expect(await page.screenshot()).toMatchImageSnapshot(snapshotOptions); }); test("pads empty space in the marquee with the space character", async () => { - await page.goto(`http://localhost:8080/#{"disableMarquee":true}`); + await page.goto(`${DOMAIN}/#{"disableMarquee":true}`); // This skin has noticeable light blue where it expects the marquee to always cover. await expect(page).toUploadFile( "#webamp-file-input", @@ -65,5 +74,5 @@ test("pads empty space in the marquee with the space character", async () => { await page.evaluate(() => window.__webamp.store.dispatch({ type: "SET_FOCUS", input: "balance" }) ); - expect(await page.screenshot()).toMatchImageSnapshot(); + expect(await page.screenshot()).toMatchImageSnapshot(snapshotOptions); }); diff --git a/package.json b/package.json index 9282aecd..6539466d 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,10 @@ "stats": "webpack --config=demo/config/webpack.prod.js --json > stats.json", "stats-library": "webpack --config=config/webpack.library.js --json > stats.json", "test": "jest --projects config/jest.unit.js", - "travis-tests": "npm run test && npm run test && npm run type-check && npm run build && npm run build-library && npm run weight", + "travis-tests": "npm run test && npm run test && npm run type-check && npm run build && npm run build-library && npm run weight && npm run integration-tests", "tdd": "jest --projects config/jest.unit.js --watch", "format": "prettier --write \"**/*.{js,ts,tsx,d.ts,css}\"", - "integration-tests": "npm run build -- --display=errors-only && jest --projects config/jest.integration.js", + "integration-tests": "npm run build -- --display=errors-only && jest --projects config/jest.integration.js --runInBand", "build-skin": "rm skins/base-2.91.wsz && cd skins/base-2.91 && zip -x .* -x 'Skining Updates.txt' -r ../base-2.91.wsz .", "build-skin-png": "rm skins/base-2.91-png.wsz && cd skins/base-2.91-png && zip -x .* -x 'Skining Updates.txt' -r ../base-2.91-png.wsz .", "compile-skin": "node scripts/compileSkin.js > css/base-skin.css",