Include UI tests for blocks with Appium

.circleci/config.yml

@@ -6,15 +6,45 @@ commands:
       - restore_cache:
           name: Restore Yarn Cache
           keys:
-            - yarn-{{ checksum "yarn.lock" }}
+            - yarn-v2-{{ arch }}-{{ checksum "yarn.lock" }}
       - run:
           name: Yarn Install
-          command: yarn install --frozen-lockfile
+          command: YARN_CACHE_FOLDER=~/.cache/yarn yarn install --frozen-lockfile
       - save_cache:
           name: Save Yarn Cache
-          key: yarn-{{ checksum "yarn.lock" }}
+          key: yarn-v2-{{ arch }}-{{ checksum "yarn.lock" }}
           paths:
             - ~/.cache/yarn
+  build-android:
+    steps:
+      - run:
+          name: Change initial HTML file
+          command: |
+            cp ./bin/tmp/initial-device-tests-html.js ./src/app/initial-html.js
+      - run:
+          name: Bundle Debug android
+          command: yarn bundle:android:test
+      - run:
+          name: Gradle assemble debug android apk
+          command: |
+            cd android
+            ./gradlew clean
+            ./gradlew assembleDebug
+  build-ios:
+    steps:
+    - run:
+        name: Change initial HTML file
+        command: |
+          cp ./bin/tmp/initial-device-tests-html.js ./src/app/initial-html.js
+    - run:
+        name: Bundle iOS
+        command: |
+          yarn bundle:ios:test
+    - run:
+        name: Generate .app file
+        command:  |
+          set +e
+          yarn react-native run-ios --configuration Release
 
 jobs:
   checks:
@@ -47,6 +77,87 @@ jobs:
       - run:
           name: Run Checks
           command: bin/ci-checks-js.sh
+  android-device-checks:
+    docker:
+    - image: circleci/android:api-27-node8-alpha
+    steps:
+      - checkout
+      - run:
+          name: Checkout Gutenberg
+          command: |
+            git submodule init
+            git submodule update --recursive
+      - yarn-install
+      - run:
+          name: Accept Licenses
+          command: |
+            yes | sdkmanager --licenses > /dev/null
+            sleep 10
+            sdkmanager --update
+          background: true
+      - run:
+          name: Set Environment Variables
+          command: |
+            echo 'export TEST_RN_PLATFORM=android' >> $BASH_ENV
+            echo 'export TEST_ENV=sauce' >> $BASH_ENV
+      - build-android
+      - run:
+          name: Upload apk to sauce labs
+          command: |
+            curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg.apk?overwrite=true --data-binary @./android/app/build/outputs/apk/debug/app-debug.apk
+      - run:
+          name: Run Device Tests
+          command: yarn device-tests
+  ios-device-checks:
+    macos:
+      xcode: "10.0"
+    steps:
+    - checkout
+    - run:
+        name: Checkout Gutenberg
+        command: |
+          git submodule init
+          git submodule update --recursive
+    - run: yarn clean:install
+    - run:
+        name: Set Environment Variables
+        command: |
+          echo 'export TEST_RN_PLATFORM=ios' >> $BASH_ENV
+          echo 'export TEST_ENV=sauce' >> $BASH_ENV
+    - restore_cache:
+        name: Restore Dependencies Cache
+        keys:
+        - dependencies-{{ checksum "react-native-aztec/ios/Cartfile.resolved" }}-{{
+          checksum "yarn.lock" }}
+        - dependencies-{{ checksum "react-native-aztec/ios/Cartfile.resolved" }}
+        - dependencies-
+    - run:
+        name: Yarn preios
+        command: yarn preios
+    - save_cache:
+        name: Save Dependencies Cache
+        key: dependencies-{{ checksum "react-native-aztec/ios/Cartfile.resolved" }}-{{
+          checksum "yarn.lock" }}
+        paths:
+        - react-native-aztec/ios/Carthage
+        - ~/.rncache
+    - build-ios
+    - run:
+        name: Zip up .app file
+        command: |
+          cp ./ios/main.jsbundle ./ios/build/Gutenberg/Build/Products/Release-iphonesimulator/gutenberg.app/main.jsbundle
+          cd ./ios/build/Gutenberg/Build/Products/Release-iphonesimulator/
+          zip -r ./Gutenberg.app.zip ./gutenberg.app
+          mv ./Gutenberg.app.zip ../../../../../../Gutenberg.app.zip
+          cd ../../../../../../
+    - run:
+        name: Upload .app to sauce labs
+        command: |
+          curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg.app.zip?overwrite=true --data-binary @./Gutenberg.app.zip
+    - run:
+        name: Run Device Tests
+        command: |
+          yarn device-tests
 
 workflows:
   gutenberg-mobile:
@@ -62,3 +173,7 @@ workflows:
           name: Test Android
           platform: android
           check-tests: true
+      - ios-device-checks:
+          name: Test iOS on Device
+      - android-device-checks:
+          name: Test Android on Device

.gitignore

@@ -23,6 +23,7 @@ yarn-error.log*
 *.apk
 *.ap_
 .gradle/
+android/app/src/main/assets/
 
 # files for the dex VM
 *.dex
@@ -95,4 +96,8 @@ buck-out/
 !.vscode/extensions.json
 
 *.pot
-bin/wp-cli.phar
+
+# e2e output log
+appium-out.log
+
+bin/wp-cli.phar

README.md

@@ -107,6 +107,23 @@ Then, open `chrome://inspect` in Chrome to attach the debugger (look into the "R
 
 This project is set up to use [jest](https://facebook.github.io/jest/) for tests. You can configure whatever testing strategy you like, but jest works out of the box. Create test files in directories called `__tests__` or with the `.test.js` extension to have the files loaded by jest. See an example test [here](https://github.com/wordpress-mobile/gutenberg-mobile/blob/develop/src/app/App.test.js). The [jest documentation](https://facebook.github.io/jest/docs/en/getting-started.html) is also a wonderful resource, as is the [React Native testing tutorial](https://facebook.github.io/jest/docs/en/tutorial-react-native.html).
 
+## UI Tests
+
+This repository uses appium to run UI tests. The tests live in `__device-tests__` and are written using Appium to run tests against simulators and real devices. To run these you'll need to check off a few things: 
+
+* For now you'll need run `yarn start`, and then either `yarn ios` or `yarn android` at least once before trying to run the tests on the respective platform
+* [Appium cli](https://github.com/appium/appium/blob/master/docs/en/about-appium/getting-started.md) installed and available globally, I'd also recommend using [appium doctor](https://github.com/appium/appium-doctor) to ensure all of appium's dependencies are good to go. You don't have to worry about starting the server yourself, the tests handle starting the server on port 4728, just be sure that the port is free or feel free to change the port number in the test file. 
+* For iOS a simulator should automatically launch but for Android you'll need to have an emulator fired up and running.
+
+After those are checked off to run the UI tests on ios run 
+
+`yarn test:ui:ios`
+
+and for android run, 
+
+`yarn test:ui:android`
+
+Alternatively if you're experiencing problems that seem to be related to the tests starting the appium server, for example errors that say `Connection Refused`, `Connection Reset` or `The requested environment is not available`. Sorry about that this is still a WIP, you can manually start the appium server via [appium desktop](https://github.com/appium/appium-desktop) or the cli, then change the port number in the tests while commenting out the `beforeAll` and `afterAll` block. 
 
 ## Static analysis and code style
 

__device-tests__/blocks/block-interaction.js

@@ -0,0 +1,73 @@
+/** @flow
+ * @format */
+
+/**
+ * External dependencies
+ */
+import wd from 'wd';
+/**
+ * Internal dependencies
+ */
+import { isAndroid } from '../helpers/utils';
+
+// Common code across used to interact with all blocks
+export default class BlockInteraction {
+	driver: wd.PromiseChainWebdriver;
+	accessibilityIdKey: string;
+	name: string;
+	blockName: string;
+	element: wd.PromiseChainWebdriver.Element;
+	accessibilityId: string;
+	accessibilityIdXPathAttrib: string;
+	static index = 0;
+
+	constructor( driver: wd.PromiseChainWebdriver, name: string = 'Unsupported Block' ) {
+		this.driver = driver;
+		this.accessibilityIdKey = 'name';
+		this.accessibilityIdXPathAttrib = 'name';
+		this.name = name;
+
+		if ( isAndroid() ) {
+			this.accessibilityIdXPathAttrib = 'content-desc';
+			this.accessibilityIdKey = 'contentDescription';
+		}
+	}
+
+	// Each subclass must include a method to do the following:
+	// * Initialise the element and accessibilityId(By calling this.setupElement in most cases)
+	// * Initialise any elements specific to interactions with that block
+	async setup() {
+		throw 'Unimplemented setup function for this block';
+	}
+
+	async getAttribute( attributeName: string ) {
+		return await this.element.getAttribute( attributeName );
+	}
+
+	// Finds the wd element for new block that was added and sets the element attribute
+	// and accessibilityId attributes on this object
+	async setupElement( blockName: string, blocks: Set<string> ) {
+		await this.driver.sleep( 2000 );
+		const blockLocator = `block-${ BlockInteraction.index }-${ blockName }`;
+		this.element = await this.driver.elementByAccessibilityId( blockLocator );
+		this.accessibilityId = await this.getAttribute( this.accessibilityIdKey );
+
+		BlockInteraction.index += 1;
+		return blocks;
+	}
+
+	// attempts to type a string to a given element, need for this stems from
+	// https://github.com/appium/appium/issues/12285#issuecomment-471872239
+	// https://github.com/facebook/WebDriverAgent/issues/1084
+	async typeString( element: wd.PromiseChainWebdriver.Element, str: string ) {
+		await element.clear();
+		if ( isAndroid() ) {
+			return await element.type( str );
+		}
+		// iOS: Problem with Appium type function requiring me to do a little hacking to get it work,
+		// as a result typing on iOS will be slower
+		for ( let i = 0; i < str.length; i++ ) {
+			await element.type( str.charAt( i ) );
+		}
+	}
+}

__device-tests__/blocks/paragraph-block-interaction.js

@@ -0,0 +1,56 @@
+/** @flow
+ * @format */
+
+/**
+ * Internal dependencies
+ */
+import BlockInteraction from './block-interaction';
+/**
+ * External dependencies
+ */
+import wd from 'wd';
+import { isAndroid } from '../helpers/utils';
+
+/**
+ * WordPress dependencies
+ */
+import { __ } from '@wordpress/i18n';
+
+export default class ParagraphBlockInteraction extends BlockInteraction {
+	// FLow complaining about type annotation on Set class here but Set<string>(); doesn't resolve
+	// $FlowFixMe
+	static blocks = new Set();
+	textViewElement: wd.PromiseChainWebdriver.Element;
+
+	constructor( driver: wd.PromiseChainWebdriver ) {
+		super( driver, __( 'Paragraph' ) );
+		this.driver = driver;
+		this.blockName = 'core/paragraph';
+	}
+
+	// gets the TextView wd element for this paragraph block and sets it to
+	// the textViewElement attribute for this object
+	async setupTextView() {
+		await this.driver.sleep( 2000 );
+		let textViewElement = 'XCUIElementTypeTextView';
+		if ( isAndroid() ) {
+			textViewElement = 'android.widget.EditText';
+		}
+		const blockLocator = `//*[@${ this.accessibilityIdXPathAttrib }="${ this.accessibilityId }"]//${ textViewElement }`;
+		this.textViewElement = await this.driver.elementByXPath( blockLocator );
+	}
+
+	async setup() {
+		await this.setupElement( this.blockName, ParagraphBlockInteraction.blocks );
+		await this.setupTextView();
+	}
+
+	async sendText( str: string ) {
+		return await this.typeString( this.textViewElement, str );
+	}
+
+	async getText() {
+		const text = await this.textViewElement.text();
+		return text.toString().trim();
+	}
+}

__device-tests__/gutenberg-editor.test.js

@@ -0,0 +1,74 @@
+/**
+ * @format
+ * */
+
+/**
+ * Internal dependencies
+ */
+import EditorPage from './pages/editor-page';
+import ParagraphBlockInteraction from './blocks/paragraph-block-interaction';
+import { setupAppium, setupDriver, isLocalEnvironment, timer } from './helpers/utils';
+
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 120000;
+
+describe( 'Gutenberg Editor tests', () => {
+	let appium;
+	let driver;
+	let editorPage;
+	let allPassed = true;
+
+	// Use reporter for setting status for saucelabs Job
+	if ( ! isLocalEnvironment() ) {
+		const reporter = {
+			specDone: async ( result ) => {
+				allPassed = allPassed && result.status !== 'failed';
+			},
+		};
+
+		jasmine.getEnv().addReporter( reporter );
+	}
+
+	beforeAll( async () => {
+		if ( isLocalEnvironment() ) {
+			appium = await setupAppium();
+		}
+
+		driver = await setupDriver();
+	} );
+
+	it( 'should be able to see visual editor', async () => {
+		editorPage = new EditorPage( driver );
+		await editorPage.expect();
+	} );
+
+	it( 'should be able to add a new Paragraph block', async () => {
+		let paragraphBlockInteraction = new ParagraphBlockInteraction( driver );
+		paragraphBlockInteraction = await editorPage.addNewBlock( paragraphBlockInteraction );
+		await paragraphBlockInteraction.sendText( 'Hello Gutenberg!' );
+		await timer( 3000 );
+		expect( await paragraphBlockInteraction.getText() ).toBe( 'Hello Gutenberg!' );
+	} );
+
+	afterAll( async () => {
+		if ( isLocalEnvironment() ) {
+			if ( driver === undefined ) {
+				if ( appium !== undefined ) {
+					await appium.kill( 'SIGINT' );
+				}
+				return;
+			}
+
+			await driver.quit();
+			await appium.kill( 'SIGINT' );
+		} else {
+			if ( driver === undefined ) {
+				if ( appium !== undefined ) {
+					await appium.kill( 'SIGINT' );
+				}
+				return;
+			}
+			driver.sauceJobStatus( allPassed );
+			await driver.quit();
+		}
+	} );
+} );