diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 704136eb4cf27..37190632b44bb 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -28,8 +28,8 @@ plugins {
 
 dependencies {
   compile "junit:junit:${versions.junit}"
-  compile "org.hamcrest:hamcrest-core:${versions.hamcrest}"
-  compile "org.hamcrest:hamcrest-library:${versions.hamcrest}"
+  compile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+  compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
 
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
@@ -81,7 +81,7 @@ tasks.dependencyLicenses.enabled = false
 tasks.dependenciesInfo.enabled = false
 
 tasks.thirdPartyAudit.excludes = [
-        //commons-logging optional dependencies
+        // commons-logging optional dependencies
         'org.apache.avalon.framework.logger.Logger',
         'org.apache.log.Hierarchy',
         'org.apache.log.Logger',
@@ -89,7 +89,15 @@ tasks.thirdPartyAudit.excludes = [
         'org.apache.log4j.Level',
         'org.apache.log4j.Logger',
         'org.apache.log4j.Priority',
-        //commons-logging provided dependencies
+        // commons-logging provided dependencies
         'javax.servlet.ServletContextEvent',
-        'javax.servlet.ServletContextListener'
+        'javax.servlet.ServletContextListener',
+        // from randomized testing
+        'org.apache.tools.ant.BuildException',
+        'org.apache.tools.ant.DirectoryScanner',
+        'org.apache.tools.ant.Task',
+        'org.apache.tools.ant.types.FileSet',
+        'org.easymock.EasyMock',
+        'org.easymock.IArgumentMatcher',
+        'org.jmock.core.Constraint'
 ]
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
index df5e8cf995d86..3aada7837d8ae 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.packaging.test;
 
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
 import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.Archives;
 import org.elasticsearch.packaging.util.Platforms;
@@ -27,9 +28,6 @@
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.FixMethodOrder;
-import org.junit.Test;
-import org.junit.runners.MethodSorters;
 
 import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.Installation;
@@ -67,8 +65,8 @@
  * Tests that apply to the archive distributions (tar, zip). To add a case for a distribution, subclass and
  * override {@link ArchiveTestCase#distribution()}. These tests should be the same across all archive distributions
  */
-@FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public abstract class ArchiveTestCase {
+@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
+public abstract class ArchiveTestCase extends PackagingTestCase {
 
     private static Installation installation;
 
@@ -86,13 +84,11 @@ public void onlyCompatibleDistributions() {
         assumeTrue("only compatible distributions", distribution().packaging.compatible);
     }
 
-    @Test
     public void test10Install() {
         installation = installArchive(distribution());
         verifyArchiveInstallation(installation, distribution());
     }
 
-    @Test
     public void test20PluginsListWithNoPlugins() {
         assumeThat(installation, is(notNullValue()));
 
@@ -103,7 +99,6 @@ public void test20PluginsListWithNoPlugins() {
         assertThat(r.stdout, isEmptyString());
     }
 
-    @Test
     public void test30AbortWhenJavaMissing() {
         assumeThat(installation, is(notNullValue()));
 
@@ -146,7 +141,6 @@ public void test30AbortWhenJavaMissing() {
         });
     }
 
-    @Test
     public void test40CreateKeystoreManually() {
         assumeThat(installation, is(notNullValue()));
 
@@ -180,7 +174,6 @@ public void test40CreateKeystoreManually() {
         });
     }
 
-    @Test
     public void test50StartAndStop() throws IOException {
         assumeThat(installation, is(notNullValue()));
 
@@ -198,7 +191,6 @@ public void test50StartAndStop() throws IOException {
         Archives.stopElasticsearch(installation);
     }
 
-    @Test
     public void test60AutoCreateKeystore() {
         assumeThat(installation, is(notNullValue()));
 
@@ -218,7 +210,6 @@ public void test60AutoCreateKeystore() {
         });
     }
 
-    @Test
     public void test70CustomPathConfAndJvmOptions() throws IOException {
         assumeThat(installation, is(notNullValue()));
 
@@ -268,7 +259,6 @@ public void test70CustomPathConfAndJvmOptions() throws IOException {
         }
     }
 
-    @Test
     public void test80RelativePathConf() throws IOException {
         assumeThat(installation, is(notNullValue()));
 
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackagingTestCase.java
new file mode 100644
index 0000000000000..77644b70f28e1
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackagingTestCase.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import com.carrotsearch.randomizedtesting.JUnit3MethodProvider;
+import com.carrotsearch.randomizedtesting.RandomizedRunner;
+import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+@RunWith(RandomizedRunner.class)
+@TestMethodProviders({
+    JUnit3MethodProvider.class
+})
+/**
+ * Class that all packaging test cases should inherit from. This makes working with the packaging tests more similar to what we're
+ * familiar with from {@link org.elasticsearch.test.ESTestCase} without having to apply its behavior that's not relevant here
+ */
+public abstract class PackagingTestCase {
+
+    protected final Log logger = LogFactory.getLog(getClass());
+
+    @Rule
+    public final TestName testNameRule = new TestName();
+
+    @Before
+    public void logTestNameBefore() {
+        logger.info("[" + testNameRule.getMethodName() + "]: before test");
+    }
+}