MineRL update

.buildkite/pipeline.yml

@@ -0,0 +1,17 @@
+steps:
+ - label: build minerl
+   env:
+     MINERL_DATA_ROOT: /tmp/minerl-data
+     JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+   command:
+    - add-apt-repository -y ppa:openjdk-r/ppa
+    - apt-get -y update
+    - apt-get -y install openjdk-8-jdk
+    - export PATH=$$JAVA_HOME/bin:$$PATH
+    - java -version
+    - ./download_minerl_data.sh
+    - ./test_build_deploy.sh
+
+   timeout_in_minutes: 15
+   agents:
+     queue: rl-cpu-v4

.env

@@ -0,0 +1 @@
+PYTHONPATH=.:$PYTHONPATH

.github/workflows/wheels3.yml

@@ -0,0 +1,139 @@
+name: Build wheels 3
+
+on: [push] #, pull_request]
+
+# todo: Support windows
+# todo: make new docker image which includes java
+# todo: get s3 upload to work.
+# todo: support sdist!
+
+jobs:
+  build_wheels_linux:
+    name:  ${{ matrix.os }} ${{ matrix.pver }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-18.04] #, windows-latest, macos-latest]
+        pver: ["cp35*x86_64", "cp36*x86_64", "cp37*x86_64", "cp38*x86_64"] # Todo add 39.
+    env:
+      # build using the manylinux2014 image
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+      CIBW_BUILD: "${{ matrix.pver }}" 
+      CIBW_SKIP: cp27-* pp27-*
+      CIBW_BEFORE_BUILD_LINUX: |
+        yum install -y  \
+          java-1.8.0-openjdk-devel 
+        # Why the fuck is there no headless-devel package like on debian!
+
+        # Add the Java dependencies to the
+        echo "$(/opt/python/cp37-cp37m/bin/pip show auditwheel | grep "Location:" | cut -d ':' -f2)/auditwheel/policy/policy.json" | xargs \
+          python -c "import sys; import json; import os; fname = sys.argv[1]; fread = open(fname, 'r'); obj = json.load(fread); fread.close(); print(obj[-1]); [o['lib_whitelist'].extend('libXcursor.so.1, libXfixes.so.3, libXi.so.6, libXrandr.so.2, libXtst.so.6, libXxf86vm.so.1, libawt.so, libawt_xawt.so, libjava.so, libjawt.so, libjvm.so, libncurses.so.5, libtinfo.so.5, libverify.so'.split(', ')) for o in obj]; print(obj[-1]['lib_whitelist']); fwrite = open(fname, 'w'); json.dump(obj, fwrite); fwrite.close(); fread = open(fname, 'r'); print(json.load(fread)[-1])"
+
+
+      CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel -v repair -w {dest_dir} {wheel}"
+      CIBW_BEFORE_BUILD: pip install .
+
+    steps:
+      - uses: actions/checkout@v2
+        with: 
+          submodules: 'recursive'
+
+      - name: Declare some variables
+        id: vars
+        shell: bash
+        run: |
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.7'
+
+      - uses: actions/setup-java@v1.3.0 # This is for MacOS and Windows
+        with:
+          java-version: '8' # The JDK version to make available on the path.
+          java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+          architecture: x64 # (x64 or x86) - defaults to x64
+
+      # TODO Run testing here! (Or we could run it in a different workflow.)
+
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==1.5.2
+
+
+      - name: Build wheels
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+      - uses: shallwefootball/s3-upload-action@master
+        with:
+          aws_key_id: ${{ secrets.AWS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+          aws_bucket: ${{ secrets.AWS_MINERL_BUCKET }}
+          source_dir: './wheelhouse'
+          destination_dir:  builds/${{ steps.vars.outputs.branch }}/${{ steps.vars.outputs.sha_short }}/
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+
+
+      - name: Declare some variables
+        id: vars
+        shell: bash
+        run: |
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.7'
+
+      - name: Build sdist
+        run: python setup.py sdist
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: dist/*.tar.gz
+
+      - uses: shallwefootball/s3-upload-action@master
+        with:
+          aws_key_id: ${{ secrets.AWS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+          aws_bucket: ${{ secrets.AWS_MINERL_BUCKET }}
+          source_dir: './wheelhouse'
+          destination_dir:  builds/${{ steps.vars.outputs.branch }}/${{ steps.vars.outputs.sha_short }}/
+
+
+
+  # # upload_pypi:
+  # #   needs: [build_wheels, build_sdist]
+  # #   runs-on: ubuntu-latest
+  # #   # upload to PyPI on every tag starting with 'v'
+  # #   # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+  # #   # alternatively, to publish when a GitHub Release is created, use the following rule:
+  # #   if: github.event_name == 'release' && github.event.action == 'published'
+  # #   steps:
+  # #     - uses: actions/download-artifact@v2
+  # #       with:
+  # #         name: artifact
+  # #         path: dist
+
+  # #     - uses: pypa/gh-action-pypi-publish@master
+  # #       with:
+  # #         user: __token__
+  # #         password: ${{ secrets.pypi_password }}
+  # #         To test: repository_url: https://test.pypi.org/legacy/

.gitignore

@@ -1,3 +1,4 @@
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -236,3 +237,54 @@ minerl_data/pipeline/tests_data/
 
 
 t/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+venv/*
+*logs*
+
+# Distribution / packaging
+.Python
+build/
+_build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+build/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# IDE files
+.idea/
+.vscode
+
+*.DS_store
+*.iml
+*Minecraft.ipr
+*Minecraft.iws
+
+*/logs/
+
+tests/logs/
+.vscode/settings.json
+*/performance
+
+
+# Jenkins
+jenkins_home/
+
+#eclipse
+Malmo/Minecraft/*.launch
+*minerl_watchers/

MANIFEST.in

@@ -1,4 +1,12 @@
 recursive-include projectname *.py
 include README
 include requirements.txt
-include LICENSE
+include LICENSE
+recursive-include minerl/env/Malmo/Minecraft/build/libs *.jar
+recursive-include minerl/env/Malmo/Minecraft/run/gradle/caches/minecraft *
+recursive-include minerl/env/Malmo/Minecraft/run/gradle/native *
+recursive-include minerl/env/Malmo/Schemas *
+recursive-include minerl/env/missions *.xml
+include minerl/env/Malmo/Minecraft/launchClient.sh
+include minerl/env/Malmo/Minecraft/options.txt
+include minerl/core/mc_constants.json

Malmo/Minecraft/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>Minecraft_</name>
+	<comment>Project Minecraft_ created by Buildship.</comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
+	</natures>
+</projectDescription>

docs/source/conf.py

@@ -89,7 +89,6 @@
 
 from docutils.parsers.rst import Directive
 
-
 class ExecDirective(Directive):
     """Execute the specified python code and insert the output into the document"""
     has_content = True

docs/source/environments/index.rst

@@ -24,12 +24,6 @@ environment!
     :end-before: inclusion-marker-do-not-remove
 
 Basic Environments
-=======================================
-
-.. warning::
-
-    The following basic environments are NOT part of the 2020 MineRL Competition! Feel free to use them for exploration
-    but agents may only be trained on `MineRL competition environments`_!
 
 .. exec::
 
@@ -132,104 +126,5 @@ Basic Environments
         print_actions_for_id(i)
 
 Competition Environments
-=======================================
-
-.. exec::
-
-    def print_actions_for_id(id):
-        import minerl
-        import gym
-        import json
-
-        def print_json(arg):
-            import json
-            arg = {":ref:`{} <{}>`".format(k,k): arg[k] for k in arg}
-            json_obj = json.dumps(arg, sort_keys=True, indent=8,
-                default=lambda o: str(o))
-            json_obj= json_obj[:-1] + "    })"
-
-            print('.. parsed-literal:: \n\n    Dict(%s\n\n\n' % json_obj)
-
-        def prep_space(space):
-            import gym
-            if isinstance(space, gym.spaces.Dict):
-                dct = {}
-                for k in space.spaces:
-                    dct[k] = prep_space(space.spaces[k])
-                return dct
-            else:
-                return space
-
-
-
-        envspec = gym.spec(id)
-
-
-        print("______________")
-        print("{}".format(id))
-        print("______________")
-
-        if 'docstr' in envspec._kwargs:
-            print(envspec._kwargs['docstr'])
-
-
-
-        action_space = prep_space(envspec._kwargs['action_space'])
-        state_space = prep_space(envspec._kwargs['observation_space'])
-
-        print(".......")
-        print("Observation Space")
-        print(".......")
-        print_json(state_space)
-
-
-        print(".......")
-        print("Action Space")
-        print(".......")
-        print_json(action_space)
-
-        print(".......")
-        print("Usage")
-        print(".......")
-
-
-        usage_str = '''.. code-block:: python
-
-            import gym
-            import minerl
-
-            # Run a random agent through the environment
-            env = gym.make("{}") # A {} env
-
-            obs = env.reset()
-            done = False
-
-            while not done:
-                # Take a no-op through the environment.
-                obs, rew, done, _ = env.step(env.action_space.noop())
-                # Do something
-
-            ######################################
-
-            # Sample some data from the dataset!
-            data = minerl.data.make("{}")
-
-            # Iterate through a single epoch using sequences of at most 32 steps
-            for obs, rew, done, act in data.seq_iter(num_epochs=1, batch_size=32):
-                # Do something
-        '''.format(id,id,id)
-        print(usage_str)
-
-
-    ids = ['MineRLTreechopVectorObf-v0',
-            'MineRLNavigateVectorObf-v0',
-            'MineRLNavigateExtremeVectorObf-v0',
-            'MineRLNavigateDenseVectorObf-v0',
-            'MineRLNavigateExtremeDenseVectorObf-v0',
-            'MineRLObtainDiamondVectorObf-v0',
-            'MineRLObtainDiamondDenseVectorObf-v0',
-            'MineRLObtainIronPickaxeVectorObf-v0',
-            'MineRLObtainIronPickaxeDenseVectorObf-v0',]
-
     for i in ids:
         print_actions_for_id(i)

minerl/data/__init__.py

@@ -23,7 +23,7 @@ def make(environment=None , data_dir=None, num_workers=4, worker_batch_size=32,
     # Ensure path is setup
     if data_dir is None and 'MINERL_DATA_ROOT' in os.environ:
         data_dir = os.environ['MINERL_DATA_ROOT']
-    elif data_dir is not None and not os.path.exists(data_dir):
+    if data_dir is not None and not os.path.exists(data_dir):
         if force_download:
             print("Provided data directory does not exist: ", data_dir)
             data_dir = download(data_dir)
@@ -38,7 +38,7 @@ def make(environment=None , data_dir=None, num_workers=4, worker_batch_size=32,
             raise ValueError("No data_dir provided and $MINERL_DATA_ROOT undefined."
                              "Specify force_download=True to download default dataset")
 
-
+    assert data_dir is not None, "data_dir is not provided, and download has failed"
     minerl.data.version.assert_version(data_dir)
 
     d = DataPipeline(