Merge pull request #1367 from AlexsLemonade/dev

[Master] Deploy

.gitignore

@@ -30,12 +30,7 @@ workers/test_volume/*
 !workers/test_volume/raw/TEST/TRANSCRIPTOME_INDEX/aegilops_tauschii_short.gtf.gz
 !workers/test_volume/raw/TEST/NO_OP/test.txt
 
-# Trying to ignore most of a directory tree but not a specific subfolder is tricky:
-# https://stackoverflow.com/questions/5533050/gitignore-exclude-folder-but-include-specific-subfolder
-!workers/test_volume/salmon_tests
-workers/test_volume/salmon_tests/*
-!workers/test_volume/salmon_tests/SRP095529/
-!workers/test_volume/salmon_tests/ZEBRAFISH_INDEX
+workers/test_volume/salmon_tests/
 
 
 # Byte-compiled / optimized / DLL files
@@ -92,6 +87,7 @@ coverage.xml
 # Django stuff:
 *.log
 local_settings.py
+common/version
 
 # Flask stuff:
 instance/
@@ -129,6 +125,9 @@ dr_env/
 # Rope project settings
 .ropeproject
 
+#VS Code
+.vscode
+
 # OSX
 .DS_Store
 ._.DS_Store
@@ -139,4 +138,4 @@ dr_env/
 # emacs backup files
 *~
 
-.vscode
+.vscode

api/data_refinery_api/serializers.py

@@ -210,6 +210,15 @@ class Meta:
 class ComputedFileListSerializer(serializers.ModelSerializer):
     result = ComputationalResultNoFilesSerializer(many=False)
     samples = DetailedExperimentSampleSerializer(many=True)
+    compendia_organism_name = serializers.CharField(source='compendia_organism__name', read_only=True)
+
+    def __init__(self, *args, **kwargs):
+        super(ComputedFileListSerializer, self).__init__(*args, **kwargs)
+        if 'context' in kwargs:
+            # only include the field `download_url` if a valid token is specified
+            # the token lookup happens in the view.
+            if 'token' not in kwargs['context']:
+                self.fields.pop('download_url')
 
     class Meta:
         model = ComputedFile
@@ -223,14 +232,21 @@ class Meta:
                     'is_qc',
                     'is_compendia',
                     'compendia_version',
+                    'compendia_organism_name',
                     'sha1',
                     's3_bucket',
                     's3_key',
                     's3_url',
+                    'download_url',
                     'created_at',
                     'last_modified',
                     'result'
                 )
+        extra_kwargs = {
+            'download_url': {
+                'help_text': 'This will contain an url to download the file. You must send a valid [token](#tag/token) in order to receive this.'
+            }
+        }
 
 ##
 # Samples
@@ -633,6 +649,9 @@ class Meta:
                     'quantile_normalize'
             )
         extra_kwargs = {
+                        'data': {
+                            'required': True,
+                        },
                         'id': {
                             'read_only': True,
                         },

api/data_refinery_api/settings.py

@@ -59,6 +59,7 @@
     'raven.contrib.django.raven_compat',
     'django_elasticsearch_dsl',
     'django_elasticsearch_dsl_drf',
+    'drf_yasg',
 
     # Local
     'data_refinery_common',
@@ -196,17 +197,19 @@
 
 RUNNING_IN_CLOUD = get_env_variable('RUNNING_IN_CLOUD') == "True"
 
+ENVIRONMENT = get_env_variable_gracefully('ENVIRONMENT')
+
 # Elastic Search
 ELASTICSEARCH_DSL = {
     'default': {
-        'hosts': get_env_variable('ELASTICSEARCH_HOST') + ":" + get_env_variable('ELASTICSEARCH_PORT') 
+        'hosts': get_env_variable('ELASTICSEARCH_HOST') + ":" + get_env_variable('ELASTICSEARCH_PORT')
     }
 }
 
 if 'test' in sys.argv:
     ELASTICSEARCH_INDEX_NAMES = {
-            'data_refinery_common.models.documents': 'experiments_test',
-        }
+        'data_refinery_common.models.documents': 'experiments_test',
+    }
 else:
     ELASTICSEARCH_INDEX_NAMES = {
         'data_refinery_common.models.documents': 'experiments',

api/data_refinery_api/urls.py

@@ -1,15 +1,18 @@
+from data_refinery_api.views import ExperimentDocumentView
+from rest_framework.routers import DefaultRouter
 from django.conf.urls import url, include
 from django.conf import settings
 from django.contrib import admin
 from django.urls import include, path
-from rest_framework.documentation import include_docs_urls
 from rest_framework.views import APIView
 from rest_framework.response import Response
 from rest_framework.reverse import reverse
 from rest_framework.urlpatterns import format_suffix_patterns
+from rest_framework import permissions
+from drf_yasg.views import get_schema_view
+from drf_yasg import openapi
 
 from data_refinery_api.views import (
-    SearchAndFilter,
     ExperimentList,
     ExperimentDetail,
     SampleList,
@@ -20,13 +23,14 @@
     SurveyJobList,
     DownloaderJobList,
     ProcessorJobList,
-    ResultsList,
+    ComputationalResultsList,
     ProcessorList,
     Stats,
     CreateDatasetView,
     DatasetView,
-    DatasetStatsView,
+    CreateApiTokenView,
     APITokenView,
+    TranscriptomeIndexList,
     TranscriptomeIndexDetail,
     QNTargetsDetail,
     QNTargetsAvailable,
@@ -42,129 +46,75 @@ class AccessUser:
 if settings.DEBUG:
     admin.site.has_permission = lambda r: setattr(r, 'user', AccessUser()) or True
 
-# This class provides a friendlier root API page.
-class APIRoot(APIView):
-    """
-    Refine.bio API
-
-    This open API provides access to all of the Refine.bio experiment, sample and result information.
-
-    """
-    def get(self, request):
-        return Response({
-            'experiments': reverse('experiments', request=request),
-            'samples': reverse('samples', request=request),
-            'organisms': reverse('organisms', request=request),
-            'platforms': reverse('platforms', request=request),
-            'institutions': reverse('institutions', request=request),
-            'jobs': reverse('jobs', request=request),
-            'stats': reverse('stats', request=request),
-            'dataset': reverse('dataset_root', request=request),
-            'token': reverse('token', request=request),
-            'search': reverse('search', request=request)
-        })
-
-# This class provides a friendlier jobs API page.
-class JobsRoot(APIView):
-    """
-    Jobs!
-    """
-    def get(self, request):
-        return Response({
-            'survey': reverse('survey_jobs', request=request),
-            'downloader': reverse('downloader_jobs', request=request),
-            'processor': reverse('processor_jobs', request=request)
-        })
-
-# This class provides a friendlier Dataset API page.
-class DatasetRoot(APIView):
-    """
-    Use the 'create' endpoint to create a new dataset,
-    then use the returned 'id' field to see and update all the fields:
-
-    `/dataset/id-1234-1234/`
-
-    """
-    def get(self, request):
-        return Response({
-            'create': reverse('create_dataset', request=request)
-        })
-
-##
-# ES
-##
-from django.conf.urls import url, include
-from rest_framework.routers import DefaultRouter
-from data_refinery_api.views import ExperimentDocumentView
+schema_view = get_schema_view(
+    openapi.Info(
+        title="Refine.bio API",
+        default_version='v1',
+        description="""
+refine.bio is a multi-organism collection of genome-wide transcriptome or gene expression data that has been obtained from publicly available repositories and uniformly processed and normalized. refine.bio allows biologists, clinicians, and machine learning researchers to search for experiments from different source repositories all in one place and build custom data sets for their questions of interest.
 
-# TODO:
-# Move this to a custom router 
-# so we can use 'name' 
-# https://www.django-rest-framework.org/api-guide/routers/
-router = DefaultRouter()
-router.register(r'',
-                    ExperimentDocumentView,
-                    base_name='esearch',
-                    )
-router.include_format_suffixes = False
+The swagger-ui view can be found [here](http://api.refine.bio/swagger/).
 
-urlpatterns = [
+The ReDoc view can be found [here](http://api.refine.bio/).
+
+Additional documentation can be found at [docs.refine.bio](http://docs.refine.bio/en/latest/).
 
+### Questions/Feedback?
+
+If you have a question or comment, please [file an issue on GitHub](https://github.com/AlexsLemonade/refinebio/issues) or send us an email at [ccdl@alexslemonade.org](mailto:ccdl@alexslemonade.org).
+        """,
+        terms_of_service="https://www.refine.bio/terms",
+        contact=openapi.Contact(email="ccdl@alexslemonade.org"),
+        license=openapi.License(name="BSD License"),
+    ),
+    public=True,
+    permission_classes=(permissions.AllowAny,),
+)
+
+urlpatterns = [
     # Primary search and filter interface
-    url(r'^search/$', SearchAndFilter.as_view(), name="search"),
+    url(r'^search/$', ExperimentDocumentView.as_view({'get': 'list'}), name="search"),
 
-    # Endpoints / Self-documentation
     url(r'^experiments/$', ExperimentList.as_view(), name="experiments"),
-    url(r'^experiments/(?P<pk>.+)/$', ExperimentDetail.as_view(), name="experiments_detail"),
+    url(r'^experiments/(?P<accession_code>.+)/$', ExperimentDetail.as_view(), name="experiments_detail"),
     url(r'^samples/$', SampleList.as_view(), name="samples"),
-    url(r'^samples/(?P<pk>[0-9]+)/$', SampleDetail.as_view(), name="samples_detail"),
+    url(r'^samples/(?P<accession_code>.+)/$', SampleDetail.as_view(), name="samples_detail"),
     url(r'^organisms/$', OrganismList.as_view(), name="organisms"),
     url(r'^platforms/$', PlatformList.as_view(), name="platforms"),
     url(r'^institutions/$', InstitutionList.as_view(), name="institutions"),
-    url(r'^results/$', ResultsList.as_view(), name="results"),
     url(r'^processors/$', ProcessorList.as_view(), name="processors"),
 
     # Deliverables
-    url(r'^dataset/$', DatasetRoot.as_view(), name="dataset_root"),
-    url(r'^dataset/create/$', CreateDatasetView.as_view(), name="create_dataset"),
-    url(r'^dataset/stats/(?P<id>[0-9a-f-]+)/$', DatasetStatsView.as_view(), name="dataset_stats"),
+    url(r'^dataset/$', CreateDatasetView.as_view(), name="create_dataset"),
     url(r'^dataset/(?P<id>[0-9a-f-]+)/$', DatasetView.as_view(), name="dataset"),
-    url(r'^token/$', APITokenView.as_view(), name="token"),
+    url(r'^token/$', CreateApiTokenView.as_view(), name="token"),
     url(r'^token/(?P<id>[0-9a-f-]+)/$', APITokenView.as_view(), name="token_id"),
 
     # Jobs
-    url(r'^jobs/$', JobsRoot.as_view(), name="jobs"),
     url(r'^jobs/survey/$', SurveyJobList.as_view(), name="survey_jobs"),
     url(r'^jobs/downloader/$', DownloaderJobList.as_view(), name="downloader_jobs"),
     url(r'^jobs/processor/$', ProcessorJobList.as_view(), name="processor_jobs"),
 
     # Dashboard Driver
     url(r'^stats/$', Stats.as_view(), name="stats"),
 
-    # Not-publically listed APIs
     # Transcriptome Indices and QN Targets
-    url(r'^transcriptome_indices', TranscriptomeIndexDetail.as_view(),
-        name="transcriptome-indices"),
-    url(r'^compendia', CompendiaDetail.as_view(),
-        name="compendia"),
-    url(r'^qn_targets_available', QNTargetsAvailable.as_view(),
-        name="qn-targets-available"),
-    url(r'^qn_targets', QNTargetsDetail.as_view(),
-        name="qn-targets"),
-    url(r'^computed_files', ComputedFilesList.as_view(),
-        name="computed-files"),
+    url(r'^transcriptome_indices/$', TranscriptomeIndexList.as_view(), name="transcriptome-indices"),
+    url(r'^transcriptome_indices/(?P<organism_name>.+)$', TranscriptomeIndexDetail.as_view(), name="transcriptome-indices-read"),
+    url(r'^qn_targets/$', QNTargetsAvailable.as_view(), name="qn-targets-available"),
+    url(r'^qn_targets/(?P<organism_name>.+)$', QNTargetsDetail.as_view(), name="qn-targets"),
+
+    url(r'^compendia/$', CompendiaDetail.as_view(), name="compendia"),
+    url(r'^computed_files/$', ComputedFilesList.as_view(), name="computed-files"),
+    url(r'^computational_results/$', ComputationalResultsList.as_view(), name="results"),
 
     # Admin
     url(r'^admin/', admin.site.urls),
 
-    # Core API schema docs
-    url(r'^docs/', include_docs_urls(title='Refine.bio API'), name="docs_schema"),
-
-    # ES
-    url(r'^es/', include(router.urls), name="esearch"),
-
-    # Root
-    url(r'^$', APIRoot.as_view(), name="api_root"),
+    # api docs
+    url(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
+    # Let's put redoc at the api root
+    url(r'^$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc')
 ]
 
 # This adds support explicitly typed endpoints such that appending '.json' returns that application type.