diff --git a/.circleci/config.yml b/.circleci/config.yml index 4529a6946..47fd098af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -329,10 +329,9 @@ workflows: branches: ignore: /.*/ tags: - # From CirlceCI Docs: - # If both only and ignore are specified the only is considered before ignore. - only: /v.*/ - ignore: /v.*-hotfix/ + # Only run deploys on versions tagged with /v.*/, unless regex + # lookahead finds `-hotfix` at the end of the version tag + only: /^v((?!-hotfix$).)*$/ hotfix-deploy: jobs: diff --git a/api/data_refinery_api/serializers.py b/api/data_refinery_api/serializers.py index 32e0c7faf..e446fbd0a 100644 --- a/api/data_refinery_api/serializers.py +++ b/api/data_refinery_api/serializers.py @@ -604,6 +604,7 @@ class DatasetSerializer(serializers.ModelSerializer): start = serializers.NullBooleanField(required=False) experiments = DatasetDetailsExperimentSerializer(source='get_experiments', many=True, read_only=True) organism_samples = serializers.SerializerMethodField(read_only=True) + worker_version = serializers.SerializerMethodField(read_only=True) def __init__(self, *args, **kwargs): super(DatasetSerializer, self).__init__(*args, **kwargs) @@ -616,6 +617,7 @@ def __init__(self, *args, **kwargs): if 'details' not in kwargs['context']['request'].query_params: self.fields.pop('experiments') self.fields.pop('organism_samples') + self.fields.pop('worker_version') # only include the field `download_url` if a valid token is specified # the token lookup happens in the view. @@ -646,7 +648,8 @@ class Meta: 'experiments', 'organism_samples', 'download_url', - 'quantile_normalize' + 'quantile_normalize', + 'worker_version' ) extra_kwargs = { 'data': { @@ -693,6 +696,10 @@ class Meta: }, 'download_url': { 'read_only': True, + }, + 'worker_version': { + 'read_only': True, + 'help_text': 'Returns the latest version of refine.bio that was used to build this dataset.' } } @@ -722,6 +729,13 @@ def get_organism_samples(self, obj): return result + def get_worker_version(self, obj): + processor_jobs = obj.processor_jobs.order_by('-created_at').values_list('worker_version', flat=True) + if processor_jobs: + return processor_jobs[0] + else: + return None + class APITokenSerializer(serializers.ModelSerializer): class Meta: diff --git a/common/data_refinery_common/migrations/0023_fix_platform_names_spaces.py b/common/data_refinery_common/migrations/0023_fix_platform_names_spaces.py new file mode 100644 index 000000000..679641f2d --- /dev/null +++ b/common/data_refinery_common/migrations/0023_fix_platform_names_spaces.py @@ -0,0 +1,22 @@ + +from django.db import migrations +from data_refinery_common.utils import get_supported_rnaseq_platforms +from django.core.paginator import Paginator + +def make_sample_platform_names_readable(apps, schema_editor): + Sample = apps.get_model('data_refinery_common', 'Sample') + + for platform_name in get_supported_rnaseq_platforms(): + munged_platform = platform_name.replace(' ', '') + Sample.objects.all().filter(platform_name=munged_platform).update(platform_name=platform_name) + print("Updating platform name from \'%s\' to \'%s\'" % (munged_platform, platform_name)) + +class Migration(migrations.Migration): + + dependencies = [ + ('data_refinery_common', '0022_auto_20190607_1505'), + ] + + operations = [ + migrations.RunPython(make_sample_platform_names_readable) + ] diff --git a/common/data_refinery_common/migrations/0024_dataset_processor_jobs.py b/common/data_refinery_common/migrations/0024_dataset_processor_jobs.py new file mode 100644 index 000000000..f80886fdd --- /dev/null +++ b/common/data_refinery_common/migrations/0024_dataset_processor_jobs.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.8 on 2019-07-26 18:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_refinery_common', '0023_fix_platform_names_spaces'), + ] + + operations = [ + migrations.AddField( + model_name='dataset', + name='processor_jobs', + field=models.ManyToManyField(through='data_refinery_common.ProcessorJobDatasetAssociation', to='data_refinery_common.ProcessorJob'), + ), + ] diff --git a/common/data_refinery_common/models/models.py b/common/data_refinery_common/models/models.py index ae8ab0159..c3b959ffe 100644 --- a/common/data_refinery_common/models/models.py +++ b/common/data_refinery_common/models/models.py @@ -1108,6 +1108,8 @@ class Dataset(models.Model): is_processed = models.BooleanField(default=False) # Result has been made is_available = models.BooleanField(default=False) # Result is ready for delivery + processor_jobs = models.ManyToManyField('data_refinery_common.ProcessorJob', through='ProcessorJobDataSetAssociation') + # Fail handling success = models.NullBooleanField(null=True) failure_reason = models.TextField() diff --git a/foreman/data_refinery_foreman/foreman/management/commands/create_missing_downloader_jobs.py b/foreman/data_refinery_foreman/foreman/management/commands/create_missing_downloader_jobs.py index e9caa38e8..f33457587 100644 --- a/foreman/data_refinery_foreman/foreman/management/commands/create_missing_downloader_jobs.py +++ b/foreman/data_refinery_foreman/foreman/management/commands/create_missing_downloader_jobs.py @@ -55,7 +55,3 @@ def handle(self, *args, **options): break page = paginator.page(page.next_page_number()) - - # 2000 samples queued up every five minutes should be fast - # enough and also not thrash the DB. - time.sleep(60 * 5)