Merge pull request #1483 from codalab/develop

Merge develop into master

src/apps/api/serializers/competitions.py

@@ -179,7 +179,7 @@ def get_used_submissions_per_day(self, obj):
                 # Get all submissions which are not failed and belongs to this user for this phase
                 qs = obj.submissions.filter(owner=user, parent__isnull=True).exclude(status='Failed')
                 # Count submissions made today
-                daily_submission_count = qs.filter(created_when__day=now().day).count()
+                daily_submission_count = qs.filter(created_when__date=now().date()).count()
                 return daily_submission_count
         return 0
 

src/apps/api/views/analytics.py

@@ -173,7 +173,7 @@ def storage_usage_history(request):
         raise PermissionDenied(detail="Admin only")
 
     storage_usage_history = {}
-    last_storage_usage_history_snapshot = StorageUsageHistory.objects.order_by("at_date").last()
+    last_storage_usage_history_snapshot = StorageUsageHistory.objects.order_by("-at_date").first()
     if last_storage_usage_history_snapshot:
         start_date = request.query_params.get("start_date", (datetime.datetime.today() - datetime.timedelta(weeks=4)).strftime("%Y-%m-%d"))
         end_date = request.query_params.get("end_date", datetime.datetime.today().strftime("%Y-%m-%d"))
@@ -209,7 +209,7 @@ def competitions_usage(request):
         raise PermissionDenied(detail="Admin only")
 
     competitions_usage = {}
-    last_competition_storage_snapshot = CompetitionStorageDataPoint.objects.order_by("at_date").last()
+    last_competition_storage_snapshot = CompetitionStorageDataPoint.objects.order_by("-at_date").first()
     if last_competition_storage_snapshot:
         start_date = request.query_params.get("start_date", (datetime.datetime.today() - datetime.timedelta(weeks=4)).strftime("%Y-%m-%d"))
         end_date = request.query_params.get("end_date", datetime.datetime.today().strftime("%Y-%m-%d"))
@@ -237,7 +237,7 @@ def competitions_usage(request):
             }
 
     response = {
-        "last_storage_calculation_date": last_competition_storage_snapshot.at_date.isoformat() if last_competition_storage_snapshot else None,
+        "last_storage_calculation_date": last_competition_storage_snapshot.created_at.isoformat() if last_competition_storage_snapshot else None,
         "competitions_usage": competitions_usage
     }
 
@@ -253,7 +253,7 @@ def users_usage(request):
         raise PermissionDenied(detail="Admin only")
 
     users_usage = {}
-    last_user_storage_snapshot = UserStorageDataPoint.objects.order_by("at_date").last()
+    last_user_storage_snapshot = UserStorageDataPoint.objects.order_by("-at_date").first()
     if last_user_storage_snapshot:
         start_date = request.query_params.get("start_date", (datetime.datetime.today() - datetime.timedelta(weeks=4)).strftime("%Y-%m-%d"))
         end_date = request.query_params.get("end_date", datetime.datetime.today().strftime("%Y-%m-%d"))
@@ -281,7 +281,7 @@ def users_usage(request):
             }
 
     response = {
-        "last_storage_calculation_date": last_user_storage_snapshot.at_date.isoformat() if last_user_storage_snapshot else None,
+        "last_storage_calculation_date": last_user_storage_snapshot.created_at.isoformat() if last_user_storage_snapshot else None,
         "users_usage": users_usage
     }
 

src/apps/competitions/models.py

@@ -3,6 +3,7 @@
 import os
 import io
 
+import botocore.exceptions
 from django.conf import settings
 from django.contrib.sites.models import Site
 from django.contrib.postgres.fields import JSONField
@@ -11,6 +12,7 @@
 from django.db.models import Q
 from django.urls import reverse
 from django.utils.timezone import now
+from decimal import Decimal
 
 from celery_config import app, app_for_vhost
 from chahub.models import ChaHubSaveMixin
@@ -466,6 +468,11 @@ def save(self, *args, **kwargs):
                 # file returns a None size, can't divide None / 1024
                 # -1 indicates an error
                 self.file_size = -1
+            except botocore.exceptions.ClientError:
+                # file might not exist in the storage
+                logger.warning(f"The data_file of SubmissionDetails id={self.id} does not exist in the storage. data_file and file_size has been cleared")
+                self.file_size = Decimal(0)
+                self.data_file = None
         return super().save(*args, **kwargs)
 
 
@@ -586,7 +593,12 @@ def save(self, ignore_submission_limit=False, **kwargs):
                 except TypeError:
                     # file returns a None size, can't divide None / 1024
                     # -1 indicates an error
-                    setattr(self, file_size_attr, -1)
+                    setattr(self, file_size_attr, Decimal(-1))
+                except botocore.exceptions.ClientError:
+                    # file might not exist in the storage
+                    logger.warning(f"The {file_path_attr} of Submission id={self.id} does not exist in the storage. {file_path_attr} and {file_size_attr} has been cleared")
+                    setattr(self, file_size_attr, Decimal(0))
+                    setattr(self, file_path_attr, None)
 
         super().save(**kwargs)
 

src/apps/datasets/models.py

@@ -1,18 +1,25 @@
 import uuid
+import botocore
+import logging
 
+import botocore.exceptions
 from django.conf import settings
 from django.contrib.sites.models import Site
 from django.db import models
 from django.db.models import Q
 from django.urls import reverse
 from django.utils.timezone import now
+from decimal import Decimal
 
 from chahub.models import ChaHubSaveMixin
 from utils.data import PathWrapper
 from utils.storage import BundleStorage
 from competitions.models import Competition
 
 
+logger = logging.getLogger()
+
+
 class Data(ChaHubSaveMixin, models.Model):
     """Data models are unqiue based on name + created_by. If no name is given, then there is no uniqueness to enforce"""
 
@@ -73,7 +80,13 @@ def save(self, *args, **kwargs):
             except TypeError:
                 # file returns a None size, can't divide None / 1024
                 # -1 indicates an error
-                self.file_size = -1
+                self.file_size = Decimal(-1)
+            except botocore.exceptions.ClientError:
+                # file might not exist in the storage
+                logger.warning(f"The data_file of Data id={self.id} does not exist in the storage. data_file and file_size has been cleared")
+                self.file_size = Decimal(0)
+                self.data_file = None
+
         if not self.name:
             self.name = f"{self.created_by.username} - {self.type}"
         return super().save(*args, **kwargs)