(WIP) Tie-breaking extensibility

kge/config-default.yaml

@@ -330,6 +330,22 @@ eval:
   # Type of evaluation (entity_ranking only at the moment)
   type: entity_ranking
 
+  # How to handle cases ties between the correct answer and other answers, e.g.,
+  #  Query: (s, p, ?).
+  #  Answers and score: a:10, b:10, c:10, d:11, e:9
+  #  Correct: 'a'.
+  #
+  # Possible options are:
+  # - worst_rank:        Use the highest rank of all answers that have the same
+  #                      score as the correct answer. In example: 4.
+  # - best_rank:         Use the lowest rank of all answers that have the same
+  #                      score as the correct answer (competition scoring). In
+  #                      example: 2. DO NOT USE THIS OPTION, it leads to
+  #                      misleading evaluation results.
+  # - rounded_mean_rank: Average between worst and best rank, rounded up
+  #                      (rounded fractional ranking). In example: 3.
+  tie_handling: rounded_mean_rank
+
   # Compute Hits@K for these choices of K
   hits_at_k_s: [1, 3, 10, 50, 100, 200, 300, 400, 500, 1000]
 
@@ -359,7 +375,6 @@ eval:
   # Other options
   pin_memory: False
 
-
 # Configuration options for model validation/selection during training. Applied
 # in addition to the options set under "eval" above.
 valid:

kge/job/entity_ranking.py

@@ -13,6 +13,11 @@ class EntityRankingJob(EvaluationJob):
 
     def __init__(self, config: Config, dataset: Dataset, parent_job, model):
         super().__init__(config, dataset, parent_job, model)
+        self.config.check(
+            "eval.tie_handling",
+            ["rounded_mean_rank", "best_rank", "worst_rank"],
+        )
+        self.tie_handling = self.config.get("eval.tie_handling")
         self.is_prepared = False
 
         if self.__class__ == EntityRankingJob:
@@ -43,7 +48,6 @@ def _prepare(self):
             num_workers=self.config.get("eval.num_workers"),
             pin_memory=self.config.get("eval.pin_memory"),
         )
-
         # let the model add some hooks, if it wants to do so
         self.model.prepare_job(self)
         self.is_prepared = True
@@ -527,8 +531,7 @@ def _get_ranks_and_num_ties(
         num_ties = torch.sum(scores == true_scores.view(-1, 1), dim=1, dtype=torch.long)
         return rank, num_ties
 
-    @staticmethod
-    def _get_ranks(rank: torch.Tensor, num_ties: torch.Tensor) -> torch.Tensor:
+    def _get_ranks(self, rank: torch.Tensor, num_ties: torch.Tensor) -> torch.Tensor:
         """Calculates the final rank from (minimum) rank and number of ties.
 
         :param rank: batch_size x 1 tensor with number of scores greater than the one of
@@ -540,8 +543,15 @@ def _get_ranks(rank: torch.Tensor, num_ties: torch.Tensor) -> torch.Tensor:
         :return: batch_size x 1 tensor of ranks
 
         """
-        ranks = rank + num_ties // 2
-        return ranks
+
+        if self.tie_handling == "rounded_mean_rank":
+            return rank + num_ties // 2
+        elif self.tie_handling == "best_rank":
+            return rank
+        elif self.tie_handling == "worst_rank":
+            return rank + num_ties - 1
+        else:
+            raise NotImplementedError
 
     def _compute_metrics(self, rank_hist, suffix=""):
         """Computes desired matrix from rank histogram"""