tail-gen-eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Based on http://nlp.cs.washington.edu/zeroshot/evaluate.py\n",
    "import pandas as pd\n",
    "import os\n",
    "import codecs\n",
    "import re\n",
    "import string\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "PUNCTUATION = set(string.punctuation)\n",
    "\n",
    "import re\n",
    "\n",
    "def remove_latin(text):\n",
    "    return re.sub(r'[^\\x00-\\x7f]',r'', text)\n",
    "\n",
    "def unk_zero_re_eval(test_file, answer_file):\n",
    "    q_aprf = unk_read_results(test_file, answer_file)\n",
    "    return pretify(q_aprf)\n",
    "\n",
    "def unk_read_results(test_set, answer_file):\n",
    "    with codecs.open(test_set, \"r\", \"utf-8\") as fin:\n",
    "        data = [line.strip().split(\"\\t\") for line in fin]\n",
    "    metadata = [x[:4] for x in data]\n",
    "    gold = [set(x[4:]) for x in data]\n",
    "\n",
    "    with codecs.open(answer_file, \"r\", \"utf-8\") as fin:\n",
    "        answers = [line.strip() for line in fin]\n",
    "\n",
    "    new_answers = []\n",
    "    for answer in answers[1:]:\n",
    "        if answer != \"no_answer\":\n",
    "            new_answers.append(answer)\n",
    "        else:\n",
    "            new_answers.append(\"\")\n",
    "\n",
    "    telemetry = []\n",
    "    for m, g, a in zip(metadata, gold, new_answers):\n",
    "        stats = score(g, a)\n",
    "        telemetry.append([m[0], m[1], str(len(g) > 0), stats])\n",
    "    return aprf(telemetry)\n",
    "\n",
    "def parse_no_answers(results):\n",
    "    p_answer = [\n",
    "        a for i, a in sorted([(int(i), a) for i, a in results[0][\"scores\"].items()])\n",
    "    ]\n",
    "    p_no_answer = [\n",
    "        a for i, a in sorted([(int(i), a) for i, a in results[0][\"na\"].items()])\n",
    "    ]\n",
    "\n",
    "    import numpy as np\n",
    "\n",
    "    return [answer > no_answer for answer, no_answer in zip(p_answer, p_no_answer)]\n",
    "\n",
    "\n",
    "def gb(collection, keyfunc):\n",
    "    return [(k, list(g)) for k, g in groupby(sorted(collection, key=keyfunc), keyfunc)]\n",
    "\n",
    "\n",
    "def aprf(g):\n",
    "    tp, tn, sys_pos, real_pos = sum(map(lambda x: x[-1], g))\n",
    "    total = len(g)\n",
    "    # a = float(tp + tn) / total\n",
    "    # nr = tn / float(total - real_pos)\n",
    "    # npr = tn / float(total - sys_pos)\n",
    "    if tp == 0:\n",
    "        p = r = f = 0.0\n",
    "    else:\n",
    "        p = tp / float(sys_pos)\n",
    "        r = tp / float(real_pos)\n",
    "        f = 2 * p * r / (p + r)\n",
    "    # return np.array((a, p, r, f, npr, nr))\n",
    "    return np.array((p, r, f))\n",
    "\n",
    "\n",
    "def score(gold, answer):\n",
    "    if len(gold) > 0:\n",
    "        gold = set.union(*[simplify(g) for g in gold])\n",
    "    answer = simplify(answer)\n",
    "    result = np.zeros(4)\n",
    "    if answer == gold:\n",
    "        if len(gold) > 0:\n",
    "            result[0] += 1\n",
    "        else:\n",
    "            result[1] += 1\n",
    "    if len(answer) > 0:\n",
    "        result[2] += 1\n",
    "    if len(gold) > 0:\n",
    "        result[3] += 1\n",
    "    return result\n",
    "\n",
    "\n",
    "def simplify(answer):\n",
    "    return set(\n",
    "        \"\".join(c for c in t if c not in PUNCTUATION)\n",
    "        for t in answer.strip().lower().split()\n",
    "    ) - {\"the\", \"a\", \"an\", \"and\", \"\"}\n",
    "\n",
    "\n",
    "def pretify(results):\n",
    "    return \" \\t \".join(\n",
    "        [\n",
    "            \": \".join((k, v))\n",
    "            for k, v in zip(\n",
    "                [\"Precision\", \"Recall\", \"F1\"],\n",
    "                map(lambda r: \"{0:.2f}%\".format(r * 100), results),\n",
    "            )\n",
    "        ]\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import GPT2LMHeadModel, GPT2TokenizerFast\n",
    "\n",
    "device = \"cuda\"\n",
    "model_id = \"gpt2-large\"\n",
    "model = GPT2LMHeadModel.from_pretrained(model_id).to(device)\n",
    "tokenizer = GPT2TokenizerFast.from_pretrained(model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "def compute_perplexity_for_questions(main_path, file):\n",
    "    ppls = []\n",
    "    df = pd.read_csv(os.path.join(main_path, file), sep=',')\n",
    "    questions = df[\"question_predictions\"].tolist()\n",
    "    for question in questions:\n",
    "        encodings = tokenizer(question, return_tensors=\"pt\")\n",
    "        input_ids = encodings.input_ids.to(device)\n",
    "        b_sz, length = input_ids.size()\n",
    "        target_ids = input_ids.clone()\n",
    "        with torch.no_grad():\n",
    "            outputs = model(input_ids, labels=target_ids)\n",
    "            neg_log_likelihood = outputs[0]\n",
    "            ppl = torch.exp(neg_log_likelihood)\n",
    "        ppls.append(ppl)\n",
    "    ppl = torch.stack(ppls).mean()\n",
    "    return ppl\n",
    "\n",
    "def gold_compute_perplexity_for_questions(main_path, file):\n",
    "    ppls = []\n",
    "    df = pd.read_csv(os.path.join(main_path, file), sep=',')\n",
    "    inputs = df[\"input_str\"].tolist()\n",
    "    for inp in inputs:\n",
    "        question = inp.split(\"context:\")[0].replace(\"question:\", \"\").strip()\n",
    "        encodings = tokenizer(question, return_tensors=\"pt\")\n",
    "        input_ids = encodings.input_ids.to(device)\n",
    "        b_sz, length = input_ids.size()\n",
    "        target_ids = input_ids.clone()\n",
    "        with torch.no_grad():\n",
    "            outputs = model(input_ids, labels=target_ids)\n",
    "            neg_log_likelihood = outputs[0]\n",
    "            ppl = torch.exp(neg_log_likelihood)\n",
    "        ppls.append(ppl)\n",
    "    ppl = torch.stack(ppls).mean()\n",
    "    return ppl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_the_prediction_files(main_path, list_of_files):\n",
    "    for file in list_of_files:\n",
    "        df = pd.read_csv(os.path.join(main_path, file), sep=',')\n",
    "        df[\"predictions_str\"].to_csv(os.path.join(\"/tmp/\", file), sep='\\t', header=True, index=False)\n",
    "\n",
    "def unk_eval_the_prediction_files(list_of_files, gold_file):\n",
    "    scores = {}\n",
    "    scores_list = []\n",
    "    precision_list = []\n",
    "    recall_list = []\n",
    "    for file in list_of_files:\n",
    "        score = unk_zero_re_eval(gold_file, os.path.join(\"/tmp/\", file))\n",
    "        arr = score.split()\n",
    "        f1_score = float(arr[-1][0:-1])\n",
    "        precision = float(arr[1][0:-1])\n",
    "        recall = float(arr[3][0:-1])\n",
    "        scores[f1_score] = file\n",
    "        scores_list.append(f1_score)\n",
    "        precision_list.append(precision)\n",
    "        recall_list.append(recall)\n",
    "\n",
    "    f1s = np.array(scores_list)\n",
    "    precisions = np.array(precision_list)\n",
    "    recalls = np.array(recall_list)\n",
    "    max_f1 = max(scores.keys())\n",
    "    return scores[max_f1],  max_f1, f1s, scores, precisions, recalls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = {}\n",
    "for fold_i in range(1, 11, 1):\n",
    "    results[fold_i] = {'mml-pgg-off-sim': {},\n",
    "                       'mml-pgg-on-sim': {},\n",
    "                       'mml-mml-off-sim': {},\n",
    "                       'mml-mml-on-sim': {}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mml-pgg-off-sim 1 mml-pgg-off-sim.fold.1.dev.predictions.step.500.csv 49.45\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 1 mml-pgg-on-sim.fold.1.dev.predictions.step.300.csv 53.57\n",
      "\n",
      "\n",
      "mml-mml-off-sim 1 mml-mml-off-sim.fold.1.dev.predictions.step.500.csv 48.79\n",
      "\n",
      "\n",
      "mml-mml-on-sim 1 mml-mml-on-sim.fold.1.dev.predictions.step.8900.csv 50.56\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 2 mml-pgg-off-sim.fold.2.dev.predictions.step.9300.csv 64.03\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 2 mml-pgg-on-sim.dev.predictions.fold.2.step.9200.csv 62.0\n",
      "\n",
      "\n",
      "mml-mml-off-sim 2 mml-mml-off-sim.dev.predictions.fold.2.step.9300.csv 64.78\n",
      "\n",
      "\n",
      "mml-mml-on-sim 2 mml-mml-on-sim.dev.predictions.fold.2.step.9400.csv 62.34\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 3 mml-pgg-off-sim.fold.3.dev.predictions.step.5300.csv 58.42\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 3 mml-pgg-on-sim.dev.predictions.fold.3.step.300.csv 57.72\n",
      "\n",
      "\n",
      "mml-mml-off-sim 3 mml-mml-off-sim.dev.predictions.fold.3.step.3800.csv 59.03\n",
      "\n",
      "\n",
      "mml-mml-on-sim 3 mml-mml-on-sim.dev.predictions.fold.3.step.2000.csv 61.22\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 4 mml-pgg-off-sim.fold.4.dev.predictions.step.1100.csv 67.0\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 4 mml-pgg-on-sim.dev.predictions.fold.4.step.500.csv 65.9\n",
      "\n",
      "\n",
      "mml-mml-off-sim 4 mml-mml-off-sim.dev.predictions.fold.4.step.2300.csv 66.22\n",
      "\n",
      "\n",
      "mml-mml-on-sim 4 mml-mml-on-sim.dev.predictions.fold.4.step.4200.csv 65.58\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 5 mml-pgg-off-sim.fold.5.dev.predictions.step.6400.csv 70.31\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 5 mml-pgg-on-sim.dev.predictions.fold.5.step.12600.csv 67.48\n",
      "\n",
      "\n",
      "mml-mml-off-sim 5 mml-mml-off-sim.dev.predictions.fold.5.step.3000.csv 64.89\n",
      "\n",
      "\n",
      "mml-mml-on-sim 5 mml-mml-on-sim.dev.predictions.fold.5.step.12800.csv 68.98\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 6 mml-pgg-off-sim.fold.6.dev.predictions.step.700.csv 66.1\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 6 mml-pgg-on-sim.dev.predictions.fold.6.step.300.csv 68.18\n",
      "\n",
      "\n",
      "mml-mml-off-sim 6 mml-mml-off-sim.dev.predictions.fold.6.step.300.csv 67.46\n",
      "\n",
      "\n",
      "mml-mml-on-sim 6 mml-mml-on-sim.dev.predictions.fold.6.step.2200.csv 66.56\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 7 mml-pgg-off-sim.fold.7.dev.predictions.step.14900.csv 61.15\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 7 mml-pgg-on-sim.dev.predictions.fold.7.step.9800.csv 64.96\n",
      "\n",
      "\n",
      "mml-mml-off-sim 7 mml-mml-off-sim.dev.predictions.fold.7.step.13400.csv 59.66\n",
      "\n",
      "\n",
      "mml-mml-on-sim 7 mml-mml-on-sim.dev.predictions.fold.7.step.16600.csv 61.81\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 8 mml-pgg-off-sim.fold.8.dev.predictions.step.3800.csv 63.41\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 8 mml-pgg-on-sim.dev.predictions.fold.8.step.17000.csv 63.89\n",
      "\n",
      "\n",
      "mml-mml-off-sim 8 mml-mml-off-sim.dev.predictions.fold.8.step.7900.csv 63.78\n",
      "\n",
      "\n",
      "mml-mml-on-sim 8 mml-mml-on-sim.dev.predictions.fold.8.step.7200.csv 60.88\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 9 mml-pgg-off-sim.fold.9.dev.predictions.step.2900.csv 64.04\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 9 mml-pgg-on-sim.dev.predictions.fold.9.step.2300.csv 64.04\n",
      "\n",
      "\n",
      "mml-mml-off-sim 9 mml-mml-off-sim.dev.predictions.fold.9.step.4700.csv 59.57\n",
      "\n",
      "\n",
      "mml-mml-on-sim 9 mml-mml-on-sim.dev.predictions.fold.9.step.1900.csv 62.96\n",
      "\n",
      "\n",
      "NEXT\n",
      "mml-pgg-off-sim 10 mml-pgg-off-sim.fold.10.dev.predictions.step.8600.csv 56.5\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 10 mml-pgg-on-sim.dev.predictions.fold.10.step.4300.csv 55.12\n",
      "\n",
      "\n",
      "mml-mml-off-sim 10 mml-mml-off-sim.dev.predictions.fold.10.step.1500.csv 54.51\n",
      "\n",
      "\n",
      "mml-mml-on-sim 10 mml-mml-on-sim.dev.predictions.fold.10.step.5800.csv 56.43\n",
      "\n",
      "\n",
      "NEXT\n"
     ]
    }
   ],
   "source": [
    "# Evaluating the dev predictions on the RE-QA dataset on all folds for the tail entity generation task.\n",
    "folders = [\"mml-pgg-off-sim\", \"mml-pgg-on-sim\", \"mml-mml-off-sim\", \"mml-mml-on-sim\"]\n",
    "\n",
    "for fold_i in range(1, 11, 1):\n",
    "    for folder in folders:\n",
    "        fold_gold_file = \"./zero-shot-extraction/relation_splits/dev.{}\".format(fold_i-1)\n",
    "        fold_path = \"~/reqa-predictions/fold_{}/{}/\".format(fold_i, folder)\n",
    "        if fold_i == 1:\n",
    "            fold_files = [\"{}.fold.{}.dev.predictions.step.{}.csv\".format(folder, fold_i, 100 * i) for i in range(1, 101, 1)]\n",
    "        elif 2 <= fold_i <= 4:\n",
    "            if folder == \"mml-pgg-off-sim\":\n",
    "                fold_files = [\"{}.fold.{}.dev.predictions.step.{}.csv\".format(folder, fold_i, 100 * i) for i in range(1, 101, 1)]\n",
    "            else:\n",
    "                fold_files = [\"{}.dev.predictions.fold.{}.step.{}.csv\".format(folder, fold_i, 100 * i) for i in range(1, 101, 1)]\n",
    "        else:\n",
    "            if folder == \"mml-pgg-off-sim\":\n",
    "                fold_files = [\"{}.fold.{}.dev.predictions.step.{}.csv\".format(folder, fold_i, 100 * i) for i in range(1, 201, 1)]\n",
    "            else:\n",
    "                fold_files = [\"{}.dev.predictions.fold.{}.step.{}.csv\".format(folder, fold_i, 100 * i) for i in range(1, 201, 1)]\n",
    "\n",
    "        preprocess_the_prediction_files(fold_path, fold_files)\n",
    "        max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "        print(folder, fold_i, max_file, max_f1)\n",
    "        print(\"\\n\")\n",
    "        results[fold_i][folder] = max_file\n",
    "    print(\"NEXT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [26.77] [23.52] [31.05]\n",
      "\n",
      "\n",
      "NEXT\n",
      "2 [25.95] [23.88] [28.4]\n",
      "\n",
      "\n",
      "NEXT\n",
      "3 [28.03] [24.05] [33.58]\n",
      "\n",
      "\n",
      "NEXT\n",
      "4 [27.49] [24.6] [31.15]\n",
      "\n",
      "\n",
      "NEXT\n",
      "5 [28.93] [25.78] [32.97]\n",
      "\n",
      "\n",
      "NEXT\n",
      "6 [26.12] [23.78] [28.97]\n",
      "\n",
      "\n",
      "NEXT\n",
      "7 [24.31] [20.6] [29.65]\n",
      "\n",
      "\n",
      "NEXT\n",
      "8 [29.7] [26.79] [33.32]\n",
      "\n",
      "\n",
      "NEXT\n",
      "9 [30.84] [27.29] [35.43]\n",
      "\n",
      "\n",
      "NEXT\n",
      "10 [26.46] [23.8] [29.78]\n",
      "\n",
      "\n",
      "NEXT\n",
      "avg f1 is: 27.459999999999997\n",
      "avg p is: 24.409\n",
      "avg re is: 31.429999999999996\n"
     ]
    }
   ],
   "source": [
    "# base predictions on the test data.\n",
    "avg_f1 = 0.0\n",
    "avg_p = 0.0\n",
    "avg_re = 0.0\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_gold_file = \"./zero-shot-extraction/relation_splits/test.{}\".format(fold_i-1)\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/\".format(fold_i)\n",
    "    fold_files = [\"base-base.test.predictions.fold.{}.csv\".format(fold_i)]\n",
    "    preprocess_the_prediction_files(fold_path, fold_files)\n",
    "    max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "    print(fold_i, f1s, precisions, recalls)\n",
    "    avg_f1 += f1s[0]\n",
    "    avg_p += precisions[0]\n",
    "    avg_re += recalls[0]\n",
    "    print(\"\\n\")\n",
    "    print(\"NEXT\")\n",
    "\n",
    "print(\"avg f1 is:\", avg_f1/10.0)\n",
    "print(\"avg p is:\", avg_p/10.0)\n",
    "print(\"avg re is:\", avg_re/10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mml-pgg-on-sim 1 mml-pgg-on-sim.test.predictions.fold.1.step.300.csv 59.57\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 2 mml-pgg-on-sim.test.predictions.fold.2.step.9200.csv 45.83\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 3 mml-pgg-on-sim.test.predictions.fold.3.step.300.csv 55.44\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 4 mml-pgg-on-sim.test.predictions.fold.4.step.500.csv 61.92\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 5 mml-pgg-on-sim.test.predictions.fold.5.step.12600.csv 53.77\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 6 mml-pgg-on-sim.test.predictions.fold.6.step.300.csv 54.33\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 7 mml-pgg-on-sim.test.predictions.fold.7.step.9800.csv 60.39\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 8 mml-pgg-on-sim.test.predictions.fold.8.step.17000.csv 56.03\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 9 mml-pgg-on-sim.test.predictions.fold.9.step.2300.csv 52.0\n",
      "\n",
      "\n",
      "mml-pgg-on-sim 10 mml-pgg-on-sim.test.predictions.fold.10.step.4300.csv 47.2\n",
      "\n",
      "\n",
      "mml-pgg-on-sim f1 54.648\n",
      "mml-pgg-on-sim p 57.0\n",
      "mml-pgg-on-sim r 52.849000000000004\n",
      "NEXT\n",
      "mml-mml-off-sim 1 mml-mml-off-sim.test.predictions.fold.1.step.500.csv 63.75\n",
      "\n",
      "\n",
      "mml-mml-off-sim 2 mml-mml-off-sim.test.predictions.fold.2.step.9300.csv 43.35\n",
      "\n",
      "\n",
      "mml-mml-off-sim 3 mml-mml-off-sim.test.predictions.fold.3.step.3800.csv 57.68\n",
      "\n",
      "\n",
      "mml-mml-off-sim 4 mml-mml-off-sim.test.predictions.fold.4.step.2300.csv 61.33\n",
      "\n",
      "\n",
      "mml-mml-off-sim 5 mml-mml-off-sim.test.predictions.fold.5.step.3000.csv 56.38\n",
      "\n",
      "\n",
      "mml-mml-off-sim 6 mml-mml-off-sim.test.predictions.fold.6.step.300.csv 56.33\n",
      "\n",
      "\n",
      "mml-mml-off-sim 7 mml-mml-off-sim.test.predictions.fold.7.step.13400.csv 53.87\n",
      "\n",
      "\n",
      "mml-mml-off-sim 8 mml-mml-off-sim.test.predictions.fold.8.step.7900.csv 58.9\n",
      "\n",
      "\n",
      "mml-mml-off-sim 9 mml-mml-off-sim.test.predictions.fold.9.step.4700.csv 51.97\n",
      "\n",
      "\n",
      "mml-mml-off-sim 10 mml-mml-off-sim.test.predictions.fold.10.step.1500.csv 55.36\n",
      "\n",
      "\n",
      "mml-mml-off-sim f1 55.891999999999996\n",
      "mml-mml-off-sim p 60.689\n",
      "mml-mml-off-sim r 52.301\n",
      "NEXT\n",
      "mml-mml-on-sim 1 mml-mml-on-sim.test.predictions.fold.1.step.8900.csv 60.63\n",
      "\n",
      "\n",
      "mml-mml-on-sim 2 mml-mml-on-sim.test.predictions.fold.2.step.9400.csv 43.11\n",
      "\n",
      "\n",
      "mml-mml-on-sim 3 mml-mml-on-sim.test.predictions.fold.3.step.2000.csv 55.83\n",
      "\n",
      "\n",
      "mml-mml-on-sim 4 mml-mml-on-sim.test.predictions.fold.4.step.4200.csv 57.36\n",
      "\n",
      "\n",
      "mml-mml-on-sim 5 mml-mml-on-sim.test.predictions.fold.5.step.12800.csv 53.29\n",
      "\n",
      "\n",
      "mml-mml-on-sim 6 mml-mml-on-sim.test.predictions.fold.6.step.2200.csv 55.9\n",
      "\n",
      "\n",
      "mml-mml-on-sim 7 mml-mml-on-sim.test.predictions.fold.7.step.16600.csv 57.11\n",
      "\n",
      "\n",
      "mml-mml-on-sim 8 mml-mml-on-sim.test.predictions.fold.8.step.7200.csv 56.71\n",
      "\n",
      "\n",
      "mml-mml-on-sim 9 mml-mml-on-sim.test.predictions.fold.9.step.1900.csv 54.5\n",
      "\n",
      "\n",
      "mml-mml-on-sim 10 mml-mml-on-sim.test.predictions.fold.10.step.5800.csv 51.54\n",
      "\n",
      "\n",
      "mml-mml-on-sim f1 54.598\n",
      "mml-mml-on-sim p 57.26800000000001\n",
      "mml-mml-on-sim r 52.31600000000001\n",
      "NEXT\n",
      "mml-pgg-off-sim 1 mml-pgg-off-sim.test.predictions.fold.1.step.500.csv 61.3\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 2 mml-pgg-off-sim.test.predictions.fold.2.step.9300.csv 45.43\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 3 mml-pgg-off-sim.test.predictions.fold.3.step.5300.csv 60.69\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 4 mml-pgg-off-sim.test.predictions.fold.4.step.1100.csv 60.57\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 5 mml-pgg-off-sim.test.predictions.fold.5.step.6400.csv 55.91\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 6 mml-pgg-off-sim.test.predictions.fold.6.step.700.csv 56.94\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 7 mml-pgg-off-sim.test.predictions.fold.7.step.14900.csv 55.25\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 8 mml-pgg-off-sim.test.predictions.fold.8.step.3800.csv 61.51\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 9 mml-pgg-off-sim.test.predictions.fold.9.step.2900.csv 50.67\n",
      "\n",
      "\n",
      "mml-pgg-off-sim 10 mml-pgg-off-sim.test.predictions.fold.10.step.8600.csv 53.71\n",
      "\n",
      "\n",
      "mml-pgg-off-sim f1 56.198\n",
      "mml-pgg-off-sim p 58.25\n",
      "mml-pgg-off-sim r 54.374\n",
      "NEXT\n"
     ]
    }
   ],
   "source": [
    "# Evaluating the test predictions on the RE-QA dataset on all folds for the tail entity generation task.\n",
    "folders = [\"mml-pgg-on-sim\", \"mml-mml-off-sim\", \"mml-mml-on-sim\", \"mml-pgg-off-sim\"]\n",
    "for folder in folders:\n",
    "    avg_f1 = {\"mml-mml-off-sim\": 0, \"mml-mml-on-sim\": 0, \"mml-pgg-on-sim\": 0, \"mml-pgg-off-sim\": 0}\n",
    "    avg_p = {\"mml-mml-off-sim\": 0, \"mml-mml-on-sim\": 0, \"mml-pgg-on-sim\": 0, \"mml-pgg-off-sim\": 0}\n",
    "    avg_r = {\"mml-mml-off-sim\": 0, \"mml-mml-on-sim\": 0, \"mml-pgg-on-sim\": 0, \"mml-pgg-off-sim\": 0}\n",
    "    for fold_i in range(1, 11, 1):\n",
    "        fold_gold_file = \"./zero-shot-extraction/relation_splits/test.{}\".format(fold_i-1)\n",
    "        fold_path = \"~/reqa-predictions/fold_{}/{}\".format(fold_i, folder)\n",
    "        old_dev_file = results[fold_i][folder]\n",
    "        new_test_file = old_dev_file.replace(\".fold.{}.dev.predictions.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        new_test_file = new_test_file.replace(\".dev.predictions.fold.{}.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        fold_files = [new_test_file]\n",
    "        preprocess_the_prediction_files(fold_path, fold_files)\n",
    "        max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "        print(folder, fold_i, max_file, max_f1)\n",
    "        avg_f1[folder] += max_f1\n",
    "        avg_p[folder] += precisions[0]\n",
    "        avg_r[folder] += recalls[0]\n",
    "        print(\"\\n\")\n",
    "\n",
    "    print(folder, \"f1\", avg_f1[folder] / 10.0)\n",
    "    print(folder, \"p\", avg_p[folder] / 10.0)\n",
    "    print(folder, \"r\", avg_r[folder] / 10.0)\n",
    "    print(\"NEXT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mml-mml-off-sim.test.predictions.fold.1.step.500.csv tensor(250.1070, device='cuda:0')\n",
      "mml-mml-off-sim.test.predictions.fold.2.step.9300.csv tensor(111.5883, device='cuda:0')\n",
      "mml-mml-off-sim.test.predictions.fold.3.step.3800.csv tensor(123.9757, device='cuda:0')\n",
      "mml-mml-off-sim.test.predictions.fold.4.step.2300.csv tensor(154.7580, device='cuda:0')\n",
      "mml-mml-off-sim.test.predictions.fold.5.step.3000.csv tensor(222.3316, device='cuda:0')\n",
      "mml-mml-off-sim.test.predictions.fold.6.step.300.csv tensor(152.8412, device='cuda:0')\n",
      "mml-mml-off-sim.test.predictions.fold.7.step.13400.csv tensor(171.4223, device='cuda:0')\n",
      "mml-mml-off-sim.test.predictions.fold.8.step.7900.csv tensor(154.2868, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "# Compute perplexity over the test generated questions for the following method on the RE-QA dataset.\n",
    "folders = [\"mml-mml-off-sim\"]\n",
    "\n",
    "for folder in folders:\n",
    "    avg_pp = {\"mml-mml-off-sim\": 0}\n",
    "    for fold_i in range(9, 11, 1):\n",
    "        fold_path = \"~/reqa-predictions/fold_{}/{}\".format(fold_i, folder)\n",
    "        old_dev_file = results[fold_i][folder]\n",
    "        new_test_file = old_dev_file.replace(\".fold.{}.dev.predictions.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        new_test_file = new_test_file.replace(\".dev.predictions.fold.{}.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        fold_file = new_test_file\n",
    "        pp = compute_perplexity_for_questions(fold_path, fold_file)\n",
    "        avg_pp[folder] += pp\n",
    "        print(fold_file, pp)\n",
    "    print(\"\\n\")\n",
    "    print(folder, \"pp\", avg_pp[folder] / 10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "148.37867999999997\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "mml-mml-off-sim.test.predictions.fold.1.step.500.csv tensor(250.1070, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.2.step.9300.csv tensor(111.5883, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.3.step.3800.csv tensor(123.9757, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.4.step.2300.csv tensor(154.7580, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.5.step.3000.csv tensor(222.3316, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.6.step.300.csv tensor(152.8412, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.7.step.13400.csv tensor(171.4223, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.8.step.7900.csv tensor(154.2868, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.9.step.4700.csv tensor(112.6208, device='cuda:0')\n",
    "mml-mml-off-sim.test.predictions.fold.10.step.1500.csv tensor(29.8551, device='cuda:0')\n",
    "'''\n",
    "\n",
    "arr = [250.1070, 111.5883, 123.9757, \n",
    "       154.7580, 222.3316, 152.8412, \n",
    "       171.4223, 154.2868, 112.6208, \n",
    "       29.8551]\n",
    "\n",
    "print(sum(arr)/10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute perplexity over the test generated questions for the following method on the RE-QA dataset.\n",
    "folders = [\"mml-pgg-off-sim\"]\n",
    "\n",
    "for folder in folders:\n",
    "    avg_pp = {\"mml-pgg-off-sim\": 0}\n",
    "    for fold_i in range(1, 11, 1):\n",
    "        fold_path = \"~/reqa-predictions/fold_{}/{}\".format(fold_i, folder)\n",
    "        old_dev_file = results[fold_i][folder]\n",
    "        new_test_file = old_dev_file.replace(\".fold.{}.dev.predictions.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        new_test_file = new_test_file.replace(\".dev.predictions.fold.{}.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        fold_file = new_test_file\n",
    "        pp = compute_perplexity_for_questions(fold_path, fold_file)\n",
    "        avg_pp[folder] += pp\n",
    "        print(fold_file, pp)\n",
    "    print(\"\\n\")\n",
    "    print(folder, \"pp\", avg_pp[folder] / 10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144.11348\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "mml-pgg-off-sim.test.predictions.fold.1.step.500.csv tensor(217.1630, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.2.step.9300.csv tensor(122.6252, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.3.step.5300.csv tensor(121.2163, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.4.step.1100.csv tensor(137.1326, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.5.step.6400.csv tensor(160.4001, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.6.step.700.csv tensor(125.5082, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.7.step.14900.csv tensor(142.9996, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.8.step.3800.csv tensor(161.9223, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.9.step.2900.csv tensor(100.0727, device='cuda:0')\n",
    "mml-pgg-off-sim.test.predictions.fold.10.step.8600.csv tensor(152.0948, device='cuda:0')\n",
    "'''\n",
    "\n",
    "arr = [217.1630, 122.6252, 121.2163, \n",
    "       137.1326, 160.4001, 125.5082, \n",
    "       142.9996, 161.9223, 100.0727, \n",
    "       152.0948]\n",
    "\n",
    "print(sum(arr)/10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute perplexity over the test generated questions for the following method on the RE-QA dataset.\n",
    "folders = [\"mml-pgg-on-sim\"]\n",
    "\n",
    "for folder in folders:\n",
    "    avg_pp = {\"mml-pgg-on-sim\": 0}\n",
    "    for fold_i in range(1, 11, 1):\n",
    "        fold_path = \"~/reqa-predictions/fold_{}/{}\".format(fold_i, folder)\n",
    "        old_dev_file = results[fold_i][folder]\n",
    "        new_test_file = old_dev_file.replace(\".fold.{}.dev.predictions.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        new_test_file = new_test_file.replace(\".dev.predictions.fold.{}.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        fold_file = new_test_file\n",
    "        pp = compute_perplexity_for_questions(fold_path, fold_file)\n",
    "        avg_pp[folder] += pp\n",
    "        print(fold_file, pp)\n",
    "    print(\"\\n\")\n",
    "    print(folder, \"pp\", avg_pp[folder] / 10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "mml-pgg-on-sim.test.predictions.fold.1.step.300.csv tensor(1344.3567, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.2.step.9200.csv tensor(238.5175, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.3.step.300.csv tensor(1374.6135, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.4.step.500.csv tensor(353.1273, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.5.step.12600.csv tensor(132.0151, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.6.step.300.csv tensor(899.9470, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.7.step.9800.csv tensor(2476.5774, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.8.step.17000.csv tensor(132.0151, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.9.step.2300.csv tensor(5960.5225, device='cuda:0')\n",
    "mml-pgg-on-sim.test.predictions.fold.10.step.4300.csv tensor(185.5247, device='cuda:0')\n",
    "\n",
    "\n",
    "mml-pgg-on-sim pp tensor(1309.7216, device='cuda:0')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute perplexity over the test generated questions for the following method on the RE-QA dataset.\n",
    "folders = [\"mml-mml-on-sim\"]\n",
    "\n",
    "for folder in folders:\n",
    "    avg_pp = {\"mml-mml-on-sim\": 0}\n",
    "    for fold_i in range(1, 11, 1):\n",
    "        fold_path = \"~/reqa-predictions/fold_{}/{}\".format(fold_i, folder)\n",
    "        old_dev_file = results[fold_i][folder]\n",
    "        new_test_file = old_dev_file.replace(\".fold.{}.dev.predictions.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        new_test_file = new_test_file.replace(\".dev.predictions.fold.{}.\".format(fold_i), \".test.predictions.fold.{}.\".format(fold_i))\n",
    "        fold_file = new_test_file\n",
    "        pp = compute_perplexity_for_questions(fold_path, fold_file)\n",
    "        avg_pp[folder] += pp\n",
    "        print(fold_file, pp)\n",
    "    print(\"\\n\")\n",
    "    print(folder, \"pp\", avg_pp[folder] / 10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "base_base_fold.1.test.predictions.step.csv tensor(215.8548, device='cuda:0')\n",
      "base_base_fold.2.test.predictions.step.csv tensor(181.5120, device='cuda:0')\n",
      "base_base_fold.3.test.predictions.step.csv tensor(128.0450, device='cuda:0')\n",
      "base_base_fold.4.test.predictions.step.csv tensor(185.9727, device='cuda:0')\n",
      "base_base_fold.5.test.predictions.step.csv tensor(196.0677, device='cuda:0')\n",
      "base_base_fold.6.test.predictions.step.csv tensor(138.5674, device='cuda:0')\n",
      "base_base_fold.7.test.predictions.step.csv tensor(219.6459, device='cuda:0')\n",
      "base_base_fold.8.test.predictions.step.csv tensor(206.5881, device='cuda:0')\n",
      "base_base_fold.9.test.predictions.step.csv tensor(163.6257, device='cuda:0')\n",
      "base_base_fold.10.test.predictions.step.csv tensor(179.0245, device='cuda:0')\n",
      "\n",
      "\n",
      "pp tensor(181.4904, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "avg_pp = 0.0\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/\".format(fold_i)\n",
    "    fold_file = \"base-base.test.predictions.fold.{}.csv\".format(fold_i)\n",
    "    pp = compute_perplexity_for_questions(fold_path, fold_file)\n",
    "    avg_pp += pp\n",
    "    print(fold_file, pp)\n",
    "\n",
    "print(\"\\n\")\n",
    "print(\"pp\", avg_pp / 10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 gold_fold.1.dev.predictions.step.800.csv 56.83\n",
      "#\n",
      "2 gold_fold.2.dev.predictions.step.2000.csv 65.76\n",
      "#\n",
      "3 gold_fold.3.dev.predictions.step.4200.csv 63.59\n",
      "#\n",
      "4 gold_fold.4.dev.predictions.step.1400.csv 64.44\n",
      "#\n",
      "5 gold_fold.5.dev.predictions.step.900.csv 67.0\n",
      "#\n",
      "6 gold_fold.6.dev.predictions.step.400.csv 69.45\n",
      "#\n",
      "7 gold_fold.7.dev.predictions.step.6100.csv 59.3\n",
      "#\n",
      "8 gold_fold.8.dev.predictions.step.7300.csv 64.1\n",
      "#\n",
      "9 gold_fold.9.dev.predictions.step.1800.csv 67.48\n",
      "#\n",
      "10 gold_fold.10.dev.predictions.step.4100.csv 61.41\n",
      "#\n"
     ]
    }
   ],
   "source": [
    "# Tail Entity Generation.\n",
    "# Evaluating the dev predictions on the RE-QA dataset using the model having access to gold templates!\n",
    "dev_max_files = {}\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_gold_file = \"./zero-shot-extraction/relation_splits/dev.{}\".format(fold_i-1)\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/gold/dev_predictions/\".format(fold_i)\n",
    "    fold_files = [\"gold_fold.{}.dev.predictions.step.{}.csv\".format(fold_i, 100 * i) for i in range(1, 506, 1)]\n",
    "    preprocess_the_prediction_files(fold_path, fold_files)\n",
    "    max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "    dev_max_files[fold_i] = max_file\n",
    "    print(fold_i, max_file, max_f1)\n",
    "    print(\"#\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [64.21] [61.01] [67.77]\n",
      "\n",
      "\n",
      "NEXT\n",
      "2 [48.65] [52.8] [45.1]\n",
      "\n",
      "\n",
      "NEXT\n",
      "3 [60.73] [60.75] [60.7]\n",
      "\n",
      "\n",
      "NEXT\n",
      "4 [67.16] [66.82] [67.5]\n",
      "\n",
      "\n",
      "NEXT\n",
      "5 [55.61] [59.76] [52.]\n",
      "\n",
      "\n",
      "NEXT\n",
      "6 [61.03] [60.72] [61.35]\n",
      "\n",
      "\n",
      "NEXT\n",
      "7 [58.4] [60.15] [56.75]\n",
      "\n",
      "\n",
      "NEXT\n",
      "8 [59.74] [63.77] [56.18]\n",
      "\n",
      "\n",
      "NEXT\n",
      "9 [56.49] [56.74] [56.25]\n",
      "\n",
      "\n",
      "NEXT\n",
      "10 [56.89] [58.55] [55.32]\n",
      "\n",
      "\n",
      "NEXT\n",
      "avg f1 is: 58.891\n",
      "avg p is: 60.10699999999999\n",
      "avg re is: 57.89200000000001\n"
     ]
    }
   ],
   "source": [
    "# Tail Entity Generation.\n",
    "# Evaluating the test predictions on the RE-QA dataset using the model having access to gold templates!\n",
    "\n",
    "avg_f1 = 0.0\n",
    "avg_p = 0.0\n",
    "avg_re = 0.0\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_gold_file = \"./zero-shot-extraction/relation_splits/test.{}\".format(fold_i-1)\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/gold/\".format(fold_i)\n",
    "    fold_files = [\"gold_fold.{}.test.predictions.step..csv\".format(fold_i)]\n",
    "    preprocess_the_prediction_files(fold_path, fold_files)\n",
    "    max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "    print(fold_i, f1s, precisions, recalls)\n",
    "    avg_f1 += f1s[0]\n",
    "    avg_p += precisions[0]\n",
    "    avg_re += recalls[0]\n",
    "    print(\"\\n\")\n",
    "    print(\"NEXT\")\n",
    "\n",
    "print(\"avg f1 is:\", avg_f1/10.0)\n",
    "print(\"avg p is:\", avg_p/10.0)\n",
    "print(\"avg re is:\", avg_re/10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [44.76] [36.19] [58.65]\n",
      "\n",
      "\n",
      "NEXT\n",
      "2 [38.57] [31.78] [49.07]\n",
      "\n",
      "\n",
      "NEXT\n",
      "3 [42.92] [35.45] [54.37]\n",
      "\n",
      "\n",
      "NEXT\n",
      "4 [47.94] [40.4] [58.93]\n",
      "\n",
      "\n",
      "NEXT\n",
      "5 [43.86] [36.67] [54.55]\n",
      "\n",
      "\n",
      "NEXT\n",
      "6 [44.04] [36.65] [55.18]\n",
      "\n",
      "\n",
      "NEXT\n",
      "7 [41.53] [33.62] [54.33]\n",
      "\n",
      "\n",
      "NEXT\n",
      "8 [46.63] [39.3] [57.3]\n",
      "\n",
      "\n",
      "NEXT\n",
      "9 [43.52] [35.54] [56.1]\n",
      "\n",
      "\n",
      "NEXT\n",
      "10 [42.35] [35.43] [52.65]\n",
      "\n",
      "\n",
      "NEXT\n",
      "avg f1 is: 43.612\n",
      "avg p is: 36.103\n",
      "avg re is: 55.113\n"
     ]
    }
   ],
   "source": [
    "# Tail Entity Generation.\n",
    "# Evaluating the test predictions on the RE-QA dataset using the model having access to gold templates without finetunning. Gold Base.\n",
    "\n",
    "avg_f1 = 0.0\n",
    "avg_p = 0.0\n",
    "avg_re = 0.0\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_gold_file = \"./zero-shot-extraction/relation_splits/test.{}\".format(fold_i-1)\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/gold/\".format(fold_i)\n",
    "    fold_files = [\"base_gold_fold.{}.test.predictions.step..csv\".format(fold_i)]\n",
    "    preprocess_the_prediction_files(fold_path, fold_files)\n",
    "    max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "    print(fold_i, f1s, precisions, recalls)\n",
    "    avg_f1 += f1s[0]\n",
    "    avg_p += precisions[0]\n",
    "    avg_re += recalls[0]\n",
    "    print(\"\\n\")\n",
    "    print(\"NEXT\")\n",
    "\n",
    "print(\"avg f1 is:\", avg_f1/10.0)\n",
    "print(\"avg p is:\", avg_p/10.0)\n",
    "print(\"avg re is:\", avg_re/10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 concat_fold.1.dev.predictions.step.1700.csv 53.96\n",
      "#\n",
      "2 concat_fold.2.dev.predictions.step.4600.csv 64.46\n",
      "#\n",
      "3 concat_fold.3.dev.predictions.step.18600.csv 61.4\n",
      "#\n",
      "4 concat_fold.4.dev.predictions.step.1400.csv 67.12\n",
      "#\n",
      "5 concat_fold.5.dev.predictions.step.7900.csv 69.91\n",
      "#\n",
      "6 concat_fold.6.dev.predictions.step.2600.csv 64.16\n",
      "#\n",
      "7 concat_fold.7.dev.predictions.step.24700.csv 63.0\n",
      "#\n",
      "8 concat_fold.8.dev.predictions.step.13800.csv 61.08\n",
      "#\n",
      "9 concat_fold.9.dev.predictions.step.17700.csv 61.6\n",
      "#\n",
      "10 concat_fold.10.dev.predictions.step.35500.csv 57.72\n",
      "#\n"
     ]
    }
   ],
   "source": [
    "# Tail Entity Generation.\n",
    "# Evaluating the dev predictions on the RE-QA dataset using the concat model.\n",
    "dev_max_files = {}\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_gold_file = \"./zero-shot-extraction/relation_splits/dev.{}\".format(fold_i-1)\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/concat/dev_predictions/\".format(fold_i)\n",
    "    fold_files = [\"concat_fold.{}.dev.predictions.step.{}.csv\".format(fold_i, 100 * i) for i in range(1, 506, 1)]\n",
    "    preprocess_the_prediction_files(fold_path, fold_files)\n",
    "    max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "    dev_max_files[fold_i] = max_file\n",
    "    print(fold_i, max_file, max_f1)\n",
    "    print(\"#\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [58.35] [61.38] [55.62]\n",
      "\n",
      "\n",
      "NEXT\n",
      "2 [42.61] [50.08] [37.08]\n",
      "\n",
      "\n",
      "NEXT\n",
      "3 [57.43] [59.32] [55.65]\n",
      "\n",
      "\n",
      "NEXT\n",
      "4 [56.75] [61.26] [52.87]\n",
      "\n",
      "\n",
      "NEXT\n",
      "5 [56.59] [60.4] [53.23]\n",
      "\n",
      "\n",
      "NEXT\n",
      "6 [56.48] [57.44] [55.55]\n",
      "\n",
      "\n",
      "NEXT\n",
      "7 [54.52] [59.09] [50.62]\n",
      "\n",
      "\n",
      "NEXT\n",
      "8 [55.51] [58.14] [53.12]\n",
      "\n",
      "\n",
      "NEXT\n",
      "9 [44.98] [56.21] [37.48]\n",
      "\n",
      "\n",
      "NEXT\n",
      "10 [53.] [55.78] [50.48]\n",
      "\n",
      "\n",
      "NEXT\n",
      "avg f1 is: 53.622\n",
      "avg p is: 57.910000000000004\n",
      "avg re is: 50.17\n"
     ]
    }
   ],
   "source": [
    "# Tail Entity Generation.\n",
    "# Evaluating the test predictions on the RE-QA dataset using the concat model!\n",
    "\n",
    "avg_f1 = 0.0\n",
    "avg_p = 0.0\n",
    "avg_re = 0.0\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_gold_file = \"./zero-shot-extraction/relation_splits/test.{}\".format(fold_i-1)\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/concat/\".format(fold_i)\n",
    "    fold_files = [\"concat_fold.{}.test.predictions.step..csv\".format(fold_i)]\n",
    "    preprocess_the_prediction_files(fold_path, fold_files)\n",
    "    max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "    print(fold_i, f1s, precisions, recalls)\n",
    "    avg_f1 += f1s[0]\n",
    "    avg_p += precisions[0]\n",
    "    avg_re += recalls[0]\n",
    "    print(\"\\n\")\n",
    "    print(\"NEXT\")\n",
    "\n",
    "print(\"avg f1 is:\", avg_f1/10.0)\n",
    "print(\"avg p is:\", avg_p/10.0)\n",
    "print(\"avg re is:\", avg_re/10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [3.06] [2.36] [4.37]\n",
      "\n",
      "\n",
      "NEXT\n",
      "2 [2.56] [1.97] [3.63]\n",
      "\n",
      "\n",
      "NEXT\n",
      "3 [1.94] [1.49] [2.78]\n",
      "\n",
      "\n",
      "NEXT\n",
      "4 [2.01] [1.54] [2.87]\n",
      "\n",
      "\n",
      "NEXT\n",
      "5 [1.66] [1.28] [2.38]\n",
      "\n",
      "\n",
      "NEXT\n",
      "6 [2.78] [2.16] [3.9]\n",
      "\n",
      "\n",
      "NEXT\n",
      "7 [3.3] [2.52] [4.78]\n",
      "\n",
      "\n",
      "NEXT\n",
      "8 [2.82] [2.21] [3.9]\n",
      "\n",
      "\n",
      "NEXT\n",
      "9 [1.58] [1.2] [2.3]\n",
      "\n",
      "\n",
      "NEXT\n",
      "10 [0.79] [0.61] [1.15]\n",
      "\n",
      "\n",
      "NEXT\n",
      "avg f1 is: 2.25\n",
      "avg p is: 1.734\n",
      "avg re is: 3.2059999999999995\n"
     ]
    }
   ],
   "source": [
    "# Tail Entity Generation.\n",
    "# Evaluating the test predictions on the RE-QA dataset using the concat model without finetunning. Concat-Base\n",
    "\n",
    "avg_f1 = 0.0\n",
    "avg_p = 0.0\n",
    "avg_re = 0.0\n",
    "for fold_i in range(1, 11, 1):\n",
    "    fold_gold_file = \"./zero-shot-extraction/relation_splits/test.{}\".format(fold_i-1)\n",
    "    fold_path = \"~/reqa-predictions/fold_{}/concat/\".format(fold_i)\n",
    "    fold_files = [\"base_concat_fold.{}.test.predictions.step..csv\".format(fold_i)]\n",
    "    preprocess_the_prediction_files(fold_path, fold_files)\n",
    "    max_file,  max_f1, f1s, scores, precisions, recalls = unk_eval_the_prediction_files(fold_files, fold_gold_file)\n",
    "    print(fold_i, f1s, precisions, recalls)\n",
    "    avg_f1 += f1s[0]\n",
    "    avg_p += precisions[0]\n",
    "    avg_re += recalls[0]\n",
    "    print(\"\\n\")\n",
    "    print(\"NEXT\")\n",
    "\n",
    "print(\"avg f1 is:\", avg_f1/10.0)\n",
    "print(\"avg p is:\", avg_p/10.0)\n",
    "print(\"avg re is:\", avg_re/10.0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}