Pre_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "import os\n",
    "import random\n",
    "import string\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raw files to .csv:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the directory containing the input files\n",
    "input_dir = 'transcript'\n",
    "\n",
    "def process_txt_file(input_file, output_file):\n",
    "    with open(input_file, 'r') as infile, open(output_file, 'w', newline='') as outfile:\n",
    "        csv_writer = csv.writer(outfile)\n",
    "        \n",
    "        header = ['time', 'Anger', 'Disgust', 'Fear', 'Happiness', 'Sadness','Surprise','Neutral']\n",
    "        csv_writer.writerow(header)\n",
    "        \n",
    "        for line in infile:\n",
    "            data = json.loads(line.strip())\n",
    "            row = [int(data['time'])] + [int(score)//1e4 for score in data['scores']]\n",
    "            csv_writer.writerow(row)\n",
    "\n",
    "# Function to recursively process nested content\n",
    "def process_content(csv_writer, content):\n",
    "    if isinstance(content, list):\n",
    "        for item in content:\n",
    "            process_content(csv_writer, item)\n",
    "    elif isinstance(content, dict):\n",
    "        role = content.get('role', '')\n",
    "        content_text = ''\n",
    "        time = int(content.get('time', '0'))\n",
    "        user_id = content.get('user_id', '')\n",
    "        if 'content' in content:\n",
    "            content_data = content['content']\n",
    "            if isinstance(content_data, list):\n",
    "                for sub_item in content_data:\n",
    "                    if isinstance(sub_item, dict) and sub_item.get('type') == 'text':\n",
    "                        content_text = sub_item.get('text', '')\n",
    "            elif isinstance(content_data, str):\n",
    "                content_text = content_data\n",
    "        csv_writer.writerow([role, content_text, time, user_id])\n",
    "\n",
    "# Function to process .json files and convert them to CSV\n",
    "def process_json_file(input_file, output_file):\n",
    "    with open(input_file, 'r') as infile, open(output_file, 'w', newline='') as outfile:\n",
    "        csv_writer = csv.writer(outfile)\n",
    "        \n",
    "        header = ['role', 'content', 'time', 'user_id']\n",
    "        csv_writer.writerow(header)\n",
    "        \n",
    "        data_list = json.load(infile)\n",
    "        \n",
    "        for item in data_list:\n",
    "            process_content(csv_writer, item)\n",
    "\n",
    "# Walk through all directories and files\n",
    "for root, dirs, files in os.walk(input_dir):\n",
    "    for filename in files:\n",
    "        if filename.endswith('.txt'):\n",
    "            input_file = os.path.join(root, filename)\n",
    "            output_file = os.path.join(root, 'processed_' + filename.replace('.txt', '.csv'))\n",
    "            process_txt_file(input_file, output_file)\n",
    "        elif filename.endswith('.json') and not filename.startswith(('pre', 'post', 'chat')):\n",
    "            input_file = os.path.join(root, filename)\n",
    "            output_file = os.path.join(root, 'processed_' + filename.replace('.json', '.csv'))\n",
    "            process_json_file(input_file, output_file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating pre-VAD avg emotion scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing folder: transcript\n",
      "Required CSV files not found in this folder.\n",
      "Processing folder: transcript/20240830_183743\n",
      "Found user data CSV: transcript/20240830_183743/processed_Emili_20240830_183743.csv\n",
      "Found raw scores CSV: transcript/20240830_183743/processed_Emili_raw_20240830_183743.csv\n",
      "Processed data saved to transcript/20240830_183743/scored_20240830_183743.csv\n",
      "Processing folder: transcript/20240906_162206\n",
      "Found user data CSV: transcript/20240906_162206/processed_Emili_20240906_162206.csv\n",
      "Found raw scores CSV: transcript/20240906_162206/processed_Emili_raw_20240906_162206.csv\n",
      "Processed data saved to transcript/20240906_162206/scored_20240906_162206.csv\n"
     ]
    }
   ],
   "source": [
    "# Updated emotion names\n",
    "EMOTION_NAMES = [\"Anger\", \"Disgust\", \"Fear\", \"Happiness\", \"Sadness\", \"Surprise\", \"Neutral\"]\n",
    "\n",
    "# Function to read CSV file into a list of dictionaries\n",
    "def read_csv_file(file_path):\n",
    "    data = []\n",
    "    with open(file_path, 'r', newline='') as csvfile:\n",
    "        reader = csv.DictReader(csvfile)\n",
    "        for row in reader:\n",
    "            data.append(row)\n",
    "    return data\n",
    "\n",
    "# Function to write data to a new CSV file\n",
    "def write_csv_file(file_path, header, data):\n",
    "    with open(file_path, 'w', newline='') as csvfile:\n",
    "        writer = csv.writer(csvfile)\n",
    "        writer.writerow(header)\n",
    "        for row in data:\n",
    "            writer.writerow(row)\n",
    "\n",
    "# Function to generate random alphanumeric ID of given length\n",
    "def generate_random_id(length):\n",
    "    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))\n",
    "\n",
    "# Function to calculate average scores for a given time range\n",
    "def calculate_average_scores(raw_scores, target_time, role, time_window):\n",
    "    sum_scores = {emotion: 0 for emotion in EMOTION_NAMES}\n",
    "    count = 0\n",
    "\n",
    "    for score_data in raw_scores:\n",
    "        score_time = int(score_data['time'])\n",
    "\n",
    "        if role == 'user':\n",
    "            if target_time - time_window <= score_time <= target_time + time_window:\n",
    "                for emotion in EMOTION_NAMES:\n",
    "                    sum_scores[emotion] += float(score_data[emotion])\n",
    "                count += 1\n",
    "        elif role == 'assistant':\n",
    "            if target_time <= score_time <= target_time + time_window:\n",
    "                for emotion in EMOTION_NAMES:\n",
    "                    sum_scores[emotion] += float(score_data[emotion])\n",
    "                count += 1\n",
    "\n",
    "    if count > 0:\n",
    "        avg_scores = {emotion: round(sum_score / count, 2) for emotion, sum_score in sum_scores.items()}\n",
    "    else:\n",
    "        avg_scores = {emotion: 0 for emotion in EMOTION_NAMES}\n",
    "\n",
    "    return avg_scores\n",
    "\n",
    "# Function to process each folder and its files\n",
    "def process_folder(root):\n",
    "    print(f\"Processing folder: {root}\")\n",
    "    user_data_csv = None\n",
    "    raw_scores_csv = None\n",
    "    output_csv = None\n",
    "\n",
    "    used_ids = set()  # Set to store used conv_ids\n",
    "\n",
    "    for filename in os.listdir(root):\n",
    "        if filename.startswith('processed_Emili_') and filename.endswith('.csv') and not filename.endswith('_condensed.csv'):\n",
    "            if filename.startswith('processed_Emili_raw_'):\n",
    "                raw_scores_csv = os.path.join(root, filename)\n",
    "            else:\n",
    "                user_data_csv = os.path.join(root, filename)\n",
    "\n",
    "    if user_data_csv and raw_scores_csv:\n",
    "        timestamp = os.path.basename(user_data_csv).split('_')[2] + '_' + os.path.basename(user_data_csv).split('_')[3].replace('.csv', '')\n",
    "        output_csv = os.path.join(root, f'scored_{timestamp}.csv')\n",
    "        print(f\"Found user data CSV: {user_data_csv}\")\n",
    "        print(f\"Found raw scores CSV: {raw_scores_csv}\")\n",
    "        user_data = read_csv_file(user_data_csv)\n",
    "        raw_scores_data = read_csv_file(raw_scores_csv)\n",
    "\n",
    "        output_header = ['Conv_id', 'time', 'role', 'user_id', 'content'] + EMOTION_NAMES\n",
    "        output_data = []\n",
    "\n",
    "        # Generate unique conv_id\n",
    "        conv_id = generate_random_id(7)\n",
    "        while conv_id in used_ids:\n",
    "            conv_id = generate_random_id(7)\n",
    "        used_ids.add(conv_id)\n",
    "\n",
    "        for user_row in user_data:\n",
    "            if user_row['role'] == 'user':\n",
    "                time_window = 5\n",
    "                target_time = int(user_row['time'])\n",
    "                avg_scores = calculate_average_scores(raw_scores_data, target_time, user_row['role'], time_window)\n",
    "                output_row = [conv_id, user_row['time'], user_row['role'], user_row['user_id'], user_row['content']] + [avg_scores[emotion] for emotion in EMOTION_NAMES]\n",
    "                output_data.append(output_row)\n",
    "            elif user_row['role'] == 'assistant':\n",
    "                time_window = 10\n",
    "                target_time = int(user_row['time'])\n",
    "                avg_scores = calculate_average_scores(raw_scores_data, target_time, user_row['role'], time_window)\n",
    "                output_row = [conv_id, user_row['time'], user_row['role'], user_row['user_id'], user_row['content']] + [avg_scores[emotion] for emotion in EMOTION_NAMES]\n",
    "                output_data.append(output_row)\n",
    "            elif user_row['role'] == 'system':\n",
    "                output_row = [conv_id, user_row['time'], user_row['role'], user_row['user_id'], user_row['content']] + [0] * len(EMOTION_NAMES)\n",
    "                output_data.append(output_row)\n",
    "\n",
    "        write_csv_file(output_csv, output_header, output_data)\n",
    "        print(f\"Processed data saved to {output_csv}\")\n",
    "    else:\n",
    "        print(\"Required CSV files not found in this folder.\")\n",
    "\n",
    "# Walk through all directories and files\n",
    "input_dir = 'transcript'\n",
    "for root, dirs, files in os.walk(input_dir):\n",
    "    process_folder(root)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating Temporal Difference and the flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_scored_files_in_directory(directory):\n",
    "    # Iterate through files in the directory\n",
    "    for filename in os.listdir(directory):\n",
    "        if filename.startswith('scored_') and filename.endswith('.csv'):\n",
    "            file_path = os.path.join(directory, filename)\n",
    "            print(f\"Processing file: {file_path}\")\n",
    "            \n",
    "            # Read the CSV file into a DataFrame\n",
    "            df = pd.read_csv(file_path)\n",
    "            \n",
    "            # Compute tot_emo_score\n",
    "            df['tot_emo_score'] = df['Happy'] * 5 + df['Neutral'] * 1 - df['Sad'] * 2 + df['Surprise'] * 1 - df['Anger'] * 2 - df['Fear'] * 2 - df['Disgust'] * 5\n",
    "            \n",
    "            # Initialize flag column with NaNs\n",
    "            df['flag'] = np.nan\n",
    "            \n",
    "            # Explicitly cast flag column to boolean\n",
    "            df['flag'] = df['flag'].astype('object')\n",
    "            \n",
    "            # Iterate through each row with role 'assistant'\n",
    "            for idx, row in df.iterrows():\n",
    "                if row['role'] == 'assistant':\n",
    "                    # Search backwards to find the previous row with role 'user'\n",
    "                    prev_user_idx = idx - 1\n",
    "                    while prev_user_idx >= 0 and df.iloc[prev_user_idx]['role'] != 'user':\n",
    "                        prev_user_idx -= 1\n",
    "                    \n",
    "                    # Check if a valid previous 'user' row was found\n",
    "                    if prev_user_idx >= 0 and df.iloc[prev_user_idx]['role'] == 'user':\n",
    "                        if row['tot_emo_score'] - df.iloc[prev_user_idx]['tot_emo_score'] >= 0:\n",
    "                            df.at[idx, 'flag'] = True\n",
    "                        else:\n",
    "                            df.at[idx, 'flag'] = False\n",
    "            \n",
    "            # Extract file timestamp from filename\n",
    "            timestamp = filename.split('_')[1].split('.')[0]  # Adjust this based on your filename pattern\n",
    "            \n",
    "            # Save the modified DataFrame to a new CSV file\n",
    "            output_filename = f'flagged_{timestamp}.csv'\n",
    "            output_path = os.path.join(directory, output_filename)\n",
    "            df.to_csv(output_path, index=False)\n",
    "            \n",
    "            print(f\"Processed data saved to {output_path}\")\n",
    "\n",
    "# Define the main directory to process\n",
    "main_directory = 'transcript'\n",
    "\n",
    "# Iterate through each directory in the main directory\n",
    "for root, dirs, files in os.walk(main_directory):\n",
    "    for directory in dirs:\n",
    "        directory_path = os.path.join(root, directory)\n",
    "        process_scored_files_in_directory(directory_path)\n",
    "\n",
    "print(\"Processing complete for all directories.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CSV to JSONL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_csv_file(csv_file_path, messages):\n",
    "    with open(csv_file_path, 'r') as csv_file:\n",
    "        csv_reader = csv.DictReader(csv_file)\n",
    "        system_message = \"\"\n",
    "\n",
    "        for row in csv_reader:\n",
    "            id = ''\n",
    "\n",
    "            if row[\"role\"] == \"user\":\n",
    "                id = f\"user_id: {row['user_id']}. \"\n",
    "            # Process the content without the column name\n",
    "            content = id + row[\"content\"].replace('\\n', ' ').replace('\\r', ' ').strip()\n",
    "\n",
    "            if row[\"role\"] == \"system\":\n",
    "                system_message += content + \" \"\n",
    "                continue\n",
    "\n",
    "            if system_message:\n",
    "                messages.append({\"role\": \"system\", \"content\": system_message.strip()})\n",
    "                system_message = \"\"\n",
    "\n",
    "            message = {\n",
    "                \"role\": row[\"role\"],\n",
    "                \"content\": content\n",
    "            }\n",
    "            if row[\"role\"] == \"assistant\":\n",
    "                message[\"weight\"] = 1 if row[\"flag\"].lower() == \"true\" else 0\n",
    "\n",
    "            messages.append(message)\n",
    "\n",
    "        if system_message:\n",
    "            messages.append({\"role\": \"system\", \"content\": system_message.strip()})\n",
    "\n",
    "\n",
    "def csv_to_jsonl(input_dir, jsonl_file_path):\n",
    "    messages = []\n",
    "\n",
    "    for root, dirs, files in os.walk(input_dir):\n",
    "        for filename in files:\n",
    "            if filename.startswith('flagged_') and filename.endswith('.csv') and not filename.endswith('_condensed.csv'):\n",
    "                csv_file_path = os.path.join(root, filename)\n",
    "                process_csv_file(csv_file_path, messages)\n",
    "\n",
    "    with open(jsonl_file_path, 'w') as jsonl_file:\n",
    "        jsonl_file.write(json.dumps({\"messages\": messages}) + '\\n')\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    input_dir = 'test_script'  # Replace with the path to your directory\n",
    "    jsonl_file_path = 'Dataset_1.jsonl'  # Replace with the desired output JSONL file path\n",
    "    csv_to_jsonl(input_dir, jsonl_file_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating avg emotion scores (Experiment 1 version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to read CSV file into a list of dictionaries\n",
    "def read_csv_file(file_path):\n",
    "    data = []\n",
    "    with open(file_path, 'r', newline='') as csvfile:\n",
    "        reader = csv.DictReader(csvfile)\n",
    "        for row in reader:\n",
    "            data.append(row)\n",
    "    return data\n",
    "\n",
    "# Function to write data to a new CSV file\n",
    "def write_csv_file(file_path, header, data):\n",
    "    with open(file_path, 'w', newline='') as csvfile:\n",
    "        writer = csv.writer(csvfile)\n",
    "        writer.writerow(header)\n",
    "        for row in data:\n",
    "            writer.writerow(row)\n",
    "\n",
    "# Function to generate random alphanumeric ID of given length\n",
    "def generate_random_id(length):\n",
    "    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))\n",
    "\n",
    "# Function to calculate average scores for a given time range\n",
    "def calculate_average_scores(raw_scores, target_time, role, time_window):\n",
    "    num_emotions = 7\n",
    "    sum_scores = [0] * num_emotions\n",
    "    count = 0\n",
    "\n",
    "    for score_data in raw_scores:\n",
    "        score_time = int(score_data['time'])\n",
    "\n",
    "        if role == 'user':\n",
    "            if target_time - time_window <= score_time <= target_time + time_window:\n",
    "                for i in range(num_emotions):\n",
    "                    sum_scores[i] += int(score_data[f'emotion_{i+1}'])\n",
    "                count += 1\n",
    "        elif role == 'assistant':\n",
    "            if target_time <= score_time <= target_time + time_window:\n",
    "                for i in range(num_emotions):\n",
    "                    sum_scores[i] += int(score_data[f'emotion_{i+1}'])\n",
    "                count += 1\n",
    "    \n",
    "    if count > 0:\n",
    "        avg_scores = [round(sum_score / count, 2) for sum_score in sum_scores]\n",
    "    else:\n",
    "        avg_scores = [0] * num_emotions\n",
    "    \n",
    "    return avg_scores\n",
    "\n",
    "# Function to process each folder and its files\n",
    "def process_folder(root):\n",
    "    print(f\"Processing folder: {root}\")\n",
    "    user_data_csv = None\n",
    "    raw_scores_csv = None\n",
    "    output_csv = None\n",
    "\n",
    "    used_ids = set()  # Set to store used conv_ids\n",
    "\n",
    "    for filename in os.listdir(root):\n",
    "        if filename.startswith('processed_Emili_') and filename.endswith('.csv') and not filename.endswith('_condensed.csv'):\n",
    "            if filename.startswith('processed_Emili_raw_'):\n",
    "                raw_scores_csv = os.path.join(root, filename)\n",
    "            else:\n",
    "                user_data_csv = os.path.join(root, filename)\n",
    "\n",
    "    if user_data_csv and raw_scores_csv:\n",
    "        timestamp = os.path.basename(user_data_csv).split('_')[2]\n",
    "        output_csv = os.path.join(root, f'scored_{timestamp}.csv')\n",
    "        print(f\"Found user data CSV: {user_data_csv}\")\n",
    "        print(f\"Found raw scores CSV: {raw_scores_csv}\")\n",
    "        user_data = read_csv_file(user_data_csv)\n",
    "        raw_scores_data = read_csv_file(raw_scores_csv)\n",
    "\n",
    "        output_header = ['Conv_id','time', 'role','user_id','content','Anger', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']\n",
    "        output_data = []\n",
    "\n",
    "        # Generate unique conv_id\n",
    "        conv_id = generate_random_id(7)\n",
    "        while conv_id in used_ids:\n",
    "            conv_id = generate_random_id(7)\n",
    "        used_ids.add(conv_id)\n",
    "\n",
    "        for user_row in user_data:\n",
    "            if user_row['role'] == 'user':\n",
    "                time_window = 5\n",
    "                target_time = int(user_row['time'])\n",
    "                avg_scores = calculate_average_scores(raw_scores_data, target_time, user_row['role'], time_window)\n",
    "                output_row = [conv_id,user_row['time'],user_row['role'], user_row['user_id'], user_row['content']] + avg_scores\n",
    "                output_data.append(output_row)\n",
    "            elif user_row['role'] == 'assistant':\n",
    "                time_window = 10\n",
    "                target_time = int(user_row['time'])\n",
    "                avg_scores = calculate_average_scores(raw_scores_data, target_time, user_row['role'], time_window)\n",
    "                output_row = [conv_id,user_row['time'],user_row['role'], user_row['user_id'], user_row['content']] + avg_scores\n",
    "                output_data.append(output_row)\n",
    "            elif user_row['role'] == 'system':\n",
    "                output_row = [conv_id,user_row['time'],user_row['role'], user_row['user_id'], user_row['content']] + [0] * 7\n",
    "                output_data.append(output_row)\n",
    "        write_csv_file(output_csv, output_header, output_data)\n",
    "        print(f\"Processed data saved to {output_csv}\")\n",
    "    else:\n",
    "        print(\"Required CSV files not found in this folder.\")\n",
    "\n",
    "# Walk through all directories and process files\n",
    "input_dir = 'transcript'\n",
    "for root, dirs, files in os.walk(input_dir):\n",
    "    process_folder(root)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating the lookup table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def load_nrc_vad_lexicon(file_path):\n",
    "    vad_lexicon = {}\n",
    "    with open(file_path, 'r', newline='', encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            word, valence, arousal, dominance = line.strip().split('\\t')\n",
    "            vad_lexicon[word.lower()] = {\n",
    "                'V': float(valence),\n",
    "                'A': float(arousal),\n",
    "                'D': float(dominance)\n",
    "            }\n",
    "    return vad_lexicon\n",
    "\n",
    "# Load the lexicon\n",
    "lexicon_path = 'NRC-VAD-Lexicon.txt'  # Make sure this path is correct\n",
    "nrc_vad_lexicon = load_nrc_vad_lexicon(lexicon_path)\n",
    "\n",
    "# Example usage\n",
    "emotions = [\n",
    "    \"Surprised\", \"Excited\", \"Angry\", \"Proud\", \"Sad\", \"Annoyed\", \"Grateful\", \"Lonely\",\n",
    "    \"Afraid\", \"Terrified\", \"Guilty\", \"Impressed\", \"Disgusted\", \"Hopeful\", \"Confident\",\n",
    "    \"Furious\", \"Anxious\", \"Anticipating\", \"Joyful\", \"Nostalgic\", \"Disappointed\",\n",
    "    \"Prepared\", \"Jealous\", \"Content\", \"Devastated\", \"Embarrassed\", \"Caring\",\n",
    "    \"Sentimental\", \"Trusting\", \"Ashamed\", \"Apprehensive\", \"Faithful\"\n",
    "]\n",
    "\n",
    "lookup_table = []\n",
    "for emotion in emotions:\n",
    "    emotion_lower = emotion.lower()\n",
    "    if emotion_lower in nrc_vad_lexicon:\n",
    "        lookup_table.append({\n",
    "            'Emotion': emotion,\n",
    "            'Valence': nrc_vad_lexicon[emotion_lower]['V'],\n",
    "            'Arousal': nrc_vad_lexicon[emotion_lower]['A'],\n",
    "            'Dominance': nrc_vad_lexicon[emotion_lower]['D']\n",
    "        })\n",
    "    else:\n",
    "        print(f\"Warning: {emotion} not found in the lexicon.\")\n",
    "\n",
    "lookup_df = pd.DataFrame(lookup_table)\n",
    "print(lookup_df)\n",
    "lookup_df.to_csv('lookup_table.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(lookup_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating the regression dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_VAD_scores(emotion, lookup_df):\n",
    "    '''\n",
    "    Function to get the Valence, Arousal, and Dominance scores for a given emotion.\n",
    "    '''\n",
    "\n",
    "    emotion_row = lookup_df[lookup_df['Emotion'] == emotion]\n",
    "    return emotion_row[['Valence', 'Arousal', 'Dominance']].values[0]\n",
    "\n",
    "#print(get_VAD_scores('Angry', lookup_df))\n",
    "\n",
    "def process_chat_emotions(chat_file:json, prechat_flag: bool ):\n",
    "    '''\n",
    "    Function to process the chat emotions and calculate the average VAD scores.\n",
    "\n",
    "    Args:\n",
    "        pre_chat_file: Path to the chat survey emotions JSON file.\n",
    "        prechat_flag: Boolean flag indicating whether pre-chat desired scores are available.\n",
    "    \n",
    "    Returns:   \n",
    "        chat_current_scores: Dictionary containing the average VAD scores for the current chat emotions.\n",
    "        pre_chat_desired_scores: Dictionary containing the average VAD scores for the pre-chat desired emotions.\n",
    "        \n",
    "    '''\n",
    "    chat_data = json.load(open(chat_file, 'r'))\n",
    "    chat_emotions = chat_data[\"current\"]\n",
    "    chat_current_scores = {'Valence':0, 'Arousal':0, 'Dominance':0}\n",
    "    num_emotions = len(chat_emotions)\n",
    "    for emotion in chat_emotions:\n",
    "        chat_current_scores['Valence'] += get_VAD_scores(emotion, lookup_df)[0]\n",
    "        chat_current_scores['Arousal'] += get_VAD_scores(emotion, lookup_df)[1]\n",
    "        chat_current_scores['Dominance'] += get_VAD_scores(emotion, lookup_df)[2]  \n",
    "\n",
    "    chat_current_scores['Valence'] /= num_emotions\n",
    "    chat_current_scores['Arousal'] /= num_emotions\n",
    "    chat_current_scores['Dominance'] /= num_emotions\n",
    "    if prechat_flag:\n",
    "        pre_chat_desired = chat_data[\"desired\"]\n",
    "        pre_chat_desired_scores = {'Valence':0, 'Arousal':0, 'Dominance':0}\n",
    "        for emotion in pre_chat_desired:\n",
    "            pre_chat_desired_scores['Valence'] += get_VAD_scores(emotion, lookup_df)[0]\n",
    "            pre_chat_desired_scores['Arousal'] += get_VAD_scores(emotion, lookup_df)[1]\n",
    "            pre_chat_desired_scores['Dominance'] += get_VAD_scores(emotion, lookup_df)[2]\n",
    "\n",
    "        pre_chat_desired_scores['Valence'] /= num_emotions\n",
    "        pre_chat_desired_scores['Arousal'] /= num_emotions\n",
    "        pre_chat_desired_scores['Dominance'] /= num_emotions\n",
    "\n",
    "        return chat_current_scores, pre_chat_desired_scores\n",
    "    \n",
    "    return chat_current_scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/f5/t4n8wk5d419dcc_kskpqnt800000gn/T/ipykernel_37870/2667711160.py:56: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  regression_data = pd.concat([regression_data, new_row], ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import json\n",
    "from datetime import datetime\n",
    "lookup_df = pd.read_csv('lookup_table.csv')\n",
    "\n",
    "def generate_regression_data(input_dir):\n",
    "    '''\n",
    "    Function to generate the regression data for the given input directory.\n",
    "    '''\n",
    "    regression_data = pd.DataFrame(columns=['conv_id', 'user_id', 'Pre_Valence', 'Pre_Arousal', 'Pre_Dominance',\n",
    "                                            'Post_Valence', 'Post_Arousal', 'Post_Dominance', 'Anger', 'Disgust', \n",
    "                                            'Fear', 'Happiness', 'Sadness', 'Surprise', 'Neutral', 'Helpfulness', \n",
    "                                            'Repetitiveness', 'Intent'])\n",
    "    \n",
    "    for root, dirs, files in os.walk(input_dir):\n",
    "        pre_chat_file = next((os.path.join(root, f) for f in files if f.startswith('pre_chat_') and f.endswith('.json')), None)\n",
    "        post_chat_file = next((os.path.join(root, f) for f in files if f.startswith('post_chat_') and f.endswith('.json')), None)\n",
    "        scored_file = next((os.path.join(root, f) for f in files if f.startswith('scored_') and f.endswith('.csv')), None)\n",
    "        \n",
    "        # Update this line to match your file naming pattern\n",
    "        eval_file = next((os.path.join(root, f) for f in files if f.startswith('chat_evaluation_') and f.endswith('.json')), None)\n",
    "        \n",
    "        if pre_chat_file and post_chat_file and scored_file and eval_file:\n",
    "            try:\n",
    "                scored_data = pd.read_csv(scored_file)\n",
    "                pre_chat_current, pre_chat_desired = process_chat_emotions(pre_chat_file, True)\n",
    "                post_chat_current = process_chat_emotions(post_chat_file, False)\n",
    "                \n",
    "                with open(eval_file, 'r') as f:\n",
    "                    eval_data = json.load(f)\n",
    "                \n",
    "                user_id = scored_data['user_id'].iloc[2] if not pd.isna(scored_data['user_id'].iloc[2]) else scored_data['user_id'].iloc[3]\n",
    "                \n",
    "                new_row = pd.DataFrame({\n",
    "                    'conv_id': [scored_data['Conv_id'].iloc[0]],\n",
    "                    'user_id': [user_id],\n",
    "                    'Pre_Valence': [pre_chat_current['Valence']],\n",
    "                    'Pre_Arousal': [pre_chat_current['Arousal']],\n",
    "                    'Pre_Dominance': [pre_chat_current['Dominance']],\n",
    "                    'Post_Valence': [post_chat_current['Valence']],\n",
    "                    'Post_Arousal': [post_chat_current['Arousal']],\n",
    "                    'Post_Dominance': [post_chat_current['Dominance']],\n",
    "                    'Anger': [scored_data['Anger'].mean()],\n",
    "                    'Disgust': [scored_data['Disgust'].mean()],\n",
    "                    'Fear': [scored_data['Fear'].mean()],\n",
    "                    'Happiness': [scored_data['Happiness'].mean()],\n",
    "                    'Sadness': [scored_data['Sadness'].mean()],\n",
    "                    'Surprise': [scored_data['Surprise'].mean()],\n",
    "                    'Neutral': [scored_data['Neutral'].mean()],\n",
    "                    'Repetitiveness': [eval_data['Repetitiveness']],\n",
    "                    'Helpfulness': [eval_data['Helpfulness']],\n",
    "                    'Intent': [eval_data['Intent']]\n",
    "                })\n",
    "                \n",
    "                regression_data = pd.concat([regression_data, new_row], ignore_index=True)\n",
    "            except Exception as e:\n",
    "                print(f\"Error processing files in {root}: {str(e)}\")\n",
    "    \n",
    "    return regression_data.round(4)\n",
    "\n",
    "# Usage\n",
    "regression_data = generate_regression_data('transcript')\n",
    "regression_data.to_csv('regression_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_csv = pd.read_csv('regression_data.csv')\n",
    "reg_csv.info()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}