clip prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5ccc869b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted Image Class: Non Islamophobic Image\n",
      "Predicted Text Class: Non Islamophobic Text\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import clip\n",
    "from torchvision import transforms\n",
    "from transformers import AutoTokenizer\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "# Load the fine-tuned CLIP model and tokenizer\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "model, preprocess = clip.load('ViT-B/32', device)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-base-patch16\")\n",
    "\n",
    "# Define the classification heads\n",
    "num_image_classes = 2  # Replace with the number of image classes in your dataset\n",
    "num_text_classes = 2   # Replace with the number of text classes in your dataset\n",
    "\n",
    "image_classifier_head = nn.Linear(512, num_image_classes).to(device)\n",
    "image_classifier_head.load_state_dict(torch.load('nature_model.pt'))\n",
    "image_classifier_head.eval()\n",
    "\n",
    "text_classifier_head = nn.Linear(512, num_text_classes).to(device)\n",
    "text_classifier_head.load_state_dict(torch.load('text_model.pt'))\n",
    "text_classifier_head.eval()\n",
    "\n",
    "# Define the mapping between class indices and class labels for images\n",
    "image_class_mapping = {\n",
    "    0: \"Islamophobic Image\",\n",
    "    1: \"Non Islamophobic Image\",\n",
    "}\n",
    "\n",
    "# Define the mapping between class indices and class labels for text\n",
    "text_class_mapping = {\n",
    "    0: \"Islamophobic Text\",\n",
    "    1: \"Non Islamophobic Text\",\n",
    "}\n",
    "\n",
    "# Define the transformation to normalize and convert image to tensor\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n",
    "])\n",
    "\n",
    "# Function to predict the class of an image\n",
    "def predict_image_class(image_path):\n",
    "    image = cv2.imread(image_path)\n",
    "    image = cv2.resize(image, (224, 224))  # Resize image to fit CLIP model input shape\n",
    "    image = transform(image).unsqueeze(0).to(device)  # Convert image to tensor and add batch dimension\n",
    "\n",
    "    with torch.no_grad():\n",
    "        image_features = model.encode_image(image)\n",
    "\n",
    "    # Convert the data type of image_features to match the classifier's weight matrix\n",
    "    image_features = image_features.to(image_classifier_head.weight.dtype)\n",
    "\n",
    "    logits = image_classifier_head(image_features)\n",
    "    probabilities = nn.functional.softmax(logits, dim=1)\n",
    "    predicted_class_index = torch.argmax(probabilities, dim=1).item()\n",
    "    predicted_class_label = image_class_mapping[predicted_class_index]\n",
    "    \n",
    "    return predicted_class_label\n",
    "\n",
    "# Function to predict the class of a text\n",
    "def predict_text_class(text):\n",
    "    text_input = clip.tokenize([text]).to(device)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        text_features = model.encode_text(text_input)\n",
    "    \n",
    "    logits = text_classifier_head(text_features)\n",
    "    probabilities = nn.functional.softmax(logits, dim=1)\n",
    "    predicted_class_index = torch.argmax(probabilities, dim=1).item()\n",
    "    predicted_class_label = text_class_mapping[predicted_class_index]\n",
    "    \n",
    "    return predicted_class_label\n",
    "\n",
    "# Test the prediction functions\n",
    "image_path = 'image.jpg'  # Replace with the path to your test image\n",
    "input_text = \"This is a test text for evaluation.\"  # Replace with your test text\n",
    "\n",
    "predicted_image_class = predict_image_class(image_path)\n",
    "predicted_text_class = predict_text_class(input_text)\n",
    "\n",
    "print(f\"Predicted Image Class: {predicted_image_class}\")\n",
    "print(f\"Predicted Text Class: {predicted_text_class}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40bdb4b4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}