notebooks/Gender Gaze Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5fbdb62a-dab5-48f5-88cb-fd7c77df5c94",
   "metadata": {},
   "source": [
    "```\n",
    "Copyright 2021 Twitter, Inc.\n",
    "SPDX-License-Identifier: Apache-2.0\n",
    "```\n",
    "\n",
    "# Gender Gaze Analysis\n",
    "\n",
    "* This notebook prepares a dataset for gender gaze analysis. \n",
    "* It selects `MAX_FOUND` number of images\n",
    "* The selected images' saliency maps are stored in the folder `./gender_gaze/annotations/{GENDER}` with the same name as the image. \n",
    "* Each image's salienct segment regions are saved in a file with a suffix `_regions`\n",
    "* Once the images are generated you can look at the saliency map images and assess if the most salient point is on the face or not as well as if any non face area is getting detected as a salient region using the `_regions` file. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e59fa01-cd77-4357-ae38-cae5d49726c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import shlex\n",
    "import subprocess\n",
    "import sys\n",
    "from collections import namedtuple\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.image as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib.collections import PatchCollection\n",
    "from matplotlib.patches import Rectangle\n",
    "\n",
    "logging.basicConfig(level=logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcb97072-7807-4677-99dd-683ffd7c08c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import platform\n",
    "\n",
    "BIN_MAPS = {\"Darwin\": \"mac\", \"Linux\": \"linux\"}\n",
    "\n",
    "HOME_DIR = Path(\"../\").expanduser()\n",
    "\n",
    "try:\n",
    "    import google.colab\n",
    "    ! pip install pandas scikit-learn scikit-image statsmodels requests dash\n",
    "    ! [[ -d image-crop-analysis ]] || git clone https://github.com/twitter-research/image-crop-analysis.git\n",
    "    HOME_DIR = Path(\"./image-crop-analysis\").expanduser()\n",
    "    IN_COLAB = True\n",
    "except:\n",
    "    IN_COLAB = False\n",
    "\n",
    "sys.path.append(str(HOME_DIR / \"src\"))\n",
    "bin_dir = HOME_DIR / Path(\"./bin\")\n",
    "bin_path = bin_dir / BIN_MAPS[platform.system()] / \"candidate_crops\"\n",
    "model_path = bin_dir / \"fastgaze.vxm\"\n",
    "data_dir = HOME_DIR / Path(\"./data/\")\n",
    "data_dir.exists()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4721634-7e72-4d8f-8d8f-3621872ea81b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(data_dir / Path(\"dataset.tsv\"), sep=\"\\t\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f03afc2e-46de-41c5-afeb-d0d31807c466",
   "metadata": {},
   "outputs": [],
   "source": [
    "from crop_api import parse_output, ImageSaliencyModel, is_symmetric, reservoir_sampling\n",
    "from image_manipulation import get_image_saliency_map, process_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d58bb8bf-9cf2-46ab-bf71-d2d1b796e722",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ImageSaliencyModel(crop_binary_path=bin_path, crop_model_path=model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "434768f0-310d-402b-b610-0382648dd0b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "MAX_FOUND = 100\n",
    "for gender in df.sex_or_gender.unique(): \n",
    "    annotation_dir = data_dir / Path(f\"./gender_gaze/annotations/{gender}\")\n",
    "    annotation_dir.mkdir(parents=True, exist_ok=True)\n",
    "    found = 0\n",
    "    for img_path in df[df.sex_or_gender == gender].sample(frac = 1, random_state=42).local_path:\n",
    "        if not img_path.lower().endswith((\".jpg\", \".jpeg\")): continue\n",
    "        if found >= MAX_FOUND: break\n",
    "        img_path = data_dir / Path(f\"./images/{img_path}\")\n",
    "        if (annotation_dir / img_path.name).exists():\n",
    "            found += 1\n",
    "            continue\n",
    "        try:\n",
    "            img, image_label_overlay, regions, threshold = get_image_saliency_map(img_path, model)\n",
    "        except TypeError as e:\n",
    "            print(img_path, e)\n",
    "            continue\n",
    "        img_shape = img.shape\n",
    "        n_regions = len([r for r in regions if r.area > 1000])\n",
    "        print(img_path.name, img_shape[0] / img_shape[1], n_regions)\n",
    "        if n_regions < 2 or (img_shape[0] / img_shape[1]) < 1.25:\n",
    "            # Only select images if it has more than 2 big regions (of area > 1000) and image is significantly tall. \n",
    "            continue\n",
    "        found += 1\n",
    "        process_image(img_path, model)\n",
    "        img_path_parts = img_path.name.rsplit(\".\", 1)\n",
    "        plt.savefig(annotation_dir / f\"{img_path_parts[0]}_regions.{img_path_parts[-1]}\", bbox_inches=\"tight\")\n",
    "        plt.close(\"all\")\n",
    "        model.plot_img_crops(img_path, aspectRatios=[1], topK=1)\n",
    "        plt.savefig(annotation_dir / img_path.name, bbox_inches=\"tight\")\n",
    "        plt.close(\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8935380-c931-4b64-95c9-3d5b5ec8a7d5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:image-crop-analysis]",
   "language": "python",
   "name": "conda-env-image-crop-analysis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}