01_load_data_to_raw.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af4af90a02ab40e592a3f3a703917cd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/108 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'title', 'text', 'url', 'wiki_id', 'views', 'paragraph_id', 'langs', 'emb'],\n",
       "    num_rows: 14995897\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_dataset(\"Cohere/wikipedia-22-12-de-embeddings\", split=\"train\")\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Nekrolog 2022</td>\n",
       "      <td>Dies ist eine Liste von im Jahr 2022 verstorbe...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12052792</td>\n",
       "      <td>12052792</td>\n",
       "      <td>4016.285400</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Deutschland (; Vollform des Staatennamens seit...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>0</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Das heutige Deutschland hat gut 84 Millionen E...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Auf dem Gebiet Deutschlands ist die Anwesenhei...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>2</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Die rasche Entwicklung vom Agrar- zum Industri...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>3</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995892</th>\n",
       "      <td>14995892</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>DB2 komplett in Richtung nativer Unterstützung...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>124</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995893</th>\n",
       "      <td>14995893</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Nachdem sich die eNF-Relationen am Ende bei de...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995894</th>\n",
       "      <td>14995894</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Jim Melton, der federführende Editor aller SQL...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995895</th>\n",
       "      <td>14995895</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Auch wenn sich das ISO-SQL-Gremium nicht dazu ...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995896</th>\n",
       "      <td>14995896</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Die Entwicklung und Implementierung eines so g...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14995897 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                                              title  \\\n",
       "0                0                                      Nekrolog 2022   \n",
       "1                1                                        Deutschland   \n",
       "2                2                                        Deutschland   \n",
       "3                3                                        Deutschland   \n",
       "4                4                                        Deutschland   \n",
       "...            ...                                                ...   \n",
       "14995892  14995892  Advanced Information Management Prototype (AIM-P)   \n",
       "14995893  14995893  Advanced Information Management Prototype (AIM-P)   \n",
       "14995894  14995894  Advanced Information Management Prototype (AIM-P)   \n",
       "14995895  14995895  Advanced Information Management Prototype (AIM-P)   \n",
       "14995896  14995896  Advanced Information Management Prototype (AIM-P)   \n",
       "\n",
       "                                                       text  \\\n",
       "0         Dies ist eine Liste von im Jahr 2022 verstorbe...   \n",
       "1         Deutschland (; Vollform des Staatennamens seit...   \n",
       "2         Das heutige Deutschland hat gut 84 Millionen E...   \n",
       "3         Auf dem Gebiet Deutschlands ist die Anwesenhei...   \n",
       "4         Die rasche Entwicklung vom Agrar- zum Industri...   \n",
       "...                                                     ...   \n",
       "14995892  DB2 komplett in Richtung nativer Unterstützung...   \n",
       "14995893  Nachdem sich die eNF-Relationen am Ende bei de...   \n",
       "14995894  Jim Melton, der federführende Editor aller SQL...   \n",
       "14995895  Auch wenn sich das ISO-SQL-Gremium nicht dazu ...   \n",
       "14995896  Die Entwicklung und Implementierung eines so g...   \n",
       "\n",
       "                                                   url   wiki_id        views  \\\n",
       "0         https://de.wikipedia.org/wiki?curid=12052792  12052792  4016.285400   \n",
       "1          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "2          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "3          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "4          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "...                                                ...       ...          ...   \n",
       "14995892  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995893  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995894  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995895  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995896  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "\n",
       "          paragraph_id  langs  \n",
       "0                    0     38  \n",
       "1                    0    295  \n",
       "2                    1    295  \n",
       "3                    2    295  \n",
       "4                    3    295  \n",
       "...                ...    ...  \n",
       "14995892           124      0  \n",
       "14995893           125      0  \n",
       "14995894           126      0  \n",
       "14995895           127      0  \n",
       "14995896           128      0  \n",
       "\n",
       "[14995897 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dataset.remove_columns(\"emb\")\n",
    "df = dataset.to_pandas()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(\"./data/wikipedia-22-12-de-no-emb-raw.pkl.gz\", compression=\"gzip\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}