scripts/1_2_py_EXTRACTION_edh-xml_files.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "W3ztOiWkjdYL"
   },
   "source": [
    "This notebook serves to get additional data from the EDH XML/epidoc files. The reason is that some information is missing in the API data.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mjj6VCcLkERG"
   },
   "outputs": [],
   "source": [
    "### REQUIREMENTS\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import zipfile\n",
    "import io\n",
    "import sddk\n",
    "from xmltodict3 import XmlTextToDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we turn to the download section of the EDH website, where we can find zip archives containing xml files with individual inscriptions. Instead of downloading them manually, we will download them directly into our Python environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# extract the download page\n",
    "resp = requests.get(\"https://edh.ub.uni-heidelberg.de/data/download\", headers={\"User-Agent\" : \"\"})\n",
    "url_text = resp.text"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "['download/edhEpidocDump_HD000001-HD010000.zip',\n 'download/edhEpidocDump_HD010001-HD020000.zip',\n 'download/edhEpidocDump_HD020001-HD030000.zip',\n 'download/edhEpidocDump_HD030001-HD040000.zip',\n 'download/edhEpidocDump_HD040001-HD050000.zip',\n 'download/edhEpidocDump_HD050001-HD060000.zip',\n 'download/edhEpidocDump_HD060001-HD070000.zip',\n 'download/edhEpidocDump_HD070001-HD080000.zip',\n 'download/edhEpidocDump_HD080001-HD082828.zip']"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract urls of individual zip archives for download\n",
    "download_urls = re.findall(\"download\\/edhEpidocDump_HD\\w+\\-\\w+\\.zip\", url_text)\n",
    "download_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD000001-HD010000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD010001-HD020000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD020001-HD030000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD030001-HD040000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD040001-HD050000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD050001-HD060000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD060001-HD070000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD070001-HD080000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD080001-HD082828.zip\n"
     ]
    },
    {
     "data": {
      "text/plain": "81883"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check how many files we have\n",
    "\n",
    "url_base = \"https://edh-www.adw.uni-heidelberg.de/\"\n",
    "\n",
    "filenames = []\n",
    "for d_url in download_urls:\n",
    "    url = url_base + d_url\n",
    "    print(url)\n",
    "    resp = requests.get(url, headers={'User-Agent': ''})\n",
    "    zipped = zipfile.ZipFile(io.BytesIO(resp.content))\n",
    "    ### names of all files within the zipped directory\n",
    "    namelist = zipped.namelist()\n",
    "    namelist = [file for file in namelist if \".xml\" in file]\n",
    "    filenames.extend(namelist)\n",
    "len(filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# collecting filenames from first url for testing\n",
    "\n",
    "url_base = \"https://edh-www.adw.uni-heidelberg.de/\"\n",
    "\n",
    "url = url_base + download_urls[0]\n",
    "resp = requests.get(url, headers={\"User-Agent\" : \"\"})\n",
    "zipped = zipfile.ZipFile(io.BytesIO(resp.content))\n",
    "### names of all files within the zipped directory\n",
    "namelist = zipped.namelist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "9928"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(namelist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_filecontent_from_filename(filename, zipped):\n",
    "    try:\n",
    "        return str(zipped.read(filename))\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "edh_filecontents = {}\n",
    "\n",
    "for filename in namelist:\n",
    "    edh_filecontents[filename] = get_filecontent_from_filename(filename, zipped)\n",
    "# transform it into dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "#soup = BeautifulSoup(edh_filecontents['HD000001.xml'])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "9928"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "outputs": [],
   "source": [
    "test_str = str(edh_filecontents[namelist[3]])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "outputs": [
    {
     "data": {
      "text/plain": "True"
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(edh_filecontents[namelist[1000]])\n",
    "# if you already have the data:\n",
    "#soup = BeautifulSoup(edh_filecontents['HD000001.xml'])\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'\n",
      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
      "<?xml-model href=\"http://www.stoa.org/epidoc/schema/latest/tei-epidoc.rng\" schematypens=\"http://relaxng.org/ns/structure/1.0\"?>\n",
      "<tei xmlns=\"http://www.tei-c.org/ns/1.0\">\n",
      " \\n\n",
      " <teiheader>\n",
      "  \\n\n",
      "  <filedesc>\n",
      "   \\n\n",
      "   <titlestmt>\n",
      "    \\n\n",
      "    <title>\n",
      "     Owner/artist inscription from Colonia Claudia Augusta? Agrippinensium &amp;ndash; K\\xc3\\xb6ln (Germania inferior)\n",
      "    </title>\n",
      "    \\n\n",
      "   </titlestmt>\n",
      "   \\n\n",
      "   <publicationstmt>\n",
      "    \\n\n",
      "    <authority>\n",
      "     Epigraphische Datenbank Heidelberg\n",
      "    </authority>\n",
      "    \\n\n",
      "    <availability>\n",
      "     \\n\n",
      "     <p>\n",
      "      Heidelberger Akademie der Wissenschaften\n",
      "     </p>\n",
      "     \\n\n",
      "     <licence target=\"http://creativecommons.org/licenses/by-sa/4.0/\">\n",
      "      This file is licensed under the Creative Commons Attribution-ShareAlike 4.0 license.\\n\n",
      "     </licence>\n",
      "     \\n\n",
      "    </availability>\n",
      "    \\n\n",
      "    <idno type=\"localID\">\n",
      "     HD081005\n",
      "    </idno>\n",
      "    <idno type=\"URI\">\n",
      "     https://edh-www.adw.uni-heidelberg.de/test/edh/inschrift/HD081005\n",
      "    </idno>\n",
      "   </publicationstmt>\n",
      "   \\n\n",
      "   <sourcedesc>\n",
      "    \\n\n",
      "    <msdesc>\n",
      "     \\n\n",
      "     <msidentifier>\n",
      "      \\n\n",
      "      <repository>\n",
      "      </repository>\n",
      "      \\n\n",
      "     </msidentifier>\n",
      "     \\n\n",
      "     <physdesc>\n",
      "      \\n\n",
      "      <objectdesc>\n",
      "       \\n\n",
      "       <supportdesc>\n",
      "        \\n\n",
      "        <support>\n",
      "         <objecttype ref=\"https://www.eagle-network.eu/voc/objtyp/lod/140\">\n",
      "          instrumentum domesticum\n",
      "         </objecttype>\n",
      "         <material ref=\"http://www.eagle-network.eu/voc/material/lod/131\">\n",
      "          Ton\n",
      "         </material>\n",
      "         <rs ref=\"http://www.eagle-network.eu/voc/decor/lod/1000\" type=\"deccoration\">\n",
      "          no\n",
      "         </rs>\n",
      "        </support>\n",
      "        \\n\n",
      "       </supportdesc>\n",
      "       \\n\n",
      "       <layoutdesc>\n",
      "        \\n\n",
      "        <layout>\n",
      "         description of text field/campus\n",
      "        </layout>\n",
      "        \\n\n",
      "       </layoutdesc>\n",
      "       \\n\n",
      "      </objectdesc>\n",
      "      \\n\n",
      "      <handdesc>\n",
      "       <ab>\n",
      "       </ab>\n",
      "      </handdesc>\n",
      "      \\n\n",
      "     </physdesc>\n",
      "     \\n\n",
      "     <history>\n",
      "      \\n\n",
      "      <origin>\n",
      "       <origplace>\n",
      "        <placename type=\"province\">\n",
      "         Germania inferior\n",
      "        </placename>\n",
      "        <placename ref=\"https://pleiades.stoa.org/places/108751\" type=\"ancient\">\n",
      "         Colonia Claudia Augusta? Agrippinensium\n",
      "        </placename>\n",
      "       </origplace>\n",
      "       <origdate>\n",
      "        <origdate datingmethod=\"http://en.wikipedia.org/wiki/Julian_calendar\" notafter-custom=\"0130\" notbefore-custom=\"0071\">\n",
      "         71 AD - 130 AD\n",
      "        </origdate>\n",
      "       </origdate>\n",
      "      </origin>\n",
      "      \\n\n",
      "      <provenance type=\"found\" when=\"2008\">\n",
      "       <placename ref=\"https://www.geonames.org/2921044\" type=\"country\">\n",
      "        Germany\n",
      "       </placename>\n",
      "       <placename ref=\"http://www.geonames.org/2886242\" type=\"modern\">\n",
      "        K\\xc3\\xb6ln\n",
      "       </placename>\n",
      "       <placename type=\"findspot\">\n",
      "        {Kurt-Hackenberg-Platz}, Arbeitsbereich 1608\n",
      "       </placename>\n",
      "       <placename type=\"region\">\n",
      "        Nordrhein-Westfalen\n",
      "       </placename>\n",
      "      </provenance>\n",
      "      \\n\n",
      "     </history>\n",
      "     \\n\n",
      "    </msdesc>\n",
      "    \\n\n",
      "   </sourcedesc>\n",
      "   \\n\n",
      "  </filedesc>\n",
      "  \\n\n",
      "  <encodingdesc>\n",
      "   \\n\n",
      "   <p>\n",
      "    Encoded following EpiDoc guidelines 9.2\n",
      "   </p>\n",
      "   \\n\n",
      "  </encodingdesc>\n",
      "  \\n\n",
      "  <profiledesc>\n",
      "   \\n\n",
      "   <langusage>\n",
      "    \\n\n",
      "    <language ident=\"en\">\n",
      "     English\n",
      "    </language>\n",
      "    \\n\n",
      "    <language ident=\"de\">\n",
      "     Deutsch\n",
      "    </language>\n",
      "    \\n\n",
      "    <language ident=\"lat\">\n",
      "     Latin\n",
      "    </language>\n",
      "    \\n\n",
      "   </langusage>\n",
      "   \\n\n",
      "   <textclass>\n",
      "    \\n\n",
      "    <keywords>\n",
      "     <term>\n",
      "      owner/artist inscription\n",
      "     </term>\n",
      "    </keywords>\n",
      "    \\n\n",
      "   </textclass>\n",
      "   \\n\n",
      "   <particdesc>\n",
      "    <p>\n",
      "    </p>\n",
      "   </particdesc>\n",
      "   \\n\n",
      "  </profiledesc>\n",
      "  \\n\n",
      "  <revisiondesc>\n",
      "   <change when=\"2019-11-14\" who=\"Feraudi\">\n",
      "    checked with photo\n",
      "   </change>\n",
      "  </revisiondesc>\n",
      "  \\n\n",
      " </teiheader>\n",
      " \\n\n",
      " <facsimile>\n",
      "  <graphic url=\"#\">\n",
      "  </graphic>\n",
      " </facsimile>\n",
      " \\n\n",
      " <text>\n",
      "  \\n\n",
      "  <body>\n",
      "   \\n\n",
      "   <div type=\"edition\" xml:space=\"preserve\">\n",
      "    <head>\n",
      "     Text\n",
      "    </head>\n",
      "    <div n=\"1\" type=\"textpart\">\n",
      "     <ab>\n",
      "      <lb n=\"1\">\n",
      "      </lb>\n",
      "      \\n\n",
      "      <expan>\n",
      "       <abbr>\n",
      "        Ol\n",
      "       </abbr>\n",
      "       <ex>\n",
      "        iva\n",
      "       </ex>\n",
      "      </expan>\n",
      "      <lb n=\"2\">\n",
      "      </lb>\n",
      "      <expan>\n",
      "       <abbr>\n",
      "        al\n",
      "       </abbr>\n",
      "       <ex>\n",
      "        ba\n",
      "       </ex>\n",
      "      </expan>\n",
      "     </ab>\n",
      "    </div>\n",
      "    <div n=\"2\" type=\"textpart\">\n",
      "     <ab>\n",
      "      <lb n=\"1\">\n",
      "      </lb>\n",
      "      <expan>\n",
      "       <abbr>\n",
      "        dul\n",
      "       </abbr>\n",
      "       <ex>\n",
      "        ci\n",
      "       </ex>\n",
      "      </expan>\n",
      "     </ab>\n",
      "    </div>\n",
      "    <div n=\"3\" type=\"textpart\">\n",
      "     <ab>\n",
      "      <lb n=\"0\">\n",
      "      </lb>\n",
      "      <gap quantity=\"1\" reason=\"lost\" unit=\"line\">\n",
      "      </gap>\n",
      "      \\n\n",
      "     </ab>\n",
      "    </div>\n",
      "   </div>\n",
      "   \\n\n",
      "   <div type=\"commentary\">\n",
      "    \\n\n",
      "    <p>\n",
      "     Pinselaufschriften (dipinto, titulus pictus). Inschrifttr\\xc3\\xa4ger: Amphore, Form Augst 21.\n",
      "    </p>\n",
      "    \\n\n",
      "   </div>\n",
      "   \\n\n",
      "   <div type=\"bibliography\">\n",
      "    \\n\n",
      "    <listbibl>\n",
      "     <bibl>\n",
      "      U. Ehmig, KJ 42, 2009, 410, Nr. 230; Foto u. Zeichung; Taf. 2, 230 (Zeichnung).\n",
      "     </bibl>\n",
      "     <bibl>\n",
      "     </bibl>\n",
      "    </listbibl>\n",
      "    \\n\n",
      "   </div>\n",
      "   \\n\n",
      "  </body>\n",
      "  \\n\n",
      " </text>\n",
      " \\n\n",
      "</tei>\n",
      "'\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# let's try to parse it as xml\n",
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "outputs": [
    {
     "data": {
      "text/plain": "<rs ref=\"http://www.eagle-network.eu/voc/decor/lod/1000\" type=\"deccoration\">no</rs>"
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.rs"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "outputs": [
    {
     "ename": "ParseError",
     "evalue": "syntax error: line 1, column 0 (<string>)",
     "output_type": "error",
     "traceback": [
      "Traceback \u001B[0;36m(most recent call last)\u001B[0m:\n",
      "  File \u001B[1;32m~/.conda/envs/workspace/lib/python3.10/site-packages/IPython/core/interactiveshell.py:3398\u001B[0m in \u001B[1;35mrun_code\u001B[0m\n    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  Input \u001B[1;32mIn [135]\u001B[0m in \u001B[1;35m<cell line: 1>\u001B[0m\n    XmlTextToDict(str(soup.find(\"listperson\")), ignore_namespace=True).get_dict()[\"listperson\"][\"person\"]\n",
      "  File \u001B[1;32m~/.conda/envs/workspace/lib/python3.10/site-packages/xmltodict3/xml_to_dict.py:216\u001B[0m in \u001B[1;35mget_dict\u001B[0m\n    xml_to_dict_node = self.get_xml_to_dict_node()\n",
      "  File \u001B[1;32m~/.conda/envs/workspace/lib/python3.10/site-packages/xmltodict3/xml_to_dict.py:227\u001B[0m in \u001B[1;35mget_xml_to_dict_node\u001B[0m\n    root_node = ElementTree.fromstring(self.xml_text)\n",
      "\u001B[0;36m  File \u001B[0;32m~/.conda/envs/workspace/lib/python3.10/xml/etree/ElementTree.py:1343\u001B[0;36m in \u001B[0;35mXML\u001B[0;36m\u001B[0m\n\u001B[0;31m    return parser.close()\u001B[0m\n",
      "\u001B[0;36m  File \u001B[0;32m<string>\u001B[0;36m\u001B[0m\n\u001B[0;31mParseError\u001B[0m\u001B[0;31m:\u001B[0m syntax error: line 1, column 0\n"
     ]
    }
   ],
   "source": [
    "XmlTextToDict(str(soup.find(\"listperson\")), ignore_namespace=True).get_dict()[\"listperson\"][\"person\"]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "data": {
      "text/plain": "[<placename type=\"province\">Baetica</placename>,\n <placename ref=\"#\" type=\"ancient\">Ipolcobulcula</placename>,\n <placename ref=\"https://www.geonames.org/2510769\" type=\"country\">Spain</placename>,\n <placename ref=\"http://www.geonames.org/2520151\" type=\"modern\">Carcabuey</placename>,\n <placename type=\"findspot\"></placename>,\n <placename type=\"region\">C\\xc3\\xb3rdoba</placename>]"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(\"placename\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<placename type=\"province\">Baetica</placename>\n",
      "<placename ref=\"#\" type=\"ancient\">Ipolcobulcula</placename>\n",
      "<placename ref=\"https://www.geonames.org/2510769\" type=\"country\">Spain</placename>\n",
      "<placename ref=\"http://www.geonames.org/2520151\" type=\"modern\">Carcabuey</placename>\n",
      "<placename type=\"findspot\"></placename>\n",
      "<placename type=\"region\">C\\xc3\\xb3rdoba</placename>\n"
     ]
    }
   ],
   "source": [
    "placenames_refs = []\n",
    "for placename in soup.find_all(\"placename\"):\n",
    "    try:\n",
    "        print(placename)\n",
    "        placenames_refs.append({\"type\" : placename[\"type\"], \"text\" : placename.get_text(), \"ref\" : placename[\"ref\"]})\n",
    "    except:\n",
    "        try:\n",
    "            placenames_refs.append({\"type\" : placename[\"type\"], \"text\" : placename.get_text()})\n",
    "        except:\n",
    "            pass"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "outputs": [
    {
     "data": {
      "text/plain": "[{'type': 'province', 'text': 'Baetica'},\n {'type': 'ancient', 'text': 'Ipolcobulcula', 'ref': '#'},\n {'type': 'country',\n  'text': 'Spain',\n  'ref': 'https://www.geonames.org/2510769'},\n {'type': 'modern',\n  'text': 'Carcabuey',\n  'ref': 'http://www.geonames.org/2520151'},\n {'type': 'findspot', 'text': ''},\n {'type': 'region', 'text': 'C\\\\xc3\\\\xb3rdoba'}]"
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "placenames_refs = []\n",
    "for placename in soup.find_all(\"placename\"):\n",
    "    placename_dict = {}\n",
    "    try:\n",
    "        placename_dict[\"type\"] = placename[\"type\"]\n",
    "        try:\n",
    "            placename_dict[\"text\"] = placename.get_text()\n",
    "        except:\n",
    "            pass\n",
    "        try:\n",
    "            placename_dict[\"ref\"] = placename[\"ref\"]\n",
    "        except:\n",
    "            pass\n",
    "    except:\n",
    "        pass\n",
    "    placenames_refs.append(placename_dict)\n",
    "placenames_refs"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "[{'type': 'province', 'text': 'Baetica'},\n {'type': 'ancient', 'text': 'Ipolcobulcula', 'ref': '#'},\n {'type': 'country',\n  'text': 'Spain',\n  'ref': 'https://www.geonames.org/2510769'},\n {'type': 'modern',\n  'text': 'Carcabuey',\n  'ref': 'http://www.geonames.org/2520151'},\n {'type': 'findspot', 'text': ''},\n {'type': 'region', 'text': 'C\\\\xc3\\\\xb3rdoba'}]"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "placenames_refs"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\n AVSLLA Marci Porci Nigri serva dominae Veneri aram posuit\\n\n"
     ]
    }
   ],
   "source": [
    "text_tag = soup.find(\"div\", attrs={\"type\" : \"edition\"})\n",
    "text_edition = \"\"\n",
    "for ab in text_tag.find_all(\"ab\"):\n",
    "    text_edition += str(ab.get_text())\n",
    "text_edition = \" \".join(\" \".join(text_edition.splitlines()).split())\n",
    "print(text_edition.rstrip())"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "outputs": [
    {
     "data": {
      "text/plain": "'http://www.eagle-network.eu/voc/material/lod/131'"
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.support.find(\"material\")[\"ref\"]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<rs ref=\"http://www.eagle-network.eu/voc/decor/lod/1000\" type=\"deccoration\">no</rs>\n"
     ]
    }
   ],
   "source": [
    "for rs in soup.find_all(\"rs\"):\n",
    "    print(rs)\n",
    "#    [\"ref\"]\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "outputs": [
    {
     "data": {
      "text/plain": "'http://www.eagle-network.eu/voc/decor/lod/1000'"
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.support.find(\"rs\")[\"ref\"]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define function for data parsing\n",
    "def get_data_from_filename(filename):\n",
    "    try:\n",
    "        soup = BeautifulSoup(edh_filecontents[filename])\n",
    "        xml_data = {}\n",
    "        idno_uri = soup.find(\"idno\", attrs={\"type\" : \"URI\"}).get_text()\n",
    "        xml_data[\"idno_uri\"] = idno_uri.rpartition(\"/\")[2]\n",
    "        try:\n",
    "            xml_data[\"idno_tm\"] = soup.find(\"idno\", attrs={\"type\" : \"TM\"}).get_text()\n",
    "        except:\n",
    "            xml_data[\"idno_tm\"] = None\n",
    "        placenames_refs = []\n",
    "        for placename in soup.find_all(\"placename\"):\n",
    "            placename_dict = {}\n",
    "            try:\n",
    "                placename_dict[\"type\"] = placename[\"type\"]\n",
    "                try:\n",
    "                    placename_dict[\"text\"] = placename.get_text()\n",
    "                except:\n",
    "                    pass\n",
    "                try:\n",
    "                    placename_dict[\"ref\"] = placename[\"ref\"]\n",
    "                except:\n",
    "                    pass\n",
    "            except:\n",
    "                pass\n",
    "            placenames_refs.append(placename_dict)\n",
    "        xml_data[\"placenames_refs\"] = placenames_refs\n",
    "        text_tag = soup.find(\"div\", attrs={\"type\" : \"edition\"})\n",
    "        text_edition = \"\"\n",
    "        for ab in text_tag.find_all(\"ab\"):\n",
    "            text_edition += str(ab.get_text())\n",
    "        text_edition = \" \".join(\" \".join(text_edition.splitlines()).split())\n",
    "        xml_data[\"text_edition\"] = text_edition\n",
    "        xml_data[\"origdate_text\"] = soup.find(\"origdate\").get_text().replace(\"\\n\", \"\")\n",
    "        try: \n",
    "            layout_execution = soup.layout.find(\"rs\")[\"ref\"]\n",
    "            xml_data[\"layout_execution\"] = layout_execution.rpartition(\"/\")[2]\n",
    "        except:\n",
    "            xml_data[\"layout_execution\"] = None\n",
    "        try:\n",
    "            xml_data[\"layout_execution_text\"] = soup.layout.rs.get_text()\n",
    "        except:\n",
    "            xml_data[\"layout_execution_text\"] = None\n",
    "        try: \n",
    "            support_objecttype = soup.support.find(\"objecttype\")[\"ref\"]\n",
    "            xml_data[\"support_objecttype\"] = support_objecttype.rpartition(\"/\")[2]\n",
    "        except:\n",
    "            xml_data[\"support_objecttype\"] = None\n",
    "        try:\n",
    "            xml_data [\"support_objecttype_text\"] = soup.support.objecttype.get_text()\n",
    "        except:\n",
    "            xml_data [\"support_objecttype_text\"] = None\n",
    "        try: \n",
    "            support_material = soup.support.find(\"material\")[\"ref\"]\n",
    "            xml_data[\"support_material\"] = support_material.rpartition(\"/\")[2]\n",
    "        except:\n",
    "            xml_data[\"support_material\"] = None\n",
    "        try:\n",
    "            xml_data[\"support_material_text\"] = soup.support.material.get_text()\n",
    "        except:\n",
    "            xml_data[\"support_material_text\"] = None\n",
    "        try: \n",
    "            support_decoration = soup.support.find(\"rs\")[\"ref\"]\n",
    "            xml_data[\"support_decoration\"] = support_decoration.rpartition(\"/\")[2]\n",
    "        except:\n",
    "            xml_data[\"support_decoration\"] = None\n",
    "        try: \n",
    "            keywords_term = soup.keywords.find(\"term\")[\"ref\"]\n",
    "            xml_data[\"keywords_term\"] = keywords_term.rpartition(\"/\")[2]\n",
    "        except:\n",
    "            xml_data[\"keywords_term\"] = None\n",
    "        try:\n",
    "            xml_data[\"keywords_term_text\"] = soup.keywords.get_text().replace(\"\\n\", \"\")\n",
    "        except:\n",
    "            xml_data[\"keywords_term_text\"] = None\n",
    "        try:\n",
    "            xml_data[\"people\"] = XmlTextToDict(str(soup.find(\"listperson\")), ignore_namespace=True).get_dict()[\"listperson\"][\"person\"]\n",
    "        except:\n",
    "            pass\n",
    "        return xml_data\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "   idno_uri idno_tm                                    placenames_refs  \\\n0  HD080001    None  [{'type': 'province', 'text': 'Germania superi...   \n1  HD080002    None  [{'type': 'province', 'text': 'Germania superi...   \n2  HD080003    None  [{'type': 'province', 'text': 'Germania superi...   \n3  HD080004    None  [{'type': 'province', 'text': 'Germania superi...   \n4  HD080005    None  [{'type': 'province', 'text': 'Germania superi...   \n5  HD080006    None  [{'type': 'province', 'text': 'Germania superi...   \n6  HD080007    None  [{'type': 'province', 'text': 'Germania superi...   \n7  HD080008    None  [{'type': 'province', 'text': 'Germania superi...   \n8  HD080009    None  [{'type': 'province', 'text': 'Germania superi...   \n9  HD080010    None  [{'type': 'province', 'text': 'Germania superi...   \n\n                             text_edition   origdate_text layout_execution  \\\n0                                  \\n E\\n   1 AD - 300 AD             None   \n1                                 \\n IT\\n  71 AD - 250 AD             None   \n2                               \\n ME D\\n   1 AD - 300 AD             None   \n3  \\n centuria Semproni Arrunti Aquilae\\n    1 AD - 50 AD             None   \n4                                 \\n CA\\n   1 AD - 300 AD             None   \n5                                  \\n M\\n  71 AD - 250 AD             None   \n6                               \\n V T \\n  71 AD - 250 AD             None   \n7                                  \\n V\\n  71 AD - 250 AD             None   \n8                                \\n V S\\n  71 AD - 250 AD             None   \n9                                  \\n R\\n  71 AD - 250 AD             None   \n\n  layout_execution_text support_objecttype support_objecttype_text  \\\n0                  None               None                    None   \n1                  None                  #                 tabula?   \n2                  None               None                    None   \n3                  None                 42                  weapon   \n4                  None                257                  tabula   \n5                  None                257                  tabula   \n6                  None                257                  tabula   \n7                  None                  #                 tabula?   \n8                  None                257                  tabula   \n9                  None                257                  tabula   \n\n  support_material support_material_text support_decoration keywords_term  \\\n0               60             Kalkstein               1000           143   \n1               60             Kalkstein               1000           143   \n2               60             Kalkstein               1000           143   \n3              109                Bronze               1000          None   \n4               60             Kalkstein               1000           143   \n5               60             Kalkstein               1000           143   \n6               60             Kalkstein               1000           143   \n7               60             Kalkstein               1000           143   \n8               60             Kalkstein               1000           143   \n9               60             Kalkstein               1000           143   \n\n         keywords_term_text                                             people  \n0                   unknown                                                NaN  \n1                   unknown                                                NaN  \n2                   unknown                                                NaN  \n3  owner/artist inscription  [{'persname': {'name': {'@type': 'nomen', '#te...  \n4                   unknown                                                NaN  \n5                   unknown                                                NaN  \n6                   unknown                                                NaN  \n7                   unknown                                                NaN  \n8                   unknown                                                NaN  \n9                   unknown                                                NaN  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>idno_uri</th>\n      <th>idno_tm</th>\n      <th>placenames_refs</th>\n      <th>text_edition</th>\n      <th>origdate_text</th>\n      <th>layout_execution</th>\n      <th>layout_execution_text</th>\n      <th>support_objecttype</th>\n      <th>support_objecttype_text</th>\n      <th>support_material</th>\n      <th>support_material_text</th>\n      <th>support_decoration</th>\n      <th>keywords_term</th>\n      <th>keywords_term_text</th>\n      <th>people</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>HD080001</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n E\\n</td>\n      <td>1 AD - 300 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>HD080002</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n IT\\n</td>\n      <td>71 AD - 250 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>#</td>\n      <td>tabula?</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>HD080003</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n ME D\\n</td>\n      <td>1 AD - 300 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>HD080004</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n centuria Semproni Arrunti Aquilae\\n</td>\n      <td>1 AD - 50 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>42</td>\n      <td>weapon</td>\n      <td>109</td>\n      <td>Bronze</td>\n      <td>1000</td>\n      <td>None</td>\n      <td>owner/artist inscription</td>\n      <td>[{'persname': {'name': {'@type': 'nomen', '#te...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>HD080005</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n CA\\n</td>\n      <td>1 AD - 300 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>HD080006</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n M\\n</td>\n      <td>71 AD - 250 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>HD080007</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n V T \\n</td>\n      <td>71 AD - 250 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>HD080008</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n V\\n</td>\n      <td>71 AD - 250 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>#</td>\n      <td>tabula?</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>HD080009</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n V S\\n</td>\n      <td>71 AD - 250 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>HD080010</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Germania superi...</td>\n      <td>\\n R\\n</td>\n      <td>71 AD - 250 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>143</td>\n      <td>unknown</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test with first ten files within the namelist\n",
    "edh_xml_data = []\n",
    "edh_filecontents = {}\n",
    "for filename in namelist[:10]:\n",
    "    edh_filecontents[filename] = get_filecontent_from_filename(filename, zipped)\n",
    "    edh_xml_data.append(get_data_from_filename(filename))\n",
    "# transform it into dataframe\n",
    "pd.DataFrame(edh_xml_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "                    columns\n0                  idno_uri\n1                   idno_tm\n2           placenames_refs\n3              text_edition\n4             origdate_text\n5          layout_execution\n6     layout_execution_text\n7        support_objecttype\n8   support_objecttype_text\n9          support_material\n10    support_material_text\n11       support_decoration\n12            keywords_term\n13       keywords_term_text\n14                   people",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>columns</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>idno_uri</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>idno_tm</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>placenames_refs</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>text_edition</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>origdate_text</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>layout_execution</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>layout_execution_text</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>support_objecttype</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>support_objecttype_text</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>support_material</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>support_material_text</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>support_decoration</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>keywords_term</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>keywords_term_text</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>people</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EDH_xml_cols = pd.DataFrame(pd.DataFrame(edh_xml_data).columns, columns=[\"columns\"])\n",
    "EDH_xml_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract xml files content from the zip files as raw strings "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD000001-HD010000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD010001-HD020000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD020001-HD030000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD030001-HD040000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD040001-HD050000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD050001-HD060000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD060001-HD070000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD070001-HD080000.zip\n",
      "https://edh-www.adw.uni-heidelberg.de/download/edhEpidocDump_HD080001-HD082828.zip\n",
      "CPU times: user 4.2 s, sys: 458 ms, total: 4.66 s\n",
      "Wall time: 11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# main loop\n",
    "\n",
    "edh_filecontents = {}\n",
    "\n",
    "url_base = \"https://edh-www.adw.uni-heidelberg.de/\"\n",
    "\n",
    "for d_url in download_urls:\n",
    "    url = url_base + d_url\n",
    "    print(url)\n",
    "    resp = requests.get(url, headers={'User-Agent': ''})\n",
    "    zipped = zipfile.ZipFile(io.BytesIO(resp.content))\n",
    "    ### names of all files within the zipped directory\n",
    "    namelist = zipped.namelist()\n",
    "    for filename in namelist:\n",
    "        try:        \n",
    "            # original: edh_xml_data.append(get_data_from_filename(filename, zipped))\n",
    "            edh_filecontents[filename] = get_filecontent_from_filename(filename, zipped)\n",
    "        except:\n",
    "            pass\n",
    "        ### index \"0\" is for main directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "81883"
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(edh_filecontents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [],
   "source": [
    "#let's connect to sciencedata and save the raw xmls there"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "connection with shared folder established with you as its owner\n",
      "endpoint variable has been configured to: https://sciencedata.dk/files/SDAM_root/SDAM_data/EDH/\n"
     ]
    }
   ],
   "source": [
    "s = sddk.cloudSession(\"sciencedata.dk\", \"SDAM_root/SDAM_data/EDH\", \"648597@au.dk\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A file with the same name (\"edh_raw_xmls_2022-11-02.json\") already exists in this location.\n",
      "Your <class 'dict'> object has been succesfully written as \"https://sciencedata.dk/files/SDAM_root/SDAM_data/EDH/edh_raw_xmls_2022-11-02.json\"\n"
     ]
    }
   ],
   "source": [
    "s.write_file(\"edh_raw_xmls_2022-11-02.json\", edh_filecontents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "outputs": [],
   "source": [
    "# read it back...\n",
    "edh_filecontents = s.read_file(\"edh_raw_xmls_2022-11-02.json\", \"dict\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "outputs": [
    {
     "data": {
      "text/plain": "81883"
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(edh_filecontents)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "outputs": [
    {
     "data": {
      "text/plain": "183726"
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lets look at what is within raw xml strings\n",
    "# how many references to http://www.eagle-network.eu/ are there?\n",
    "eagle_n = 0\n",
    "for filename in edh_filecontents.keys():\n",
    "    eagle_n = eagle_n + str(edh_filecontents[filename]).count(\"http://www.eagle-network.eu/\")\n",
    "eagle_n # in april 183726"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "edh_xml_lens = []\n",
    "for filename in edh_filecontents.keys():\n",
    "    edh_xml_lens.append((filename, len(edh_filecontents[filename])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[('HD016471.xml', 3458),\n ('HD026527.xml', 3459),\n ('HD015306.xml', 3472),\n ('HD051245.xml', 3475),\n ('HD051249.xml', 3475),\n ('HD051252.xml', 3477),\n ('HD051253.xml', 3477),\n ('HD051246.xml', 3488),\n ('HD051250.xml', 3488),\n ('HD019186.xml', 3491),\n ('HD019254.xml', 3495),\n ('HD033450.xml', 3496),\n ('HD007072.xml', 3497),\n ('HD051247.xml', 3501),\n ('HD019257.xml', 3502),\n ('HD007096.xml', 3503),\n ('HD007093.xml', 3508),\n ('HD007099.xml', 3508),\n ('HD007458.xml', 3508),\n ('HD073406.xml', 3509),\n ('HD010645.xml', 3512),\n ('HD029145.xml', 3513),\n ('HD029148.xml', 3513),\n ('HD050900.xml', 3514),\n ('HD051248.xml', 3514),\n ('HD014773.xml', 3516),\n ('HD012717.xml', 3517),\n ('HD022906.xml', 3517),\n ('HD031206.xml', 3517),\n ('HD076796.xml', 3517),\n ('HD017988.xml', 3522),\n ('HD048216.xml', 3522),\n ('HD015992.xml', 3523),\n ('HD048231.xml', 3525),\n ('HD048253.xml', 3525),\n ('HD017575.xml', 3527),\n ('HD051251.xml', 3527),\n ('HD011124.xml', 3528),\n ('HD017116.xml', 3529),\n ('HD048240.xml', 3529),\n ('HD021168.xml', 3530),\n ('HD021402.xml', 3531),\n ('HD011954.xml', 3533),\n ('HD017395.xml', 3536),\n ('HD018182.xml', 3536),\n ('HD017852.xml', 3537),\n ('HD015241.xml', 3539),\n ('HD014818.xml', 3540),\n ('HD014761.xml', 3541),\n ('HD013708.xml', 3542),\n ('HD007909.xml', 3543),\n ('HD027024.xml', 3543),\n ('HD013726.xml', 3544),\n ('HD007390.xml', 3547),\n ('HD007393.xml', 3547),\n ('HD012500.xml', 3547),\n ('HD016004.xml', 3548),\n ('HD010811.xml', 3549),\n ('HD018695.xml', 3549),\n ('HD020467.xml', 3549),\n ('HD021226.xml', 3550),\n ('HD023771.xml', 3550),\n ('HD027021.xml', 3550),\n ('HD027094.xml', 3551),\n ('HD019095.xml', 3552),\n ('HD015627.xml', 3553),\n ('HD016671.xml', 3553),\n ('HD019893.xml', 3553),\n ('HD023765.xml', 3553),\n ('HD012494.xml', 3554),\n ('HD012575.xml', 3554),\n ('HD012497.xml', 3556),\n ('HD018659.xml', 3556),\n ('HD011139.xml', 3557),\n ('HD018441.xml', 3557),\n ('HD009079.xml', 3559),\n ('HD011455.xml', 3559),\n ('HD026805.xml', 3559),\n ('HD013780.xml', 3561),\n ('HD031071.xml', 3561),\n ('HD013713.xml', 3562),\n ('HD019294.xml', 3562),\n ('HD028266.xml', 3563),\n ('HD017057.xml', 3564),\n ('HD017689.xml', 3564),\n ('HD008205.xml', 3565),\n ('HD013928.xml', 3566),\n ('HD017162.xml', 3566),\n ('HD013720.xml', 3567),\n ('HD021738.xml', 3568),\n ('HD010594.xml', 3569),\n ('HD013677.xml', 3569),\n ('HD019036.xml', 3569),\n ('HD013717.xml', 3570),\n ('HD015294.xml', 3570),\n ('HD023054.xml', 3570),\n ('HD025386.xml', 3570),\n ('HD027609.xml', 3571),\n ('HD014285.xml', 3572),\n ('HD019678.xml', 3572),\n ('HD013699.xml', 3573),\n ('HD014752.xml', 3573),\n ('HD020372.xml', 3573),\n ('HD028377.xml', 3573),\n ('HD017757.xml', 3574),\n ('HD023995.xml', 3574),\n ('HD022251.xml', 3575),\n ('HD049610.xml', 3575),\n ('HD050885.xml', 3575),\n ('HD008708.xml', 3576),\n ('HD015357.xml', 3576),\n ('HD019681.xml', 3576),\n ('HD051219.xml', 3577),\n ('HD009459.xml', 3579),\n ('HD016984.xml', 3579),\n ('HD018159.xml', 3579),\n ('HD019161.xml', 3579),\n ('HD019897.xml', 3579),\n ('HD054517.xml', 3579),\n ('HD009579.xml', 3580),\n ('HD017858.xml', 3580),\n ('HD018800.xml', 3580),\n ('HD031203.xml', 3580),\n ('HD044842.xml', 3580),\n ('HD048241.xml', 3580),\n ('HD007222.xml', 3581),\n ('HD013671.xml', 3581),\n ('HD014267.xml', 3581),\n ('HD014654.xml', 3581),\n ('HD016180.xml', 3581),\n ('HD018523.xml', 3581),\n ('HD022420.xml', 3581),\n ('HD054515.xml', 3581),\n ('HD012152.xml', 3582),\n ('HD018938.xml', 3582),\n ('HD011552.xml', 3584),\n ('HD017754.xml', 3584),\n ('HD026823.xml', 3584),\n ('HD044841.xml', 3584),\n ('HD054516.xml', 3584),\n ('HD013883.xml', 3585),\n ('HD017483.xml', 3585),\n ('HD011407.xml', 3586),\n ('HD013877.xml', 3586),\n ('HD013880.xml', 3586),\n ('HD011537.xml', 3587),\n ('HD020219.xml', 3587),\n ('HD015375.xml', 3588),\n ('HD028686.xml', 3588),\n ('HD010258.xml', 3589),\n ('HD017969.xml', 3589),\n ('HD024898.xml', 3589),\n ('HD030240.xml', 3589),\n ('HD013524.xml', 3590),\n ('HD009900.xml', 3591),\n ('HD011951.xml', 3591),\n ('HD014861.xml', 3591),\n ('HD020458.xml', 3591),\n ('HD021339.xml', 3591),\n ('HD020452.xml', 3592),\n ('HD033135.xml', 3592),\n ('HD051841.xml', 3592),\n ('HD053898.xml', 3592),\n ('HD015109.xml', 3593),\n ('HD020477.xml', 3593),\n ('HD020712.xml', 3593),\n ('HD020455.xml', 3594),\n ('HD022248.xml', 3594),\n ('HD048204.xml', 3594),\n ('HD050901.xml', 3594),\n ('HD007819.xml', 3595),\n ('HD014096.xml', 3595),\n ('HD028158.xml', 3595),\n ('HD028551.xml', 3595),\n ('HD040726.xml', 3595),\n ('HD007975.xml', 3596),\n ('HD011784.xml', 3596),\n ('HD012647.xml', 3596),\n ('HD012984.xml', 3596),\n ('HD018148.xml', 3596),\n ('HD007206.xml', 3597),\n ('HD007221.xml', 3597),\n ('HD012066.xml', 3597),\n ('HD012638.xml', 3597),\n ('HD062077.xml', 3597),\n ('HD076751.xml', 3597),\n ('HD010895.xml', 3598),\n ('HD014282.xml', 3598),\n ('HD016825.xml', 3598),\n ('HD012644.xml', 3599),\n ('HD022422.xml', 3599),\n ('HD022939.xml', 3599),\n ('HD023400.xml', 3599),\n ('HD007203.xml', 3600),\n ('HD007368.xml', 3600),\n ('HD013729.xml', 3600),\n ('HD015303.xml', 3600),\n ('HD016814.xml', 3600),\n ('HD021637.xml', 3600),\n ('HD021810.xml', 3600),\n ('HD030225.xml', 3600),\n ('HD011458.xml', 3601),\n ('HD012656.xml', 3601),\n ('HD012981.xml', 3601),\n ('HD013807.xml', 3601),\n ('HD030222.xml', 3601),\n ('HD030723.xml', 3601),\n ('HD010402.xml', 3602),\n ('HD012659.xml', 3602),\n ('HD013889.xml', 3602),\n ('HD015495.xml', 3602),\n ('HD016034.xml', 3602),\n ('HD048213.xml', 3602),\n ('HD007090.xml', 3603),\n ('HD012090.xml', 3603),\n ('HD018173.xml', 3603),\n ('HD011930.xml', 3604),\n ('HD049616.xml', 3604),\n ('HD010104.xml', 3605),\n ('HD014707.xml', 3605),\n ('HD009848.xml', 3606),\n ('HD017146.xml', 3606),\n ('HD017373.xml', 3606),\n ('HD018147.xml', 3606),\n ('HD021261.xml', 3606),\n ('HD022022.xml', 3606),\n ('HD027898.xml', 3606),\n ('HD051913.xml', 3606),\n ('HD012708.xml', 3607),\n ('HD012772.xml', 3607),\n ('HD013647.xml', 3607),\n ('HD022213.xml', 3607),\n ('HD024492.xml', 3607),\n ('HD032817.xml', 3607),\n ('HD011699.xml', 3608),\n ('HD012125.xml', 3608),\n ('HD021462.xml', 3608),\n ('HD022714.xml', 3608),\n ('HD027348.xml', 3608),\n ('HD032718.xml', 3608),\n ('HD048252.xml', 3608),\n ('HD008918.xml', 3609),\n ('HD010763.xml', 3609),\n ('HD017311.xml', 3609),\n ('HD020793.xml', 3609),\n ('HD020863.xml', 3609),\n ('HD021600.xml', 3609),\n ('HD023933.xml', 3609),\n ('HD025681.xml', 3609),\n ('HD010393.xml', 3610),\n ('HD012503.xml', 3610),\n ('HD014212.xml', 3610),\n ('HD014221.xml', 3610),\n ('HD016031.xml', 3610),\n ('HD019803.xml', 3610),\n ('HD048081.xml', 3610),\n ('HD052545.xml', 3610),\n ('HD012796.xml', 3611),\n ('HD017178.xml', 3611),\n ('HD018606.xml', 3611),\n ('HD019663.xml', 3611),\n ('HD020549.xml', 3611),\n ('HD021118.xml', 3611),\n ('HD021451.xml', 3611),\n ('HD023786.xml', 3611),\n ('HD059786.xml', 3611),\n ('HD011909.xml', 3612),\n ('HD011949.xml', 3612),\n ('HD012921.xml', 3612),\n ('HD013530.xml', 3612),\n ('HD016904.xml', 3612),\n ('HD019453.xml', 3612),\n ('HD023207.xml', 3612),\n ('HD023909.xml', 3612),\n ('HD018259.xml', 3613),\n ('HD023359.xml', 3613),\n ('HD024408.xml', 3613),\n ('HD032601.xml', 3613),\n ('HD032823.xml', 3613),\n ('HD048212.xml', 3613),\n ('HD008069.xml', 3614),\n ('HD011981.xml', 3614),\n ('HD015491.xml', 3614),\n ('HD016954.xml', 3614),\n ('HD021890.xml', 3614),\n ('HD022087.xml', 3614),\n ('HD046149.xml', 3614),\n ('HD015738.xml', 3615),\n ('HD019432.xml', 3615),\n ('HD020077.xml', 3615),\n ('HD020872.xml', 3615),\n ('HD030219.xml', 3615),\n ('HD032442.xml', 3615),\n ('HD053036.xml', 3615),\n ('HD054082.xml', 3615),\n ('HD021899.xml', 3616),\n ('HD023783.xml', 3616),\n ('HD024372.xml', 3616),\n ('HD032190.xml', 3616),\n ('HD067772.xml', 3616),\n ('HD014273.xml', 3617),\n ('HD015777.xml', 3617),\n ('HD016351.xml', 3617),\n ('HD020866.xml', 3617),\n ('HD021242.xml', 3617),\n ('HD023792.xml', 3617),\n ('HD032439.xml', 3617),\n ('HD015139.xml', 3618),\n ('HD017700.xml', 3618),\n ('HD020893.xml', 3618),\n ('HD023795.xml', 3618),\n ('HD027420.xml', 3618),\n ('HD032436.xml', 3618),\n ('HD006996.xml', 3619),\n ('HD007230.xml', 3619),\n ('HD012056.xml', 3619),\n ('HD013357.xml', 3619),\n ('HD017249.xml', 3619),\n ('HD018770.xml', 3619),\n ('HD023798.xml', 3619),\n ('HD032445.xml', 3619),\n ('HD007257.xml', 3620),\n ('HD007387.xml', 3620),\n ('HD017300.xml', 3620),\n ('HD021757.xml', 3620),\n ('HD022500.xml', 3620),\n ('HD022849.xml', 3620),\n ('HD045201.xml', 3621),\n ('HD013494.xml', 3622),\n ('HD015885.xml', 3622),\n ('HD017848.xml', 3622),\n ('HD018902.xml', 3622),\n ('HD020634.xml', 3622),\n ('HD024583.xml', 3622),\n ('HD045196.xml', 3622),\n ('HD007350.xml', 3623),\n ('HD014224.xml', 3623),\n ('HD014227.xml', 3623),\n ('HD019998.xml', 3623),\n ('HD024022.xml', 3623),\n ('HD045197.xml', 3623),\n ('HD045198.xml', 3623),\n ('HD008922.xml', 3624),\n ('HD015056.xml', 3624),\n ('HD017975.xml', 3624),\n ('HD022354.xml', 3624),\n ('HD025498.xml', 3624),\n ('HD045200.xml', 3624),\n ('HD023601.xml', 3625),\n ('HD023693.xml', 3625),\n ('HD027358.xml', 3625),\n ('HD031511.xml', 3625),\n ('HD054518.xml', 3625),\n ('HD010255.xml', 3626),\n ('HD011130.xml', 3626),\n ('HD011808.xml', 3626),\n ('HD013056.xml', 3626),\n ('HD013862.xml', 3626),\n ('HD018142.xml', 3626),\n ('HD030798.xml', 3626),\n ('HD054519.xml', 3626),\n ('HD059705.xml', 3626),\n ('HD003356.xml', 3627),\n ('HD013372.xml', 3627),\n ('HD014749.xml', 3627),\n ('HD015760.xml', 3627),\n ('HD026358.xml', 3627),\n ('HD048219.xml', 3627),\n ('HD014218.xml', 3628),\n ('HD022426.xml', 3628),\n ('HD048214.xml', 3628),\n ('HD012161.xml', 3629),\n ('HD012915.xml', 3629),\n ('HD015121.xml', 3629),\n ('HD015253.xml', 3629),\n ('HD015630.xml', 3629),\n ('HD020195.xml', 3629),\n ('HD024588.xml', 3629),\n ('HD073937.xml', 3629),\n ('HD015127.xml', 3630),\n ('HD020461.xml', 3630),\n ('HD030804.xml', 3630),\n ('HD015396.xml', 3631),\n ('HD016684.xml', 3631),\n ('HD026550.xml', 3631),\n ('HD006121.xml', 3632),\n ('HD007025.xml', 3632),\n ('HD014720.xml', 3632),\n ('HD015643.xml', 3632),\n ('HD018049.xml', 3632),\n ('HD019489.xml', 3632),\n ('HD022417.xml', 3632),\n ('HD049617.xml', 3632),\n ('HD014719.xml', 3633),\n ('HD015052.xml', 3633),\n ('HD027240.xml', 3633),\n ('HD030216.xml', 3633),\n ('HD041979.xml', 3633),\n ('HD012906.xml', 3634),\n ('HD013065.xml', 3634),\n ('HD014290.xml', 3634),\n ('HD014770.xml', 3634),\n ('HD028224.xml', 3634),\n ('HD052879.xml', 3634),\n ('HD016010.xml', 3635),\n ('HD016681.xml', 3635),\n ('HD016796.xml', 3635),\n ('HD017021.xml', 3635),\n ('HD030717.xml', 3635),\n ('HD045199.xml', 3635),\n ('HD051842.xml', 3635),\n ('HD007436.xml', 3636),\n ('HD013282.xml', 3636),\n ('HD013892.xml', 3636),\n ('HD015869.xml', 3636),\n ('HD015924.xml', 3636),\n ('HD016469.xml', 3636),\n ('HD018460.xml', 3636),\n ('HD018621.xml', 3636),\n ('HD019908.xml', 3636),\n ('HD026968.xml', 3636),\n ('HD030807.xml', 3636),\n ('HD046539.xml', 3636),\n ('HD011615.xml', 3637),\n ('HD017308.xml', 3637),\n ('HD017795.xml', 3637),\n ('HD019905.xml', 3637),\n ('HD023471.xml', 3637),\n ('HD028548.xml', 3637),\n ('HD028909.xml', 3637),\n ('HD030795.xml', 3637),\n ('HD031965.xml', 3637),\n ('HD033456.xml', 3637),\n ('HD046487.xml', 3637),\n ('HD010263.xml', 3638),\n ('HD012662.xml', 3638),\n ('HD013092.xml', 3638),\n ('HD017677.xml', 3638),\n ('HD019212.xml', 3638),\n ('HD021640.xml', 3638),\n ('HD033348.xml', 3638),\n ('HD011452.xml', 3639),\n ('HD016687.xml', 3639),\n ('HD017553.xml', 3639),\n ('HD018944.xml', 3639),\n ('HD020857.xml', 3639),\n ('HD029658.xml', 3639),\n ('HD032586.xml', 3639),\n ('HD004707.xml', 3640),\n ('HD008074.xml', 3640),\n ('HD008122.xml', 3640),\n ('HD011932.xml', 3640),\n ('HD012140.xml', 3640),\n ('HD017565.xml', 3640),\n ('HD017857.xml', 3640),\n ('HD017863.xml', 3640),\n ('HD022495.xml', 3640),\n ('HD022915.xml', 3640),\n ('HD030024.xml', 3640),\n ('HD031065.xml', 3640),\n ('HD031371.xml', 3640),\n ('HD032223.xml', 3640),\n ('HD049275.xml', 3640),\n ('HD059736.xml', 3640),\n ('HD071495.xml', 3640),\n ('HD009851.xml', 3641),\n ('HD017514.xml', 3641),\n ('HD022918.xml', 3641),\n ('HD029085.xml', 3641),\n ('HD033585.xml', 3641),\n ('HD007365.xml', 3642),\n ('HD011145.xml', 3642),\n ('HD014850.xml', 3642),\n ('HD015300.xml', 3642),\n ('HD015500.xml', 3642),\n ('HD017216.xml', 3642),\n ('HD026293.xml', 3642),\n ('HD041981.xml', 3642),\n ('HD010399.xml', 3643),\n ('HD013107.xml', 3643),\n ('HD013672.xml', 3643),\n ('HD015027.xml', 3643),\n ('HD020843.xml', 3643),\n ('HD022219.xml', 3643),\n ('HD029043.xml', 3643),\n ('HD029551.xml', 3643),\n ('HD006937.xml', 3644),\n ('HD007770.xml', 3644),\n ('HD013666.xml', 3644),\n ('HD013841.xml', 3644),\n ('HD016690.xml', 3644),\n ('HD016822.xml', 3644),\n ('HD017807.xml', 3644),\n ('HD027114.xml', 3644),\n ('HD012900.xml', 3645),\n ('HD013603.xml', 3645),\n ('HD020415.xml', 3645),\n ('HD024397.xml', 3645),\n ('HD027932.xml', 3645),\n ('HD032814.xml', 3645),\n ('HD032820.xml', 3645),\n ('HD051864.xml', 3645),\n ('HD006950.xml', 3646),\n ('HD009047.xml', 3646),\n ('HD009071.xml', 3646),\n ('HD009101.xml', 3646),\n ('HD010775.xml', 3646),\n ('HD011132.xml', 3646),\n ('HD012747.xml', 3646),\n ('HD013609.xml', 3646),\n ('HD014337.xml', 3646),\n ('HD017801.xml', 3646),\n ('HD019687.xml', 3646),\n ('HD021585.xml', 3646),\n ('HD026305.xml', 3646),\n ('HD032763.xml', 3646),\n ('HD048852.xml', 3646),\n ('HD053895.xml', 3646),\n ('HD012074.xml', 3647),\n ('HD014890.xml', 3647),\n ('HD032826.xml', 3647),\n ('HD046709.xml', 3647),\n ('HD046710.xml', 3647),\n ('HD055611.xml', 3647),\n ('HD012909.xml', 3648),\n ('HD013606.xml', 3648),\n ('HD019684.xml', 3648),\n ('HD028683.xml', 3648),\n ('HD029040.xml', 3648),\n ('HD046340.xml', 3648),\n ('HD073399.xml', 3648),\n ('HD007818.xml', 3649),\n ('HD009800.xml', 3649),\n ('HD009879.xml', 3649),\n ('HD013050.xml', 3649),\n ('HD015497.xml', 3649),\n ('HD020558.xml', 3649),\n ('HD030333.xml', 3649),\n ('HD044602.xml', 3649),\n ('HD051346.xml', 3649),\n ('HD006494.xml', 3650),\n ('HD008919.xml', 3650),\n ('HD009159.xml', 3650),\n ('HD009845.xml', 3650),\n ('HD012132.xml', 3650),\n ('HD012799.xml', 3650),\n ('HD016177.xml', 3650),\n ('HD016793.xml', 3650),\n ('HD017174.xml', 3650),\n ('HD017374.xml', 3650),\n ('HD017715.xml', 3650),\n ('HD020967.xml', 3650),\n ('HD027126.xml', 3650),\n ('HD044508.xml', 3650),\n ('HD051912.xml', 3650),\n ('HD059747.xml', 3650),\n ('HD006665.xml', 3651),\n ('HD007010.xml', 3651),\n ('HD013723.xml', 3651),\n ('HD016257.xml', 3651),\n ('HD019140.xml', 3651),\n ('HD021842.xml', 3651),\n ('HD006001.xml', 3652),\n ('HD013053.xml', 3652),\n ('HD013639.xml', 3652),\n ('HD016808.xml', 3652),\n ('HD017653.xml', 3652),\n ('HD020552.xml', 3652),\n ('HD031716.xml', 3652),\n ('HD062528.xml', 3652),\n ('HD011446.xml', 3653),\n ('HD011960.xml', 3653),\n ('HD014801.xml', 3653),\n ('HD014955.xml', 3653),\n ('HD015399.xml', 3653),\n ('HD015414.xml', 3653),\n ('HD015633.xml', 3653),\n ('HD015785.xml', 3653),\n ('HD017442.xml', 3653),\n ('HD020215.xml', 3653),\n ('HD022462.xml', 3653),\n ('HD026854.xml', 3653),\n ('HD048853.xml', 3653),\n ('HD050899.xml', 3653),\n ('HD059787.xml', 3653),\n ('HD006452.xml', 3654),\n ('HD015893.xml', 3654),\n ('HD015899.xml', 3654),\n ('HD016644.xml', 3654),\n ('HD023583.xml', 3654),\n ('HD024396.xml', 3654),\n ('HD028896.xml', 3654),\n ('HD030330.xml', 3654),\n ('HD032808.xml', 3654),\n ('HD045283.xml', 3654),\n ('HD012320.xml', 3655),\n ('HD013398.xml', 3655),\n ('HD015657.xml', 3655),\n ('HD015916.xml', 3655),\n ('HD017205.xml', 3655),\n ('HD017623.xml', 3655),\n ('HD019512.xml', 3655),\n ('HD020242.xml', 3655),\n ('HD021986.xml', 3655),\n ('HD022656.xml', 3655),\n ('HD048854.xml', 3655),\n ('HD076801.xml', 3655),\n ('HD009513.xml', 3656),\n ('HD011413.xml', 3656),\n ('HD011543.xml', 3656),\n ('HD012128.xml', 3656),\n ('HD013541.xml', 3656),\n ('HD021659.xml', 3656),\n ('HD022840.xml', 3656),\n ('HD032559.xml', 3656),\n ('HD045101.xml', 3656),\n ('HD007605.xml', 3657),\n ('HD009243.xml', 3657),\n ('HD009612.xml', 3657),\n ('HD013653.xml', 3657),\n ('HD017738.xml', 3657),\n ('HD021643.xml', 3657),\n ('HD021774.xml', 3657),\n ('HD022971.xml', 3657),\n ('HD048218.xml', 3657),\n ('HD007078.xml', 3658),\n ('HD009720.xml', 3658),\n ('HD014020.xml', 3658),\n ('HD017389.xml', 3658),\n ('HD006581.xml', 3659),\n ('HD007996.xml', 3659),\n ('HD008986.xml', 3659),\n ('HD009138.xml', 3659),\n ('HD011440.xml', 3659),\n ('HD011938.xml', 3659),\n ('HD012071.xml', 3659),\n ('HD014394.xml', 3659),\n ('HD014406.xml', 3659),\n ('HD014726.xml', 3659),\n ('HD015435.xml', 3659),\n ('HD017118.xml', 3659),\n ('HD017313.xml', 3659),\n ('HD018126.xml', 3659),\n ('HD020550.xml', 3659),\n ('HD023777.xml', 3659),\n ('HD024195.xml', 3659),\n ('HD025351.xml', 3659),\n ('HD026049.xml', 3659),\n ('HD028077.xml', 3659),\n ('HD031644.xml', 3659),\n ('HD046095.xml', 3659),\n ('HD046160.xml', 3659),\n ('HD048251.xml', 3659),\n ('HD050888.xml', 3659),\n ('HD054913.xml', 3659),\n ('HD072871.xml', 3659),\n ('HD007660.xml', 3660),\n ('HD014767.xml', 3660),\n ('HD015309.xml', 3660),\n ('HD016999.xml', 3660),\n ('HD017403.xml', 3660),\n ('HD023780.xml', 3660),\n ('HD043483.xml', 3660),\n ('HD049405.xml', 3660),\n ('HD059744.xml', 3660),\n ('HD007722.xml', 3661),\n ('HD009606.xml', 3661),\n ('HD009717.xml', 3661),\n ('HD020979.xml', 3661),\n ('HD022073.xml', 3661),\n ('HD022260.xml', 3661),\n ('HD033036.xml', 3661),\n ('HD047965.xml', 3661),\n ('HD049863.xml', 3661),\n ('HD049866.xml', 3661),\n ('HD049868.xml', 3661),\n ('HD007967.xml', 3662),\n ('HD009567.xml', 3662),\n ('HD010010.xml', 3662),\n ('HD017155.xml', 3662),\n ('HD017512.xml', 3662),\n ('HD022774.xml', 3662),\n ('HD023789.xml', 3662),\n ('HD030075.xml', 3662),\n ('HD031224.xml', 3662),\n ('HD049864.xml', 3662),\n ('HD050886.xml', 3662),\n ('HD071508.xml', 3662),\n ('HD007241.xml', 3663),\n ('HD008336.xml', 3663),\n ('HD009202.xml', 3663),\n ('HD009591.xml', 3663),\n ('HD015120.xml', 3663),\n ('HD015255.xml', 3663),\n ('HD015370.xml', 3663),\n ('HD016669.xml', 3663),\n ('HD017312.xml', 3663),\n ('HD017960.xml', 3663),\n ('HD019014.xml', 3663),\n ('HD019028.xml', 3663),\n ('HD021517.xml', 3663),\n ('HD023930.xml', 3663),\n ('HD030843.xml', 3663),\n ('HD053341.xml', 3663),\n ('HD007703.xml', 3664),\n ('HD008913.xml', 3664),\n ('HD009107.xml', 3664),\n ('HD009199.xml', 3664),\n ('HD010580.xml', 3664),\n ('HD010708.xml', 3664),\n ('HD030771.xml', 3664),\n ('HD010013.xml', 3665),\n ('HD011941.xml', 3665),\n ('HD013208.xml', 3665),\n ('HD016363.xml', 3665),\n ('HD019052.xml', 3665),\n ('HD020954.xml', 3665),\n ('HD021656.xml', 3665),\n ('HD024373.xml', 3665),\n ('HD072890.xml', 3665),\n ('HD009065.xml', 3666),\n ('HD011215.xml', 3666),\n ('HD014985.xml', 3666),\n ('HD016620.xml', 3666),\n ('HD017121.xml', 3666),\n ('HD017262.xml', 3666),\n ('HD030801.xml', 3666),\n ('HD054528.xml', 3666),\n ('HD006990.xml', 3667),\n ('HD008297.xml', 3667),\n ('HD015103.xml', 3667),\n ('HD018832.xml', 3667),\n ('HD018881.xml', 3667),\n ('HD019079.xml', 3667),\n ('HD019509.xml', 3667),\n ('HD021400.xml', 3667),\n ('HD021667.xml', 3667),\n ('HD045328.xml', 3667),\n ('HD048197.xml', 3667),\n ('HD048220.xml', 3667),\n ('HD049611.xml', 3667),\n ('HD049696.xml', 3667),\n ('HD071501.xml', 3667),\n ('HD006952.xml', 3668),\n ('HD007987.xml', 3668),\n ('HD009178.xml', 3668),\n ('HD010221.xml', 3668),\n ('HD013539.xml', 3668),\n ('HD015964.xml', 3668),\n ('HD017191.xml', 3668),\n ('HD019264.xml', 3668),\n ('HD027039.xml', 3668),\n ('HD048215.xml', 3668),\n ('HD006275.xml', 3669),\n ('HD006998.xml', 3669),\n ('HD007821.xml', 3669),\n ('HD009608.xml', 3669),\n ('HD010773.xml', 3669),\n ('HD011085.xml', 3669),\n ('HD012131.xml', 3669),\n ('HD014395.xml', 3669),\n ('HD015961.xml', 3669),\n ('HD020222.xml', 3669),\n ('HD023590.xml', 3669),\n ('HD030792.xml', 3669),\n ('HD051656.xml', 3669),\n ('HD051929.xml', 3669),\n ('HD010254.xml', 3670),\n ('HD012096.xml', 3670),\n ('HD015097.xml', 3670),\n ('HD021461.xml', 3670),\n ('HD022596.xml', 3670),\n ('HD047189.xml', 3670),\n ('HD005560.xml', 3671),\n ('HD007427.xml', 3671),\n ('HD009200.xml', 3671),\n ('HD009558.xml', 3671),\n ('HD019495.xml', 3671),\n ('HD020233.xml', 3671),\n ('HD021221.xml', 3671),\n ('HD047191.xml', 3671),\n ('HD006441.xml', 3672),\n ('HD007761.xml', 3672),\n ('HD009462.xml', 3672),\n ('HD010113.xml', 3672),\n ('HD014802.xml', 3672),\n ('HD017387.xml', 3672),\n ('HD017962.xml', 3672),\n ('HD018691.xml', 3672),\n ('HD019843.xml', 3672),\n ('HD021326.xml', 3672),\n ('HD023677.xml', 3672),\n ('HD027117.xml', 3672),\n ('HD051655.xml', 3672),\n ('HD053131.xml', 3672),\n ('HD010396.xml', 3673),\n ('HD010644.xml', 3673),\n ('HD011090.xml', 3673),\n ('HD011404.xml', 3673),\n ('HD011861.xml', 3673),\n ('HD012206.xml', 3673),\n ('HD013804.xml', 3673),\n ('HD017049.xml', 3673),\n ('HD018156.xml', 3673),\n ('HD031647.xml', 3673),\n ('HD050591.xml', 3673),\n ('HD010040.xml', 3674),\n ('HD014804.xml', 3674),\n ('HD015130.xml', 3674),\n ('HD015408.xml', 3674),\n ('HD015503.xml', 3674),\n ('HD017098.xml', 3674),\n ('HD017821.xml', 3674),\n ('HD018395.xml', 3674),\n ('HD020464.xml', 3674),\n ('HD021963.xml', 3674),\n ('HD007969.xml', 3675),\n ('HD009843.xml', 3675),\n ('HD012877.xml', 3675),\n ('HD014907.xml', 3675),\n ('HD016986.xml', 3675),\n ('HD017817.xml', 3675),\n ('HD019313.xml', 3675),\n ('HD030305.xml', 3675),\n ('HD030594.xml', 3675),\n ('HD052570.xml', 3675),\n ('HD007981.xml', 3676),\n ('HD008546.xml', 3676),\n ('HD009849.xml', 3676),\n ('HD014691.xml', 3676),\n ('HD016798.xml', 3676),\n ('HD017092.xml', 3676),\n ('HD017384.xml', 3676),\n ('HD018158.xml', 3676),\n ('HD020802.xml', 3676),\n ('HD030600.xml', 3676),\n ('HD048209.xml', 3676),\n ('HD054525.xml', 3676),\n ('HD054527.xml', 3676),\n ('HD073068.xml', 3676),\n ('HD011999.xml', 3677),\n ('HD012171.xml', 3677),\n ('HD012853.xml', 3677),\n ('HD013886.xml', 3677),\n ('HD014685.xml', 3677),\n ('HD016396.xml', 3677),\n ('HD020200.xml', 3677),\n ('HD020869.xml', 3677),\n ('HD024544.xml', 3677),\n ('HD027901.xml', 3677),\n ('HD028933.xml', 3677),\n ('HD031068.xml', 3677),\n ('HD006359.xml', 3678),\n ('HD006940.xml', 3678),\n ('HD007801.xml', 3678),\n ('HD014997.xml', 3678),\n ('HD015061.xml', 3678),\n ('HD018647.xml', 3678),\n ('HD021174.xml', 3678),\n ('HD021405.xml', 3678),\n ('HD021498.xml', 3678),\n ('HD026530.xml', 3678),\n ('HD030791.xml', 3678),\n ('HD031230.xml', 3678),\n ('HD007255.xml', 3679),\n ('HD008765.xml', 3679),\n ('HD009537.xml', 3679),\n ('HD011927.xml', 3679),\n ('HD012641.xml', 3679),\n ('HD012814.xml', 3679),\n ('HD014184.xml', 3679),\n ('HD016439.xml', 3679),\n ('HD023768.xml', 3679),\n ('HD023948.xml', 3679),\n ('HD029394.xml', 3679),\n ('HD008624.xml', 3680),\n ('HD010279.xml', 3680),\n ('HD012400.xml', 3680),\n ('HD014961.xml', 3680),\n ('HD015018.xml', 3680),\n ('HD015312.xml', 3680),\n ('HD016757.xml', 3680),\n ('HD016874.xml', 3680),\n ('HD017214.xml', 3680),\n ('HD017748.xml', 3680),\n ('HD017792.xml', 3680),\n ('HD018586.xml', 3680),\n ('HD023382.xml', 3680),\n ('HD059801.xml', 3680),\n ('HD006240.xml', 3681),\n ('HD007439.xml', 3681),\n ('HD008534.xml', 3681),\n ('HD008821.xml', 3681),\n ('HD010720.xml', 3681),\n ('HD011136.xml', 3681),\n ('HD014817.xml', 3681),\n ('HD014849.xml', 3681),\n ('HD015915.xml', 3681),\n ('HD024438.xml', 3681),\n ('HD029547.xml', 3681),\n ('HD007123.xml', 3682),\n ('HD007575.xml', 3682),\n ('HD008868.xml', 3682),\n ('HD009038.xml', 3682),\n ('HD014352.xml', 3682),\n ('HD016653.xml', 3682),\n ('HD019385.xml', 3682),\n ('HD020612.xml', 3682),\n ('HD021096.xml', 3682),\n ('HD021523.xml', 3682),\n ('HD029196.xml', 3682),\n ('HD031539.xml', 3682),\n ('HD033534.xml', 3682),\n ('HD009080.xml', 3683),\n ('HD010230.xml', 3683),\n ('HD011083.xml', 3683),\n ('HD015872.xml', 3683),\n ('HD018161.xml', 3683),\n ('HD018935.xml', 3683),\n ('HD019210.xml', 3683),\n ('HD020339.xml', 3683),\n ('HD029115.xml', 3683),\n ('HD032811.xml', 3683),\n ('HD046098.xml', 3683),\n ('HD059766.xml', 3683),\n ('HD012757.xml', 3684),\n ('HD014253.xml', 3684),\n ('HD014704.xml', 3684),\n ('HD017739.xml', 3684),\n ('HD018139.xml', 3684),\n ('HD019608.xml', 3684),\n ('HD020258.xml', 3684),\n ('HD022498.xml', 3684),\n ('HD023410.xml', 3684),\n ('HD029589.xml', 3684),\n ('HD032829.xml', 3684),\n ('HD051860.xml', 3684),\n ('HD005262.xml', 3685),\n ('HD008415.xml', 3685),\n ('HD008621.xml', 3685),\n ('HD008903.xml', 3685),\n ('HD023099.xml', 3685),\n ('HD023404.xml', 3685),\n ('HD026742.xml', 3685),\n ('HD048235.xml', 3685),\n ('HD049277.xml', 3685),\n ('HD009177.xml', 3686),\n ('HD011747.xml', 3686),\n ('HD014506.xml', 3686),\n ('HD015392.xml', 3686),\n ('HD018917.xml', 3686),\n ('HD019904.xml', 3686),\n ('HD020252.xml', 3686),\n ('HD020547.xml', 3686),\n ('HD021123.xml', 3686),\n ('HD021883.xml', 3686),\n ('HD022235.xml', 3686),\n ('HD023150.xml', 3686),\n ('HD023153.xml', 3686),\n ('HD048249.xml', 3686),\n ('HD055605.xml', 3686),\n ('HD007978.xml', 3687),\n ('HD009074.xml', 3687),\n ('HD010173.xml', 3687),\n ('HD010215.xml', 3687),\n ('HD017608.xml', 3687),\n ('HD017788.xml', 3687),\n ('HD018521.xml', 3687),\n ('HD020068.xml', 3687),\n ('HD022318.xml', 3687),\n ('HD027573.xml', 3687),\n ('HD028339.xml', 3687),\n ('HD033351.xml', 3687),\n ('HD007972.xml', 3688),\n ('HD009068.xml', 3688),\n ('HD009557.xml', 3688),\n ('HD010886.xml', 3688),\n ('HD013030.xml', 3688),\n ('HD014292.xml', 3688),\n ('HD018766.xml', 3688),\n ('HD019146.xml', 3688),\n ('HD020212.xml', 3688),\n ('HD021447.xml', 3688),\n ('HD023491.xml', 3688),\n ('HD023785.xml', 3688),\n ('HD026074.xml', 3688),\n ('HD026080.xml', 3688),\n ('HD028155.xml', 3688),\n ('HD048760.xml', 3688),\n ('HD053896.xml', 3688),\n ('HD011088.xml', 3689),\n ('HD012165.xml', 3689),\n ('HD012518.xml', 3689),\n ('HD014482.xml', 3689),\n ('HD018431.xml', 3689),\n ('HD018486.xml', 3689),\n ('HD018862.xml', 3689),\n ('HD019521.xml', 3689),\n ('HD022805.xml', 3689),\n ('HD027402.xml', 3689),\n ...]"
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# perhaps there are some strenge values\n",
    "sorted(edh_xml_lens, key=lambda x: x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[('HD000721.xml', 41649),\n ('HD043295.xml', 42599),\n ('HD043289.xml', 44271),\n ('HD044445.xml', 46700),\n ('HD026775.xml', 51213),\n ('HD017350.xml', 53152),\n ('HD056719.xml', 54588),\n ('HD043480.xml', 68146),\n ('HD032316.xml', 102219)]"
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(edh_xml_lens, key=lambda x: x[1])[-10:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parse the xml data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "edh_xml_data = []\n",
    "for filename in edh_filecontents.keys():\n",
    "    edh_xml_data.append(get_data_from_filename(filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "outputs": [
    {
     "data": {
      "text/plain": "81883"
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove empty\n",
    "#edh_xml_data = [elem for elem in edh_xml_data if elem != None]\n",
    "# how many we have\n",
    "# last time we had 81143 and then 81156\n",
    "len(edh_xml_data)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "outputs": [
    {
     "data": {
      "text/plain": "[None]"
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# look et invalid\n",
    "[el for el in edh_xml_data if el == None]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "outputs": [
    {
     "data": {
      "text/plain": "81882"
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's take valid only:\n",
    "edh_xml_data = [elem for elem in edh_xml_data if elem != None]\n",
    "len(edh_xml_data)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "outputs": [
    {
     "data": {
      "text/plain": "   idno_uri idno_tm                                    placenames_refs  \\\n0  HD000001    None  [{'type': 'province', 'text': 'Latium et Campa...   \n1  HD000002    None  [{'type': 'province', 'text': 'Roma'}, {'type'...   \n2  HD000003    None  [{'type': 'province', 'text': 'Baetica'}, {'ty...   \n3  HD000004    None  [{'type': 'province', 'text': 'Baetica'}, {'ty...   \n4  HD000005    None  [{'type': 'province', 'text': 'Roma'}, {'type'...   \n\n                                        text_edition    origdate_text  \\\n0  \\n Dis Manibus Noniae Publi filiae Optatae et ...   71 AD - 130 AD   \n1       \\n Caius Sextius Paris qui vixit annis LXX\\n   51 AD - 200 AD   \n2  \\n Publio Mummio Publi filio Galeria Sisennae ...  131 AD - 170 AD   \n3  \\n AVSLLA Marci Porci Nigri serva dominae Vene...  151 AD - 200 AD   \n4  \\n libertus Successus Luci libertus Irenaeus C...    1 AD - 200 AD   \n\n  layout_execution layout_execution_text support_objecttype  \\\n0             None                  None                257   \n1             None                  None                257   \n2             None                  None                 57   \n3             None                  None                 29   \n4             None                  None                250   \n\n  support_objecttype_text support_material            support_material_text  \\\n0                  tabula             None  Marmor, ge\\xc3\\xa4dert / farbig   \n1                  tabula               48                           Marmor   \n2             statue base               48                           Marmor   \n3                   altar               60                        Kalkstein   \n4                   stele             None                             None   \n\n  support_decoration keywords_term     keywords_term_text  \\\n0               1000            92                epitaph   \n1               1000            92                epitaph   \n2               1000            69  honorific inscription   \n3               1000           372     votive inscription   \n4               1000            92                epitaph   \n\n                                              people  \n0  [{'persname': {'name': [{'@type': 'nomen', '#t...  \n1  {'persname': {'name': [{'@type': 'praenomen', ...  \n2  {'persname': {'name': [{'@type': 'praenomen', ...  \n3  [{'persname': {'name': {'@type': 'cognomen', '...  \n4  [{'persname': {'name': [{'@type': 'praenomen',...  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>idno_uri</th>\n      <th>idno_tm</th>\n      <th>placenames_refs</th>\n      <th>text_edition</th>\n      <th>origdate_text</th>\n      <th>layout_execution</th>\n      <th>layout_execution_text</th>\n      <th>support_objecttype</th>\n      <th>support_objecttype_text</th>\n      <th>support_material</th>\n      <th>support_material_text</th>\n      <th>support_decoration</th>\n      <th>keywords_term</th>\n      <th>keywords_term_text</th>\n      <th>people</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>HD000001</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Latium et Campa...</td>\n      <td>\\n Dis Manibus Noniae Publi filiae Optatae et ...</td>\n      <td>71 AD - 130 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>None</td>\n      <td>Marmor, ge\\xc3\\xa4dert / farbig</td>\n      <td>1000</td>\n      <td>92</td>\n      <td>epitaph</td>\n      <td>[{'persname': {'name': [{'@type': 'nomen', '#t...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>HD000002</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Roma'}, {'type'...</td>\n      <td>\\n Caius Sextius Paris qui vixit annis LXX\\n</td>\n      <td>51 AD - 200 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>48</td>\n      <td>Marmor</td>\n      <td>1000</td>\n      <td>92</td>\n      <td>epitaph</td>\n      <td>{'persname': {'name': [{'@type': 'praenomen', ...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>HD000003</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Baetica'}, {'ty...</td>\n      <td>\\n Publio Mummio Publi filio Galeria Sisennae ...</td>\n      <td>131 AD - 170 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>57</td>\n      <td>statue base</td>\n      <td>48</td>\n      <td>Marmor</td>\n      <td>1000</td>\n      <td>69</td>\n      <td>honorific inscription</td>\n      <td>{'persname': {'name': [{'@type': 'praenomen', ...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>HD000004</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Baetica'}, {'ty...</td>\n      <td>\\n AVSLLA Marci Porci Nigri serva dominae Vene...</td>\n      <td>151 AD - 200 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>29</td>\n      <td>altar</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>372</td>\n      <td>votive inscription</td>\n      <td>[{'persname': {'name': {'@type': 'cognomen', '...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>HD000005</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Roma'}, {'type'...</td>\n      <td>\\n libertus Successus Luci libertus Irenaeus C...</td>\n      <td>1 AD - 200 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>250</td>\n      <td>stele</td>\n      <td>None</td>\n      <td>None</td>\n      <td>1000</td>\n      <td>92</td>\n      <td>epitaph</td>\n      <td>[{'persname': {'name': [{'@type': 'praenomen',...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make a dataframe from\n",
    "edh_xml_data_df = pd.DataFrame(edh_xml_data)\n",
    "edh_xml_data_df.head(5)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "outputs": [
    {
     "data": {
      "text/plain": "81882"
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(edh_xml_data_df)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "outputs": [],
   "source": [
    "def try_to_get_placename(listofdicts, placename_type):\n",
    "    try:\n",
    "        placename = [d[\"text\"] for d in listofdicts if d[\"type\"]==placename_type][0]\n",
    "    except:\n",
    "        placename = \"\"\n",
    "    return placename"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "outputs": [],
   "source": [
    "edh_xml_data_df[\"province_label\"] = edh_xml_data_df[\"placenames_refs\"].apply(lambda x: try_to_get_placename(x, \"province\"))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "outputs": [
    {
     "data": {
      "text/plain": "396"
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edh_xml_data_df[\"province_label\"].str.contains(\"\\?\", na=False).sum()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "outputs": [],
   "source": [
    "def try_to_get_pleiades(listofdicts):\n",
    "    listofdicts = [d for d in listofdicts if \"ref\" in d.keys()]\n",
    "    try:\n",
    "        pleiades_id = [d[\"ref\"] for d in listofdicts if \"pleiades\" in d[\"ref\"]][0].rpartition(\"/\")[2]\n",
    "    except:\n",
    "        pleiades_id = \"\"\n",
    "    return pleiades_id"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "outputs": [
    {
     "data": {
      "text/plain": "'432808'"
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try_to_get_pleiades(test_listd)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "outputs": [
    {
     "data": {
      "text/plain": "   idno_uri idno_tm                                    placenames_refs  \\\n0  HD000001    None  [{'type': 'province', 'text': 'Latium et Campa...   \n1  HD000002    None  [{'type': 'province', 'text': 'Roma'}, {'type'...   \n2  HD000003    None  [{'type': 'province', 'text': 'Baetica'}, {'ty...   \n3  HD000004    None  [{'type': 'province', 'text': 'Baetica'}, {'ty...   \n4  HD000005    None  [{'type': 'province', 'text': 'Roma'}, {'type'...   \n\n                                        text_edition    origdate_text  \\\n0  \\n Dis Manibus Noniae Publi filiae Optatae et ...   71 AD - 130 AD   \n1       \\n Caius Sextius Paris qui vixit annis LXX\\n   51 AD - 200 AD   \n2  \\n Publio Mummio Publi filio Galeria Sisennae ...  131 AD - 170 AD   \n3  \\n AVSLLA Marci Porci Nigri serva dominae Vene...  151 AD - 200 AD   \n4  \\n libertus Successus Luci libertus Irenaeus C...    1 AD - 200 AD   \n\n  layout_execution layout_execution_text support_objecttype  \\\n0             None                  None                257   \n1             None                  None                257   \n2             None                  None                 57   \n3             None                  None                 29   \n4             None                  None                250   \n\n  support_objecttype_text support_material            support_material_text  \\\n0                  tabula             None  Marmor, ge\\xc3\\xa4dert / farbig   \n1                  tabula               48                           Marmor   \n2             statue base               48                           Marmor   \n3                   altar               60                        Kalkstein   \n4                   stele             None                             None   \n\n  support_decoration keywords_term     keywords_term_text  \\\n0               1000            92                epitaph   \n1               1000            92                epitaph   \n2               1000            69  honorific inscription   \n3               1000           372     votive inscription   \n4               1000            92                epitaph   \n\n                                              people  \\\n0  [{'persname': {'name': [{'@type': 'nomen', '#t...   \n1  {'persname': {'name': [{'@type': 'praenomen', ...   \n2  {'persname': {'name': [{'@type': 'praenomen', ...   \n3  [{'persname': {'name': {'@type': 'cognomen', '...   \n4  [{'persname': {'name': [{'@type': 'praenomen',...   \n\n                 province_label pleiades_id  \n0  Latium et Campania (Regio I)      432808  \n1                          Roma      423025  \n2                       Baetica              \n3                       Baetica              \n4                          Roma      423025  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>idno_uri</th>\n      <th>idno_tm</th>\n      <th>placenames_refs</th>\n      <th>text_edition</th>\n      <th>origdate_text</th>\n      <th>layout_execution</th>\n      <th>layout_execution_text</th>\n      <th>support_objecttype</th>\n      <th>support_objecttype_text</th>\n      <th>support_material</th>\n      <th>support_material_text</th>\n      <th>support_decoration</th>\n      <th>keywords_term</th>\n      <th>keywords_term_text</th>\n      <th>people</th>\n      <th>province_label</th>\n      <th>pleiades_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>HD000001</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Latium et Campa...</td>\n      <td>\\n Dis Manibus Noniae Publi filiae Optatae et ...</td>\n      <td>71 AD - 130 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>None</td>\n      <td>Marmor, ge\\xc3\\xa4dert / farbig</td>\n      <td>1000</td>\n      <td>92</td>\n      <td>epitaph</td>\n      <td>[{'persname': {'name': [{'@type': 'nomen', '#t...</td>\n      <td>Latium et Campania (Regio I)</td>\n      <td>432808</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>HD000002</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Roma'}, {'type'...</td>\n      <td>\\n Caius Sextius Paris qui vixit annis LXX\\n</td>\n      <td>51 AD - 200 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>257</td>\n      <td>tabula</td>\n      <td>48</td>\n      <td>Marmor</td>\n      <td>1000</td>\n      <td>92</td>\n      <td>epitaph</td>\n      <td>{'persname': {'name': [{'@type': 'praenomen', ...</td>\n      <td>Roma</td>\n      <td>423025</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>HD000003</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Baetica'}, {'ty...</td>\n      <td>\\n Publio Mummio Publi filio Galeria Sisennae ...</td>\n      <td>131 AD - 170 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>57</td>\n      <td>statue base</td>\n      <td>48</td>\n      <td>Marmor</td>\n      <td>1000</td>\n      <td>69</td>\n      <td>honorific inscription</td>\n      <td>{'persname': {'name': [{'@type': 'praenomen', ...</td>\n      <td>Baetica</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>HD000004</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Baetica'}, {'ty...</td>\n      <td>\\n AVSLLA Marci Porci Nigri serva dominae Vene...</td>\n      <td>151 AD - 200 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>29</td>\n      <td>altar</td>\n      <td>60</td>\n      <td>Kalkstein</td>\n      <td>1000</td>\n      <td>372</td>\n      <td>votive inscription</td>\n      <td>[{'persname': {'name': {'@type': 'cognomen', '...</td>\n      <td>Baetica</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>HD000005</td>\n      <td>None</td>\n      <td>[{'type': 'province', 'text': 'Roma'}, {'type'...</td>\n      <td>\\n libertus Successus Luci libertus Irenaeus C...</td>\n      <td>1 AD - 200 AD</td>\n      <td>None</td>\n      <td>None</td>\n      <td>250</td>\n      <td>stele</td>\n      <td>None</td>\n      <td>None</td>\n      <td>1000</td>\n      <td>92</td>\n      <td>epitaph</td>\n      <td>[{'persname': {'name': [{'@type': 'praenomen',...</td>\n      <td>Roma</td>\n      <td>423025</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edh_xml_data_df[\"pleiades_id\"] = edh_xml_data_df[\"placenames_refs\"].apply(try_to_get_pleiades)\n",
    "edh_xml_data_df.head(5)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Save the data locally and to sciencedata"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "outputs": [],
   "source": [
    "edh_xml_data_df.to_json(\"../data/large_data/edh_xml_data.json\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A file with the same name (\"edh_xml_data_2022-11-02.json\") already exists in this location.\n",
      "Your <class 'pandas.core.frame.DataFrame'> object has been succesfully written as \"https://sciencedata.dk/files/SDAM_root/SDAM_data/EDH/edh_xml_data_2022-11-02.json\"\n"
     ]
    }
   ],
   "source": [
    "s.write_file(\"edh_xml_data_2022-11-02.json\", edh_xml_data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "machine_shape": "hm",
   "name": "1_2_py_EXTRACTION_edh-xml_files.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}