inspect-etl8b2.ipynb

{
	"cells": [
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"import os\n",
				"import time\n",
				"import math\n",
				"import re\n",
				"import struct\n",
				"import traceback\n",
				"import codecs\n",
				"\n",
				"import bitstring\n",
				"\n",
				"import numpy as np\n",
				"import matplotlib.pyplot as plt\n",
				"# import cv2\n",
				"import pandas as pd\n",
				"\n",
				"from PIL import Image\n",
				"import IPython.display as display\n",
				"\n",
				"from JIS0208 import JIS0208"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"os.listdir('.')"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"file_list = [\n",
				"    'ETL8B2C1',\n",
				"    'ETL8B2C2',\n",
				"    'ETL8B2C3',\n",
				"]"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"filename = file_list[0]\n",
				"print(filename)\n",
				"file_stream = bitstring.ConstBitStream(filename=filename)"
			]
		},
		{
			"cell_type": "markdown",
			"metadata": {},
			"source": [
				"**The first record of each file is zero padded dummy.**"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"record = file_stream.readlist('int:16,uint:16,uint:32,bytes:504')\n",
				"record_dict = {\n",
				"    'sheet_number': record[0], # type: int - Serial Sheet Number\n",
				"    'JIS_KANJI_CODE': record[1], # type: binary - JIS Kanji Code (JIS X 0208)\n",
				"    'JIS_READING': record[2], # type: ASCII - JIS Typical Reading (e.g. 'AI.M')\n",
				"    'image_data': record[3], # type: bytes\n",
				"}\n",
				"\n",
				"width = 64\n",
				"height = 63\n",
				"\n",
				"np_img = np.array(Image.frombytes('1', (width, height), record_dict['image_data'], 'raw')).astype(int)\n",
				"plt.imshow(np_img)\n",
				"plt.colorbar()"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"record = file_stream.readlist('int:16,uint:16,uint:32,bytes:504')\n",
				"record_dict = {\n",
				"    'sheet_number': record[0], # type: int - Serial Sheet Number\n",
				"    'JIS_KANJI_CODE': record[1], # type: binary - JIS Kanji Code (JIS X 0208)\n",
				"    'JIS_READING': record[2], # type: ASCII - JIS Typical Reading (e.g. 'AI.M')\n",
				"    'image_data': record[3], # type: bytes\n",
				"}\n",
				"\n",
				"width = 64\n",
				"height = 63\n",
				"\n",
				"np_img = np.array(Image.frombytes('1', (width, height), record_dict['image_data'], 'raw')).astype(int)\n",
				"plt.imshow(np_img)\n",
				"plt.colorbar()"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"hex(record_dict['JIS_KANJI_CODE'])"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"total_samples = 0\n",
				"record_count = {}\n",
				"\n",
				"for filename in file_list:\n",
				"\n",
				"    file_stream = bitstring.ConstBitStream(filename=filename)\n",
				"    # skip the first record in each file\n",
				"    file_stream.readlist('bytes:512')\n",
				"\n",
				"    while True:\n",
				"        try:\n",
				"            record = file_stream.readlist('int:16,int:16,uint:32,bytes:504')\n",
				"        except:\n",
				"            break\n",
				"        \n",
				"        total_samples += 1\n",
				"        record_dict = {\n",
				"            'sheet_number': record[0], # type: int - Serial Sheet Number\n",
				"            'JIS_CODE': record[1], # type: binary - JIS Kanji Code (JIS X 0208)\n",
				"            'JIS_READING': record[2], # type: ASCII - JIS Typical Reading (e.g. 'AI.M')\n",
				"            'image_data': record[3], # type: bytes\n",
				"        }\n",
				"\n",
				"        jis_code = record_dict['JIS_CODE']\n",
				"        \n",
				"        if jis_code in JIS0208.keys():\n",
				"            unicode_char = JIS0208[jis_code]\n",
				"        else:\n",
				"            unicode_char = 'JIS_CODE=' + repr(jis_code)\n",
				"\n",
				"        if unicode_char in record_count.keys():\n",
				"            record_count[unicode_char] += 1\n",
				"        else:\n",
				"            record_count[unicode_char] = 1"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"total_samples"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"len(record_count.keys())"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"record_count"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"csv_filename = 'classes.tsv'\n",
				"with open(csv_filename, mode='w', encoding='utf-8') as f:\n",
				"    f.write('class\\tnum_samples\\n')\n",
				"    for k in record_count.keys():\n",
				"        log_str = f'{k}\\t{record_count[k]}\\n'\n",
				"        f.write(log_str)\n",
				"# sort classes\n",
				"pd_df = pd.read_csv(csv_filename, encoding='utf-8', sep='\\t')\n",
				"pd_df = pd_df.sort_values(['class'])\n",
				"pd_df.to_csv(csv_filename, encoding='utf-8', index=False, sep='\\t')"
			]
		}
	],
	"metadata": {
		"kernelspec": {
			"display_name": "Python 3",
			"language": "python",
			"name": "python3"
		},
		"language_info": {
			"codemirror_mode": {
				"name": "ipython",
				"version": 3
			},
			"file_extension": ".py",
			"mimetype": "text/x-python",
			"name": "python",
			"nbconvert_exporter": "python",
			"pygments_lexer": "ipython3"
		}
	},
	"nbformat": 4,
	"nbformat_minor": 2
}