inspect-etl7.ipynb

{
	"cells": [
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"import os\n",
				"import time\n",
				"import math\n",
				"import re\n",
				"import struct\n",
				"import traceback\n",
				"\n",
				"import numpy as np\n",
				"import matplotlib.pyplot as plt\n",
				"# import cv2\n",
				"import pandas as pd\n",
				"\n",
				"from PIL import Image\n",
				"import IPython.display as display"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"os.listdir('.')"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"file_list = [\n",
				"    'ETL7LC_1',\n",
				"    'ETL7LC_2',\n",
				"    'ETL7SC_1',\n",
				"    'ETL7SC_2',\n",
				"]"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"filename = file_list[0]\n",
				"filename"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"RECORD_LENGTH = 2052 # bytes"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"with open(filename, 'rb') as data_file:\n",
				"    record_string = data_file.read(RECORD_LENGTH)"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"print(type(record_string), len(record_string))"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"UNPACK_STRING = '>H2sH6BI4H4B4x2016s4x'"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"record = struct.unpack(UNPACK_STRING, record_string)\n",
				"print(type(record), len(record))"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"record_dict = {\n",
				"    'index': record[0], # type: int\n",
				"    'character_name': record[1], # type: bytes - ascii encoding (e.g. A, KA, 0, $)\n",
				"    'sheet_index': record[2], # type: int\n",
				"    'JIS_X_0201_CODE': record[3], # type: binary - https://en.wikipedia.org/wiki/JIS_X_0201\n",
				"    'EBCDIC_CODE': record[4], # type: binary - https://en.wikipedia.org/wiki/EBCDIC\n",
				"    'image_quality': record[5], # type: int - [0, 1, 2, 3] with 0 is clean\n",
				"    'group_quality': record[6], # type: int - [0, 1, 2] with 0 is clean\n",
				"    'writer_gender': record[7], # type: int - 1 is male, 2 is female. JIS X 0303 encoding?\n",
				"    'writer_age': record[8], # type: int - \n",
				"    'serial_data_index': record[9], # type: int\n",
				"    'industry_classification_code': record[10], # type: int - JIS X 0403\n",
				"    'occupation_classification_code': record[11], # type: int - JIS X 0404\n",
				"    'data_of_sheet_gathering': record[12], # type: int - data of sheet gathering (19)YYMM\n",
				"    'date_of_scan': record[13], # type: int - date of scan (19)YYMM\n",
				"    'x_pos': record[14], # type: int - x coordinate of scan position on sheet (>= 1)\n",
				"    'y_pos': record[15], # type: int - y coordinate of scan position on sheet (>= 1)\n",
				"    'minimum_intensity_level': record[16], # type: int - value in range (0-255)\n",
				"    'maximum_scanned_level': record[17], # type: int - value in range (0-255)\n",
				"    'image_data': record[18], # type: bytes\n",
				"}"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"for key in record_dict.keys():\n",
				"    print(key, type(record_dict[key]))"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"record_dict['character_name']"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"record_dict['character_name'].decode('ascii').strip()"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"width = 64\n",
				"height = 63\n",
				"img = np.array(Image.frombytes('F', (width, height), record_dict['image_data'], 'bit', 4))"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"plt.imshow(img)\n",
				"plt.colorbar()"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"total_samples = 0\n",
				"record_count = {}\n",
				"RECORD_LENGTH = 2052 # bytes\n",
				"UNPACK_STRING = '>H2sH6BI4H4B4x2016s4x'\n",
				"\n",
				"for filename in file_list:\n",
				"    f = open(filename, 'rb')\n",
				"\n",
				"    while True:\n",
				"        record_string = f.read(RECORD_LENGTH)\n",
				"\n",
				"        if len(record_string) < RECORD_LENGTH:\n",
				"            break\n",
				"        total_samples += 1\n",
				"        record = struct.unpack(UNPACK_STRING, record_string)\n",
				"\n",
				"        record_dict = {\n",
				"            'index': record[0], # type: int\n",
				"            'character_name': record[1], # type: bytes - ascii encoding (e.g. A, KA, 0, $)\n",
				"            'sheet_index': record[2], # type: int\n",
				"            'JIS_X_0201_CODE': record[3], # type: binary - https://en.wikipedia.org/wiki/JIS_X_0201\n",
				"            'EBCDIC_CODE': record[4], # type: binary - https://en.wikipedia.org/wiki/EBCDIC\n",
				"            'image_quality': record[5], # type: int - [0, 1, 2, 3] with 0 is clean\n",
				"            'group_quality': record[6], # type: int - [0, 1, 2] with 0 is clean\n",
				"            'writer_gender': record[7], # type: int - 1 is male, 2 is female. JIS X 0303 encoding?\n",
				"            'writer_age': record[8], # type: int - \n",
				"            'serial_data_index': record[9], # type: int\n",
				"            'industry_classification_code': record[10], # type: int - JIS X 0403\n",
				"            'occupation_classification_code': record[11], # type: int - JIS X 0404\n",
				"            'data_of_sheet_gathering': record[12], # type: int - data of sheet gathering (19)YYMM\n",
				"            'date_of_scan': record[13], # type: int - date of scan (19)YYMM\n",
				"            'x_pos': record[14], # type: int - x coordinate of scan position on sheet (>= 1)\n",
				"            'y_pos': record[15], # type: int - y coordinate of scan position on sheet (>= 1)\n",
				"            'minimum_intensity_level': record[16], # type: int - value in range (0-255)\n",
				"            'maximum_scanned_level': record[17], # type: int - value in range (0-255)\n",
				"            'image_data': record[18], # type: bytes\n",
				"        }\n",
				"\n",
				"#         name = record_dict['character_name'].decode('ascii').strip()\n",
				"        name = record_dict['character_name'].decode('ascii')\n",
				"\n",
				"        if name in record_count.keys():\n",
				"            record_count[name] += 1\n",
				"        else:\n",
				"            record_count[name] = 1\n",
				"\n",
				"    f.close()"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"total_samples"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"len(record_count.keys())"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {
				"scrolled": true
			},
			"outputs": [],
			"source": [
				"record_count"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"csv_filename = 'classes.tsv'\n",
				"with open(csv_filename, mode='w', encoding='utf-8') as f:\n",
				"    f.write('class\\tnum_samples\\n')\n",
				"    for k in record_count.keys():\n",
				"        log_str = f'{k}\\t{record_count[k]}\\n'\n",
				"        f.write(log_str)\n",
				"# sort classes\n",
				"# pd_df = pd.read_csv(csv_filename, encoding='utf-8', sep='\\t')\n",
				"# pd_df = pd_df.sort_values(['class'])\n",
				"# pd_df.to_csv(csv_filename, encoding='utf-8', index=False, sep='\\t')"
			]
		}
	],
	"metadata": {
		"kernelspec": {
			"display_name": "Python 3",
			"language": "python",
			"name": "python3"
		},
		"language_info": {
			"codemirror_mode": {
				"name": "ipython",
				"version": 3
			},
			"file_extension": ".py",
			"mimetype": "text/x-python",
			"name": "python",
			"nbconvert_exporter": "python",
			"pygments_lexer": "ipython3"
		}
	},
	"nbformat": 4,
	"nbformat_minor": 2
}