inspect-etl3.ipynb

{
	"cells": [
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"import os\n",
				"import re\n",
				"import io\n",
				"import time\n",
				"import math\n",
				"import struct\n",
				"import traceback\n",
				"import codecs\n",
				"import json\n",
				"from typing import List, Dict\n",
				"\n",
				"# external dependencies\n",
				"import numpy as np\n",
				"import matplotlib.pyplot as plt\n",
				"import bitstring\n",
				"from tqdm import tqdm\n",
				"\n",
				"from PIL import Image\n",
				"import IPython.display as display\n",
				"\n",
				"# local modules\n",
				"import shared"
			]
		},
		{
			"cell_type": "markdown",
			"metadata": {},
			"source": [
				"# Prerequisite\n",
				"\n",
				"Download the dataset and unpack them as shown in the bellow file structure.\n",
				"\n",
				"```\n",
				".\n",
				"└── ETL3\n",
				"    ├── ETL3C_1\n",
				"    ├── ETL3C_2\n",
				"    └── ETL3INFO\n",
				"```"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"part_filepaths = [\n",
				"    'ETL3/ETL3C_1',\n",
				"    'ETL3/ETL3C_2',\n",
				"]\n",
				"\n",
				"# the code below is for Restart and Run All people (include me)\n",
				"for part_filepath in part_filepaths:\n",
				"    if not os.path.exists(part_filepath):\n",
				"        raise Exception(part_filepath, 'does not exist!')"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"part_filepath = part_filepaths[0]\n",
				"part_filepath"
			]
		},
		{
			"cell_type": "markdown",
			"metadata": {},
			"source": [
				"Below is the data format for the `ETL3`, `ETL4`, and `ETL5`.\n",
				"\n",
				"```\n",
				"/---[form_c.htm 2001-09.04]-------------------------------------------------------------------\n",
				"\n",
				"\n",
				"\t\t\t\tC-Type Data Format\t(ETL3, ETL4, ETL5)\n",
				"\n",
				"\n",
				"1. File Format (Fixed Record Length without Control Words)\n",
				"\n",
				"                   <----------> Logical record (2952 bytes) (1byte = 8bits)\n",
				"     ------------------------------------------------------------------\n",
				"     |  Sample 1  |  Sample 2  |  Sample 3  |    ....    |  Sample N  |  (No. of records = N)\n",
				"     ------------------------------------------------------------------\n",
				"\n",
				"\n",
				"2. Contents of Logical Record (3936 characters = 2952 bytes) (1character = 6bits)\n",
				"\n",
				" --------------------------------------------------------------------------------------------\n",
				"|             |No. of|        |                                                              |\n",
				"|   Character |Char- |  Type  |            Contents of Logical Record                        |\n",
				"|   Position  |acters|        |                                                              |\n",
				"|============================================================================================|\n",
				"|    1 -    6 |    6 | Integer| Serial Data Number                                           |\n",
				"|    7 -   12 |    6 | Integer| Serial Sheet Number                                          |\n",
				"|   13 -   18 |    6 | Binary | JIS Code (Effective bits = Left 8 bits) (JIS X 0201)         |\n",
				"|I  19 -   24 |    6 | Binary | EBCDIC Code (Effective bits = Left 8 bits)                   |\n",
				"|D  25 -   28 |    4 | T56Code| 4 Character Code ( ex. \"N  0\", \"A  A\", \"S  +\", \"K KA\" )      |\n",
				"|   29 -   30 |    2 | T56Code| Spaces                                                       |\n",
				"|P  31 -   36 |    6 | Integer| Evaluation of Individual Character Image (0=clean, 1, 2, 3)  |\n",
				"|a  37 -   42 |    6 | Integer| Evaluation of Character Group (0=clean, 1, 2)                |\n",
				"|r  43 -   48 |    6 | Integer| Sample Position Y on Sheet                                   |\n",
				"|t  49 -   54 |    6 | Integer| Sample Position X on Sheet                                   |\n",
				"|   55 -   60 |    6 | Integer| Male-Female Code ( 1=male, 2=female ) (JIS X 0303)           |\n",
				"|   61 -   66 |    6 | Integer| Age of Writer                                                |\n",
				"|   67 -   72 |    6 | Integer| Industry Classification Code (JIS X 0403)                    |\n",
				"|   73 -   78 |    6 | Integer| Occupation Classification Code (JIS X 0404)                  |\n",
				"|   79 -   84 |    6 | Integer| Sheet Gatherring Date                                        |\n",
				"|   85 -   90 |    6 | Integer| Scanning Date                                                |\n",
				"|   91 -   96 |    6 | Integer| Number of X-Axis Sampling Points                             |\n",
				"|   97 -  102 |    6 | Integer| Number of Y-Axis Sampling Points                             |\n",
				"|  103 -  108 |    6 | Integer| Number of Levels of Pixel                                    |\n",
				"|  109 -  114 |    6 | Integer| Magnification of Scanning Lenz                               |\n",
				"|  115 -  120 |    6 | Integer| Serial Data Number (old)                                     |\n",
				"|  121 -  288 |  168 |        | (undefined)                                                  |\n",
				"|-------------|------|--------|--------------------------------------------------------------|\n",
				"|  289 - 3936 | 3648 | Packed | 16 Gray Level (4bit/pixel) Image Data                        |\n",
				"|             |      |        | 72(X-axis size) * 76(Y-axis size) = 5472 pixels              |\n",
				" --------------------------------------------------------------------------------------------\n",
				"\n",
				"------------------------------------------------------------[form_c.htm]---------------------/\n",
				"```"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"infile = bitstring.ConstBitStream(filename=part_filepath)"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"bitstring_unpack_str = ','.join([\n",
				"    'uint:36', # serial data number - [0]\n",
				"    'uint:36', # serial sheet number - [1]\n",
				"    'uint:36', # JIS Code - [2]\n",
				"    'uint:36', # EBCDIC Code - [3]\n",
				"    '4*uint:6', # 4 Character Code - [4:8]\n",
				"    'pad:12', # Spaces\n",
				"    'uint:36', # Evaluation of Individual Character Image (0=clean, 1, 2, 3) - [8]\n",
				"    'uint:36', # Evaluation of Character Group (0=clean, 1, 2) - [9]\n",
				"    'uint:36', # Sample Position Y on Sheet - [10]\n",
				"    'uint:36', # Sample Position X on Sheet - [11]\n",
				"    'uint:36', # Male-Female Code (1=male, 2=female) (JIS X 0303) - [12]\n",
				"    'uint:36', # Age of Writer - [13]\n",
				"    'uint:36', # Industry Classification Code (JIS X 0403) - [14]\n",
				"    'uint:36', # Occupation Classification Code (JIS X 0404) - [15]\n",
				"    'uint:36', # Sheet Gathering Date - [16]\n",
				"    'uint:36', # Scanning Date - [17]\n",
				"    'uint:36', # Number of X-Axis Sampling Points - [18]\n",
				"    'uint:36', # Number of Y-Axis Sampling Points - [19]\n",
				"    'uint:36', # Number of Levels of Pixel - [20]\n",
				"    'uint:36', # Magnification of Scanning Lenz - [21]\n",
				"    'uint:36', # Serial Data Number (old) - [22]\n",
				"    'pad:1008', # (undefined)\n",
				"    'bytes:2736', # 16 Gray Level (4bit/pixel) Image Data 72(X-axis size) * 76(Y-axis size) = 5472 pixels - [23]\n",
				"])"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"RECORD_LENGTH = 6 * 3936 # in bits\n",
				"IMG_WIDTH = 72\n",
				"IMG_HEIGHT = 76"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"skip = 0\n",
				"infile.pos = skip * RECORD_LENGTH"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {
				"scrolled": true
			},
			"outputs": [],
			"source": [
				"record = infile.readlist(bitstring_unpack_str)\n",
				"\n",
				"print(type(record), len(record))\n",
				"\n",
				"# skip the last image data as it is too large to print out\n",
				"for idx, v in enumerate(record[:-1]):\n",
				"    print(f'{idx} - {v}')"
			]
		},
		{
			"cell_type": "markdown",
			"metadata": {},
			"source": [
				"# 72x76 16-bit image"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"pil_image = Image.frombytes('F', (IMG_WIDTH, IMG_HEIGHT), record[23], 'bit', 4)\n",
				"np_image = np.array(pil_image)\n",
				"plt.imshow(np_image, cmap='gray')\n",
				"plt.colorbar()"
			]
		},
		{
			"cell_type": "markdown",
			"metadata": {},
			"source": [
				"# grayscale `[0-255]` range"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"pil_image = Image.frombytes('F', (IMG_WIDTH, IMG_HEIGHT), record[23], 'bit', 4)\n",
				"np_image = np.array(pil_image, dtype=np.uint8)\n",
				"print(np_image.dtype, np_image.shape)\n",
				"\n",
				"np_image = np_image * 16 # 256/16\n",
				"print(np_image.dtype, np_image.shape)\n",
				"\n",
				"plt.imshow(np_image, cmap='gray')\n",
				"plt.colorbar()"
			]
		},
		{
			"cell_type": "markdown",
			"metadata": {},
			"source": [
				"# Compress the image in PNG format"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"def encode_image_as_png_bytes(data: bytes):\n",
				"    pil_image = Image.frombytes('F', (IMG_WIDTH, IMG_HEIGHT), data, 'bit', 4)\n",
				"    np_image = np.array(pil_image, dtype=np.uint8)\n",
				"    np_image = np_image * 16 # 256/16\n",
				"    pil_image = Image.fromarray(np_image)\n",
				"\n",
				"    buffer = io.BytesIO()\n",
				"    pil_image.save(buffer, format='PNG')\n",
				"    png_encoded_image = buffer.getvalue()\n",
				"\n",
				"    return buffer.getvalue()"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"png_image = encode_image_as_png_bytes(record[23])\n",
				"image_filepath = 'tmp.png'\n",
				"with open(image_filepath, mode='wb') as outfile:\n",
				"    outfile.write(png_image)\n",
				"\n",
				"display.display(display.Image(image_filepath))"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"{\n",
				"    'serial data number': record[0],\n",
				"    'serial sheet number': record[1],\n",
				"    'JIS Code': ''.join(map(shared.T56, record[4:8])),\n",
				"    'Evaluation of Individual Character Image (0=clean, 1, 2, 3)': record[8],\n",
				"    'Evaluation of Character Group (0=clean, 1, 2)': record[9],\n",
				"    'Sample Position Y on Sheet': record[10],\n",
				"    'Sample Position X on Sheet': record[11],\n",
				"    'Male-Female Code (1=male, 2=female) (JIS X 0303)': record[12],\n",
				"    'Age of Writer': record[13],\n",
				"    'Industry Classification Code (JIS X 0403)': record[14],\n",
				"    'Occupation Classification Code (JIS X 0404)': record[15],\n",
				"    'Sheet Gathering Date': record[16],\n",
				"    'Scanning Date': record[17],\n",
				"    'Number of X-Axis Sampling Points': record[18],\n",
				"    'Number of Y-Axis Sampling Points': record[19],\n",
				"    'Number of Levels of Pixel': record[20],\n",
				"    'Magnification of Scanning Lenz': record[21],\n",
				"    'Serial Data Number (old)': record[22],\n",
				"    'PNG_encoded_image': encode_image_as_png_bytes(record[23]),\n",
				"}"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"records_metadata = []\n",
				"serialized_dataset_filepath = f'etl3{shared.XFormat.EXTENSION}'\n",
				"\n",
				"if os.path.exists(serialized_dataset_filepath):\n",
				"    raise Exception(serialized_dataset_filepath + ' is already existed!')\n",
				"\n",
				"with open(serialized_dataset_filepath, mode='wb') as outfile:\n",
				"    pbar = tqdm(part_filepaths)\n",
				"    \n",
				"    for part_filepath in pbar:\n",
				"        pbar.set_description(part_filepath)\n",
				"\n",
				"        infile = bitstring.ConstBitStream(filename=part_filepath)\n",
				"        infile.pos = 0\n",
				"\n",
				"        while True:\n",
				"            try:\n",
				"                record = infile.readlist(bitstring_unpack_str)\n",
				"            except:\n",
				"                # print(record)\n",
				"                # traceback.print_exc()\n",
				"                break\n",
				"\n",
				"            record_data = {\n",
				"                'serial data number': record[0],\n",
				"                'serial sheet number': record[1],\n",
				"                'JIS Code': ''.join(map(shared.T56, record[4:8])),\n",
				"                'Evaluation of Individual Character Image (0=clean, 1, 2, 3)': record[8],\n",
				"                'Evaluation of Character Group (0=clean, 1, 2)': record[9],\n",
				"                'Sample Position Y on Sheet': record[10],\n",
				"                'Sample Position X on Sheet': record[11],\n",
				"                'Male-Female Code (1=male, 2=female) (JIS X 0303)': record[12],\n",
				"                'Age of Writer': record[13],\n",
				"                'Industry Classification Code (JIS X 0403)': record[14],\n",
				"                'Occupation Classification Code (JIS X 0404)': record[15],\n",
				"                'Sheet Gathering Date': record[16],\n",
				"                'Scanning Date': record[17],\n",
				"                'Number of X-Axis Sampling Points': record[18],\n",
				"                'Number of Y-Axis Sampling Points': record[19],\n",
				"                'Number of Levels of Pixel': record[20],\n",
				"                'Magnification of Scanning Lenz': record[21],\n",
				"                'Serial Data Number (old)': record[22],\n",
				"                'PNG_encoded_image': encode_image_as_png_bytes(record[23]),\n",
				"            }\n",
				"\n",
				"            serialized_record_datatype, serialized_record = shared.XFormat.serialize_obj(record_data)\n",
				"            record_byte_count = len(serialized_record)\n",
				"            record_seek_start = outfile.tell()\n",
				"\n",
				"            outfile.write(serialized_record_datatype)\n",
				"            outfile.write(shared.XFormat.serialize_int(record_byte_count))\n",
				"            outfile.write(serialized_record)\n",
				"\n",
				"            record_seek_end = outfile.tell()\n",
				"\n",
				"            metadata_record = {\n",
				"                'serial data number': record[0],\n",
				"                'serial sheet number': record[1],\n",
				"                'JIS Code': ''.join(map(shared.T56, record[4:8])),\n",
				"                'Evaluation of Individual Character Image (0=clean, 1, 2, 3)': record[8],\n",
				"                'Evaluation of Character Group (0=clean, 1, 2)': record[9],\n",
				"                'Sample Position Y on Sheet': record[10],\n",
				"                'Sample Position X on Sheet': record[11],\n",
				"                'Male-Female Code (1=male, 2=female) (JIS X 0303)': record[12],\n",
				"                'Age of Writer': record[13],\n",
				"                'Industry Classification Code (JIS X 0403)': record[14],\n",
				"                'Occupation Classification Code (JIS X 0404)': record[15],\n",
				"                'Sheet Gathering Date': record[16],\n",
				"                'Scanning Date': record[17],\n",
				"                'Number of X-Axis Sampling Points': record[18],\n",
				"                'Number of Y-Axis Sampling Points': record[19],\n",
				"                'Number of Levels of Pixel': record[20],\n",
				"                'Magnification of Scanning Lenz': record[21],\n",
				"                'Serial Data Number (old)': record[22],\n",
				"                'seek_start': record_seek_start,\n",
				"                'seek_end': record_seek_end,\n",
				"            }\n",
				"\n",
				"            records_metadata.append(metadata_record)\n",
				"\n",
				"            pbar.set_description(part_filepath + ' - ' + str(metadata_record['serial data number']) + ' - ' + metadata_record['JIS Code'])"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"metadata_filepath = 'etl3-metadata.json'\n",
				"if os.path.exists(metadata_filepath):\n",
				"    raise Exception(metadata_filepath + 'is already existed!')\n",
				"\n",
				"with open(metadata_filepath, mode='w', encoding='utf-8') as out_stream:\n",
				"    json.dump(records_metadata, out_stream, ensure_ascii=False, indent='\\t')"
			]
		},
		{
			"cell_type": "code",
			"execution_count": null,
			"metadata": {},
			"outputs": [],
			"source": [
				"len(records_metadata)"
			]
		}
	],
	"metadata": {
		"kernelspec": {
			"display_name": "Python 3",
			"language": "python",
			"name": "python3"
		},
		"language_info": {
			"codemirror_mode": {
				"name": "ipython",
				"version": 3
			},
			"file_extension": ".py",
			"mimetype": "text/x-python",
			"name": "python",
			"nbconvert_exporter": "python",
			"pygments_lexer": "ipython3"
		}
	},
	"nbformat": 4,
	"nbformat_minor": 2
}