170 lines
6.3 KiB
Python
170 lines
6.3 KiB
Python
__author__ = "Adel Daouzli"
|
|
__licence__ = "GPLv3"
|
|
|
|
import os
|
|
from pnmimage import PnmImage
|
|
|
|
class LettersData(object):
|
|
def __init__(self, data_folder="data/", filename_expected_data="list_expected_data.txt", car_filename_fmt="ext_ln{}_car{}.pgm"):
|
|
self.filename_expected_data = filename_expected_data
|
|
self.car_filename_fmt = car_filename_fmt
|
|
self.data_folder = data_folder
|
|
self.expected_data_list = []
|
|
self.expected_letters = []
|
|
self.letter_to_int = {}
|
|
self.letters_classes = {}
|
|
self.extract_vocab = []
|
|
self.vocab = {}
|
|
self._images = []
|
|
self._expected_values = []
|
|
self.nb_vocab = 0
|
|
self.input_image_size = 0
|
|
self.selection_vocab = {}
|
|
self.selection_batch = []
|
|
self.letter_to_vector = {}
|
|
self.vector_to_letter = {}
|
|
|
|
def _gen_ext_filenames_list(self):
|
|
for ln in range(100):
|
|
for car in range(100):
|
|
n = self.data_folder + self.car_filename_fmt.format(ln, car)
|
|
if os.path.exists(n) and n.replace(data_folder, '') in self.filenames_list:
|
|
yield(n)
|
|
|
|
def _get_expected_letters_list(self):
|
|
"""Get expected letters list from expected data file.
|
|
|
|
:return: list of tuples with the letter and the corresponding filename
|
|
|
|
"""
|
|
data_list = []
|
|
name = self.filename_expected_data
|
|
with open(name) as f:
|
|
lst = [(line.split(" ")[0], line.split(" ", 1)[1].replace('\n', '')) for line in f.readlines() if line.strip() != '']
|
|
# list of tuples with (letter, filename)
|
|
data_list = [(e[1], e[0]) for e in lst]
|
|
return data_list
|
|
|
|
def _extract_data(self):
|
|
"""Extract data.
|
|
Create list of expected letters/filenames, base letters with info, classes.
|
|
"""
|
|
self.expected_data_list = self._get_expected_letters_list()
|
|
"""list of tuples with letter and filename"""
|
|
self.extract_vocab = sorted(set([l[0] for l in self.expected_data_list]))
|
|
"""list of data vocab letters sorted"""
|
|
self.nb_vocab = len(self.extract_vocab)
|
|
|
|
self.vocab = {}
|
|
letters_list = [l[0] for l in self.expected_data_list]
|
|
for i, c in enumerate(self.extract_vocab):
|
|
vec = [0] * self.nb_vocab
|
|
vec[i] = 1
|
|
self.vocab[c] = {
|
|
'index': i,
|
|
'count': letters_list.count(c),
|
|
'vector': vec
|
|
}
|
|
self.letter_to_int = {c: i for i, c in enumerate(self.extract_vocab)}
|
|
"""dict of letters with index of each"""
|
|
self.letters_classes = {}
|
|
"""dict of letters with vector representation of each letter"""
|
|
|
|
for letter, idx in sorted(self.letter_to_int.items(), key=lambda x: x[0]):
|
|
cls = [0] * self.nb_vocab
|
|
cls[idx] = 1
|
|
self.letters_classes[letter] = cls
|
|
|
|
def get_vocab_with_min_count(self, min_count):
|
|
"""Get the vocab. A dictionary of letters with count, index and vector.
|
|
The index is re-computed and also the vector to match the number of subelements
|
|
"""
|
|
subvocab = {}
|
|
i = 0
|
|
for c, info in sorted(self.vocab.items(), key=lambda x: x[0]):
|
|
if info['count'] >= min_count:
|
|
subvocab[c] = {
|
|
'count': info['count'],
|
|
'index': i,
|
|
'vector': None
|
|
}
|
|
i += 1
|
|
nb_vocab = len(subvocab)
|
|
self.letter_to_vector = {}
|
|
self.vector_to_letter = {}
|
|
for c in sorted(subvocab):
|
|
vec = [0] * nb_vocab
|
|
vec[subvocab[c]['index']] = 1
|
|
subvocab[c]['vector'] = vec
|
|
self.letter_to_vector[c] = vec
|
|
return subvocab
|
|
|
|
def get_letter_of_vector(self, vector):
|
|
"""Get the letter corresponding to a given vector.
|
|
|
|
:param vector: a flatten list representing the vector representation of a letter
|
|
:return: the found letter else None
|
|
"""
|
|
ret = None
|
|
assert(type(vector) is list)
|
|
for letter, vec in self.letter_to_vector.items():
|
|
if vec == vector:
|
|
ret = letter
|
|
break
|
|
return ret
|
|
|
|
def get_batches(self, min_count=0, mini_batch_size=None):
|
|
"""Get the selection data based on min count of letters in the dataset
|
|
|
|
:param min_count: minimal count of same letters to be added (default 0 for the whole dataset)
|
|
:param mini_batch_size: size of a mini batch, if None the whole dataset size (default None)
|
|
if whole size is not factor of the mini batch size then the last mini batch
|
|
has a size < mini batch size
|
|
:return: a list of mini batches (at least list of one)
|
|
"""
|
|
if mini_batch_size is None:
|
|
mini_batch_size = len(self.expected_data_list)
|
|
self.selection_vocab = self.get_vocab_with_min_count(min_count)
|
|
self.selection_batch = []
|
|
X = []
|
|
Y = []
|
|
bsize = 0
|
|
for letter, name in self.expected_data_list:
|
|
path = '{}{}'.format(self.data_folder, name)
|
|
if letter in self.selection_vocab:
|
|
img = PnmImage()
|
|
if img.load(path) == False:
|
|
print("ERROR: failed to open '{}'".format(path))
|
|
image_size = img.get_size()
|
|
bsize += 1
|
|
X.append(img.get_data_bin())
|
|
Y.append(self.selection_vocab[letter]['vector'])
|
|
if bsize >= mini_batch_size:
|
|
self.selection_batch.append((X, Y))
|
|
X = []
|
|
Y = []
|
|
bsize = 0
|
|
if bsize > 0:
|
|
self.selection_batch.append((X, Y))
|
|
return self.selection_batch
|
|
|
|
def process(self):
|
|
self._extract_data()
|
|
|
|
def get_data_with_min_count(self, min_count):
|
|
vocab = self.get_vocab_with_min_count(min_count)
|
|
#todo
|
|
return images, expected_values
|
|
|
|
def get_vocab(self):
|
|
"""Get the vocab. A dictionary of letters with count, index and vector"""
|
|
return self.vocab
|
|
|
|
def get_classes(self):
|
|
return self.letters_classes
|
|
|
|
def get_class_element_size(self):
|
|
return self.nb_vocab
|
|
|
|
def get_input_image_size(self):
|
|
return self.input_image_size |