projects
/
culture.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Update.
[culture.git]
/
tasks.py
diff --git
a/tasks.py
b/tasks.py
index
c7348d5
..
cbc8e6b
100755
(executable)
--- a/
tasks.py
+++ b/
tasks.py
@@
-1429,7
+1429,7
@@
class Grid(Task):
def tensorize(self, descr):
token_descr = [s.strip().split(" ") for s in descr]
l = max([len(s) for s in token_descr])
def tensorize(self, descr):
token_descr = [s.strip().split(" ") for s in descr]
l = max([len(s) for s in token_descr])
- token_descr = [s + ["
<nul>
"] * (l - len(s)) for s in token_descr]
+ token_descr = [s + ["
#
"] * (l - len(s)) for s in token_descr]
id_descr = [[self.token2id[u] for u in s] for s in token_descr]
return torch.tensor(id_descr, device=self.device)
id_descr = [[self.token2id[u] for u in s] for s in token_descr]
return torch.tensor(id_descr, device=self.device)
@@
-1440,7
+1440,7
@@
class Grid(Task):
# trim all the tensors in the tuple z to remove as much token from
# left and right in the first tensor. If z is a tuple, all its
# elements are trimed according to the triming for the first
# trim all the tensors in the tuple z to remove as much token from
# left and right in the first tensor. If z is a tuple, all its
# elements are trimed according to the triming for the first
- def trim(self, z, token="
<nul>
"):
+ def trim(self, z, token="
#
"):
n = self.token2id[token]
if type(z) == tuple:
x = z[0]
n = self.token2id[token]
if type(z) == tuple:
x = z[0]
@@
-1459,8
+1459,7
@@
class Grid(Task):
nb_train_samples,
nb_test_samples,
batch_size,
nb_train_samples,
nb_test_samples,
batch_size,
- height,
- width,
+ size,
logger=None,
device=torch.device("cpu"),
):
logger=None,
device=torch.device("cpu"),
):
@@
-1468,7
+1467,7
@@
class Grid(Task):
self.device = device
self.batch_size = batch_size
self.device = device
self.batch_size = batch_size
- self.grid_factory = grid.GridFactory(
height=height, width=width
)
+ self.grid_factory = grid.GridFactory(
size=size
)
if logger is not None:
logger(
if logger is not None:
logger(
@@
-1483,7
+1482,7
@@
class Grid(Task):
)
# Build the tokenizer
)
# Build the tokenizer
- tokens =
{}
+ tokens =
set()
for d in [self.train_descr, self.test_descr]:
for s in d:
for t in s.strip().split(" "):
for d in [self.train_descr, self.test_descr]:
for s in d:
for t in s.strip().split(" "):
@@
-1492,12
+1491,12
@@
class Grid(Task):
# the same descr
tokens = list(tokens)
tokens.sort()
# the same descr
tokens = list(tokens)
tokens.sort()
- tokens = ["
<nul>
"] + tokens
+ tokens = ["
#
"] + tokens
self.token2id = dict([(t, n) for n, t in enumerate(tokens)])
self.id2token = dict([(n, t) for n, t in enumerate(tokens)])
self.token2id = dict([(t, n) for n, t in enumerate(tokens)])
self.id2token = dict([(n, t) for n, t in enumerate(tokens)])
- self.t_nul = self.token2id["
<nul>
"]
- self.t_true = self.token2id["
<true>
"]
- self.t_false = self.token2id["
<false>
"]
+ self.t_nul = self.token2id["
#
"]
+ self.t_true = self.token2id["
true
"]
+ self.t_false = self.token2id["
false
"]
# Tokenize the train and test sets
self.train_input = self.tensorize(self.train_descr)
# Tokenize the train and test sets
self.train_input = self.tensorize(self.train_descr)
@@
-1540,8
+1539,8
@@
class Grid(Task):
nb_total = ar_mask.sum().item()
nb_correct = ((correct == result).long() * ar_mask).sum().item()
nb_total = ar_mask.sum().item()
nb_correct = ((correct == result).long() * ar_mask).sum().item()
- logger(f"test_performance {nb_total=} {nb_correct=}")
- logger(f"main_test_accuracy {nb_correct / nb_total}")
+ logger(f"test_performance {n
_epoch} {n
b_total=} {nb_correct=}")
+ logger(f"main_test_accuracy {n
_epoch} {n
b_correct / nb_total}")
######################################################################
######################################################################