From: Francois Fleuret Date: Sun, 7 Aug 2022 19:50:15 +0000 (+0200) Subject: Added the small-weight embedding initialization. X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=c3621f9a75cd4d79410d90a29dc9fdec401eaa2d;p=mygpt.git Added the small-weight embedding initialization. --- diff --git a/mygpt.py b/mygpt.py index 7ff1035..3bce361 100755 --- a/mygpt.py +++ b/mygpt.py @@ -97,6 +97,10 @@ class MyGPT(nn.Module): AddPositionalEncoding(len_max), ) + # Small embedding initialization + with torch.no_grad(): + self.embedding[0].weight.normal_(0, 2e-2) + trunk_blocks = [ ] for _ in range(nb_blocks):