+default_model_args = {
+ "17K": {
+ "dim_model": 32,
+ "dim_keys": 32,
+ "dim_hidden": 32,
+ "nb_heads": 2,
+ "nb_blocks": 2,
+ },
+ "37M": {
+ "dim_model": 512,
+ "dim_keys": 64,
+ "dim_hidden": 2048,
+ "nb_heads": 8,
+ "nb_blocks": 12,
+ },
+ "122M": {
+ "dim_model": 768,
+ "dim_keys": 64,
+ "dim_hidden": 2048,
+ "nb_heads": 8,
+ "nb_blocks": 24,
+ },
+ "352M": {
+ "dim_model": 1024,
+ "dim_keys": 64,
+ "dim_hidden": 2048,
+ "nb_heads": 8,
+ "nb_blocks": 48,
+ },
+}
+
+if args.model in default_model_args:
+ for k, v in default_model_args[args.model].items():
+ if getattr(args, k) is None:
+ setattr(args, k, v)
+else:
+ raise ValueError(f"Unknown model {args.model}")
+
+######################################################################
+