Add pre-training hyperparams

README.md

@@ -36,6 +36,13 @@ training_data:
 - corpora:
   - (+Abs) Springer and Elsevier abstracts in the duration of 1990-2020
   - (+Abs+Full) Springer and Elsevier abstracts and open access full publication text in the duration of 1990-2020
+pre-training-hyperparams:
+- MAX_LEN = 512 # Default of BERT Tokenizer
+- MLM_PROP = 0.15 # Data Collator
+- num_train_epochs = 3 # the minimum sufficient epochs found on many articles && default of trainer here
+- per_device_train_batch_size = 16 # the maximumn that could be held by V100 on Ara with 512 MAX_LEN was 8 in the old run
+- per_device_eval_batch_size = 16 # usually as above
+- gradient_accumulation_steps = 4 # this will grant a minim batch size 16 * 4 * nGPUs.
 ---
 
 # BiodivBERT