Trained using SimCSE
#!/bin/bash
# pip install -r requirements.txt
NUM_GPU=4
# Randomly set a port number
# If you encounter "address already used" error, just run again or manually set an available port id.
PORT_ID=$(expr $RANDOM + 1000)
# Allow multiple threads
export OMP_NUM_THREADS=8
# --model_name_or_path vesteinn/DanskBERT \
# --model_name_or_path bert-base-uncased \
python -m torch.distributed.launch --nproc_per_node $NUM_GPU --master_port $PORT_ID train.py \
--train_file data/dfm_paragraphs.txt \
--model_name_or_path chcaa/dfm-encoder-large-v1 \
--output_dir result/dfm-sentence-encoder-large-v2 \
--num_train_epochs 1 \
--per_device_train_batch_size 32 \
--learning_rate 1e-5 \
--max_seq_length 32 \
--evaluation_strategy steps \
--metric_for_best_model stsb_spearman \
--load_best_model_at_end \
--pooler_type avg \
--mlp_only_train \
--do_mlm \
--overwrite_output_dir \
--temp 0.05 \
--do_train \
--fp16 \
"$@"