add job generation

Author: jqueguiner

Dockerfile

@@ -18,11 +18,31 @@ RUN pip install git+https://github.com/huggingface/transformers.git
 
 RUN mkdir -p /workspace/wav2vec/
 
-COPY finetune.sh /workspace/wav2vec/
-COPY run_common_voice.py /workspace/wav2vec/
+COPY finetune.sh run_common_voice.py  finetune_with_params.sh /workspace/wav2vec/
+
 COPY home-server.html /usr/bin/home-server.html
 
 RUN chown -R 42420:42420 /workspace
 
+#Default training env variables
+ENV model_name_or_path="facebook/wav2vec2-large-xlsr-53" \
+    dataset_config_name="fr" \
+    output_dir="/workspace/output_models/wav2vec2-large-xlsr-french-demo" \
+    cache_dir="/workspace/data" \
+    num_train_epochs="1" \
+    per_device_train_batch_size="32" \
+    evaluation_strategy="steps" \
+    learning_rate="3e-4" \
+    warmup_steps="500" \
+    save_steps="10" \
+    eval_steps="10" \
+    save_total_limit="1" \
+    logging_steps="10" \
+    feat_proj_dropout="0.0" \
+    layerdrop="0.1" \
+    max_train_samples=100 \
+    max_val_samples=100
+
+WORKDIR /workspace/wav2vec
 ENTRYPOINT []
 CMD ["supervisord", "-n", "-u", "42420", "-c", "/etc/supervisor/supervisor.conf"]

finetune_with_params.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+    
+python /workspace/wav2vec/run_common_voice.py \
+    --model_name_or_path=$model_name_or_path \
+    --dataset_config_name=$dataset_config_name \
+    --output_dir=$output_dir \
+    --cache_dir=$cache_dir \
+    --overwrite_output_dir \
+    --num_train_epochs=$num_train_epochs \
+    --per_device_train_batch_size=$per_device_train_batch_size \
+    --evaluation_strategy=$evaluation_strategy \
+    --learning_rate=$learning_rate \
+    --warmup_steps=$warmup_steps \
+    --fp16 \
+    --freeze_feature_extractor \
+    --save_steps=$save_steps \
+    --eval_steps=$eval_steps \
+    --save_total_limit=$save_total_limit \
+    --logging_steps=$logging_steps \
+    --group_by_length \
+    --feat_proj_dropout=$feat_proj_dropout \
+    --layerdrop=$layerdrop \
+    --gradient_checkpointing \
+    --do_train \
+    --do_eval \
+    --max_train_samples $max_train_samples \
+    --max_val_samples $max_val_samples
+

generate_all_trainings.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[8]:
+
+
+import os
+import csv
+
+
+# In[20]:
+
+
+with open('wav2vec_languages.csv') as csv_file:
+    csv_reader = csv.reader(csv_file, delimiter=',')
+    # This skips the first row of the CSV file because it's a header
+    next(csv_reader)
+    for (language_code, language_full_name) in csv_reader:
+        print(f"#Launching Training for {language_code}-{language_full_name}")
+        cmd = f"ovhai job run --gpu 1 --name '{language_code}-{language_full_name}' --volume output_models@GRA:/workspace/output_models:RW:cache -e model_name_or_path='facebook/wav2vec2-large-xlsr-53' -e dataset_config_name={language_code} -e output_dir='/workspace/output_models/wav2vec2-large-xlsr-{language_code}-{language_full_name}-demo' -e cache_dir='/workspace/data' databuzzword/hf-wav2vec -- sh /workspace/wav2vec/finetune_with_params.sh"
+        print(cmd)
+        stream = os.popen(cmd)
+        output = stream.read()
+        output
+
+
+# In[3]:
+
+
+
+
+
+# In[ ]:
+
+
+
+

wav2vec_languages.csv

@@ -0,0 +1,61 @@
+language_code,language_full_name
+ab,abkhazian
+ar,arabic
+as,assamese
+br,breton
+ca,catalan
+cnh,cnh
+cs,czech
+cv,chuvash
+cy,welsh
+de,german
+dv,divehi
+el,greek
+en,english
+eo,esperanto
+es,spanish
+et,estonian
+eu,basque
+fa,persian
+fi,finnish
+fr,french
+fy-NL,western_frisian-netherlands
+ga-IE,irish-ireland
+hi,hindi
+hsb,upper_sorbian
+hu,hungarian
+ia,interlingua
+id,indonesian
+it,italian
+ja,japanese
+ka,georgian
+kab,kabyle
+ky,kyrgyz
+lg,ganda
+lt,lithuanian
+lv,latvian
+mn,mongolian
+mt,maltese
+nl,dutch
+or,odia
+pa-IN,punjabi-india
+pl,polish
+pt,portuguese
+rm-sursilv,romansh_sursilv
+rm-vallader,romansh_vallader
+ro,romanian
+ru,russian
+rw,kinyarwanda
+sah,sakha
+sl,slovenian
+sv-SE,swedish-sweden
+ta,tamil
+th,thai
+tr,turkish
+tt,tatar
+uk,ukrainian
+vi,vietnamese
+vot,votic
+zh-CN,chinese-china
+zh-HK,chinese-hong_kong_sar_china
+zh-TW,chinese-taiwan