Skip to content

Commit 7037eb2

Browse files
Adel-MoumenAdel-Moumen
authored
Add TF32 flags + general improvements in DDP backend (#2682)
* add TF32 flags * reduce a bit the DDP backend * remove global variable * add allow / id in wandb * add checks * improve barrier depending on the context * add check in if_main_process() * destroy properly ddp group * 2f instead of 4f * add ddp_destroy_process_group * comment * remove ddp destroy * add ddp check properly * remove unused parameter and fix issues with DDP and pretrainer * add ctxt manager to make sure that global variable is always decreased even with an exception being raised * fix issue with pretrainer and DDP * follow up latter issue - now DDP works with pretrained models. * add logger adapter in SpeechBrain * add example of having logger adapter for fetch.py * remove setting logger lvl * add `standalone` * JESUS! THE PRE-COMMITs ARE SPAMMING MY MAILS F. * add example * :( he deserved it ! * replace everywhere `get_logger` to use SB one + fix circular imports + fix missing logging lib when needed * rename MainProcessGuard to MainProcessContext * warning_once * docstring * fix logger config * docstring * fix whisper normalizer fn * progressbar on ddp
1 parent eb13b9e commit 7037eb2

225 files changed

Lines changed: 757 additions & 596 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/multigpu.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ Using SpeechBrain, this would look like:
2727

2828
```bash
2929
cd recipes/<dataset>/<task>/
30-
torchrun --nproc_per_node=4 experiment.py hyperparams.yaml
30+
torchrun --standalone --nproc_per_node=4 experiment.py hyperparams.yaml
3131
```
3232

3333
... where `nproc_per_node` is the the number of processes to spawn/GPUs to use.

‎recipes/AISHELL-1/ASR/CTC/train_with_wav2vec.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,16 @@
1414
* Yingzhi WANG 2022
1515
"""
1616

17-
import logging
1817
import sys
1918

2019
import torch
2120
from hyperpyyaml import load_hyperpyyaml
2221

2322
import speechbrain as sb
2423
from speechbrain.utils.distributed import run_on_main
24+
from speechbrain.utils.logger import get_logger
2525

26-
logger = logging.getLogger(__name__)
26+
logger = get_logger(__name__)
2727

2828

2929
# Define training procedure

‎recipes/AISHELL-1/ASR/seq2seq/train.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@
33
AISHELL-1 seq2seq model recipe. (Adapted from the LibriSpeech recipe.)
44
"""
55

6-
import logging
76
import sys
87

98
import torch
109
from hyperpyyaml import load_hyperpyyaml
1110

1211
import speechbrain as sb
1312
from speechbrain.utils.distributed import run_on_main
13+
from speechbrain.utils.logger import get_logger
1414

15-
logger = logging.getLogger(__name__)
15+
logger = get_logger(__name__)
1616

1717

1818
# Define training procedure
@@ -299,7 +299,7 @@ def text_pipeline(wrd):
299299
) = dataio_prepare(hparams)
300300

301301
# We download and pretrain the tokenizer
302-
run_on_main(hparams["pretrainer"].collect_files)
302+
hparams["pretrainer"].collect_files()
303303
hparams["pretrainer"].load_collected()
304304

305305
# Trainer initialization

‎recipes/AISHELL-1/ASR/transformer/train.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@
88
* Titouan Parcollet 2021
99
"""
1010

11-
import logging
1211
import sys
1312

1413
import torch
1514
from hyperpyyaml import load_hyperpyyaml
1615

1716
import speechbrain as sb
1817
from speechbrain.utils.distributed import run_on_main
18+
from speechbrain.utils.logger import get_logger
1919

20-
logger = logging.getLogger(__name__)
20+
logger = get_logger(__name__)
2121

2222

2323
# Define training procedure
@@ -421,7 +421,7 @@ def text_pipeline(wrd):
421421
) = dataio_prepare(hparams)
422422

423423
# We download and pretrain the tokenizer
424-
run_on_main(hparams["pretrainer"].collect_files)
424+
hparams["pretrainer"].collect_files()
425425
hparams["pretrainer"].load_collected()
426426

427427
# Trainer initialization

‎recipes/AISHELL-1/ASR/transformer/train_with_wav2vect.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,16 @@
66
77
"""
88

9-
import logging
109
import sys
1110

1211
import torch
1312
from hyperpyyaml import load_hyperpyyaml
1413

1514
import speechbrain as sb
1615
from speechbrain.utils.distributed import run_on_main
16+
from speechbrain.utils.logger import get_logger
1717

18-
logger = logging.getLogger(__name__)
18+
logger = get_logger(__name__)
1919

2020

2121
# Define training procedure
@@ -422,7 +422,7 @@ def text_pipeline(wrd):
422422
) = dataio_prepare(hparams)
423423

424424
# We download and pretrain the tokenizer
425-
run_on_main(hparams["pretrainer"].collect_files)
425+
hparams["pretrainer"].collect_files()
426426
hparams["pretrainer"].load_collected()
427427

428428
# Trainer initialization

‎recipes/AISHELL-1/aishell_prepare.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,14 @@
1111
import csv
1212
import functools
1313
import glob
14-
import logging
1514
import os
1615
import shutil
1716

1817
from speechbrain.dataio.dataio import read_audio_info
18+
from speechbrain.utils.logger import get_logger
1919
from speechbrain.utils.parallel import parallel_map
2020

21-
logger = logging.getLogger(__name__)
21+
logger = get_logger(__name__)
2222

2323

2424
def extract_and_cleanup_wav_files(

‎recipes/AMI/Diarization/experiment.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020

2121
import glob
2222
import json
23-
import logging
2423
import os
2524
import pickle
2625
import shutil
@@ -37,11 +36,12 @@
3736
from speechbrain.processing.PLDA_LDA import StatObject_SB
3837
from speechbrain.utils.DER import DER
3938
from speechbrain.utils.distributed import run_on_main
39+
from speechbrain.utils.logger import get_logger
4040

4141
np.random.seed(1234)
4242

4343
# Logger setup
44-
logger = logging.getLogger(__name__)
44+
logger = get_logger(__name__)
4545
current_dir = os.path.dirname(os.path.abspath(__file__))
4646
sys.path.append(os.path.dirname(current_dir))
4747

‎recipes/AMI/ami_prepare.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@
88

99
import glob
1010
import json
11-
import logging
1211
import os
1312
import xml.etree.ElementTree as et
1413

1514
from ami_splits import get_AMI_split
1615

1716
from speechbrain.dataio.dataio import load_pkl, save_pkl
17+
from speechbrain.utils.logger import get_logger
1818

19-
logger = logging.getLogger(__name__)
19+
logger = get_logger(__name__)
2020
SAMPLERATE = 16000
2121

2222

‎recipes/Aishell1Mix/separation/train.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
"""
2323

2424
import csv
25-
import logging
2625
import os
2726
import sys
2827

@@ -37,6 +36,7 @@
3736
import speechbrain.nnet.schedulers as schedulers
3837
from speechbrain.core import AMPConfig
3938
from speechbrain.utils.distributed import run_on_main
39+
from speechbrain.utils.logger import get_logger
4040

4141

4242
# from: recipes/LibriMix/separation/train.py
@@ -568,7 +568,7 @@ def audio_pipeline_noise(noise_wav):
568568
sb.utils.distributed.ddp_init_group(run_opts)
569569

570570
# Logger info
571-
logger = logging.getLogger(__name__)
571+
logger = get_logger(__name__)
572572

573573
# If device is cpu use precision='bf16'
574574

‎recipes/AudioMNIST/audiomnist_prepare.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
import csv
1616
import json
17-
import logging
1817
import math
1918
import os
2019
from functools import partial
@@ -28,6 +27,7 @@
2827
import speechbrain as sb
2928
from speechbrain.dataio.dataio import load_pkl, save_pkl
3029
from speechbrain.utils.data_utils import download_file
30+
from speechbrain.utils.logger import get_logger
3131
from speechbrain.utils.superpowers import run_shell
3232

3333
DEFAULT_SPLITS = ["train", "valid", "test"]
@@ -40,7 +40,7 @@
4040
OPT_FILE = "opt_audiomnist_prepare.pkl"
4141

4242

43-
logger = logging.getLogger(__name__)
43+
logger = get_logger(__name__)
4444

4545

4646
def prepare_audiomnist(

0 commit comments

Comments
 (0)