file glob

Signed-off-by: ssbuild <[email protected]>
ssbuild · Nov 3, 2023 · bea540f · bea540f
1 parent 007ca53
commit bea540f
Show file tree

Hide file tree

Showing 8 changed files with 11 additions and 5 deletions.
diff --git a/data_utils.py b/data_utils.py
@@ -1,10 +1,12 @@
 # @Time    : 2023/1/22 16:22
 # @Author  : tk
 # @FileName: data_utils.py
+
 import sys
 import os
 sys.path.append(os.path.abspath(os.path.dirname(__file__)))
 
+import glob
 import copy
 import json
 import random
@@ -142,6 +144,7 @@ def _get_messages(self, lines):
     # 读取文件
     def on_get_corpus(self, files: typing.List, mode: str):
         D = []
+        files = sum([glob.glob(file) for file in files], [])
         for file in files:
             with open(file, mode='r', encoding='utf-8', newline='\n') as f:
                 lines = f.readlines()

diff --git a/scripts/config/train_ac.yaml b/scripts/config/train_ac.yaml
@@ -34,7 +34,8 @@ seed: 42
 fp16: true
 do_train: true
 train_file:
-- ../data/finetune_train_examples.json
+- ../data/*.json
+
 do_eval: false
 do_predict: false
 per_device_train_batch_size: 2

diff --git a/scripts/config/train_cl.yaml b/scripts/config/train_cl.yaml
@@ -39,7 +39,7 @@ seed: 42
 fp16: true
 do_train: true
 train_file:
-- ../data/finetune_train_examples.json
+- ../data/*.json
 do_eval: false
 do_predict: false
 per_device_train_batch_size: 2

diff --git a/scripts/config/train_hf.yaml b/scripts/config/train_hf.yaml
@@ -33,7 +33,7 @@ seed: 42
 fp16: true
 do_train: true
 train_file:
-- ../data/finetune_train_examples.json
+- ../data/*.json
 do_eval: false
 do_predict: false
 per_device_train_batch_size: 2

diff --git a/scripts/config/train_pl.yaml b/scripts/config/train_pl.yaml
@@ -23,7 +23,7 @@ data_backend: parquet
 convert_onnx: false
 do_train: true
 train_file:
-- ../data/finetune_train_examples.json
+- ../data/*.json
 max_epochs: 20
 max_steps: -1
 

diff --git a/scripts/train_full.sh b/scripts/train_full.sh
@@ -11,6 +11,7 @@ export enable_ptv2=false
 export enable_lora=false
 export load_in_bit=0
 
+#export CUDA_VISIBLE_DEVICES="0,1,2,3"
 
 
 

diff --git a/scripts/train_lora.sh b/scripts/train_lora.sh
@@ -11,6 +11,7 @@ export enable_ptv2=false
 export enable_lora=true
 export load_in_bit=4
 
+#export CUDA_VISIBLE_DEVICES="0,1,2,3"
 
 usage() { echo "Usage: $0 [-m <train|dataset>]" 1>&2; exit 1; }
 

diff --git a/scripts/train_ptv2.sh b/scripts/train_ptv2.sh
@@ -11,7 +11,7 @@ export enable_ptv2=true
 export enable_lora=false
 export load_in_bit=0
 
-
+#export CUDA_VISIBLE_DEVICES="0,1,2,3"
 
 usage() { echo "Usage: $0 [-m <train|dataset>]" 1>&2; exit 1; }