File size: 503 Bytes
14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
#!/usr/bin/env python3
import glob
import os
from datasets import load_dataset
folder_path = "isp-data-json/" # Replace with your folder path
# Search for all .json files in the folder
json_files = glob.glob(os.path.join(folder_path, "*.jsonl"))
path = "pminervini/inverse-scaling"
for json_path in json_files:
base_name = os.path.basename(json_path)
name = base_name.split("_")[0]
ds = load_dataset("json", data_files={"data": json_path})
ds.push_to_hub(path, config_name=name)
|