注意版本！！

python == 3.8.6

torch == 1.10.0

transformers == 4.36.2

datasets == 2.15.0

import json
# 数据集下载地址：https://www.cluebenchmarks.com/introduce.html
# 细粒度命名实体识别->下载# 将数据转为  BIO 标注形式
def dimension_label(path, save_path, labels_path=None):label_dict = ['O']with open(save_path, "a", encoding="utf-8") as w:with open(path, "r", encoding="utf-8") as r:for line in r:line = json.loads(line)text = line['text']label = line['label']text_label = ['O'] * len(text)for label_key in label:  # 遍历实体标签B_label = "B-" + label_keyI_label = "I-" + label_keyif B_label not in label_dict:label_dict.append(B_label)if I_label not in label_dict:label_dict.append(I_label)label_item = label[label_key]for entity in label_item:  # 遍历实体position = label_item[entity]start = position[0][0]end = position[0][1]text_label[start] = B_labelfor i in range(start + 1, end + 1):text_label[i] = I_labelline = {"text": text,"label": text_label}line = json.dumps(line, ensure_ascii=False)w.write(line + "\n")w.flush()if labels_path:  # 保存 label ，后续训练和预测时使用label_map = {}for i,label in enumerate(label_dict):label_map[label] = iwith open(labels_path, "w", encoding="utf-8") as w:labels = json.dumps(label_map, ensure_ascii=False)w.write(labels + "\n")w.flush()
if __name__ == '__main__':path = "./cluener_public/dev.json"save_path = "./data/dev.json"dimension_label(path, save_path)path = "./cluener_public/train.json"save_path = "./data/train.json"labels_path = "./data/labels.json"dimension_label(path, save_path, labels_path)

# 处理数据集构建 Dataset
from torch.utils.data import Dataset, DataLoader
import torch
import json
class NERDataset(Dataset):