分别使用两种框架，加载预训练模型，对句对进行分类

数据下载：千言数据集：文本相似度

1. Paddle

可以使用 paddlenlp 直接加载预训练模型，比较方便

# %%
# 比赛地址
# https://aistudio.baidu.com/aistudio/competition/detail/45
import time
import os
import numpy as np
import paddle
import paddlenlp
import paddle.nn.functional as F
import paddle.distributed as dist  # 并行
from functools import partial
from paddlenlp.data import Stack, Pad, Tuple
import paddle.nn as nn
from paddlenlp.datasets import load_dataset
from paddlenlp.transformers import LinearDecayWithWarmupdef read(data, datasetname, predict=False): # 将数据转成迭代器if not predict:for d in data:label = d["label"]if datasetname != "lcqmc":text1, text2 = d["sentence1"], d["sentence2"]else:text1, text2 = d["query"], d["title"]yield {"label": label, "text1": text1, "text2": text2}else:for d in data:if datasetname != "lcqmc":text1, text2 = d["sentence1"], d["sentence2"]else:text1, text2 = d["query"], d["title"]yield {"text1": text1, "text2": text2}def convert_data(data, tokenizer, datasetname, max_seq_len=512, is_test=False): # 数据转码为模型的输入text1, text2 = data["text1"], data["text2"]encoded_inputs = tokenizer(text=text1, text_pair=text2, max_seq_len=max_seq_len)input_ids = encoded_inputs["input_ids"]token_type_ids = encoded_inputs["token_type_ids"]if not is_test:label = np.array([data["label"]], dtype="int64")return input_ids, token_type_ids, labelreturn input_ids, token_type_idsclass PretrainedModel(nn.Layer): # 预训练模型 + FCdef __init__(self, pretrained_model, dropout=None):super().__init__()self.ptm = pretrained_modelself.dropout = nn.Dropout(dropout if dropout is not None else 0.1)self.clf = nn.Linear(self.ptm.config["hidden_size"], 2)def forward(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)cls_embedding = self.dropout(cls_embedding)logits = self.clf(cls_embedding)probs = F.softmax(logits)return probsclass Recongnizer(): # 识别器类def __init__(self, datasetname, state_dict_path=None):  # 传入模型参数路径self.seed = 100paddle.seed = self.seedself.batch_size = 128self.epochs = 20self.max_seq_len = 512self.datasetname = datasetname# paddlenlp自带的一键加载数据self.train_ds, self.dev_ds, self.test_ds = load_dataset(datasetname, splits=["train", "dev", "test"])# 使用预训练模型的tokenizerself.tokenizer = paddlenlp.transformers.ErnieGramTokenizer.from_pretrained("ernie-gram-zh")# https://gitee.com/paddlepaddle/PaddleNLP/blob/develop/docs/model_zoo/transformers.rst# 加载预训练模型self.pretrained_model = paddlenlp.transformers.ErnieGramModel.from_pretrained("ernie-gram-zh")self.model = PretrainedModel(self.pretrained_model)if state_dict_path: # 如果传入了模型参数，直接加载参数try:state_dict = paddle.load(state_dict_path)self.model.set_dict(state_dict)except:print("加载模型参数失败！")self.pathname = "checkpoint"self.global_step = 0isExists = os.path.exists(self.pathname)if not isExists:os.mkdir(self.pathname)self.save_dir = ""self.save_param_path = ""def fit(self):# 加载数据集train_ds = load_dataset(read, data=self.train_ds, datasetname=self.datasetname, lazy=False)dev_ds = load_dataset(read, data=self.dev_ds, datasetname=self.datasetname, lazy=False)test_ds = load_dataset(read, data=self.test_ds, datasetname=self.datasetname, predict=True, lazy=False)# 展示数据for i, example in enumerate(train_ds):if i < 5:print(example)input_ids, token_type_ids, label = convert_data(train_ds[0], self.tokenizer, self.datasetname)print(input_ids)# [1, 692, 811, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2, 329, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2]print(token_type_ids)# [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]print(label)# [1]# 数据转换函数trans_func = partial(convert_data, tokenizer=self.tokenizer, datasetname=self.datasetname,max_seq_len=self.max_seq_len)# 对数据进行批量打包+padbatchify_fn = lambda samples, fn=Tuple(Pad(axis=0, pad_val=self.tokenizer.pad_token_id),Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id),Stack(dtype="int64")): [d for d in fn(samples)]# 将长度不同的多个句子padding到统一长度，取N个输入数据中的最大长度# 长度是指的： 一个batch中的最大长度，主要考虑性能开销# 取样器batch_sampler = paddle.io.DistributedBatchSampler(train_ds, batch_size=self.batch_size, shuffle=True)# 数据加载器train_data_loader = paddle.io.DataLoader(dataset=train_ds.map(trans_func),batch_sampler=batch_sampler,collate_fn=batchify_fn,return_list=True)batch_sampler = paddle.io.BatchSampler(dev_ds, batch_size=self.batch_size, shuffle=False)dev_data_loader = paddle.io.DataLoader(dataset=dev_ds.map(trans_func),batch_sampler=batch_sampler,collate_fn=batchify_fn,return_list=True)num_training_steps = len(train_data_loader) * self.epochs# 学习率lr_scheduler = LinearDecayWithWarmup(5e-5, num_training_steps, 0.0)# 衰减的参数decay_params = [p.name for n, p in self.model.named_parameters()if not any(nd in n for nd in ["bias", "norm"])]# 梯度剪切clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0)# 优化器optimizer = paddle.optimizer.AdamW(learning_rate=lr_scheduler,parameters=self.model.parameters(),weight_decay=0.0,apply_decay_param_fun=lambda x: x in decay_params,grad_clip=clip)# 损失函数criterion = paddle.nn.loss.CrossEntropyLoss()# 评估准确率metric = paddle.metric.Precision()t_start = time.time()F1 = 0 # 最大F1值for epoch in range(1, self.epochs + 1):for step, batch in enumerate(train_data_loader, start=1):input_ids, token_type_ids, labels = batchprobs = self.model(input_ids=input_ids, token_type_ids=token_type_ids)loss = criterion(probs, labels)metric.update(np.argmax(probs, axis=1), labels)acc = metric.accumulate()self.global_step += 1if self.global_step % 10 == 0:print("训练步数 %d, epoch: %d, batch: %d, loss: %.5f, acc: %.5f, speed: %.2f step/s"% (self.global_step, epoch, step, loss, acc,10 / (time.time() - t_start)))t_start = time.time()loss.backward()optimizer.step()lr_scheduler.step()optimizer.clear_grad()if self.global_step % 100 == 0:_, F1, improve = self.evaluate(criterion, metric, dev_data_loader, F1, "dev")if improve:print("评估结果 F1值 : {:.3f} , 模型保存于：".format(F1) + self.save_param_path)else:print("最好结果 F1值 : {:.3f} , 当前评估没有提升！".format(F1))print("-----训练完成------")# 用最好的模型参数，提交预测state_dict = paddle.load(self.save_param_path)self.model.set_dict(state_dict)self.writeToFile(test_ds)@paddle.no_grad()def evaluate(self, criterion, metric, data_loader, F1, phase="dev"):self.model.eval()metric.reset()recall = paddle.metric.Recall()recall.reset()losses = []prob_list = []for batch in data_loader:input_ids, token_type_ids, labels = batchprobs = self.model(input_ids=input_ids, token_type_ids=token_type_ids)prob_list.extend(probs)loss = criterion(probs, labels)losses.append(loss.numpy())metric.update(np.argmax(probs, axis=1), labels)recall.update(np.argmax(probs, axis=1), labels)acc = metric.accumulate()rec = recall.accumulate()f1 = 0 if (acc + rec) == 0.0 else 2 * acc * rec / (acc + rec)improve = Falseif f1 > F1: # 保存 F1 值最大的时候的模型参数F1 = f1improve = Trueself.save_dir = os.path.join(self.pathname, "best_model_state")self.save_param_path = os.path.join(self.save_dir, "model_state_pdparams_F1_" + str(round(F1, 4)))paddle.save(self.model.state_dict(), self.save_param_path)self.tokenizer.save_pretrained(self.save_dir)print("评估 {} loss: {:.5}, acc: {:.5}, recall: {:.5}".format(phase, np.mean(losses), acc, rec))self.model.train()metric.reset()return prob_list, F1, improvedef predict(self, text1, text2): # 单条 句对预测encoded_inputs = self.tokenizer(text=text1, text_pair=text2, max_seq_len=self.max_seq_len)input_ids = encoded_inputs["input_ids"]token_type_ids = encoded_inputs["token_type_ids"]predict_data_loader = [(input_ids, token_type_ids)]batch_probs = []self.model.eval()with paddle.no_grad():for batch_data in predict_data_loader:input_ids, token_type_ids = batch_datainput_ids = paddle.to_tensor([input_ids])token_type_ids = paddle.to_tensor([token_type_ids])batch_prob = self.model(input_ids=input_ids, token_type_ids=token_type_ids)batch_prob = F.softmax(batch_prob, axis=1).numpy()batch_probs.append(batch_prob)batch_probs = np.concatenate(batch_probs, axis=0)return batch_probsdef writeToFile(self, test_ds): # 对测试集进行预测，写入文件with open(self.datasetname + ".tsv", "w", encoding="utf-8") as f:f.write("index\tprediction\n")for i, d in enumerate(test_ds):prob = self.predict(d["text1"], d["text2"])label = 1 if prob[0][1] >= 0.5 else 0f.write(str(i) + "\t" + str(label) + "\n")if __name__ == "__main__":dist.init_parallel_env()  # 初始化并行环境# 启动命令 python -m paddle.distributed.launch --gpus '0,1' xxx.py &# 并行训练设置# https://aistudio.baidu.com/aistudio/projectdetail/1222066# https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/02_paddle2.0_develop/06_device_cn.htmldatasetnames = ["lcqmc", "bq_corpus", "paws-x"]  #for name in datasetnames:model = Recongnizer(name)model.fit()

2. PyTorch

预训练模型下载：https://huggingface.co/nghuyong/ernie-1.0

# %%
# 比赛地址
# https://aistudio.baidu.com/aistudio/competition/detail/45
import time
import os
import numpy as np
import torch
from datetime import timedelta
import torch.nn as nn
import torch.nn.functional as F
from transformers import AutoTokenizer, AutoModel
from sklearn import metricsnp.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed_all(4)
torch.backends.cudnn.deterministic = True  # 保证每次运行结果一样def get_time_dif(start_time): # 辅助函数，获取训练时长"""获取已经使用的时间"""end_time = time.time()time_dif = end_time - start_timereturn timedelta(seconds=int(round(time_dif)))def load(file, test=False): # 加载数据集data = []with open(file, 'r', encoding='utf-8') as f:for line in f:res = line[:-1].split('\t')if not test:data.append({"text1": res[0], "text2": res[1], "label": int(res[2])})else:data.append({"text1": res[0], "text2": res[1]})return datadef load_dataset(datasetname): # 加载数据集train_ds = load(os.path.join(datasetname, "train.tsv"))dev_ds = load(os.path.join(datasetname, "dev.tsv"))test_ds = load(os.path.join(datasetname, "test.tsv"), True)return train_ds, dev_ds, test_dsdef convert_data(data, tokenizer, max_seq_len=512, is_test=False): # 转换数据集为模型可用的编码if isinstance(data, dict):text1, text2 = data["text1"], data["text2"]if not is_test:label = [data["label"]]else:text1, text2, label = [], [], []for d in data:if not is_test:label.append(d["label"])text1.append(d["text1"])text2.append(d["text2"])# tokenizer 参数说明见下面链接# https://huggingface.co/transformers/main_classes/tokenizer.html?highlight=tokenizerx = tokenizer(text=text1, text_pair=text2, add_special_tokens=True, padding='longest',max_length=max_seq_len, return_tensors='np', return_token_type_ids=True, return_attention_mask=True,truncation=True)input_ids, token_type_ids, mask = x["input_ids"], x["token_type_ids"], x["attention_mask"]if not is_test:return input_ids, token_type_ids, mask, np.array(label, dtype="int64")return input_ids, token_type_ids, maskclass DatasetIterator(object): # 数据迭代器def __init__(self, tokenizer, data, batch_size, device, max_seq_len=512, pred=False):self.batch_size = batch_sizeself.pred = predself.tokenizer = tokenizerself.max_seq_len = max_seq_lenself.dataset = []self.batch_data = []self.ct = 0for d in data:  # 对数据进行 batch 化，每 batch_size 个一组self.batch_data.append(d)self.ct += 1if self.ct == self.batch_size:self.dataset.append(self.batch_data)self.ct = 0self.batch_data = []if self.ct != 0:self.dataset.append(self.batch_data)self.n_batches = len(self.dataset)self.index = 0self.device = devicedef _to_tensor(self, datas): # 转化为 tensorinput_ids = torch.LongTensor(np.array(datas[0])).to(self.device)token_type_ids = torch.LongTensor(np.array(datas[1])).to(self.device)mask = torch.LongTensor(np.array(datas[2])).to(self.device)if not self.pred:label = datas[3]label = torch.LongTensor(np.array(label)).to(self.device)return (input_ids, token_type_ids, mask), labelelse:return (input_ids, token_type_ids, mask)def __next__(self):if self.index < self.n_batches:batches = self.dataset[self.index]self.index += 1batches = convert_data(batches, self.tokenizer, self.max_seq_len, self.pred)batches = self._to_tensor(batches)return batcheselse:self.index = 0raise StopIterationdef __iter__(self):return selfdef __len__(self):return self.n_batchesdef bulid_iterator(tokenizer, dataset, batch_size, device, max_seq_len=512, pred=False):iter = DatasetIterator(tokenizer, dataset, batch_size, device, max_seq_len, pred)return iterclass PretrainedModel(nn.Module): # 预训练模型 + FCdef __init__(self, pretrained_model, dropout=None):super(PretrainedModel, self).__init__()self.ptm = pretrained_modelfor param in self.ptm.parameters():param.requires_grad = True  # 打开 finetune 开关self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)self.clf = nn.Linear(768, 2)def forward(self, x):out = self.ptm(input_ids=x[0], attention_mask=x[2], token_type_ids=x[1])cls_embedding = self.dropout(out['pooler_output'])logits = self.clf(cls_embedding)probs = F.softmax(logits, dim=1)return probsclass Recongnizer(): # 识别器类def __init__(self, datasetname, state_dict_path=None):  # 传入模型参数路径self.seed = torch.initial_seed()self.datasetname = datasetnameself.batch_size = 64self.epochs = 3self.max_seq_len = 512self.lr = 1e-5self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')self.train_ds, self.dev_ds, self.test_ds = load_dataset(self.datasetname)# 使用预训练模型的tokenizer, 见 https://huggingface.co/nghuyong/ernie-1.0self.tokenizer = AutoTokenizer.from_pretrained("../ERNIE_pretrain")self.pretrained_model = AutoModel.from_pretrained("../ERNIE_pretrain")self.model = PretrainedModel(self.pretrained_model).to(self.device)if state_dict_path:try:state_dict = torch.load(state_dict_path)self.model.load_state_dict(state_dict)except:print("加载模型参数失败！")self.pathname = "checkpoint_pt"self.global_step = 0isExists = os.path.exists(self.pathname)if not isExists:os.mkdir(self.pathname)self.save_param_path = os.path.join(self.pathname, "best_params_pt")def fit(self):# 加载数据集train_ds = bulid_iterator(self.tokenizer, self.train_ds, self.batch_size, self.device, self.max_seq_len)dev_ds = bulid_iterator(self.tokenizer, self.dev_ds, self.batch_size, self.device, self.max_seq_len)test_ds = bulid_iterator(self.tokenizer, self.test_ds, self.batch_size, self.device, self.max_seq_len, pred=True)# 展示数据for i, example in enumerate(train_ds):if i < 2:(input_ids, token_type_ids, mask), label = exampleprint(input_ids)# [1, 692, 811, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2, 329, 445, 2001, 497, 5, 654, 21, 692, 811, 614, 356, 314, 5, 291, 21, 2]print(token_type_ids)# [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]print(label)# [1]param_optimizer = list(self.model.named_parameters())no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']# 优化的参数optimizer_grouped_parameters = [{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_deacy': 0.0}]# 优化器optimizer = torch.optim.AdamW(params=optimizer_grouped_parameters, lr=self.lr)t_start = time.time()dev_best_loss = float('inf')self.model.train()for epoch in range(1, self.epochs + 1):for step, batch in enumerate(train_ds, start=1):x, labels = batchprobs = self.model(x)self.model.zero_grad()loss = F.cross_entropy(probs, labels)loss.backward(retain_graph=False)optimizer.step()self.global_step += 1if self.global_step % 10 == 0:true = labels.data.cpu()predit = torch.max(probs.data, 1)[1].cpu()train_acc = metrics.accuracy_score(true, predit)dev_acc, dev_loss = self.evaluate(dev_ds)if dev_loss < dev_best_loss: # 保存 dev 上 loss 最小的模型dev_best_loss = dev_losstorch.save(self.model.state_dict(), self.save_param_path)improve = '*'else:improve = ''time_dif = get_time_dif(t_start)msg = 'Iter:{0:>6}, Train Loss:{1:>5.2}, Train Acc:{2:>6.2}, Val Loss:{3:>5.2}, Val Acc:{4:>6.2%}, Time:{5} {6}'print(msg.format(self.global_step, loss.item(), train_acc, dev_loss, dev_acc, time_dif, improve))self.model.train()print("-----训练完成------")# 用最好的模型参数，测评一次state_dict = torch.load(self.save_param_path)self.model.load_state_dict(state_dict)self.model.eval()self.writeToFile(self.test_ds)def evaluate(self, data_loader, test=False): # 评估self.model.eval()loss_total = 0predict_all = np.array([], dtype=int)labels_all = np.array([], dtype=int)with torch.no_grad():for x, labels in data_loader:outputs = self.model(x)loss = F.cross_entropy(outputs, labels)loss_total = loss_total + losslabels = labels.data.cpu().numpy()predict = torch.max(outputs.data, 1)[1].cpu().numpy()labels_all = np.append(labels_all, labels)predict_all = np.append(predict_all, predict)acc = metrics.accuracy_score(labels_all, predict_all)if test:return acc, loss_total / len(data_loader), predict_allreturn acc, loss_total / len(data_loader)def predict(self, text1, text2):  # 单条句对预测d = {"text1": text1, "text2": text2}input_ids, token_type_ids, mask = convert_data(d, self.tokenizer, max_seq_len=self.max_seq_len, is_test=True)input_ids = torch.LongTensor(input_ids).to(self.device)token_type_ids = torch.LongTensor(token_type_ids).to(self.device)mask = torch.LongTensor(mask).to(self.device)predict_data_loader = [(input_ids, token_type_ids, mask)]batch_probs = []self.model.eval()with torch.no_grad():for x in predict_data_loader:batch_prob = self.model(x)batch_probs.append(batch_prob.cpu().numpy())batch_probs = np.concatenate(batch_probs, axis=0)return batch_probsdef writeToFile(self, test_ds): # 将预测结果写入文件with open(self.datasetname + ".tsv", "w", encoding="utf-8") as f:f.write("index\tprediction\n")for i, d in enumerate(test_ds):prob = self.predict(d["text1"], d["text2"])label = 1 if prob[0][1] >= 0.5 else 0f.write(str(i) + "\t" + str(label) + "\n")if __name__ == "__main__":datasetnames = ["lcqmc", "bq_corpus", "paws-x"]  #for name in datasetnames:model = Recongnizer(name)model.fit()

3. 提交结果