基于langchain和向量库微调qwen2-1.5B学习行业专属知识

现在有大量的word文档数据需要让大模型学习形成专业业务领域的知识后对外提供问答服务，以下是基于前问Qwen2-1.5B模型进行langchain RAG向量搜索+大模型问答的实现过程。

第一步：原始数据处理

原始word文档数据里面包含了封面、目录、概述、专业知识章节、表格、备注、附录等内容，还好没有图像，这些数据直接做分词向量化处理的时候因为上下文环境的问题会造成信息丢失，比如现在有一大段文字内容是关于如何制作面条的，关于制作面条的过程讲解的很细致，但是缺失了一个关键问题，就是这是关于中国面条的知识还是关于制作意大利面的知识呢？在做了向量化处理以后，如果用户的问题是如何制作意大利面，文字向量化处理的时候缺失了文章所属的专业领域分类信息就会造成数据检索失败或不准确，所以我这里是先将word文档内的内容按照段落转换成了excel格式，每一段数据一行，然后手工的对数据进行了一个概述补充，即补充了这些文字知识对应的是专业领域内的哪个领域的描述信息。

以下是word文档转为excel的处理程序：

import pandas as pd
from docx import Document# 解析Word文档并保存段落到Excel文件
def parse_word_and_save_to_excel(word_file_path, excel_file_path, prefix):doc = Document(word_file_path)paragraphs = [prefix + p.text.strip() for p in doc.paragraphs if p.text.strip()]df = pd.DataFrame(paragraphs, columns=["paragraph"])df.to_excel(excel_file_path, index=False, engine='openpyxl')print(f"段落数据已保存到 {excel_file_path}")if __name__ == "__main__":word_file_path = "/data/xxx/doc/新加坡.docx"  # 请填写您的Word文档路径excel_file_path = "/data/xxx/新加坡段落数据.xlsx"  # 输出的Excel文件路径prefix = "在新加坡,"  # 前缀字符串parse_word_and_save_to_excel(word_file_path, excel_file_path, prefix)

得到excel数据以后手动的对内容进行补充修改完善调整，涉及到表格数据的，涉及到列表内容的有可能需要完善修改一下，另外就是刚刚提到的，补充这段文字和业务领域的关联，删除没用的垃圾数据，如“目录”，“附录”等等没有用的数据。下一步就是产生问答对数据，这里采用的方式是去调用其他的更高级的模型来生成问答对数据内容，正好赶上智谱清言API注册就给一千万token，所以这里使用的是chatglm4的模型去生成问答对数据，代码如下：

import json
import requests
import pandas as pd
from tqdm import tqdm# 设置HTTP代理
proxies = {"http": "http://x.1.2.3:1234","https": "http://x.1.2.3:1234"
}# API key 和 URL
api_key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # 请填写您的API Key
api_url = "https://open.bigmodel.cn/api/paas/v4/chat/completions"# 调用智谱AI API生成问答对
def generate_qa_pairs(paragraph, model="glm-4"):headers = {"Authorization": f"Bearer {api_key}","Content-Type": "application/json"}prompt = f"请基于以下段落生成尽可能多的问答对，并以 [{json.dumps({'q': '', 'a': ''})}] 结构化JSON格式返回：{paragraph}"data = {"model": model,"messages": [{"role": "user", "content": prompt}],"stream": False}response = requests.post(api_url, headers=headers, json=data, proxies=proxies)response.raise_for_status()return response.json()["choices"][0]["message"]["content"]# 保存结果为JSON文件
def save_to_json(data, output_file):with open(output_file, 'w', encoding='utf-8') as f:json.dump(data, f, ensure_ascii=False, indent=4)def process_paragraphs_from_excel(excel_file_path, output_file, start_percent, end_percent):df = pd.read_excel(excel_file_path, engine='openpyxl')paragraphs = df['paragraph'].tolist()total_paragraphs = len(paragraphs)start_index = int(total_paragraphs * start_percent / 100)end_index = int(total_paragraphs * end_percent / 100)selected_paragraphs = paragraphs[start_index:end_index]qa_pairs = []for paragraph in tqdm(selected_paragraphs, desc=f"Processing paragraphs {start_percent}% to {end_percent}%"):try:#去除中文双引号及英文双引号cleaned_paragraph = paragraph.replace('“', '').replace('”', '').replace('"', '')qa_content = generate_qa_pairs(cleaned_paragraph)qa_pairs.append({"paragraph": paragraph,"qa_pairs": json.loads(qa_content)})except requests.exceptions.RequestException as e:print(f"Error processing paragraph: {paragraph}\nError: {e}\nserver response:{qa_content}")except json.JSONDecodeError as e:print(f"Error decoding JSON for paragraph: {paragraph}\nError: {e}\nserver response:{qa_content}")save_to_json(qa_pairs, output_file)print(f"问答对已保存到 {output_file}")if __name__ == "__main__":excel_file_path = "/data/xxxxx/新加坡段落数据.xlsx"  # 输入的Excel文件路径output_json_file = "/data/xxxx/新加坡_qa_pairs.json"  # 输出的JSON文件路径start_percent = 0  # 起始百分比end_percent = 100  # 结束百分比process_paragraphs_from_excel(excel_file_path, output_json_file, start_percent, end_percent)

经过上面的步骤，word文档里面的知识就变成了问答对及上下文json数据，大概格式如下：

  [{"paragraph": "这里是上下文文字数据信息","qa_pairs": [{"q": "示例问题1？","a": "示例问题答案"},{"q": "示例问题2？","a": "示例问题答案"}]}
]

第二步：langchain+向量检索喂给大模型

我这里是使用的Qwen2 1.5b模型，因为垃圾显卡只能跑这个模型了，另外是提前走向量搜索，大模型就做个总结也用不了多少算力。

先安装依赖

pip install flask flask-cors torch numpy transformers langchain langchain-community langchain-huggingface

下面是程序代码：

from flask import Flask, request, jsonify
from flask_cors import CORS
import json
import os
import re
import torch
import numpy as np
from transformers import AutoModelForCausalLM, AutoTokenizer
from abc import ABC
from langchain.llms.base import LLM
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain_community.vectorstores import FAISS
from langchain_community.document_loaders import TextLoader
from langchain_huggingface import HuggingFaceEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.docstore.document import Document
from langchain.prompts.prompt import PromptTemplate
from langchain.chains import RetrievalQA
from typing import Any, List, Mapping, Optional, Tuple# Flask应用初始化
app = Flask(__name__)
CORS(app)  # 允许跨域device = "cuda"  # 将模型加载到cuda设备上# 从指定路径加载模型和分词器
model_path = "/data/model/Qwen2-1.5B-Instruct/"
model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype="auto", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_path)# 定义Qwen模型类，继承自LangChain的LLM基类
class Qwen(LLM, ABC):max_token: int = 10000  # 最大token数temperature: float = 0.01  # 温度参数，用于控制生成文本的多样性top_p = 0.9  # top-p采样参数history_len: int = 3  # 对话历史长度def __init__(self):super().__init__()@propertydef _llm_type(self) -> str:return "Qwen"@propertydef _history_len(self) -> int:return self.history_lendef set_history_len(self, history_len: int = 10) -> None:self.history_len = history_lendef _call(self,prompt: str,stop: Optional[List[str]] = None,run_manager: Optional[CallbackManagerForLLMRun] = None,) -> str:# 构造对话消息messages = [{"role": "system", "content": "你是一个非常专业的人工智能助手."},{"role": "user", "content": prompt}]# 将消息应用模板并生成文本text = tokenizer.apply_chat_template(messages,tokenize=False,add_generation_prompt=True)# 将文本转换为模型输入model_inputs = tokenizer([text], return_tensors="pt").to(device)# 生成响应文本generated_ids = model.generate(model_inputs.input_ids,max_new_tokens=512)# 获取生成的文本generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]return response@propertydef _identifying_params(self) -> Mapping[str, Any]:"""获取识别参数"""return {"max_token": self.max_token,"temperature": self.temperature,"top_p": self.top_p,"history_len": self.history_len}# 中文文本拆分器类，继承自CharacterTextSplitter
class ChineseTextSplitter(CharacterTextSplitter):def __init__(self, pdf: bool = False, **kwargs):super().__init__(**kwargs)self.pdf = pdfdef split_text(self, text: str) -> List[str]:if self.pdf:# 处理PDF文本格式text = re.sub(r"\n{3,}", "\n", text)text = re.sub('\s', ' ', text)text = text.replace("\n\n", "")# 定义句子分隔模式sent_sep_pattern = re.compile('([﹒﹔﹖﹗．。！？]["’”」』]{0,2}|(?=["‘“「『]{1,2}|$))')sent_list = []for ele in sent_sep_pattern.split(text):if sent_sep_pattern.match(ele) and sent_list:sent_list[-1] += eleelif ele:sent_list.append(ele)return sent_list# 加载文件并进行拆分
def load_file(filepath):loader = TextLoader(filepath, autodetect_encoding=True)textsplitter = ChineseTextSplitter(pdf=False)docs = loader.load_and_split(textsplitter)write_check_file(filepath, docs)return docs# 将拆分后的文档写入检查文件
def write_check_file(filepath, docs):folder_path = os.path.join(os.path.dirname(filepath), "tmp_files")if not os.path.exists(folder_path):os.makedirs(folder_path)fp = os.path.join(folder_path, 'load_file.txt')with open(fp, 'a+', encoding='utf-8') as fout:fout.write("filepath=%s,len=%s" % (filepath, len(docs)))fout.write('\n')for i in docs:fout.write(str(i))fout.write('\n')fout.close()# 将连续的列表分离成多个子列表
def separate_list(ls: List[int]) -> List[List[int]]:lists = []ls1 = [ls[0]]for i in range(1, len(ls)):if ls[i - 1] + 1 == ls[i]:ls1.append(ls[i])else:lists.append(ls1)ls1 = [ls[i]]lists.append(ls1)return lists# FAISS向量搜索包装类
class FAISSWrapper(FAISS):chunk_size = 250chunk_conent = Truescore_threshold = 0def similarity_search_with_score_by_vector(self, embedding: List[float], k: int = 4, filter=None,fetch_k: Optional[int] = None) -> List[Tuple[Document, float]]:if filter:embedding = [e for e, f in zip(embedding, filter) if f]if fetch_k is not None:k = fetch_kscores, indices = self.index.search(np.array([embedding], dtype=np.float32), k)docs = []id_set = set()store_len = len(self.index_to_docstore_id)for j, i in enumerate(indices[0]):if i == -1 or 0 < self.score_threshold < scores[0][j]:# 当返回的文档数量不足时会发生这种情况continue_id = self.index_to_docstore_id[i]doc = self.docstore.search(_id)if not self.chunk_conent:if not isinstance(doc, Document):raise ValueError(f"Could not find document for id {_id}, got {doc}")doc.metadata["score"] = int(scores[0][j])docs.append(doc)continueid_set.add(i)docs_len = len(doc.page_content)for k in range(1, max(i, store_len - i)):break_flag = Falsefor l in [i + k, i - k]:if 0 <= l < len(self.index_to_docstore_id):_id0 = self.index_to_docstore_id[l]doc0 = self.docstore.search(_id0)if docs_len + len(doc0.page_content) > self.chunk_size:break_flag = Truebreakelif doc0.metadata["source"] == doc.metadata["source"]:docs_len += len(doc0.page_content)id_set.add(l)if break_flag:breakif not self.chunk_conent:return docsif len(id_set) == 0 and self.score_threshold > 0:return []id_list = sorted(list(id_set))id_lists = separate_list(id_list)for id_seq in id_lists:for id in id_seq:if id == id_seq[0]:_id = self.index_to_docstore_id[id]doc = self.docstore.search(_id)else:_id0 = self.index_to_docstore_id[id]doc0 = self.docstore.search(_id0)doc.page_content += " " + doc0.page_contentif not isinstance(doc, Document):raise ValueError(f"Could not find document for id {_id}, got {doc}")doc_score = min([scores[0][id] for id in [indices[0].tolist().index(i) for i in id_seq if i in indices[0]]])doc.metadata["score"] = int(doc_score)docs.append((doc, doc_score))#print(docs)  #在这里可以看向量搜索结果,然后大模型根据这个结果去总结答案return docs# 加载QA数据
def load_qa_data(filepath):with open(filepath, 'r', encoding='utf-8') as f:data = json.load(f)return data# 获取文档的上下文信息
def get_context(document):return document.metadata.get("context_str", "No relevant information available.")@app.route('/query', methods=['POST'])
def handle_query():data = request.jsonquery = data.get("query")if not query:return jsonify({"error": "No query provided"}), 400# 使用检索到的上下文构造请求并调用模型response = qa.invoke({"query": query})return jsonify({"response": response})if __name__ == '__main__':# 加载QA数据qa_data_path = '/data/xxxxx/xxxx_qa_pairs.json'qa_data = load_qa_data(qa_data_path)# 处理每个QA对docs = []for item in qa_data:paragraph = item["paragraph"]for qa_pair in item["qa_pairs"]:question = qa_pair["q"]expected_answer = qa_pair["a"]docs.append(Document(page_content=paragraph, metadata={"source": paragraph,"context_str": paragraph,"question": question, "expected_answer": expected_answer}))# 嵌入模型名称EMBEDDING_MODEL = 'text2vec'PROMPT_TEMPLATE = """已知信息:{context_str} 根据上面的已知信息, respond to the user's question concisely and professionally. If an answer cannot be derived from it, say 'The question cannot be answered with the given information' or 'Not enough relevant information has been provided,' and do not include fabricated details in the answer. 问题是 {question}"""# 嵌入运行设备EMBEDDING_DEVICE = "cuda"# 从向量存储返回top-k文本块VECTOR_SEARCH_TOP_K = 5 #向量搜索结果太多的话可以改小点CHAIN_TYPE = 'stuff'embedding_model_dict = {"text2vec": "/data/model/text2vec-base-chinese/",}llm = Qwen()embeddings = HuggingFaceEmbeddings(model_name=embedding_model_dict[EMBEDDING_MODEL], model_kwargs={'device': EMBEDDING_DEVICE})# 初始化FAISS向量存储docsearch = FAISSWrapper.from_documents(docs, embeddings)prompt = PromptTemplate(template=PROMPT_TEMPLATE, input_variables=["context_str", "question"])chain_type_kwargs = {"prompt": prompt, "document_variable_name": "context_str"}qa = RetrievalQA.from_chain_type(llm=llm,chain_type=CHAIN_TYPE, retriever=docsearch.as_retriever(search_kwargs={"k": VECTOR_SEARCH_TOP_K}), chain_type_kwargs=chain_type_kwargs)app.run(host='0.0.0.0', port=5000)  # 启动Flask服务

上面的代码用到了text2vec,下载地址：

https://huggingface.co/shibing624/text2vec-base-chinese

千问大模型下载地址：

https://huggingface.co/Qwen/Qwen2-1.5B-Instruct

千问模型帮助文档资料：

https://qwen.readthedocs.io/en/latest/framework/Langchain.html

上面的代码的原始版本来源是（没法直接运行）：

https://qwen.readthedocs.io/en/latest/_sources/framework/Langchain.rst.txt

运行程序以后可以通过postman调用，调用的测试数据样式可以是下面这样：