LLamaSharp加载llama.cpp转化好的模型

新建.net8控制台项目

在这里插入图片描述

安装依赖包

LLamaSharp和LLamaSharp.Backend.Cpu
在这里插入图片描述

准备好转化好的模型

没有的话参考这篇文章https://blog.csdn.net/qq_36437991/article/details/137248622
在这里插入图片描述

编写代码

using LLama;
using LLama.Common;
using LLama.Native;namespace llamasharpstu01;class Program
{static async Task Main(string[] args){//模型量化// string srcPath = "<model.bin>";// string dstPath = "<model_q4_0.bin>";// // LLamaQuantizer.Quantize(srcPath, dstPath, "q4_0");// LLamaQuantizer.Quantize(srcPath, dstPath, LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0);//将其更改为您自己的模型路径。string modelPath = @"G:\Cpp\qwenchattest10.5b-q5_k_m.gguf";//modelPath = @"G:\Cpp\qwenctest10.5b.gguf";modelPath = @"G:\Cpp\qwenchat0.5b-q4_0.gguf";var parameters = new ModelParams(modelPath){//最长的聊天时间作为记忆。ContextSize = 1024,//要卸载到GPU的层数。请根据您的GPU内存进行调整。GpuLayerCount = 5,Encoding = Encoding.UTF8};using var model = LLamaWeights.LoadFromFile(parameters);using var context = model.CreateContext(parameters);var executor = new InteractiveExecutor(context);//添加聊天记录作为提示，告诉AI如何行动。var chatHistory = new ChatHistory();// chatHistory.AddMessage(AuthorRole.System, "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.");// chatHistory.AddMessage(AuthorRole.User, "Hello, Bob.");// chatHistory.AddMessage(AuthorRole.Assistant, "Hello. How may I help you today?");ChatSession session = new(executor, chatHistory);InferenceParams inferenceParams = new InferenceParams(){//答案中出现的token不应超过256个。-1表示无限制MaxTokens = 256,AntiPrompts = new List<string> { "User:" }};Console.ForegroundColor = ConsoleColor.Yellow;Console.Write("The chat session has started.\nUser: ");Console.ForegroundColor = ConsoleColor.Green;string userInput = Console.ReadLine() ?? "";while (userInput != "exit"){await foreach (var text in session.ChatAsync(new ChatHistory.Message(AuthorRole.User, userInput), inferenceParams)){Console.ForegroundColor = ConsoleColor.White;Console.Write(text);}Console.WriteLine("");Console.ForegroundColor = ConsoleColor.Green;userInput = Console.ReadLine() ?? "";}}
}