java Springboot使用扣子Coze实现实时音频对话智能客服

一、背景

因公司业务需求，需要使用智能客服实时接听顾客电话。

现在已经完成的操作是，智能体已接入系统进行对练，所以本文章不写对联相关的功能。只有coze对接～

扣子提供了试用Realtime WebSocket，点击右上角setting配置好智能体token之后就可以试用了

注意：只有扣子专业版支持实时音视频，所以需要开通专业版，开发测试阶段可以先充值1元买1000资源点对接测试，注意超额会单独收费哦，

二、准备工作

1、发布智能体为AI服务

a.登陆扣子平台注册账号

扣子扣子是新一代 AI 大模型智能体开发平台。整合了插件、长短期记忆、工作流、卡片等丰富能力，扣子能帮你低门槛、快速搭建个性化或具备商业价值的智能体，并发布到豆包、飞书等各个平台。https://www.coze.cn/home b. 在左侧导航栏中选择工作空间，并在页面顶部空间列表中选择个人空间或团队空间

c. 在项目开发页面，新建智能体

d.创建智能体完成之后，点击右上角的发布，在发布页面，选择API选项，然后点击发布

c.获取智能体ID,后续开发要用

点开工作空间->项目开发->你的智能体，点进新建的智能体，链接地址后的数字则为智能体ID

2、获取访问令牌

因公司业务需要经过对比我们选用了JWT方式，开发测试阶段也可以选择个人访问令牌

a.在扣子API页面，进入授权-> Oauth应用页面->创建新应用，注意客户端类型为服务端应用

b.保存后进行下一步授权，将自动生成的公钥复制保存好，自动下载的私钥也要存储好，后续接口认证会用到！

3、安装Java SDK，参考扣子官网

扣子扣子是新一代 AI 大模型智能体开发平台。整合了插件、长短期记忆、工作流、卡片等丰富能力，扣子能帮你低门槛、快速搭建个性化或具备商业价值的智能体，并发布到豆包、飞书等各个平台。https://www.coze.cn/open/docs/developer_guides/java_installation

三、实践开发

1、添加maven依赖

 <dependency><groupId>com.coze</groupId><artifactId>coze-api</artifactId><version>0.3.0</version>
</dependency><!-- 以下非必须！！！！我把私钥文件放到resources下了，所以打包需要加上这个类型->
<build><resources><resource><directory>src/main/resources</directory><includes><include>**/*.pem</include></includes><filtering>false</filtering></resource></resources>
</build>

2、获取token

@Slf4j
@Component
public class CozeOAuth {/*** JWT鉴权token*/public String getJWTToken() {String token = "";try {// 获取私钥文件String jwtOauthPrivateKeyFilePath = "这里是你的私钥文件地址";ClassLoader classLoader = this.getClass().getClassLoader();java.net.URL resourceUrl = classLoader.getResource(jwtOauthPrivateKeyFilePath);if (resourceUrl == null) {log.info("私钥资源文件未找到，{}", jwtOauthPrivateKeyFilePath);return token;}String jwtOauthPrivateKey = new String(Files.readAllBytes(Paths.get(resourceUrl.toURI())), StandardCharsets.UTF_8);JWTOAuthClient oauth = new JWTOAuthClient.JWTOAuthBuilder().clientID("这里是你之前创建的OAuth应用Id").privateKey(jwtOauthPrivateKey).publicKey("这里是你的公钥").baseURL(com.coze.openapi.service.config.Consts.COZE_CN_BASE_URL).build();// 获取tokenOAuthToken resp = oauth.getAccessToken();System.out.println(resp);if (Objects.nonNull(resp)) {token = resp.getAccessToken();}} catch (Exception e) {log.error("获取coze JWT token异常！", e);}log.info("获取coze JWT token：{}", token);return token;}}

3、创建新类继承WebsocketsChatCallbackHandler，接收扣子服务端返回消息并做业务处理

@Slf4j
public class MyWebsocketsChatCallbackHandler extends WebsocketsChatCallbackHandler {public void onChatCreated(WebsocketsChatClient client, ChatCreatedEvent event) {log.info("扣子服务端返回，对话连接成功,{}", JSON.toJSONString(event));}public void onChatUpdated(WebsocketsChatClient client, ChatUpdatedEvent event) {log.info("扣子服务端返回，对话配置成功,{}", JSON.toJSONString(event));}public void onConversationChatCreated(WebsocketsChatClient client, ConversationChatCreatedEvent event) {log.info("扣子服务端返回，对话开始,{}", JSON.toJSONString(event));}public void onConversationChatInProgress(WebsocketsChatClient client, ConversationChatInProgressEvent event) {log.info("扣子服务端返回，对话正在处理,{}", JSON.toJSONString(event));}public void onConversationMessageDelta(WebsocketsChatClient client, ConversationMessageDeltaEvent event) {log.info("扣子服务端返回，增量消息,{}", JSON.toJSONString(event));}public void onConversationAudioDelta(WebsocketsChatClient client, ConversationAudioDeltaEvent event) {log.info("扣子服务端返回，增量语音,{}", JSON.toJSONString(event));// TODO 处理实际业务，比如返回给用户的语音}public void onConversationMessageCompleted(WebsocketsChatClient client, ConversationMessageCompletedEvent event) {log.info("扣子服务端返回，消息完成,{}", JSON.toJSONString(event));}public void onConversationAudioCompleted(WebsocketsChatClient client, ConversationAudioCompletedEvent event) {log.info("扣子服务端返回，语音回复完成,{}", JSON.toJSONString(event));}public void onConversationChatCompleted(WebsocketsChatClient client, ConversationChatCompletedEvent event) {log.info("扣子服务端返回，对话完成,{}", JSON.toJSONString(event));}public void onConversationChatFailed(WebsocketsChatClient client, ConversationChatFailedEvent event) {log.info("扣子服务端返回，对话失败,{}", JSON.toJSONString(event));}public void onInputAudioBufferCompleted(WebsocketsChatClient client, InputAudioBufferCompletedEvent event) {log.info("扣子服务端返回，流式提交的音频完成,{}", JSON.toJSONString(event));}public void onInputAudioBufferCleared(WebsocketsChatClient client, InputAudioBufferClearedEvent event) {log.info("扣子服务端返回，清除缓冲区音频成功,{}", JSON.toJSONString(event));}public void onConversationCleared(WebsocketsChatClient client, ConversationClearedEvent event) {log.info("扣子服务端返回，上下文清除完成,{}", JSON.toJSONString(event));}public void onConversationChatCanceled(WebsocketsChatClient client, ConversationChatCanceledEvent event) {log.info("扣子服务端返回，智能体输出中断,{}", JSON.toJSONString(event));}public void onConversationAudioTranscriptUpdate(WebsocketsChatClient client, ConversationAudioTranscriptUpdateEvent event) {log.info("扣子服务端返回，用户语音识别字幕,{}", JSON.toJSONString(event));}public void onConversationAudioTranscriptCompleted(WebsocketsChatClient client, ConversationAudioTranscriptCompletedEvent event) {log.info("扣子服务端返回，用户语音识别完成,{}", JSON.toJSONString(event));}public void onConversationChatRequiresAction(WebsocketsChatClient client, ConversationChatRequiresActionEvent event) {log.info("扣子服务端返回，端插件请求,{}", JSON.toJSONString(event));}public void onInputAudioBufferSpeechStarted(WebsocketsChatClient client, InputAudioBufferSpeechStartedEvent event) {log.info("扣子服务端返回，用户开始说话,{}", JSON.toJSONString(event));}public void onInputAudioBufferSpeechStopped(WebsocketsChatClient client, InputAudioBufferSpeechStoppedEvent event) {log.info("扣子服务端返回，用户结束说话,{}", JSON.toJSONString(event));}public void onClosing(WebsocketsChatClient client, int code, String reason) {log.info("扣子服务端返回，onClosing,code：{}，reason：{}",code, reason);}public void onClosed(WebsocketsChatClient client, int code, String reason) {log.info("扣子服务端返回，onClosed,code:{},reason：{}", code,reason);}public void onError(WebsocketsChatClient client, ErrorEvent event) {log.info("扣子服务端返回，onError,event：{}", JSON.toJSONString(event));}public void onFailure(WebsocketsChatClient client, Throwable t) {log.info("扣子服务端返回，onFailure,event：{}", JSON.toJSONString(t));}public void onClientException(WebsocketsChatClient client, Throwable t) {log.info("扣子服务端返回，onFailure,event：{}", JSON.toJSONString(t.getMessage()));}
}

4、创建工具类

抽取跟业务无关的代码到该类中

@Slf4j
@Component
public class WebSocketUtils {@Resourceprivate CozeOAuth cozeOAuth;/*** 更新对话配置 请求参数*/public ChatUpdateEventData initChatUpdateEventData() {// 对话配置ChatConfig chatConfig = new ChatConfig();chatConfig.setAutoSaveHistory(true);// 输入音频格式InputAudio inputAudio = new InputAudio("pcm", "g711a", 8000, 1, 16);PCMConfig pcmConfig = new PCMConfig(100,8000);// 输出音频格式OutputAudio outputAudio = new OutputAudio("pcm", pcmConfig, null, null, null);// 转检测配置// server_vad 模式下，VAD 检测到语音之前要包含的音频量，单位为 ms。默认为 600ms。// server_vad 模式下，检测语音停止的静音持续时间，单位为 ms。默认为 500msTurnDetection turnDetection = new TurnDetection("server_vad", 300, 500);return cChatUpdateEventData.builder().inputAudio(inputAudio).outputAudio(outputAudio).chatConfig(chatConfig).turnDetection(turnDetection).build();}public CozeAPI getCozeApi(){return new CozeAPI.Builder().baseURL(Consts.COZE_CN_BASE_URL).auth(new TokenAuth(cozeOAuth.getJWTToken())).readTimeout(10000).build();}
}

5、使用websocket双向流式对话

我们用到了第三方的用户进线传输，直接sip协议拿包，将包传输给扣子，之后再将扣子的增量语音返回给第三方就行。所以选择了websocket的方式

        byte[] buffer = new byte[1500];CozeAPI cozeAPI = webSocketUtils.getCozeApi();WebsocketsChatClient websocketsChatClient = cozeAPI.websockets().chat().create(new WebsocketsChatCreateReq("这里是你的智能体ID", new MyWebsocketsChatCallbackHandler()));// 更新对话配置               websocketsChatClient.chatUpdate(webSocketUtils.initChatUpdateEventData());// 此处可以根据实际业务接收语音流byte[] audioData = Files.readAllBytes(Paths.get("/音频.pcm"));// 流式上传音频片段websocketsChatClient.inputAudioBufferAppend(audioData);