实现了电脑窗口切换，截图识别未读消息，与语言模型交互后，将答案带入到钉钉窗口中。

偷个懒，直接贴代码了，后续不断完善注释，如果遇到读不懂的地方，欢迎交流。

# -*- coding: UTF-8 -*-
import time
# 必备的注释文件
import pygetwindow
from PIL import ImageGrab,Image
import time
import cv2
import numpy as np
import pyautogui
import easyocr
import  os
import pytesseract
import zhipuai
from zhipuai import ZhipuAI
import pyperclip
# from win10toast import ToastNotifier
import tkinter as tk
# def toastmsg(msg):
#
#     toaster = ToastNotifier()
#     toaster.show_toast("钉钉回复工具", msg, duration=10)
# 打开对话框
def openchat(xm,ym):# # 显示结果# cv2.imshow('Detected Red Points', image)# cv2.waitKey(0)# cv2.destroyAllWindows()# 要点击屏幕上的那个点# 移动鼠标到图标位置pyautogui.moveTo(xm, ym, duration=1)time.sleep(2)# 点击图标pyautogui.click(xm, ym)
# 识别对话框中的文字
def watchtext(imgurl):print('识别图片')# 读取图片image = cv2.imread(imgurl)# 图片预处理，例如灰度化、二值化等gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]# 使用pytesseract识别文字pytesseract.pytesseract.tesseract_cmd = r'D:\Program Files\Tesseract-OCR\tesseract.exe'text = pytesseract.image_to_string(thresh, lang='chi_sim')print(text)# 另外一个工具# 设为中英文混合识别：ch_sim enreader = easyocr.Reader(['ch_sim', 'en'], gpu=False)# 识别图片## print(str(os.path) + '/' + imgurl)# result = reader.readtext(imgurl, detail=0)# for i in result:#     # 输出识别出的信息#     # 输出识别出的信息#     # print('输出识别出的信息')#     print(i, end='')#     做一下图片的裁剪再识别：ch_sim enimg = Image.open(imgurl)# 获取图片大小img_size = img.size# h = img_size[1] #图片高度# w = img_size[0] #图片宽度# 设置截取部分相对位置x = 0.20 * img_size[0]+200y = 0.1 * img_size[1]# y = 350w = 1 * img_size[0]-400h = 1* img_size[1]-720# 截取图片cropped = img.crop((x, y, x + w, y + h))  # (x1,y1,x2,y2)# 保存截图图片，命名为test.pngcropped.save('test01.png')# 设为中英文混合识别：ch_sim enreader = easyocr.Reader(['ch_sim', 'en'], gpu=False, verbose=False)# 路径改为用户需要识别的图片的路径result = reader.readtext('test01.png', detail=0)for i in result:# 输出识别出的信息# 输出识别出的信息# print('输出识别出的信息')print(i, end='')return result
# 截图保存
def getmscreen():windowsjiantou  = pygetwindow.getWindowsWithTitle('XXXX')windowsjiantou[0].show()w = windowsjiantou[0]w.activate()# 获取桌面窗口的坐标和尺寸left, top, width, height = w.left, w.top, w.width, w.heightw.activate()w.show()# 将窗口最大化w.maximize()# 下面的单位是5秒time.sleep(0.5)print('运行到了这里')# 使用ImageGrab.grab()方法截取桌面screenshot = ImageGrab.grab(bbox=(left, top, left + width, top + height))# 获取当前时间的时间戳timestamp = time.time()print("当前时间戳：", timestamp)imgurl =  str(timestamp)+'desktop_screenshot.png'# 保存截图# screenshot.save(imgurl)img = pyautogui.screenshot()img.save(str(timestamp)+'desktop_screenshot.png')return imgurl
def getchat(questiontext):print(questiontext)# 接入质谱AI的APIclient = ZhipuAI(api_key=" . ")  # 请填写您自己的APIKeyresponse = client.chat.completions.create(model="glm-4",  # 填写需要调用的模型名称  OA表单中选不到项目的添加方法messages=[{"role": "user", "content": questiontext},],tools=[{"type": "retrieval","retrieval": {"knowledge_id": " ","prompt_template": "从文档\n\"\"\"\n{{knowledge}}\n\"\"\"\n中找问题\n\"\"\"\n{{question}}\n\"\"\"\n的答案，找到答案就仅使用文档语句回答问题，找不到答案就用自身知识回答并且告诉用户该信息不是来自文档。\n不要复述问题，直接开始回答。"}}],stream=True,)resstr = ""for chunk in response:# print(chunk.choices[0].delta)resstr = resstr + str(chunk.choices[0].delta.content)# print(chunk.choices[0].delta.content)print(resstr)# 做一个data，把数据返回去return resstr
def pasttext(text):windowsjiantou  = pygetwindow.getWindowsWithTitle('XXXX')windowsjiantou[0].show()w = windowsjiantou[0]w.activate()# 移动鼠标到目标位置（这里以屏幕坐标为例）pyautogui.moveTo(600, 900)# 模拟鼠标点击pyautogui.click()# 模拟键盘输入# pyautogui.typewrite('你好www', interval=0.2)# # 模拟按下Win键# pyautogui.press("win")# # 输入中文输入法的名称，例如“微软拼音输入法”# pyautogui.typewrite("微软拼音输入法")# # 模拟按下回车键# pyautogui.press("enter")# # 等待中文输入法启动# pyautogui.sleep(1)# # 输入中文字符# pyautogui.typewrite("你好，世界！")pyperclip.copy(text)time.sleep(0.5)pyautogui.hotkey('ctrl', 'v')# pyperclip.paste()
def capture():# toastmsg('程序运行中')# 获取桌面窗口# desktop_window = pygetwindow.getDesktopWindow()desktop_window = pygetwindow.getAllWindows()desktop_window_title = pygetwindow.getAllTitles()for window in desktop_window_title:print(window)windowsjiantou  = pygetwindow.getWindowsWithTitle('XXXX')windowsjiantou[0].show()w = windowsjiantou[0]w.activate()# 获取桌面窗口的坐标和尺寸left, top, width, height = w.left, w.top, w.width, w.heightw.activate()w.show()# 将窗口最大化w.maximize()# 下面的单位是5秒time.sleep(0.5)print('运行到了这里')# 使用ImageGrab.grab()方法截取桌面screenshot = ImageGrab.grab(bbox=(left, top, left + width, top + height))# 获取当前时间的时间戳timestamp = time.time()print("当前时间戳：", timestamp)# 保存截图screenshot.save(str(timestamp)+'desktop_screenshot.png')# 读取图片上的红点# 识别图片imgs =str(timestamp)+'desktop_screenshot.png'# 读取图像image = cv2.imread(imgs)# 读取图像# 将图像从BGR转换为HSV颜色空间hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)# 定义红色的HSV范围lower_red1 = np.array([0, 120, 70])upper_red1 = np.array([10, 255, 255])lower_red2 = np.array([170, 120, 70])upper_red2 = np.array([180, 255, 255])# 创建掩码mask1 = cv2.inRange(hsv, lower_red1, upper_red1)mask2 = cv2.inRange(hsv, lower_red2, upper_red2)mask = cv2.bitwise_or(mask1, mask2)# 形态学操作以去除噪声kernel = np.ones((5, 5), np.uint8)mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)mask = cv2.dilate(mask, kernel, iterations=1)# 寻找轮廓  这里满足要求的轮廓已经放到这里数组里了contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)myusecolours = []# 绘制轮廓for contour in contours:# 计算轮廓的面积# 先留下面积大于100的轮廓area = cv2.contourArea(contour)if area > 50:  # 可以根据实际情况调整这个阈值print('面积大于50')# 计算轮廓的周长perimeter = cv2.arcLength(contour, True)# 计算轮廓的近似形状# approxPolyDP 函数用于计算轮廓的近似形状# approxPolyDPapprox = cv2.approxPolyDP(contour, 0.04 * perimeter, True)# 如果轮廓是圆形，那么近似形状的顶点数量应该接近于0# 但是这里我直接用半径来判断if len(approx) < 10:(x, y), radius = cv2.minEnclosingCircle(contour)center = (int(x), int(y))radius = int(radius)if radius > 5:  # 可以根据实际情况调整这个阈值  圆角值改小了一点# 使用cv2.circle() 在原图上绘制筛选后的圆形轮廓。print('绘制了一个图形print')cv2.circle(image, center, radius, (0, 255, 0), 2)# 这里是通过考验的contour# 获取contour 的坐标print(contour)myusecolours.append(contour)# 显示结果# cv2.imshow('Contours', image)# cv2.waitKey(0)# cv2.destroyAllWindows()print('----')myusecolours02 =myusecoloursmyusecolours02.reverse()print(len(myusecolours02))print(len(myusecolours02))if len(myusecolours02) == 0:returncontoursmsg = myusecolours02[-1]# if len(myusecolours02) < 3:#     contoursmsg = myusecolours02[2]### # 获取第一条未读消息# if len(myusecolours02) < 2:#     contoursmsg = myusecolours02[1]## if len(myusecolours02) < 1:#     contoursmsg = myusecolours02[0]# 获取坐标x, y, w, h = cv2.boundingRect(contoursmsg)# 打印边界框坐标print(f"Bounding box coordinates: x={x}, y={y}, w={w}, h={h}")# 得到中心点的位置(xm, ym), radius = cv2.minEnclosingCircle(contoursmsg)print(f"Bounding box coordinates: ----------------------------  x={xm}, y={ym}")# 打开对话框openchat(xm,ym)# 截图imgurl = getmscreen()# 识别对话框中的文字textcontent = watchtext(imgurl)# print(textcontent)textcontent02 = ''for item in textcontent:print(item+'\n')textcontent02= textcontent02+item+''# 获取最后一条消息# textcontent.reverse()# lasttext = textcontent[0]# print('最新的一条消息')# print(lasttext)# 调用API开始聊天--最后一条消息textcontent.reverse()textcontent01 = textcontent[0]answer = getchat(textcontent01)# 调用API开始聊天--所有识别的内容# answer = getchat(textcontent02)# 将内容粘贴到钉钉窗口中pasttext(answer)# toastmsg('程序运完毕')# print(desktop_window)# print(desktop_window_title)# # 获取桌面窗口的坐标和尺寸# left, top, width, height = desktop_window.left, desktop_window.top, desktop_window.width, desktop_window.height## # 使用ImageGrab.grab()方法截取桌面# screenshot = ImageGrab.grab(bbox=(left, top, left + width, top + height))## # 保存截图# screenshot.save('desktop_screenshot.png')
def say_hello():capture()
if __name__ == '__main__':# 先来屏幕截图capture()# root = tk.Tk()# root.geometry("400x500")# # 禁止用户调整窗口大小# root.resizable(False, False)## label = tk.Label(root, text=" ", font=("Microsoft YaHei", 16))# label.pack(pady=20)### label = tk.Label(root, text="点击 接管电脑 后，程序会识别未读消息并到知识库中进行检索填充回复。对信息修改勾，可以进行发送，或者设置自动发送",wraplength=300, font=("Microsoft YaHei", 16))# label.pack(pady=20)#### button = tk.Button(root, text="接管电脑", command=say_hello)# button.pack(pady=20)## root.mainloop()