影刀RPA鼠标键盘精准控制：解决复杂桌面应用操作难题
作者：林焱 | 发布平台：CSDN / 公众号 / 掘金
前言：当网页自动化遇到瓶颈
大多数RPA教程聚焦于网页自动化，但真实企业环境中存在大量桌面应用：
- 老旧的Windows ERP客户端
- 财务软件（用友、金蝶）
- CAD/设计软件批处理
- 医疗影像系统
这些系统没有Web界面，无法用CSS选择器定位，只能通过鼠标键盘模拟来实现自动化。本文深入讲解影刀RPA的桌面应用操作技术。
基础工具：pyautogui + win32api
[video(video-7WpPfjeG-1782319941974)(type-csdn)(url-live.csdn.net/v/embed/526…)]
[video(video-Ej9BjuKE-1782319948825)(type-csdn)(url-live.csdn.net/v/embed/526…)]
import shadowbot as sb
import pyautogui
import win32gui
import win32con
import win32api
import ctypes
import time
# 关键设置
pyautogui.PAUSE = 0.1 # 每次操作后暂停0.1秒
pyautogui.FAILSAFE = True # 鼠标移到左上角紧急停止
pyautogui.MINIMUM_DURATION = 0.1 # 鼠标移动最短时间
def safe_click(x, y, button='left', clicks=1, interval=0.1):
"""安全点击（含验证）"""
# 移动到目标位置
pyautogui.moveTo(x, y, duration=0.3)
time.sleep(0.1)
# 执行点击
pyautogui.click(x, y, button=button, clicks=clicks, interval=interval)
time.sleep(0.1)
sb.log.debug(f"点击操作：({x}, {y}), {button}键, {clicks}次")
技巧一：窗口精准定位
通过窗口标题找到目标程序
class WindowManager:
"""窗口管理工具"""
@staticmethod
def find_window(title_pattern, exact=False):
"""按标题查找窗口"""
import re
results = []
def enum_callback(hwnd, results):
if win32gui.IsWindowVisible(hwnd):
title = win32gui.GetWindowText(hwnd)
if exact:
if title == title_pattern:
results.append(hwnd)
else:
if title_pattern.lower() in title.lower():
results.append(hwnd)
win32gui.EnumWindows(enum_callback, results)
return results
@staticmethod
def activate_window(hwnd):
"""激活并置前窗口"""
# 如果窗口最小化，先恢复
if win32gui.IsIconic(hwnd):
win32gui.ShowWindow(hwnd, win32con.SW_RESTORE)
# 置于前台
win32gui.SetForegroundWindow(hwnd)
time.sleep(0.3)
return True
@staticmethod
def get_window_rect(hwnd):
"""获取窗口位置和大小"""
rect = win32gui.GetWindowRect(hwnd)
return {
"left": rect[0],
"top": rect[1],
"right": rect[2],
"bottom": rect[3],
"width": rect[2] - rect[0],
"height": rect[3] - rect[1],
}
@staticmethod
def resize_window(hwnd, width, height, x=None, y=None):
"""调整窗口大小"""
rect = win32gui.GetWindowRect(hwnd)
new_x = x if x is not None else rect[0]
new_y = y if y is not None else rect[1]
win32gui.MoveWindow(hwnd, new_x, new_y, width, height, True)
time.sleep(0.2)
wm = WindowManager()
# 找到用友财务软件
ufida_windows = wm.find_window("用友")
if ufida_windows:
hwnd = ufida_windows[0]
wm.activate_window(hwnd)
win_rect = wm.get_window_rect(hwnd)
sb.log.info(f"找到用友窗口：{win_rect}")
技巧二：基于坐标的相对定位
固定坐标会随屏幕分辨率变化而失效，应使用相对坐标：
class RelativeCoordinator:
"""基于窗口的相对坐标系统"""
def __init__(self, hwnd):
self.hwnd = hwnd
self._update_rect()
def _update_rect(self):
rect = win32gui.GetWindowRect(self.hwnd)
self.left = rect[0]
self.top = rect[1]
self.width = rect[2] - rect[0]
self.height = rect[3] - rect[1]
def abs_pos(self, rel_x, rel_y):
"""相对坐标转绝对坐标"""
self._update_rect() # 每次都刷新，防止窗口移动
if 0 <= rel_x <= 1: # 比例形式
abs_x = self.left + int(self.width * rel_x)
else: # 像素偏移形式
abs_x = self.left + rel_x
if 0 <= rel_y <= 1:
abs_y = self.top + int(self.height * rel_y)
else:
abs_y = self.top + rel_y
return abs_x, abs_y
def click(self, rel_x, rel_y, **kwargs):
"""在相对位置点击"""
abs_x, abs_y = self.abs_pos(rel_x, rel_y)
safe_click(abs_x, abs_y, **kwargs)
def input_at(self, rel_x, rel_y, text):
"""在相对位置输入文字"""
self.click(rel_x, rel_y)
time.sleep(0.2)
# 清除原有内容
pyautogui.hotkey('ctrl', 'a')
time.sleep(0.1)
# 使用剪贴板输入（避免中文输入法问题）
import pyperclip
pyperclip.copy(text)
pyautogui.hotkey('ctrl', 'v')
time.sleep(0.1)
# 使用示例：操作用友财务
hwnd_list = wm.find_window("用友T3")
if hwnd_list:
coord = RelativeCoordinator(hwnd_list[0])
# 点击"总账"菜单（位于窗口宽度20%，高度5%处）
coord.click(0.20, 0.05)
time.sleep(0.5)
# 在凭证编号输入框输入
coord.input_at(0.30, 0.35, "2024-001")
技巧三：图像识别定位（不依赖坐标）
当界面布局可能变化时，用截图模板匹配来定位元素：
import cv2
import numpy as np
from PIL import ImageGrab
class ImageBasedLocator:
"""基于图像识别的元素定位"""
def __init__(self, screenshot_region=None):
self.region = screenshot_region # (left, top, right, bottom)
def take_screenshot(self):
"""截取屏幕"""
if self.region:
screenshot = ImageGrab.grab(bbox=self.region)
else:
screenshot = ImageGrab.grab()
return np.array(screenshot)
def find_element(self, template_path, confidence=0.85):
"""在屏幕上找到模板图像的位置"""
screenshot = self.take_screenshot()
template = cv2.imread(template_path)
if template is None:
raise FileNotFoundError(f"模板文件不存在：{template_path}")
# 转换颜色空间
screenshot_gray = cv2.cvtColor(screenshot, cv2.COLOR_RGB2GRAY)
template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
# 模板匹配
result = cv2.matchTemplate(screenshot_gray, template_gray, cv2.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
if max_val >= confidence:
# 找到元素，返回中心坐标
h, w = template.shape[:2]
center_x = max_loc[0] + w // 2
center_y = max_loc[1] + h // 2
# 如果指定了区域，需要加上偏移
if self.region:
center_x += self.region[0]
center_y += self.region[1]
return center_x, center_y, max_val
else:
return None, None, max_val
def find_all_elements(self, template_path, confidence=0.85):
"""找到所有匹配的元素"""
screenshot = self.take_screenshot()
template = cv2.imread(template_path)
screenshot_gray = cv2.cvtColor(screenshot, cv2.COLOR_RGB2GRAY)
template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
result = cv2.matchTemplate(screenshot_gray, template_gray, cv2.TM_CCOEFF_NORMED)
h, w = template.shape[:2]
locations = np.where(result >= confidence)
points = []
for pt in zip(*locations[::-1]):
center_x = pt[0] + w // 2
center_y = pt[1] + h // 2
if self.region:
center_x += self.region[0]
center_y += self.region[1]
points.append((center_x, center_y))
# 去重（相近坐标合并）
unique_points = []
for pt in points:
if not any(abs(pt[0] - u[0]) < 20 and abs(pt[1] - u[1]) < 20 for u in unique_points):
unique_points.append(pt)
return unique_points
def click_element(self, template_path, confidence=0.85, max_wait=10):
"""找到并点击元素"""
start_time = time.time()
while time.time() - start_time < max_wait:
x, y, score = self.find_element(template_path, confidence)
if x is not None:
safe_click(x, y)
sb.log.info(f"点击元素成功（置信度：{score:.2f}）：{template_path}")
return True
time.sleep(0.5)
sb.log.warning(f"超时未找到元素：{template_path}")
return False
# 使用示例
locator = ImageBasedLocator()
# 点击"保存"按钮（不依赖坐标）
locator.click_element("templates/save_button.png", confidence=0.9)
# 找到所有"删除"按钮
delete_buttons = locator.find_all_elements("templates/delete_icon.png")
sb.log.info(f"找到 {len(delete_buttons)} 个删除按钮")
技巧四：键盘快捷键自动化
常用软件快捷键封装
class KeyboardAutomation:
"""键盘操作封装"""
@staticmethod
def select_all():
pyautogui.hotkey('ctrl', 'a')
time.sleep(0.1)
@staticmethod
def copy():
pyautogui.hotkey('ctrl', 'c')
time.sleep(0.1)
@staticmethod
def paste():
pyautogui.hotkey('ctrl', 'v')
time.sleep(0.1)
@staticmethod
def undo():
pyautogui.hotkey('ctrl', 'z')
time.sleep(0.1)
@staticmethod
def save():
pyautogui.hotkey('ctrl', 's')
time.sleep(0.5)
@staticmethod
def tab_next(times=1):
for _ in range(times):
pyautogui.press('tab')
time.sleep(0.1)
@staticmethod
def navigate_table(direction):
"""在表格中导航"""
key_map = {
'right': 'tab',
'left': ('shift', 'tab'),
'down': 'down',
'up': 'up',
'next_row': 'return',
}
key = key_map.get(direction, 'tab')
if isinstance(key, tuple):
pyautogui.hotkey(*key)
else:
pyautogui.press(key)
time.sleep(0.1)
@staticmethod
def type_with_clipboard(text):
"""通过剪贴板输入文字（支持中文）"""
import pyperclip
pyperclip.copy(str(text))
pyautogui.hotkey('ctrl', 'v')
time.sleep(0.1)
@staticmethod
def press_function_key(n):
"""按功能键 F1-F12"""
pyautogui.press(f'f{n}')
time.sleep(0.2)
kbd = KeyboardAutomation()
def fill_form_with_keyboard(form_data):
"""用键盘Tab键逐字段填写表单"""
fields = list(form_data.values())
for i, value in enumerate(fields):
# 输入当前字段
kbd.type_with_clipboard(str(value))
time.sleep(0.1)
# Tab到下一个字段
if i < len(fields) - 1:
kbd.tab_next()
# 最后按Enter提交
pyautogui.press('enter')
time.sleep(0.5)
实战：自动操作用友财务软件录入凭证
def input_accounting_voucher(voucher_data):
"""在用友T3中录入记账凭证"""
# 查找并激活用友窗口
hwnd_list = wm.find_window("用友T3")
if not hwnd_list:
sb.log.error("用友T3未启动")
return False
hwnd = hwnd_list[0]
wm.activate_window(hwnd)
coord = RelativeCoordinator(hwnd)
locator = ImageBasedLocator(region=tuple(wm.get_window_rect(hwnd).values())[:4])
# 1. 点击"总账" → "凭证" → "填制凭证"
coord.click(0.05, 0.035) # 总账菜单
time.sleep(0.5)
locator.click_element("templates/menu_voucher.png")
time.sleep(0.3)
locator.click_element("templates/menu_fill_voucher.png")
time.sleep(1.0)
# 2. 填写凭证头
coord.input_at(0.12, 0.12, voucher_data["日期"])
kbd.tab_next()
kbd.type_with_clipboard(voucher_data["摘要"])
kbd.tab_next()
# 3. 填写凭证明细行
for entry in voucher_data["明细"]:
kbd.type_with_clipboard(entry["科目代码"])
kbd.tab_next()
if entry.get("借方金额"):
kbd.type_with_clipboard(str(entry["借方金额"]))
kbd.tab_next()
if entry.get("贷方金额"):
kbd.type_with_clipboard(str(entry["贷方金额"]))
kbd.tab_next(2)
# 4. 保存凭证
kbd.save()
# 5. 验证保存成功
time.sleep(1.0)
x, y, score = locator.find_element("templates/save_success.png")
if x is not None:
sb.log.info(f"凭证保存成功：{voucher_data['摘要']}")
# 截图留档
screenshot = locator.take_screenshot()
cv2.imwrite(f"voucher_{datetime.now().strftime('%Y%m%d%H%M%S')}.png",
cv2.cvtColor(screenshot, cv2.COLOR_RGB2BGR))
return True
else:
sb.log.error("凭证保存状态未知，需人工确认")
return False
# 批量录入凭证
vouchers = sb.excel.read_all("待录入凭证.xlsx")
for v in vouchers:
result = input_accounting_voucher(v)
if not result:
sb.log.warning(f"凭证 {v.get('摘要')} 可能需要人工确认")
time.sleep(2) # 每条凭证后等待2秒
常见问题与解决方案
| 问题 | 原因 | 解决方案 |
|---|---|---|
| 点击偏移 | 屏幕分辨率不同 | 使用相对坐标或图像识别 |
| 中文输入乱码 | 输入法干扰 | 使用剪贴板粘贴方式 |
| 找不到窗口 | 被遮挡/最小化 | 使用win32api强制置前 |
| 操作太快 | 软件响应慢 | 增加wait/sleep时间 |
| 软件崩溃 | 内存/兼容性 | 添加崩溃检测与重启逻辑 |
总结
桌面应用自动化的核心技能：
- 窗口管理：win32gui定位、激活、调整窗口
- 坐标系统：使用相对坐标避免硬编码
- 图像识别：cv2模板匹配，不依赖固定坐标
- 键盘模拟：Tab键导航、快捷键操作
- 鼠标控制：平滑移动、随机间隔、防检测
作者：林焱 | 转载请注明出处
