Please enter the commit message for your changes. Lines starting
with '' will be ignored, and an empty message aborts the commit. On branch master Initial commit Changes to be committed: new file: .gitignore new file: Cargo.lock new file: Cargo.toml new file: LICENSE new file: README.md new file: justfile new file: src/adb.rs new file: src/answerer.rs new file: src/context.rs new file: src/logging.rs new file: src/main.rs new file: src/page.rs new file: src/utils.rs
This commit is contained in:
RainNight-PrincessHighness
+276
@@ -0,0 +1,276 @@
|
||||
// 257bf64 unauthorized
|
||||
// 257bf64 device
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use image::RgbaImage;
|
||||
use imageproc::{point::Point, rect::Rect};
|
||||
use more_asserts::{assert_ge, assert_gt};
|
||||
use rand::{SeedableRng, rngs::StdRng};
|
||||
use rand_distr::Distribution;
|
||||
|
||||
use crate::page::RectExtra;
|
||||
|
||||
pub struct Adb {
|
||||
adb: String,
|
||||
device: String,
|
||||
rng: StdRng,
|
||||
}
|
||||
|
||||
impl Adb {
|
||||
pub fn new(adb: &str) -> Self {
|
||||
let adb = if adb.find(std::path::MAIN_SEPARATOR).is_none() {
|
||||
adb.to_owned()
|
||||
} else {
|
||||
which(adb)
|
||||
};
|
||||
let rng = StdRng::from_rng(&mut rand::rng());
|
||||
Adb {
|
||||
adb,
|
||||
device: "".to_owned(),
|
||||
rng,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tap(&self, point: Point<i32>) {
|
||||
self.shell(&[
|
||||
"shell",
|
||||
"input",
|
||||
"tap",
|
||||
&point.x.to_string(),
|
||||
&point.y.to_string(),
|
||||
]);
|
||||
}
|
||||
|
||||
pub fn tap_random(&mut self, area: &Rect) {
|
||||
let point = self.random_point(area);
|
||||
self.tap(point);
|
||||
}
|
||||
|
||||
fn random_point(&mut self, area: &Rect) -> Point<i32> {
|
||||
assert_ge!(area.left(), 0);
|
||||
assert_ge!(area.top(), 0);
|
||||
assert_ge!(area.width(), 3);
|
||||
assert_ge!(area.height(), 3);
|
||||
if area.width() < 5 || area.height() < 5 {
|
||||
return area.center();
|
||||
}
|
||||
const RATE: u32 = 6;
|
||||
let area = Rect::at(
|
||||
area.left() + (area.width() / RATE) as i32,
|
||||
area.top() + (area.height() / RATE) as i32,
|
||||
)
|
||||
.of_size(
|
||||
area.width() - area.width() / RATE * 2,
|
||||
area.height() - area.height() / RATE * 2,
|
||||
);
|
||||
|
||||
const STD_DEV: f64 = 0.25;
|
||||
let normal = rand_distr::Normal::new(0.5, STD_DEV).unwrap();
|
||||
let (x, y) = loop {
|
||||
let x = normal.sample(&mut self.rng);
|
||||
let y = normal.sample(&mut self.rng);
|
||||
if (0.0..1.0).contains(&x) && (0.0..1.0).contains(&y) {
|
||||
break (x, y);
|
||||
}
|
||||
};
|
||||
let x = area.width() as f64 * x;
|
||||
let y = area.height() as f64 * y;
|
||||
Point::new(area.left() + x as i32, area.top() + y as i32)
|
||||
}
|
||||
|
||||
pub fn screencap(&self) -> RgbaImage {
|
||||
let out = self.shell(&[
|
||||
"exec-out", "screencap", "-p",
|
||||
"-d", "4630946457447247251",
|
||||
]);
|
||||
|
||||
const PNG_MAGIC: [u8; 8] = [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A];
|
||||
if out.len() < 8 || out[..8] != PNG_MAGIC {
|
||||
let head = &out[..out.len().min(80)];
|
||||
panic!(
|
||||
"screencap not PNG: len={}, head={:02X?}, text={:?}",
|
||||
out.len(),
|
||||
head,
|
||||
String::from_utf8_lossy(head),
|
||||
);
|
||||
}
|
||||
|
||||
let screen = image::load_from_memory(&out).unwrap();
|
||||
assert!(screen.color() == image::ColorType::Rgba8);
|
||||
screen.into_rgba8()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn screen_size(&self) -> (u32, u32) {
|
||||
let out = self.shell(&["shell", "wm", "size"]);
|
||||
let out = String::from_utf8_lossy(&out);
|
||||
let mut size = out.split_ascii_whitespace().last().unwrap().split('x');
|
||||
let width = size.next().unwrap().parse::<u32>().unwrap();
|
||||
let height = size.next().unwrap().parse::<u32>().unwrap();
|
||||
assert_gt!(width, 0);
|
||||
assert_gt!(height, 0);
|
||||
(width, height)
|
||||
}
|
||||
|
||||
pub fn set_device(&mut self, device: String) {
|
||||
self.device = device;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn devices(&self) -> Vec<Device> {
|
||||
let stdout = self.shell(&["devices"]);
|
||||
let stdout = String::from_utf8_lossy(&stdout);
|
||||
let mut devices = vec![];
|
||||
for line in stdout.lines().skip(1) {
|
||||
let device = line.split_whitespace().next().unwrap_or_default();
|
||||
if device.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if line.ends_with("device") {
|
||||
devices.push(Device::Device(device.to_string()));
|
||||
} else if line.ends_with("unauthorized") {
|
||||
devices.push(Device::Unauthorized(device.to_string()));
|
||||
} else {
|
||||
log::warn!("Unknown device: {}", line);
|
||||
}
|
||||
}
|
||||
log::info!("Devices: {:?}", devices);
|
||||
devices
|
||||
}
|
||||
|
||||
fn shell(&self, args: &[&str]) -> Vec<u8> {
|
||||
let mut cmd = Command::new(&self.adb);
|
||||
if !self.device.is_empty() {
|
||||
cmd.arg("-s").arg(&self.device);
|
||||
}
|
||||
command(cmd.args(args))
|
||||
}
|
||||
}
|
||||
|
||||
fn which(name: &str) -> String {
|
||||
let mut cmd = if cfg!(windows) {
|
||||
let mut cmd = Command::new("powershell.exe");
|
||||
cmd.arg("-Command");
|
||||
cmd.arg(format!("(Get-Command '{}').Source", name));
|
||||
cmd
|
||||
} else if cfg!(unix) {
|
||||
let mut cmd = Command::new("bash");
|
||||
cmd.arg("-c");
|
||||
cmd.arg(format!("command -v '{}'", name));
|
||||
cmd
|
||||
} else {
|
||||
panic!("Unsupported platform");
|
||||
};
|
||||
String::from_utf8_lossy(&command(&mut cmd))
|
||||
.trim()
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
fn command(cmd: &mut Command) -> Vec<u8> {
|
||||
let out = cmd.output().unwrap_or_else(|e| {
|
||||
panic!(
|
||||
"Failed to execute command: {} {:?}: {}",
|
||||
cmd.get_program().to_str().unwrap(),
|
||||
cmd.get_args(),
|
||||
e
|
||||
)
|
||||
});
|
||||
if out.status.success() {
|
||||
out.stdout
|
||||
} else {
|
||||
log::error!(
|
||||
"{} {:?}: {:?}",
|
||||
cmd.get_program().to_str().unwrap(),
|
||||
cmd.get_args(),
|
||||
out.status,
|
||||
);
|
||||
panic!(
|
||||
"{} {:?}\n{}",
|
||||
cmd.get_program().to_str().unwrap(),
|
||||
cmd.get_args(),
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-adb"))]
|
||||
mod tests {
|
||||
use more_asserts::assert_le;
|
||||
|
||||
use crate::logging;
|
||||
|
||||
use super::*;
|
||||
#[test]
|
||||
fn test_devices() {
|
||||
logging::init_for_test();
|
||||
|
||||
let mut adb = Adb::new("adb");
|
||||
let devices = adb.devices();
|
||||
assert!(!devices.is_empty(), "No devices found");
|
||||
for device in devices {
|
||||
if let Device::Device(device) = device {
|
||||
adb.set_device(device.clone());
|
||||
let out = adb.shell(&["shell", "pwd"]);
|
||||
let out = String::from_utf8_lossy(&out);
|
||||
println!("Device: {}: {}", device, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_screen_size() {
|
||||
logging::init_for_test();
|
||||
let adb = Adb::new("adb");
|
||||
let (width, height) = adb.screen_size();
|
||||
println!("Screen size: {}x{}", width, height);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_screencap() {
|
||||
logging::init_for_test();
|
||||
let adb = Adb::new("adb");
|
||||
let screen = adb.screencap();
|
||||
println!("Screen size: {:?}", screen.dimensions());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_screencap_save() {
|
||||
logging::init_for_test();
|
||||
let adb = Adb::new("adb");
|
||||
let screen = adb.screencap();
|
||||
screen.save("screen.png").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_random_point() {
|
||||
logging::init_for_test();
|
||||
let mut adb = Adb::new("adb");
|
||||
let area = Rect::at(0, 0).of_size(200, 100);
|
||||
for i in 0..10000 {
|
||||
let point = adb.random_point(&area);
|
||||
assert_ge!(point.x, 33);
|
||||
assert_le!(point.x, 166);
|
||||
assert_ge!(point.y, 16);
|
||||
assert_le!(point.y, 83);
|
||||
if i < 10 {
|
||||
println!("Point: {:3}, {:3}", point.x, point.y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tap() {
|
||||
logging::init_for_test();
|
||||
let adb = Adb::new("adb");
|
||||
let (width, height) = adb.screen_size();
|
||||
adb.tap(Point::new((width / 2) as i32, (height / 2) as i32));
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
pub enum Device {
|
||||
Unauthorized(String),
|
||||
Device(String),
|
||||
}
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
use std::{io::Cursor, time::Duration};
|
||||
|
||||
use base64::{Engine, prelude::BASE64_STANDARD};
|
||||
use image::{ImageFormat, RgbImage, RgbaImage, buffer::ConvertBuffer};
|
||||
use rand::RngExt;
|
||||
use reqwest::{
|
||||
blocking::Client,
|
||||
header::{HeaderMap, HeaderName, HeaderValue},
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::{
|
||||
context::Context, json_at, json_value_as_i64, json_value_as_str, json_value_as_vec, parse_json,
|
||||
};
|
||||
|
||||
pub struct Multimodal {
|
||||
url: String,
|
||||
model: String,
|
||||
key: String,
|
||||
client: Client,
|
||||
thinking: bool,
|
||||
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
}
|
||||
|
||||
impl Multimodal {
|
||||
pub fn from_args(ctx: &Context) -> Self {
|
||||
let url = ctx.api_url.clone();
|
||||
let model = ctx.api_model.clone();
|
||||
let key = ctx.api_key.clone();
|
||||
Self::new(url, model, key, ctx.answer_thinking)
|
||||
}
|
||||
pub fn new(url: String, model: String, key: String, thinking: bool) -> Self {
|
||||
let client = Client::new();
|
||||
Self {
|
||||
url,
|
||||
model,
|
||||
key,
|
||||
client,
|
||||
thinking,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn answer(&mut self, question: &RgbaImage) -> Option<Answer> {
|
||||
let resp = self.post(&question.convert());
|
||||
log::trace!("{}", serde_json::to_string(&resp).unwrap());
|
||||
let choices = json_value_as_vec!(json_at!(resp, "choices").unwrap()).unwrap();
|
||||
let choice = &choices[0];
|
||||
let usage = json_at!(resp, "usage").unwrap();
|
||||
let message = json_value_as_str!(json_at!(choice, "message", "content").unwrap()).unwrap();
|
||||
let prompt_tokens =
|
||||
json_value_as_i64!(json_at!(usage, "prompt_tokens").unwrap()).unwrap() as u64;
|
||||
let completion_tokens =
|
||||
json_value_as_i64!(json_at!(usage, "completion_tokens").unwrap()).unwrap() as u64;
|
||||
log::debug!("answer: {message:?}");
|
||||
log::debug!(
|
||||
"tokens: prompt: {prompt_tokens}, completion: {completion_tokens}, total: {}",
|
||||
prompt_tokens + completion_tokens
|
||||
);
|
||||
self.prompt_tokens += prompt_tokens;
|
||||
self.completion_tokens += completion_tokens;
|
||||
log::trace!(
|
||||
"acc tokens: prompt: {}, completion: {}, total: {}",
|
||||
self.prompt_tokens,
|
||||
self.completion_tokens,
|
||||
self.tokens()
|
||||
);
|
||||
|
||||
log::debug!("raw answer: {}", message);
|
||||
parse_answer(message)
|
||||
}
|
||||
|
||||
pub fn input_tokens(&self) -> u64 {
|
||||
self.prompt_tokens
|
||||
}
|
||||
|
||||
pub fn output_tokens(&self) -> u64 {
|
||||
self.completion_tokens
|
||||
}
|
||||
|
||||
pub fn tokens(&self) -> u64 {
|
||||
self.prompt_tokens + self.completion_tokens
|
||||
}
|
||||
|
||||
fn post(&self, question: &RgbImage) -> serde_json::Value {
|
||||
let img_base64 = image_to_jpeg_to_base64(question);
|
||||
|
||||
let headers = new_headers(&[
|
||||
("Content-Type", "application/json"),
|
||||
("Authorization", &format!("Bearer {}", self.key)),
|
||||
]);
|
||||
let prompt = "回答图片里的选择题，你的回答会被代码解析，直接输出你认为最合适的选项字母，仅输出选项字母，不需要多余的解释，即使不确定也必须选择一个选项。";
|
||||
let body = json!({
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": format!("data:image/jpeg;base64,{}", img_base64),
|
||||
},
|
||||
},
|
||||
{
|
||||
"type":"text",
|
||||
"text": prompt,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
"thinking": {
|
||||
"type": if self.thinking { "enabled" } else { "disabled" },
|
||||
},
|
||||
});
|
||||
let body = body.to_string();
|
||||
log::debug!("request {}", self.url);
|
||||
let resp = self
|
||||
.client
|
||||
.post(&self.url)
|
||||
.headers(headers)
|
||||
.body(body)
|
||||
.timeout(Duration::from_secs(600));
|
||||
let resp = resp.send().unwrap();
|
||||
|
||||
let resp_status = resp.status();
|
||||
let resp = resp.error_for_status();
|
||||
let resp = resp.unwrap_or_else(|_| panic!("status: {resp_status}"));
|
||||
|
||||
parse_json!(resp.text().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
fn new_headers(headers: &[(&str, &str)]) -> HeaderMap {
|
||||
let mut req_headers = HeaderMap::new();
|
||||
for (key, value) in headers {
|
||||
let name = HeaderName::from_bytes(key.as_bytes()).unwrap();
|
||||
let value = HeaderValue::from_str(value).unwrap();
|
||||
req_headers.insert(name, value);
|
||||
}
|
||||
req_headers
|
||||
}
|
||||
|
||||
pub fn image_to_jpeg_to_base64(img: &RgbImage) -> String {
|
||||
let mut buf = Vec::new();
|
||||
let mut writer = Cursor::new(&mut buf);
|
||||
img.write_to(&mut writer, ImageFormat::Jpeg).unwrap();
|
||||
BASE64_STANDARD.encode(&buf)
|
||||
}
|
||||
|
||||
#[derive(clap::ValueEnum, Clone, Copy, Debug)]
|
||||
pub enum Answer {
|
||||
A = 0,
|
||||
B = 1,
|
||||
C = 2,
|
||||
D = 3,
|
||||
}
|
||||
|
||||
impl Answer {
|
||||
pub fn random() -> Self {
|
||||
match rand::rng().random_range(0..4) {
|
||||
0 => Answer::A,
|
||||
1 => Answer::B,
|
||||
2 => Answer::C,
|
||||
_ => Answer::D,
|
||||
}
|
||||
}
|
||||
pub fn to_str<'a>(self) -> &'a str {
|
||||
match self {
|
||||
Answer::A => "A",
|
||||
Answer::B => "B",
|
||||
Answer::C => "C",
|
||||
Answer::D => "D",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_answer(arg_answer: &str) -> Option<Answer> {
|
||||
let answer = arg_answer.trim();
|
||||
let answer = if answer.len() == 1 {
|
||||
answer.chars().next().unwrap()
|
||||
} else {
|
||||
let pos = answer.find("答案")?;
|
||||
let mut ans = ' ';
|
||||
for c in answer[pos..].chars() {
|
||||
if c.is_ascii_alphabetic() {
|
||||
ans = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ans
|
||||
};
|
||||
|
||||
match answer.to_ascii_uppercase() {
|
||||
'A' => Some(Answer::A),
|
||||
'B' => Some(Answer::B),
|
||||
'C' => Some(Answer::C),
|
||||
'D' => Some(Answer::D),
|
||||
_ => {
|
||||
log::warn!("Unknown answer: {arg_answer}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::logging::LogFormat;
|
||||
|
||||
#[derive(clap::Parser, Debug)]
|
||||
pub struct Context {
|
||||
#[arg(long, default_value_t = log::Level::Info, env = "BILI_LV6_HARDCORE_LOG_LEVEL")]
|
||||
pub log_level: log::Level,
|
||||
|
||||
#[arg(long, default_value = "simple", env = "BILI_LV6_HARDCORE_LOG_FORMAT")]
|
||||
pub log_format: LogFormat,
|
||||
|
||||
/// 连接指定的设备，adb 的 -s 参数的值
|
||||
#[arg(long)]
|
||||
pub device: Option<String>,
|
||||
|
||||
#[arg(long, default_value = "adb", env = "BILI_LV6_HARDCORE_ADB")]
|
||||
pub adb: String,
|
||||
|
||||
// 遇到无法识别的答案，容许随机选择的比例
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = 0.03,
|
||||
env = "BILI_LV6_HARDCORE_ANSWER_FALLBACK_RATIO"
|
||||
)]
|
||||
pub answer_fallback_ratio: f32,
|
||||
// 遇到无法识别的答案，重试的次数
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = 1,
|
||||
env = "BILI_LV6_HARDCORE_ANSWER_FALLBACK_RETRY_COUNT"
|
||||
)]
|
||||
pub answer_retry_limit: u32,
|
||||
|
||||
// 是否开启模型的思考功能
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
env = "BILI_LV6_HARDCORE_ANSWER_THINKING"
|
||||
)]
|
||||
pub answer_thinking: bool,
|
||||
|
||||
#[arg(long, env = "BILI_LV6_HARDCORE_API_URL")]
|
||||
pub api_url: String,
|
||||
#[arg(long, env = "BILI_LV6_HARDCORE_API_MODEL")]
|
||||
pub api_model: String,
|
||||
#[arg(long, env = "BILI_LV6_HARDCORE_API_KEY")]
|
||||
pub api_key: String,
|
||||
|
||||
/// 每百万输入token的费用
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "1.5",
|
||||
env = "BILI_LV6_HARDCORE_API_COST_INPUT_PER_MILLION_TOKENS"
|
||||
)]
|
||||
pub api_cost_input: f64,
|
||||
/// 每百万输出token的费用
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "4.5",
|
||||
env = "BILI_LV6_HARDCORE_API_COST_OUTPUT_PER_MILLION_TOKENS"
|
||||
)]
|
||||
pub api_cost_output: f64,
|
||||
|
||||
/// 调试用，保存未识别的截图
|
||||
#[arg(long, env = "BILI_LV6_HARDCORE_DEBUG_SAVE_PATH")]
|
||||
pub debug_save_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Context {
|
||||
pub fn check(&mut self) {
|
||||
assert!(self.answer_fallback_ratio >= 0.0 && self.answer_fallback_ratio <= 1.0);
|
||||
}
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
use chrono::{DateTime, Local};
|
||||
use const_format::formatcp;
|
||||
|
||||
use std::{
|
||||
io::Write,
|
||||
path::MAIN_SEPARATOR,
|
||||
sync::{LazyLock, Mutex},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use crate::context::Context;
|
||||
|
||||
pub fn init(args: &Context, start_time: Instant) {
|
||||
let logger = Logger::new(args.log_format, start_time);
|
||||
|
||||
log::set_boxed_logger(Box::new(logger)).unwrap();
|
||||
log::set_max_level(args.log_level.to_level_filter());
|
||||
log::info!("System time {}", DateTimeFormat(Local::now()));
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn init_for_test() {
|
||||
static ONCE: LazyLock<()> = LazyLock::new(|| {
|
||||
let logger = Logger::new(LogFormat::Complete, Instant::now());
|
||||
|
||||
log::set_boxed_logger(Box::new(logger)).unwrap();
|
||||
log::set_max_level(log::LevelFilter::Trace);
|
||||
});
|
||||
let _ = &*ONCE;
|
||||
}
|
||||
|
||||
struct Logger {
|
||||
format: LogFormat,
|
||||
start_time: Instant,
|
||||
last_time: Mutex<Instant>,
|
||||
}
|
||||
|
||||
impl Logger {
|
||||
fn new(format: LogFormat, start_time: Instant) -> Self {
|
||||
Self {
|
||||
format,
|
||||
start_time,
|
||||
last_time: Mutex::new(start_time),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl log::Log for Logger {
|
||||
fn enabled(&self, metadata: &log::Metadata) -> bool {
|
||||
metadata.level() <= log::max_level()
|
||||
}
|
||||
|
||||
fn log(&self, record: &log::Record) {
|
||||
if self.format.is_simple() {
|
||||
println!("{}", record.args());
|
||||
return;
|
||||
}
|
||||
|
||||
let current_time = Instant::now();
|
||||
let since_last;
|
||||
{
|
||||
let mut last_time = self.last_time.lock().unwrap();
|
||||
since_last = current_time.duration_since(*last_time);
|
||||
*last_time = current_time;
|
||||
}
|
||||
let since_last = since_last.as_micros() as f32 / 1000.0;
|
||||
let elapsed_time =
|
||||
DurationFormat(current_time.duration_since(self.start_time).as_micros() as u64);
|
||||
let log_level = format_level(record.level());
|
||||
let source_file = SourceFileFormat::new(
|
||||
record.file().unwrap_or_default(),
|
||||
record.line().unwrap_or_default(),
|
||||
);
|
||||
let message = format!(
|
||||
"{elapsed_time} {log_level} {} {source_file} {since_last:.3}ms\n",
|
||||
record.args()
|
||||
);
|
||||
|
||||
print!("{}", message);
|
||||
}
|
||||
|
||||
fn flush(&self) {
|
||||
std::io::stdout().flush().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(clap::ValueEnum, Clone, Copy, Debug)]
|
||||
pub enum LogFormat {
|
||||
Simple,
|
||||
Complete,
|
||||
}
|
||||
|
||||
impl LogFormat {
|
||||
fn is_simple(&self) -> bool {
|
||||
matches!(self, LogFormat::Simple)
|
||||
}
|
||||
#[allow(dead_code)]
|
||||
fn is_complete(&self) -> bool {
|
||||
matches!(self, LogFormat::Complete)
|
||||
}
|
||||
}
|
||||
|
||||
fn format_level(level: log::Level) -> char {
|
||||
match level {
|
||||
log::Level::Error => 'E',
|
||||
log::Level::Warn => 'W',
|
||||
log::Level::Info => 'I',
|
||||
log::Level::Debug => 'D',
|
||||
log::Level::Trace => 'T',
|
||||
}
|
||||
}
|
||||
|
||||
struct DurationFormat(u64);
|
||||
|
||||
impl std::fmt::Display for DurationFormat {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let millis = self.0 / 1000;
|
||||
let second = millis / 1000;
|
||||
let minute = second / 60;
|
||||
let hour = minute / 60;
|
||||
write!(
|
||||
f,
|
||||
"{}:{:02}:{:02}.{:03},{:03}",
|
||||
hour,
|
||||
minute % 60,
|
||||
second % 60,
|
||||
millis % 1000,
|
||||
self.0 % 1000
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
struct DateTimeFormat(DateTime<Local>);
|
||||
|
||||
impl std::fmt::Display for DateTimeFormat {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let micros = self.0.timestamp_subsec_micros() % 1_000_000;
|
||||
let millis = micros / 1000;
|
||||
write!(
|
||||
f,
|
||||
"{}.{:03},{:03}",
|
||||
self.0.format("%Y-%m-%d %H:%M:%S"),
|
||||
millis,
|
||||
micros % 1000
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
struct SourceFileFormat<'a> {
|
||||
file: &'a str,
|
||||
line: u32,
|
||||
}
|
||||
|
||||
impl<'a> SourceFileFormat<'a> {
|
||||
fn new(file: &'a str, line: u32) -> Self {
|
||||
Self { file, line }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> std::fmt::Display for SourceFileFormat<'a> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut name = self.file;
|
||||
let mod_file = formatcp!("{}mod.rs", MAIN_SEPARATOR);
|
||||
if name.ends_with(mod_file) {
|
||||
name = &name[..name.len() - mod_file.len()];
|
||||
}
|
||||
if let Some(pos) = name.rfind(MAIN_SEPARATOR) {
|
||||
name = &name[pos + 1..];
|
||||
}
|
||||
write!(f, "{}:{}", name, self.line)
|
||||
}
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
// #![deny(clippy::unwrap_used)]
|
||||
mod adb;
|
||||
mod answerer;
|
||||
mod context;
|
||||
mod logging;
|
||||
mod page;
|
||||
mod utils;
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use clap::Parser;
|
||||
use image::{GenericImageView, GrayImage, RgbaImage, buffer::ConvertBuffer};
|
||||
use imageproc::drawing::draw_hollow_rect_mut;
|
||||
|
||||
use adb::Adb;
|
||||
use answerer::{Answer, Multimodal};
|
||||
use context::Context;
|
||||
use page::PageQuestion;
|
||||
|
||||
// sk-7bc0515c63a64d238a30deddb55f79bf
|
||||
// https://api.deepseek.com
|
||||
// deepseek-flash
|
||||
fn main() {
|
||||
let ctx = global_init();
|
||||
|
||||
let mut adb = Adb::new(&ctx.adb);
|
||||
if let Some(device) = &ctx.device {
|
||||
adb.set_device(device.to_owned());
|
||||
}
|
||||
wait_question_page(&adb);
|
||||
|
||||
let mut answerer = Multimodal::from_args(&ctx);
|
||||
let mut question_count = 0u32;
|
||||
let mut fallback_count = 0u32;
|
||||
const INTERVAL: Duration = Duration::from_millis(750);
|
||||
loop {
|
||||
let mut res = None;
|
||||
const IDENTIFY_TRY_LIMIT: usize = 5;
|
||||
for i in 0..IDENTIFY_TRY_LIMIT {
|
||||
if i != 0 {
|
||||
std::thread::sleep(INTERVAL);
|
||||
}
|
||||
if let Some(val) = identify_screen(&adb, &ctx.debug_save_path) {
|
||||
res = Some(val);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if res.is_none() {
|
||||
log::info!("No question page detected");
|
||||
break;
|
||||
}
|
||||
let (question, page) = res.unwrap();
|
||||
|
||||
question_count += 1;
|
||||
|
||||
let mut retry_count = 0u32;
|
||||
let ans = loop {
|
||||
let ans = answerer.answer(&question);
|
||||
if let Some(ans) = ans {
|
||||
break ans;
|
||||
}
|
||||
if retry_count < ctx.answer_retry_limit {
|
||||
retry_count += 1;
|
||||
log::info!(
|
||||
"Failed to get a valid answer, retrying... ({} / {})",
|
||||
retry_count,
|
||||
ctx.answer_retry_limit
|
||||
);
|
||||
continue;
|
||||
}
|
||||
fallback_count += 1;
|
||||
let limit = ctx.answer_fallback_ratio;
|
||||
let ratio = fallback_count as f32 / question_count as f32;
|
||||
assert!(
|
||||
ratio <= limit,
|
||||
"Fallback ratio exceeded: {ratio:.3} > {limit:.3}",
|
||||
);
|
||||
let ans = Answer::random();
|
||||
log::warn!(
|
||||
"Failed to parse answer, use random answer: {}, fallback count: {}/{}({:.3})",
|
||||
ans.to_str(),
|
||||
fallback_count,
|
||||
question_count,
|
||||
ratio
|
||||
);
|
||||
break ans;
|
||||
};
|
||||
let choice = page.choice(ans);
|
||||
log::info!("{:03}: answer: {:?}, tap screen", question_count, ans);
|
||||
adb.tap_random(choice);
|
||||
std::thread::sleep(INTERVAL);
|
||||
}
|
||||
log::info!(
|
||||
"cost: question: {}: tokens: input: {}, output: {}, total: {}, {:.3}RMB",
|
||||
question_count,
|
||||
answerer.input_tokens(),
|
||||
answerer.output_tokens(),
|
||||
answerer.tokens(),
|
||||
answerer.input_tokens() as f64 / 1_000_000f64 * ctx.api_cost_input
|
||||
+ answerer.output_tokens() as f64 / 1_000_000f64 * ctx.api_cost_output
|
||||
);
|
||||
}
|
||||
|
||||
fn wait_question_page(adb: &Adb) {
|
||||
loop {
|
||||
log::info!("Waiting for question page...");
|
||||
if identify_screen(adb, &None).is_some() {
|
||||
log::info!("Question page detected");
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
|
||||
fn identify_screen(adb: &Adb, save_error: &Option<PathBuf>) -> Option<(RgbaImage, PageQuestion)> {
|
||||
let screen = adb.screencap();
|
||||
let mut edges = edge_detection(&screen);
|
||||
match PageQuestion::match_page(&edges) {
|
||||
Ok(question) => {
|
||||
let core = &question.core;
|
||||
let core = screen
|
||||
.view(
|
||||
core.left() as u32,
|
||||
core.top() as u32,
|
||||
core.width(),
|
||||
core.height(),
|
||||
)
|
||||
.to_image();
|
||||
Some((core, question))
|
||||
}
|
||||
Err(rects) => {
|
||||
if let Some(save_path) = save_error {
|
||||
if !save_path.exists() {
|
||||
std::fs::create_dir_all(save_path).unwrap();
|
||||
}
|
||||
for rect in &rects {
|
||||
draw_hollow_rect_mut(&mut edges, *rect, image::Luma([255]));
|
||||
}
|
||||
let name = format!(
|
||||
"screen-{}.jpg",
|
||||
chrono::Local::now().format("%y%m%d-%H%M%S")
|
||||
);
|
||||
let file = save_path.join(name);
|
||||
log::debug!("save error image: {:?}", file);
|
||||
edges.save(file).unwrap();
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn edge_detection(img: &RgbaImage) -> GrayImage {
|
||||
let gray: GrayImage = img.convert();
|
||||
imageproc::edges::canny(&gray, 50.0, 150.0)
|
||||
}
|
||||
|
||||
fn global_init() -> Context {
|
||||
let start_time = std::time::Instant::now();
|
||||
dotenv();
|
||||
let mut ctx = Context::parse();
|
||||
ctx.check();
|
||||
logging::init(&ctx, start_time);
|
||||
ctx
|
||||
}
|
||||
|
||||
fn dotenv() {
|
||||
let env_file = std::env::var("BILI_LV6_HARDCORE_DOTENV").unwrap_or(".env".to_owned());
|
||||
let env_file = Path::new(&env_file);
|
||||
if env_file.is_file() {
|
||||
dotenvy::from_path(env_file).unwrap();
|
||||
}
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
use std::{fmt::Display, ops::Sub};
|
||||
|
||||
use image::GrayImage;
|
||||
use imageproc::{
|
||||
contours::{Contour, find_contours},
|
||||
point::Point,
|
||||
rect::Rect,
|
||||
};
|
||||
use num_traits::Bounded;
|
||||
|
||||
use crate::answerer::Answer;
|
||||
|
||||
pub(crate) struct PageQuestion {
|
||||
pub(crate) core: Rect,
|
||||
pub(crate) check_boxes: Vec<Rect>,
|
||||
}
|
||||
|
||||
impl PageQuestion {
|
||||
pub(crate) fn choice(&self, ans: Answer) -> &Rect {
|
||||
&self.check_boxes[ans as usize]
|
||||
}
|
||||
|
||||
pub(crate) fn match_page(edges: &GrayImage) -> Result<Self, Vec<Rect>> {
|
||||
let width = edges.width();
|
||||
let height = edges.height();
|
||||
|
||||
let contours_vec = find_contours::<i32>(edges);
|
||||
log::debug!("contours: {}", contours_vec.len());
|
||||
|
||||
let contours = contours_vec.iter().filter(|x| {
|
||||
if x.points.len() < width as usize {
|
||||
return false;
|
||||
}
|
||||
let rect = bounding_rect(x);
|
||||
if rect.width() < width / 2 {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
});
|
||||
|
||||
let rects: Vec<_> = contours.map(bounding_rect).collect();
|
||||
log::debug!("rects: {}", rects.len());
|
||||
log::trace!("rects: {:?}", rects);
|
||||
let choices = nms(&rects);
|
||||
log::debug!("nms: {}", choices.len());
|
||||
log::trace!("choices: {:?}", choices);
|
||||
let choices = choices.into_iter().filter(|x| {
|
||||
// 排除宽高比过大或者过小的框
|
||||
let ratio = x.width() as f32 / x.height() as f32;
|
||||
(5.0..=8.0).contains(&ratio)
|
||||
});
|
||||
let choices: Vec<_> = choices.collect();
|
||||
log::debug!("choices: {}", choices.len());
|
||||
|
||||
const CHOICES_NUMBER: usize = 4;
|
||||
if choices.len() != CHOICES_NUMBER {
|
||||
return Err(choices);
|
||||
}
|
||||
|
||||
// 所有的框需要差不多宽并且左右间距是一致的，左右间距一致防止截到过渡动画
|
||||
let same_w = is_difference_small(choices.iter().map(|x| x.width()), 3);
|
||||
let same_l = is_difference_small(choices.iter().map(|x| x.left()), 3);
|
||||
let same_lr_pad = (choices[0].left() - (width as i32 - choices[0].right())).abs() < 3;
|
||||
if !same_w || !same_l || !same_lr_pad {
|
||||
log::debug!(
|
||||
"not same: w: {}, l: {}, lr: {}",
|
||||
same_w,
|
||||
same_l,
|
||||
same_lr_pad
|
||||
);
|
||||
return Err(choices);
|
||||
}
|
||||
|
||||
let mut choices = choices;
|
||||
choices.sort_by_key(|x| x.top());
|
||||
for i in 1..choices.len() {
|
||||
if choices[i - 1].bottom() >= choices[i].top() {
|
||||
log::warn!("overlap: {} {}", choices[i - 1].bottom(), choices[i].top());
|
||||
return Err(choices);
|
||||
}
|
||||
}
|
||||
|
||||
let core = location_core(
|
||||
&contours_vec,
|
||||
(choices[0].top() as u32 + choices[0].height()) as usize,
|
||||
choices[0].height() as usize,
|
||||
width as usize,
|
||||
height as usize,
|
||||
);
|
||||
log::trace!("core: {:?}", core);
|
||||
|
||||
Ok(PageQuestion {
|
||||
core,
|
||||
check_boxes: choices,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn is_difference_small<T>(data: impl Iterator<Item = T>, threshold: T) -> bool
|
||||
where
|
||||
T: Bounded + PartialOrd + Copy + Sub<Output = T> + Display,
|
||||
{
|
||||
let mut min = T::max_value();
|
||||
let mut max = T::min_value();
|
||||
let mut n = 0usize;
|
||||
for val in data {
|
||||
n += 1;
|
||||
if val < min {
|
||||
min = val;
|
||||
}
|
||||
if val > max {
|
||||
max = val;
|
||||
}
|
||||
}
|
||||
let small = n == 0 || max - min <= threshold;
|
||||
if !small {
|
||||
log::debug!("difference_small: {min}, {max}");
|
||||
}
|
||||
small
|
||||
}
|
||||
|
||||
fn location_core(
|
||||
contours: &[Contour<i32>],
|
||||
center: usize,
|
||||
box_h: usize,
|
||||
img_w: usize,
|
||||
img_h: usize,
|
||||
) -> Rect {
|
||||
let mut point_count = vec![0; img_h];
|
||||
for contour in contours {
|
||||
for point in &contour.points {
|
||||
point_count[point.y as usize] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let mut top_y = 0;
|
||||
let mut begin = center;
|
||||
const THRESHOLD: i32 = 42;
|
||||
for i in (0..center).rev() {
|
||||
if point_count[i] < THRESHOLD {
|
||||
continue;
|
||||
}
|
||||
if begin - i > box_h + 1 {
|
||||
let off = box_h / 4 * 3;
|
||||
top_y = begin.max(off) - off;
|
||||
break;
|
||||
}
|
||||
begin = i;
|
||||
}
|
||||
let mut bottom_y = img_h;
|
||||
let mut begin = center;
|
||||
for (i, item) in point_count.iter().enumerate().take(img_h).skip(center) {
|
||||
if *item < THRESHOLD {
|
||||
continue;
|
||||
}
|
||||
if i - begin > box_h + 1 {
|
||||
bottom_y = (begin + box_h / 2).min(img_h);
|
||||
break;
|
||||
}
|
||||
begin = i;
|
||||
}
|
||||
const ALIGN: usize = 28;
|
||||
let h = bottom_y - top_y;
|
||||
assert!(h > ALIGN, "height: {}", h);
|
||||
let h = h / ALIGN * ALIGN;
|
||||
let mut top_y = top_y + (ALIGN - 1) / ALIGN * ALIGN;
|
||||
let bottom_y = bottom_y + (ALIGN - 1) / ALIGN * ALIGN;
|
||||
|
||||
if bottom_y - top_y > h {
|
||||
top_y += ALIGN / 2;
|
||||
}
|
||||
|
||||
Rect::at(0, top_y as i32).of_size(img_w as u32, h as u32)
|
||||
}
|
||||
|
||||
fn nms(rects: &[Rect]) -> Vec<Rect> {
|
||||
const IOU_THRESHOLD: f32 = 0.6;
|
||||
let mut res = vec![];
|
||||
for src in rects {
|
||||
if let Some(pos) = res
|
||||
.iter()
|
||||
.position(|target| src.iou(target) > IOU_THRESHOLD)
|
||||
{
|
||||
if src.area() > res[pos].area() {
|
||||
res[pos] = *src;
|
||||
}
|
||||
} else {
|
||||
res.push(*src);
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
fn bounding_rect(contour: &Contour<i32>) -> Rect {
|
||||
let mut min_x = i32::MAX;
|
||||
let mut min_y = i32::MAX;
|
||||
let mut max_x = i32::MIN;
|
||||
let mut max_y = i32::MIN;
|
||||
|
||||
for point in &contour.points {
|
||||
if point.x < min_x {
|
||||
min_x = point.x;
|
||||
}
|
||||
if point.y < min_y {
|
||||
min_y = point.y;
|
||||
}
|
||||
if point.x > max_x {
|
||||
max_x = point.x;
|
||||
}
|
||||
if point.y > max_y {
|
||||
max_y = point.y;
|
||||
}
|
||||
}
|
||||
|
||||
Rect::at(min_x, min_y).of_size((max_x + 1 - min_x) as u32, (max_y + 1 - min_y) as u32)
|
||||
}
|
||||
|
||||
pub(crate) trait RectExtra {
|
||||
fn area(&self) -> u64;
|
||||
fn iou(&self, other: &Self) -> f32;
|
||||
fn center(&self) -> Point<i32>;
|
||||
#[allow(dead_code)]
|
||||
fn contains(&self, point: Point<i32>) -> bool;
|
||||
}
|
||||
|
||||
impl RectExtra for Rect {
|
||||
fn area(&self) -> u64 {
|
||||
self.width() as u64 * self.height() as u64
|
||||
}
|
||||
|
||||
fn iou(&self, other: &Self) -> f32 {
|
||||
let x1 = self.left().max(other.left());
|
||||
let y1 = self.top().max(other.top());
|
||||
let x2 = self.right().min(other.right());
|
||||
let y2 = self.bottom().min(other.bottom());
|
||||
if x1 > x2 || y1 > y2 {
|
||||
0.0
|
||||
} else {
|
||||
let inter_area = (x2 - x1 + 1) * (y2 - y1 + 1);
|
||||
let union_area = self.area() + other.area() - inter_area as u64;
|
||||
(inter_area as f64 / union_area as f64) as f32
|
||||
}
|
||||
}
|
||||
|
||||
fn center(&self) -> Point<i32> {
|
||||
let x = self.left() + self.width() as i32 / 2;
|
||||
let y = self.top() + self.height() as i32 / 2;
|
||||
Point::new(x, y)
|
||||
}
|
||||
|
||||
fn contains(&self, point: Point<i32>) -> bool {
|
||||
point.x >= self.left()
|
||||
&& point.x <= self.right()
|
||||
&& point.y >= self.top()
|
||||
&& point.y <= self.bottom()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
#[macro_export]
|
||||
macro_rules! parse_json {
|
||||
($text:expr) => {{
|
||||
let text = &$text;
|
||||
serde_json::from_str::<serde_json::Value>(text).expect(text)
|
||||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! json_at {
|
||||
($json:expr, $key:expr) => {{
|
||||
let json = &$json;
|
||||
let key = &$key;
|
||||
let value = &json[key];
|
||||
if !value.is_null() {
|
||||
Ok(value)
|
||||
} else {
|
||||
Err(format!("missing key: {}, in {}", key, json))
|
||||
}
|
||||
}};
|
||||
($json:expr, $key:expr, $($keys:expr),+) => {{
|
||||
json_at!($json, $key).and_then(|x| json_at!(x, $($keys),+))
|
||||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! json_value_as_i64 {
|
||||
($json:expr) => {{
|
||||
let value = &$json;
|
||||
match value {
|
||||
serde_json::Value::Number(val) => Ok(val.as_i64().unwrap()),
|
||||
_ => Err(format!("cannot convert to int: {}", value)),
|
||||
}
|
||||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! json_value_as_vec {
|
||||
($json:expr) => {{
|
||||
let value = &$json;
|
||||
match value {
|
||||
serde_json::Value::Array(val) => Ok(val),
|
||||
_ => Err(format!("cannot convert to array: {}", value)),
|
||||
}
|
||||
}};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! json_value_as_str {
|
||||
($json:expr) => {{
|
||||
let value = &$json;
|
||||
match value {
|
||||
serde_json::Value::String(val) => Ok(val.as_str()),
|
||||
_ => Err(format!("cannot convert to str: {}", value)),
|
||||
}
|
||||
}};
|
||||
}
|
||||
Reference in New Issue
Block a user