chakcy_code_repository/queue_sqlite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

modify: 协程调度器

Browse Source
This commit is contained in:
chakcy
2025-09-20 19:07:52 +08:00
parent 2a516ec5a3
commit 7189d6e2a4
25 changed files with 916 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/src/queue_operation.rs
+25
View File
@@ -461,6 +461,31 @@ impl QueueOperation {
Ok(())
}
pub fn remove_expired_messages(&self, days: i64) -> PyResult<()> {
let now: DateTime<Utc> = Utc::now();
let expire_time = now - Duration::days(days);
let iso_format = expire_time.to_rfc3339();
let conn = self
.pool
.get()
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
conn.execute(
"
UPDATE messages
SET is_deleted = 1
WHERE is_deleted = 0
AND expire_time IS NOT NULL
AND expire_time < ?1
",
params![iso_format],
)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(())
}
pub fn clean_expired_messages(&self) -> PyResult<()> {
let now: DateTime<Utc> = Utc::now();
let iso_format = now.to_rfc3339();
src/queue_sqlite/constant/message_status.py
-1
View File
@@ -7,4 +7,3 @@ class MessageStatus(IntEnum):
COMPLETED = 2
FAILED = 3
RETRYING = 4
src/queue_sqlite/constant/message_type.py
+2 -2
View File
@@ -2,5 +2,5 @@ from enum import Enum
class MessageType(Enum):
TASK = "task" # 任务
LISTEN = "listen" # 监听
TASK = "task" # 任务
LISTEN = "listen" # 监听
src/queue_sqlite/core/__init__.pyi
+1
View File
@@ -44,6 +44,7 @@ class core:
def delete_message(self, message_id: str): ...
def clean_old_messages(self, days: int): ...
def clean_expired_messages(self): ...
def remove_expired_messages(self, days: int): ...
class TaskMounter:
def __init__(self, task_class: type[TaskMounter]): ...
src/queue_sqlite/core/core.pyd
BIN
View File
Binary file not shown.
src/queue_sqlite/core/core.so
BIN
View File
Binary file not shown.
src/queue_sqlite/cycle/async_task_cycle.py
+97
View File
@@ -0,0 +1,97 @@
from ..model import MessageItem
from ..constant import MessageStatus
import json
import asyncio
import functools
def retry_async(max_retries=3, delay=1):
"""
异步重试装饰器
Args:
max_retries: 最大重试次数
delay: 重试间隔（秒）
"""
def decorator(func):
@functools.wraps(func)
async def wrapper(self, *args, **kwargs):
last_exception = None
# 使用message_item中的retry_count作为最大重试次数
retries = getattr(self.message_item, "retry_count", max_retries)
# 至少尝试一次（retries+1），最多尝试max_retries+1次
actual_retries = min(retries, max_retries) if max_retries > 0 else retries
for attempt in range(actual_retries + 1):
try:
return await func(self, *args, **kwargs)
except Exception as e:
last_exception = e
if attempt < actual_retries:
await asyncio.sleep(delay)
else:
break
if last_exception:
raise last_exception
else:
raise Exception("Unknown error occurred during async task execution")
return wrapper
return decorator
class AsyncTaskCycle:
def __init__(self, message_item: MessageItem, callback):
self.message_item = message_item
self.callback = callback
self.task_result = None
self.task_status = None
self.task_error = None
@retry_async(max_retries=3, delay=1)
async def run(self):
try:
task_result = await self.callback(self.message_item) # type: ignore
except Exception as e:
self.task_result = None
self.task_status = MessageStatus.FAILED
self.task_error = str(e)
else:
self.task_result = task_result
self.task_status = MessageStatus.COMPLETED
self.task_error = None
def get_task_result(self):
if isinstance(self.task_result, (dict, list)):
try:
return json.dumps(self.task_result)
except:
return json.dumps({"result": str(self.task_result)})
elif isinstance(self.task_result, str):
try:
json.loads(self.task_result)
return self.task_result
except:
return json.dumps({"result": self.task_result})
elif isinstance(self.task_result, (int, float, bool)):
return json.dumps({"result": self.task_result})
elif self.task_result is None:
return "null"
else:
return json.dumps({"result": str(self.task_result)})
def get_task_status(self):
return self.task_status
def get_task_error(self):
return self.task_error
def get_task_message_item(self):
return self.message_item
def get_task_callback(self):
return self.callback
src/queue_sqlite/cycle/task_cycle.py
+52 -9
View File
@@ -3,6 +3,50 @@ from typing import Callable, Optional
from ..constant import MessageStatus
import json
import functools
import time
def retry_sync(max_retries=3, delay=1):
"""
同步重试装饰器
Args:
max_retries: 最大重试次数
delay: 重试间隔（秒）
"""
def decorator(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
last_exception = None
# 使用message_item中的retry_count作为最大重试次数
retries = getattr(self.message_item, "retry_count", max_retries)
# 至少尝试一次（retries+1），最多尝试max_retries+1次
actual_retries = min(retries, max_retries) if max_retries > 0 else retries
for attempt in range(actual_retries + 1):
try:
return func(self, *args, **kwargs)
except Exception as e:
last_exception = e
if attempt < actual_retries:
time.sleep(delay)
else:
break
# 如果没有捕获到异常，则创建一个新的异常
if last_exception is not None:
raise last_exception
else:
raise Exception("Unknown error occurred during sync task execution")
return wrapper
return decorator
class TaskCycle:
def __init__(self, message_item: MessageItem, callback: Optional[Callable]):
self.message_item = message_item
@@ -11,15 +55,16 @@ class TaskCycle:
self.task_status = None
self.task_error = None
@retry_sync(max_retries=3, delay=1)
def run(self):
try:
task_result = self.callback(self.message_item) # type: ignore
task_result = self.callback(self.message_item) # type: ignore
except Exception as e:
self.task_result = None
self.task_status = MessageStatus.FAILED
self.task_error = str(e)
else:
self.task_result = task_result # type: ignore
self.task_result = task_result # type: ignore
self.task_status = MessageStatus.COMPLETED
self.task_error = None
@@ -28,20 +73,20 @@ class TaskCycle:
try:
return json.dumps(self.task_result)
except:
return json.dumps({'result': str(self.task_result)})
return json.dumps({"result": str(self.task_result)})
elif isinstance(self.task_result, str):
try:
json.loads(self.task_result)
return self.task_result
except:
return json.dumps({'result': self.task_result})
return json.dumps({"result": self.task_result})
elif isinstance(self.task_result, (int, float, bool)):
return json.dumps({'result': self.task_result})
return json.dumps({"result": self.task_result})
elif self.task_result is None:
return 'null'
return "null"
else:
return json.dumps({'result': str(self.task_result)})
return json.dumps({"result": str(self.task_result)})
def get_task_status(self):
return self.task_status
@@ -54,5 +99,3 @@ class TaskCycle:
def get_task_callback(self):
return self.callback
src/queue_sqlite/queue_operation/listen_operation.py
+33 -21
View File
@@ -19,12 +19,21 @@ class ListenOperation:
def create_table(self):
if len(self.listen_fields) == 0:
return
sql = f"""
conn = self._get_connection()
# 分别执行每个SQL语句而不是使用executescript
conn.execute(
"""
CREATE TABLE IF NOT EXISTS listen_table (
id INTEGER PRIMARY KEY AUTOINCREMENT,
key Text,
value JSON
);
)
"""
)
conn.execute(
"""
CREATE TABLE IF NOT EXISTS change_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
table_name TEXT,
@@ -34,18 +43,28 @@ class ListenOperation:
new_value TEXT,
is_delete integer DEFAULT 0,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TRIGGER IF NOT EXISTS track_value_change
AFTER UPDATE OF value ON listen_table -- 监听特定列
FOR EACH ROW
WHEN OLD.value <> NEW.value -- 仅当值实际变化时触发
BEGIN
INSERT INTO change_log (table_name, row_id, column_name, old_value, new_value)
VALUES ('listen_table', NEW.id, 'key', OLD.key, NEW.key);
END;
)
"""
conn = self._get_connection()
conn.executescript(sql)
)
# 检查触发器是否已存在，如果不存在则创建
try:
conn.execute(
"""
CREATE TRIGGER IF NOT EXISTS track_value_change
AFTER UPDATE OF value ON listen_table
FOR EACH ROW
WHEN OLD.value <> NEW.value
BEGIN
INSERT INTO change_log (table_name, row_id, column_name, old_value, new_value)
VALUES ('listen_table', NEW.id, 'key', OLD.key, NEW.key);
END
"""
)
except sqlite3.Error:
# 如果触发器创建失败，我们继续执行其他操作
pass
conn.execute("PRAGMA journal_mode=WAL;")
conn.execute("PRAGMA synchronous=NORMAL;")
conn.execute("PRAGMA cache_size=-20000;")
@@ -54,15 +73,8 @@ class ListenOperation:
conn.execute("PRAGMA busy_timeout=5000;")
conn.commit()
# 删除原有数据
# sql = f"""
# DELETE FROM listen_table
# """
# conn.execute(sql)
# conn.commit()
for listen_field in self.listen_fields:
sql = f"""
sql = """
INSERT INTO
listen_table (key, value)
VALUES
src/queue_sqlite/queue_operation/queue_operation.py
+5
View File
@@ -120,3 +120,8 @@ class QueueOperation:
def clean_expired_messages(self, shard_index: int):
conn = self._get_shard_conn(shard_index)
conn.clean_expired_messages()
# 彻底删除消息
def remove_expired_messages(self, shard_index: int, remove_days: int = 30):
conn = self._get_shard_conn(shard_index)
conn.remove_expired_messages(remove_days)
src/queue_sqlite/scheduler/__init__.py
+11 -8
View File
@@ -1,24 +1,27 @@
from .standard import StandardQueueScheduler
from ._async import AsyncQueueScheduler
from .base import BaseScheduler
from ..model import MessageItem
from typing import Callable
import multiprocessing
SCHEDULER_TYPES = {"standard": StandardQueueScheduler, "async": AsyncQueueScheduler}
class QueueScheduler(BaseScheduler):
def __init__(
self,
receive_thread_num=1,
task_thread_num=1,
task_thread_num=multiprocessing.cpu_count() * 2,
shard_num=4,
scheduler_type="standard",
):
self.scheduler = None
if scheduler_type == "standard":
self.scheduler = StandardQueueScheduler(
receive_thread_num, task_thread_num, shard_num
)
else:
pass
scheduler_class = SCHEDULER_TYPES.get(scheduler_type, None)
if not scheduler_class:
raise ValueError(f"Invalid scheduler type: {scheduler_type}")
self.scheduler = scheduler_class(receive_thread_num, task_thread_num, shard_num)
if self.scheduler:
self.queue_operation = self.scheduler.queue_operation
src/queue_sqlite/scheduler/_async/__init__.py
+49
View File
@@ -0,0 +1,49 @@
from .async_receive_scheduler import AsyncReceiveScheduler
from .async_task_scheduler import AsyncTaskScheduler
from .async_listen_data_scheduler import AsyncListenDataScheduler
from ...model import MessageItem
from ...queue_operation.listen_operation import ListenOperation
from typing import Callable
from ...queue_operation import QueueOperation
import os
from ..base import BaseScheduler
from ..cleanup_scheduler import CleanupScheduler
class AsyncQueueScheduler(BaseScheduler):
def __init__(self, receive_thread_num=1, task_thread_num=1, shard_num=4):
self.queue_operation = QueueOperation(shard_num=shard_num)
self.receive_scheduler = AsyncReceiveScheduler(
self.queue_operation, receive_thread_num
)
self.listen_operation = ListenOperation(
os.path.join(self.queue_operation.db_dir, "listen.db")
)
self.listen_operation.create_table()
self.listen_scheduler = AsyncListenDataScheduler(self.listen_operation)
self.task_scheduler = AsyncTaskScheduler(self.queue_operation, task_thread_num)
self.cleanup_scheduler = CleanupScheduler(self.queue_operation)
def send_message(self, message: MessageItem, callback: Callable):
self.receive_scheduler.send_message(message, callback)
def update_listen_data(self, key, value):
self.listen_operation.update_listen_data(key, value)
def get_listen_datas(self):
return self.listen_operation.get_values()
def get_listen_data(self, key):
return self.listen_operation.get_value(key)
def start(self):
self.receive_scheduler.start_receive_thread()
self.task_scheduler.start_task_thread()
def stop(self):
self.receive_scheduler.stop_receive_thread()
self.task_scheduler.stop_task_thread()
__all__ = ["AsyncQueueScheduler"]
src/queue_sqlite/scheduler/_async/async_listen_data_scheduler.py
+67
View File
@@ -0,0 +1,67 @@
from ...queue_operation.listen_operation import ListenOperation
import asyncio
from ...mounter.listen_mounter import ListenMounter
import threading
import multiprocessing
import concurrent.futures
class AsyncListenDataScheduler:
def __init__(self, listen_operation: ListenOperation):
self.listen_operation = listen_operation
self.is_running = False
self.thread_num = multiprocessing.cpu_count()
self.listen_thread = None
async def _process_listen_data(self, key, value, delete_id):
listen_function = ListenMounter.get_Listener_function(key)
if listen_function:
if asyncio.iscoroutinefunction(listen_function):
await listen_function(value)
else:
try:
loop = asyncio.get_event_loop()
with concurrent.futures.ThreadPoolExecutor(
max_workers=self.thread_num
) as executor:
await loop.run_in_executor(executor, listen_function, value)
listen_function(value)
except Exception as e:
ValueError(f"Error in {key} listener function: {e}")
finally:
self.listen_operation.delete_change_log(delete_id=delete_id)
async def listen(self):
async with asyncio.Semaphore(self.thread_num):
while self.is_running:
status, change_data_items = self.listen_operation.listen_data()
tasks = []
if status:
for data in change_data_items:
key = data[6]
new_value = data[7]
delete_id = data[0]
tasks.append(
asyncio.create_task(
self._process_listen_data(key, new_value, delete_id)
)
)
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
else:
await asyncio.sleep(0.05)
def start_listen_data(self):
if self.is_running:
return
self.is_running = True
self.listen_thread = threading.Thread(target=self.listen)
self.listen_thread.start()
def stop_listen_data(self):
if not self.is_running:
return
self.is_running = False
if self.listen_thread and self.listen_thread.is_alive():
self.listen_thread.join(timeout=2)
src/queue_sqlite/scheduler/_async/async_receive_scheduler.py
+102
View File
@@ -0,0 +1,102 @@
from ...queue_operation import QueueOperation
from ...model import MessageItem
from typing import Callable
import asyncio
import threading
import concurrent.futures
class AsyncReceiveScheduler:
def __init__(self, queue_operation: QueueOperation, receive_thread_num: int = 1):
self.callbacks = dict()
self.receive_thread_num = receive_thread_num
self.is_running = False
self.lock = threading.Lock()
self.queue_operation = queue_operation
self.receive_thread = None # 单一轮询线程
def send_message(self, message: MessageItem, callback: Callable = None): # type: ignore
if callback is None:
callback = lambda message: print(message)
# self.queue_operation.enqueue(message.to_dict())
self.queue_operation.enqueue(message.to_dict_by_core())
with self.lock:
self.callbacks[message.id] = callback
async def receive_message(self):
"""单一轮询线程，并行执行回调"""
def callback_default(message):
return message
async with asyncio.Semaphore(self.receive_thread_num):
while self.is_running:
message_list = self.queue_operation.get_completed_messages()
tasks = []
if message_list:
for message in message_list:
message = MessageItem.from_dict(message)
with self.lock:
callback = self.callbacks.pop(message.id, None)
if callback is None:
callback = callback_default
tasks.append(
asyncio.create_task(self._safe_callback(callback, message))
)
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
else:
await asyncio.sleep(0.05)
async def _safe_callback(self, callback, message):
"""安全执行回调函数"""
try:
# 检查回调函数是否是异步函数
if asyncio.iscoroutinefunction(callback):
await callback(message)
else:
# 如果是同步函数，则在执行器中运行以避免阻塞事件循环
loop = asyncio.get_event_loop()
with concurrent.futures.ThreadPoolExecutor(
max_workers=self.receive_thread_num
) as executor:
await loop.run_in_executor(executor, callback, message)
except Exception as e:
print(f"回调执行错误: {str(e)}")
finally:
self.queue_operation.delete_message(message.id)
def _run_receive_loop(self):
"""在新事件循环中运行接收消息循环"""
# 创建新的事件循环
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
# 运行异步接收函数直到停止
loop.run_until_complete(self.receive_message())
except Exception as e:
print(f"任务调度出错: {str(e)}")
finally:
loop.close()
def start_receive_thread(self):
if self.is_running:
return
self.is_running = True
# 创建单一轮询线程
self.receive_thread = threading.Thread(
target=self._run_receive_loop, daemon=True
)
self.receive_thread.start()
def stop_receive_thread(self):
if not self.is_running:
return
self.is_running = False
# 等待轮询线程结束
if self.receive_thread and self.receive_thread.is_alive():
self.receive_thread.join(timeout=5.0)
src/queue_sqlite/scheduler/_async/async_task_scheduler.py
+137
View File
@@ -0,0 +1,137 @@
from ...queue_operation import QueueOperation
from ...model import MessageItem
from ...constant import MessageStatus
from ...cycle.async_task_cycle import AsyncTaskCycle
from ...mounter.task_mounter import TaskMounter
import asyncio
from datetime import datetime
import threading
import multiprocessing
class AsyncTaskScheduler:
def __init__(
self,
queue_operation: QueueOperation,
task_thread_num: int = multiprocessing.cpu_count() * 2,
):
self.task_thread_num = task_thread_num
self.is_running = False
self.queue_operation = queue_operation
self.task_thread = None # 单一轮询线程
async def _process_message(self, message: MessageItem):
"""处理单个消息"""
try:
if message.destination == "client":
message.status = MessageStatus.COMPLETED
message.result = {"result": "success"}
message.updatetime = datetime.now()
return message
task_function = TaskMounter.get_task_function(message.destination)
if task_function is None:
raise ValueError(
f"Task function not found for destination: {message.destination}"
)
task_cycle = AsyncTaskCycle(message, task_function)
await task_cycle.run()
message.status = task_cycle.get_task_status() # type: ignore
if message.status == MessageStatus.FAILED:
message.result = {"error": task_cycle.get_task_error()}
else:
message.result = task_cycle.get_task_result() # type: ignore
message.updatetime = datetime.now()
except Exception as e:
message.status = MessageStatus.FAILED
message.result = {"error": str(e)}
message.updatetime = datetime.now()
return message
def _process_messages(self, message):
"""更新任务结果到数据库"""
try:
self.queue_operation.update_result(message.id, message.result)
self.queue_operation.update_status(message.id, message.status)
except Exception as e:
print(f"任务结果更新失败: {str(e)}")
async def task_callback(self):
"""单一轮询线程，并行执行任务"""
# 使用信号量控制并发任务数
semaphore = asyncio.Semaphore(self.task_thread_num)
async def limited_process_message(message):
async with semaphore:
return await self._process_message(MessageItem.from_dict(message))
while self.is_running:
try:
message_list = self.queue_operation.dequeue(
size=self.task_thread_num * 2
)
if not message_list:
# 使用指数退避策略减少轮询开销
await asyncio.sleep(0.05) # 增加延迟减少CPU使用
continue
# 并发处理消息
tasks = [
asyncio.create_task(limited_process_message(message))
for message in message_list
]
# 等待所有任务完成
completed, _ = await asyncio.wait(
tasks,
return_when=asyncio.ALL_COMPLETED,
timeout=30.0, # 添加超时防止无限等待
)
# 批量处理结果
for task in completed:
try:
message = task.result()
self._process_messages(message)
except Exception as e:
print(f"处理任务结果时出错: {str(e)}")
except Exception as e:
print(f"任务调度出错: {str(e)}")
await asyncio.sleep(0.1) # 出错时短暂延迟
def _run_task_loop(self):
"""在新事件循环中运行任务处理循环"""
# 创建新的事件循环
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
# 运行异步任务处理函数直到停止
loop.run_until_complete(self.task_callback())
except Exception as e:
print(f"任务调度出错: {str(e)}")
finally:
loop.close()
def start_task_thread(self):
if self.is_running:
return
self.is_running = True
self.task_thread = threading.Thread(target=self._run_task_loop)
self.task_thread.daemon = True
self.task_thread.start()
def stop_task_thread(self):
if not self.is_running:
return
self.is_running = False
if self.task_thread:
self.task_thread.join(timeout=5.0)
self.task_thread = None
src/queue_sqlite/scheduler/async/__init__.py
View File
src/queue_sqlite/scheduler/standard/cleanup_scheduler.py → src/queue_sqlite/scheduler/cleanup_scheduler.py
+12 -7
View File
@@ -1,26 +1,31 @@
# 新建清理调度器
import threading
import time
from ...queue_operation import QueueOperation
from ..queue_operation import QueueOperation
class CleanupScheduler:
def __init__(self, queue_operation: QueueOperation, interval_minutes=60):
def __init__(
self, queue_operation: QueueOperation, interval_minutes=60, remove_days=30
):
self.queue_operation = queue_operation
self.interval = interval_minutes * 60 # 转换为秒
self.is_running = False
self.cleanup_thread = None
self.remove_days = remove_days
for i in range(self.queue_operation.shard_num):
# 清理过期但未处理的消息
self.queue_operation.clean_expired_messages(i)
# 彻底删除30天前的消息
self.queue_operation.remove_expired_messages(i, self.remove_days)
def cleanup_expired_messages(self):
"""清理过期消息"""
while self.is_running:
try:
# 清理7天前的已完成/失败消息
for i in range(self.queue_operation.shard_num):
self.queue_operation.clean_old_messages(i, 7)
# 清理过期但未处理的消息
for i in range(self.queue_operation.shard_num):
# 清理过期但未处理的消息
self.queue_operation.clean_expired_messages(i)
except Exception as e:
src/queue_sqlite/scheduler/standard/__init__.py
+1 -1
View File
@@ -5,7 +5,7 @@ from ...model import MessageItem
from typing import Callable
from ...queue_operation import QueueOperation
from ...queue_operation.listen_operation import ListenOperation
from .cleanup_scheduler import CleanupScheduler
from ..cleanup_scheduler import CleanupScheduler
import os
from ..base import BaseScheduler
src/queue_sqlite/scheduler/standard/listen_data_scheduler.py
+2 -1
View File
@@ -2,13 +2,14 @@ from ...queue_operation.listen_operation import ListenOperation
from concurrent.futures import ThreadPoolExecutor
from ...mounter.listen_mounter import ListenMounter
import threading
import multiprocessing
class ListenDataScheduler:
def __init__(self, listen_operation: ListenOperation):
self.listen_operation = listen_operation
self.is_running = False
self.executor = ThreadPoolExecutor(max_workers=1)
self.executor = ThreadPoolExecutor(max_workers=multiprocessing.cpu_count())
self.listen_thread = None
def _process_listen_data(self, key, value, delete_id):
src/queue_sqlite/scheduler/standard/task_scheduler.py
+6 -1
View File
@@ -7,10 +7,15 @@ from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
import time
import threading
import multiprocessing
class TaskScheduler:
def __init__(self, queue_operation: QueueOperation, task_thread_num: int = 2):
def __init__(
self,
queue_operation: QueueOperation,
task_thread_num: int = multiprocessing.cpu_count() * 2,
):
self.task_thread_num = task_thread_num
self.is_running = False
self.executor = ThreadPoolExecutor(max_workers=task_thread_num) # 并行执行任务
tests/async_taks/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from .example import async_example
__all__ = ["async_example"]
tests/async_taks/example.py
+19
View File
@@ -0,0 +1,19 @@
from queue_sqlite.mounter.task_mounter import TaskMounter
from queue_sqlite.model import MessageItem
@TaskMounter.task(meta={"task_name": "test"})
async def async_example(message_item: MessageItem):
def fibonacci_generator():
a, b = 0, 1
while True:
yield a
a, b = b, a + b
# 示例：获取前10项
fib = fibonacci_generator()
message_item.result = {"fibonacci": [next(fib) for _ in range(500)]}
return message_item.to_json()
# 输出：[0, 1, 1, 2, 3, 5, 8, 13, 21, 34]
tests/test_async_scheduler.py
+75
View File
@@ -0,0 +1,75 @@
from queue_sqlite.model import MessageItem
from queue_sqlite.scheduler import QueueScheduler
import time
from async_taks import *
import threading
import psutil
class TestAsyncScheduler:
callback_counter = 0
lock = threading.Lock()
is_monitoring = False
@classmethod
def monitor_resources(cls):
"""监控资源使用情况"""
while cls.is_monitoring:
cpu = psutil.cpu_percent()
memory = psutil.virtual_memory().percent
print(f"CPU使用率: {cpu}%, 内存使用率: {memory}%")
time.sleep(0.5)
@classmethod
async def _callback(cls, message_item: MessageItem):
with cls.lock:
cls.callback_counter += 1
# print(f"callback: {message_item}")
# print(f"callback: {message_item.id}")
# print(f"callback: {message_item.expire_time}")
# print(f"callback: {message_item.tags}")
@classmethod
def test_async_scheduler(cls):
"""测试异步调度器"""
TASK_NUM = 10000
messages = []
for i in range(TASK_NUM):
message_item = MessageItem(content={"num": i}, destination="async_example")
messages.append(message_item)
cls.is_monitoring = True
monitor_thread = threading.Thread(target=cls.monitor_resources, daemon=True)
monitor_thread.start()
queue_scheduler = QueueScheduler(
receive_thread_num=4,
task_thread_num=8,
shard_num=12,
scheduler_type="async",
)
start_time = time.perf_counter()
queue_scheduler.start()
for message_item in messages:
message_item: MessageItem
queue_scheduler.send_message(message_item, cls._callback)
while (
queue_scheduler.queue_operation.get_queue_length() > 0
or cls.callback_counter < TASK_NUM
): # 30秒超时
time.sleep(0.5)
print(f"当前队列长度：{queue_scheduler.queue_operation.get_queue_length()}")
print(f"当前回调数量：{cls.callback_counter}")
queue_scheduler.stop()
print(f"当前队列长度：{queue_scheduler.queue_operation.get_queue_length()}")
print(f"测试函数：斐波那契数列")
print(f"次数：{TASK_NUM}")
print(f"总耗时: {time.perf_counter() - start_time:.2f}秒")
cls.is_monitoring = False
monitor_thread.join(timeout=1)
to_json_utils.py
+214
View File
@@ -0,0 +1,214 @@
import sqlite3
import json
import tkinter as tk
from tkinter import ttk, filedialog, messagebox
from datetime import datetime
class SQLiteToJSONConverter:
def __init__(self, root):
self.root = root
self.root.title("SQLite 表转 JSON 工具")
self.root.geometry("800x600")
# 变量
self.db_path = tk.StringVar()
self.tables = []
self.selected_table = tk.StringVar()
self.create_widgets()
def create_widgets(self):
# 主框架
main_frame = ttk.Frame(self.root, padding="10")
main_frame.grid(row=0, column=0, sticky=(tk.W, tk.E, tk.N, tk.S))
# 数据库选择部分
db_frame = ttk.LabelFrame(main_frame, text="数据库选择", padding="5")
db_frame.grid(row=0, column=0, columnspan=2, sticky=(tk.W, tk.E), pady=5)
ttk.Entry(db_frame, textvariable=self.db_path, width=70).grid(
row=0, column=0, padx=5
)
ttk.Button(db_frame, text="浏览", command=self.browse_db).grid(
row=0, column=1, padx=5
)
ttk.Button(db_frame, text="连接", command=self.connect_db).grid(
row=0, column=2, padx=5
)
# 表选择部分
table_frame = ttk.LabelFrame(main_frame, text="表选择", padding="5")
table_frame.grid(row=1, column=0, columnspan=2, sticky=(tk.W, tk.E), pady=5)
ttk.Label(table_frame, text="选择表:").grid(row=0, column=0, sticky=tk.W)
self.table_combo = ttk.Combobox(
table_frame, textvariable=self.selected_table, state="readonly"
)
self.table_combo.grid(row=0, column=1, sticky=(tk.W, tk.E), padx=5)
ttk.Button(table_frame, text="加载数据", command=self.load_table_data).grid(
row=0, column=2, padx=5
)
# 数据显示部分
data_frame = ttk.LabelFrame(main_frame, text="表数据", padding="5")
data_frame.grid(
row=2, column=0, columnspan=2, sticky=(tk.W, tk.E, tk.N, tk.S), pady=5
)
# 创建Treeview显示数据
columns = ("#0",)
self.tree = ttk.Treeview(data_frame, columns=columns, show="headings")
vsb = ttk.Scrollbar(data_frame, orient="vertical", command=self.tree.yview)
hsb = ttk.Scrollbar(data_frame, orient="horizontal", command=self.tree.xview)
self.tree.configure(yscrollcommand=vsb.set, xscrollcommand=hsb.set)
self.tree.grid(row=0, column=0, sticky=(tk.W, tk.E, tk.N, tk.S))
vsb.grid(row=0, column=1, sticky=(tk.N, tk.S))
hsb.grid(row=1, column=0, sticky=(tk.W, tk.E))
# 按钮部分
button_frame = ttk.Frame(main_frame)
button_frame.grid(row=3, column=0, columnspan=2, pady=10)
ttk.Button(button_frame, text="导出为JSON", command=self.export_to_json).pack(
side=tk.LEFT, padx=5
)
ttk.Button(button_frame, text="清空", command=self.clear_all).pack(
side=tk.LEFT, padx=5
)
ttk.Button(button_frame, text="退出", command=self.root.quit).pack(
side=tk.LEFT, padx=5
)
# 配置权重
self.root.columnconfigure(0, weight=1)
self.root.rowconfigure(0, weight=1)
main_frame.columnconfigure(0, weight=1)
main_frame.rowconfigure(2, weight=1)
data_frame.columnconfigure(0, weight=1)
data_frame.rowconfigure(0, weight=1)
def browse_db(self):
file_path = filedialog.askopenfilename(
title="选择SQLite数据库文件",
filetypes=[
("SQLite数据库", "*.db *.sqlite *.sqlite3"),
("所有文件", "*.*"),
],
)
if file_path:
self.db_path.set(file_path)
def connect_db(self):
if not self.db_path.get():
messagebox.showerror("错误", "请先选择数据库文件")
return
try:
conn = sqlite3.connect(self.db_path.get())
cursor = conn.cursor()
# 获取所有表名
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
self.tables = [table[0] for table in cursor.fetchall()]
self.table_combo["values"] = self.tables
if self.tables:
self.selected_table.set(self.tables[0])
conn.close()
messagebox.showinfo(
"成功", f"成功连接到数据库，找到 {len(self.tables)} 个表"
)
except sqlite3.Error as e:
messagebox.showerror("数据库错误", f"连接数据库时出错: {str(e)}")
def load_table_data(self):
if not self.selected_table.get():
messagebox.showerror("错误", "请先选择表")
return
try:
conn = sqlite3.connect(self.db_path.get())
conn.row_factory = sqlite3.Row # 这样可以使用列名访问数据
cursor = conn.cursor()
# 获取表数据
cursor.execute(f"SELECT * FROM {self.selected_table.get()}")
rows = cursor.fetchall()
# 清空Treeview
for item in self.tree.get_children():
self.tree.delete(item)
# 设置列
columns = [description[0] for description in cursor.description]
self.tree["columns"] = columns
for col in columns:
self.tree.heading(col, text=col)
self.tree.column(col, width=100, minwidth=50)
# 添加数据
for row in rows:
self.tree.insert("", "end", values=tuple(row))
conn.close()
except sqlite3.Error as e:
messagebox.showerror("数据库错误", f"加载数据时出错: {str(e)}")
def export_to_json(self):
if not self.selected_table.get():
messagebox.showerror("错误", "请先选择表")
return
# 选择保存位置
file_path = filedialog.asksaveasfilename(
title="保存JSON文件",
defaultextension=".json",
filetypes=[("JSON文件", "*.json"), ("所有文件", "*.*")],
)
if not file_path:
return
try:
conn = sqlite3.connect(self.db_path.get())
conn.row_factory = sqlite3.Row # 使用列名访问数据
cursor = conn.cursor()
# 获取表数据
cursor.execute(f"SELECT * FROM {self.selected_table.get()}")
rows = cursor.fetchall()
# 转换为字典列表
result = [dict(row) for row in rows]
# 写入JSON文件
with open(file_path, "w", encoding="utf-8") as f:
json.dump(result, f, indent=4, ensure_ascii=False)
conn.close()
messagebox.showinfo("成功", f"数据已成功导出到 {file_path}")
except sqlite3.Error as e:
messagebox.showerror("数据库错误", f"导出数据时出错: {str(e)}")
except Exception as e:
messagebox.showerror("错误", f"保存文件时出错: {str(e)}")
def clear_all(self):
self.db_path.set("")
self.tables = []
self.table_combo["values"] = []
self.selected_table.set("")
for item in self.tree.get_children():
self.tree.delete(item)
self.tree["columns"] = ("#0",)
if __name__ == "__main__":
root = tk.Tk()
app = SQLiteToJSONConverter(root)
root.mainloop()