modify: 解决 message_item result 字段重复序列化问题
This commit is contained in:
parent
2f68ecac71
commit
80cde8aff3
@ -8,11 +8,11 @@ from ..core import core
|
|||||||
from ..constant import MessageStatus
|
from ..constant import MessageStatus
|
||||||
|
|
||||||
|
|
||||||
class QueueOperation():
|
class QueueOperation:
|
||||||
def __init__(self, shard_num: int = 4, queue_name: str = "default"):
|
def __init__(self, shard_num: int = 4, queue_name: str = "default"):
|
||||||
self.shard_num = shard_num
|
self.shard_num = shard_num
|
||||||
self.db_dir = os.path.join("cache", queue_name)
|
self.db_dir = os.path.join("cache", queue_name)
|
||||||
self.shard_connections = threading.local()
|
self.shard_connections = threading.local()
|
||||||
if not os.path.exists(self.db_dir):
|
if not os.path.exists(self.db_dir):
|
||||||
os.makedirs(self.db_dir)
|
os.makedirs(self.db_dir)
|
||||||
self.init_shards()
|
self.init_shards()
|
||||||
@ -103,9 +103,9 @@ class QueueOperation():
|
|||||||
conn.update_status(message_id, status)
|
conn.update_status(message_id, status)
|
||||||
|
|
||||||
# 更新消息结果
|
# 更新消息结果
|
||||||
def update_result(self, message_id: str, result: dict):
|
def update_result(self, message_id: str, result: str):
|
||||||
conn = self._get_shard_conn(self._get_shard_index(message_id))
|
conn = self._get_shard_conn(self._get_shard_index(message_id))
|
||||||
conn.update_result(message_id, json.dumps(result))
|
conn.update_result(message_id, result)
|
||||||
|
|
||||||
# 删除消息
|
# 删除消息
|
||||||
def delete_message(self, message_id: str):
|
def delete_message(self, message_id: str):
|
||||||
@ -121,4 +121,3 @@ class QueueOperation():
|
|||||||
def clean_expired_messages(self, shard_index: int):
|
def clean_expired_messages(self, shard_index: int):
|
||||||
conn = self._get_shard_conn(shard_index)
|
conn = self._get_shard_conn(shard_index)
|
||||||
conn.clean_expired_messages()
|
conn.clean_expired_messages()
|
||||||
|
|
||||||
@ -30,11 +30,11 @@ class TaskScheduler:
|
|||||||
task_cycle = TaskCycle(message, task_function)
|
task_cycle = TaskCycle(message, task_function)
|
||||||
task_cycle.run()
|
task_cycle.run()
|
||||||
|
|
||||||
message.status = task_cycle.get_task_status() # type: ignore
|
message.status = task_cycle.get_task_status() # type: ignore
|
||||||
if message.status == MessageStatus.FAILED:
|
if message.status == MessageStatus.FAILED:
|
||||||
message.result = {"error": task_cycle.get_task_error()}
|
message.result = {"error": task_cycle.get_task_error()}
|
||||||
else:
|
else:
|
||||||
message.result = task_cycle.get_task_result() # type: ignore
|
message.result = task_cycle.get_task_result() # type: ignore
|
||||||
message.updatetime = datetime.now()
|
message.updatetime = datetime.now()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
message.status = MessageStatus.FAILED
|
message.status = MessageStatus.FAILED
|
||||||
@ -55,13 +55,15 @@ class TaskScheduler:
|
|||||||
"""单一轮询线程,并行执行任务"""
|
"""单一轮询线程,并行执行任务"""
|
||||||
while self.is_running:
|
while self.is_running:
|
||||||
try:
|
try:
|
||||||
message_list = self.queue_operation.dequeue(size=self.task_thread_num * 2)
|
message_list = self.queue_operation.dequeue(
|
||||||
|
size=self.task_thread_num * 2
|
||||||
|
)
|
||||||
if message_list:
|
if message_list:
|
||||||
# 并行处理所有获取到的消息
|
# 并行处理所有获取到的消息
|
||||||
for message in message_list:
|
for message in message_list:
|
||||||
self.executor.submit(
|
self.executor.submit(
|
||||||
lambda m: self._update_result(self._process_message(m)),
|
lambda m: self._update_result(self._process_message(m)),
|
||||||
MessageItem.from_dict(message)
|
MessageItem.from_dict(message),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
time.sleep(0.1) # 适当休眠
|
time.sleep(0.1) # 适当休眠
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
from queue_sqlite.model import MessageItem
|
from queue_sqlite.model import MessageItem
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
class TestMessageItem:
|
class TestMessageItem:
|
||||||
@ -15,4 +16,46 @@ class TestMessageItem:
|
|||||||
@classmethod
|
@classmethod
|
||||||
def test_from_dict(cls):
|
def test_from_dict(cls):
|
||||||
messageItem = MessageItem.from_dict(cls.messageItem.to_dict())
|
messageItem = MessageItem.from_dict(cls.messageItem.to_dict())
|
||||||
|
data = {
|
||||||
|
"id": "ddb66277-503c-4921-8e7f-5091eace49e3",
|
||||||
|
"type": "task",
|
||||||
|
"status": 2,
|
||||||
|
"content": '{"num": 1}',
|
||||||
|
"createtime": "2025-09-12T18:41:08.221531",
|
||||||
|
"updatetime": "2025-09-12T18:41:08.221531",
|
||||||
|
"result": '"{\\"message\\": \\"\\\\u6d4b\\\\u8bd5\\\\u6210\\\\u529f\\"}"',
|
||||||
|
"priority": 1,
|
||||||
|
"source": "client",
|
||||||
|
"destination": "test",
|
||||||
|
"retry_count": 0,
|
||||||
|
"expire_time": "null",
|
||||||
|
"tags": "null",
|
||||||
|
"metadata": "{}",
|
||||||
|
}
|
||||||
print(messageItem.to_dict())
|
print(messageItem.to_dict())
|
||||||
|
print(MessageItem.from_dict(data).to_dict())
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def test_from_json(cls):
|
||||||
|
json_str = """
|
||||||
|
{
|
||||||
|
"content": {
|
||||||
|
"num": 1
|
||||||
|
},
|
||||||
|
"createtime": "2025-09-12T16:42:50.663248",
|
||||||
|
"destination": "test",
|
||||||
|
"expire_time": null,
|
||||||
|
"id": "b83c3d72-0b06-4c34-ab4f-32d696aa3875",
|
||||||
|
"metadata": {},
|
||||||
|
"priority": 1,
|
||||||
|
"result": "{\"message\": \"\\u6d4b\\u8bd5\\u6210\\u529f\"}",
|
||||||
|
"retry_count": 0,
|
||||||
|
"source": "client",
|
||||||
|
"status": 2,
|
||||||
|
"tags": null,
|
||||||
|
"type": "task",
|
||||||
|
"updatetime": "2025-09-12T16:42:50.663248"
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
messageItem = MessageItem.from_dict(json.loads(json_str))
|
||||||
|
print(messageItem.to_json())
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user