diff --git a/dsRagAnything/Tools/T2_raganything_example.py b/dsRagAnything/Tools/T2_raganything_example.py new file mode 100644 index 00000000..874da535 --- /dev/null +++ b/dsRagAnything/Tools/T2_raganything_example.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +""" +Example script demonstrating the integration of MinerU parser with RAGAnything + +This example shows how to: +1. Process parsed documents with RAGAnything +2. Perform multimodal queries on the processed documents +3. Handle different types of content (text, images, tables) +""" + +import os +import argparse +import asyncio +import logging +import logging.config +from pathlib import Path + +# Add project root directory to Python path +import sys + +sys.path.append(str(Path(__file__).parent.parent)) + +from lightrag.llm.openai import openai_complete_if_cache, openai_embed +from lightrag.utils import EmbeddingFunc, logger, set_verbose_debug +from raganything import RAGAnything, RAGAnythingConfig + + +def configure_logging(): + """Configure logging for the application""" + # Get log directory path from environment variable or use current directory + log_dir = os.getenv("LOG_DIR", os.getcwd()) + log_file_path = os.path.abspath(os.path.join(log_dir, "raganything_example.log")) + + print(f"\nRAGAnything example log file: {log_file_path}\n") + os.makedirs(os.path.dirname(log_dir), exist_ok=True) + + # Get log file max size and backup count from environment variables + log_max_bytes = int(os.getenv("LOG_MAX_BYTES", 10485760)) # Default 10MB + log_backup_count = int(os.getenv("LOG_BACKUP_COUNT", 5)) # Default 5 backups + + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": "%(levelname)s: %(message)s", + }, + "detailed": { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", + }, + }, + "handlers": { + "console": { + "formatter": "default", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr", + }, + "file": { + "formatter": "detailed", + "class": "logging.handlers.RotatingFileHandler", + "filename": log_file_path, + "maxBytes": log_max_bytes, + "backupCount": log_backup_count, + "encoding": "utf-8", + }, + }, + "loggers": { + "lightrag": { + "handlers": ["console", "file"], + "level": "INFO", + "propagate": False, + }, + }, + } + ) + + # Set the logger level to INFO + logger.setLevel(logging.INFO) + # Enable verbose debug if needed + set_verbose_debug(os.getenv("VERBOSE", "false").lower() == "true") + + +async def process_with_rag( + file_path: str, + output_dir: str, + working_dir: str = None, +): + """ + Process document with RAGAnything + + Args: + file_path: Path to the document + output_dir: Output directory for RAG results + api_key: OpenAI API key + base_url: Optional base URL for API + working_dir: Working directory for RAG storage + """ + try: + # Create RAGAnything configuration + config = RAGAnythingConfig( + working_dir=working_dir or "./rag_storage", + mineru_parse_method="auto", + enable_image_processing=True, + enable_table_processing=True, + enable_equation_processing=True, + ) + + # Define LLM model function + def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs): + return openai_complete_if_cache( + "deepseek-chat", + prompt, + system_prompt=system_prompt, + history_messages=history_messages, + api_key="sk-44ae895eeb614aa1a9c6460579e322f1", + base_url="https://api.deepseek.com", + **kwargs, + ) + + # Define vision model function for image processing + def vision_model_func( + prompt, system_prompt=None, history_messages=[], image_data=None, **kwargs + ): + if image_data: + return openai_complete_if_cache( + "GLM-4.1V-9B-Thinking", + "", + system_prompt=None, + history_messages=[], + messages=[ + {"role": "system", "content": system_prompt} + if system_prompt + else None, + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_data}" + }, + }, + ], + } + if image_data + else {"role": "user", "content": prompt}, + ], + api_key="sk-pbqibyjwhrgmnlsmdygplahextfaclgnedetybccknxojlyl", + base_url='https://api.siliconflow.cn/v1/chat/completions', + **kwargs, + ) + else: + return llm_model_func(prompt, system_prompt, history_messages, **kwargs) + + # Define embedding function + embedding_func = EmbeddingFunc( + embedding_dim=3072, + max_token_size=8192, + func=lambda texts: openai_embed( + texts, + model="BAAI/bge-m3", + api_key="sk-pbqibyjwhrgmnlsmdygplahextfaclgnedetybccknxojlyl", + base_url="https://api.siliconflow.cn/v1/embeddings", + ), + ) + + # Initialize RAGAnything with new dataclass structure + rag = RAGAnything( + config=config, + llm_model_func=llm_model_func, + vision_model_func=vision_model_func, + embedding_func=embedding_func, + ) + + # Process document + await rag.process_document_complete( + file_path=file_path, output_dir=output_dir, parse_method="auto" + ) + + # Example queries + queries = [ + "What is the main content of the document?", + "Describe the images and figures in the document", + "Tell me about the experimental results and data tables", + ] + + logger.info("\nQuerying processed document:") + for query in queries: + logger.info(f"\nQuery: {query}") + result = await rag.query_with_multimodal(query, mode="hybrid") + logger.info(f"Answer: {result}") + + except Exception as e: + logger.error(f"Error processing with RAG: {str(e)}") + import traceback + + logger.error(traceback.format_exc()) + + +def main(): + file_path="../Txt/黄海的个人简历.txt" + output="../Txt/output" + working_dir="../Txt/working_dir" + # Process with RAG + asyncio.run( + process_with_rag( + file_path, output, working_dir + ) + ) + + +if __name__ == "__main__": + # Configure logging first + configure_logging() + + print("RAGAnything Example") + print("=" * 30) + print("Processing document with multimodal RAG pipeline") + print("=" * 30) + + main() diff --git a/dsRagAnything/Tools/T2_End_to_End.py b/dsRagAnything/Tools/T3_End_to_End.py similarity index 100% rename from dsRagAnything/Tools/T2_End_to_End.py rename to dsRagAnything/Tools/T3_End_to_End.py diff --git a/dsRagAnything/Tools/output/黄琬乔2023蓝桥杯省赛准考证/auto/黄琬乔2023蓝桥杯省赛准考证_origin.pdf b/dsRagAnything/Tools/output/黄琬乔2023蓝桥杯省赛准考证/auto/黄琬乔2023蓝桥杯省赛准考证_origin.pdf index 2f618958..4dc56c5a 100644 Binary files a/dsRagAnything/Tools/output/黄琬乔2023蓝桥杯省赛准考证/auto/黄琬乔2023蓝桥杯省赛准考证_origin.pdf and b/dsRagAnything/Tools/output/黄琬乔2023蓝桥杯省赛准考证/auto/黄琬乔2023蓝桥杯省赛准考证_origin.pdf differ diff --git a/dsRagAnything/Tools/rag_storage/kv_store_doc_status.json b/dsRagAnything/Tools/rag_storage/kv_store_doc_status.json index 04f1f7a2..46eefbbc 100644 --- a/dsRagAnything/Tools/rag_storage/kv_store_doc_status.json +++ b/dsRagAnything/Tools/rag_storage/kv_store_doc_status.json @@ -1,12 +1,12 @@ { "doc-c5b36c7bc30ea3197cf3de1efee747b5": { "status": "failed", - "error": "unsupported operand type(s) for ** or pow(): 'str' and 'dict'", + "error": "404 page not found", "content": "第十四届蓝桥杯大赛青少组省赛准考证\n\n比赛信息:\n\n注意事项\n\n一、参赛选手须在赛前按照《线上比赛操作指南》中的要求进行准备。(指南请自行下载)。\n\n二、比赛系统自下载准考证当日起开始公测,请参赛选手务必在赛前使用准考证上的账号密码提前登录系统进行练习测试。\n\n三、准考证可下载后,因设备原因或未测试环境造成不能参赛的情况,由参赛选手自行承担。\n\n四、电脑配置要求:必须有前置摄像头,麦克风,请尽量使用 2017 年以后购买的电脑(不包含微软 Surface 平板电脑),win10或以上版本,MAC OS10.9 或以上版本。\n\n五、准备一台带摄像头的移动设备(手机等)用于监考,安装“微信”软件(微信版本 8.0.30或以上版本)。\n\n六、提前在电脑下载谷歌浏览器最新版本(浏览器务必从官网下载 https://www.Google.cn/chrome/)。\n\n七、比赛当天晚于开始时间 30分钟登录的将被取消比赛资格。\n\n八、比赛采用多重防作弊措施,比赛过程中包括但不限于以下作弊行为:\n\n1.选手冒名顶替参赛; \n2.授权他人远程登录; \n3.使用分屏; \n4.不开“微考试助手”; \n5.离开“微考试助手”摄像头 5 分钟以上; \n6.故意遮挡摄像头或者关闭摄像头 5 分钟以上; \n7.故意关闭麦克风及摄像头行为; \n8.比赛环境中出现非参赛选手及任何与题目有关的声音; \n9.监控画面未按照比赛要求设置;\n\n10.比赛过程中出现缩小比赛平台并打开其他文件、资料、网页等;有上述行为的参赛选手,组委会有权降级或者取消参赛选手成绩。九、比赛时间不中断,允许准备草稿纸、笔以便答题时演算使用。\n\n十、选手做编程题的方式:\n\n1.Scratch 程序可以在网页直接编写(Scratch 3.0 版)。2.Python 和 $\\mathrm { C } { + } { + }$ 程序既可以在网页编写,也可以在本地 IDE编程环境编写,编写之后拷贝到网页“代码编写区域”。(参加比赛的考生应在电脑上自行安装对应编程科目的编程 IDE:Python3.7 以上, ${ \\mathrm { C } } { + } { + } 5 . 1 . 1$ 以上)3.EV3 和 Arduino 程序使用本地编程软件编写。将程序源文件保存到桌面,然后在比赛系统中上传。(EV3 编程软件采用乐高 Mindstorms 图形化编程环境,Arduino 统一使用 Mixly 0.997 及以上版本软件或 Arduino IDE1.8.5 版)4.EV3 器材为 45544 或 9898 核心套装。十一、《线上比赛操作指南》请扫描下方二维码查看。\n\n《线上比赛操作指南》\n\n十二、交卷后将无法继续作答,请确认不再继续答题后点击交卷。 \n十三、登录比赛系统前关闭电脑上所有软件和弹窗,以免造成卡顿和闪退情况。 \n十四、比赛中系统后台全程监控。 \n十五、比赛咨询电话 400-055-9099。 \n十六、上述内容解释权归组委会所有。", "content_summary": "第十四届蓝桥杯大赛青少组省赛准考证\n\n比赛信息:\n\n注意事项\n\n一、参赛选手须在赛前按照《线上比赛操作指南》中的要求进行准备。(指南请自行下载)。\n\n二、比赛系统自下载准考证当日起开始公测,请参赛选手务必在赛前使用准考证上的账号密码提前登录系统进行练习测试。\n\n三、准考证可下载后,因设备原因或未测试环境造成不能参赛的情况,由参赛选手自行承担。\n\n四、电脑配置要求:必须有前置摄像头,麦克风,请尽量使用 2017 年以后购买的电脑(不包含微软 Surface 平板电脑),win10或以上版本,MAC...", "content_length": 1271, "created_at": "2025-07-05T00:03:14.674003+00:00", - "updated_at": "2025-07-05T00:03:15.263263+00:00", + "updated_at": "2025-07-05T00:08:21.564739+00:00", "file_path": "黄琬乔2023蓝桥杯省赛准考证.pdf" } } \ No newline at end of file diff --git a/dsRagAnything/Txt/黄海的个人简历.txt b/dsRagAnything/Txt/黄海的个人简历.txt new file mode 100644 index 00000000..ec9817ea --- /dev/null +++ b/dsRagAnything/Txt/黄海的个人简历.txt @@ -0,0 +1,6 @@ +当前会话的使用者是黄海,男,汉族,2000-1-1出生于吉林省长春市 +毕业于东北师范大学计算教育专业,硕士毕业于北京大学计算系。获得美国麻省理工学院计算机与人工智能博士学位。 + +擅长AI人工智能技术研究,对于人工智能在军事领域应用有突出贡献。 + +数学是小学六年级水平,对于分数计算、约分存在一定不足,需要加强此方面的训练。 \ No newline at end of file