#!/usr/bin/env python """ Example script demonstrating the integration of MinerU parser with RAGAnything This example shows how to: 1. Process parsed documents with RAGAnything 2. Perform multimodal queries on the processed documents 3. Handle different types of content (text, images, tables) """ import os import argparse import asyncio import logging import logging.config from pathlib import Path # Add project root directory to Python path import sys sys.path.append(str(Path(__file__).parent.parent)) from lightrag.llm.openai import openai_complete_if_cache, openai_embed from lightrag.utils import EmbeddingFunc, logger, set_verbose_debug from raganything import RAGAnything, RAGAnythingConfig def configure_logging(): """Configure logging for the application""" # Get log directory path from environment variable or use current directory log_dir = os.getenv("LOG_DIR", os.getcwd()) log_file_path = os.path.abspath(os.path.join(log_dir, "raganything_example.log")) print(f"\nRAGAnything example log file: {log_file_path}\n") os.makedirs(os.path.dirname(log_dir), exist_ok=True) # Get log file max size and backup count from environment variables log_max_bytes = int(os.getenv("LOG_MAX_BYTES", 10485760)) # Default 10MB log_backup_count = int(os.getenv("LOG_BACKUP_COUNT", 5)) # Default 5 backups logging.config.dictConfig( { "version": 1, "disable_existing_loggers": False, "formatters": { "default": { "format": "%(levelname)s: %(message)s", }, "detailed": { "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", }, }, "handlers": { "console": { "formatter": "default", "class": "logging.StreamHandler", "stream": "ext://sys.stderr", }, "file": { "formatter": "detailed", "class": "logging.handlers.RotatingFileHandler", "filename": log_file_path, "maxBytes": log_max_bytes, "backupCount": log_backup_count, "encoding": "utf-8", }, }, "loggers": { "lightrag": { "handlers": ["console", "file"], "level": "INFO", "propagate": False, }, }, } ) # Set the logger level to INFO logger.setLevel(logging.INFO) # Enable verbose debug if needed set_verbose_debug(os.getenv("VERBOSE", "false").lower() == "true") async def process_with_rag( file_path: str, output_dir: str, working_dir: str = None, ): """ Process document with RAGAnything Args: file_path: Path to the document output_dir: Output directory for RAG results api_key: OpenAI API key base_url: Optional base URL for API working_dir: Working directory for RAG storage """ try: # Create RAGAnything configuration config = RAGAnythingConfig( working_dir=working_dir or "./rag_storage", mineru_parse_method="auto", enable_image_processing=True, enable_table_processing=True, enable_equation_processing=True, ) # Define LLM model function def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs): return openai_complete_if_cache( "deepseek-chat", prompt, system_prompt=system_prompt, history_messages=history_messages, api_key="sk-44ae895eeb614aa1a9c6460579e322f1", base_url="https://api.deepseek.com", **kwargs, ) # Define vision model function for image processing def vision_model_func( prompt, system_prompt=None, history_messages=[], image_data=None, **kwargs ): if image_data: return openai_complete_if_cache( "GLM-4.1V-9B-Thinking", "", system_prompt=None, history_messages=[], messages=[ {"role": "system", "content": system_prompt} if system_prompt else None, { "role": "user", "content": [ {"type": "text", "text": prompt}, { "type": "image_url", "image_url": { "url": f"data:image/jpeg;base64,{image_data}" }, }, ], } if image_data else {"role": "user", "content": prompt}, ], api_key="sk-pbqibyjwhrgmnlsmdygplahextfaclgnedetybccknxojlyl", base_url='https://api.siliconflow.cn/v1/chat/completions', **kwargs, ) else: return llm_model_func(prompt, system_prompt, history_messages, **kwargs) # Define embedding function embedding_func = EmbeddingFunc( embedding_dim=3072, max_token_size=8192, func=lambda texts: openai_embed( texts, model="BAAI/bge-m3", api_key="sk-pbqibyjwhrgmnlsmdygplahextfaclgnedetybccknxojlyl", base_url="https://api.siliconflow.cn/v1/embeddings", ), ) # Initialize RAGAnything with new dataclass structure rag = RAGAnything( config=config, llm_model_func=llm_model_func, vision_model_func=vision_model_func, embedding_func=embedding_func, ) # Process document await rag.process_document_complete( file_path=file_path, output_dir=output_dir, parse_method="auto" ) # Example queries queries = [ "What is the main content of the document?", "Describe the images and figures in the document", "Tell me about the experimental results and data tables", ] logger.info("\nQuerying processed document:") for query in queries: logger.info(f"\nQuery: {query}") result = await rag.query_with_multimodal(query, mode="hybrid") logger.info(f"Answer: {result}") except Exception as e: logger.error(f"Error processing with RAG: {str(e)}") import traceback logger.error(traceback.format_exc()) def main(): file_path="../Txt/黄海的个人简历.txt" output="../Txt/output" working_dir="../Txt/working_dir" # Process with RAG asyncio.run( process_with_rag( file_path, output, working_dir ) ) if __name__ == "__main__": # Configure logging first configure_logging() print("RAGAnything Example") print("=" * 30) print("Processing document with multimodal RAG pipeline") print("=" * 30) main()