diff --git a/.dockerignore b/.dockerignore index 8e0ed179..3f0066a9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,49 @@ +# Environment files .env +.env.local +.env.development +.env.production + +# Auth files .drive.json +*.json + +# Dependencies +node_modules/ +*/node_modules/ +**/node_modules/ + +# Python cache +__pycache__/ +*/__pycache__/ +**/__pycache__/ +*.pyc +*.pyo +*.pyd +.Python + +# Build outputs +build/ +dist/ +.next/ +out/ + +# Development files +.git/ +.gitignore +README.md +*.md +.vscode/ +.idea/ + +# Logs +*.log +logs/ + +# OS files +.DS_Store +Thumbs.db + +# Temporary files +tmp/ +temp/ diff --git a/.env.example b/.env.example index 865e1fa3..6bb49c99 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,24 @@ -# flow id from the the openrag flow json -FLOW_ID=1098eea1-6649-4e1d-aed1-b77249fb8dd0 +# make one like so https://docs.langflow.org/api-keys-and-authentication#langflow-secret-key +LANGFLOW_SECRET_KEY= + +# flow ids for chat and ingestion flows +LANGFLOW_CHAT_FLOW_ID=1098eea1-6649-4e1d-aed1-b77249fb8dd0 +LANGFLOW_INGEST_FLOW_ID=5488df7c-b93f-4f87-a446-b67028bc0813 NUDGES_FLOW_ID=ebc01d31-1976-46ce-a385-b0240327226c + # Set a strong admin password for OpenSearch; a bcrypt hash is generated at # container startup from this value. Do not commit real secrets. +# must match the hashed password in secureconfig, must change for secure deployment!!! OPENSEARCH_PASSWORD= -# make here https://console.cloud.google.com/apis/credentials + +# make here https://console.cloud.google.com/apis/credentials GOOGLE_OAUTH_CLIENT_ID= GOOGLE_OAUTH_CLIENT_SECRET= + # Azure app registration credentials for SharePoint/OneDrive MICROSOFT_GRAPH_OAUTH_CLIENT_ID= MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET= + # OPTIONAL: dns routable from google (etc.) to handle continous ingest (something like ngrok works). This enables continous ingestion WEBHOOK_BASE_URL= diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..fe76467a --- /dev/null +++ b/Makefile @@ -0,0 +1,210 @@ +# OpenRAG Development Makefile +# Provides easy commands for development workflow + +.PHONY: help dev dev-cpu dev-local infra stop clean build logs shell-backend shell-frontend install test backend frontend install-be install-fe build-be build-fe logs-be logs-fe logs-lf logs-os shell-be shell-lf shell-os restart status health db-reset flow-upload quick setup + +# Default target +help: + @echo "OpenRAG Development Commands" + @echo "" + @echo "Development:" + @echo " dev - Start full stack with GPU support (docker compose)" + @echo " dev-cpu - Start full stack with CPU only (docker compose)" + @echo " dev-local - Start infrastructure only, run backend/frontend locally" + @echo " infra - Start infrastructure services only (alias for dev-local)" + @echo " stop - Stop all containers" + @echo " restart - Restart all containers" + @echo "" + @echo "Local Development:" + @echo " backend - Run backend locally (requires infrastructure)" + @echo " frontend - Run frontend locally" + @echo " install - Install all dependencies" + @echo " install-be - Install backend dependencies (uv)" + @echo " install-fe - Install frontend dependencies (npm)" + @echo "" + @echo "Utilities:" + @echo " build - Build all Docker images" + @echo " clean - Stop containers and remove volumes" + @echo " logs - Show logs from all containers" + @echo " logs-be - Show backend container logs" + @echo " logs-lf - Show langflow container logs" + @echo " shell-be - Shell into backend container" + @echo " shell-lf - Shell into langflow container" + @echo "" + @echo "Testing:" + @echo " test - Run backend tests" + @echo " lint - Run linting checks" + @echo "" + +# Development environments +dev: + @echo "🚀 Starting OpenRAG with GPU support..." + docker-compose up -d + @echo "✅ Services started!" + @echo " Backend: http://localhost:8000" + @echo " Frontend: http://localhost:3000" + @echo " Langflow: http://localhost:7860" + @echo " OpenSearch: http://localhost:9200" + @echo " Dashboards: http://localhost:5601" + +dev-cpu: + @echo "🚀 Starting OpenRAG with CPU only..." + docker-compose -f docker-compose-cpu.yml up -d + @echo "✅ Services started!" + @echo " Backend: http://localhost:8000" + @echo " Frontend: http://localhost:3000" + @echo " Langflow: http://localhost:7860" + @echo " OpenSearch: http://localhost:9200" + @echo " Dashboards: http://localhost:5601" + +dev-local: + @echo "🔧 Starting infrastructure only (for local development)..." + docker-compose up -d opensearch dashboards langflow + @echo "✅ Infrastructure started!" + @echo " Langflow: http://localhost:7860" + @echo " OpenSearch: http://localhost:9200" + @echo " Dashboards: http://localhost:5601" + @echo "" + @echo "Now run 'make backend' and 'make frontend' in separate terminals" + +infra: + @echo "🔧 Starting infrastructure services only..." + docker-compose up -d opensearch dashboards langflow + @echo "✅ Infrastructure services started!" + @echo " Langflow: http://localhost:7860" + @echo " OpenSearch: http://localhost:9200" + @echo " Dashboards: http://localhost:5601" + +# Container management +stop: + @echo "🛑 Stopping all containers..." + docker-compose down + docker-compose -f docker-compose-cpu.yml down 2>/dev/null || true + +restart: stop dev + +clean: stop + @echo "🧹 Cleaning up containers and volumes..." + docker-compose down -v --remove-orphans + docker-compose -f docker-compose-cpu.yml down -v --remove-orphans 2>/dev/null || true + docker system prune -f + +# Local development +backend: + @echo "🐍 Starting backend locally..." + @if [ ! -f .env ]; then echo "⚠️ .env file not found. Copy .env.example to .env first"; exit 1; fi + uv run python src/main.py + +frontend: + @echo "⚛️ Starting frontend locally..." + @if [ ! -d "frontend/node_modules" ]; then echo "📦 Installing frontend dependencies first..."; cd frontend && npm install; fi + cd frontend && npx next dev + +# Installation +install: install-be install-fe + @echo "✅ All dependencies installed!" + +install-be: + @echo "📦 Installing backend dependencies..." + uv sync + +install-fe: + @echo "📦 Installing frontend dependencies..." + cd frontend && npm install + +# Building +build: + @echo "🔨 Building Docker images..." + docker-compose build + +build-be: + @echo "🔨 Building backend image..." + docker build -t openrag-backend -f Dockerfile.backend . + +build-fe: + @echo "🔨 Building frontend image..." + docker build -t openrag-frontend -f Dockerfile.frontend . + +# Logging and debugging +logs: + @echo "📋 Showing all container logs..." + docker-compose logs -f + +logs-be: + @echo "📋 Showing backend logs..." + docker-compose logs -f openrag-backend + +logs-fe: + @echo "📋 Showing frontend logs..." + docker-compose logs -f openrag-frontend + +logs-lf: + @echo "📋 Showing langflow logs..." + docker-compose logs -f langflow + +logs-os: + @echo "📋 Showing opensearch logs..." + docker-compose logs -f opensearch + +# Shell access +shell-be: + @echo "🐚 Opening shell in backend container..." + docker-compose exec openrag-backend /bin/bash + +shell-lf: + @echo "🐚 Opening shell in langflow container..." + docker-compose exec langflow /bin/bash + +shell-os: + @echo "🐚 Opening shell in opensearch container..." + docker-compose exec opensearch /bin/bash + +# Testing and quality +test: + @echo "🧪 Running backend tests..." + uv run pytest + +lint: + @echo "🔍 Running linting checks..." + cd frontend && npm run lint + @echo "Frontend linting complete" + +# Service status +status: + @echo "📊 Container status:" + @docker-compose ps 2>/dev/null || echo "No containers running" + +health: + @echo "🏥 Health check:" + @echo "Backend: $$(curl -s http://localhost:8000/health 2>/dev/null || echo 'Not responding')" + @echo "Langflow: $$(curl -s http://localhost:7860/health 2>/dev/null || echo 'Not responding')" + @echo "OpenSearch: $$(curl -s -k -u admin:$(shell grep OPENSEARCH_PASSWORD .env | cut -d= -f2) https://localhost:9200 2>/dev/null | jq -r .tagline 2>/dev/null || echo 'Not responding')" + +# Database operations +db-reset: + @echo "🗄️ Resetting OpenSearch indices..." + curl -X DELETE "http://localhost:9200/documents" -u admin:$$(grep OPENSEARCH_PASSWORD .env | cut -d= -f2) || true + curl -X DELETE "http://localhost:9200/knowledge_filters" -u admin:$$(grep OPENSEARCH_PASSWORD .env | cut -d= -f2) || true + @echo "Indices reset. Restart backend to recreate." + +# Flow management +flow-upload: + @echo "📁 Uploading flow to Langflow..." + @if [ -z "$(FLOW_FILE)" ]; then echo "Usage: make flow-upload FLOW_FILE=path/to/flow.json"; exit 1; fi + curl -X POST "http://localhost:7860/api/v1/flows" \ + -H "Content-Type: application/json" \ + -d @$(FLOW_FILE) + +# Quick development shortcuts +quick: dev-local + @echo "🚀 Quick start: infrastructure running!" + @echo "Run these in separate terminals:" + @echo " make backend" + @echo " make frontend" + +# Environment setup +setup: + @echo "⚙️ Setting up development environment..." + @if [ ! -f .env ]; then cp .env.example .env && echo "📝 Created .env from template"; fi + @$(MAKE) install + @echo "✅ Setup complete! Run 'make dev' to start." \ No newline at end of file diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index 0414a523..132cb233 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -15,10 +15,10 @@ services: bash -c " # Start OpenSearch in background /usr/share/opensearch/opensearch-docker-entrypoint.sh opensearch & - + # Wait a bit for OpenSearch to start, then apply security config sleep 10 && /usr/share/opensearch/setup-security.sh & - + # Wait for background processes wait " @@ -53,7 +53,8 @@ services: - LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - - FLOW_ID=${FLOW_ID} + - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} + - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - OPENSEARCH_PORT=9200 - OPENSEARCH_USERNAME=admin @@ -98,7 +99,8 @@ services: - LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY} - JWT="dummy" - OPENRAG-QUERY-FILTER="{}" - - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER + - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} + - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD - LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} diff --git a/docker-compose.yml b/docker-compose.yml index 23499cb9..62bb8d2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,10 +15,10 @@ services: bash -c " # Start OpenSearch in background /usr/share/opensearch/opensearch-docker-entrypoint.sh opensearch & - + # Wait a bit for OpenSearch to start, then apply security config sleep 10 && /usr/share/opensearch/setup-security.sh & - + # Wait for background processes wait " @@ -52,7 +52,8 @@ services: - LANGFLOW_PUBLIC_URL=${LANGFLOW_PUBLIC_URL} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - - FLOW_ID=${FLOW_ID} + - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} + - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - OPENSEARCH_PORT=9200 - OPENSEARCH_USERNAME=admin @@ -98,7 +99,8 @@ services: - LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY} - JWT="dummy" - OPENRAG-QUERY-FILTER="{}" - - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER + - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} + - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD - LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json new file mode 100644 index 00000000..dd039a37 --- /dev/null +++ b/flows/ingestion_flow.json @@ -0,0 +1,2032 @@ +{ + "data": { + "edges": [ + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "SplitText", + "id": "SplitText-QIKhg", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "ingest_data", + "id": "OpenSearchHybrid-Ve6bS", + "inputTypes": [ + "Data", + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__SplitText-QIKhg{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "SplitText-QIKhg", + "sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "OpenSearchHybrid-Ve6bS", + "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "OpenAIEmbeddings", + "id": "OpenAIEmbeddings-joRJ6", + "name": "embeddings", + "output_types": [ + "Embeddings" + ] + }, + "targetHandle": { + "fieldName": "embedding", + "id": "OpenSearchHybrid-Ve6bS", + "inputTypes": [ + "Embeddings" + ], + "type": "other" + } + }, + "id": "xy-edge__OpenAIEmbeddings-joRJ6{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}", + "selected": false, + "source": "OpenAIEmbeddings-joRJ6", + "sourceHandle": "{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}", + "target": "OpenSearchHybrid-Ve6bS", + "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "File", + "id": "File-PSU37", + "name": "message", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "data_inputs", + "id": "SplitText-QIKhg", + "inputTypes": [ + "Data", + "DataFrame", + "Message" + ], + "type": "other" + } + }, + "id": "xy-edge__File-PSU37{œdataTypeœ:œFileœ,œidœ:œFile-PSU37œ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}-SplitText-QIKhg{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", + "selected": false, + "source": "File-PSU37", + "sourceHandle": "{œdataTypeœ:œFileœ,œidœ:œFile-PSU37œ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}", + "target": "SplitText-QIKhg", + "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" + } + ], + "nodes": [ + { + "data": { + "description": "Split text into chunks based on specified criteria.", + "display_name": "Split Text", + "id": "SplitText-QIKhg", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Split text into chunks based on specified criteria.", + "display_name": "Split Text", + "documentation": "https://docs.langflow.org/components-processing#split-text", + "edited": true, + "field_order": [ + "data_inputs", + "chunk_overlap", + "chunk_size", + "separator", + "text_key", + "keep_separator" + ], + "frozen": false, + "icon": "scissors-line-dashed", + "legacy": false, + "lf_version": "1.5.0.post2", + "metadata": { + "code_hash": "65a90e1f4fe6", + "dependencies": { + "dependencies": [ + { + "name": "langchain_text_splitters", + "version": "0.3.9" + }, + { + "name": "langflow", + "version": "1.5.0.post2" + } + ], + "total_dependencies": 2 + }, + "module": "custom_components.split_text" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Chunks", + "group_outputs": false, + "hidden": null, + "method": "split_text", + "name": "dataframe", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "chunk_overlap": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Chunk Overlap", + "dynamic": false, + "info": "Number of characters to overlap between chunks.", + "list": false, + "list_add_label": "Add More", + "name": "chunk_overlap", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 200 + }, + "chunk_size": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Chunk Size", + "dynamic": false, + "info": "The maximum length of each chunk. Text is first split by separator, then chunks are merged up to this size. Individual splits larger than this won't be further divided.", + "list": false, + "list_add_label": "Add More", + "name": "chunk_size", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 1000 + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from langchain_text_splitters import CharacterTextSplitter\n\nfrom langflow.custom.custom_component.component import Component\nfrom langflow.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output\nfrom langflow.schema.data import Data\nfrom langflow.schema.dataframe import DataFrame\nfrom langflow.schema.message import Message\nfrom langflow.utils.util import unescape_string\n\n\nclass SplitTextComponent(Component):\n display_name: str = \"Split Text\"\n description: str = \"Split text into chunks based on specified criteria.\"\n documentation: str = \"https://docs.langflow.org/components-processing#split-text\"\n icon = \"scissors-line-dashed\"\n name = \"SplitText\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Input\",\n info=\"The data with texts to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n IntInput(\n name=\"chunk_overlap\",\n display_name=\"Chunk Overlap\",\n info=\"Number of characters to overlap between chunks.\",\n value=200,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n info=(\n \"The maximum length of each chunk. Text is first split by separator, \"\n \"then chunks are merged up to this size. \"\n \"Individual splits larger than this won't be further divided.\"\n ),\n value=1000,\n ),\n MessageTextInput(\n name=\"separator\",\n display_name=\"Separator\",\n info=(\n \"The character to split on. Use \\\\n for newline. \"\n \"Examples: \\\\n\\\\n for paragraphs, \\\\n for lines, . for sentences\"\n ),\n value=\"\\n\",\n ),\n MessageTextInput(\n name=\"text_key\",\n display_name=\"Text Key\",\n info=\"The key to use for the text column.\",\n value=\"text\",\n advanced=True,\n ),\n DropdownInput(\n name=\"keep_separator\",\n display_name=\"Keep Separator\",\n info=\"Whether to keep the separator in the output chunks and where to place it.\",\n options=[\"False\", \"True\", \"Start\", \"End\"],\n value=\"False\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Chunks\", name=\"dataframe\", method=\"split_text\"),\n ]\n\n def _docs_to_data(self, docs) -> list[Data]:\n data_list = [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n return data_list\n\n def _fix_separator(self, separator: str) -> str:\n \"\"\"Fix common separator issues and convert to proper format.\"\"\"\n if separator == \"/n\":\n return \"\\n\"\n if separator == \"/t\":\n return \"\\t\"\n return separator\n\n def split_text_base(self):\n separator = self._fix_separator(self.separator)\n separator = unescape_string(separator)\n\n if isinstance(self.data_inputs, DataFrame):\n if not len(self.data_inputs):\n msg = \"DataFrame is empty\"\n raise TypeError(msg)\n\n self.data_inputs.text_key = self.text_key\n try:\n documents = self.data_inputs.to_lc_documents()\n except Exception as e:\n msg = f\"Error converting DataFrame to documents: {e}\"\n raise TypeError(msg) from e\n elif isinstance(self.data_inputs, Message):\n self.data_inputs = [self.data_inputs.to_data()]\n return self.split_text_base()\n else:\n if not self.data_inputs:\n msg = \"No data inputs provided\"\n raise TypeError(msg)\n\n documents = []\n if isinstance(self.data_inputs, Data):\n self.data_inputs.text_key = self.text_key\n documents = [self.data_inputs.to_lc_document()]\n else:\n try:\n documents = [input_.to_lc_document() for input_ in self.data_inputs if isinstance(input_, Data)]\n if not documents:\n msg = f\"No valid Data inputs found in {type(self.data_inputs)}\"\n raise TypeError(msg)\n except AttributeError as e:\n msg = f\"Invalid input type in collection: {e}\"\n raise TypeError(msg) from e\n try:\n # Convert string 'False'/'True' to boolean\n keep_sep = self.keep_separator\n if isinstance(keep_sep, str):\n if keep_sep.lower() == \"false\":\n keep_sep = False\n elif keep_sep.lower() == \"true\":\n keep_sep = True\n # 'start' and 'end' are kept as strings\n self.log(documents)\n splitter = CharacterTextSplitter(\n chunk_overlap=self.chunk_overlap,\n chunk_size=self.chunk_size,\n separator=separator,\n keep_separator=keep_sep,\n )\n return splitter.split_documents(documents)\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n def split_text(self) -> DataFrame:\n return DataFrame(self._docs_to_data(self.split_text_base()))\n" + }, + "data_inputs": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Input", + "dynamic": false, + "info": "The data with texts to split in chunks.", + "input_types": [ + "Data", + "DataFrame", + "Message" + ], + "list": false, + "list_add_label": "Add More", + "name": "data_inputs", + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "keep_separator": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Keep Separator", + "dynamic": false, + "info": "Whether to keep the separator in the output chunks and where to place it.", + "name": "keep_separator", + "options": [ + "False", + "True", + "Start", + "End" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "False" + }, + "separator": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Separator", + "dynamic": false, + "info": "The character to split on. Use \\n for newline. Examples: \\n\\n for paragraphs, \\n for lines, . for sentences", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "separator", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "\n" + }, + "text_key": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Text Key", + "dynamic": false, + "info": "The key to use for the text column.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "text_key", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "text" + } + }, + "tool_mode": false + }, + "selected_output": "chunks", + "type": "SplitText" + }, + "dragging": false, + "height": 475, + "id": "SplitText-QIKhg", + "measured": { + "height": 475, + "width": 320 + }, + "position": { + "x": 1729.1788373023007, + "y": 1330.8003441546418 + }, + "positionAbsolute": { + "x": 1683.4543896546102, + "y": 1350.7871623588553 + }, + "selected": false, + "type": "genericNode", + "width": 320 + }, + { + "data": { + "id": "OpenAIEmbeddings-joRJ6", + "node": { + "base_classes": [ + "Embeddings" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Generate embeddings using OpenAI models.", + "display_name": "OpenAI Embeddings", + "documentation": "", + "edited": false, + "field_order": [ + "default_headers", + "default_query", + "chunk_size", + "client", + "deployment", + "embedding_ctx_length", + "max_retries", + "model", + "model_kwargs", + "openai_api_key", + "openai_api_base", + "openai_api_type", + "openai_api_version", + "openai_organization", + "openai_proxy", + "request_timeout", + "show_progress_bar", + "skip_empty", + "tiktoken_model_name", + "tiktoken_enable", + "dimensions" + ], + "frozen": false, + "icon": "OpenAI", + "legacy": false, + "lf_version": "1.5.0.post2", + "metadata": { + "code_hash": "2691dee277c9", + "dependencies": { + "dependencies": [ + { + "name": "langchain_openai", + "version": "0.3.23" + }, + { + "name": "langflow", + "version": "1.5.0.post2" + } + ], + "total_dependencies": 2 + }, + "module": "langflow.components.openai.openai.OpenAIEmbeddingsComponent" + }, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Embedding Model", + "group_outputs": false, + "method": "build_embeddings", + "name": "embeddings", + "selected": "Embeddings", + "tool_mode": true, + "types": [ + "Embeddings" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "chunk_size": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Chunk Size", + "dynamic": false, + "info": "", + "list": false, + "name": "chunk_size", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "int", + "value": 1000 + }, + "client": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Client", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "client", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from langchain_openai import OpenAIEmbeddings\n\nfrom langflow.base.embeddings.model import LCEmbeddingsModel\nfrom langflow.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom langflow.field_typing import Embeddings\nfrom langflow.io import BoolInput, DictInput, DropdownInput, FloatInput, IntInput, MessageTextInput, SecretStrInput\n\n\nclass OpenAIEmbeddingsComponent(LCEmbeddingsModel):\n display_name = \"OpenAI Embeddings\"\n description = \"Generate embeddings using OpenAI models.\"\n icon = \"OpenAI\"\n name = \"OpenAIEmbeddings\"\n\n inputs = [\n DictInput(\n name=\"default_headers\",\n display_name=\"Default Headers\",\n advanced=True,\n info=\"Default headers to use for the API request.\",\n ),\n DictInput(\n name=\"default_query\",\n display_name=\"Default Query\",\n advanced=True,\n info=\"Default query parameters to use for the API request.\",\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n MessageTextInput(name=\"client\", display_name=\"Client\", advanced=True),\n MessageTextInput(name=\"deployment\", display_name=\"Deployment\", advanced=True),\n IntInput(name=\"embedding_ctx_length\", display_name=\"Embedding Context Length\", advanced=True, value=1536),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", value=3, advanced=True),\n DropdownInput(\n name=\"model\",\n display_name=\"Model\",\n advanced=False,\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=\"text-embedding-3-small\",\n ),\n DictInput(name=\"model_kwargs\", display_name=\"Model Kwargs\", advanced=True),\n SecretStrInput(name=\"openai_api_key\", display_name=\"OpenAI API Key\", value=\"OPENAI_API_KEY\", required=True),\n MessageTextInput(name=\"openai_api_base\", display_name=\"OpenAI API Base\", advanced=True),\n MessageTextInput(name=\"openai_api_type\", display_name=\"OpenAI API Type\", advanced=True),\n MessageTextInput(name=\"openai_api_version\", display_name=\"OpenAI API Version\", advanced=True),\n MessageTextInput(\n name=\"openai_organization\",\n display_name=\"OpenAI Organization\",\n advanced=True,\n ),\n MessageTextInput(name=\"openai_proxy\", display_name=\"OpenAI Proxy\", advanced=True),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n BoolInput(name=\"skip_empty\", display_name=\"Skip Empty\", advanced=True),\n MessageTextInput(\n name=\"tiktoken_model_name\",\n display_name=\"TikToken Model Name\",\n advanced=True,\n ),\n BoolInput(\n name=\"tiktoken_enable\",\n display_name=\"TikToken Enable\",\n advanced=True,\n value=True,\n info=\"If False, you must have transformers installed.\",\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n return OpenAIEmbeddings(\n client=self.client or None,\n model=self.model,\n dimensions=self.dimensions or None,\n deployment=self.deployment or None,\n api_version=self.openai_api_version or None,\n base_url=self.openai_api_base or None,\n openai_api_type=self.openai_api_type or None,\n openai_proxy=self.openai_proxy or None,\n embedding_ctx_length=self.embedding_ctx_length,\n api_key=self.openai_api_key or None,\n organization=self.openai_organization or None,\n allowed_special=\"all\",\n disallowed_special=\"all\",\n chunk_size=self.chunk_size,\n max_retries=self.max_retries,\n timeout=self.request_timeout or None,\n tiktoken_enabled=self.tiktoken_enable,\n tiktoken_model_name=self.tiktoken_model_name or None,\n show_progress_bar=self.show_progress_bar,\n model_kwargs=self.model_kwargs,\n skip_empty=self.skip_empty,\n default_headers=self.default_headers or None,\n default_query=self.default_query or None,\n )\n" + }, + "default_headers": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Default Headers", + "dynamic": false, + "info": "Default headers to use for the API request.", + "list": false, + "name": "default_headers", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_input": true, + "type": "dict", + "value": {} + }, + "default_query": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Default Query", + "dynamic": false, + "info": "Default query parameters to use for the API request.", + "list": false, + "name": "default_query", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_input": true, + "type": "dict", + "value": {} + }, + "deployment": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Deployment", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "deployment", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "dimensions": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Dimensions", + "dynamic": false, + "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", + "list": false, + "name": "dimensions", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "int", + "value": "" + }, + "embedding_ctx_length": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Embedding Context Length", + "dynamic": false, + "info": "", + "list": false, + "name": "embedding_ctx_length", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "int", + "value": 1536 + }, + "max_retries": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Max Retries", + "dynamic": false, + "info": "", + "list": false, + "name": "max_retries", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "int", + "value": 3 + }, + "model": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "display_name": "Model", + "dynamic": false, + "info": "", + "name": "model", + "options": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "text-embedding-3-small" + }, + "model_kwargs": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Model Kwargs", + "dynamic": false, + "info": "", + "list": false, + "name": "model_kwargs", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_input": true, + "type": "dict", + "value": {} + }, + "openai_api_base": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI API Base", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "openai_api_base", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_api_key": { + "_input_type": "SecretStrInput", + "advanced": false, + "display_name": "OpenAI API Key", + "dynamic": false, + "info": "", + "input_types": [], + "load_from_db": true, + "name": "openai_api_key", + "password": true, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "str", + "value": "OPENAI_API_KEY" + }, + "openai_api_type": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI API Type", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "openai_api_type", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_api_version": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI API Version", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "openai_api_version", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_organization": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI Organization", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "openai_organization", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_proxy": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI Proxy", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "openai_proxy", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "request_timeout": { + "_input_type": "FloatInput", + "advanced": true, + "display_name": "Request Timeout", + "dynamic": false, + "info": "", + "list": false, + "name": "request_timeout", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "float", + "value": "" + }, + "show_progress_bar": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Show Progress Bar", + "dynamic": false, + "info": "", + "list": false, + "name": "show_progress_bar", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + }, + "skip_empty": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Skip Empty", + "dynamic": false, + "info": "", + "list": false, + "name": "skip_empty", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + }, + "tiktoken_enable": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "TikToken Enable", + "dynamic": false, + "info": "If False, you must have transformers installed.", + "list": false, + "name": "tiktoken_enable", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "tiktoken_model_name": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "TikToken Model Name", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "load_from_db": false, + "name": "tiktoken_model_name", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + } + }, + "tool_mode": false + }, + "selected_output": "embeddings", + "type": "OpenAIEmbeddings" + }, + "dragging": false, + "height": 320, + "id": "OpenAIEmbeddings-joRJ6", + "measured": { + "height": 320, + "width": 320 + }, + "position": { + "x": 1704.8491676318172, + "y": 1879.144249471858 + }, + "positionAbsolute": { + "x": 1690.9220896443658, + "y": 1866.483269483266 + }, + "selected": false, + "type": "genericNode", + "width": 320 + }, + { + "data": { + "id": "note-Bm5Xw", + "node": { + "description": "### 💡 Add your OpenAI API key here 👇", + "display_name": "", + "documentation": "", + "template": { + "backgroundColor": "transparent" + } + }, + "type": "note" + }, + "dragging": false, + "height": 324, + "id": "note-Bm5Xw", + "measured": { + "height": 324, + "width": 324 + }, + "position": { + "x": 1692.2322233423606, + "y": 1821.9077961087607 + }, + "positionAbsolute": { + "x": 1692.2322233423606, + "y": 1821.9077961087607 + }, + "selected": false, + "type": "noteNode", + "width": 324 + }, + { + "data": { + "id": "File-PSU37", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Loads content from files with optional advanced document processing and export using Docling.", + "display_name": "File", + "documentation": "https://docs.langflow.org/components-data#file", + "edited": true, + "field_order": [ + "path", + "file_path", + "separator", + "silent_errors", + "delete_server_file_after_processing", + "ignore_unsupported_extensions", + "ignore_unspecified_files", + "advanced_mode", + "pipeline", + "ocr_engine", + "md_image_placeholder", + "md_page_break_placeholder", + "doc_key", + "use_multithreading", + "concurrency_multithreading", + "markdown" + ], + "frozen": false, + "icon": "file-text", + "last_updated": "2025-09-09T02:18:48.064Z", + "legacy": false, + "lf_version": "1.5.0.post2", + "metadata": { + "code_hash": "086578fbbd54", + "dependencies": { + "dependencies": [ + { + "name": "langflow", + "version": "1.5.0.post2" + }, + { + "name": "anyio", + "version": "4.10.0" + } + ], + "total_dependencies": 2 + }, + "module": "custom_components.file" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Raw Content", + "group_outputs": false, + "hidden": null, + "method": "load_files_message", + "name": "message", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + }, + { + "allows_loop": false, + "cache": true, + "display_name": "File Path", + "group_outputs": false, + "hidden": null, + "method": "load_files_path", + "name": "path", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "advanced_mode": { + "_input_type": "BoolInput", + "advanced": false, + "display_name": "Advanced Parser", + "dynamic": false, + "info": "Enable advanced document processing and export with Docling for PDFs, images, and office documents. Available only for single file processing.", + "list": false, + "list_add_label": "Add More", + "name": "advanced_mode", + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "\"\"\"Enhanced file component with clearer structure and Docling isolation.\n\nNotes:\n-----\n- Functionality is preserved with minimal behavioral changes.\n- ALL Docling parsing/export runs in a separate OS process to prevent memory\n growth and native library state from impacting the main Langflow process.\n- Standard text/structured parsing continues to use existing BaseFileComponent\n utilities (and optional threading via `parallel_load_data`).\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport subprocess\nimport sys\nimport textwrap\nfrom copy import deepcopy\nfrom typing import TYPE_CHECKING, Any\n\nfrom langflow.base.data.base_file import BaseFileComponent\nfrom langflow.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data\nfrom langflow.io import (\n BoolInput,\n DropdownInput,\n FileInput,\n IntInput,\n MessageTextInput,\n Output,\n StrInput,\n)\nfrom langflow.schema.data import Data\nfrom langflow.schema.message import Message\nimport anyio\nfrom langflow.services.storage.utils import build_content_type_from_extension\nif TYPE_CHECKING:\n from langflow.schema import DataFrame\n\n\nclass FileComponent(BaseFileComponent):\n \"\"\"File component with optional Docling processing (isolated in a subprocess).\"\"\"\n\n display_name = \"File\"\n description = \"Loads content from files with optional advanced document processing and export using Docling.\"\n documentation: str = \"https://docs.langflow.org/components-data#file\"\n icon = \"file-text\"\n name = \"File\"\n\n # Docling-supported/compatible extensions; TEXT_FILE_TYPES are supported by the base loader.\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n *TEXT_FILE_TYPES,\n ]\n\n # Fixed export settings used when markdown export is requested.\n EXPORT_FORMAT = \"Markdown\"\n IMAGE_MODE = \"placeholder\"\n\n # ---- Inputs / Outputs (kept as close to original as possible) -------------------\n _base_inputs = deepcopy(BaseFileComponent._base_inputs)\n for input_item in _base_inputs:\n if isinstance(input_item, FileInput) and input_item.name == \"path\":\n input_item.real_time_refresh = True\n break\n\n inputs = [\n *_base_inputs,\n BoolInput(\n name=\"advanced_mode\",\n display_name=\"Advanced Parser\",\n value=False,\n real_time_refresh=True,\n info=(\n \"Enable advanced document processing and export with Docling for PDFs, images, and office documents. \"\n \"Available only for single file processing.\"\n ),\n show=False,\n ),\n DropdownInput(\n name=\"pipeline\",\n display_name=\"Pipeline\",\n info=\"Docling pipeline to use\",\n options=[\"standard\", \"vlm\"],\n value=\"standard\",\n advanced=True,\n ),\n DropdownInput(\n name=\"ocr_engine\",\n display_name=\"OCR Engine\",\n info=\"OCR engine to use. Only available when pipeline is set to 'standard'.\",\n options=[\"\", \"easyocr\"],\n value=\"\",\n show=False,\n advanced=True,\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder between pages in the markdown output.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n show=False,\n ),\n # Deprecated input retained for backward-compatibility.\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"[Deprecated] Use Multithreading\",\n advanced=True,\n value=True,\n info=\"Set 'Processing Concurrency' greater than 1 to enable multithreading.\",\n ),\n IntInput(\n name=\"concurrency_multithreading\",\n display_name=\"Processing Concurrency\",\n advanced=True,\n info=\"When multiple files are being processed, the number of files to process concurrently.\",\n value=1,\n ),\n BoolInput(\n name=\"markdown\",\n display_name=\"Markdown Export\",\n info=\"Export processed documents to Markdown format. Only available when advanced mode is enabled.\",\n value=False,\n show=False,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n ]\n\n # ------------------------------ UI helpers --------------------------------------\n\n def _path_value(self, template: dict) -> list[str]:\n \"\"\"Return the list of currently selected file paths from the template.\"\"\"\n return template.get(\"path\", {}).get(\"file_path\", [])\n\n def update_build_config(\n self,\n build_config: dict[str, Any],\n field_value: Any,\n field_name: str | None = None,\n ) -> dict[str, Any]:\n \"\"\"Show/hide Advanced Parser and related fields based on selection context.\"\"\"\n if field_name == \"path\":\n paths = self._path_value(build_config)\n file_path = paths[0] if paths else \"\"\n file_count = len(field_value) if field_value else 0\n\n # Advanced mode only for single (non-tabular) file\n allow_advanced = file_count == 1 and not file_path.endswith((\".csv\", \".xlsx\", \".parquet\"))\n build_config[\"advanced_mode\"][\"show\"] = allow_advanced\n if not allow_advanced:\n build_config[\"advanced_mode\"][\"value\"] = False\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = False\n\n elif field_name == \"advanced_mode\":\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = bool(field_value)\n\n return build_config\n\n def update_outputs(self, frontend_node: dict[str, Any], field_name: str, field_value: Any) -> dict[str, Any]: # noqa: ARG002\n \"\"\"Dynamically show outputs based on file count/type and advanced mode.\"\"\"\n if field_name not in [\"path\", \"advanced_mode\"]:\n return frontend_node\n\n template = frontend_node.get(\"template\", {})\n paths = self._path_value(template)\n if not paths:\n return frontend_node\n\n frontend_node[\"outputs\"] = []\n if len(paths) == 1:\n file_path = paths[0] if field_name == \"path\" else frontend_node[\"template\"][\"path\"][\"file_path\"][0]\n if file_path.endswith((\".csv\", \".xlsx\", \".parquet\")):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"dataframe\", method=\"load_files_structured\"),\n )\n elif file_path.endswith(\".json\"):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"json\", method=\"load_files_json\"),\n )\n\n advanced_mode = frontend_node.get(\"template\", {}).get(\"advanced_mode\", {}).get(\"value\", False)\n if advanced_mode:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Output\", name=\"advanced\", method=\"load_files_advanced\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Markdown\", name=\"markdown\", method=\"load_files_markdown\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n # Multiple files => DataFrame output; advanced parser disabled\n frontend_node[\"outputs\"].append(Output(display_name=\"Files\", name=\"dataframe\", method=\"load_files\"))\n\n return frontend_node\n\n # ------------------------------ Core processing ----------------------------------\n\n def _is_docling_compatible(self, file_path: str) -> bool:\n \"\"\"Lightweight extension gate for Docling-compatible types.\"\"\"\n docling_exts = (\n \".adoc\",\n \".asciidoc\",\n \".asc\",\n \".bmp\",\n \".csv\",\n \".dotx\",\n \".dotm\",\n \".docm\",\n \".docx\",\n \".htm\",\n \".html\",\n \".jpeg\",\n \".json\",\n \".md\",\n \".pdf\",\n \".png\",\n \".potx\",\n \".ppsx\",\n \".pptm\",\n \".potm\",\n \".ppsm\",\n \".pptx\",\n \".tiff\",\n \".txt\",\n \".xls\",\n \".xlsx\",\n \".xhtml\",\n \".xml\",\n \".webp\",\n )\n return file_path.lower().endswith(docling_exts)\n\n def _process_docling_in_subprocess(self, file_path: str) -> Data | None:\n \"\"\"Run Docling in a separate OS process and map the result to a Data object.\n\n We avoid multiprocessing pickling by launching `python -c \"