gmakstutis/openrag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
openrag/flows/ingestion_flow.json
Gabriel Luiz Freitas Almeida ec5092a54a Add ingestion flow for OpenSearch integration 
This commit introduces a new JSON configuration file for the OpenSearch ingestion flow, detailing the data processing pipeline. The flow includes components for splitting text, generating embeddings, and ingesting data into OpenSearch, enhancing the capabilities for Retrieval Augmented Generation (RAG) tasks. The configuration is designed to support various input types and provides detailed metadata for each component, ensuring robust and well-documented integration.
2025-09-04 18:03:16 -03:00

1588 lines
No EOL
101 KiB
JSON
Raw Blame History

{
"data": {
"edges": [
{
"animated": false,
"className": "",
"data": {
"sourceHandle": {
"dataType": "File",
"id": "File-PSU37",
"name": "message",
"output_types": [
"Message"
]
},
"targetHandle": {
"fieldName": "data_inputs",
"id": "SplitText-QIKhg",
"inputTypes": [
"Data",
"DataFrame",
"Message"
],
"type": "other"
}
},
"id": "reactflow__edge-File-PSU37{\u0153dataType\u0153:\u0153File\u0153,\u0153id\u0153:\u0153File-PSU37\u0153,\u0153name\u0153:\u0153message\u0153,\u0153output_types\u0153:[\u0153Message\u0153]}-SplitText-QIKhg{\u0153fieldName\u0153:\u0153data_inputs\u0153,\u0153id\u0153:\u0153SplitText-QIKhg\u0153,\u0153inputTypes\u0153:[\u0153Data\u0153,\u0153DataFrame\u0153,\u0153Message\u0153],\u0153type\u0153:\u0153other\u0153}",
"selected": false,
"source": "File-PSU37",
"sourceHandle": "{\u0153dataType\u0153:\u0153File\u0153,\u0153id\u0153:\u0153File-PSU37\u0153,\u0153name\u0153:\u0153message\u0153,\u0153output_types\u0153:[\u0153Message\u0153]}",
"target": "SplitText-QIKhg",
"targetHandle": "{\u0153fieldName\u0153:\u0153data_inputs\u0153,\u0153id\u0153:\u0153SplitText-QIKhg\u0153,\u0153inputTypes\u0153:[\u0153Data\u0153,\u0153DataFrame\u0153,\u0153Message\u0153],\u0153type\u0153:\u0153other\u0153}"
},
{
"animated": false,
"className": "",
"data": {
"sourceHandle": {
"dataType": "OpenAIEmbeddings",
"id": "OpenAIEmbeddings-joRJ6",
"name": "embeddings",
"output_types": [
"Embeddings"
]
},
"targetHandle": {
"fieldName": "embedding",
"id": "OpenSearch-Mkw1W",
"inputTypes": [
"Embeddings"
],
"type": "other"
}
},
"id": "xy-edge__OpenAIEmbeddings-joRJ6{\u0153dataType\u0153:\u0153OpenAIEmbeddings\u0153,\u0153id\u0153:\u0153OpenAIEmbeddings-joRJ6\u0153,\u0153name\u0153:\u0153embeddings\u0153,\u0153output_types\u0153:[\u0153Embeddings\u0153]}-OpenSearch-Mkw1W{\u0153fieldName\u0153:\u0153embedding\u0153,\u0153id\u0153:\u0153OpenSearch-Mkw1W\u0153,\u0153inputTypes\u0153:[\u0153Embeddings\u0153],\u0153type\u0153:\u0153other\u0153}",
"selected": false,
"source": "OpenAIEmbeddings-joRJ6",
"sourceHandle": "{\u0153dataType\u0153:\u0153OpenAIEmbeddings\u0153,\u0153id\u0153:\u0153OpenAIEmbeddings-joRJ6\u0153,\u0153name\u0153:\u0153embeddings\u0153,\u0153output_types\u0153:[\u0153Embeddings\u0153]}",
"target": "OpenSearch-Mkw1W",
"targetHandle": "{\u0153fieldName\u0153:\u0153embedding\u0153,\u0153id\u0153:\u0153OpenSearch-Mkw1W\u0153,\u0153inputTypes\u0153:[\u0153Embeddings\u0153],\u0153type\u0153:\u0153other\u0153}"
},
{
"animated": false,
"className": "",
"data": {
"sourceHandle": {
"dataType": "SplitText",
"id": "SplitText-QIKhg",
"name": "dataframe",
"output_types": [
"DataFrame"
]
},
"targetHandle": {
"fieldName": "ingest_data",
"id": "OpenSearch-Mkw1W",
"inputTypes": [
"Data",
"DataFrame"
],
"type": "other"
}
},
"id": "xy-edge__SplitText-QIKhg{\u0153dataType\u0153:\u0153SplitText\u0153,\u0153id\u0153:\u0153SplitText-QIKhg\u0153,\u0153name\u0153:\u0153dataframe\u0153,\u0153output_types\u0153:[\u0153DataFrame\u0153]}-OpenSearch-Mkw1W{\u0153fieldName\u0153:\u0153ingest_data\u0153,\u0153id\u0153:\u0153OpenSearch-Mkw1W\u0153,\u0153inputTypes\u0153:[\u0153Data\u0153,\u0153DataFrame\u0153],\u0153type\u0153:\u0153other\u0153}",
"selected": false,
"source": "SplitText-QIKhg",
"sourceHandle": "{\u0153dataType\u0153:\u0153SplitText\u0153,\u0153id\u0153:\u0153SplitText-QIKhg\u0153,\u0153name\u0153:\u0153dataframe\u0153,\u0153output_types\u0153:[\u0153DataFrame\u0153]}",
"target": "OpenSearch-Mkw1W",
"targetHandle": "{\u0153fieldName\u0153:\u0153ingest_data\u0153,\u0153id\u0153:\u0153OpenSearch-Mkw1W\u0153,\u0153inputTypes\u0153:[\u0153Data\u0153,\u0153DataFrame\u0153],\u0153type\u0153:\u0153other\u0153}"
}
],
"nodes": [
{
"data": {
"description": "Split text into chunks based on specified criteria.",
"display_name": "Split Text",
"id": "SplitText-QIKhg",
"node": {
"base_classes": [
"Data"
],
"beta": false,
"conditional_paths": [],
"custom_fields": {},
"description": "Split text into chunks based on specified criteria.",
"display_name": "Split Text",
"documentation": "",
"edited": false,
"field_order": [
"data_inputs",
"chunk_overlap",
"chunk_size",
"separator"
],
"frozen": false,
"icon": "scissors-line-dashed",
"legacy": false,
"lf_version": "1.1.1",
"metadata": {
"code_hash": "dbf2e9d2319d",
"dependencies": {
"dependencies": [
{
"name": "langchain_text_splitters",
"version": "0.3.9"
},
{
"name": "langflow",
"version": "1.5.0.post2"
}
],
"total_dependencies": 2
},
"module": "langflow.components.processing.split_text.SplitTextComponent"
},
"output_types": [],
"outputs": [
{
"allows_loop": false,
"cache": true,
"display_name": "Chunks",
"group_outputs": false,
"method": "split_text",
"name": "dataframe",
"selected": "DataFrame",
"tool_mode": true,
"types": [
"DataFrame"
],
"value": "__UNDEFINED__"
}
],
"pinned": false,
"template": {
"_type": "Component",
"chunk_overlap": {
"advanced": false,
"display_name": "Chunk Overlap",
"dynamic": false,
"info": "Number of characters to overlap between chunks.",
"list": false,
"name": "chunk_overlap",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "int",
"value": 200
},
"chunk_size": {
"advanced": false,
"display_name": "Chunk Size",
"dynamic": false,
"info": "The maximum length of each chunk. Text is first split by separator, then chunks are merged up to this size. Individual splits larger than this won't be further divided.",
"list": false,
"name": "chunk_size",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "int",
"value": 1000
},
"code": {
"advanced": true,
"dynamic": true,
"fileTypes": [],
"file_path": "",
"info": "",
"list": false,
"load_from_db": false,
"multiline": true,
"name": "code",
"password": false,
"placeholder": "",
"required": true,
"show": true,
"title_case": false,
"type": "code",
"value": "from langchain_text_splitters import CharacterTextSplitter\n\nfrom langflow.custom.custom_component.component import Component\nfrom langflow.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output\nfrom langflow.schema.data import Data\nfrom langflow.schema.dataframe import DataFrame\nfrom langflow.schema.message import Message\nfrom langflow.utils.util import unescape_string\n\n\nclass SplitTextComponent(Component):\n display_name: str = \"Split Text\"\n description: str = \"Split text into chunks based on specified criteria.\"\n documentation: str = \"https://docs.langflow.org/components-processing#split-text\"\n icon = \"scissors-line-dashed\"\n name = \"SplitText\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Input\",\n info=\"The data with texts to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n IntInput(\n name=\"chunk_overlap\",\n display_name=\"Chunk Overlap\",\n info=\"Number of characters to overlap between chunks.\",\n value=200,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n info=(\n \"The maximum length of each chunk. Text is first split by separator, \"\n \"then chunks are merged up to this size. \"\n \"Individual splits larger than this won't be further divided.\"\n ),\n value=1000,\n ),\n MessageTextInput(\n name=\"separator\",\n display_name=\"Separator\",\n info=(\n \"The character to split on. Use \\\\n for newline. \"\n \"Examples: \\\\n\\\\n for paragraphs, \\\\n for lines, . for sentences\"\n ),\n value=\"\\n\",\n ),\n MessageTextInput(\n name=\"text_key\",\n display_name=\"Text Key\",\n info=\"The key to use for the text column.\",\n value=\"text\",\n advanced=True,\n ),\n DropdownInput(\n name=\"keep_separator\",\n display_name=\"Keep Separator\",\n info=\"Whether to keep the separator in the output chunks and where to place it.\",\n options=[\"False\", \"True\", \"Start\", \"End\"],\n value=\"False\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Chunks\", name=\"dataframe\", method=\"split_text\"),\n ]\n\n def _docs_to_data(self, docs) -> list[Data]:\n return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n\n def _fix_separator(self, separator: str) -> str:\n \"\"\"Fix common separator issues and convert to proper format.\"\"\"\n if separator == \"/n\":\n return \"\\n\"\n if separator == \"/t\":\n return \"\\t\"\n return separator\n\n def split_text_base(self):\n separator = self._fix_separator(self.separator)\n separator = unescape_string(separator)\n\n if isinstance(self.data_inputs, DataFrame):\n if not len(self.data_inputs):\n msg = \"DataFrame is empty\"\n raise TypeError(msg)\n\n self.data_inputs.text_key = self.text_key\n try:\n documents = self.data_inputs.to_lc_documents()\n except Exception as e:\n msg = f\"Error converting DataFrame to documents: {e}\"\n raise TypeError(msg) from e\n elif isinstance(self.data_inputs, Message):\n self.data_inputs = [self.data_inputs.to_data()]\n return self.split_text_base()\n else:\n if not self.data_inputs:\n msg = \"No data inputs provided\"\n raise TypeError(msg)\n\n documents = []\n if isinstance(self.data_inputs, Data):\n self.data_inputs.text_key = self.text_key\n documents = [self.data_inputs.to_lc_document()]\n else:\n try:\n documents = [input_.to_lc_document() for input_ in self.data_inputs if isinstance(input_, Data)]\n if not documents:\n msg = f\"No valid Data inputs found in {type(self.data_inputs)}\"\n raise TypeError(msg)\n except AttributeError as e:\n msg = f\"Invalid input type in collection: {e}\"\n raise TypeError(msg) from e\n try:\n # Convert string 'False'/'True' to boolean\n keep_sep = self.keep_separator\n if isinstance(keep_sep, str):\n if keep_sep.lower() == \"false\":\n keep_sep = False\n elif keep_sep.lower() == \"true\":\n keep_sep = True\n # 'start' and 'end' are kept as strings\n\n splitter = CharacterTextSplitter(\n chunk_overlap=self.chunk_overlap,\n chunk_size=self.chunk_size,\n separator=separator,\n keep_separator=keep_sep,\n )\n return splitter.split_documents(documents)\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n def split_text(self) -> DataFrame:\n return DataFrame(self._docs_to_data(self.split_text_base()))\n"
},
"data_inputs": {
"advanced": false,
"display_name": "Input",
"dynamic": false,
"info": "The data with texts to split in chunks.",
"input_types": [
"Data",
"DataFrame",
"Message"
],
"list": false,
"name": "data_inputs",
"placeholder": "",
"required": true,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "other",
"value": ""
},
"keep_separator": {
"_input_type": "DropdownInput",
"advanced": true,
"combobox": false,
"dialog_inputs": {},
"display_name": "Keep Separator",
"dynamic": false,
"info": "Whether to keep the separator in the output chunks and where to place it.",
"name": "keep_separator",
"options": [
"False",
"True",
"Start",
"End"
],
"options_metadata": [],
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "str",
"value": "False"
},
"separator": {
"advanced": false,
"display_name": "Separator",
"dynamic": false,
"info": "The character to split on. Use \\n for newline. Examples: \\n\\n for paragraphs, \\n for lines, . for sentences",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "separator",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": "\n"
},
"text_key": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "Text Key",
"dynamic": false,
"info": "The key to use for the text column.",
"input_types": [
"Message"
],
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
"name": "text_key",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": "text"
}
}
},
"selected_output": "chunks",
"type": "SplitText"
},
"dragging": false,
"height": 475,
"id": "SplitText-QIKhg",
"measured": {
"height": 475,
"width": 320
},
"position": {
"x": 1692.461995335383,
"y": 1328.2681481569232
},
"positionAbsolute": {
"x": 1683.4543896546102,
"y": 1350.7871623588553
},
"selected": false,
"type": "genericNode",
"width": 320
},
{
"data": {
"id": "OpenAIEmbeddings-joRJ6",
"node": {
"base_classes": [
"Embeddings"
],
"beta": false,
"conditional_paths": [],
"custom_fields": {},
"description": "Generate embeddings using OpenAI models.",
"display_name": "OpenAI Embeddings",
"documentation": "",
"edited": false,
"field_order": [
"default_headers",
"default_query",
"chunk_size",
"client",
"deployment",
"embedding_ctx_length",
"max_retries",
"model",
"model_kwargs",
"openai_api_key",
"openai_api_base",
"openai_api_type",
"openai_api_version",
"openai_organization",
"openai_proxy",
"request_timeout",
"show_progress_bar",
"skip_empty",
"tiktoken_model_name",
"tiktoken_enable",
"dimensions"
],
"frozen": false,
"icon": "OpenAI",
"legacy": false,
"lf_version": "1.1.1",
"metadata": {
"code_hash": "2691dee277c9",
"dependencies": {
"dependencies": [
{
"name": "langchain_openai",
"version": "0.3.23"
},
{
"name": "langflow",
"version": "1.5.0.post2"
}
],
"total_dependencies": 2
},
"module": "langflow.components.openai.openai.OpenAIEmbeddingsComponent"
},
"output_types": [],
"outputs": [
{
"allows_loop": false,
"cache": true,
"display_name": "Embedding Model",
"group_outputs": false,
"method": "build_embeddings",
"name": "embeddings",
"selected": "Embeddings",
"tool_mode": true,
"types": [
"Embeddings"
],
"value": "__UNDEFINED__"
}
],
"pinned": false,
"template": {
"_type": "Component",
"chunk_size": {
"_input_type": "IntInput",
"advanced": true,
"display_name": "Chunk Size",
"dynamic": false,
"info": "",
"list": false,
"name": "chunk_size",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "int",
"value": 1000
},
"client": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "Client",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "client",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"code": {
"advanced": true,
"dynamic": true,
"fileTypes": [],
"file_path": "",
"info": "",
"list": false,
"load_from_db": false,
"multiline": true,
"name": "code",
"password": false,
"placeholder": "",
"required": true,
"show": true,
"title_case": false,
"type": "code",
"value": "from langchain_openai import OpenAIEmbeddings\n\nfrom langflow.base.embeddings.model import LCEmbeddingsModel\nfrom langflow.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom langflow.field_typing import Embeddings\nfrom langflow.io import BoolInput, DictInput, DropdownInput, FloatInput, IntInput, MessageTextInput, SecretStrInput\n\n\nclass OpenAIEmbeddingsComponent(LCEmbeddingsModel):\n display_name = \"OpenAI Embeddings\"\n description = \"Generate embeddings using OpenAI models.\"\n icon = \"OpenAI\"\n name = \"OpenAIEmbeddings\"\n\n inputs = [\n DictInput(\n name=\"default_headers\",\n display_name=\"Default Headers\",\n advanced=True,\n info=\"Default headers to use for the API request.\",\n ),\n DictInput(\n name=\"default_query\",\n display_name=\"Default Query\",\n advanced=True,\n info=\"Default query parameters to use for the API request.\",\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n MessageTextInput(name=\"client\", display_name=\"Client\", advanced=True),\n MessageTextInput(name=\"deployment\", display_name=\"Deployment\", advanced=True),\n IntInput(name=\"embedding_ctx_length\", display_name=\"Embedding Context Length\", advanced=True, value=1536),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", value=3, advanced=True),\n DropdownInput(\n name=\"model\",\n display_name=\"Model\",\n advanced=False,\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=\"text-embedding-3-small\",\n ),\n DictInput(name=\"model_kwargs\", display_name=\"Model Kwargs\", advanced=True),\n SecretStrInput(name=\"openai_api_key\", display_name=\"OpenAI API Key\", value=\"OPENAI_API_KEY\", required=True),\n MessageTextInput(name=\"openai_api_base\", display_name=\"OpenAI API Base\", advanced=True),\n MessageTextInput(name=\"openai_api_type\", display_name=\"OpenAI API Type\", advanced=True),\n MessageTextInput(name=\"openai_api_version\", display_name=\"OpenAI API Version\", advanced=True),\n MessageTextInput(\n name=\"openai_organization\",\n display_name=\"OpenAI Organization\",\n advanced=True,\n ),\n MessageTextInput(name=\"openai_proxy\", display_name=\"OpenAI Proxy\", advanced=True),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n BoolInput(name=\"skip_empty\", display_name=\"Skip Empty\", advanced=True),\n MessageTextInput(\n name=\"tiktoken_model_name\",\n display_name=\"TikToken Model Name\",\n advanced=True,\n ),\n BoolInput(\n name=\"tiktoken_enable\",\n display_name=\"TikToken Enable\",\n advanced=True,\n value=True,\n info=\"If False, you must have transformers installed.\",\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n return OpenAIEmbeddings(\n client=self.client or None,\n model=self.model,\n dimensions=self.dimensions or None,\n deployment=self.deployment or None,\n api_version=self.openai_api_version or None,\n base_url=self.openai_api_base or None,\n openai_api_type=self.openai_api_type or None,\n openai_proxy=self.openai_proxy or None,\n embedding_ctx_length=self.embedding_ctx_length,\n api_key=self.openai_api_key or None,\n organization=self.openai_organization or None,\n allowed_special=\"all\",\n disallowed_special=\"all\",\n chunk_size=self.chunk_size,\n max_retries=self.max_retries,\n timeout=self.request_timeout or None,\n tiktoken_enabled=self.tiktoken_enable,\n tiktoken_model_name=self.tiktoken_model_name or None,\n show_progress_bar=self.show_progress_bar,\n model_kwargs=self.model_kwargs,\n skip_empty=self.skip_empty,\n default_headers=self.default_headers or None,\n default_query=self.default_query or None,\n )\n"
},
"default_headers": {
"_input_type": "DictInput",
"advanced": true,
"display_name": "Default Headers",
"dynamic": false,
"info": "Default headers to use for the API request.",
"list": false,
"name": "default_headers",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_input": true,
"type": "dict",
"value": {}
},
"default_query": {
"_input_type": "DictInput",
"advanced": true,
"display_name": "Default Query",
"dynamic": false,
"info": "Default query parameters to use for the API request.",
"list": false,
"name": "default_query",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_input": true,
"type": "dict",
"value": {}
},
"deployment": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "Deployment",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "deployment",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"dimensions": {
"_input_type": "IntInput",
"advanced": true,
"display_name": "Dimensions",
"dynamic": false,
"info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.",
"list": false,
"name": "dimensions",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "int",
"value": ""
},
"embedding_ctx_length": {
"_input_type": "IntInput",
"advanced": true,
"display_name": "Embedding Context Length",
"dynamic": false,
"info": "",
"list": false,
"name": "embedding_ctx_length",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "int",
"value": 1536
},
"max_retries": {
"_input_type": "IntInput",
"advanced": true,
"display_name": "Max Retries",
"dynamic": false,
"info": "",
"list": false,
"name": "max_retries",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "int",
"value": 3
},
"model": {
"_input_type": "DropdownInput",
"advanced": false,
"combobox": false,
"display_name": "Model",
"dynamic": false,
"info": "",
"name": "model",
"options": [
"text-embedding-3-small",
"text-embedding-3-large",
"text-embedding-ada-002"
],
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "str",
"value": "text-embedding-3-small"
},
"model_kwargs": {
"_input_type": "DictInput",
"advanced": true,
"display_name": "Model Kwargs",
"dynamic": false,
"info": "",
"list": false,
"name": "model_kwargs",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_input": true,
"type": "dict",
"value": {}
},
"openai_api_base": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "OpenAI API Base",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "openai_api_base",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"openai_api_key": {
"_input_type": "SecretStrInput",
"advanced": false,
"display_name": "OpenAI API Key",
"dynamic": false,
"info": "",
"input_types": [],
"load_from_db": true,
"name": "openai_api_key",
"password": true,
"placeholder": "",
"required": true,
"show": true,
"title_case": false,
"type": "str",
"value": "OPENAI_API_KEY"
},
"openai_api_type": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "OpenAI API Type",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "openai_api_type",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"openai_api_version": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "OpenAI API Version",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "openai_api_version",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"openai_organization": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "OpenAI Organization",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "openai_organization",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"openai_proxy": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "OpenAI Proxy",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "openai_proxy",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"request_timeout": {
"_input_type": "FloatInput",
"advanced": true,
"display_name": "Request Timeout",
"dynamic": false,
"info": "",
"list": false,
"name": "request_timeout",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "float",
"value": ""
},
"show_progress_bar": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Show Progress Bar",
"dynamic": false,
"info": "",
"list": false,
"name": "show_progress_bar",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "bool",
"value": false
},
"skip_empty": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Skip Empty",
"dynamic": false,
"info": "",
"list": false,
"name": "skip_empty",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "bool",
"value": false
},
"tiktoken_enable": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "TikToken Enable",
"dynamic": false,
"info": "If False, you must have transformers installed.",
"list": false,
"name": "tiktoken_enable",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "bool",
"value": true
},
"tiktoken_model_name": {
"_input_type": "MessageTextInput",
"advanced": true,
"display_name": "TikToken Model Name",
"dynamic": false,
"info": "",
"input_types": [
"Message"
],
"list": false,
"load_from_db": false,
"name": "tiktoken_model_name",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
}
},
"tool_mode": false
},
"selected_output": "embeddings",
"type": "OpenAIEmbeddings"
},
"dragging": false,
"height": 320,
"id": "OpenAIEmbeddings-joRJ6",
"measured": {
"height": 320,
"width": 320
},
"position": {
"x": 1690.9220896443658,
"y": 1866.483269483266
},
"positionAbsolute": {
"x": 1690.9220896443658,
"y": 1866.483269483266
},
"selected": false,
"type": "genericNode",
"width": 320
},
{
"data": {
"id": "note-Bm5Xw",
"node": {
"description": "### \ud83d\udca1 Add your OpenAI API key here \ud83d\udc47",
"display_name": "",
"documentation": "",
"template": {
"backgroundColor": "transparent"
}
},
"type": "note"
},
"dragging": false,
"height": 324,
"id": "note-Bm5Xw",
"measured": {
"height": 324,
"width": 324
},
"position": {
"x": 1692.2322233423606,
"y": 1821.9077961087607
},
"positionAbsolute": {
"x": 1692.2322233423606,
"y": 1821.9077961087607
},
"selected": false,
"type": "noteNode",
"width": 324
},
{
"data": {
"id": "File-PSU37",
"node": {
"base_classes": [
"Message"
],
"beta": false,
"conditional_paths": [],
"custom_fields": {},
"description": "Loads content from one or more files as a DataFrame.",
"display_name": "File",
"documentation": "",
"edited": false,
"field_order": [
"path",
"file_path",
"separator",
"silent_errors",
"delete_server_file_after_processing",
"ignore_unsupported_extensions",
"ignore_unspecified_files",
"use_multithreading",
"concurrency_multithreading"
],
"frozen": false,
"icon": "file-text",
"last_updated": "2025-09-03T06:37:14.082Z",
"legacy": false,
"metadata": {},
"minimized": false,
"output_types": [],
"outputs": [
{
"allows_loop": false,
"cache": true,
"display_name": "Raw Content",
"group_outputs": false,
"method": "load_files_message",
"name": "message",
"options": null,
"required_inputs": null,
"selected": "Message",
"tool_mode": true,
"types": [
"Message"
],
"value": "__UNDEFINED__"
}
],
"pinned": false,
"template": {
"_type": "Component",
"code": {
"advanced": true,
"dynamic": true,
"fileTypes": [],
"file_path": "",
"info": "",
"list": false,
"load_from_db": false,
"multiline": true,
"name": "code",
"password": false,
"placeholder": "",
"required": true,
"show": true,
"title_case": false,
"type": "code",
"value": "\"\"\"Enhanced file component with clearer structure and Docling isolation.\n\nNotes:\n-----\n- Functionality is preserved with minimal behavioral changes.\n- ALL Docling parsing/export runs in a separate OS process to prevent memory\n growth and native library state from impacting the main Langflow process.\n- Standard text/structured parsing continues to use existing BaseFileComponent\n utilities (and optional threading via `parallel_load_data`).\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport subprocess\nimport sys\nimport textwrap\nfrom copy import deepcopy\nfrom typing import TYPE_CHECKING, Any\n\nfrom langflow.base.data.base_file import BaseFileComponent\nfrom langflow.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data\nfrom langflow.io import (\n BoolInput,\n DropdownInput,\n FileInput,\n IntInput,\n MessageTextInput,\n Output,\n StrInput,\n)\nfrom langflow.schema.data import Data\nfrom langflow.schema.message import Message\n\nif TYPE_CHECKING:\n from langflow.schema import DataFrame\n\n\nclass FileComponent(BaseFileComponent):\n \"\"\"File component with optional Docling processing (isolated in a subprocess).\"\"\"\n\n display_name = \"File\"\n description = \"Loads content from files with optional advanced document processing and export using Docling.\"\n documentation: str = \"https://docs.langflow.org/components-data#file\"\n icon = \"file-text\"\n name = \"File\"\n\n # Docling-supported/compatible extensions; TEXT_FILE_TYPES are supported by the base loader.\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n *TEXT_FILE_TYPES,\n ]\n\n # Fixed export settings used when markdown export is requested.\n EXPORT_FORMAT = \"Markdown\"\n IMAGE_MODE = \"placeholder\"\n\n # ---- Inputs / Outputs (kept as close to original as possible) -------------------\n _base_inputs = deepcopy(BaseFileComponent._base_inputs)\n for input_item in _base_inputs:\n if isinstance(input_item, FileInput) and input_item.name == \"path\":\n input_item.real_time_refresh = True\n break\n\n inputs = [\n *_base_inputs,\n BoolInput(\n name=\"advanced_mode\",\n display_name=\"Advanced Parser\",\n value=False,\n real_time_refresh=True,\n info=(\n \"Enable advanced document processing and export with Docling for PDFs, images, and office documents. \"\n \"Available only for single file processing.\"\n ),\n show=False,\n ),\n DropdownInput(\n name=\"pipeline\",\n display_name=\"Pipeline\",\n info=\"Docling pipeline to use\",\n options=[\"standard\", \"vlm\"],\n value=\"standard\",\n advanced=True,\n ),\n DropdownInput(\n name=\"ocr_engine\",\n display_name=\"OCR Engine\",\n info=\"OCR engine to use. Only available when pipeline is set to 'standard'.\",\n options=[\"\", \"easyocr\"],\n value=\"\",\n show=False,\n advanced=True,\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"<!-- image -->\",\n advanced=True,\n show=False,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder between pages in the markdown output.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n show=False,\n ),\n # Deprecated input retained for backward-compatibility.\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"[Deprecated] Use Multithreading\",\n advanced=True,\n value=True,\n info=\"Set 'Processing Concurrency' greater than 1 to enable multithreading.\",\n ),\n IntInput(\n name=\"concurrency_multithreading\",\n display_name=\"Processing Concurrency\",\n advanced=True,\n info=\"When multiple files are being processed, the number of files to process concurrently.\",\n value=1,\n ),\n BoolInput(\n name=\"markdown\",\n display_name=\"Markdown Export\",\n info=\"Export processed documents to Markdown format. Only available when advanced mode is enabled.\",\n value=False,\n show=False,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n ]\n\n # ------------------------------ UI helpers --------------------------------------\n\n def _path_value(self, template: dict) -> list[str]:\n \"\"\"Return the list of currently selected file paths from the template.\"\"\"\n return template.get(\"path\", {}).get(\"file_path\", [])\n\n def update_build_config(\n self,\n build_config: dict[str, Any],\n field_value: Any,\n field_name: str | None = None,\n ) -> dict[str, Any]:\n \"\"\"Show/hide Advanced Parser and related fields based on selection context.\"\"\"\n if field_name == \"path\":\n paths = self._path_value(build_config)\n file_path = paths[0] if paths else \"\"\n file_count = len(field_value) if field_value else 0\n\n # Advanced mode only for single (non-tabular) file\n allow_advanced = file_count == 1 and not file_path.endswith((\".csv\", \".xlsx\", \".parquet\"))\n build_config[\"advanced_mode\"][\"show\"] = allow_advanced\n if not allow_advanced:\n build_config[\"advanced_mode\"][\"value\"] = False\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = False\n\n elif field_name == \"advanced_mode\":\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = bool(field_value)\n\n return build_config\n\n def update_outputs(self, frontend_node: dict[str, Any], field_name: str, field_value: Any) -> dict[str, Any]: # noqa: ARG002\n \"\"\"Dynamically show outputs based on file count/type and advanced mode.\"\"\"\n if field_name not in [\"path\", \"advanced_mode\"]:\n return frontend_node\n\n template = frontend_node.get(\"template\", {})\n paths = self._path_value(template)\n if not paths:\n return frontend_node\n\n frontend_node[\"outputs\"] = []\n if len(paths) == 1:\n file_path = paths[0] if field_name == \"path\" else frontend_node[\"template\"][\"path\"][\"file_path\"][0]\n if file_path.endswith((\".csv\", \".xlsx\", \".parquet\")):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"dataframe\", method=\"load_files_structured\"),\n )\n elif file_path.endswith(\".json\"):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"json\", method=\"load_files_json\"),\n )\n\n advanced_mode = frontend_node.get(\"template\", {}).get(\"advanced_mode\", {}).get(\"value\", False)\n if advanced_mode:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Output\", name=\"advanced\", method=\"load_files_advanced\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Markdown\", name=\"markdown\", method=\"load_files_markdown\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n # Multiple files => DataFrame output; advanced parser disabled\n frontend_node[\"outputs\"].append(Output(display_name=\"Files\", name=\"dataframe\", method=\"load_files\"))\n\n return frontend_node\n\n # ------------------------------ Core processing ----------------------------------\n\n def _is_docling_compatible(self, file_path: str) -> bool:\n \"\"\"Lightweight extension gate for Docling-compatible types.\"\"\"\n docling_exts = (\n \".adoc\",\n \".asciidoc\",\n \".asc\",\n \".bmp\",\n \".csv\",\n \".dotx\",\n \".dotm\",\n \".docm\",\n \".docx\",\n \".htm\",\n \".html\",\n \".jpeg\",\n \".json\",\n \".md\",\n \".pdf\",\n \".png\",\n \".potx\",\n \".ppsx\",\n \".pptm\",\n \".potm\",\n \".ppsm\",\n \".pptx\",\n \".tiff\",\n \".txt\",\n \".xls\",\n \".xlsx\",\n \".xhtml\",\n \".xml\",\n \".webp\",\n )\n return file_path.lower().endswith(docling_exts)\n\n def _process_docling_in_subprocess(self, file_path: str) -> Data | None:\n \"\"\"Run Docling in a separate OS process and map the result to a Data object.\n\n We avoid multiprocessing pickling by launching `python -c \"<script>\"` and\n passing JSON config via stdin. The child prints a JSON result to stdout.\n \"\"\"\n if not file_path:\n return None\n\n args: dict[str, Any] = {\n \"file_path\": file_path,\n \"markdown\": bool(self.markdown),\n \"image_mode\": str(self.IMAGE_MODE),\n \"md_image_placeholder\": str(self.md_image_placeholder),\n \"md_page_break_placeholder\": str(self.md_page_break_placeholder),\n \"pipeline\": str(self.pipeline),\n \"ocr_engine\": str(self.ocr_engine) if getattr(self, \"ocr_engine\", \"\") else None,\n }\n\n # The child is a tiny, self-contained script to keep memory/state isolated.\n child_script = textwrap.dedent(\n r\"\"\"\n import json, sys\n\n def try_imports():\n # Strategy 1: latest layout\n try:\n from docling.datamodel.base_models import ConversionStatus, InputFormat # type: ignore\n from docling.document_converter import DocumentConverter # type: ignore\n from docling_core.types.doc import ImageRefMode # type: ignore\n return ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, \"latest\"\n except Exception:\n pass\n # Strategy 2: alternative layout\n try:\n from docling.document_converter import DocumentConverter # type: ignore\n try:\n from docling_core.types import ConversionStatus, InputFormat # type: ignore\n except Exception:\n try:\n from docling.datamodel import ConversionStatus, InputFormat # type: ignore\n except Exception:\n class ConversionStatus: SUCCESS = \"success\"\n class InputFormat:\n PDF=\"pdf\"; IMAGE=\"image\"\n try:\n from docling_core.types.doc import ImageRefMode # type: ignore\n except Exception:\n class ImageRefMode:\n PLACEHOLDER=\"placeholder\"; EMBEDDED=\"embedded\"\n return ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, \"alternative\"\n except Exception:\n pass\n # Strategy 3: basic converter only\n try:\n from docling.document_converter import DocumentConverter # type: ignore\n class ConversionStatus: SUCCESS = \"success\"\n class InputFormat:\n PDF=\"pdf\"; IMAGE=\"image\"\n class ImageRefMode:\n PLACEHOLDER=\"placeholder\"; EMBEDDED=\"embedded\"\n return ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, \"basic\"\n except Exception as e:\n raise ImportError(f\"Docling imports failed: {e}\") from e\n\n def create_converter(strategy, input_format, DocumentConverter, pipeline, ocr_engine):\n if strategy == \"latest\" and pipeline == \"standard\":\n try:\n from docling.datamodel.pipeline_options import PdfPipelineOptions # type: ignore\n from docling.document_converter import PdfFormatOption # type: ignore\n pipe = PdfPipelineOptions()\n if ocr_engine:\n try:\n from docling.models.factories import get_ocr_factory # type: ignore\n pipe.do_ocr = True\n fac = get_ocr_factory(allow_external_plugins=False)\n pipe.ocr_options = fac.create_options(kind=ocr_engine)\n except Exception:\n pipe.do_ocr = False\n fmt = {}\n if hasattr(input_format, \"PDF\"):\n fmt[getattr(input_format, \"PDF\")] = PdfFormatOption(pipeline_options=pipe)\n if hasattr(input_format, \"IMAGE\"):\n fmt[getattr(input_format, \"IMAGE\")] = PdfFormatOption(pipeline_options=pipe)\n return DocumentConverter(format_options=fmt)\n except Exception:\n return DocumentConverter()\n return DocumentConverter()\n\n def export_markdown(document, ImageRefMode, image_mode, img_ph, pg_ph):\n try:\n mode = getattr(ImageRefMode, image_mode.upper(), image_mode)\n return document.export_to_markdown(\n image_mode=mode,\n image_placeholder=img_ph,\n page_break_placeholder=pg_ph,\n )\n except Exception:\n try:\n return document.export_to_text()\n except Exception:\n return str(document)\n\n def to_rows(doc_dict):\n rows = []\n for t in doc_dict.get(\"texts\", []):\n prov = t.get(\"prov\") or []\n page_no = None\n if prov and isinstance(prov, list) and isinstance(prov[0], dict):\n page_no = prov[0].get(\"page_no\")\n rows.append({\n \"page_no\": page_no,\n \"label\": t.get(\"label\"),\n \"text\": t.get(\"text\"),\n \"level\": t.get(\"level\"),\n })\n return rows\n\n def main():\n cfg = json.loads(sys.stdin.read())\n file_path = cfg[\"file_path\"]\n markdown = cfg[\"markdown\"]\n image_mode = cfg[\"image_mode\"]\n img_ph = cfg[\"md_image_placeholder\"]\n pg_ph = cfg[\"md_page_break_placeholder\"]\n pipeline = cfg[\"pipeline\"]\n ocr_engine = cfg.get(\"ocr_engine\")\n meta = {\"file_path\": file_path}\n\n try:\n ConversionStatus, InputFormat, DocumentConverter, ImageRefMode, strategy = try_imports()\n converter = create_converter(strategy, InputFormat, DocumentConverter, pipeline, ocr_engine)\n try:\n res = converter.convert(file_path)\n except Exception as e:\n print(json.dumps({\"ok\": False, \"error\": f\"Docling conversion error: {e}\", \"meta\": meta}))\n return\n\n ok = False\n if hasattr(res, \"status\"):\n try:\n ok = (res.status == ConversionStatus.SUCCESS) or (str(res.status).lower() == \"success\")\n except Exception:\n ok = (str(res.status).lower() == \"success\")\n if not ok and hasattr(res, \"document\"):\n ok = getattr(res, \"document\", None) is not None\n if not ok:\n print(json.dumps({\"ok\": False, \"error\": \"Docling conversion failed\", \"meta\": meta}))\n return\n\n doc = getattr(res, \"document\", None)\n if doc is None:\n print(json.dumps({\"ok\": False, \"error\": \"Docling produced no document\", \"meta\": meta}))\n return\n\n if markdown:\n text = export_markdown(doc, ImageRefMode, image_mode, img_ph, pg_ph)\n print(json.dumps({\"ok\": True, \"mode\": \"markdown\", \"text\": text, \"meta\": meta}))\n return\n\n # structured\n try:\n doc_dict = doc.export_to_dict()\n except Exception as e:\n print(json.dumps({\"ok\": False, \"error\": f\"Docling export_to_dict failed: {e}\", \"meta\": meta}))\n return\n\n rows = to_rows(doc_dict)\n print(json.dumps({\"ok\": True, \"mode\": \"structured\", \"doc\": rows, \"meta\": meta}))\n except Exception as e:\n print(\n json.dumps({\n \"ok\": False,\n \"error\": f\"Docling processing error: {e}\",\n \"meta\": {\"file_path\": file_path},\n })\n )\n\n if __name__ == \"__main__\":\n main()\n \"\"\"\n )\n\n # Validate file_path to avoid command injection or unsafe input\n if not isinstance(args[\"file_path\"], str) or any(c in args[\"file_path\"] for c in [\";\", \"|\", \"&\", \"$\", \"`\"]):\n return Data(data={\"error\": \"Unsafe file path detected.\", \"file_path\": args[\"file_path\"]})\n\n proc = subprocess.run( # noqa: S603\n [sys.executable, \"-u\", \"-c\", child_script],\n input=json.dumps(args).encode(\"utf-8\"),\n capture_output=True,\n check=False,\n )\n\n if not proc.stdout:\n err_msg = proc.stderr.decode(\"utf-8\", errors=\"replace\") or \"no output from child process\"\n return Data(data={\"error\": f\"Docling subprocess error: {err_msg}\", \"file_path\": file_path})\n\n try:\n result = json.loads(proc.stdout.decode(\"utf-8\"))\n except Exception as e: # noqa: BLE001\n err_msg = proc.stderr.decode(\"utf-8\", errors=\"replace\")\n return Data(\n data={\"error\": f\"Invalid JSON from Docling subprocess: {e}. stderr={err_msg}\", \"file_path\": file_path},\n )\n\n if not result.get(\"ok\"):\n return Data(data={\"error\": result.get(\"error\", \"Unknown Docling error\"), **result.get(\"meta\", {})})\n\n meta = result.get(\"meta\", {})\n if result.get(\"mode\") == \"markdown\":\n exported_content = str(result.get(\"text\", \"\"))\n return Data(\n text=exported_content,\n data={\"exported_content\": exported_content, \"export_format\": self.EXPORT_FORMAT, **meta},\n )\n\n rows = list(result.get(\"doc\", []))\n return Data(data={\"doc\": rows, \"export_format\": self.EXPORT_FORMAT, **meta})\n\n def process_files(\n self,\n file_list: list[BaseFileComponent.BaseFile],\n ) -> list[BaseFileComponent.BaseFile]:\n \"\"\"Process input files.\n\n - Single file + advanced_mode => Docling in a separate process.\n - Otherwise => standard parsing in current process (optionally threaded).\n \"\"\"\n if not file_list:\n msg = \"No files to process.\"\n raise ValueError(msg)\n\n def process_file_standard(file_path: str, *, silent_errors: bool = False) -> Data | None:\n try:\n return parse_text_file_to_data(file_path, silent_errors=silent_errors)\n except FileNotFoundError as e:\n self.log(f\"File not found: {file_path}. Error: {e}\")\n if not silent_errors:\n raise\n return None\n except Exception as e:\n self.log(f\"Unexpected error processing {file_path}: {e}\")\n if not silent_errors:\n raise\n return None\n\n # Advanced path: only for a single Docling-compatible file\n if len(file_list) == 1:\n file_path = str(file_list[0].path)\n if self.advanced_mode and self._is_docling_compatible(file_path):\n advanced_data: Data | None = self._process_docling_in_subprocess(file_path)\n\n # --- UNNEST: expand each element in `doc` to its own Data row\n payload = getattr(advanced_data, \"data\", {}) or {}\n doc_rows = payload.get(\"doc\")\n if isinstance(doc_rows, list):\n rows: list[Data | None] = [\n Data(\n data={\n \"file_path\": file_path,\n **(item if isinstance(item, dict) else {\"value\": item}),\n },\n )\n for item in doc_rows\n ]\n return self.rollup_data(file_list, rows)\n\n # If not structured, keep as-is (e.g., markdown export or error dict)\n return self.rollup_data(file_list, [advanced_data])\n\n # Standard multi-file (or single non-advanced) path\n concurrency = 1 if not self.use_multithreading else max(1, self.concurrency_multithreading)\n file_paths = [str(f.path) for f in file_list]\n self.log(f\"Starting parallel processing of {len(file_paths)} files with concurrency: {concurrency}.\")\n my_data = parallel_load_data(\n file_paths,\n silent_errors=self.silent_errors,\n load_function=process_file_standard,\n max_concurrency=concurrency,\n )\n return self.rollup_data(file_list, my_data)\n\n # ------------------------------ Output helpers -----------------------------------\n\n def load_files_advanced(self) -> DataFrame:\n \"\"\"Load files using advanced Docling processing and export to an advanced format.\"\"\"\n self.markdown = False\n return self.load_files()\n\n def load_files_markdown(self) -> Message:\n \"\"\"Load files using advanced Docling processing and export to Markdown format.\"\"\"\n self.markdown = True\n result = self.load_files()\n return Message(text=str(result.text[0]))\n"
},
"concurrency_multithreading": {
"_input_type": "IntInput",
"advanced": true,
"display_name": "Processing Concurrency",
"dynamic": false,
"info": "When multiple files are being processed, the number of files to process concurrently.",
"list": false,
"list_add_label": "Add More",
"name": "concurrency_multithreading",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "int",
"value": 1
},
"delete_server_file_after_processing": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Delete Server File After Processing",
"dynamic": false,
"info": "If true, the Server File Path will be deleted after processing.",
"list": false,
"list_add_label": "Add More",
"name": "delete_server_file_after_processing",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": true
},
"file_path": {
"_input_type": "HandleInput",
"advanced": true,
"display_name": "Server File Path",
"dynamic": false,
"info": "Data object with a 'file_path' property pointing to server file or a Message object with a path to the file. Supercedes 'Path' but supports same file types.",
"input_types": [
"Data",
"Message"
],
"list": true,
"list_add_label": "Add More",
"name": "file_path",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "other",
"value": ""
},
"ignore_unspecified_files": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Ignore Unspecified Files",
"dynamic": false,
"info": "If true, Data with no 'file_path' property will be ignored.",
"list": false,
"list_add_label": "Add More",
"name": "ignore_unspecified_files",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": false
},
"ignore_unsupported_extensions": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Ignore Unsupported Extensions",
"dynamic": false,
"info": "If true, files with unsupported extensions will not be processed.",
"list": false,
"list_add_label": "Add More",
"name": "ignore_unsupported_extensions",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": true
},
"path": {
"_input_type": "FileInput",
"advanced": false,
"display_name": "Files",
"dynamic": false,
"fileTypes": [
"txt",
"md",
"mdx",
"csv",
"json",
"yaml",
"yml",
"xml",
"html",
"htm",
"pdf",
"docx",
"py",
"sh",
"sql",
"js",
"ts",
"tsx",
"zip",
"tar",
"tgz",
"bz2",
"gz"
],
"file_path": [],
"info": "Supported file extensions: txt, md, mdx, csv, json, yaml, yml, xml, html, htm, pdf, docx, py, sh, sql, js, ts, tsx; optionally bundled in file extensions: zip, tar, tgz, bz2, gz",
"list": true,
"list_add_label": "Add More",
"name": "path",
"placeholder": "",
"real_time_refresh": true,
"required": false,
"show": true,
"temp_file": false,
"title_case": false,
"trace_as_metadata": true,
"type": "file",
"value": ""
},
"separator": {
"_input_type": "StrInput",
"advanced": true,
"display_name": "Separator",
"dynamic": false,
"info": "Specify the separator to use between multiple outputs in Message format.",
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
"name": "separator",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "str",
"value": "\n\n"
},
"silent_errors": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Silent Errors",
"dynamic": false,
"info": "If true, errors will not raise an exception.",
"list": false,
"list_add_label": "Add More",
"name": "silent_errors",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": false
},
"use_multithreading": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "[Deprecated] Use Multithreading",
"dynamic": false,
"info": "Set 'Processing Concurrency' greater than 1 to enable multithreading.",
"list": false,
"list_add_label": "Add More",
"name": "use_multithreading",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": true
}
},
"tool_mode": false
},
"showNode": true,
"type": "File"
},
"dragging": false,
"id": "File-PSU37",
"measured": {
"height": 230,
"width": 320
},
"position": {
"x": 1330.7650978046952,
"y": 1431.5905495627503
},
"selected": false,
"type": "genericNode"
},
{
"data": {
"description": "OpenSearch Vector Store with advanced, customizable search capabilities.",
"display_name": "OpenSearch",
"id": "OpenSearch-Mkw1W",
"node": {
"base_classes": [
"Data",
"DataFrame",
"VectorStore"
],
"beta": false,
"conditional_paths": [],
"custom_fields": {},
"description": "OpenSearch Vector Store with advanced, customizable search capabilities.",
"display_name": "OpenSearch",
"documentation": "",
"edited": false,
"field_order": [
"opensearch_url",
"index_name",
"ingest_data",
"search_query",
"should_cache_vector_store",
"embedding",
"search_type",
"number_of_results",
"search_score_threshold",
"username",
"password",
"use_ssl",
"verify_certs",
"hybrid_search_query"
],
"frozen": false,
"icon": "OpenSearch",
"legacy": false,
"metadata": {
"code_hash": "972b714acf6b",
"dependencies": {
"dependencies": [
{
"name": "langchain_community",
"version": "0.3.21"
},
{
"name": "langflow",
"version": "1.5.0.post2"
}
],
"total_dependencies": 2
},
"module": "custom_components.opensearch"
},
"minimized": false,
"output_types": [],
"outputs": [
{
"allows_loop": false,
"cache": true,
"display_name": "Search Results",
"group_outputs": false,
"method": "search_documents",
"name": "search_results",
"options": null,
"required_inputs": null,
"selected": "Data",
"tool_mode": true,
"types": [
"Data"
],
"value": "__UNDEFINED__"
},
{
"allows_loop": false,
"cache": true,
"display_name": "DataFrame",
"group_outputs": false,
"method": "as_dataframe",
"name": "dataframe",
"options": null,
"required_inputs": null,
"selected": "DataFrame",
"tool_mode": true,
"types": [
"DataFrame"
],
"value": "__UNDEFINED__"
},
{
"allows_loop": false,
"cache": true,
"display_name": "Vector Store Connection",
"group_outputs": false,
"hidden": true,
"method": "as_vector_store",
"name": "vectorstoreconnection",
"options": null,
"required_inputs": null,
"selected": "VectorStore",
"tool_mode": true,
"types": [
"VectorStore"
],
"value": "__UNDEFINED__"
}
],
"pinned": false,
"template": {
"_type": "Component",
"code": {
"advanced": true,
"dynamic": true,
"fileTypes": [],
"file_path": "",
"info": "",
"list": false,
"load_from_db": false,
"multiline": true,
"name": "code",
"password": false,
"placeholder": "",
"required": true,
"show": true,
"title_case": false,
"type": "code",
"value": "import json\nfrom typing import Any\n\nfrom langchain_community.vectorstores import OpenSearchVectorSearch\n\nfrom langflow.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom langflow.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom langflow.io import (\n BoolInput,\n DropdownInput,\n FloatInput,\n HandleInput,\n IntInput,\n MultilineInput,\n SecretStrInput,\n StrInput,\n)\nfrom langflow.schema.data import Data\nfrom fastapi.encoders import jsonable_encoder\nfrom langchain_core.documents import Document\nimport json\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store with advanced, customizable search capabilities.\"\"\"\n\n display_name: str = \"OpenSearch\"\n description: str = \"OpenSearch Vector Store with advanced, customizable search capabilities.\"\n name = \"OpenSearch\"\n icon = \"OpenSearch\"\n\n inputs = [\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=\"URL for OpenSearch cluster (e.g. https://192.168.1.1:9200).\",\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"documents\",\n info=\"The index name where the vectors will be stored in OpenSearch cluster.\",\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Vector Field\",\n value=\"chunk_embedding\",\n info=\"Document field embeddings are stored in.\",\n advanced=True,\n ),\n StrInput(\n name=\"text_field\",\n display_name=\"Text Field\", \n value=\"text\",\n info=\"Document field the text of the document is stored in.\",\n advanced=True,\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\"],\n value=\"nmslib\",\n info=\"Vector search engine to use.\",\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Space Type\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=\"Distance metric for vector similarity.\",\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=100,\n info=\"Size of the dynamic list used during k-NN graph creation.\",\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=\"Number of bidirectional links created for each new element.\",\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs,\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n DropdownInput(\n name=\"search_type\",\n display_name=\"Search Type\",\n options=[\"similarity\", \"similarity_score_threshold\", \"mmr\"],\n value=\"similarity\",\n advanced=True,\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Number of Results\",\n info=\"Number of results to return.\",\n advanced=True,\n value=4,\n ),\n FloatInput(\n name=\"search_score_threshold\",\n display_name=\"Search Score Threshold\",\n info=\"Minimum similarity score threshold for search results.\",\n value=0.0,\n advanced=True,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n advanced=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"Password\",\n value=\"OPENSEARCH_PASSWORD\",\n advanced=True,\n ),\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL\",\n value=True,\n advanced=True,\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify Certificates\",\n value=False,\n advanced=True,\n ),\n MultilineInput(\n name=\"hybrid_search_query\",\n display_name=\"Hybrid Search Query\",\n value=\"\",\n advanced=True,\n info=(\n \"Provide a custom hybrid search query in JSON format. This allows you to combine \"\n \"vector similarity and keyword matching.\"\n ),\n ),\n ]\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearchVectorSearch:\n \"\"\"Builds the OpenSearch Vector Store object.\"\"\"\n try:\n from langchain_community.vectorstores import OpenSearchVectorSearch\n except ImportError as e:\n error_message = f\"Failed to import required modules: {e}\"\n self.log(error_message)\n raise ImportError(error_message) from e\n\n try:\n opensearch = OpenSearchVectorSearch(\n index_name=self.index_name,\n embedding_function=self.embedding,\n opensearch_url=self.opensearch_url,\n http_auth=(self.username, self.password),\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n engine=self.engine,\n vector_field=self.vector_field,\n text_field=self.text_field,\n space_type=self.space_type,\n ef_construction=self.ef_construction,\n m=self.m,\n )\n except Exception as e:\n error_message = f\"Failed to create OpenSearchVectorSearch instance: {e}\"\n self.log(error_message)\n raise RuntimeError(error_message) from e\n\n if self.ingest_data:\n self._add_documents_to_vector_store(opensearch)\n\n return opensearch\n\n def _add_documents_to_vector_store(self, vector_store: \"OpenSearchVectorSearch\") -> None:\n \"\"\"Adds documents to the Vector Store.\"\"\"\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n documents = []\n for _input in self.ingest_data or []:\n if isinstance(_input, Data):\n doc = Document(\n page_content=_input.get_text(), \n metadata=jsonable_encoder(_input.data) if _input.data else {}\n )\n documents.append(doc)\n else:\n error_message = f\"Expected Data object, got {type(_input)}\"\n self.log(error_message)\n raise TypeError(error_message)\n\n if documents and self.embedding is not None:\n self.log(f\"Adding {len(documents)} documents to the Vector Store.\")\n try:\n vector_store.add_documents(\n documents, \n vector_field=self.vector_field,\n text_field=self.text_field\n )\n except Exception as e:\n error_message = f\"Error adding documents to Vector Store: {e}\"\n self.log(error_message)\n raise RuntimeError(error_message) from e\n else:\n self.log(\"No documents to add to the Vector Store.\")\n\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Search for similar documents in the vector store or retrieve all documents if no query is provided.\"\"\"\n \n vector_store = self.build_vector_store()\n try:\n query = query or \"\"\n\n if self.hybrid_search_query.strip():\n try:\n hybrid_query = json.loads(self.hybrid_search_query)\n except json.JSONDecodeError as e:\n error_message = f\"Invalid hybrid search query JSON: {e}\"\n self.log(error_message)\n raise ValueError(error_message) from e\n\n results = vector_store.client.search(index=self.index_name, body=hybrid_query)\n\n processed_results = []\n for hit in results.get(\"hits\", {}).get(\"hits\", []):\n source = hit.get(\"_source\", {})\n text = source.get(self.text_field, \"\")\n metadata = source.get(\"metadata\", {})\n\n if isinstance(text, dict):\n text = text.get(\"text\", \"\")\n\n processed_results.append(\n {\n \"page_content\": text,\n \"metadata\": metadata,\n }\n )\n return processed_results\n\n search_kwargs = {\n \"k\": self.number_of_results,\n \"vector_field\": self.vector_field,\n \"text_field\": self.text_field\n }\n search_type = self.search_type.lower()\n\n if search_type == \"similarity\":\n results = vector_store.similarity_search(query, **search_kwargs)\n return [{\"page_content\": doc.page_content, \"metadata\": doc.metadata} for doc in results]\n if search_type == \"similarity_score_threshold\":\n search_kwargs[\"score_threshold\"] = self.search_score_threshold\n results = vector_store.similarity_search_with_relevance_scores(query, **search_kwargs)\n return [\n {\n \"page_content\": doc.page_content,\n \"metadata\": doc.metadata,\n \"score\": score,\n }\n for doc, score in results\n ]\n if search_type == \"mmr\":\n results = vector_store.max_marginal_relevance_search(query, **search_kwargs)\n return [{\"page_content\": doc.page_content, \"metadata\": doc.metadata} for doc in results]\n\n except Exception as e:\n error_message = f\"Error during search: {e}\"\n self.log(error_message)\n raise RuntimeError(error_message) from e\n\n error_message = f\"Error during search. Invalid search type: {self.search_type}\"\n self.log(error_message)\n raise ValueError(error_message)\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search for documents in the vector store based on the search input.\n\n If no search input is provided, retrieve all documents.\n \"\"\"\n try:\n query = self.search_query.strip() if self.search_query else None\n results = self.search(query)\n retrieved_data = [\n Data(\n file_path=result[\"metadata\"].get(\"file_path\", \"\"),\n text=result[\"page_content\"],\n )\n for result in results\n ]\n except Exception as e:\n error_message = f\"Error during document search: {e}\"\n self.log(error_message)\n raise RuntimeError(error_message) from e\n\n self.status = retrieved_data\n return retrieved_data\n"
},
"embedding": {
"_input_type": "HandleInput",
"advanced": false,
"display_name": "Embedding",
"dynamic": false,
"info": "",
"input_types": [
"Embeddings"
],
"list": false,
"list_add_label": "Add More",
"name": "embedding",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "other",
"value": ""
},
"hybrid_search_query": {
"_input_type": "MultilineInput",
"advanced": true,
"copy_field": false,
"display_name": "Hybrid Search Query",
"dynamic": false,
"info": "Provide a custom hybrid search query in JSON format. This allows you to combine vector similarity and keyword matching.",
"input_types": [
"Message"
],
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
"multiline": true,
"name": "hybrid_search_query",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "str",
"value": ""
},
"index_name": {
"_input_type": "StrInput",
"advanced": false,
"display_name": "Index Name",
"dynamic": false,
"info": "The index name where the vectors will be stored in OpenSearch cluster.",
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
"name": "index_name",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "str",
"value": "documents"
},
"ingest_data": {
"_input_type": "HandleInput",
"advanced": false,
"display_name": "Ingest Data",
"dynamic": false,
"info": "",
"input_types": [
"Data",
"DataFrame"
],
"list": true,
"list_add_label": "Add More",
"name": "ingest_data",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"trace_as_metadata": true,
"type": "other",
"value": ""
},
"number_of_results": {
"_input_type": "IntInput",
"advanced": true,
"display_name": "Number of Results",
"dynamic": false,
"info": "Number of results to return.",
"list": false,
"list_add_label": "Add More",
"name": "number_of_results",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "int",
"value": 4
},
"opensearch_url": {
"_input_type": "StrInput",
"advanced": false,
"display_name": "OpenSearch URL",
"dynamic": false,
"info": "URL for OpenSearch cluster (e.g. https://192.168.1.1:9200).",
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
"name": "opensearch_url",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "str",
"value": "http://opensearch:9200"
},
"password": {
"_input_type": "SecretStrInput",
"advanced": true,
"display_name": "Password",
"dynamic": false,
"info": "",
"input_types": [],
"load_from_db": true,
"name": "password",
"password": true,
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"type": "str",
"value": "OPENSEARCH_PASSWORD"
},
"search_query": {
"_input_type": "QueryInput",
"advanced": false,
"display_name": "Search Query",
"dynamic": false,
"info": "Enter a query to run a similarity search.",
"input_types": [
"Message"
],
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
"name": "search_query",
"placeholder": "Enter a query...",
"required": false,
"show": true,
"title_case": false,
"tool_mode": true,
"trace_as_input": true,
"trace_as_metadata": true,
"type": "query",
"value": ""
},
"search_score_threshold": {
"_input_type": "FloatInput",
"advanced": true,
"display_name": "Search Score Threshold",
"dynamic": false,
"info": "Minimum similarity score threshold for search results.",
"list": false,
"list_add_label": "Add More",
"name": "search_score_threshold",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "float",
"value": 0
},
"search_type": {
"_input_type": "DropdownInput",
"advanced": true,
"combobox": false,
"dialog_inputs": {},
"display_name": "Search Type",
"dynamic": false,
"info": "",
"name": "search_type",
"options": [
"similarity",
"similarity_score_threshold",
"mmr"
],
"options_metadata": [],
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"toggle": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "str",
"value": "similarity"
},
"should_cache_vector_store": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Cache Vector Store",
"dynamic": false,
"info": "If True, the vector store will be cached for the current build of the component. This is useful for components that have multiple output methods and want to share the same vector store.",
"list": false,
"list_add_label": "Add More",
"name": "should_cache_vector_store",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": true
},
"use_ssl": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Use SSL",
"dynamic": false,
"info": "",
"list": false,
"list_add_label": "Add More",
"name": "use_ssl",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": true
},
"username": {
"_input_type": "StrInput",
"advanced": true,
"display_name": "Username",
"dynamic": false,
"info": "",
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
"name": "username",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "str",
"value": "admin"
},
"verify_certs": {
"_input_type": "BoolInput",
"advanced": true,
"display_name": "Verify Certificates",
"dynamic": false,
"info": "",
"list": false,
"list_add_label": "Add More",
"name": "verify_certs",
"placeholder": "",
"required": false,
"show": true,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
"type": "bool",
"value": false
}
},
"tool_mode": false
},
"selected_output": "search_results",
"showNode": true,
"type": "OpenSearch"
},
"dragging": false,
"id": "OpenSearch-Mkw1W",
"measured": {
"height": 518,
"width": 320
},
"position": {
"x": 2136.4456339674302,
"y": 1460.3160066924486
},
"selected": false,
"type": "genericNode"
}
],
"viewport": {
"x": -1173.5436043881646,
"y": -1289.0306227762003,
"zoom": 1.0020797567291742
}
},
"description": "Load your data for chat context with Retrieval Augmented Generation.",
"endpoint_name": null,
"id": "5488df7c-b93f-4f87-a446-b67028bc0813",
"is_component": false,
"last_tested_version": "1.5.0.post2",
"name": "OpenSearch Ingestion Flow",
"tags": [
"openai",
"astradb",
"rag",
"q-a"
]
}
Reference in a new issue View git blame Copy permalink