3879 lines
No EOL
232 KiB
JSON
3879 lines
No EOL
232 KiB
JSON
{
|
|
"data": {
|
|
"edges": [
|
|
{
|
|
"animated": false,
|
|
"className": "",
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "SplitText",
|
|
"id": "SplitText-QIKhg",
|
|
"name": "dataframe",
|
|
"output_types": [
|
|
"DataFrame"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "ingest_data",
|
|
"id": "OpenSearchHybrid-Ve6bS",
|
|
"inputTypes": [
|
|
"Data",
|
|
"DataFrame"
|
|
],
|
|
"type": "other"
|
|
}
|
|
},
|
|
"id": "xy-edge__SplitText-QIKhg{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}",
|
|
"selected": false,
|
|
"source": "SplitText-QIKhg",
|
|
"sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}",
|
|
"target": "OpenSearchHybrid-Ve6bS",
|
|
"targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}"
|
|
},
|
|
{
|
|
"animated": false,
|
|
"className": "",
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "OpenAIEmbeddings",
|
|
"id": "OpenAIEmbeddings-joRJ6",
|
|
"name": "embeddings",
|
|
"output_types": [
|
|
"Embeddings"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "embedding",
|
|
"id": "OpenSearchHybrid-Ve6bS",
|
|
"inputTypes": [
|
|
"Embeddings"
|
|
],
|
|
"type": "other"
|
|
}
|
|
},
|
|
"id": "xy-edge__OpenAIEmbeddings-joRJ6{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}",
|
|
"selected": false,
|
|
"source": "OpenAIEmbeddings-joRJ6",
|
|
"sourceHandle": "{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}",
|
|
"target": "OpenSearchHybrid-Ve6bS",
|
|
"targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}"
|
|
},
|
|
{
|
|
"animated": false,
|
|
"className": "",
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "ChatInput",
|
|
"id": "ChatInput-WLvBD",
|
|
"name": "message",
|
|
"output_types": [
|
|
"Message"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "urls",
|
|
"id": "URLComponent-lnA0q",
|
|
"inputTypes": [
|
|
"Message"
|
|
],
|
|
"type": "str"
|
|
}
|
|
},
|
|
"id": "xy-edge__ChatInput-WLvBD{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}-URLComponent-lnA0q{œfieldNameœ:œurlsœ,œidœ:œURLComponent-lnA0qœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}",
|
|
"selected": false,
|
|
"source": "ChatInput-WLvBD",
|
|
"sourceHandle": "{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}",
|
|
"target": "URLComponent-lnA0q",
|
|
"targetHandle": "{œfieldNameœ:œurlsœ,œidœ:œURLComponent-lnA0qœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}"
|
|
},
|
|
{
|
|
"animated": false,
|
|
"className": "",
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "URLComponent",
|
|
"id": "URLComponent-lnA0q",
|
|
"name": "page_results",
|
|
"output_types": [
|
|
"DataFrame"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "df",
|
|
"id": "DataFrameOperations-hqIoy",
|
|
"inputTypes": [
|
|
"DataFrame"
|
|
],
|
|
"type": "other"
|
|
}
|
|
},
|
|
"id": "xy-edge__URLComponent-lnA0q{œdataTypeœ:œURLComponentœ,œidœ:œURLComponent-lnA0qœ,œnameœ:œpage_resultsœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-hqIoy{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}",
|
|
"selected": false,
|
|
"source": "URLComponent-lnA0q",
|
|
"sourceHandle": "{œdataTypeœ:œURLComponentœ,œidœ:œURLComponent-lnA0qœ,œnameœ:œpage_resultsœ,œoutput_typesœ:[œDataFrameœ]}",
|
|
"target": "DataFrameOperations-hqIoy",
|
|
"targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}"
|
|
},
|
|
{
|
|
"animated": false,
|
|
"className": "",
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "ChatInput",
|
|
"id": "ChatInput-WLvBD",
|
|
"name": "message",
|
|
"output_types": [
|
|
"Message"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "new_column_value",
|
|
"id": "DataFrameOperations-hqIoy",
|
|
"inputTypes": [
|
|
"Message"
|
|
],
|
|
"type": "str"
|
|
}
|
|
},
|
|
"id": "xy-edge__ChatInput-WLvBD{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}-DataFrameOperations-hqIoy{œfieldNameœ:œnew_column_valueœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}",
|
|
"selected": false,
|
|
"source": "ChatInput-WLvBD",
|
|
"sourceHandle": "{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}",
|
|
"target": "DataFrameOperations-hqIoy",
|
|
"targetHandle": "{œfieldNameœ:œnew_column_valueœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}"
|
|
},
|
|
{
|
|
"animated": false,
|
|
"className": "",
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "DataFrameOperations",
|
|
"id": "DataFrameOperations-hqIoy",
|
|
"name": "output",
|
|
"output_types": [
|
|
"DataFrame"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "df",
|
|
"id": "DataFrameOperations-A98BL",
|
|
"inputTypes": [
|
|
"DataFrame"
|
|
],
|
|
"type": "other"
|
|
}
|
|
},
|
|
"id": "xy-edge__DataFrameOperations-hqIoy{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-hqIoyœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-A98BL{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-A98BLœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}",
|
|
"selected": false,
|
|
"source": "DataFrameOperations-hqIoy",
|
|
"sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-hqIoyœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}",
|
|
"target": "DataFrameOperations-A98BL",
|
|
"targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-A98BLœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}"
|
|
},
|
|
{
|
|
"animated": false,
|
|
"className": "",
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "DataFrameOperations",
|
|
"id": "DataFrameOperations-A98BL",
|
|
"name": "output",
|
|
"output_types": [
|
|
"DataFrame"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "data_inputs",
|
|
"id": "SplitText-QIKhg",
|
|
"inputTypes": [
|
|
"Data",
|
|
"DataFrame",
|
|
"Message"
|
|
],
|
|
"type": "other"
|
|
}
|
|
},
|
|
"id": "xy-edge__DataFrameOperations-A98BL{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-A98BLœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-SplitText-QIKhg{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}",
|
|
"selected": false,
|
|
"source": "DataFrameOperations-A98BL",
|
|
"sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-A98BLœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}",
|
|
"target": "SplitText-QIKhg",
|
|
"targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}"
|
|
},
|
|
{
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "SplitText",
|
|
"id": "SplitText-QIKhg",
|
|
"name": "dataframe",
|
|
"output_types": [
|
|
"DataFrame"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "df",
|
|
"id": "DataFrameOperations-RhKoe",
|
|
"inputTypes": [
|
|
"DataFrame"
|
|
],
|
|
"type": "other"
|
|
}
|
|
},
|
|
"id": "xy-edge__SplitText-QIKhg{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-RhKoe{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-RhKoeœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}",
|
|
"source": "SplitText-QIKhg",
|
|
"sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}",
|
|
"target": "DataFrameOperations-RhKoe",
|
|
"targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-RhKoeœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}"
|
|
},
|
|
{
|
|
"data": {
|
|
"sourceHandle": {
|
|
"dataType": "DataFrameOperations",
|
|
"id": "DataFrameOperations-RhKoe",
|
|
"name": "output",
|
|
"output_types": [
|
|
"DataFrame"
|
|
]
|
|
},
|
|
"targetHandle": {
|
|
"fieldName": "input_value",
|
|
"id": "ChatOutput-Q1dhr",
|
|
"inputTypes": [
|
|
"Data",
|
|
"DataFrame",
|
|
"Message"
|
|
],
|
|
"type": "other"
|
|
}
|
|
},
|
|
"id": "xy-edge__DataFrameOperations-RhKoe{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-RhKoeœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-ChatOutput-Q1dhr{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Q1dhrœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}",
|
|
"source": "DataFrameOperations-RhKoe",
|
|
"sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-RhKoeœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}",
|
|
"target": "ChatOutput-Q1dhr",
|
|
"targetHandle": "{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Q1dhrœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}"
|
|
}
|
|
],
|
|
"nodes": [
|
|
{
|
|
"data": {
|
|
"description": "Split text into chunks based on specified criteria.",
|
|
"display_name": "Split Text",
|
|
"id": "SplitText-QIKhg",
|
|
"node": {
|
|
"base_classes": [
|
|
"DataFrame"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Split text into chunks based on specified criteria.",
|
|
"display_name": "Split Text",
|
|
"documentation": "https://docs.langflow.org/components-processing#split-text",
|
|
"edited": true,
|
|
"field_order": [
|
|
"data_inputs",
|
|
"chunk_overlap",
|
|
"chunk_size",
|
|
"separator",
|
|
"text_key",
|
|
"keep_separator"
|
|
],
|
|
"frozen": false,
|
|
"icon": "scissors-line-dashed",
|
|
"legacy": false,
|
|
"lf_version": "1.6.0",
|
|
"metadata": {
|
|
"code_hash": "f2867efda61f",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "langchain_text_splitters",
|
|
"version": "0.3.9"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 2
|
|
},
|
|
"module": "custom_components.split_text"
|
|
},
|
|
"minimized": false,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Chunks",
|
|
"group_outputs": false,
|
|
"hidden": null,
|
|
"method": "split_text",
|
|
"name": "dataframe",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "DataFrame",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"DataFrame"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"chunk_overlap": {
|
|
"_input_type": "IntInput",
|
|
"advanced": false,
|
|
"display_name": "Chunk Overlap",
|
|
"dynamic": false,
|
|
"info": "Number of characters to overlap between chunks.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "chunk_overlap",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 200
|
|
},
|
|
"chunk_size": {
|
|
"_input_type": "IntInput",
|
|
"advanced": false,
|
|
"display_name": "Chunk Size",
|
|
"dynamic": false,
|
|
"info": "The maximum length of each chunk. Text is first split by separator, then chunks are merged up to this size. Individual splits larger than this won't be further divided.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "chunk_size",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 1000
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "from langchain_text_splitters import CharacterTextSplitter\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.util import unescape_string\n\n\nclass SplitTextComponent(Component):\n display_name: str = \"Split Text\"\n description: str = \"Split text into chunks based on specified criteria.\"\n documentation: str = \"https://docs.langflow.org/components-processing#split-text\"\n icon = \"scissors-line-dashed\"\n name = \"SplitText\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Input\",\n info=\"The data with texts to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n IntInput(\n name=\"chunk_overlap\",\n display_name=\"Chunk Overlap\",\n info=\"Number of characters to overlap between chunks.\",\n value=200,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n info=(\n \"The maximum length of each chunk. Text is first split by separator, \"\n \"then chunks are merged up to this size. \"\n \"Individual splits larger than this won't be further divided.\"\n ),\n value=1000,\n ),\n MessageTextInput(\n name=\"separator\",\n display_name=\"Separator\",\n info=(\n \"The character to split on. Use \\\\n for newline. \"\n \"Examples: \\\\n\\\\n for paragraphs, \\\\n for lines, . for sentences\"\n ),\n value=\"\\n\",\n ),\n MessageTextInput(\n name=\"text_key\",\n display_name=\"Text Key\",\n info=\"The key to use for the text column.\",\n value=\"text\",\n advanced=True,\n ),\n DropdownInput(\n name=\"keep_separator\",\n display_name=\"Keep Separator\",\n info=\"Whether to keep the separator in the output chunks and where to place it.\",\n options=[\"False\", \"True\", \"Start\", \"End\"],\n value=\"False\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Chunks\", name=\"dataframe\", method=\"split_text\"),\n ]\n\n def _docs_to_data(self, docs) -> list[Data]:\n return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n\n def _fix_separator(self, separator: str) -> str:\n \"\"\"Fix common separator issues and convert to proper format.\"\"\"\n if separator == \"/n\":\n return \"\\n\"\n if separator == \"/t\":\n return \"\\t\"\n return separator\n\n def split_text_base(self):\n separator = self._fix_separator(self.separator)\n separator = unescape_string(separator)\n\n if isinstance(self.data_inputs, DataFrame):\n if not len(self.data_inputs):\n msg = \"DataFrame is empty\"\n raise TypeError(msg)\n\n self.data_inputs.text_key = self.text_key\n try:\n documents = self.data_inputs.to_lc_documents()\n except Exception as e:\n msg = f\"Error converting DataFrame to documents: {e}\"\n raise TypeError(msg) from e\n elif isinstance(self.data_inputs, Message):\n self.data_inputs = [self.data_inputs.to_data()]\n return self.split_text_base()\n else:\n if not self.data_inputs:\n msg = \"No data inputs provided\"\n raise TypeError(msg)\n\n documents = []\n if isinstance(self.data_inputs, Data):\n self.data_inputs.text_key = self.text_key\n documents = [self.data_inputs.to_lc_document()]\n else:\n try:\n documents = [input_.to_lc_document() for input_ in self.data_inputs if isinstance(input_, Data)]\n if not documents:\n msg = f\"No valid Data inputs found in {type(self.data_inputs)}\"\n raise TypeError(msg)\n except AttributeError as e:\n msg = f\"Invalid input type in collection: {e}\"\n raise TypeError(msg) from e\n try:\n # Convert string 'False'/'True' to boolean\n keep_sep = self.keep_separator\n if isinstance(keep_sep, str):\n if keep_sep.lower() == \"false\":\n keep_sep = False\n elif keep_sep.lower() == \"true\":\n keep_sep = True\n # 'start' and 'end' are kept as strings\n\n splitter = CharacterTextSplitter(\n chunk_overlap=self.chunk_overlap,\n chunk_size=self.chunk_size,\n separator=separator,\n keep_separator=keep_sep,\n )\n return splitter.split_documents(documents)\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n def split_text(self) -> DataFrame:\n return DataFrame(self._docs_to_data(self.split_text_base()))\n"
|
|
},
|
|
"data_inputs": {
|
|
"_input_type": "HandleInput",
|
|
"advanced": false,
|
|
"display_name": "Input",
|
|
"dynamic": false,
|
|
"info": "The data with texts to split in chunks.",
|
|
"input_types": [
|
|
"Data",
|
|
"DataFrame",
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "data_inputs",
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"trace_as_metadata": true,
|
|
"type": "other",
|
|
"value": ""
|
|
},
|
|
"keep_separator": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": true,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Keep Separator",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "Whether to keep the separator in the output chunks and where to place it.",
|
|
"name": "keep_separator",
|
|
"options": [
|
|
"False",
|
|
"True",
|
|
"Start",
|
|
"End"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "False"
|
|
},
|
|
"separator": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Separator",
|
|
"dynamic": false,
|
|
"info": "The character to split on. Use \\n for newline. Examples: \\n\\n for paragraphs, \\n for lines, . for sentences",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "separator",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "\n"
|
|
},
|
|
"text_key": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Text Key",
|
|
"dynamic": false,
|
|
"info": "The key to use for the text column.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "text_key",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "text"
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"selected_output": "chunks",
|
|
"type": "SplitText"
|
|
},
|
|
"dragging": false,
|
|
"height": 475,
|
|
"id": "SplitText-QIKhg",
|
|
"measured": {
|
|
"height": 475,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 2299.485091096586,
|
|
"y": 1430.4506304359015
|
|
},
|
|
"positionAbsolute": {
|
|
"x": 1683.4543896546102,
|
|
"y": 1350.7871623588553
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode",
|
|
"width": 320
|
|
},
|
|
{
|
|
"data": {
|
|
"description": "Generate embeddings using OpenAI models.",
|
|
"display_name": "OpenAI Embeddings",
|
|
"id": "OpenAIEmbeddings-joRJ6",
|
|
"node": {
|
|
"base_classes": [
|
|
"Embeddings"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Generate embeddings using OpenAI models.",
|
|
"display_name": "OpenAI Embeddings",
|
|
"documentation": "",
|
|
"edited": false,
|
|
"field_order": [
|
|
"default_headers",
|
|
"default_query",
|
|
"chunk_size",
|
|
"client",
|
|
"deployment",
|
|
"embedding_ctx_length",
|
|
"max_retries",
|
|
"model",
|
|
"model_kwargs",
|
|
"openai_api_key",
|
|
"openai_api_base",
|
|
"openai_api_type",
|
|
"openai_api_version",
|
|
"openai_organization",
|
|
"openai_proxy",
|
|
"request_timeout",
|
|
"show_progress_bar",
|
|
"skip_empty",
|
|
"tiktoken_model_name",
|
|
"tiktoken_enable",
|
|
"dimensions"
|
|
],
|
|
"frozen": false,
|
|
"icon": "OpenAI",
|
|
"legacy": false,
|
|
"lf_version": "1.6.0",
|
|
"metadata": {
|
|
"code_hash": "8a658ed6d4c9",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "langchain_openai",
|
|
"version": "0.3.23"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 2
|
|
},
|
|
"module": "custom_components.openai_embeddings"
|
|
},
|
|
"minimized": false,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Embedding Model",
|
|
"group_outputs": false,
|
|
"method": "build_embeddings",
|
|
"name": "embeddings",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "Embeddings",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"Embeddings"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"chunk_size": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "Chunk Size",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "chunk_size",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 1000
|
|
},
|
|
"client": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Client",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "client",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "from langchain_openai import OpenAIEmbeddings\n\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import BoolInput, DictInput, DropdownInput, FloatInput, IntInput, MessageTextInput, SecretStrInput\n\n\nclass OpenAIEmbeddingsComponent(LCEmbeddingsModel):\n display_name = \"OpenAI Embeddings\"\n description = \"Generate embeddings using OpenAI models.\"\n icon = \"OpenAI\"\n name = \"OpenAIEmbeddings\"\n\n inputs = [\n DictInput(\n name=\"default_headers\",\n display_name=\"Default Headers\",\n advanced=True,\n info=\"Default headers to use for the API request.\",\n ),\n DictInput(\n name=\"default_query\",\n display_name=\"Default Query\",\n advanced=True,\n info=\"Default query parameters to use for the API request.\",\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n MessageTextInput(name=\"client\", display_name=\"Client\", advanced=True),\n MessageTextInput(name=\"deployment\", display_name=\"Deployment\", advanced=True),\n IntInput(name=\"embedding_ctx_length\", display_name=\"Embedding Context Length\", advanced=True, value=1536),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", value=3, advanced=True),\n DropdownInput(\n name=\"model\",\n display_name=\"Model\",\n advanced=False,\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=\"text-embedding-3-small\",\n ),\n DictInput(name=\"model_kwargs\", display_name=\"Model Kwargs\", advanced=True),\n SecretStrInput(name=\"openai_api_key\", display_name=\"OpenAI API Key\", value=\"OPENAI_API_KEY\", required=True),\n MessageTextInput(name=\"openai_api_base\", display_name=\"OpenAI API Base\", advanced=True),\n MessageTextInput(name=\"openai_api_type\", display_name=\"OpenAI API Type\", advanced=True),\n MessageTextInput(name=\"openai_api_version\", display_name=\"OpenAI API Version\", advanced=True),\n MessageTextInput(\n name=\"openai_organization\",\n display_name=\"OpenAI Organization\",\n advanced=True,\n ),\n MessageTextInput(name=\"openai_proxy\", display_name=\"OpenAI Proxy\", advanced=True),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n BoolInput(name=\"skip_empty\", display_name=\"Skip Empty\", advanced=True),\n MessageTextInput(\n name=\"tiktoken_model_name\",\n display_name=\"TikToken Model Name\",\n advanced=True,\n ),\n BoolInput(\n name=\"tiktoken_enable\",\n display_name=\"TikToken Enable\",\n advanced=True,\n value=True,\n info=\"If False, you must have transformers installed.\",\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n return OpenAIEmbeddings(\n client=self.client or None,\n model=self.model,\n dimensions=self.dimensions or None,\n deployment=self.deployment or None,\n api_version=self.openai_api_version or None,\n base_url=self.openai_api_base or None,\n openai_api_type=self.openai_api_type or None,\n openai_proxy=self.openai_proxy or None,\n embedding_ctx_length=self.embedding_ctx_length,\n api_key=self.openai_api_key or None,\n organization=self.openai_organization or None,\n allowed_special=\"all\",\n disallowed_special=\"all\",\n chunk_size=self.chunk_size,\n max_retries=self.max_retries,\n timeout=self.request_timeout or None,\n tiktoken_enabled=self.tiktoken_enable,\n tiktoken_model_name=self.tiktoken_model_name or None,\n show_progress_bar=self.show_progress_bar,\n model_kwargs=self.model_kwargs,\n skip_empty=self.skip_empty,\n default_headers=self.default_headers or None,\n default_query=self.default_query or None,\n )\n"
|
|
},
|
|
"default_headers": {
|
|
"_input_type": "DictInput",
|
|
"advanced": true,
|
|
"display_name": "Default Headers",
|
|
"dynamic": false,
|
|
"info": "Default headers to use for the API request.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "default_headers",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"type": "dict",
|
|
"value": {}
|
|
},
|
|
"default_query": {
|
|
"_input_type": "DictInput",
|
|
"advanced": true,
|
|
"display_name": "Default Query",
|
|
"dynamic": false,
|
|
"info": "Default query parameters to use for the API request.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "default_query",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"type": "dict",
|
|
"value": {}
|
|
},
|
|
"deployment": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Deployment",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "deployment",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"dimensions": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "Dimensions",
|
|
"dynamic": false,
|
|
"info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "dimensions",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": ""
|
|
},
|
|
"embedding_ctx_length": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "Embedding Context Length",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "embedding_ctx_length",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 1536
|
|
},
|
|
"max_retries": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "Max Retries",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "max_retries",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 3
|
|
},
|
|
"model": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": false,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Model",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "",
|
|
"name": "model",
|
|
"options": [
|
|
"text-embedding-3-small",
|
|
"text-embedding-3-large",
|
|
"text-embedding-ada-002"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "text-embedding-3-small"
|
|
},
|
|
"model_kwargs": {
|
|
"_input_type": "DictInput",
|
|
"advanced": true,
|
|
"display_name": "Model Kwargs",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "model_kwargs",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"type": "dict",
|
|
"value": {}
|
|
},
|
|
"openai_api_base": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "OpenAI API Base",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "openai_api_base",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"openai_api_key": {
|
|
"_input_type": "SecretStrInput",
|
|
"advanced": false,
|
|
"display_name": "OpenAI API Key",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [],
|
|
"load_from_db": true,
|
|
"name": "openai_api_key",
|
|
"password": true,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "str",
|
|
"value": "OPENAI_API_KEY"
|
|
},
|
|
"openai_api_type": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "OpenAI API Type",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "openai_api_type",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"openai_api_version": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "OpenAI API Version",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "openai_api_version",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"openai_organization": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "OpenAI Organization",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "openai_organization",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"openai_proxy": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "OpenAI Proxy",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "openai_proxy",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"request_timeout": {
|
|
"_input_type": "FloatInput",
|
|
"advanced": true,
|
|
"display_name": "Request Timeout",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "request_timeout",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "float",
|
|
"value": ""
|
|
},
|
|
"show_progress_bar": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Show Progress Bar",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "show_progress_bar",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": false
|
|
},
|
|
"skip_empty": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Skip Empty",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "skip_empty",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": false
|
|
},
|
|
"tiktoken_enable": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "TikToken Enable",
|
|
"dynamic": false,
|
|
"info": "If False, you must have transformers installed.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "tiktoken_enable",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"tiktoken_model_name": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "TikToken Model Name",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "tiktoken_model_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"selected_output": "embeddings",
|
|
"type": "OpenAIEmbeddings"
|
|
},
|
|
"dragging": false,
|
|
"height": 320,
|
|
"id": "OpenAIEmbeddings-joRJ6",
|
|
"measured": {
|
|
"height": 320,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 1870.4219509914485,
|
|
"y": 2193.4259215896764
|
|
},
|
|
"positionAbsolute": {
|
|
"x": 1690.9220896443658,
|
|
"y": 1866.483269483266
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode",
|
|
"width": 320
|
|
},
|
|
{
|
|
"data": {
|
|
"id": "note-Bm5Xw",
|
|
"node": {
|
|
"description": "### 💡 Add your OpenAI API key here 👇",
|
|
"display_name": "",
|
|
"documentation": "",
|
|
"template": {
|
|
"backgroundColor": "transparent"
|
|
}
|
|
},
|
|
"type": "note"
|
|
},
|
|
"dragging": false,
|
|
"height": 324,
|
|
"id": "note-Bm5Xw",
|
|
"measured": {
|
|
"height": 324,
|
|
"width": 324
|
|
},
|
|
"position": {
|
|
"x": 1868.5365759938197,
|
|
"y": 2117.7924922977318
|
|
},
|
|
"positionAbsolute": {
|
|
"x": 1692.2322233423606,
|
|
"y": 1821.9077961087607
|
|
},
|
|
"selected": false,
|
|
"type": "noteNode",
|
|
"width": 324
|
|
},
|
|
{
|
|
"data": {
|
|
"id": "OpenSearchHybrid-Ve6bS",
|
|
"node": {
|
|
"base_classes": [
|
|
"Data",
|
|
"DataFrame",
|
|
"VectorStore"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Store and search documents using OpenSearch with hybrid semantic and keyword search capabilities.",
|
|
"display_name": "OpenSearch",
|
|
"documentation": "",
|
|
"edited": true,
|
|
"field_order": [
|
|
"docs_metadata",
|
|
"opensearch_url",
|
|
"index_name",
|
|
"engine",
|
|
"space_type",
|
|
"ef_construction",
|
|
"m",
|
|
"ingest_data",
|
|
"search_query",
|
|
"should_cache_vector_store",
|
|
"embedding",
|
|
"vector_field",
|
|
"number_of_results",
|
|
"filter_expression",
|
|
"auth_mode",
|
|
"username",
|
|
"password",
|
|
"jwt_token",
|
|
"jwt_header",
|
|
"bearer_prefix",
|
|
"use_ssl",
|
|
"verify_certs"
|
|
],
|
|
"frozen": false,
|
|
"icon": "OpenSearch",
|
|
"legacy": false,
|
|
"lf_version": "1.6.0",
|
|
"metadata": {
|
|
"code_hash": "08d808984c3d",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "opensearchpy",
|
|
"version": "2.8.0"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 2
|
|
},
|
|
"module": "custom_components.opensearch"
|
|
},
|
|
"minimized": false,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Search Results",
|
|
"group_outputs": false,
|
|
"hidden": null,
|
|
"method": "search_documents",
|
|
"name": "search_results",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"tool_mode": true,
|
|
"types": [
|
|
"Data"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
},
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "DataFrame",
|
|
"group_outputs": false,
|
|
"hidden": null,
|
|
"method": "as_dataframe",
|
|
"name": "dataframe",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "DataFrame",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"DataFrame"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
},
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Vector Store Connection",
|
|
"group_outputs": false,
|
|
"hidden": false,
|
|
"method": "as_vector_store",
|
|
"name": "vectorstoreconnection",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"tool_mode": true,
|
|
"types": [
|
|
"VectorStore"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"auth_mode": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": false,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Authentication Mode",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "Authentication method: 'basic' for username/password authentication, or 'jwt' for JSON Web Token (Bearer) authentication.",
|
|
"load_from_db": false,
|
|
"name": "auth_mode",
|
|
"options": [
|
|
"basic",
|
|
"jwt"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"real_time_refresh": true,
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "jwt"
|
|
},
|
|
"bearer_prefix": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Prefix 'Bearer '",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "bearer_prefix",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "from __future__ import annotations\n\nimport json\nimport uuid\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, MultilineInput, SecretStrInput, StrInput, TableInput\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports document\n ingestion, vector embeddings, and advanced filtering with authentication options.\n\n Features:\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Hybrid search combining KNN vector similarity and keyword matching\n - Flexible authentication (Basic auth, JWT tokens)\n - Advanced filtering and aggregations\n - Metadata injection during document ingestion\n \"\"\"\n\n display_name: str = \"OpenSearch\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with hybrid semantic and keyword search capabilities.\"\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"docs_metadata\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n \"load_from_db\": True\n },\n ],\n value=[],\n # advanced=True,\n input_types=[\"Data\"]\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"jvector\", \"nmslib\", \"faiss\", \"lucene\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'jvector' is recommended for most use cases. \"\n \"Note: Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n StrInput(\n name=\"vector_field\",\n display_name=\"Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=\"Name of the field in OpenSearch documents that stores the vector embeddings for similarity search.\",\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"basic\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=False,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=False,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=True,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=True,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n ]\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n }\n }\n },\n }\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our “vector store.”\n self.log(self.ingest_data)\n client = self.build_client()\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings\n - Creates appropriate index mappings\n - Bulk inserts documents with vectors\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n docs = self.ingest_data or []\n if not docs:\n self.log(\"No documents to ingest.\")\n return\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Generate embeddings\n vectors = self.embedding.embed_documents(texts)\n\n if not vectors:\n self.log(\"No vectors generated from documents.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=self.vector_field,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with proper KNN mapping...\")\n\n # Use the LangChain-style bulk ingestion\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=self.vector_field,\n text_field=\"text\",\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n self.log(f\"Successfully indexed {len(return_ids)} documents.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n # ---------- search (single hybrid path matching your tool) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform hybrid search combining vector similarity and keyword matching.\n\n This method executes a sophisticated search that combines:\n - K-nearest neighbor (KNN) vector similarity search (70% weight)\n - Multi-field keyword search with fuzzy matching (30% weight)\n - Optional filtering and score thresholds\n - Aggregations for faceted search results\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression (can be either A or B shape; see _coerce_filter_clauses)\n filter_obj = None\n if getattr(self, \"filter_expression\", \"\") and self.filter_expression.strip():\n try:\n filter_obj = json.loads(self.filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Embed the query\n vec = self.embedding.embed_query(q)\n\n # Build filter clauses (accept both shapes)\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Respect the tool's limit/threshold defaults\n limit = (filter_obj or {}).get(\"limit\", self.number_of_results)\n score_threshold = (filter_obj or {}).get(\"score_threshold\", 0)\n\n # Build the same hybrid body as your SearchService\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"knn\": {\n self.vector_field: {\n \"vector\": vec,\n \"k\": 10, # fixed to match the tool\n \"boost\": 0.7,\n }\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3,\n }\n },\n ],\n \"minimum_should_match\": 1,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": \"filename\", \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n if filter_clauses:\n body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n\n if isinstance(score_threshold, (int, float)) and score_threshold > 0:\n # top-level min_score (matches your tool)\n body[\"min_score\"] = score_threshold\n\n resp = client.search(index=self.index_name, body=body)\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the search using the\n configured search_query and returns results in Langflow's Data format.\n\n Returns:\n List of Data objects containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n raw = self.search(self.search_query or \"\")\n return [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n self.log(self.ingest_data)\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
|
|
},
|
|
"docs_metadata": {
|
|
"_input_type": "TableInput",
|
|
"advanced": false,
|
|
"display_name": "Document Metadata",
|
|
"dynamic": false,
|
|
"info": "Additional metadata key-value pairs to be added to all ingested documents. Useful for tagging documents with source information, categories, or other custom attributes.",
|
|
"input_types": [
|
|
"Data"
|
|
],
|
|
"is_list": true,
|
|
"list_add_label": "Add More",
|
|
"name": "docs_metadata",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"table_icon": "Table",
|
|
"table_schema": [
|
|
{
|
|
"description": "Key name",
|
|
"display_name": "Key",
|
|
"formatter": "text",
|
|
"name": "key",
|
|
"type": "str"
|
|
},
|
|
{
|
|
"description": "Value of the metadata",
|
|
"display_name": "Value",
|
|
"formatter": "text",
|
|
"load_from_db": true,
|
|
"name": "value",
|
|
"type": "str"
|
|
}
|
|
],
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"trigger_icon": "Table",
|
|
"trigger_text": "Open table",
|
|
"type": "table",
|
|
"value": [
|
|
{
|
|
"key": "owner_name",
|
|
"value": "OWNER_NAME"
|
|
},
|
|
{
|
|
"key": "owner",
|
|
"value": "OWNER"
|
|
}
|
|
]
|
|
},
|
|
"ef_construction": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "EF Construction",
|
|
"dynamic": false,
|
|
"info": "Size of the dynamic candidate list during index construction. Higher values improve recall but increase indexing time and memory usage.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "ef_construction",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 512
|
|
},
|
|
"embedding": {
|
|
"_input_type": "HandleInput",
|
|
"advanced": false,
|
|
"display_name": "Embedding",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Embeddings"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "embedding",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"trace_as_metadata": true,
|
|
"type": "other",
|
|
"value": ""
|
|
},
|
|
"engine": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": true,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Vector Engine",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "Vector search engine for similarity calculations. 'jvector' is recommended for most use cases. Note: Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.",
|
|
"load_from_db": false,
|
|
"name": "engine",
|
|
"options": [
|
|
"jvector",
|
|
"nmslib",
|
|
"faiss",
|
|
"lucene"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "nmslib"
|
|
},
|
|
"filter_expression": {
|
|
"_input_type": "MultilineInput",
|
|
"advanced": false,
|
|
"copy_field": false,
|
|
"display_name": "Search Filters (JSON)",
|
|
"dynamic": false,
|
|
"info": "Optional JSON configuration for search filtering, result limits, and score thresholds.\n\nFormat 1 - Explicit filters:\n{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, {\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\n\nFormat 2 - Context-style mapping:\n{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\n\nUse __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "filter_expression",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"index_name": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Index Name",
|
|
"dynamic": false,
|
|
"info": "The OpenSearch index name where documents will be stored and searched. Will be created automatically if it doesn't exist.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "index_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "documents"
|
|
},
|
|
"ingest_data": {
|
|
"_input_type": "HandleInput",
|
|
"advanced": false,
|
|
"display_name": "Ingest Data",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [
|
|
"Data",
|
|
"DataFrame"
|
|
],
|
|
"list": true,
|
|
"list_add_label": "Add More",
|
|
"name": "ingest_data",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"trace_as_metadata": true,
|
|
"type": "other",
|
|
"value": ""
|
|
},
|
|
"jwt_header": {
|
|
"_input_type": "StrInput",
|
|
"advanced": true,
|
|
"display_name": "JWT Header Name",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "jwt_header",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "Authorization"
|
|
},
|
|
"jwt_token": {
|
|
"_input_type": "SecretStrInput",
|
|
"advanced": false,
|
|
"display_name": "JWT Token",
|
|
"dynamic": false,
|
|
"info": "Valid JSON Web Token for authentication. Will be sent in the Authorization header (with optional 'Bearer ' prefix).",
|
|
"input_types": [],
|
|
"load_from_db": true,
|
|
"name": "jwt_token",
|
|
"password": true,
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "str",
|
|
"value": "JWT"
|
|
},
|
|
"m": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "M Parameter",
|
|
"dynamic": false,
|
|
"info": "Number of bidirectional connections for each vector in the HNSW graph. Higher values improve search quality but increase memory usage and indexing time.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "m",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 16
|
|
},
|
|
"number_of_results": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "Default Result Limit",
|
|
"dynamic": false,
|
|
"info": "Default maximum number of search results to return when no limit is specified in the filter expression.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "number_of_results",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 15
|
|
},
|
|
"opensearch_url": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "OpenSearch URL",
|
|
"dynamic": false,
|
|
"info": "The connection URL for your OpenSearch cluster (e.g., http://localhost:9200 for local development or your cloud endpoint).",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "opensearch_url",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "https://opensearch:9200"
|
|
},
|
|
"password": {
|
|
"_input_type": "SecretStrInput",
|
|
"advanced": false,
|
|
"display_name": "OpenSearch Password",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"input_types": [],
|
|
"load_from_db": false,
|
|
"name": "password",
|
|
"password": true,
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"search_query": {
|
|
"_input_type": "QueryInput",
|
|
"advanced": false,
|
|
"display_name": "Search Query",
|
|
"dynamic": false,
|
|
"info": "Enter a query to run a similarity search.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "search_query",
|
|
"placeholder": "Enter a query...",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": true,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "query",
|
|
"value": ""
|
|
},
|
|
"should_cache_vector_store": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Cache Vector Store",
|
|
"dynamic": false,
|
|
"info": "If True, the vector store will be cached for the current build of the component. This is useful for components that have multiple output methods and want to share the same vector store.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "should_cache_vector_store",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"space_type": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": true,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Distance Metric",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, 'cosinesimil' for cosine similarity, 'innerproduct' for dot product.",
|
|
"name": "space_type",
|
|
"options": [
|
|
"l2",
|
|
"l1",
|
|
"cosinesimil",
|
|
"linf",
|
|
"innerproduct"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "l2"
|
|
},
|
|
"use_ssl": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Use SSL/TLS",
|
|
"dynamic": false,
|
|
"info": "Enable SSL/TLS encryption for secure connections to OpenSearch.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "use_ssl",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"username": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Username",
|
|
"dynamic": false,
|
|
"info": "",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "username",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "admin"
|
|
},
|
|
"vector_field": {
|
|
"_input_type": "StrInput",
|
|
"advanced": true,
|
|
"display_name": "Vector Field Name",
|
|
"dynamic": false,
|
|
"info": "Name of the field in OpenSearch documents that stores the vector embeddings for similarity search.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "vector_field",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "chunk_embedding"
|
|
},
|
|
"verify_certs": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Verify SSL Certificates",
|
|
"dynamic": false,
|
|
"info": "Verify SSL certificates when connecting. Disable for self-signed certificates in development environments.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "verify_certs",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": false
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"selected_output": "dataframe",
|
|
"showNode": true,
|
|
"type": "OpenSearchVectorStoreComponent"
|
|
},
|
|
"dragging": false,
|
|
"id": "OpenSearchHybrid-Ve6bS",
|
|
"measured": {
|
|
"height": 822,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 2694.183983837566,
|
|
"y": 1425.777807367294
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode"
|
|
},
|
|
{
|
|
"data": {
|
|
"id": "URLComponent-lnA0q",
|
|
"node": {
|
|
"base_classes": [
|
|
"DataFrame",
|
|
"Message"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Fetch content from one or more web pages, following links recursively.",
|
|
"display_name": "URL",
|
|
"documentation": "https://docs.langflow.org/components-data#url",
|
|
"edited": true,
|
|
"field_order": [
|
|
"urls",
|
|
"max_depth",
|
|
"prevent_outside",
|
|
"use_async",
|
|
"format",
|
|
"timeout",
|
|
"headers",
|
|
"filter_text_html",
|
|
"continue_on_failure",
|
|
"check_response_status",
|
|
"autoset_encoding"
|
|
],
|
|
"frozen": false,
|
|
"icon": "layout-template",
|
|
"legacy": false,
|
|
"lf_version": "1.6.0",
|
|
"metadata": {
|
|
"code_hash": "4c72ce0f2e34",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "requests",
|
|
"version": "2.32.5"
|
|
},
|
|
{
|
|
"name": "bs4",
|
|
"version": "4.12.3"
|
|
},
|
|
{
|
|
"name": "langchain_community",
|
|
"version": "0.3.21"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 4
|
|
},
|
|
"module": "custom_components.url"
|
|
},
|
|
"minimized": false,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Extracted Pages",
|
|
"group_outputs": false,
|
|
"hidden": null,
|
|
"method": "fetch_content",
|
|
"name": "page_results",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "DataFrame",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"DataFrame"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
},
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Raw Content",
|
|
"group_outputs": false,
|
|
"hidden": null,
|
|
"method": "fetch_content_as_message",
|
|
"name": "raw_results",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "Message",
|
|
"tool_mode": false,
|
|
"types": [
|
|
"Message"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"autoset_encoding": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Autoset Encoding",
|
|
"dynamic": false,
|
|
"info": "If enabled, automatically sets the encoding of the request.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "autoset_encoding",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"check_response_status": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Check Response Status",
|
|
"dynamic": false,
|
|
"info": "If enabled, checks the response status of the request.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "check_response_status",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": false
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "import importlib\nimport re\n\nimport requests\nfrom bs4 import BeautifulSoup\nfrom langchain_community.document_loaders import RecursiveUrlLoader\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/components-data#url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=\"Output Format. Use 'Text' to extract the text from the HTML or 'HTML' for the raw HTML content.\",\n options=[\"Text\", \"HTML\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> str:\n \"\"\"Ensures the given string is a valid URL.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n str: The normalized URL\n\n Raises:\n ValueError: If the URL is invalid\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n return url\n\n def _create_loader(self, url: str) -> RecursiveUrlLoader:\n \"\"\"Creates a RecursiveUrlLoader instance with the configured settings.\n\n Args:\n url: The URL to load\n\n Returns:\n RecursiveUrlLoader: Configured loader instance\n \"\"\"\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n extractor = (lambda x: x) if self.format == \"HTML\" else (lambda x: BeautifulSoup(x, \"lxml\").get_text())\n\n return RecursiveUrlLoader(\n url=url,\n max_depth=self.max_depth,\n prevent_outside=self.prevent_outside,\n use_async=self.use_async,\n extractor=extractor,\n timeout=self.timeout,\n headers=headers_dict,\n check_response_status=self.check_response_status,\n continue_on_failure=self.continue_on_failure,\n base_url=url, # Add base_url to ensure consistent domain crawling\n autoset_encoding=self.autoset_encoding, # Enable automatic encoding detection\n exclude_dirs=[], # Allow customization of excluded directories\n link_regex=None, # Allow customization of link filtering\n )\n\n def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs.\n\n Returns:\n List[Data]: List of Data objects containing the fetched content\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n urls = list({self.ensure_url(url) for url in self.urls if url.strip()})\n logger.debug(f\"URLs: {urls}\")\n if not urls:\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n all_docs = []\n for url in urls:\n logger.debug(f\"Loading documents from {url}\")\n\n try:\n loader = self._create_loader(url)\n docs = loader.load()\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} documents from {url}\")\n all_docs.extend(docs)\n\n except requests.exceptions.RequestException as e:\n logger.exception(f\"Error loading documents from {url}: {e}\")\n continue\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # data = [Data(text=doc.page_content, **doc.metadata) for doc in all_docs]\n data = [\n {\n \"text\": safe_convert(doc.page_content, clean_data=True),\n \"url\": doc.metadata.get(\"source\", \"\"),\n \"title\": doc.metadata.get(\"title\", \"\"),\n \"description\": doc.metadata.get(\"description\", \"\"),\n \"content_type\": doc.metadata.get(\"content_type\", \"\"),\n \"language\": doc.metadata.get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else e\n msg = f\"Error loading documents: {error_msg!s}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n return data\n\n def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n return DataFrame(data=self.fetch_url_contents())\n\n def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n"
|
|
},
|
|
"continue_on_failure": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Continue on Failure",
|
|
"dynamic": false,
|
|
"info": "If enabled, continues crawling even if some requests fail.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "continue_on_failure",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"filter_text_html": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Filter Text/HTML",
|
|
"dynamic": false,
|
|
"info": "If enabled, filters out text/css content type from the results.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "filter_text_html",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"format": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": true,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Output Format",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "Output Format. Use 'Text' to extract the text from the HTML or 'HTML' for the raw HTML content.",
|
|
"name": "format",
|
|
"options": [
|
|
"Text",
|
|
"HTML"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "Text"
|
|
},
|
|
"headers": {
|
|
"_input_type": "TableInput",
|
|
"advanced": true,
|
|
"display_name": "Headers",
|
|
"dynamic": false,
|
|
"info": "The headers to send with the request",
|
|
"input_types": [
|
|
"DataFrame"
|
|
],
|
|
"is_list": true,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "headers",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"table_icon": "Table",
|
|
"table_schema": [
|
|
{
|
|
"description": "Header name",
|
|
"display_name": "Header",
|
|
"name": "key",
|
|
"type": "str"
|
|
},
|
|
{
|
|
"description": "Header value",
|
|
"display_name": "Value",
|
|
"name": "value",
|
|
"type": "str"
|
|
}
|
|
],
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"trigger_icon": "Table",
|
|
"trigger_text": "Open table",
|
|
"type": "table",
|
|
"value": [
|
|
{
|
|
"key": "User-Agent",
|
|
"value": "langflow"
|
|
}
|
|
]
|
|
},
|
|
"max_depth": {
|
|
"_input_type": "SliderInput",
|
|
"advanced": false,
|
|
"display_name": "Depth",
|
|
"dynamic": false,
|
|
"info": "Controls how many 'clicks' away from the initial page the crawler will go:\n- depth 1: only the initial page\n- depth 2: initial page + all pages linked directly from it\n- depth 3: initial page + direct links + links found on those direct link pages\nNote: This is about link traversal, not URL path depth.",
|
|
"max_label": " ",
|
|
"max_label_icon": "None",
|
|
"min_label": " ",
|
|
"min_label_icon": "None",
|
|
"name": "max_depth",
|
|
"placeholder": "",
|
|
"range_spec": {
|
|
"max": 5,
|
|
"min": 1,
|
|
"step": 1,
|
|
"step_type": "float"
|
|
},
|
|
"required": false,
|
|
"show": true,
|
|
"slider_buttons": false,
|
|
"slider_buttons_options": [],
|
|
"slider_input": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"type": "slider",
|
|
"value": 1
|
|
},
|
|
"prevent_outside": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Prevent Outside",
|
|
"dynamic": false,
|
|
"info": "If enabled, only crawls URLs within the same domain as the root URL. This helps prevent the crawler from going to external websites.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "prevent_outside",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"timeout": {
|
|
"_input_type": "IntInput",
|
|
"advanced": true,
|
|
"display_name": "Timeout",
|
|
"dynamic": false,
|
|
"info": "Timeout for the request in seconds.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "timeout",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 30
|
|
},
|
|
"urls": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "URLs",
|
|
"dynamic": false,
|
|
"info": "Enter one or more URLs to crawl recursively, by clicking the '+' button.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": true,
|
|
"list_add_label": "Add URL",
|
|
"load_from_db": false,
|
|
"name": "urls",
|
|
"placeholder": "Enter a URL...",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": true,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"use_async": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Use Async",
|
|
"dynamic": false,
|
|
"info": "If enabled, uses asynchronous loading which can be significantly faster but might use more system resources.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "use_async",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"selected_output": "page_results",
|
|
"showNode": true,
|
|
"type": "URLComponent"
|
|
},
|
|
"dragging": false,
|
|
"id": "URLComponent-lnA0q",
|
|
"measured": {
|
|
"height": 292,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 1249.8241608743583,
|
|
"y": 1270.7229143090308
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode"
|
|
},
|
|
{
|
|
"data": {
|
|
"description": "Get chat inputs from the Playground.",
|
|
"display_name": "Chat Input",
|
|
"id": "ChatInput-WLvBD",
|
|
"node": {
|
|
"base_classes": [
|
|
"Message"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Get chat inputs from the Playground.",
|
|
"display_name": "Chat Input",
|
|
"documentation": "https://docs.langflow.org/components-io#chat-input",
|
|
"edited": false,
|
|
"field_order": [
|
|
"input_value",
|
|
"should_store_message",
|
|
"sender",
|
|
"sender_name",
|
|
"session_id",
|
|
"files"
|
|
],
|
|
"frozen": false,
|
|
"icon": "MessagesSquare",
|
|
"legacy": false,
|
|
"lf_version": "1.6.0",
|
|
"metadata": {
|
|
"code_hash": "f701f686b325",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 1
|
|
},
|
|
"module": "custom_components.chat_input"
|
|
},
|
|
"minimized": true,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Chat Message",
|
|
"group_outputs": false,
|
|
"method": "message_response",
|
|
"name": "message",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "Message",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"Message"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "from lfx.base.data.utils import IMG_FILE_TYPES, TEXT_FILE_TYPES\nfrom lfx.base.io.chat import ChatComponent\nfrom lfx.inputs.inputs import BoolInput\nfrom lfx.io import (\n DropdownInput,\n FileInput,\n MessageTextInput,\n MultilineInput,\n Output,\n)\nfrom lfx.schema.message import Message\nfrom lfx.utils.constants import (\n MESSAGE_SENDER_AI,\n MESSAGE_SENDER_NAME_USER,\n MESSAGE_SENDER_USER,\n)\n\n\nclass ChatInput(ChatComponent):\n display_name = \"Chat Input\"\n description = \"Get chat inputs from the Playground.\"\n documentation: str = \"https://docs.langflow.org/components-io#chat-input\"\n icon = \"MessagesSquare\"\n name = \"ChatInput\"\n minimized = True\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Input Text\",\n value=\"\",\n info=\"Message to be passed as input.\",\n input_types=[],\n ),\n BoolInput(\n name=\"should_store_message\",\n display_name=\"Store Messages\",\n info=\"Store the message in the history.\",\n value=True,\n advanced=True,\n ),\n DropdownInput(\n name=\"sender\",\n display_name=\"Sender Type\",\n options=[MESSAGE_SENDER_AI, MESSAGE_SENDER_USER],\n value=MESSAGE_SENDER_USER,\n info=\"Type of sender.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"sender_name\",\n display_name=\"Sender Name\",\n info=\"Name of the sender.\",\n value=MESSAGE_SENDER_NAME_USER,\n advanced=True,\n ),\n MessageTextInput(\n name=\"session_id\",\n display_name=\"Session ID\",\n info=\"The session ID of the chat. If empty, the current session ID parameter will be used.\",\n advanced=True,\n ),\n FileInput(\n name=\"files\",\n display_name=\"Files\",\n file_types=TEXT_FILE_TYPES + IMG_FILE_TYPES,\n info=\"Files to be sent with the message.\",\n advanced=True,\n is_list=True,\n temp_file=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Chat Message\", name=\"message\", method=\"message_response\"),\n ]\n\n async def message_response(self) -> Message:\n # Ensure files is a list and filter out empty/None values\n files = self.files if self.files else []\n if files and not isinstance(files, list):\n files = [files]\n # Filter out None/empty values\n files = [f for f in files if f is not None and f != \"\"]\n\n message = await Message.create(\n text=self.input_value,\n sender=self.sender,\n sender_name=self.sender_name,\n session_id=self.session_id,\n files=files,\n )\n if self.session_id and isinstance(message, Message) and self.should_store_message:\n stored_message = await self.send_message(\n message,\n )\n self.message.value = stored_message\n message = stored_message\n\n self.status = message\n return message\n"
|
|
},
|
|
"files": {
|
|
"_input_type": "FileInput",
|
|
"advanced": true,
|
|
"display_name": "Files",
|
|
"dynamic": false,
|
|
"fileTypes": [
|
|
"csv",
|
|
"json",
|
|
"pdf",
|
|
"txt",
|
|
"md",
|
|
"mdx",
|
|
"yaml",
|
|
"yml",
|
|
"xml",
|
|
"html",
|
|
"htm",
|
|
"docx",
|
|
"py",
|
|
"sh",
|
|
"sql",
|
|
"js",
|
|
"ts",
|
|
"tsx",
|
|
"jpg",
|
|
"jpeg",
|
|
"png",
|
|
"bmp",
|
|
"image"
|
|
],
|
|
"file_path": "",
|
|
"info": "Files to be sent with the message.",
|
|
"list": true,
|
|
"list_add_label": "Add More",
|
|
"name": "files",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"temp_file": true,
|
|
"title_case": false,
|
|
"trace_as_metadata": true,
|
|
"type": "file",
|
|
"value": ""
|
|
},
|
|
"input_value": {
|
|
"_input_type": "MultilineInput",
|
|
"advanced": false,
|
|
"copy_field": false,
|
|
"display_name": "Input Text",
|
|
"dynamic": false,
|
|
"info": "Message to be passed as input.",
|
|
"input_types": [],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "input_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "www.langflow.org"
|
|
},
|
|
"sender": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": true,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Sender Type",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "Type of sender.",
|
|
"name": "sender",
|
|
"options": [
|
|
"Machine",
|
|
"User"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "User"
|
|
},
|
|
"sender_name": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Sender Name",
|
|
"dynamic": false,
|
|
"info": "Name of the sender.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "sender_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "User"
|
|
},
|
|
"session_id": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Session ID",
|
|
"dynamic": false,
|
|
"info": "The session ID of the chat. If empty, the current session ID parameter will be used.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "session_id",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"should_store_message": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Store Messages",
|
|
"dynamic": false,
|
|
"info": "Store the message in the history.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "should_store_message",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"showNode": true,
|
|
"type": "ChatInput"
|
|
},
|
|
"dragging": false,
|
|
"id": "ChatInput-WLvBD",
|
|
"measured": {
|
|
"height": 204,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 884.822226410103,
|
|
"y": 1554.894873551992
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode"
|
|
},
|
|
{
|
|
"data": {
|
|
"id": "DataFrameOperations-hqIoy",
|
|
"node": {
|
|
"base_classes": [
|
|
"DataFrame"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Perform various operations on a DataFrame.",
|
|
"display_name": "DataFrame Operations",
|
|
"documentation": "https://docs.langflow.org/components-processing#dataframe-operations",
|
|
"edited": false,
|
|
"field_order": [
|
|
"df",
|
|
"operation",
|
|
"column_name",
|
|
"filter_value",
|
|
"filter_operator",
|
|
"ascending",
|
|
"new_column_name",
|
|
"new_column_value",
|
|
"columns_to_select",
|
|
"num_rows",
|
|
"replace_value",
|
|
"replacement_value"
|
|
],
|
|
"frozen": false,
|
|
"icon": "table",
|
|
"last_updated": "2025-10-03T14:31:53.355Z",
|
|
"legacy": false,
|
|
"lf_version": "1.6.0",
|
|
"metadata": {
|
|
"code_hash": "b4d6b19b6eef",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "pandas",
|
|
"version": "2.2.3"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 2
|
|
},
|
|
"module": "lfx.components.processing.dataframe_operations.DataFrameOperationsComponent"
|
|
},
|
|
"minimized": false,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "DataFrame",
|
|
"group_outputs": false,
|
|
"method": "perform_operation",
|
|
"name": "output",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "DataFrame",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"DataFrame"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"ascending": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": false,
|
|
"display_name": "Sort Ascending",
|
|
"dynamic": true,
|
|
"info": "Whether to sort in ascending order.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "ascending",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"DataFrame Operations\"\n description = \"Perform various operations on a DataFrame.\"\n documentation: str = \"https://docs.langflow.org/components-processing#dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"DataFrame\",\n info=\"The input DataFrame to operate on.\",\n required=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"DataFrame\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n\n return build_config\n\n def perform_operation(self) -> DataFrame:\n df_copy = self.df.copy()\n\n # Handle SortableListInput format for operation\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list) and len(operation_input) > 0:\n op = operation_input[0].get(\"name\", \"\")\n else:\n op = \"\"\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n"
|
|
},
|
|
"column_name": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Column Name",
|
|
"dynamic": true,
|
|
"info": "The column name to use for the operation.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "column_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"columns_to_select": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Columns to Select",
|
|
"dynamic": true,
|
|
"info": "",
|
|
"list": true,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "columns_to_select",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"df": {
|
|
"_input_type": "DataFrameInput",
|
|
"advanced": false,
|
|
"display_name": "DataFrame",
|
|
"dynamic": false,
|
|
"info": "The input DataFrame to operate on.",
|
|
"input_types": [
|
|
"DataFrame"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "df",
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "other",
|
|
"value": ""
|
|
},
|
|
"filter_operator": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": false,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Filter Operator",
|
|
"dynamic": true,
|
|
"external_options": {},
|
|
"info": "The operator to apply for filtering rows.",
|
|
"name": "filter_operator",
|
|
"options": [
|
|
"equals",
|
|
"not equals",
|
|
"contains",
|
|
"not contains",
|
|
"starts with",
|
|
"ends with",
|
|
"greater than",
|
|
"less than"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "equals"
|
|
},
|
|
"filter_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Filter Value",
|
|
"dynamic": true,
|
|
"info": "The value to filter rows by.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "filter_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"new_column_name": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "New Column Name",
|
|
"dynamic": true,
|
|
"info": "The new column name when renaming or adding a column.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "new_column_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "filename"
|
|
},
|
|
"new_column_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "New Column Value",
|
|
"dynamic": true,
|
|
"info": "The value to populate the new column with.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "new_column_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"num_rows": {
|
|
"_input_type": "IntInput",
|
|
"advanced": false,
|
|
"display_name": "Number of Rows",
|
|
"dynamic": true,
|
|
"info": "Number of rows to return (for head/tail).",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "num_rows",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 5
|
|
},
|
|
"operation": {
|
|
"_input_type": "SortableListInput",
|
|
"advanced": false,
|
|
"display_name": "Operation",
|
|
"dynamic": false,
|
|
"info": "Select the DataFrame operation to perform.",
|
|
"limit": 1,
|
|
"name": "operation",
|
|
"options": [
|
|
{
|
|
"icon": "plus",
|
|
"name": "Add Column"
|
|
},
|
|
{
|
|
"icon": "minus",
|
|
"name": "Drop Column"
|
|
},
|
|
{
|
|
"icon": "filter",
|
|
"name": "Filter"
|
|
},
|
|
{
|
|
"icon": "arrow-up",
|
|
"name": "Head"
|
|
},
|
|
{
|
|
"icon": "pencil",
|
|
"name": "Rename Column"
|
|
},
|
|
{
|
|
"icon": "replace",
|
|
"name": "Replace Value"
|
|
},
|
|
{
|
|
"icon": "columns",
|
|
"name": "Select Columns"
|
|
},
|
|
{
|
|
"icon": "arrow-up-down",
|
|
"name": "Sort"
|
|
},
|
|
{
|
|
"icon": "arrow-down",
|
|
"name": "Tail"
|
|
},
|
|
{
|
|
"icon": "copy-x",
|
|
"name": "Drop Duplicates"
|
|
}
|
|
],
|
|
"placeholder": "Select Operation",
|
|
"real_time_refresh": true,
|
|
"required": false,
|
|
"search_category": [],
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "sortableList",
|
|
"value": [
|
|
{
|
|
"chosen": false,
|
|
"icon": "plus",
|
|
"name": "Add Column",
|
|
"selected": false
|
|
}
|
|
]
|
|
},
|
|
"replace_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Value to Replace",
|
|
"dynamic": true,
|
|
"info": "The value to replace in the column.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "replace_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"replacement_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Replacement Value",
|
|
"dynamic": true,
|
|
"info": "The value to replace with.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "replacement_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"showNode": true,
|
|
"type": "DataFrameOperations"
|
|
},
|
|
"dragging": false,
|
|
"id": "DataFrameOperations-hqIoy",
|
|
"measured": {
|
|
"height": 399,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 1601.8752590736613,
|
|
"y": 1442.944202002645
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode"
|
|
},
|
|
{
|
|
"data": {
|
|
"id": "DataFrameOperations-A98BL",
|
|
"node": {
|
|
"base_classes": [
|
|
"DataFrame"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Perform various operations on a DataFrame.",
|
|
"display_name": "DataFrame Operations",
|
|
"documentation": "https://docs.langflow.org/components-processing#dataframe-operations",
|
|
"edited": false,
|
|
"field_order": [
|
|
"df",
|
|
"operation",
|
|
"column_name",
|
|
"filter_value",
|
|
"filter_operator",
|
|
"ascending",
|
|
"new_column_name",
|
|
"new_column_value",
|
|
"columns_to_select",
|
|
"num_rows",
|
|
"replace_value",
|
|
"replacement_value"
|
|
],
|
|
"frozen": false,
|
|
"icon": "table",
|
|
"last_updated": "2025-10-03T14:31:53.355Z",
|
|
"legacy": false,
|
|
"lf_version": "1.6.0",
|
|
"metadata": {
|
|
"code_hash": "b4d6b19b6eef",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "pandas",
|
|
"version": "2.2.3"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 2
|
|
},
|
|
"module": "lfx.components.processing.dataframe_operations.DataFrameOperationsComponent"
|
|
},
|
|
"minimized": false,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "DataFrame",
|
|
"group_outputs": false,
|
|
"method": "perform_operation",
|
|
"name": "output",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "DataFrame",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"DataFrame"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"ascending": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": false,
|
|
"display_name": "Sort Ascending",
|
|
"dynamic": true,
|
|
"info": "Whether to sort in ascending order.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "ascending",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"DataFrame Operations\"\n description = \"Perform various operations on a DataFrame.\"\n documentation: str = \"https://docs.langflow.org/components-processing#dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"DataFrame\",\n info=\"The input DataFrame to operate on.\",\n required=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"DataFrame\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n\n return build_config\n\n def perform_operation(self) -> DataFrame:\n df_copy = self.df.copy()\n\n # Handle SortableListInput format for operation\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list) and len(operation_input) > 0:\n op = operation_input[0].get(\"name\", \"\")\n else:\n op = \"\"\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n"
|
|
},
|
|
"column_name": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Column Name",
|
|
"dynamic": true,
|
|
"info": "The column name to use for the operation.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "column_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"columns_to_select": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Columns to Select",
|
|
"dynamic": true,
|
|
"info": "",
|
|
"list": true,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "columns_to_select",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"df": {
|
|
"_input_type": "DataFrameInput",
|
|
"advanced": false,
|
|
"display_name": "DataFrame",
|
|
"dynamic": false,
|
|
"info": "The input DataFrame to operate on.",
|
|
"input_types": [
|
|
"DataFrame"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "df",
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "other",
|
|
"value": ""
|
|
},
|
|
"filter_operator": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": false,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Filter Operator",
|
|
"dynamic": true,
|
|
"external_options": {},
|
|
"info": "The operator to apply for filtering rows.",
|
|
"name": "filter_operator",
|
|
"options": [
|
|
"equals",
|
|
"not equals",
|
|
"contains",
|
|
"not contains",
|
|
"starts with",
|
|
"ends with",
|
|
"greater than",
|
|
"less than"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "equals"
|
|
},
|
|
"filter_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Filter Value",
|
|
"dynamic": true,
|
|
"info": "The value to filter rows by.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "filter_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"new_column_name": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "New Column Name",
|
|
"dynamic": true,
|
|
"info": "The new column name when renaming or adding a column.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "new_column_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "mimetype"
|
|
},
|
|
"new_column_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "New Column Value",
|
|
"dynamic": true,
|
|
"info": "The value to populate the new column with.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "new_column_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "text/html"
|
|
},
|
|
"num_rows": {
|
|
"_input_type": "IntInput",
|
|
"advanced": false,
|
|
"display_name": "Number of Rows",
|
|
"dynamic": true,
|
|
"info": "Number of rows to return (for head/tail).",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "num_rows",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 5
|
|
},
|
|
"operation": {
|
|
"_input_type": "SortableListInput",
|
|
"advanced": false,
|
|
"display_name": "Operation",
|
|
"dynamic": false,
|
|
"info": "Select the DataFrame operation to perform.",
|
|
"limit": 1,
|
|
"name": "operation",
|
|
"options": [
|
|
{
|
|
"icon": "plus",
|
|
"name": "Add Column"
|
|
},
|
|
{
|
|
"icon": "minus",
|
|
"name": "Drop Column"
|
|
},
|
|
{
|
|
"icon": "filter",
|
|
"name": "Filter"
|
|
},
|
|
{
|
|
"icon": "arrow-up",
|
|
"name": "Head"
|
|
},
|
|
{
|
|
"icon": "pencil",
|
|
"name": "Rename Column"
|
|
},
|
|
{
|
|
"icon": "replace",
|
|
"name": "Replace Value"
|
|
},
|
|
{
|
|
"icon": "columns",
|
|
"name": "Select Columns"
|
|
},
|
|
{
|
|
"icon": "arrow-up-down",
|
|
"name": "Sort"
|
|
},
|
|
{
|
|
"icon": "arrow-down",
|
|
"name": "Tail"
|
|
},
|
|
{
|
|
"icon": "copy-x",
|
|
"name": "Drop Duplicates"
|
|
}
|
|
],
|
|
"placeholder": "Select Operation",
|
|
"real_time_refresh": true,
|
|
"required": false,
|
|
"search_category": [],
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "sortableList",
|
|
"value": [
|
|
{
|
|
"chosen": false,
|
|
"icon": "plus",
|
|
"name": "Add Column",
|
|
"selected": false
|
|
}
|
|
]
|
|
},
|
|
"replace_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Value to Replace",
|
|
"dynamic": true,
|
|
"info": "The value to replace in the column.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "replace_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"replacement_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Replacement Value",
|
|
"dynamic": true,
|
|
"info": "The value to replace with.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "replacement_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"showNode": true,
|
|
"type": "DataFrameOperations"
|
|
},
|
|
"dragging": false,
|
|
"id": "DataFrameOperations-A98BL",
|
|
"measured": {
|
|
"height": 399,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 1946.8185577395595,
|
|
"y": 1432.2126327108165
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode"
|
|
},
|
|
{
|
|
"data": {
|
|
"id": "ChatOutput-Q1dhr",
|
|
"node": {
|
|
"base_classes": [
|
|
"Message"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Display a chat message in the Playground.",
|
|
"display_name": "Chat Output",
|
|
"documentation": "https://docs.langflow.org/components-io#chat-output",
|
|
"edited": false,
|
|
"field_order": [
|
|
"input_value",
|
|
"should_store_message",
|
|
"sender",
|
|
"sender_name",
|
|
"session_id",
|
|
"data_template",
|
|
"clean_data"
|
|
],
|
|
"frozen": false,
|
|
"icon": "MessagesSquare",
|
|
"legacy": false,
|
|
"metadata": {
|
|
"code_hash": "9647f4d2f4b4",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "orjson",
|
|
"version": "3.10.15"
|
|
},
|
|
{
|
|
"name": "fastapi",
|
|
"version": "0.117.1"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 3
|
|
},
|
|
"module": "lfx.components.input_output.chat_output.ChatOutput"
|
|
},
|
|
"minimized": true,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "Output Message",
|
|
"group_outputs": false,
|
|
"method": "message_response",
|
|
"name": "message",
|
|
"selected": "Message",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"Message"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"clean_data": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Basic Clean Data",
|
|
"dynamic": false,
|
|
"info": "Whether to clean data before converting to string.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "clean_data",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "from collections.abc import Generator\nfrom typing import Any\n\nimport orjson\nfrom fastapi.encoders import jsonable_encoder\n\nfrom lfx.base.io.chat import ChatComponent\nfrom lfx.helpers.data import safe_convert\nfrom lfx.inputs.inputs import BoolInput, DropdownInput, HandleInput, MessageTextInput\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.schema.properties import Source\nfrom lfx.template.field.base import Output\nfrom lfx.utils.constants import (\n MESSAGE_SENDER_AI,\n MESSAGE_SENDER_NAME_AI,\n MESSAGE_SENDER_USER,\n)\n\n\nclass ChatOutput(ChatComponent):\n display_name = \"Chat Output\"\n description = \"Display a chat message in the Playground.\"\n documentation: str = \"https://docs.langflow.org/components-io#chat-output\"\n icon = \"MessagesSquare\"\n name = \"ChatOutput\"\n minimized = True\n\n inputs = [\n HandleInput(\n name=\"input_value\",\n display_name=\"Inputs\",\n info=\"Message to be passed as output.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n BoolInput(\n name=\"should_store_message\",\n display_name=\"Store Messages\",\n info=\"Store the message in the history.\",\n value=True,\n advanced=True,\n ),\n DropdownInput(\n name=\"sender\",\n display_name=\"Sender Type\",\n options=[MESSAGE_SENDER_AI, MESSAGE_SENDER_USER],\n value=MESSAGE_SENDER_AI,\n advanced=True,\n info=\"Type of sender.\",\n ),\n MessageTextInput(\n name=\"sender_name\",\n display_name=\"Sender Name\",\n info=\"Name of the sender.\",\n value=MESSAGE_SENDER_NAME_AI,\n advanced=True,\n ),\n MessageTextInput(\n name=\"session_id\",\n display_name=\"Session ID\",\n info=\"The session ID of the chat. If empty, the current session ID parameter will be used.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"data_template\",\n display_name=\"Data Template\",\n value=\"{text}\",\n advanced=True,\n info=\"Template to convert Data to Text. If left empty, it will be dynamically set to the Data's text key.\",\n ),\n BoolInput(\n name=\"clean_data\",\n display_name=\"Basic Clean Data\",\n value=True,\n advanced=True,\n info=\"Whether to clean data before converting to string.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Output Message\",\n name=\"message\",\n method=\"message_response\",\n ),\n ]\n\n def _build_source(self, id_: str | None, display_name: str | None, source: str | None) -> Source:\n source_dict = {}\n if id_:\n source_dict[\"id\"] = id_\n if display_name:\n source_dict[\"display_name\"] = display_name\n if source:\n # Handle case where source is a ChatOpenAI object\n if hasattr(source, \"model_name\"):\n source_dict[\"source\"] = source.model_name\n elif hasattr(source, \"model\"):\n source_dict[\"source\"] = str(source.model)\n else:\n source_dict[\"source\"] = str(source)\n return Source(**source_dict)\n\n async def message_response(self) -> Message:\n # First convert the input to string if needed\n text = self.convert_to_string()\n\n # Get source properties\n source, _, display_name, source_id = self.get_properties_from_source_component()\n\n # Create or use existing Message object\n if isinstance(self.input_value, Message):\n message = self.input_value\n # Update message properties\n message.text = text\n else:\n message = Message(text=text)\n\n # Set message properties\n message.sender = self.sender\n message.sender_name = self.sender_name\n message.session_id = self.session_id\n message.flow_id = self.graph.flow_id if hasattr(self, \"graph\") else None\n message.properties.source = self._build_source(source_id, display_name, source)\n\n # Store message if needed\n if self.session_id and self.should_store_message:\n stored_message = await self.send_message(message)\n self.message.value = stored_message\n message = stored_message\n\n self.status = message\n return message\n\n def _serialize_data(self, data: Data) -> str:\n \"\"\"Serialize Data object to JSON string.\"\"\"\n # Convert data.data to JSON-serializable format\n serializable_data = jsonable_encoder(data.data)\n # Serialize with orjson, enabling pretty printing with indentation\n json_bytes = orjson.dumps(serializable_data, option=orjson.OPT_INDENT_2)\n # Convert bytes to string and wrap in Markdown code blocks\n return \"```json\\n\" + json_bytes.decode(\"utf-8\") + \"\\n```\"\n\n def _validate_input(self) -> None:\n \"\"\"Validate the input data and raise ValueError if invalid.\"\"\"\n if self.input_value is None:\n msg = \"Input data cannot be None\"\n raise ValueError(msg)\n if isinstance(self.input_value, list) and not all(\n isinstance(item, Message | Data | DataFrame | str) for item in self.input_value\n ):\n invalid_types = [\n type(item).__name__\n for item in self.input_value\n if not isinstance(item, Message | Data | DataFrame | str)\n ]\n msg = f\"Expected Data or DataFrame or Message or str, got {invalid_types}\"\n raise TypeError(msg)\n if not isinstance(\n self.input_value,\n Message | Data | DataFrame | str | list | Generator | type(None),\n ):\n type_name = type(self.input_value).__name__\n msg = f\"Expected Data or DataFrame or Message or str, Generator or None, got {type_name}\"\n raise TypeError(msg)\n\n def convert_to_string(self) -> str | Generator[Any, None, None]:\n \"\"\"Convert input data to string with proper error handling.\"\"\"\n self._validate_input()\n if isinstance(self.input_value, list):\n clean_data: bool = getattr(self, \"clean_data\", False)\n return \"\\n\".join([safe_convert(item, clean_data=clean_data) for item in self.input_value])\n if isinstance(self.input_value, Generator):\n return self.input_value\n return safe_convert(self.input_value)\n"
|
|
},
|
|
"data_template": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Data Template",
|
|
"dynamic": false,
|
|
"info": "Template to convert Data to Text. If left empty, it will be dynamically set to the Data's text key.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "data_template",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "{text}"
|
|
},
|
|
"input_value": {
|
|
"_input_type": "HandleInput",
|
|
"advanced": false,
|
|
"display_name": "Inputs",
|
|
"dynamic": false,
|
|
"info": "Message to be passed as output.",
|
|
"input_types": [
|
|
"Data",
|
|
"DataFrame",
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "input_value",
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"trace_as_metadata": true,
|
|
"type": "other",
|
|
"value": ""
|
|
},
|
|
"sender": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": true,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Sender Type",
|
|
"dynamic": false,
|
|
"external_options": {},
|
|
"info": "Type of sender.",
|
|
"name": "sender",
|
|
"options": [
|
|
"Machine",
|
|
"User"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "Machine"
|
|
},
|
|
"sender_name": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Sender Name",
|
|
"dynamic": false,
|
|
"info": "Name of the sender.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "sender_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "AI"
|
|
},
|
|
"session_id": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": true,
|
|
"display_name": "Session ID",
|
|
"dynamic": false,
|
|
"info": "The session ID of the chat. If empty, the current session ID parameter will be used.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "session_id",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"should_store_message": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": true,
|
|
"display_name": "Store Messages",
|
|
"dynamic": false,
|
|
"info": "Store the message in the history.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "should_store_message",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"showNode": false,
|
|
"type": "ChatOutput"
|
|
},
|
|
"dragging": false,
|
|
"id": "ChatOutput-Q1dhr",
|
|
"measured": {
|
|
"height": 48,
|
|
"width": 192
|
|
},
|
|
"position": {
|
|
"x": 3135.060664388748,
|
|
"y": 2541.1604512818694
|
|
},
|
|
"selected": false,
|
|
"type": "genericNode"
|
|
},
|
|
{
|
|
"data": {
|
|
"id": "DataFrameOperations-RhKoe",
|
|
"node": {
|
|
"base_classes": [
|
|
"DataFrame"
|
|
],
|
|
"beta": false,
|
|
"conditional_paths": [],
|
|
"custom_fields": {},
|
|
"description": "Perform various operations on a DataFrame.",
|
|
"display_name": "DataFrame Operations",
|
|
"documentation": "https://docs.langflow.org/components-processing#dataframe-operations",
|
|
"edited": false,
|
|
"field_order": [
|
|
"df",
|
|
"operation",
|
|
"column_name",
|
|
"filter_value",
|
|
"filter_operator",
|
|
"ascending",
|
|
"new_column_name",
|
|
"new_column_value",
|
|
"columns_to_select",
|
|
"num_rows",
|
|
"replace_value",
|
|
"replacement_value"
|
|
],
|
|
"frozen": false,
|
|
"icon": "table",
|
|
"last_updated": "2025-10-03T14:40:28.935Z",
|
|
"legacy": false,
|
|
"metadata": {
|
|
"code_hash": "b4d6b19b6eef",
|
|
"dependencies": {
|
|
"dependencies": [
|
|
{
|
|
"name": "pandas",
|
|
"version": "2.2.3"
|
|
},
|
|
{
|
|
"name": "lfx",
|
|
"version": null
|
|
}
|
|
],
|
|
"total_dependencies": 2
|
|
},
|
|
"module": "lfx.components.processing.dataframe_operations.DataFrameOperationsComponent"
|
|
},
|
|
"minimized": false,
|
|
"output_types": [],
|
|
"outputs": [
|
|
{
|
|
"allows_loop": false,
|
|
"cache": true,
|
|
"display_name": "DataFrame",
|
|
"group_outputs": false,
|
|
"method": "perform_operation",
|
|
"name": "output",
|
|
"options": null,
|
|
"required_inputs": null,
|
|
"selected": "DataFrame",
|
|
"tool_mode": true,
|
|
"types": [
|
|
"DataFrame"
|
|
],
|
|
"value": "__UNDEFINED__"
|
|
}
|
|
],
|
|
"pinned": false,
|
|
"template": {
|
|
"_type": "Component",
|
|
"ascending": {
|
|
"_input_type": "BoolInput",
|
|
"advanced": false,
|
|
"display_name": "Sort Ascending",
|
|
"dynamic": true,
|
|
"info": "Whether to sort in ascending order.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "ascending",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "bool",
|
|
"value": true
|
|
},
|
|
"code": {
|
|
"advanced": true,
|
|
"dynamic": true,
|
|
"fileTypes": [],
|
|
"file_path": "",
|
|
"info": "",
|
|
"list": false,
|
|
"load_from_db": false,
|
|
"multiline": true,
|
|
"name": "code",
|
|
"password": false,
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"type": "code",
|
|
"value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"DataFrame Operations\"\n description = \"Perform various operations on a DataFrame.\"\n documentation: str = \"https://docs.langflow.org/components-processing#dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"DataFrame\",\n info=\"The input DataFrame to operate on.\",\n required=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"DataFrame\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n\n return build_config\n\n def perform_operation(self) -> DataFrame:\n df_copy = self.df.copy()\n\n # Handle SortableListInput format for operation\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list) and len(operation_input) > 0:\n op = operation_input[0].get(\"name\", \"\")\n else:\n op = \"\"\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n"
|
|
},
|
|
"column_name": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Column Name",
|
|
"dynamic": true,
|
|
"info": "The column name to use for the operation.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "column_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"columns_to_select": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "Columns to Select",
|
|
"dynamic": true,
|
|
"info": "",
|
|
"list": true,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "columns_to_select",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"df": {
|
|
"_input_type": "DataFrameInput",
|
|
"advanced": false,
|
|
"display_name": "DataFrame",
|
|
"dynamic": false,
|
|
"info": "The input DataFrame to operate on.",
|
|
"input_types": [
|
|
"DataFrame"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "df",
|
|
"placeholder": "",
|
|
"required": true,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "other",
|
|
"value": ""
|
|
},
|
|
"filter_operator": {
|
|
"_input_type": "DropdownInput",
|
|
"advanced": false,
|
|
"combobox": false,
|
|
"dialog_inputs": {},
|
|
"display_name": "Filter Operator",
|
|
"dynamic": true,
|
|
"external_options": {},
|
|
"info": "The operator to apply for filtering rows.",
|
|
"name": "filter_operator",
|
|
"options": [
|
|
"equals",
|
|
"not equals",
|
|
"contains",
|
|
"not contains",
|
|
"starts with",
|
|
"ends with",
|
|
"greater than",
|
|
"less than"
|
|
],
|
|
"options_metadata": [],
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"toggle": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": "equals"
|
|
},
|
|
"filter_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Filter Value",
|
|
"dynamic": true,
|
|
"info": "The value to filter rows by.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "filter_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"new_column_name": {
|
|
"_input_type": "StrInput",
|
|
"advanced": false,
|
|
"display_name": "New Column Name",
|
|
"dynamic": true,
|
|
"info": "The new column name when renaming or adding a column.",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "new_column_name",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"new_column_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "New Column Value",
|
|
"dynamic": true,
|
|
"info": "The value to populate the new column with.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "new_column_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"num_rows": {
|
|
"_input_type": "IntInput",
|
|
"advanced": false,
|
|
"display_name": "Number of Rows",
|
|
"dynamic": true,
|
|
"info": "Number of rows to return (for head/tail).",
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"name": "num_rows",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "int",
|
|
"value": 5
|
|
},
|
|
"operation": {
|
|
"_input_type": "SortableListInput",
|
|
"advanced": false,
|
|
"display_name": "Operation",
|
|
"dynamic": false,
|
|
"info": "Select the DataFrame operation to perform.",
|
|
"limit": 1,
|
|
"name": "operation",
|
|
"options": [
|
|
{
|
|
"icon": "plus",
|
|
"name": "Add Column"
|
|
},
|
|
{
|
|
"icon": "minus",
|
|
"name": "Drop Column"
|
|
},
|
|
{
|
|
"icon": "filter",
|
|
"name": "Filter"
|
|
},
|
|
{
|
|
"icon": "arrow-up",
|
|
"name": "Head"
|
|
},
|
|
{
|
|
"icon": "pencil",
|
|
"name": "Rename Column"
|
|
},
|
|
{
|
|
"icon": "replace",
|
|
"name": "Replace Value"
|
|
},
|
|
{
|
|
"icon": "columns",
|
|
"name": "Select Columns"
|
|
},
|
|
{
|
|
"icon": "arrow-up-down",
|
|
"name": "Sort"
|
|
},
|
|
{
|
|
"icon": "arrow-down",
|
|
"name": "Tail"
|
|
},
|
|
{
|
|
"icon": "copy-x",
|
|
"name": "Drop Duplicates"
|
|
}
|
|
],
|
|
"placeholder": "Select Operation",
|
|
"real_time_refresh": true,
|
|
"required": false,
|
|
"search_category": [],
|
|
"show": true,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_metadata": true,
|
|
"type": "sortableList",
|
|
"value": [
|
|
{
|
|
"chosen": false,
|
|
"icon": "arrow-up",
|
|
"name": "Head",
|
|
"selected": false
|
|
}
|
|
]
|
|
},
|
|
"replace_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Value to Replace",
|
|
"dynamic": true,
|
|
"info": "The value to replace in the column.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "replace_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
},
|
|
"replacement_value": {
|
|
"_input_type": "MessageTextInput",
|
|
"advanced": false,
|
|
"display_name": "Replacement Value",
|
|
"dynamic": true,
|
|
"info": "The value to replace with.",
|
|
"input_types": [
|
|
"Message"
|
|
],
|
|
"list": false,
|
|
"list_add_label": "Add More",
|
|
"load_from_db": false,
|
|
"name": "replacement_value",
|
|
"placeholder": "",
|
|
"required": false,
|
|
"show": false,
|
|
"title_case": false,
|
|
"tool_mode": false,
|
|
"trace_as_input": true,
|
|
"trace_as_metadata": true,
|
|
"type": "str",
|
|
"value": ""
|
|
}
|
|
},
|
|
"tool_mode": false
|
|
},
|
|
"showNode": true,
|
|
"type": "DataFrameOperations"
|
|
},
|
|
"dragging": false,
|
|
"id": "DataFrameOperations-RhKoe",
|
|
"measured": {
|
|
"height": 317,
|
|
"width": 320
|
|
},
|
|
"position": {
|
|
"x": 2773.2060092972047,
|
|
"y": 2337.54590413581
|
|
},
|
|
"selected": true,
|
|
"type": "genericNode"
|
|
}
|
|
],
|
|
"viewport": {
|
|
"x": -1399.3594540015256,
|
|
"y": -1163.4584036787048,
|
|
"zoom": 0.7595394813583742
|
|
}
|
|
},
|
|
"description": "This flow is to ingest the URL to open search.",
|
|
"endpoint_name": null,
|
|
"id": "72c3d17c-2dac-4a73-b48a-6518473d7830",
|
|
"is_component": false,
|
|
"last_tested_version": "1.6.0",
|
|
"name": "OpenSearch URL Ingestion Flow",
|
|
"mcp_enabled": true,
|
|
"tags": [
|
|
"openai",
|
|
"astradb",
|
|
"rag",
|
|
"q-a"
|
|
]
|
|
} |