Fix linting
This commit is contained in:
parent
d979e9078f
commit
2767212ba0
3 changed files with 28 additions and 16 deletions
|
|
@ -139,7 +139,7 @@ EMBEDDING_BINDING_HOST=http://localhost:11434
|
||||||
# AZURE_EMBEDDING_ENDPOINT=your_endpoint
|
# AZURE_EMBEDDING_ENDPOINT=your_endpoint
|
||||||
# AZURE_EMBEDDING_API_KEY=your_api_key
|
# AZURE_EMBEDDING_API_KEY=your_api_key
|
||||||
|
|
||||||
### Jina AI Embedding
|
### Jina AI Embedding
|
||||||
EMBEDDING_BINDING=jina
|
EMBEDDING_BINDING=jina
|
||||||
EMBEDDING_BINDING_HOST=https://api.jina.ai/v1/embeddings
|
EMBEDDING_BINDING_HOST=https://api.jina.ai/v1/embeddings
|
||||||
EMBEDDING_MODEL=jina-embeddings-v4
|
EMBEDDING_MODEL=jina-embeddings-v4
|
||||||
|
|
|
||||||
|
|
@ -89,7 +89,13 @@ def create_app(args):
|
||||||
]:
|
]:
|
||||||
raise Exception("llm binding not supported")
|
raise Exception("llm binding not supported")
|
||||||
|
|
||||||
if args.embedding_binding not in ["lollms", "ollama", "openai", "azure_openai", "jina"]:
|
if args.embedding_binding not in [
|
||||||
|
"lollms",
|
||||||
|
"ollama",
|
||||||
|
"openai",
|
||||||
|
"azure_openai",
|
||||||
|
"jina",
|
||||||
|
]:
|
||||||
raise Exception("embedding binding not supported")
|
raise Exception("embedding binding not supported")
|
||||||
|
|
||||||
# Set default hosts if not provided
|
# Set default hosts if not provided
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,7 @@ async def fetch_data(url, headers, data):
|
||||||
request_info=response.request_info,
|
request_info=response.request_info,
|
||||||
history=response.history,
|
history=response.history,
|
||||||
status=response.status,
|
status=response.status,
|
||||||
message=f"Jina API error: {error_text}"
|
message=f"Jina API error: {error_text}",
|
||||||
)
|
)
|
||||||
response_json = await response.json()
|
response_json = await response.json()
|
||||||
data_list = response_json.get("data", [])
|
data_list = response_json.get("data", [])
|
||||||
|
|
@ -69,10 +69,10 @@ async def jina_embed(
|
||||||
"""
|
"""
|
||||||
if api_key:
|
if api_key:
|
||||||
os.environ["JINA_API_KEY"] = api_key
|
os.environ["JINA_API_KEY"] = api_key
|
||||||
|
|
||||||
if "JINA_API_KEY" not in os.environ:
|
if "JINA_API_KEY" not in os.environ:
|
||||||
raise ValueError("JINA_API_KEY environment variable is required")
|
raise ValueError("JINA_API_KEY environment variable is required")
|
||||||
|
|
||||||
url = base_url or "https://api.jina.ai/v1/embeddings"
|
url = base_url or "https://api.jina.ai/v1/embeddings"
|
||||||
headers = {
|
headers = {
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
|
|
@ -84,29 +84,35 @@ async def jina_embed(
|
||||||
"dimensions": dimensions,
|
"dimensions": dimensions,
|
||||||
"input": texts,
|
"input": texts,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Only add optional parameters if they have non-default values
|
# Only add optional parameters if they have non-default values
|
||||||
if late_chunking:
|
if late_chunking:
|
||||||
data["late_chunking"] = late_chunking
|
data["late_chunking"] = late_chunking
|
||||||
|
|
||||||
logger.debug(f"Jina embedding request: {len(texts)} texts, dimensions: {dimensions}")
|
logger.debug(
|
||||||
|
f"Jina embedding request: {len(texts)} texts, dimensions: {dimensions}"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data_list = await fetch_data(url, headers, data)
|
data_list = await fetch_data(url, headers, data)
|
||||||
|
|
||||||
if not data_list:
|
if not data_list:
|
||||||
logger.error("Jina API returned empty data list")
|
logger.error("Jina API returned empty data list")
|
||||||
raise ValueError("Jina API returned empty data list")
|
raise ValueError("Jina API returned empty data list")
|
||||||
|
|
||||||
if len(data_list) != len(texts):
|
if len(data_list) != len(texts):
|
||||||
logger.error(f"Jina API returned {len(data_list)} embeddings for {len(texts)} texts")
|
logger.error(
|
||||||
raise ValueError(f"Jina API returned {len(data_list)} embeddings for {len(texts)} texts")
|
f"Jina API returned {len(data_list)} embeddings for {len(texts)} texts"
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
f"Jina API returned {len(data_list)} embeddings for {len(texts)} texts"
|
||||||
|
)
|
||||||
|
|
||||||
embeddings = np.array([dp["embedding"] for dp in data_list])
|
embeddings = np.array([dp["embedding"] for dp in data_list])
|
||||||
logger.debug(f"Jina embeddings generated: shape {embeddings.shape}")
|
logger.debug(f"Jina embeddings generated: shape {embeddings.shape}")
|
||||||
|
|
||||||
return embeddings
|
return embeddings
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Jina embedding error: {e}")
|
logger.error(f"Jina embedding error: {e}")
|
||||||
raise
|
raise
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue