gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle azure models as well

Browse source
This commit is contained in:
Rita Aleksziev 2025-01-07 15:00:58 +01:00
parent a774191ed3
commit fb13a1b61a
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cognee/tasks/chunks/chunk_by_paragraph.py
View file

@ -29,6 +29,8 @@ def chunk_by_paragraph(
for paragraph_id, sentence, word_count, end_type in chunk_by_sentence(data, maximum_length=paragraph_length): for paragraph_id, sentence, word_count, end_type in chunk_by_sentence(data, maximum_length=paragraph_length):
# Check if this sentence would exceed length limit # Check if this sentence would exceed length limit
if embedding_model: if embedding_model:
if embedding_model.startswith("azure/"):
embedding_model = embedding_model.split("/")[-1]
tokenizer = tiktoken.encoding_for_model(embedding_model) tokenizer = tiktoken.encoding_for_model(embedding_model)
token_count = len(tokenizer.encode(sentence)) token_count = len(tokenizer.encode(sentence))
else: else: