bb8cb692e0
<!-- .github/pull_request_template.md --> ## Description - Enable custom tasks in corpus building ## DCO Affirmation I affirm that all code in every commit of this pull request conforms to the terms of the Topoteretes Developer Certificate of Origin <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced a configurable option to specify the task retrieval strategy during corpus building. - Enhanced the workflow with integrated task fetching, featuring a default retrieval mechanism. - Updated evaluation configuration to support customizable task selection for more flexible operations. - Added a new abstract base class for defining various task retrieval strategies. - Introduced a new enumeration to map task getter types to their corresponding classes. - **Dependencies** - Added a new dependency for downloading files from Google Drive. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
23 lines
862 B
Python
23 lines
862 B
Python
from enum import Enum
|
|
from typing import Type
|
|
|
|
from evals.eval_framework.benchmark_adapters.hotpot_qa_adapter import HotpotQAAdapter
|
|
from evals.eval_framework.benchmark_adapters.musique_adapter import MusiqueQAAdapter
|
|
from evals.eval_framework.benchmark_adapters.dummy_adapter import DummyAdapter
|
|
from evals.eval_framework.benchmark_adapters.twowikimultihop_adapter import TwoWikiMultihopAdapter
|
|
|
|
|
|
class BenchmarkAdapter(Enum):
|
|
DUMMY = ("Dummy", DummyAdapter)
|
|
HOTPOTQA = ("HotPotQA", HotpotQAAdapter)
|
|
MUSIQUE = ("Musique", MusiqueQAAdapter)
|
|
TWOWIKIMULTIHOP = ("TwoWikiMultiHop", TwoWikiMultihopAdapter)
|
|
|
|
def __new__(cls, adapter_name: str, adapter_class: Type):
|
|
obj = object.__new__(cls)
|
|
obj._value_ = adapter_name
|
|
obj.adapter_class = adapter_class
|
|
return obj
|
|
|
|
def __str__(self):
|
|
return self.value
|