Index

A

accumulator (retrieval_qa_benchmark.utils.profiler.Profiler attribute)

answer (retrieval_qa_benchmark.schema.QAPrediction attribute) (retrieval_qa_benchmark.schema.QARecord attribute)

B

BaseDataset.Config (class in retrieval_qa_benchmark.schema) BaseEvaluator.Config (class in retrieval_qa_benchmark.schema) BaseTransform.Config (class in retrieval_qa_benchmark.schema) bm25_filter() (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher method) boot_time_key (retrieval_qa_benchmark.models.TGI_LLM attribute) build() (retrieval_qa_benchmark.models.ChatGPT class method) (retrieval_qa_benchmark.models.GPT class method) (retrieval_qa_benchmark.models.RemoteLLM class method) (retrieval_qa_benchmark.models.TGI_LLM class method) (retrieval_qa_benchmark.schema.BaseDataset class method) (retrieval_qa_benchmark.schema.BaseLLM class method) (retrieval_qa_benchmark.schema.TransformGraph class method)

build_agent_template() (retrieval_qa_benchmark.transforms.AgentRouter method) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter method)

C

chain() (retrieval_qa_benchmark.schema.BaseTransform method) (retrieval_qa_benchmark.transforms.AgentRouter method) (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 method) (retrieval_qa_benchmark.transforms.ContextWithFaiss method) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid method) (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid method) (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB method) (retrieval_qa_benchmark.transforms.LangChainListSQLDB method) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB method) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter method) (retrieval_qa_benchmark.transforms.LangChainSQLChecker method) check_status() (retrieval_qa_benchmark.schema.BaseTransform method) checker_prompt (retrieval_qa_benchmark.transforms.LangChainSQLChecker attribute) children (retrieval_qa_benchmark.schema.BaseTransform attribute) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) choices (retrieval_qa_benchmark.schema.QAPrediction attribute) (retrieval_qa_benchmark.schema.QARecord attribute)

clear() (retrieval_qa_benchmark.utils.profiler.Profiler method) client (retrieval_qa_benchmark.models.TGI_LLM attribute) comment (retrieval_qa_benchmark.schema.LLMHistory attribute) completion_time_key (retrieval_qa_benchmark.models.TGI_LLM attribute) completion_tokens (retrieval_qa_benchmark.schema.BaseLLMOutput attribute) (retrieval_qa_benchmark.schema.LLMHistory attribute) (retrieval_qa_benchmark.schema.QAPrediction attribute) context (retrieval_qa_benchmark.schema.QAPrediction attribute) (retrieval_qa_benchmark.schema.QARecord attribute) context_template (retrieval_qa_benchmark.schema.BaseLLM attribute) (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 attribute) (retrieval_qa_benchmark.transforms.ContextWithFaiss attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid attribute) convert_record() (retrieval_qa_benchmark.schema.BaseLLM method) counter (retrieval_qa_benchmark.utils.profiler.Profiler attribute) created_by (retrieval_qa_benchmark.schema.LLMHistory attribute)

D

dataset (retrieval_qa_benchmark.schema.BaseEvaluator attribute) dataset_name (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 attribute) (retrieval_qa_benchmark.transforms.ContextWithFaiss attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher attribute)

dataset_split (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher attribute) Datasets (retrieval_qa_benchmark.utils.registry.Registry attribute) descrption (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainListSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainSQLChecker attribute)

E

el_auth (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) el_host (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) emb_filter() (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher method) embedding_name (retrieval_qa_benchmark.transforms.ContextWithFaiss attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute)

entry_id (retrieval_qa_benchmark.schema.TransformGraph attribute) eval_set (retrieval_qa_benchmark.schema.BaseDataset attribute) Evaluators (retrieval_qa_benchmark.utils.registry.Registry attribute) execute_action() (retrieval_qa_benchmark.transforms.AgentRouter method) (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB method) (retrieval_qa_benchmark.transforms.LangChainListSQLDB method) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB method) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter method) (retrieval_qa_benchmark.transforms.LangChainSQLChecker method) extra (retrieval_qa_benchmark.schema.BaseDataset.Config attribute) (retrieval_qa_benchmark.schema.LLMHistory attribute)

F

faiss_bm25_hybrid_filter() (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher method) faiss_bm25_union_filter() (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher method) field_targets() (retrieval_qa_benchmark.schema.BaseTransform method)

format_agent_template() (retrieval_qa_benchmark.transforms.AgentRouter method) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter method) format_instructions (retrieval_qa_benchmark.transforms.AgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute)

G

generate() (retrieval_qa_benchmark.models.ChatGPT method) (retrieval_qa_benchmark.models.GPT method) (retrieval_qa_benchmark.models.RemoteLLM method) (retrieval_qa_benchmark.models.TGI_LLM method) (retrieval_qa_benchmark.schema.BaseLLM method) generated (retrieval_qa_benchmark.schema.BaseLLMOutput attribute) (retrieval_qa_benchmark.schema.LLMHistory attribute) (retrieval_qa_benchmark.schema.QAPrediction attribute)

get_next_state() (retrieval_qa_benchmark.transforms.AgentRouter method) (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB method) (retrieval_qa_benchmark.transforms.LangChainListSQLDB method) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB method) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter method) (retrieval_qa_benchmark.transforms.LangChainSQLChecker method)

H

host (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute)

I

id (retrieval_qa_benchmark.schema.QAPrediction attribute) (retrieval_qa_benchmark.schema.QARecord attribute) index_path (retrieval_qa_benchmark.transforms.ContextWithFaiss attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher attribute)

index_search() (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher method) is_raw_rank (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) iterator() (retrieval_qa_benchmark.schema.BaseDataset method)

K

kw_topk (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute)

L

llm (retrieval_qa_benchmark.schema.BaseEvaluator attribute) llm_model (retrieval_qa_benchmark.transforms.AgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLChecker attribute)

LLMs (retrieval_qa_benchmark.utils.registry.Registry attribute)

M

matched (retrieval_qa_benchmark.schema.QAPrediction attribute) matcher (retrieval_qa_benchmark.evaluators.MCMAEvaluator attribute) (retrieval_qa_benchmark.evaluators.MCSAEvaluator attribute) (retrieval_qa_benchmark.schema.BaseEvaluator attribute) module retrieval_qa_benchmark.evaluators retrieval_qa_benchmark.models retrieval_qa_benchmark.schema retrieval_qa_benchmark.transforms retrieval_qa_benchmark.transforms.searchers

N

name (retrieval_qa_benchmark.schema.BaseDataset attribute) (retrieval_qa_benchmark.schema.BaseLLM attribute) (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainListSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainSQLChecker attribute) nodes (retrieval_qa_benchmark.schema.TransformGraph attribute) nprobe (retrieval_qa_benchmark.transforms.ContextWithFaiss attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher attribute)

num_filtered (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute) num_selected (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 attribute) (retrieval_qa_benchmark.transforms.ContextWithFaiss attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid attribute)

O

out_file (retrieval_qa_benchmark.schema.BaseEvaluator attribute)

P

para_id_list_to_entry() (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher method) para_id_to_entry() (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher method) parse_extra() (retrieval_qa_benchmark.transforms.AgentRouter method) (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB method) (retrieval_qa_benchmark.transforms.LangChainListSQLDB method) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB method) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter method) (retrieval_qa_benchmark.transforms.LangChainSQLChecker method) password (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute) port (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute)

prefix (retrieval_qa_benchmark.transforms.AgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) preproc_question4query() (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 method) (retrieval_qa_benchmark.transforms.ContextWithFaiss method) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid method) (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid method) profile_avg (retrieval_qa_benchmark.schema.QAPrediction attribute) profile_count (retrieval_qa_benchmark.schema.QAPrediction attribute) profile_dataset() (retrieval_qa_benchmark.utils.profiler.Profiler method) profile_function() (retrieval_qa_benchmark.utils.profiler.Profiler method) profile_model() (retrieval_qa_benchmark.utils.profiler.Profiler method) profile_time (retrieval_qa_benchmark.schema.QAPrediction attribute) profile_transform() (retrieval_qa_benchmark.utils.profiler.Profiler method) Profiler.Config (class in retrieval_qa_benchmark.utils.profiler) prompt_tokens (retrieval_qa_benchmark.schema.BaseLLMOutput attribute) (retrieval_qa_benchmark.schema.LLMHistory attribute) (retrieval_qa_benchmark.schema.QAPrediction attribute)

Q

QAPrediction.Config (class in retrieval_qa_benchmark.schema) QARecord.Config (class in retrieval_qa_benchmark.schema)

question (retrieval_qa_benchmark.schema.QAPrediction attribute) (retrieval_qa_benchmark.schema.QARecord attribute)

R

rank_dict (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid attribute) record_template (retrieval_qa_benchmark.schema.BaseLLM attribute) (retrieval_qa_benchmark.transforms.AgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) register_dataset() (retrieval_qa_benchmark.utils.registry.Registry method) register_evaluator() (retrieval_qa_benchmark.utils.registry.Registry method) register_model() (retrieval_qa_benchmark.utils.registry.Registry method) register_transform() (retrieval_qa_benchmark.utils.registry.Registry method) Registry.Config (class in retrieval_qa_benchmark.utils.registry) result (retrieval_qa_benchmark.schema.ToolHistory attribute) retrieval_qa_benchmark.evaluators module

retrieval_qa_benchmark.models module retrieval_qa_benchmark.schema module retrieval_qa_benchmark.transforms module retrieval_qa_benchmark.transforms.searchers module retrieve() (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher method) run_args (retrieval_qa_benchmark.schema.BaseLLM attribute)

S

search() (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25HybridSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher method) (retrieval_qa_benchmark.transforms.searchers.FaissSearcher method) (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher method) sep_chr (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 attribute) (retrieval_qa_benchmark.transforms.ContextWithFaiss attribute) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid attribute) (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid attribute) set_children() (retrieval_qa_benchmark.schema.BaseTransform method) (retrieval_qa_benchmark.transforms.AgentRouter method) (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB method) (retrieval_qa_benchmark.transforms.LangChainListSQLDB method) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB method) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter method) (retrieval_qa_benchmark.transforms.LangChainSQLChecker method)

sql_dialect (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLChecker attribute) sql_topk (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) stack (retrieval_qa_benchmark.schema.QAPrediction attribute) (retrieval_qa_benchmark.schema.QARecord attribute) stream (retrieval_qa_benchmark.models.TGI_LLM attribute) suffix (retrieval_qa_benchmark.transforms.AgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) system_prompt (retrieval_qa_benchmark.models.ChatGPT attribute) (retrieval_qa_benchmark.models.GPT attribute) (retrieval_qa_benchmark.models.RemoteLLM attribute) (retrieval_qa_benchmark.models.TGI_LLM attribute)

T

table_name (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute) text_preprocess (retrieval_qa_benchmark.transforms.searchers.ElSearchSearcher attribute) (retrieval_qa_benchmark.transforms.searchers.FaissElSearchBM25UnionSearcher attribute) TGI_LLM.Config (class in retrieval_qa_benchmark.models) thought (retrieval_qa_benchmark.schema.ToolHistory attribute) tokenizer_type (retrieval_qa_benchmark.schema.BaseLLM property) tool (retrieval_qa_benchmark.schema.ToolHistory attribute) tool_inputs (retrieval_qa_benchmark.schema.ToolHistory attribute) transform (retrieval_qa_benchmark.schema.BaseEvaluator attribute)

transform_choices() (retrieval_qa_benchmark.schema.BaseTransform method) transform_context() (retrieval_qa_benchmark.transforms.ContextWithElasticBM25 method) (retrieval_qa_benchmark.transforms.ContextWithFaiss method) (retrieval_qa_benchmark.transforms.ContextWithFaissESHybrid method) (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid method) Transforms (retrieval_qa_benchmark.utils.registry.Registry attribute) two_staged (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute) type (retrieval_qa_benchmark.schema.QAPrediction attribute) (retrieval_qa_benchmark.schema.QARecord attribute)

U

url (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainListSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainSQLChecker attribute)

username (retrieval_qa_benchmark.transforms.searchers.MyScaleSearcher attribute)

V

verbose (retrieval_qa_benchmark.transforms.AgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainInfoSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainListSQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainQuerySQLDB attribute) (retrieval_qa_benchmark.transforms.LangChainSQLAgentRouter attribute) (retrieval_qa_benchmark.transforms.LangChainSQLChecker attribute)

W

with_title (retrieval_qa_benchmark.transforms.ContextWithRRFHybrid attribute)