1
2
3
4
5
6
7
8
9
10

from llama_index.readers.web import SimpleWebPageReader
from llama_index.core.node_parser import SentenceSplitterlanguage = "EN"
url_doc = "https://eur-lex.europa.eu/legal-content/"+language+"/TXT/HTML/?uri=CELEX:52021PC0206"documents = SimpleWebPageReader(html_to_text=True).load_data([url_doc])parser = SentenceSplitter(chunk_size=1000)
nodes = parser.get_nodes_from_documents(documents, show_progress=True)

1
2
3
4
5
6
7
8
9
10
11
12
13

prompts={}
prompts["EN"] = """\
Context information is below.---------------------
{context_str}
---------------------Given the context information and not prior knowledge, generate only questions based on the below query.You are a Teacher/ Professor. Your task is to setup {num_questions_per_chunk} questions for an upcoming quiz/examination.
The questions should be diverse in nature across the document. Restrict the questions to the context information provided."
"""

1
2
3
4
5
6
7
8
9

from llama_index.llms import OpenAI
from llama_index.legacy.finetuning import generate_qa_embedding_pairsqa_dataset = generate_qa_embedding_pairs(llm=OpenAI(model="gpt-3.5-turbo-0125",additional_kwargs={'seed':42}),nodes=nodes,qa_generate_prompt_tmpl = prompts[language],num_questions_per_chunk=2
)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

def evaluate(dataset, embed_model, insert_batch_size=1000, top_k=5):# Get corpus, queries, and relevant documents from the qa_dataset objectcorpus = dataset.corpusqueries = dataset.queriesrelevant_docs = dataset.relevant_docs# Create TextNode objects for each document in the corpus and create a VectorStoreIndex to efficiently store and retrieve embeddingsnodes = [TextNode(id_=id_, text=text) for id_, text in corpus.items()]index = VectorStoreIndex(nodes, embed_model=embed_model, insert_batch_size=insert_batch_size)retriever = index.as_retriever(similarity_top_k=top_k)# Prepare to collect evaluation resultseval_results = []# Iterate over each query in the dataset to evaluate retrieval performancefor query_id, query in tqdm(queries.items()):# Retrieve the top_k most similar documents for the current query and extract the IDs of the retrieved documentsretrieved_nodes = retriever.retrieve(query)retrieved_ids = [node.node.node_id for node in retrieved_nodes]# Check if the expected document was among the retrieved documentsexpected_id = relevant_docs[query_id][0]is_hit = expected_id in retrieved_ids  # assume 1 relevant doc per query# Calculate the Mean Reciprocal Rank (MRR) and append to resultsif is_hit:rank = retrieved_ids.index(expected_id) + 1mrr = 1 / rankelse:mrr = 0eval_results.append(mrr)# Return the average MRR across all queries as the final evaluation metricreturn np.average(eval_results)

1
2
3
4

from llama_index.embeddings.openai import OpenAIEmbeddingembed_model = OpenAIEmbedding(model=model_spec['model_name'],dimensions=model_spec['dimensions'])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

embeddings_model_spec = {
}embeddings_model_spec['OAI-Large-256']={'model_name':'text-embedding-3-large','dimensions':256}
embeddings_model_spec['OAI-Large-3072']={'model_name':'text-embedding-3-large','dimensions':3072}
embeddings_model_spec['OAI-Small']={'model_name':'text-embedding-3-small','dimensions':1536}
embeddings_model_spec['OAI-ada-002']={'model_name':'text-embedding-ada-002','dimensions':None}results = []languages = ["EN", "FR", "CS", "HU"]# Loop through all languages
for language in languages:# Load datasetfile_name=language+"_dataset.json"qa_dataset = EmbeddingQAFinetuneDataset.from_json(file_name)# Loop through all modelsfor model_name, model_spec in embeddings_model_spec.items():# Get modelembed_model = OpenAIEmbedding(model=model_spec['model_name'],dimensions=model_spec['dimensions'])# Assess embedding score (in terms of MRR)score = evaluate(qa_dataset, embed_model)results.append([language, model_name, score])df_results = pd.DataFrame(results, columns = ["Language" ,"Embedding model", "MRR"])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

embeddings_model_spec = {
}embeddings_model_spec['E5-mistral-7b']={'model_name':'intfloat/e5-mistral-7b-instruct','max_length':32768, 'pooling_type':'last_token', 'normalize': True, 'batch_size':1, 'kwargs': {'load_in_4bit':True, 'bnb_4bit_compute_dtype':torch.float16}}
embeddings_model_spec['ML-E5-large']={'model_name':'intfloat/multilingual-e5-large','max_length':512, 'pooling_type':'mean', 'normalize': True, 'batch_size':1, 'kwargs': {'device_map': 'cuda', 'torch_dtype':torch.float16}}
embeddings_model_spec['BGE-M3']={'model_name':'BAAI/bge-m3','max_length':8192, 'pooling_type':'cls', 'normalize': True, 'batch_size':1, 'kwargs': {'device_map': 'cuda', 'torch_dtype':torch.float16}}
embeddings_model_spec['Nomic-Embed']={'model_name':'nomic-ai/nomic-embed-text-v1','max_length':8192, 'pooling_type':'mean', 'normalize': True, 'batch_size':1, 'kwargs': {'device_map': 'cuda', 'trust_remote_code' : True}}results = []languages = ["EN", "FR", "CS", "HU"]# Loop through all models
for model_name, model_spec in embeddings_model_spec.items():print("Processing model : "+str(model_spec))# Get modeltokenizer = AutoTokenizer.from_pretrained(model_spec['model_name'])embed_model = AutoModel.from_pretrained(model_spec['model_name'], **model_spec['kwargs'])if model_name=="Nomic-Embed":embed_model.to('cuda')# Loop through all languagesfor language in languages:# Load datasetfile_name=language+"_dataset.json"qa_dataset = EmbeddingQAFinetuneDataset.from_json(file_name)start_time_assessment=time.time()# Assess embedding score (in terms of hit rate at k=5)score = evaluate(qa_dataset, tokenizer, embed_model, model_spec['normalize'], model_spec['max_length'], model_spec['pooling_type'])# Get duration of score assessmentduration_assessment = time.time()-start_time_assessmentresults.append([language, model_name, score, duration_assessment])df_results = pd.DataFrame(results, columns = ["Language" ,"Embedding model", "MRR", "Duration"])