应用程序任务驱动：详细解析LLM的评估指标 原创

1 >>> from blanc import BlancHelp, BlancTune
2 >>> document = "Jack drove his minivan to the bazaar to purchase milk and honey for his large family."
3 >>> summary = "Jack bought milk and honey."
4 >>> blanc_help = BlancHelp()
5 >>> blanc_tune = BlancTune(finetune_mask_evenly=False, show_progress_bar=False)
6 >>> blanc_help.eval_once(document, summary)
7 0.2222222222222222
8 >>> blanc_tune.eval_once(document, summary)
9 0.3333333333333333
1.
2.
3.
4.
5.
6.
7.
8.
9.

1 from ref_free_metrics.supert import Supert
2 from utils.data_reader import CorpusReader
3
4 # read docs and summaries
5 reader = CorpusReader('data/topic_1')
6 source_docs = reader()
7 summaries = reader.readSummaries() 
8
9 # compute the Supert scores
10 supert = Supert(source_docs) 
11 scores = supert(summaries)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

1
2 >>> predictions = ["hello there general kenobi", "foo bar foobar"]
3 >>> references = [
4 ...     ["hello there general kenobi", "hello there !"],
5 ...     ["foo bar foobar"]
6 ... ]
7 >>> bleu = evaluate.load("bleu")
8 >>> results = bleu.compute(predictions=predictions, references=references)
9 >>> print(results)
10 {'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6}
11
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

1 >>> rouge = evaluate.load('rouge')
2 >>> predictions = ["hello goodbye", "ankh morpork"]
3 >>> references = ["goodbye", "general kenobi"]
4 >>> results = rouge.compute(predictions=predictions,
5 ...                         references=references,
6 ...                         use_aggregator=False)
7 >>> print(list(results.keys()))
8 ['rouge1', 'rouge2', 'rougeL', 'rougeLsum']
9 >>> print(results["rouge1"])
10 [0.5, 0.0]
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.

1 from evaluate import load
2 bertscore = load("bertscore")
3 predictions = ["hello world", "general kenobi"]
4 references = ["hello world", "general kenobi"]
5 results = bertscore.compute(predictions=predictions, references=references, model_type="distilbert-base-uncased")
6 print(results)
7 {'precision': [1.0, 1.0], 'recall': [1.0, 1.0], 'f1': [1.0, 1.0], 'hashcode': 'distilbert-base-uncased_L5_no-idf_version=0.3.10(hug_trans=4.10.3)'}
8
Huggingface的BERTScore指标的基本用法示例代码。
1.
2.
3.
4.
5.
6.
7.
8.
9.

1 from datasets import Dataset 
2 import os
3 from ragas import evaluate
4 from ragas.metrics import faithfulness, answer_correctness
5
6 os.environ["OPENAI_API_KEY"] = "your-openai-key"
7
8 data_samples = {
9    'question': ['When was the first super bowl?', 'Who won the most super bowls?'],
10    'answer': ['The first superbowl was held on Jan 15, 1967', 'The most super bowls have been won by The New England Patriots'],
11    'contexts' : [['The First AFL–NFL World Championship Game was an American football game played on January 15, 1967, at the Los Angeles Memorial Coliseum in Los Angeles,'], 
12    ['The Green Bay Packers...Green Bay, Wisconsin.','The Packers compete...Football Conference']],
13    'ground_truth': ['The first superbowl was held on January 15, 1967', 'The New England Patriots have won the Super Bowl a record six times']
14 }
15
16 dataset = Dataset.from_dict(data_samples)
17
18 score = evaluate(dataset,metrics=[faithfulness,answer_correctness])
19 score.to_pandas()
RAG忠诚度的例子。
Python 
1 from ares import ARES
2
3 ues_idp_config = {
4    "in_domain_prompts_dataset": "nq_few_shot_prompt_for_judge_scoring.tsv",
5    "unlabeled_evaluation_set": "nq_unlabeled_output.tsv", 
6    "model_choice" : "gpt-3.5-turbo-0125"
7 } 
8
9 ares = ARES(ues_idp=ues_idp_config)
10 results = ares.ues_idp()
11 print(results)
12 # {'Context Relevance Scores': [Score], 'Answer Faithfulness Scores': [Score], 'Answer Relevance Scores': [
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.

1 from qafacteval import QAFactEval
2 kwargs = {"cuda_device": 0, "use_lerc_quip": True, \
3        "verbose": True, "generation_batch_size": 32, \
4        "answering_batch_size": 32, "lerc_batch_size": 8}
5
6 model_folder = "" # path to models downloaded with download_models.sh
7 metric = QAFactEval(
8    lerc_quip_path=f"{model_folder}/quip-512-mocha",
9    generation_model_path=f"{model_folder}/generation/model.tar.gz",
10    answering_model_dir=f"{model_folder}/answering",
11    lerc_model_path=f"{model_folder}/lerc/model.tar.gz",
12lerc_pretrained_model_path=f"{model_folder}/lerc/pretraining.tar.gz",
13    **kwargs
14 )
15
16 results = metric.score_batch_qafacteval(["This is a source document"], [["This is a summary."]], return_qa_pairs=True)
17 score = results[0][0]['qa-eval']['lerc_quip']
18
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.

1 from deepeval import evaluate
2 from deepeval.metrics import SummarizationMetric
3 from deepeval.test_case import LLMTestCase
4 ...
5
6 test_case = LLMTestCase(input=input, actual_output=actual_output)
7 metric = SummarizationMetric(
8    threshold=0.5,
9    model="gpt-4",
10    assessment_questions=[
11        "Is the coverage score based on a percentage of 'yes' answers?",
12        "Does the score ensure the summary's accuracy with the source?",
13        "Does a higher score mean a more comprehensive summary?"
14    ]
15 )
16
17 metric.measure(test_case)
18 print(metric.score)
19 print(metric.reason)
20
21 # or evaluate test cases in bulk
22 evaluate([test_case], [metric])
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.