From 741785a3d9cb69f8d4dbc12a092bd082fc460702 Mon Sep 17 00:00:00 2001
From: Ting <wtmlon@foxmail.com>
Date: Fri, 29 Nov 2024 14:12:38 +0800
Subject: [PATCH] support pp accuracy calculation (#9379)

* support pp accuracy calculation

* add pp accuracy ci

* add comment

* update

* mv logits accumulation to cpu

* refactor code

* code refactor

* remove ci, not support yet

* update
---
 paddlenlp/trainer/trainer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/paddlenlp/trainer/trainer.py b/paddlenlp/trainer/trainer.py
index 2418b84791f8..5d44c35767c7 100644
--- a/paddlenlp/trainer/trainer.py
+++ b/paddlenlp/trainer/trainer.py
@@ -3197,7 +3197,9 @@ def evaluation_loop(
 
         # Metrics!
         if self.compute_metrics is not None and all_preds is not None and all_labels is not None:
-            metrics = self.compute_metrics(EvalPrediction(predictions=all_preds, label_ids=all_labels))
+            # all_labels maybe is a tuple when prediction_steps output label_mask
+            batch_labels = all_labels[0] if isinstance(all_labels, (list, tuple)) else all_labels
+            metrics = self.compute_metrics(EvalPrediction(predictions=all_preds, label_ids=batch_labels))
         else:
             metrics = {}