[Project]Evaluation Matric 수정하기

수정코드

        # calculate recall and precision with tp and fp
        cls_all_tp = np.sum(tp)
        cls_all_fp = np.sum(fp)
        tp = np.cumsum(tp, axis=1)
        fp = np.cumsum(fp, axis=1)
        eps = np.finfo(np.float32).eps
        recalls = tp / np.maximum(num_gts[:, np.newaxis], eps) # num_gts가 gt수니까
        precisions = tp / np.maximum((tp + fp), eps)

        # calculate AP
        # scale_ranges 지정 되어 있으면 rec, prec, num_gts 값 변경
        if scale_ranges is None:
            recalls = recalls[0, :]
            precisions = precisions[0, :]
            num_gts = num_gts.item()
        mode = 'area' if not use_07_metric else '11points'
        ap = average_precision(recalls, precisions, mode)
        # calculate F1 (wrong)
        # f1_score = 2 * precisions * recalls / np.maximum((precisions+recalls),eps)
        # f1_score = f1_score.mean()
        print(f"cls_all_tp : {cls_all_tp}")
        print("="*20)
        print(f"cls_all_fp : {cls_all_fp}")
        # re calculate F1 
        cls_all_rec = cls_all_tp / np.maximum(num_gts, eps)
        cls_all_pre = cls_all_tp / np.maximum((cls_all_tp + cls_all_fp), eps)
        f1_score = 2 * cls_all_pre * cls_all_rec / np.maximum((cls_all_pre+cls_all_rec),eps)

결과

cls_all_tp : 11159.0
====================
cls_all_fp : 25422.0
cls_all_tp : 12266.0
====================
cls_all_fp : 105994.0
cls_all_tp : 118864.0
====================
cls_all_fp : 171550.0
0.56824106 0.38241543397228606 [{'num_gts': 13924, 'num_dets': 36581, 'recall': array([7.1818e-05, 1.4364e-04, 2.1546e-04, ..., 8.0142e-01, 8.0142e-01, 8.0142e-01]), 'precision': array([1.    , 1.    , 1.    , ..., 0.3051, 0.3051, 0.305 ], dtype=float32), 'ap': 0.7537326, 'F1': 0.44189683411811864}, {'num_gts': 27520, 'num_dets': 118260, 'recall': array([3.6337e-05, 7.2674e-05, 1.0901e-04, ..., 4.4571e-01, 4.4571e-01, 4.4571e-01]), 'precision': array([1.    , 1.    , 1.    , ..., 0.1037, 0.1037, 0.1037], dtype=float32), 'ap': 0.27206123, 'F1': 0.1682809676824649}, {'num_gts': 152226, 'num_dets': 290414, 'recall': array([6.5692e-06, 1.3138e-05, 1.9708e-05, ..., 7.8084e-01, 7.8084e-01, 7.8084e-01]), 'precision': array([1.    , 1.    , 1.    , ..., 0.4093, 0.4093, 0.4093], dtype=float32), 'ap': 0.6789294, 'F1': 0.5370685001162745}]
2022-10-07 15:09:21,153 - mmrotate - INFO - 
+----------+--------+--------+--------+-----------+-------+-------+
| class    | gts    | dets   | recall | precision | ap    | F1    |
+----------+--------+--------+--------+-----------+-------+-------+
| Airplane | 13924  | 36581  | 0.801  | 0.305     | 0.754 | 0.442 |
| Ship     | 27520  | 118260 | 0.446  | 0.104     | 0.272 | 0.168 |
| Vehicle  | 152226 | 290414 | 0.781  | 0.409     | 0.679 | 0.537 |
+----------+--------+--------+--------+-----------+-------+-------+
| mAP, mF1 |        |        |        |           | 0.568 | 0.382 |
+----------+--------+--------+--------+-----------+-------+-------+
2022-10-07 15:09:21,159 - mmrotate - INFO - Exp name: oriented_rcnn_r50_fpn_1x_dota_le90.py
2022-10-07 15:09:21,159 - mmrotate - INFO - Epoch(val) [1][7653]	mAP: 0.5682410597801208, F1Score: 0.3824

recall, precision도 맞춰서 변경

        # calculate AP
        # scale_ranges 지정 되어 있으면 rec, prec, num_gts 값 변경
        if scale_ranges is None:
            recalls = recalls[0, :]
            precisions = precisions[0, :]
            num_gts = num_gts.item()
        mode = 'area' if not use_07_metric else '11points'
        ap = average_precision(recalls, precisions, mode)
        # calculate F1 (wrong)
        # f1_score = 2 * precisions * recalls / np.maximum((precisions+recalls),eps)
        # f1_score = f1_score.mean()
        
        # recall, precision도 직접 계산값 사용
        # 참고 : https://towardsdatascience.com/micro-macro-weighted-averages-of-f1-score-clearly-explained-b603420b292f
        # re calculate F1 
        cls_all_rec = cls_all_tp / np.maximum(num_gts, eps)
        cls_all_pre = cls_all_tp / np.maximum((cls_all_tp + cls_all_fp), eps)
        f1_score = 2 * cls_all_pre * cls_all_rec / np.maximum((cls_all_pre+cls_all_rec),eps)


        eval_results.append({
            'num_gts': num_gts,
            'num_dets': num_dets,
            'recall': cls_all_rec,
            'precision': cls_all_pre,
            'ap': ap,
            'F1' : f1_score
        })
    pool.close()

결과

2022-10-07 15:55:54,335 - mmrotate - INFO - Saving checkpoint at 1 epochs
[>>] 7653/7653, 11.9 task/s, elapsed: 642s, ETA:     0s0.54283684 0.434323229252455 [{'num_gts': 13924, 'num_dets': 23704, 'recall': 0.8027147371444987, 'precision': 0.4715238, 'ap': 0.7731695, 'F1': 0.5940788759229012}, {'num_gts': 27520, 'num_dets': 49964, 'recall': 0.3596293604651163, 'precision': 0.19808263, 'ap': 0.21362929, 'F1': 0.2554591968806364}, {'num_gts': 152226, 'num_dets': 366785, 'recall': 0.7729822763522657, 'precision': 0.3208092, 'ap': 0.6417116, 'F1': 0.45343161495382733}]
2022-10-07 16:07:14,088 - mmrotate - INFO - 
+----------+--------+--------+--------+-----------+-------+-------+
| class    | gts    | dets   | recall | precision | ap    | F1    |
+----------+--------+--------+--------+-----------+-------+-------+
| Airplane | 13924  | 23704  | 0.803  | 0.472     | 0.773 | 0.594 |
| Ship     | 27520  | 49964  | 0.360  | 0.198     | 0.214 | 0.255 |
| Vehicle  | 152226 | 366785 | 0.773  | 0.321     | 0.642 | 0.453 |
+----------+--------+--------+--------+-----------+-------+-------+
| mAP, mF1 |        |        |        |           | 0.543 | 0.434 |
+----------+--------+--------+--------+-----------+-------+-------+
0.54283684 0.434323229252455