-
Notifications
You must be signed in to change notification settings - Fork 2
/
metrics.py
213 lines (170 loc) · 8.56 KB
/
metrics.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
# Code in this file is modified from the following link:
# https://github.com/MouseLand/cellpose/blob/main/cellpose/metrics.py
# Copyright © 2020 Howard Hughes Medical Institute
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
# Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
# Neither the name of HHMI nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import numpy as np
from scipy.optimize import linear_sum_assignment
def mask_ious(masks_true, masks_pred):
"""return best-matched masks"""
iou = _intersection_over_union(masks_true, masks_pred)[1:, 1:]
n_min = min(iou.shape[0], iou.shape[1])
costs = -(iou >= 0.5).astype(float) - iou / (2 * n_min)
true_ind, pred_ind = linear_sum_assignment(costs)
iout = np.zeros(masks_true.max())
iout[true_ind] = iou[true_ind, pred_ind]
preds = np.zeros(masks_true.max(), "int")
preds[true_ind] = pred_ind + 1
return iout, preds
def average_precision(masks_true, masks_pred, threshold=[0.5, 0.75, 0.9]):
"""average precision estimation: AP = TP / (TP + FP + FN)
This function is based heavily on the *fast* stardist matching functions
(https://github.com/mpicbg-csbd/stardist/blob/master/stardist/matching.py)
Parameters
------------
masks_true: list of ND-arrays (int) or ND-array (int)
where 0=NO masks; 1,2... are mask labels
masks_pred: list of ND-arrays (int) or ND-array (int)
ND-array (int) where 0=NO masks; 1,2... are mask labels
Returns
------------
ap: array [len(masks_true) x len(threshold)]
average precision at thresholds
tp: array [len(masks_true) x len(threshold)]
number of true positives at thresholds
fp: array [len(masks_true) x len(threshold)]
number of false positives at thresholds
fn: array [len(masks_true) x len(threshold)]
number of false negatives at thresholds
"""
not_list = False
if not isinstance(masks_true, list):
masks_true = [masks_true]
masks_pred = [masks_pred]
not_list = True
if not isinstance(threshold, list) and not isinstance(threshold, np.ndarray):
threshold = [threshold]
if len(masks_true) != len(masks_pred):
raise ValueError(
"metrics.average_precision requires len(masks_true)==len(masks_pred)"
)
ap = np.zeros((len(masks_true), len(threshold)), np.float32)
tp = np.zeros((len(masks_true), len(threshold)), np.float32)
fp = np.zeros((len(masks_true), len(threshold)), np.float32)
fn = np.zeros((len(masks_true), len(threshold)), np.float32)
n_true = np.array(list(map(np.max, masks_true)))
n_pred = np.array(list(map(np.max, masks_pred)))
for n in range(len(masks_true)):
# _,mt = np.reshape(np.unique(masks_true[n], return_index=True), masks_pred[n].shape)
if n_pred[n] > 0:
iou = _intersection_over_union(masks_true[n], masks_pred[n])[1:, 1:]
for k, th in enumerate(threshold):
tp[n, k] = _true_positive(iou, th)
fp[n] = n_pred[n] - tp[n]
fn[n] = n_true[n] - tp[n]
if (tp[n] + fp[n] + fn[n]) == 0:
if ap[n] == 0:
ap[n] = 1.0
else:
ap[n] = 0.0
else:
ap[n] = tp[n] / (tp[n] + fp[n] + fn[n])
if not_list:
ap, tp, fp, fn = ap[0], tp[0], fp[0], fn[0]
return ap, tp, fp, fn
def _label_overlap(x, y):
"""fast function to get pixel overlaps between masks in x and y
Parameters
------------
x: ND-array, int
where 0=NO masks; 1,2... are mask labels
y: ND-array, int
where 0=NO masks; 1,2... are mask labels
Returns
------------
overlap: ND-array, int
matrix of pixel overlaps of size [x.max()+1, y.max()+1]
"""
# put label arrays into standard form then flatten them
# x = (utils.format_labels(x)).ravel()
# y = (utils.format_labels(y)).ravel()
x = x.ravel()
y = y.ravel()
# preallocate a 'contact map' matrix
overlap = np.zeros((1 + x.max(), 1 + y.max()), dtype=np.uint)
# loop over the labels in x and add to the corresponding
# overlap entry. If label A in x and label B in y share P
# pixels, then the resulting overlap is P
# len(x)=len(y), the number of pixels in the whole image
for i in range(len(x)):
overlap[x[i], y[i]] += 1
return overlap
def _intersection_over_union(masks_true, masks_pred):
"""intersection over union of all mask pairs
Parameters
------------
masks_true: ND-array, int
ground truth masks, where 0=NO masks; 1,2... are mask labels
masks_pred: ND-array, int
predicted masks, where 0=NO masks; 1,2... are mask labels
Returns
------------
iou: ND-array, float
matrix of IOU pairs of size [x.max()+1, y.max()+1]
------------
How it works:
The overlap matrix is a lookup table of the area of intersection
between each set of labels (true and predicted). The true labels
are taken to be along axis 0, and the predicted labels are taken
to be along axis 1. The sum of the overlaps along axis 0 is thus
an array giving the total overlap of the true labels with each of
the predicted labels, and likewise the sum over axis 1 is the
total overlap of the predicted labels with each of the true labels.
Because the label 0 (background) is included, this sum is guaranteed
to reconstruct the total area of each label. Adding this row and
column vectors gives a 2D array with the areas of every label pair
added together. This is equivalent to the union of the label areas
except for the duplicated overlap area, so the overlap matrix is
subtracted to find the union matrix.
"""
overlap = _label_overlap(masks_true, masks_pred)
n_pixels_pred = np.sum(overlap, axis=0, keepdims=True)
n_pixels_true = np.sum(overlap, axis=1, keepdims=True)
iou = overlap / (n_pixels_pred + n_pixels_true - overlap)
iou[np.isnan(iou)] = 0.0
return iou
def _true_positive(iou, th):
"""true positive at threshold th
Parameters
------------
iou: float, ND-array
array of IOU pairs
th: float
threshold on IOU for positive label
Returns
------------
tp: float
number of true positives at threshold
------------
How it works:
(1) Find minimum number of masks
(2) Define cost matrix; for a given threshold, each element is negative
the higher the IoU is (perfect IoU is 1, worst is 0). The second term
gets more negative with higher IoU, but less negative with greater
n_min (but that's a constant...)
(3) Solve the linear sum assignment problem. The costs array defines the cost
of matching a true label with a predicted label, so the problem is to
find the set of pairings that minimizes this cost. The scipy.optimize
function gives the ordered lists of corresponding true and predicted labels.
(4) Extract the IoUs fro these parings and then threshold to get a boolean array
whose sum is the number of true positives that is returned.
"""
n_min = min(iou.shape[0], iou.shape[1])
costs = -(iou >= th).astype(float) - iou / (2 * n_min)
true_ind, pred_ind = linear_sum_assignment(costs)
match_ok = iou[true_ind, pred_ind] >= th
tp = match_ok.sum()
return tp