48 lines
1.8 KiB
Python
48 lines
1.8 KiB
Python
"""Global sample mapping and refusal of truncated/non-finite runner output."""
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from harness.eval import cls
|
|
|
|
|
|
class Package:
|
|
def __init__(self, n):
|
|
self.items = [{'file': f'item-{i}.jpg', 'label': i % 7, 'index': i} for i in range(n)]
|
|
def image(self, item):
|
|
return item['index']
|
|
|
|
|
|
class Runner:
|
|
def batch(self, xs):
|
|
result = []
|
|
for value in xs:
|
|
logits = np.arange(1000, dtype=np.float32) * -.001
|
|
logits[int(value) % 7] = 100
|
|
result.append([logits])
|
|
return result
|
|
|
|
|
|
class ClassificationPredictions(unittest.TestCase):
|
|
def test_predictions_keep_global_ids_across_chunks(self):
|
|
package = Package(300)
|
|
baseline, measured = {}, {}
|
|
with patch.object(cls, 'preprocess_cls', side_effect=lambda image, **kw: image):
|
|
cls.run_fp32(Runner(), package, '1x3x224x224', baseline)
|
|
cls.run_quant(Runner(), package, '1x3x224x224', baseline, measured)
|
|
rows = measured['_sample_predictions']
|
|
self.assertEqual(len(rows), 300)
|
|
self.assertEqual([row['sample_id'] for row in rows], [it['file'] for it in package.items])
|
|
self.assertTrue(all(row['top5'][0] == i % 7 for i, row in enumerate(rows)))
|
|
self.assertEqual(measured['metrics']['top1'], 100)
|
|
|
|
def test_truncated_and_nonfinite_outputs_are_rejected(self):
|
|
package = Package(2)
|
|
for results in ([[np.ones(1000)]], [[np.full(1000, np.nan)], [np.ones(1000)]]):
|
|
with patch.object(cls, 'preprocess_cls', return_value=np.zeros((1, 3, 224, 224))), \
|
|
patch.object(cls, 'run_batched', return_value=results), self.assertRaises(ValueError):
|
|
cls.predict(Runner(), package, '1x3x224x224')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|