data-gen-bpu/scripts/test_classification_predictions.py

48 lines
1.8 KiB
Python

"""Global sample mapping and refusal of truncated/non-finite runner output."""
import unittest
from unittest.mock import patch
import numpy as np
from harness.eval import cls
class Package:
def __init__(self, n):
self.items = [{'file': f'item-{i}.jpg', 'label': i % 7, 'index': i} for i in range(n)]
def image(self, item):
return item['index']
class Runner:
def batch(self, xs):
result = []
for value in xs:
logits = np.arange(1000, dtype=np.float32) * -.001
logits[int(value) % 7] = 100
result.append([logits])
return result
class ClassificationPredictions(unittest.TestCase):
def test_predictions_keep_global_ids_across_chunks(self):
package = Package(300)
baseline, measured = {}, {}
with patch.object(cls, 'preprocess_cls', side_effect=lambda image, **kw: image):
cls.run_fp32(Runner(), package, '1x3x224x224', baseline)
cls.run_quant(Runner(), package, '1x3x224x224', baseline, measured)
rows = measured['_sample_predictions']
self.assertEqual(len(rows), 300)
self.assertEqual([row['sample_id'] for row in rows], [it['file'] for it in package.items])
self.assertTrue(all(row['top5'][0] == i % 7 for i, row in enumerate(rows)))
self.assertEqual(measured['metrics']['top1'], 100)
def test_truncated_and_nonfinite_outputs_are_rejected(self):
package = Package(2)
for results in ([[np.ones(1000)]], [[np.full(1000, np.nan)], [np.ones(1000)]]):
with patch.object(cls, 'preprocess_cls', return_value=np.zeros((1, 3, 224, 224))), \
patch.object(cls, 'run_batched', return_value=results), self.assertRaises(ValueError):
cls.predict(Runner(), package, '1x3x224x224')
if __name__ == '__main__':
unittest.main()