From f256eca04cc3bd4553cd55d4154e498a3b990ba2 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Tue, 20 Oct 2026 16:24:48 +0100 Subject: [PATCH] feat(medcat-trainer): metrics report filter, on row, on low performing metrics --- .../src/components/metrics/ConceptSummary.vue | 541 +++++++++++++++--- .../components/metrics/ConceptSummary.spec.ts | 191 +++++++ 2 files changed, 655 insertions(+), 77 deletions(-) create mode 100644 medcat-trainer/webapp/frontend/src/tests/components/metrics/ConceptSummary.spec.ts diff --git a/medcat-trainer/webapp/frontend/src/components/metrics/ConceptSummary.vue b/medcat-trainer/webapp/frontend/src/components/metrics/ConceptSummary.vue index 65b2900af..f51a4e15f 100644 --- a/medcat-trainer/webapp/frontend/src/components/metrics/ConceptSummary.vue +++ b/medcat-trainer/webapp/frontend/src/components/metrics/ConceptSummary.vue @@ -1,39 +1,71 @@ @@ -117,9 +161,9 @@ v-model="item.cui_prec" color="#32AB60" background-color="#46B480" - height="30px" + height="16" :max="1.0"> - {{item.cui_prec.toFixed(1)}} + {{item.cui_prec.toFixed(2)}} @@ -141,6 +185,14 @@ {{ item.fps }} + @@ -193,6 +245,18 @@ import AnnoResult from '@/components/anns/AnnoResult.vue' import Modal from '@/components/common/Modal.vue' +const VARIATION_PREVIEW_COUNT = 2 +const VARIATION_TERM_LIMIT = 28 + +const SUMMARY_METRICS = [ + { key: 'macro-f1', label: 'Macro F1', kind: 'macro', metric: 'cui_f1' }, + { key: 'macro-prec', label: 'Macro Precision', kind: 'macro', metric: 'cui_prec' }, + { key: 'macro-rec', label: 'Macro Recall', kind: 'macro', metric: 'cui_rec' }, + { key: 'micro-f1', label: 'Micro F1', kind: 'micro', metric: 'cui_f1' }, + { key: 'micro-prec', label: 'Micro Precision', kind: 'micro', metric: 'cui_prec' }, + { key: 'micro-rec', label: 'Micro Recall', kind: 'micro', metric: 'cui_rec' } +] + export default { name: 'ConceptSummary', components: { @@ -217,59 +281,218 @@ export default { title: null, type: null }, + f1Threshold: 0.7, + excludedKeys: [], + sortBy: [{ key: 'cui_f1', order: 'asc' }], headers: [ - {value: 'concept_name', title: 'Concept Name'}, - { value: 'concept_count' }, - { value: 'variations' }, - { value: 'variation_values', title: ''}, - { value: 'count_variations_ratio', title: 'Variation Ratio' }, - { value: 'cui', title: 'CUI' }, - { value: 'cui_f1', title: 'F1' }, - { value: 'cui_prec', title: 'Prec.' }, - { value: 'cui_rec', title: 'Rec.' }, - { value: 'tps', title: 'TPs' }, - { value: 'fns', title: 'FNs' }, - { value: 'fps', title: 'FPs' } + { key: 'concept_name', value: 'concept_name', title: 'Concept Name'}, + { key: 'concept_count', value: 'concept_count' }, + { key: 'variations', value: 'variations' }, + { key: 'variation_values', value: 'variation_values', title: ''}, + { key: 'count_variations_ratio', value: 'count_variations_ratio', title: 'Variation Ratio' }, + { key: 'cui', value: 'cui', title: 'CUI' }, + { key: 'cui_f1', value: 'cui_f1', title: 'F1' }, + { key: 'cui_prec', value: 'cui_prec', title: 'Prec.' }, + { key: 'cui_rec', value: 'cui_rec', title: 'Rec.' }, + { key: 'tps', value: 'tps', title: 'TPs' }, + { key: 'fns', value: 'fns', title: 'FNs' }, + { key: 'fps', value: 'fps', title: 'FPs' }, + { key: 'exclude', value: 'exclude', title: 'Exclude', sortable: false } ] } }, + computed: { + sourceItems() { + return this.items || [] + }, + includedItems() { + return this.sourceItems.filter(item => !this.excludedKeys.includes(this.conceptKey(item))) + }, + excludedItems() { + return this.sourceItems.filter(item => this.excludedKeys.includes(this.conceptKey(item))) + }, + hasExclusions() { + return this.excludedItems.length > 0 + }, + lowPerformingItems() { + return this.includedItems.filter(item => this.isLowPerforming(item)) + }, + lowPerformingCount() { + return this.lowPerformingItems.length + }, + excludeLowPerformersLabel() { + if (this.lowPerformingCount === 0) { + return 'No concepts below this F1' + } + if (this.lowPerformingCount === 1) { + return 'Exclude 1 low performer' + } + return `Exclude ${this.lowPerformingCount} low performers` + }, + summaryMetrics() { + return SUMMARY_METRICS.map(definition => { + const current = this.scoreFor(this.includedItems, definition.kind, definition.metric) + const baseline = this.scoreFor(this.sourceItems, definition.kind, definition.metric) + const delta = current === null || baseline === null ? null : current - baseline + return { ...definition, current, baseline, delta } + }) + } + }, + watch: { + items() { + this.excludedKeys = [] + }, + f1Threshold(value) { + if (!Number.isFinite(value)) { + this.f1Threshold = 0.7 + return + } + const clamped = Math.min(1, Math.max(0, Math.round(value * 100) / 100)) + if (clamped !== value) { + this.f1Threshold = clamped + } + } + }, methods: { - calculateMacroAverage(metric) { - if (!this.items || this.items.length === 0) { - return '0.0' + conceptKey(item) { + return `${item.cui || ''}::${item.concept_name || ''}` + }, + variationList(item) { + const values = item.value + if (Array.isArray(values)) { + return values.map(value => String(value)).filter(value => value.length > 0) + } + if (values === null || values === undefined || values === '') { + return [] + } + return [String(values)] + }, + shortVariation(value) { + if (value.length <= VARIATION_TERM_LIMIT) { + return value + } + return `${value.slice(0, VARIATION_TERM_LIMIT - 1)}…` + }, + variationPreview(item) { + const values = this.variationList(item) + if (!values.length) { + return '' + } + const shown = values.slice(0, VARIATION_PREVIEW_COUNT).map(value => this.shortVariation(value)) + const hidden = values.length - shown.length + if (hidden > 0) { + return `${shown.join(', ')} +${hidden}` + } + return shown.join(', ') + }, + variationTooltip(item) { + return this.variationList(item).join(', ') + }, + isLowPerforming(item) { + const f1 = Number(item.cui_f1) + return Number.isFinite(f1) && f1 < this.f1Threshold + }, + excludeConcept(item) { + const key = this.conceptKey(item) + if (!this.excludedKeys.includes(key)) { + this.excludedKeys = [...this.excludedKeys, key] } - const sum = this.items.reduce((acc, item) => acc + item[metric], 0) - return ((sum / this.items.length) * 100).toFixed(1) }, - calculateMicroAverage(metric) { - if (!this.items || this.items.length === 0) { - return '0.0' + excludeLowPerformers() { + const keys = this.lowPerformingItems.map(item => this.conceptKey(item)) + this.excludedKeys = [...this.excludedKeys, ...keys] + }, + restoreConcept(item) { + const key = this.conceptKey(item) + this.excludedKeys = this.excludedKeys.filter(excluded => excluded !== key) + }, + restoreAll() { + this.excludedKeys = [] + }, + rowProps({ item }) { + if (this.isLowPerforming(item)) { + return { class: 'low-perf-row' } + } + return {} + }, + scoreFor(items, kind, metric) { + if (this.sourceItems.length === 0) { + return 0 + } + if (!items.length) { + return null } + if (kind === 'macro') { + return this.macroAverage(items, metric) + } + return this.microAverage(items)[metric] + }, + macroAverage(items, metric) { + const values = items + .map(item => Number(item[metric])) + .filter(value => Number.isFinite(value)) + if (!values.length) { + return null + } + return values.reduce((acc, value) => acc + value, 0) / values.length + }, + microAverage(items) { let totalTPs = 0 let totalFPs = 0 let totalFNs = 0 - this.items.forEach(item => { - totalTPs += item.tps - totalFPs += item.fps - totalFNs += item.fns + items.forEach(item => { + totalTPs += Number(item.tps) || 0 + totalFPs += Number(item.fps) || 0 + totalFNs += Number(item.fns) || 0 }) - let precision = totalTPs / (totalTPs + totalFPs) - let recall = totalTPs / (totalTPs + totalFNs) - let f1 = 2 * (precision * recall) / (precision + recall) - - switch(metric) { - case 'cui_prec': - return (precision * 100).toFixed(1) - case 'cui_rec': - return (recall * 100).toFixed(1) - case 'cui_f1': - return (f1 * 100).toFixed(1) - default: - return '0.0' + const precision = this.ratio(totalTPs, totalTPs + totalFPs) + const recall = this.ratio(totalTPs, totalTPs + totalFNs) + let f1 = null + if (precision !== null && recall !== null && (precision + recall) > 0) { + f1 = (2 * precision * recall) / (precision + recall) + } else if (precision === 0 && recall === 0) { + f1 = 0 + } + + return { + cui_prec: precision, + cui_rec: recall, + cui_f1: f1 } }, + ratio(numerator, denominator) { + if (!denominator) { + return null + } + const value = numerator / denominator + return Number.isFinite(value) ? value : null + }, + formatPercent(value) { + if (value === null || !Number.isFinite(value)) { + return '—' + } + return `${(value * 100).toFixed(1)}%` + }, + formatDelta(delta) { + const points = delta * 100 + const sign = points > 0 ? '+' : '' + return `${sign}${points.toFixed(1)} pts` + }, + formatRatio(value) { + const numeric = Number(value) + if (!Number.isFinite(numeric)) { + return '—' + } + return numeric.toFixed(2) + }, + deltaClass(delta) { + if (delta === null || !Number.isFinite(delta) || Math.abs(delta) < 0.0005) { + return 'flat' + } + return delta > 0 ? 'up' : 'down' + }, openExamples(exampleType, item) { if (exampleType === 'tp_examples') { this.modalData.title = `${item.concept_name} (${item.cui}) - True Positive Model Predictions` @@ -307,13 +530,15 @@ export default { } .perf-progress { - width: 50px; + width: 54px; } .res-btn { - height: 25px; - padding: 3px 10px !important; + height: 20px; + padding: 0 8px !important; border: 0 !important; + font-size: 12px; + line-height: 20px; } .res-btn:hover { @@ -328,4 +553,166 @@ export default { .examples-modal .modal-container { width: 90%; } - \ No newline at end of file + + + diff --git a/medcat-trainer/webapp/frontend/src/tests/components/metrics/ConceptSummary.spec.ts b/medcat-trainer/webapp/frontend/src/tests/components/metrics/ConceptSummary.spec.ts new file mode 100644 index 000000000..94903b712 --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/tests/components/metrics/ConceptSummary.spec.ts @@ -0,0 +1,191 @@ +import { describe, expect, it } from 'vitest' +import { mount } from '@vue/test-utils' +import ConceptSummary from '@/components/metrics/ConceptSummary.vue' + +const concepts = [ + { + cui: 'C0001', + concept_name: 'Low concept', + concept_count: 8, + variations: 2, + value: ['low', 'low concept'], + count_variations_ratio: 4, + cui_f1: 0.2, + cui_prec: 0.25, + cui_rec: 0.2, + tps: 1, + fps: 3, + fns: 4, + tp_examples: [], + fp_examples: [], + fn_examples: [] + }, + { + cui: 'C0002', + concept_name: 'Strong concept', + concept_count: 4, + variations: 1, + value: ['strong'], + count_variations_ratio: 4, + cui_f1: 1, + cui_prec: 1, + cui_rec: 1, + tps: 4, + fps: 0, + fns: 0, + tp_examples: [], + fp_examples: [], + fn_examples: [] + }, + { + cui: 'C0003', + concept_name: 'Mid concept', + concept_count: 6, + variations: 2, + value: ['mid', 'middle'], + count_variations_ratio: 3, + cui_f1: 0.8, + cui_prec: 0.8, + cui_rec: 0.8, + tps: 4, + fps: 1, + fns: 1, + tp_examples: [], + fp_examples: [], + fn_examples: [] + } +] + +const mountSummary = () => mount(ConceptSummary, { + props: { + items: concepts.map(concept => ({ ...concept, value: [...concept.value] })), + docs2text: {} + }, + global: { + stubs: { + 'v-row': { template: '
' }, + 'v-card': { template: '
' }, + 'v-card-title': { template: '

' }, + 'v-card-text': { template: '
' }, + 'v-tooltip': { + template: '' + }, + 'v-progress-linear': { template: '
' }, + 'v-data-table': { + props: ['items'], + template: ` +
+
+ + +
+
+ ` + }, + 'font-awesome-icon': true, + modal: true, + 'anno-result': true + } + } +}) + +describe('metrics ConceptSummary.vue', () => { + it('recalculates scores when a low-performing concept is excluded and restored', async () => { + const wrapper = mountSummary() + + expect(Number((wrapper.get('#low-f1-threshold').element as HTMLInputElement).value)).toBe(0.7) + expect(wrapper.text()).toContain('66.7%') + expect(wrapper.text()).toContain('Exclude 1 low performer') + expect(wrapper.text()).not.toContain('full set') + + await wrapper.get('[aria-label="Exclude Low concept"]').trigger('click') + + expect(wrapper.text()).toContain('90.0%') + expect(wrapper.text()).toContain('+23.3 pts') + expect(wrapper.text()).toContain('full set 66.7%') + expect(wrapper.text()).toContain('1 concept excluded from these scores') + expect(wrapper.text()).toContain('2 of 3 concepts in these scores') + expect(wrapper.find('[aria-label="Exclude Low concept"]').exists()).toBe(false) + + await wrapper.get('[aria-label="Restore Low concept"]').trigger('click') + + expect(wrapper.text()).toContain('66.7%') + expect(wrapper.text()).not.toContain('full set') + expect(wrapper.text()).toContain('3 of 3 concepts in these scores') + }) + + it('excludes every concept under the F1 threshold and can restore them together', async () => { + const wrapper = mountSummary() + + await wrapper.get('#low-f1-threshold').setValue('0.9') + await wrapper.get('button.btn-outline-danger').trigger('click') + + expect(wrapper.text()).toContain('100.0%') + expect(wrapper.text()).toContain('2 concepts excluded from these scores') + expect(wrapper.text()).toContain('Low concept') + expect(wrapper.text()).toContain('Mid concept') + expect(wrapper.find('[aria-label="Exclude Strong concept"]').exists()).toBe(true) + + await wrapper.get('button.btn-outline-primary').trigger('click') + + expect(wrapper.text()).toContain('3 of 3 concepts in these scores') + expect(wrapper.text()).not.toContain('excluded from these scores') + }) + + it('keeps a long variation list on one line and shows the full set in the tooltip', () => { + const many = Array.from({ length: 12 }, (_, index) => `variation term ${index} with extra words`) + const wrapper = mount(ConceptSummary, { + props: { + items: [{ + ...concepts[0], + value: many + }], + docs2text: {} + }, + global: { + stubs: { + 'v-row': { template: '
' }, + 'v-card': { template: '
' }, + 'v-card-title': { template: '

' }, + 'v-card-text': { template: '
' }, + 'v-tooltip': { + template: '' + }, + 'v-progress-linear': { template: '
' }, + 'v-data-table': { + props: ['items'], + template: ` +
+
+ +
+
+ ` + }, + 'font-awesome-icon': true, + modal: true, + 'anno-result': true + } + } + }) + + const preview = wrapper.get('.variation-values').text() + expect(preview).toBe('variation term 0 with extra…, variation term 1 with extra… +10') + expect(preview).not.toContain('variation term 11') + expect(wrapper.get('.tooltip-body').text()).toContain(many[11]) + }) + + it('shows an em dash when every concept is excluded', async () => { + const wrapper = mountSummary() + const vm = wrapper.vm as unknown as { + excludeConcept: (item: { cui: string, concept_name: string }) => void + includedItems: Array<{ cui: string, concept_name: string }> + } + + vm.includedItems.forEach(item => vm.excludeConcept(item)) + await wrapper.vm.$nextTick() + + expect(wrapper.text()).toContain('0 of 3 concepts in these scores') + expect(wrapper.text()).toContain('—') + }) +})