gonum.org/v1/gonum@v0.15.1-0.20240517103525-f853624cb1bb/stat/roc_test.go

// Copyright ©2016 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import (
	"fmt"
	"math"
	"slices"
	"testing"

	"golang.org/x/exp/rand"

	"gonum.org/v1/gonum/floats"
)

func TestROC(t *testing.T) {
	const tol = 1e-14

	cases := []struct {
		y          []float64
		c          []bool
		w          []float64
		cutoffs    []float64
		wantTPR    []float64
		wantFPR    []float64
		wantThresh []float64
	}{
		// Test cases were informed by using sklearn metrics.roc_curve when
		// cutoffs is nil, but all test cases (including when cutoffs is not
		// nil) were calculated manually.
		// Some differences exist between unweighted ROCs from our function
		// and metrics.roc_curve which appears to use integer cutoffs in that
		// case. sklearn also appears to do some magic that trims leading zeros
		// sometimes.
		{ // 0
			y:          []float64{0, 3, 5, 6, 7.5, 8},
			c:          []bool{false, true, false, true, true, true},
			wantTPR:    []float64{0, 0.25, 0.5, 0.75, 0.75, 1, 1},
			wantFPR:    []float64{0, 0, 0, 0, 0.5, 0.5, 1},
			wantThresh: []float64{math.Inf(1), 8, 7.5, 6, 5, 3, 0},
		},
		{ // 1
			y:          []float64{0, 3, 5, 6, 7.5, 8},
			c:          []bool{false, true, false, true, true, true},
			w:          []float64{4, 1, 6, 3, 2, 2},
			wantTPR:    []float64{0, 0.25, 0.5, 0.875, 0.875, 1, 1},
			wantFPR:    []float64{0, 0, 0, 0, 0.6, 0.6, 1},
			wantThresh: []float64{math.Inf(1), 8, 7.5, 6, 5, 3, 0},
		},
		{ // 2
			y:          []float64{0, 3, 5, 6, 7.5, 8},
			c:          []bool{false, true, false, true, true, true},
			cutoffs:    []float64{-1, 2, 4, 6, 8},
			wantTPR:    []float64{0.25, 0.75, 0.75, 1, 1},
			wantFPR:    []float64{0, 0, 0.5, 0.5, 1},
			wantThresh: []float64{8, 6, 4, 2, -1},
		},
		{ // 3
			y:          []float64{0, 3, 5, 6, 7.5, 8},
			c:          []bool{false, true, false, true, true, true},
			cutoffs:    []float64{-1, 1, 2, 3, 4, 5, 6, 7, 8},
			wantTPR:    []float64{0.25, 0.5, 0.75, 0.75, 0.75, 1, 1, 1, 1},
			wantFPR:    []float64{0, 0, 0, 0.5, 0.5, 0.5, 0.5, 0.5, 1},
			wantThresh: []float64{8, 7, 6, 5, 4, 3, 2, 1, -1},
		},
		{ // 4
			y:          []float64{0, 3, 5, 6, 7.5, 8},
			c:          []bool{false, true, false, true, true, true},
			w:          []float64{4, 1, 6, 3, 2, 2},
			cutoffs:    []float64{-1, 2, 4, 6, 8},
			wantTPR:    []float64{0.25, 0.875, 0.875, 1, 1},
			wantFPR:    []float64{0, 0, 0.6, 0.6, 1},
			wantThresh: []float64{8, 6, 4, 2, -1},
		},
		{ // 5
			y:          []float64{0, 3, 5, 6, 7.5, 8},
			c:          []bool{false, true, false, true, true, true},
			w:          []float64{4, 1, 6, 3, 2, 2},
			cutoffs:    []float64{-1, 1, 2, 3, 4, 5, 6, 7, 8},
			wantTPR:    []float64{0.25, 0.5, 0.875, 0.875, 0.875, 1, 1, 1, 1},
			wantFPR:    []float64{0, 0, 0, 0.6, 0.6, 0.6, 0.6, 0.6, 1},
			wantThresh: []float64{8, 7, 6, 5, 4, 3, 2, 1, -1},
		},
		{ // 6
			y:          []float64{0, 3, 6, 6, 6, 8},
			c:          []bool{false, true, false, true, true, true},
			wantTPR:    []float64{0, 0.25, 0.75, 1, 1},
			wantFPR:    []float64{0, 0, 0.5, 0.5, 1},
			wantThresh: []float64{math.Inf(1), 8, 6, 3, 0},
		},
		{ // 7
			y:          []float64{0, 3, 6, 6, 6, 8},
			c:          []bool{false, true, false, true, true, true},
			w:          []float64{4, 1, 6, 3, 2, 2},
			wantTPR:    []float64{0, 0.25, 0.875, 1, 1},
			wantFPR:    []float64{0, 0, 0.6, 0.6, 1},
			wantThresh: []float64{math.Inf(1), 8, 6, 3, 0},
		},
		{ // 8
			y:          []float64{0, 3, 6, 6, 6, 8},
			c:          []bool{false, true, false, true, true, true},
			cutoffs:    []float64{-1, 2, 4, 6, 8},
			wantTPR:    []float64{0.25, 0.75, 0.75, 1, 1},
			wantFPR:    []float64{0, 0.5, 0.5, 0.5, 1},
			wantThresh: []float64{8, 6, 4, 2, -1},
		},
		{ // 9
			y:          []float64{0, 3, 6, 6, 6, 8},
			c:          []bool{false, true, false, true, true, true},
			cutoffs:    []float64{-1, 1, 2, 3, 4, 5, 6, 7, 8},
			wantTPR:    []float64{0.25, 0.25, 0.75, 0.75, 0.75, 1, 1, 1, 1},
			wantFPR:    []float64{0, 0, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1},
			wantThresh: []float64{8, 7, 6, 5, 4, 3, 2, 1, -1},
		},
		{ // 10
			y:          []float64{0, 3, 6, 6, 6, 8},
			c:          []bool{false, true, false, true, true, true},
			w:          []float64{4, 1, 6, 3, 2, 2},
			cutoffs:    []float64{-1, 2, 4, 6, 8},
			wantTPR:    []float64{0.25, 0.875, 0.875, 1, 1},
			wantFPR:    []float64{0, 0.6, 0.6, 0.6, 1},
			wantThresh: []float64{8, 6, 4, 2, -1},
		},
		{ // 11
			y:          []float64{0, 3, 6, 6, 6, 8},
			c:          []bool{false, true, false, true, true, true},
			w:          []float64{4, 1, 6, 3, 2, 2},
			cutoffs:    []float64{-1, 1, 2, 3, 4, 5, 6, 7, 8},
			wantTPR:    []float64{0.25, 0.25, 0.875, 0.875, 0.875, 1, 1, 1, 1},
			wantFPR:    []float64{0, 0, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 1},
			wantThresh: []float64{8, 7, 6, 5, 4, 3, 2, 1, -1},
		},
		{ // 12
			y:          []float64{0.1, 0.35, 0.4, 0.8},
			c:          []bool{true, false, true, false},
			wantTPR:    []float64{0, 0, 0.5, 0.5, 1},
			wantFPR:    []float64{0, 0.5, 0.5, 1, 1},
			wantThresh: []float64{math.Inf(1), 0.8, 0.4, 0.35, 0.1},
		},
		{ // 13
			y:          []float64{0.1, 0.35, 0.4, 0.8},
			c:          []bool{false, false, true, true},
			wantTPR:    []float64{0, 0.5, 1, 1, 1},
			wantFPR:    []float64{0, 0, 0, 0.5, 1},
			wantThresh: []float64{math.Inf(1), 0.8, 0.4, 0.35, 0.1},
		},
		{ // 14
			y:          []float64{0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 10},
			c:          []bool{false, true, false, false, true, true, false},
			cutoffs:    []float64{-1, 2.5, 5, 7.5, 10},
			wantTPR:    []float64{0, 0, 0, 0, 1},
			wantFPR:    []float64{0.25, 0.25, 0.25, 0.25, 1},
			wantThresh: []float64{10, 7.5, 5, 2.5, -1},
		},
		{ // 15
			y:          []float64{1, 2},
			c:          []bool{false, false},
			wantTPR:    []float64{math.NaN(), math.NaN(), math.NaN()},
			wantFPR:    []float64{0, 0.5, 1},
			wantThresh: []float64{math.Inf(1), 2, 1},
		},
		{ // 16
			y:          []float64{1, 2},
			c:          []bool{false, false},
			cutoffs:    []float64{-1, 2},
			wantTPR:    []float64{math.NaN(), math.NaN()},
			wantFPR:    []float64{0.5, 1},
			wantThresh: []float64{2, -1},
		},
		{ // 17
			y:          []float64{1, 2},
			c:          []bool{false, false},
			cutoffs:    []float64{0, 1.2, 1.4, 1.6, 1.8, 2},
			wantTPR:    []float64{math.NaN(), math.NaN(), math.NaN(), math.NaN(), math.NaN(), math.NaN()},
			wantFPR:    []float64{0.5, 0.5, 0.5, 0.5, 0.5, 1},
			wantThresh: []float64{2, 1.8, 1.6, 1.4, 1.2, 0},
		},
		{ // 18
			y:          []float64{1},
			c:          []bool{false},
			wantTPR:    []float64{math.NaN(), math.NaN()},
			wantFPR:    []float64{0, 1},
			wantThresh: []float64{math.Inf(1), 1},
		},
		{ // 19
			y:          []float64{1},
			c:          []bool{false},
			cutoffs:    []float64{-1, 1},
			wantTPR:    []float64{math.NaN(), math.NaN()},
			wantFPR:    []float64{1, 1},
			wantThresh: []float64{1, -1},
		},
		{ // 20
			y:          []float64{1},
			c:          []bool{true},
			wantTPR:    []float64{0, 1},
			wantFPR:    []float64{math.NaN(), math.NaN()},
			wantThresh: []float64{math.Inf(1), 1},
		},
		{ // 21
			y:          []float64{},
			c:          []bool{},
			wantTPR:    nil,
			wantFPR:    nil,
			wantThresh: nil,
		},
		{ // 22
			y:          []float64{},
			c:          []bool{},
			cutoffs:    []float64{-1, 2.5, 5, 7.5, 10},
			wantTPR:    nil,
			wantFPR:    nil,
			wantThresh: nil,
		},
		{ // 23
			y:          []float64{0.1, 0.35, 0.4, 0.8},
			c:          []bool{true, false, true, false},
			cutoffs:    []float64{-1, 0.1, 0.35, 0.4, 0.8, 0.9, 1},
			wantTPR:    []float64{0, 0, 0, 0.5, 0.5, 1, 1},
			wantFPR:    []float64{0, 0, 0.5, 0.5, 1, 1, 1},
			wantThresh: []float64{1, 0.9, 0.8, 0.4, 0.35, 0.1, -1},
		},
		{ // 24
			y:          []float64{0.1, 0.35, 0.4, 0.8},
			c:          []bool{true, false, true, false},
			cutoffs:    []float64{math.Inf(-1), 0.1, 0.36, 0.8},
			wantTPR:    []float64{0, 0.5, 1, 1},
			wantFPR:    []float64{0.5, 0.5, 1, 1},
			wantThresh: []float64{0.8, 0.36, 0.1, math.Inf(-1)},
		},
		{ // 25
			y:          []float64{0, 3, 5, 6, 7.5, 8},
			c:          []bool{false, true, false, true, true, true},
			cutoffs:    make([]float64, 0, 10),
			wantTPR:    []float64{0, 0.25, 0.5, 0.75, 0.75, 1, 1},
			wantFPR:    []float64{0, 0, 0, 0, 0.5, 0.5, 1},
			wantThresh: []float64{math.Inf(1), 8, 7.5, 6, 5, 3, 0},
		},
		{ // 26
			y:          []float64{0.1, 0.35, 0.4, 0.8},
			c:          []bool{true, false, true, false},
			cutoffs:    []float64{-1, 0.1, 0.35, 0.4, 0.8, 0.9, 1, 1.1, 1.2},
			wantTPR:    []float64{0, 0, 0, 0, 0, 0.5, 0.5, 1, 1},
			wantFPR:    []float64{0, 0, 0, 0, 0.5, 0.5, 1, 1, 1},
			wantThresh: []float64{1.2, 1.1, 1, 0.9, 0.8, 0.4, 0.35, 0.1, -1},
		},
	}
	for i, test := range cases {
		gotTPR, gotFPR, gotThresh := ROC(test.cutoffs, test.y, test.c, test.w)
		if !floats.Same(gotTPR, test.wantTPR) && !floats.EqualApprox(gotTPR, test.wantTPR, tol) {
			t.Errorf("%d: unexpected TPR got:%v want:%v", i, gotTPR, test.wantTPR)
		}
		if !floats.Same(gotFPR, test.wantFPR) && !floats.EqualApprox(gotFPR, test.wantFPR, tol) {
			t.Errorf("%d: unexpected FPR got:%v want:%v", i, gotFPR, test.wantFPR)
		}
		if !floats.Same(gotThresh, test.wantThresh) {
			t.Errorf("%d: unexpected thresholds got:%#v want:%v", i, gotThresh, test.wantThresh)
		}
	}
}

func TestTOC(t *testing.T) {
	cases := []struct {
		c       []bool
		w       []float64
		wantMin []float64
		wantMax []float64
		wantTOC []float64
	}{
		{ // 0
			// This is the example given in the paper's supplement.
			// http://www2.clarku.edu/~rpontius/TOCexample2.xlsx
			// It is also shown in the WP article.
			// https://en.wikipedia.org/wiki/Total_operating_characteristic#/media/File:TOC_labeled.png
			c: []bool{
				false, false, false, false, false, false,
				false, false, false, false, false, false,
				false, false, true, true, true, true,
				true, true, true, false, false, true,
				false, true, false, false, true, false,
			},
			wantMin: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
			wantMax: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
			wantTOC: []float64{0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10},
		},
		{ // 1
			c:       []bool{},
			wantMin: nil,
			wantMax: nil,
			wantTOC: nil,
		},
		{ // 2
			c: []bool{
				true, true, true, true, true,
			},
			wantMin: []float64{0, 1, 2, 3, 4, 5},
			wantMax: []float64{0, 1, 2, 3, 4, 5},
			wantTOC: []float64{0, 1, 2, 3, 4, 5},
		},
		{ // 3
			c: []bool{
				false, false, false, false, false,
			},
			wantMin: []float64{0, 0, 0, 0, 0, 0},
			wantMax: []float64{0, 0, 0, 0, 0, 0},
			wantTOC: []float64{0, 0, 0, 0, 0, 0},
		},
		{ // 4
			c:       []bool{false, false, false, true, false, true},
			w:       []float64{2, 2, 3, 6, 1, 4},
			wantMin: []float64{0, 0, 0, 3, 6, 8, 10},
			wantMax: []float64{0, 4, 5, 10, 10, 10, 10},
			wantTOC: []float64{0, 4, 4, 10, 10, 10, 10},
		},
	}
	for i, test := range cases {
		gotMin, gotTOC, gotMax := TOC(test.c, test.w)
		if !floats.Same(gotMin, test.wantMin) {
			t.Errorf("%d: unexpected minimum bound got:%v want:%v", i, gotMin, test.wantMin)
		}
		if !floats.Same(gotMax, test.wantMax) {
			t.Errorf("%d: unexpected maximum bound got:%v want:%v", i, gotMax, test.wantMax)
		}
		if !floats.Same(gotTOC, test.wantTOC) {
			t.Errorf("%d: unexpected TOC got:%v want:%v", i, gotTOC, test.wantTOC)
		}
	}
}

func BenchmarkROC(b *testing.B) {
	sizes := []int{empty, small, medium, large}
	for _, cutoffsSize := range sizes {
		for _, ySize := range sizes {
			classesSize := ySize
			for _, weightsSize := range slices.Compact([]int{empty, ySize}) {
				benchmarkROC(b, cutoffsSize, ySize, classesSize, weightsSize)
			}
		}
	}
}

func benchmarkROC(b *testing.B, cutoffsSize int, ySize int, classesSize int, weightsSize int) bool {
	return b.Run(
		fmt.Sprintf(
			"cutoffs=%d,y=%d,classes=%d,weights=%d",
			cutoffsSize, ySize, classesSize, weightsSize),
		func(b *testing.B) {
			src := rand.NewSource(1)

			cutoffs := randomFloats(cutoffsSize, src)
			slices.Sort(cutoffs)

			y := randomFloats(ySize, src)
			slices.Sort(y)

			classes := randomBools(classesSize, src)

			var weights []float64
			if weightsSize != empty {
				weights = randomFloats(weightsSize, src)
			}

			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				ROC(cutoffs, y, classes, weights)
			}
		})
}

func randomFloats(l int, src rand.Source) []float64 {
	rnd := rand.New(src)
	s := make([]float64, l)
	for i := range s {
		s[i] = rnd.Float64()
	}
	return s
}

func randomBools(l int, src rand.Source) []bool {
	rnd := rand.New(src)
	s := make([]bool, l)
	for i := range s {
		s[i] = rnd.Int31n(2) == 1
	}
	return s
}