001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/**
031 * 
032 */
033package org.openimaj.experiment.evaluation.agreement;
034
035import gnu.trove.list.array.TDoubleArrayList;
036import gnu.trove.map.hash.TObjectDoubleHashMap;
037import gnu.trove.map.hash.TObjectIntHashMap;
038
039import java.util.HashMap;
040import java.util.List;
041import java.util.Map;
042
043/**
044 * Calculates Fleiss inter-rater agreement - a version of Cohen's kappa that
045 * works with multiple raters. Note that it is technially not a kappa as it does
046 * not reduce to Cohen's kappa when used with only 2 raters.
047 * 
048 * @see "http://en.wikipedia.org/wiki/Fleiss'_kappa"
049 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
050 * @created 16 Aug 2013
051 */
052public class FleissInterraterAgreement
053{
054        /**
055         * Calculate Fleiss interrater agreement
056         * 
057         * @see "http://en.wikipedia.org/wiki/Fleiss'_kappa"
058         * @param data
059         *            The rater's annotations
060         * @return The Fleiss Kappa
061         */
062        public static <K, A> double calculate(
063                        List<Map<K, A>> data)
064        {
065                // Map from subject to category count (the table)
066                final Map<K, TObjectIntHashMap<A>> table = new HashMap<K, TObjectIntHashMap<A>>();
067
068                // Total assignments to each category, Pj*Nn
069                final TObjectDoubleHashMap<A> cats = new TObjectDoubleHashMap<A>();
070
071                // Total ratings made
072                int totalCount = 0;
073
074                // Generate the counts table
075                // Loop through all the raters
076                for (final Map<K, A> ratings : data)
077                {
078                        // Loop through the subjects that this rater rated
079                        for (final K subject : ratings.keySet())
080                        {
081                                final A annotation = ratings.get(subject);
082                                if (annotation != null)
083                                {
084                                        TObjectIntHashMap<A> count = table.get(subject);
085                                        if (count == null)
086                                        {
087                                                count = new TObjectIntHashMap<A>();
088                                                table.put(subject, count);
089                                        }
090
091                                        count.putIfAbsent(annotation, 0);
092                                        count.increment(annotation);
093
094                                        cats.putIfAbsent(annotation, 0);
095                                        cats.increment(annotation);
096
097                                        totalCount++;
098                                }
099                        }
100                }
101
102                // Normalise Pj by N*n (totalCount) and square (part of working out
103                // PeBar)
104                final TDoubleArrayList Pj = new TDoubleArrayList();
105                for (final A x : cats.keySet())
106                        Pj.add((cats.get(x) / totalCount) *
107                                        (cats.get(x) / totalCount));
108
109                final int n = data.size();
110
111                final TDoubleArrayList Pis = new TDoubleArrayList();
112                for (final K subject : table.keySet())
113                {
114                        double Pi = 0;
115                        for (final A cat : table.get(subject).keySet())
116                        {
117                                final int nij = table.get(subject).get(cat);
118                                Pi += nij * (nij - 1);
119                        }
120                        Pi /= (n * (n - 1));
121                        Pis.add(Pi);
122                }
123
124                final double Pbar = Pis.sum() / Pis.size();
125                final double PeBar = Pj.sum();
126
127                final double kappa = (Pbar - PeBar) / (1 - PeBar);
128                return kappa;
129        }
130}