001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/**
031 *
032 */
033package org.openimaj.experiment.evaluation.agreement;
034
035import java.util.Map;
036
037/**
038 * Calculates the percentage interrater agreement for a given dataset between
039 * two (and only two) raters.
040 * 
041 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
042 * @created 12 Aug 2013
043 */
044public class PercentageInterraterAgreement
045{
046        /**
047         * The input should be a {@link Map} for each rater where the keys represent
048         * all the subjects that were rated by the raters and the values represent
049         * the annotations given by the raters. Agreement between the raters is
050         * determined by {@link #equals(Object)} for the INSTANCE type. Annotations
051         * for subjects which are not in both sets are ignored.
052         * 
053         * @param rater1
054         *            The annotations from rater 1
055         * @param rater2
056         *            The annotations from rater 2
057         * @return The percentage agreement
058         */
059        public static <K, A> double calculate(
060                        final Map<K, A> rater1,
061                        final Map<K, A> rater2)
062        {
063                int totalCount = 0;
064                int agreementCount = 0;
065
066                for (final K subjectKey : rater1.keySet())
067                {
068                        // We can only form an agreement if both raters rated this
069                        // specific subject, so let's check
070                        if (rater2.keySet().contains(subjectKey))
071                        {
072                                final A r1a = rater1.get(subjectKey);
073                                final A r2a = rater2.get(subjectKey);
074
075                                // It's possible that the key exists but is mapped to
076                                // a null value (for example, if majority voting was used
077                                // to generate the set and there was no majority).
078                                if (r1a == null || r2a == null)
079                                        continue;
080
081                                // Get the answers from the raters
082                                final A annotation1 = r1a;
083                                final A annotation2 = r2a;
084
085                                // Count the agreements
086                                if (annotation1.equals(annotation2))
087                                        agreementCount++;
088
089                                // Keep a running total
090                                totalCount++;
091                        }
092                }
093
094                final double PrA = agreementCount / (double) totalCount;
095                return PrA;
096        }
097}