#
# agreement-fishing.R, 12 Jul 26
#
# Data from:
# Determining the Distribution of Maintenance Categories: Survey versus Measurement
# Stephen R. Schach and Bo Jin and Liguo Yu and Gillian Z. Heller and Jeff Offutt
# and
# Two datasets of defect reports labeled by a crowd of annotators of unknown reliability
# Jerónimo Hernández-González and Daniel Rodriguez and Iñaki Inza and Rachel Harrison and Jose A. Lozano


library(colorspace)
library(plyr)
library(irr)
library(irrCAC)


par(bty=l)
par(las=1)
par(pch=+)
pal_col=rainbow(2)

# Convert a contingency table into a list of values
contin_to_list=function(contin)
{
t_df=as.data.frame(as.table(contin))
# Yukky way of replicating a dataframe
t=adply(t_df, 1, function(df) df[rep(1, times=df$Freq[1]), ])
t$Freq=NULL
return(t)
}


# Maintenance categories
cross_rater=read.csv(maint-agreement.csv.xz)
rownames(cross_rater)=colnames(cross_rater)

mmc2=read.csv(mmc2.csv)
mmc2$summary=NULL
mmc2$description=NULL

mmc5=read.csv(mmc5.csv)
mmc5$summary=NULL
mmc5$description=NULL


# Cohen's kappa
kappa2(contin_to_list(data.matrix(cross_rater)))


# Conger's kappa
conger.kappa.raw(mmc2)
conger.kappa.raw(mmc5)

