#
# failure-fshing.R, 27 Jun 26
# Data from:
# N-Version Programming with Coding Agents
# Javier Ron and Benoit Baudry and and Martin Monperrus

library("colorspace")
library("plyr")


par(bty="l")
par(las=1)
par(pch="+")
pal_col=rainbow(2)


fail_dist=function(df)
{
# Don't subtract 1 to prevent zeros appearing
return(data.frame(version_id=df$version_id[1], fdist=diff(df$test_id)))
}


one_exp=function(N)
{
p=rexp(N, 1)

return(p)
}


two_exp=function(N, alpha_p)
{
p=rexp(N*1/(1+alpha_p), 1)
q=rexp(N*alpha_p^2/(1+alpha_p), alpha_p)

return(c(p, q))
}


# An exponential QQ plot to check whether distances have an exponential distributions
QQ_plot=function(df)
{
sdist=sort(df$fdist)
n=nrow(df)

q_i=-mean(sdist)*log(1-((1:n)-0.5)/n)

plot(q_i, sdist, log="xy", col=pal_col[2],
	xlab="Predicted distance", ylab="Actual distance")

qmod=glm(log(sdist) ~ log(q_i))
print(summary(qmod))
pred=predict(qmod)

lines(q_i, exp(pred), col=pal_col[1])

}


# test_id,version_id,agent,run_id,language,passed,exception
# 73,opencode__m_qwen_qwen3.5-flash-02-23__l_python__run000,opencode,0,python,False,

fa=read.csv("failures.csv")

mod_f_dist=ddply(fa, .(version_id), fail_dist)


# Implementations with different numbers of failures
cc4_5r=subset(mod_f_dist, version_id == "claude_code__m_claude-haiku-4.5__l_rust__run000")
cc4_5p=subset(mod_f_dist, version_id == "claude_code__m_claude-sonnet-4.5__l_pascal__run000")
cur_2p=subset(mod_f_dist, version_id == "cursor__m_composer-2__l_python__run000")
oc3_5p=subset(mod_f_dist, version_id == "opencode__m_qwen_qwen3.5-flash-02-23__l_python__run000")

QQ_plot(cc4_5r)
# QQ_plot(cc4_5p)
# QQ_plot(cur_2p)
# QQ_plot(oc3_5p)

dev.copy(dev=png, file="cc4_5r_QQplot.png")
dev.off()


# Predict the QQ values from the distance
n=length(cc4_5r$fdist)
sdist=sort(cc4_5r$fdist)
qq_i=(1:n)/(n+1)

e2_mod=nls(qq_i ~ 1-a*exp(l1*sdist)-(1-a)*exp(l2*sdist), trace=TRUE,
			start=list(a=0.1, l1=-0.0004192725, l2=-1e-6))
summary(e2_mod)

# Two exponentials

N=1e3
plot(sort(one_exp(N)), log="y", col=pal_col[2])
lines(sort(two_exp(N, 0.5)), col=pal_col[1])
lines(sort(two_exp(N, 0.75)), col=pal_col[1])
lines(sort(two_exp(N, 0.9)), col=pal_col[1])
lines(sort(two_exp(N, 1.1)), col=pal_col[1])

