#
# mergewords.awk, 23 Mar 10

BEGIN {
	stop_list["and"]=1
	stop_list["any"]=1
	stop_list["are"]=1
	stop_list["at"]=1
	stop_list["be"]=1
	stop_list["else"]=1
	stop_list["in"]=1
	stop_list["is"]=1
	stop_list["it"]=1
	stop_list["of"]=1
	stop_list["that"]=1
	stop_list["the"]=1
	stop_list["than"]=1
	stop_list["then"]=1
	stop_list["to"]=1
	chevron[1]=">"
	chevron[2]=">>"
	chevron[3]=">>>"
	chevron[4]=">>>>"
	chevron[5]=">>>>>"
	chevron[6]=">>>>>>"
	}

$1 == "w" {
	wrd=tolower($2)
	if (stop_list[wrd] == 1)
	   next
	if (word_list[wrd] == 0)
	   {
	   msg_word[cur_msg " | " wrd]++
	   }
	if (msg_words[wrd] == 1)
	   word_in_msg[wrd]++
	word_list[wrd]=1
	next
	}

	{
	word_list[1]=1
	delete word_list
	msg_words[1]=1
	delete msg_words
	for (m=1; m <= NF; m++)
	   msg_words[tolower($m)]=1
	cur_msg=$0
	next
	}

END {
	for (mw in msg_word)
	   if (msg_word[mw] > 1)
	      print mw " " msg_word[mw]
	for (mw in word_in_msg)
	   print mw " " word_in_msg[mw]
	}

