#!/usr/bin/perl
use warnings;
use strict;
use LM::NGramLangModel;
use utf8;

=head1 task2.pl

Skript pro reseni ukolu cislo 2.

=head2 Pouziti

./task2.pl file

=cut

my $file = $ARGV[0];

# kontrola vstupnich parametru
if ( ! defined ($file) ) { 
	print "Usage: task2.pl inputFile\n";
	exit (3);
}

if ( ! -f $file ) { 
	print "Input file $file doesn't exist.\n";
	exit (3);
};


# priprava nazvu jednotlivych vstupnich souboru
my $filePrepared = $file . ".prepared";
my $fileTrain = $file . ".train";
my $fileHeldOut = $file . ".heldout";
my $fileTest = $file . ".test";

# pokud neexistuji, tak je vytvorim
print STDERR "Prepare input data $file\n";
if ( ! -f $filePrepared ) { 
	exec ("./s_task2_prepare.sh $file");
} else { 
	print STDERR "\tAlready done\n";
}
print "\n";

# vytvoreni jazykoveho modelu
my $langModel = LM::NGramLangModel->new(3, 0, $fileTrain);

# spocitani pravdepodobnosti
$langModel->computeCounts();
$langModel->computeProbs();


# nazev souboru s lambdama
my $fileLambdas = "res.task2.".$file.".lambdas";

# pokud neexistuji lambdy, tak je spocitej
if ( ! -f $fileLambdas ) {
	# EM-smoothing
	$langModel->emSmooth($fileHeldOut);

	# ulozeni lambd
	$langModel->storeLambdas($fileLambdas);
} else { 
	$langModel->loadLambdas($fileLambdas);
}

# 1. cast ukolu 
# ziskani lambd
my @lambdasHeldout = $langModel->getLambdas();
print "λ0\tλ1\tλ2\tλ3\n";
print join("\t", map { formatF($_); } @lambdasHeldout), "\n";


# 2. cast ukolu
# nazev souboru s lambdama
print "\n=======================\n";

my $fileLambdasTrain = "res.task2.".$file.".lambdas.train";
$langModel->setLambdas([0.25, 0.25, 0.25, 0.25]);
# pokud neexistuji lambdy, tak je spocitej
if ( ! -f $fileLambdasTrain ) {
	$langModel->{smoothed} = 0;
	# EM-smoothing
	$langModel->emSmooth($fileTrain);

	# ulozeni lambd
	$langModel->storeLambdas($fileLambdasTrain);
} else { 
	$langModel->loadLambdas($fileLambdasTrain);
}

my @lambdasTrain = $langModel->getLambdas();
print "λ0\tλ1\tλ2\tλ3\n";
print join("\t", map { formatF($_); } @lambdasTrain), "\n";

# nastav puvodni lambdy
$langModel->setLambdas(\@lambdasHeldout);



my $sum = $lambdasHeldout[0] + $lambdasHeldout[1] + $lambdasHeldout[2];
my @lambdas = ();


# 3. cast ukolu
# zmena rozlozeni jednotlivych lambd - boost
print "\n=======================\n";
print "Boost\tλ0\tλ1\tλ2\tλ3\tCross entr.\tPerplexity\n";

# hodnoty ze zadani
my @boost = (0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99);
for my $b (@boost) { 

	# spocitej lambdy
	my $bC = (1 - $lambdasHeldout[3]) * $b;
	my $s = 0;
	for my $i (0 .. 2) { 
		$lambdas[$i] = $lambdasHeldout[$i] - $bC * $lambdasHeldout[$i] / $sum;
		$s += $lambdas[$i];
	}
	$lambdas[3] = $lambdasHeldout[3] + $bC;
	$s += $lambdas[3];

	# spocitej cross entropii
	$langModel->setLambdas(\@lambdas);
	my $crossEntropy = $langModel->getCrossEntropy($fileTest);

	print $b, "\t", join("\t", map { formatF($_); } @lambdas), "\t", formatF($crossEntropy), "\t", formatF(2 ** $crossEntropy),"\n";
}

# 4.cast ukolu - pokracovani
# zmena rozlozeni lambd - discount
print "\n=======================\n";
print "Discount\tλ0\tλ1\tλ2\tλ3\tCross entr.\tPerplexity\n";

my @discount = (1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0);
for my $d (@discount) { 

	# spocitej lambdy
	$lambdas[3] = $lambdasHeldout[3] * $d;
	my $dC = $lambdasHeldout[3] - $lambdas[3];
	my $s = 0;
	for my $i (0 .. 2) { 
		$lambdas[$i] = $lambdasHeldout[$i] + $dC * $lambdasHeldout[$i] / $sum;
		$s += $lambdas[$i];
	}
	$s += $lambdas[3];

	# spocitej cross entropii
	$langModel->setLambdas(\@lambdas);
	my $crossEntropy = $langModel->getCrossEntropy($fileTest);

	print $d, "\t", join("\t", map { formatF($_); } @lambdas), "\t", formatF($crossEntropy), "\t", formatF(2 ** $crossEntropy),"\n";
}


# 5. cast ukolu 
# doplnkovy ukol - spocitani pokryti
print "\n=======================\n";
print "Coverage\nPos\tUn. Unigram\tUn. Bigram\tUn. Trigram\tUnigram\tBigram\tTrigram\n";

my @coverage = $langModel->getCoverage($fileTest);
for my $line (@coverage) { 
	my @cols = @{$line};
	my $pos = shift @cols;
	print $pos, "\t", join("\t", map { formatF(100 * $_); } @cols), "\n";
}

print "\n=======================\n";
print "Data\tUnigram\tBigram\tTrigram\n";
print "train", "\t", $langModel->{uniqueNgrams}{1}, "\t", $langModel->{uniqueNgrams}{2}, "\t", $langModel->{uniqueNgrams}{3}, "\n";

my $heldoutModel = LM::NGramLangModel->new(3, 0, $fileHeldOut);
$heldoutModel->computeCounts();
print "heldout", "\t", $heldoutModel->{uniqueNgrams}{1}, "\t", $heldoutModel->{uniqueNgrams}{2}, "\t", $heldoutModel->{uniqueNgrams}{3}, "\n";

my $testModel = LM::NGramLangModel->new(3, 0, $fileTest);
$testModel->computeCounts();
print "test", "\t", $testModel->{uniqueNgrams}{1}, "\t", $testModel->{uniqueNgrams}{2}, "\t", $testModel->{uniqueNgrams}{3}, "\n";

# nesmysl - nejde scitat n-gramy z jednotlivych souboru
# print "all", "\t", $langModel->{uniqueNgrams}{1} + $heldoutModel->{uniqueNgrams}{1} + $testModel->{uniqueNgrams}{1}, 
# 			 "\t", $langModel->{uniqueNgrams}{2} + $heldoutModel->{uniqueNgrams}{2} + $testModel->{uniqueNgrams}{2},
# 			 "\t", $langModel->{uniqueNgrams}{3} + $heldoutModel->{uniqueNgrams}{3} + $testModel->{uniqueNgrams}{3}, "\n";

sub formatF
{
	my $f = shift;
	return sprintf("%2.3f", $f);
}


