#!/usr/bin/perl
# parseKos.perl - Utility for parsing DailyKos archive pages
# Copyright (C) 2006, 2007, 2008 Jon F. Garfunkel <jon at civilities.net>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    http://www.gnu.org/copyleft/gpl.html



###########################################
# Topic - for indexing topic information
#
package Topic;
  sub new {
    my($class) = shift;
    my ($desc) = shift;
    bless {
	"description"=> $desc,
	"KosPosts2002"   => 0,
	"KosPosts2003"   => 0,
	"OtherPosts"     => 0,
	"TotalPosts"     => 0,
	"Comments2002"   => 0,
	"Comments2003"   => 0,
	"Comments"   => 0,
	"WordCount2002"  => 0,
	"WordCountKos2003"  => 0,
	"WordCount2003"  => 0,
	"ZeroLinked"  => 0,
	"Onelinked"  => 0,
	"Manylinked"  => 0,
	"NumLinks"  => 0
   }, $class;
}



#############################################
package main;

use CSV;
use Switch;

##############################################
# Basic stuff

# Who's running the program?
$MyName = "Jon Garfunkel";

# Who's our daddy?
$urlbase = "http://www.dailykos.net/archives";

# Where's our stash?
$sourcepath = "pages";


###############################################
# These are the 16 topics we'll be looking for...

@topicnames = 
("iraq","bushies","primary","congress","states","calif","dems","economy","domestic",
		"gwot","world","media","misc","democracy","site","open");

%topics = ();

$topics{"bushies"}    = Topic->new("Bush administration");
$topics{"iraq"}       = Topic->new("Iraq war");
$topics{"gwot"}       = Topic->new("Afghanistan, 9/11, war on terror");
$topics{"world"}      = Topic->new("World affairs (exluding wars)");
$topics{"domestic"}   = Topic->new("Domestic Affairs");
$topics{"economy"}    = Topic->new("Economy");
$topics{"dems"}       = Topic->new("Democratic Party");
$topics{"democracy"}  = Topic->new("Democracy: Voting, Free Speech, etc.");
$topics{"congress"}   = Topic->new("Congressional politics");
$topics{"states"}     = Topic->new("State/local politics");
$topics{"calif"}      = Topic->new("California politics");
$topics{"primary"}    = Topic->new("2004 Democratic Presidential Primary");
$topics{"media"}      = Topic->new("Media");
$topics{"misc"}       = Topic->new("Baseball, etc.");
$topics{"site"}       = Topic->new("Site issues and announcements");
$topics{"open"}       = Topic->new("Open Thread");


###############################################
# Initialize some files

# Let's write out an Atom/RSS feed

$rdf = "DailyKos.atom";
open (RDF,"> $rdf");
print RDF "<?xml version='1.0' encoding='utf-8'?><feed xmlns='http://www.w3.org/2005/Atom'>\n";
print RDF "<title>DailyKos archive May 26, 2002 - October 14, 2003</title>\n";


# Also, a CSV is nice to take home

$csvfile = "DailyKos.csv";
open (CSV,"> $csvfile");
CSVinit();
$csvline = CSVjoin("id", "Author", "Date", "Category", "Title", "Wordcount", "Comments");
print CSV "$csvline\n";;

# And an index ByMonth

$monthfile = "ByMonth.html";
open (BYMONTH, "> $monthfile");
select(BYMONTH);
print_header("By Month");

    print "<tr><td></td><th colspan='16'>Posts in each topic</th><td></td>";
    print "<th colspan='3'>posts with # hyperlinks</th></tr>";
    print "<tr><td>Month</td>";
    foreach (@topicnames) {
      $t = $topics{$_};
      $desc = $t->{'description'};
      print "<td><a href='kos_$_.html' title='$desc'>$_</a></td>";
    }
    print "<td>comments</td>";
    print "<td title='Posts with zero hyperlinks'>zero</td>";
    print "<td title='Posts with one hyperlink'>one</td>";
    print "<td title='Posts with multiple hyperlinks'>many</td>";
    print "<td>total posts</td>";
    print "</tr>\n";

####### Also, one page for each topic

foreach (@topicnames) {
  $f = "kos_" . $_ . ".html";
  open (TOPIC, ">$f");
  select TOPIC;
  print_header($topics{$_}->{'description'});
  print "<tr><th>Date</th><th>Title</th><th>Author</th><th>wordcount</th>";
  print "<th>comments</th><th>Links</th><th span='Internal Links'>In. Links</th></tr>";

  close TOPIC;
}


####################################
# Initialize some vars for monthly summary

$lastMonth = "";
$lastMonthLabel = "";
$MonthRowCount = 0;
$lastDay = "";

%TopicsMonth = ();
$MonthComments = 0;
%TopicsCount = ();
%Authors = ();

$NumLinks = 0;
$ZeroLinked = 0;
$OneLinked = 0;
$ManyLinked = 0;

open (MONTH, ">/dev/null");

######################################
#
# Let's loop on through
#
#

$start = 1;
$end = 4571;
print STDERR "parsing 4571 articles...\n";
for ($i = $start; $i< $end; $i += 1) {
  print STDERR "." if ($i % 100 == 0);
  $id = sprintf("%06d",$i);
  $file = sprintf("%06d.html", $i);
  $url = "$urlbase/$file";

  # open file  
  $file = "$sourcepath/$file";
  # print STDERR "opening $file";

  next if (!-e $file);  
  open (FILE, "< $file");

  # initialize variables
  $numposts = 0;
  $date = "";
  $author = "kos";
  $subject = "";
  $comments = 0;

  $wordcount = 0;
  @urls = ();
  @links = ();
  $internalLinks = 0;
  $flag = "";

  $line=0;  

  # the very first post is about Kos. The rest we have to figure out
  $subject = "site" if $i == 1;

  $state = 0;

  while (<FILE>) {

    switch ($state) {

      #####################
      case 0 { 
        # first we find a title. We can use <TITLE> but let's take the one from the body
        if (m|<span class="title">(.+)$|) { 
	  $title = $1;
          
          # clean it up
          $cutoff = rindex($title, "<");
          $title = substr($title, 0, $cutoff);

          # We'll guess the subject from the title
	  $subject = guess_subject($title);
          $state = 1;
          $line = 0;	
        }
        # break;
        
      }

      #######################3
      case 1 {
        # We're now in the body of the post

        # There must be an author here somewhere. Bueller? Anybody?
        $a = guess_author($_);
        $author = $a if ($a ne "");

        # Let's look at the paragraphs and the occasional italicized line
        if ((m/^<p>/i) || (m/^<i>/)) {
          if ($subject eq "") {
            # We're going to keep quessing
            $subject = guess_subject($_);

            # and as we do, we can give a hint to the output as to where we found the keyword
            $line++;
            $flag = "*" . $line if $subject ne "";
          }
        }

        # Take another guess at the date. Some of the content includes rdf:dc
        # Most don't. 

        $d = guess_date($_);
        if ($d ne "") {
	  $date = $d;
          $date =~ s/\s+$//;
          
          # and then the comments are here 
          if (m|Comments</a> \((\d+)\)|) {
            $comments = $1;
            # $internalLinks--; # because this is a link
            $state = 3;
            next;
          }
          # otherwise, maybe comments are in the next line
          $state = 2;
        }

        # find some hyperlinks
        if (m|href="([^"]+)">([^<]+)</a>|) {
          push (@links, "<a class='extlink' href='$1'>$2</a>");
        }

        @newlinks = m|href="http://([^/"]+)[/"]|gi;
        foreach $nl (@newlinks) {
          push(@urls,$nl);
          $internalLinks++ if ($nl =~ /dailykos/);
          $urlcount[$nl]++;
        }

        # take a rough count of the words
        @words = split(/\s/, $_);
        $count = @words;
        $wordcount += $count if ($date eq ""); 

      }
      ###########################
      case 2 {
          if (m|Comments</a> \((\d+)\)|) {
            $comments = $1;
            $state = 3;
          }
      }
      # default: break;
    }
  }
  # we done with this file
  close FILE;

  $wordcount -= 2 if ($wordcount > 1);
  next if $title eq "Test";

  #################################
  # Now interpret the data


  # Find a date, loser
  ($Month, $Day, $Year, $Time, $M) = split(/ /,$date);
  chop($Day);
  $YM = $Year . "_" . $Month;
  $MonthFile = "kos_$YM.html";
  $Mon = substr($Month, 0, 3);

  # One last adjustment to the subject
  if ($subject eq "election") {
    $subject = ($Year eq "2003") ? "primary" : "congress";
  }

  $title = "untitled" if $title eq "";
  $title =~ s/"/'/g;

  if (($Year == 2002) && ($title =~ /Gephardt/)) {
    $subject = "iraq";  # When Gephardt sold the Dems out
  }
  $subject = "misc" if $subject eq "";

  ##########################
  # Adjustments

  # make some adjustments to the author
  $author = "kos" if $i==4101; 			# correcting for mis-identification
  $author = "Steve Gilliard" if $i==2240;
  $author = "Steve Soto" if $i==1021;
  $author = "Steve Soto" if $i==1078;
  $author = "Billmon" if $i==801;
  $author = "Billmon" if $i==952;
  $author = "Billmon" if $i==1054;
  $author = "Billmon" if $i==1808;
  $author = "Billmon" if $i==1845;
  $author = "kos" if $i==1915;
  $author = "RonK" if $i==3922;
  $author = "RonK" if $i==4106;
  $author = "RonK" if $i==4135;
  $author = "RonK" if $i==4145;

  # several of the "Open Thread" posts were substantive; most site issues
  $subject = "congress" if $i==655;
  $subject = "iraq" if $i==1460;
  $subject = "iraq" if $i==2240;
  $subject = "iraq" if $i==4219;
  $subject = "site" if $i==481;  # reveals name to be Markos
  $subject = "site" if $i==596;
  $subject = "site" if $i==1168;
  $subject = "site" if $i==2022;
  $subject = "site" if $i==2308;
  $subject = "site" if $i==2330;
  $subject = "site" if $i==3148;
  $subject = "site" if $i==3331;
  $subject = "site" if $i==3407;
  $subject = "site" if $i==3677;
  $subject = "site" if $i==3799;
  $subject = "site" if $i==3910;

  $subject = "misc" if $i==4232;
  $subject = "misc" if $i==4331;

  $subject = "bushies" if $i==25;
  $subject = "media" if $i==634;

  # "polls" are often wrong...
  $subject = "primary" if $i==2231;
  $subject = "primary" if $i==3822;
  $subject = "primary" if $i==4153;
  $subject = "states" if $i==4334;


  $subject = "congress" if $i==172;
  $subject = "congress" if $i==343;
  $subject = "states" if $i==452;
  $subject = "states" if $i==456;
  $subject = "states" if $i==484;

  ###################################
  # Increment our counters

  $TopicsCount{$subject}++;
  $TopicsMonth{$subject}++;
  $Authors{$author}++;
  $MonthComments += $comments;
  
  $PostsThisMonth++;

  $t = $topics{$subject};
  if ($Year eq "2002") {
    $t->{'KosPosts2002'}++;
    $t->{'Comments2002'} += $comments;
    $t->{'WordCount2002'} += $wordcount;

  } else {
    if ($author eq "kos") {
      $t->{'KosPosts2003'}++;    
      $t->{'WordCountKos2003'} += $wordcount;
    } else {
      $t->{'OtherPosts'}++;
      $t->{'WordCount2003'} += $wordcount;
    }
    $t->{'Comments2003'} += $comments;
  }
  $numlinks = @links;
  $t->{'NumLinks'} += $numlinks;

  if ($numlinks == 0) {
    $t->{'ZeroLinked'}++;
    $ZeroLinked++;
  } elsif ($numlinks == 1) {
    $t->{'OneLinked'}++;
    $OneLinked++;
  } else {
    $t->{'ManyLinked'}++;
    $ManyLinked++;
  }

  # for debugging - $flag indicates with a # if the keyword was found in the paragraphs
  # print STDERR "$id $date s:$subject$flag  $title  -by $author w:$wordcount\tc:$comments\n";

  # write out CSV and RDF out...
  $csvline = CSVjoin($id, $author, $date, $subject, $title, $wordcount, $comments);
  print CSV "$csvline\n";;
  print RDF "<entry>\n\t<link rel='alternate' type='text/html' href='$url'/>\n";
  print RDF "\t<title>$title</title>\n\t<author><name>$author</name></author>";
  print RDF "\n\t<category>$subject</category>\n\t<pubDate>$date</pubDate>\n";
  print RDF "</entry>\n";

  # now write some neat HTML files. Ideally, we ought to process the RDF instead.
  # We'll do one file per month, for starters.
  # As well as one per topic

  $title = substr($title,0,84);
  open (TOPIC, ">> kos_$subject.html");

  $TopicRowClass = ($TopicsCount{$subject} % 2 == 1) ? "oddrow" : "";

  $allLinks = @urls;

  select(TOPIC);
  print "<tr class='$TopicRowClass'><td nowrap>";
  print "<a href='$MonthFile#day$Day'>$Year $Mon $Day</a> $Time $M ";
  print "</td><td><b><a href='$url'>$title</a></b></td>\n";
  print "    <td nowrap><i>$author</i></td><td align='right'>$wordcount</td>";
  print "<td align='right'>$comments</td>";
  print "<td align='right'>$allLinks</td><td align='right'>$internalLinks</td>";
  print "</tr>\n";
  close TOPIC;


# we could print out the links, too. Another time.
#  print TOPIC "<div id='details-$id' style='display:none'>";
  foreach (@links) {
    m|<a href='http://([^/]+)|;
    $site = $1;
    #    print TOPIC "<li>$site: $_</li>\n";
  }
#  print TOPIC "</div><br>\n";
  
  ################################
  # if we're a new month

  if ($YM ne $lastMonth) {
    select(MONTH);
    print "</table>";
    if ($TopicsMonth{'open'} > 0) {
      print "<p><i>Note: <A href='kos_open.html'>Open Thread</a> posts not included above.</i></p>";
    }
    print "<p><b>Summary</b><table><tr>";


    foreach $t (@topicnames) {
      $c = $TopicsMonth{$t};
      print "<td>$t:$c</td>" if ($c > 0);
    }

    print "</tr></table>\n";
    print "<p><a href='$MonthFile'>$Month $Year</a></p>";
    print_footer();
    close MONTH;

    if ($i > 1) {   # write a row
      select (BYMONTH);
      $MonthRowCount++;

      $rowclass = ($MonthRowCount % 2 == 1) ? "oddrow" : "";

      print "<tr class='$rowclass'><td><a href='kos_$lastMonth.html'>$lastMonthLabel</a></td>";
      foreach $t (@topicnames) {
        $c = $TopicsMonth{$t};
        print "<td align='right'>$c</td>";
      }
      print "<td align='right'>$MonthComments</td>";
      print "<td align='right'>$ZeroLinked</td>";
      print "<td align='right'>$OneLinked</td>";
      print "<td align='right'>$ManyLinked</td>";
      print "<td align='right'>$PostsThisMonth</td>";
      print "</tr>\n";

      %TopicsMonth = ();
      $PostsThisMonth = 0;
      $MonthComments = 0;
      $ZeroLinked = 0;
      $OneLinked = 0;
      $ManyLinked = 0;
    }

    open (MONTH, "> $MonthFile");
    $monthcounter = 0;
    select(MONTH);

    $lastMonthLabel = "$Month $Year";
    print_header($lastMonthLabel);
    print "<tr><th>Date</th><th>topic</th><th>Title</th><th>Author</th>";
    print "<th>wordcount</th><th>comments</th>";
    print "<th>Links</th><th span='Internal Links'>In. Links</th></tr>";
    $lastMonth = $YM;
  }

  next if ($subject eq "open");

  select(MONTH);

  $rowclass = ($monthcounter % 2 == 1) ? "oddrow" : "";

  print "<tr class='$rowclass'><td nowrap>$Year $Mon $Day $Time $M ";

  if ($Day ne $lastDay) {
    print "<a name=\"day$Day\"> </a>\n";
    $lastDay = $Day;
  }

  print "</td><td><a href='kos_$subject.html'>$subject</a> </td>";
  print "<td><b><a href='$url'>$title</a></b> </td>\n";
  print "<td><i>$author</i> </td><td align='right'>$wordcount</td>";
  print "<td align='right'>$comments</td>";
  print "<td align='right'>$allLinks</td><td align='right'>$internalLinks</td>";
  print "</tr>\n";
  $monthcounter++;

}
print STDERR "done\n";

print RDF "</channel></rss>";

select(MONTH);
print "</table><p>Summary</p><table><tr>";
foreach $t (@topicnames) {
    $c = $TopicsMonth{$t};
    print "<td>$t:$c</td>" if ($c > 0);
}
print "</tr></table>\n";
print_footer();
close MONTH;
close CSV;
close RDF;

#############################################
# Print the ByTopic Summary

open (SUMMARY , ">ByTopic.html");
select(SUMMARY);
print_header("By Topic");

print "<tr><td></td><th colspan='4'>Posts by author/year</th>";
print "<th colspan='2'># Comments</th>";
print "<th colspan='2'>Comments/Post</th>\n";
print "<th colspan='3'>Wordcount</th>\n";
print "<th colspan='5'># Links</th></tr>\n";


print "<tr><th align='left'>Topic</td>";
print "<th>Total</th><th>Kos 2002</th><th>Kos 2003</th><th>Others</th>";
print "<th> 2002</th><th> 2003</th>";
print "<th> 2002</th><th> 2003</th>";
print "<th>Kos 2002</th><th> Kos 2003</th><th>Others</th>";
print "<th>per post </th>";
print "<th>per 1K words</th>";
print "<th>zero</th>";
print "<th>one</th>";
print "<th>many</th>";
print "</tr>\n";

$total = Topic->new("Total");
#$totalxot = Topic->new("Total (except Open Thread)");
$sumline = 0;

foreach (@topicnames) {
  $f = "kos_" . $_ . ".html";
  # print STDERR "Writing $f\n";
  open (TOPIC, ">>$f");
  select(TOPIC);
  print "</table>\n";
  print_footer();
  close TOPIC;

  select(SUMMARY);
  $sumline++;

  # print some subtotals
  print_summary_line($total, "Subtotal") if ($_ eq "open" || $_ eq "site"); 

  print_summary_line($topics{$_});
  
}

$comment_note = "<p>Note: comments were down or turned off between July 1 2003 and August 16 2003</i></p>";
# close out summary
print_summary_line($total, "Total");
print  "</table>";
print $comment_note;
print_footer();

close SUMMARY;


select (BYMONTH);
print "<tr><td><a href='kos_$lastMonth.html'>$lastMonthLabel</a></td>";
foreach $t (@topicnames) {
      $c = $TopicsMonth{$t};
      print "<td align='right'>$c</td>";
}
print "<td align='right'>$MonthComments</td>";
print "<td align='right'>$ZeroLinked</td>";
print "<td align='right'>$OneLinked</td>";
print "<td align='right'>$ManyLinked</td>";
print "<td align='right'>$PostsThisMonth</td>";
print "</tr>";
print "<tr class='total'><td>Total</td>";
foreach $t (@topicnames) {
  print "<td align='right'>$topics{$t}->{'TotalPosts'}</td>";
}
print "<td align='right'>$total->{'Comments'}</td>";
print "<td align='right'>$total->{'ZeroLinked'}</td>";
print "<td align='right'>$total->{'OneLinked'}</td>";
print "<td align='right'>$total->{'ManyLinked'}</td>";
print "<td align='right'>$total->{'TotalPosts'}</td>";

print "</tr>";
# now print total


print "</table>\n";

print $comment_note;
print_footer();
close (BYMONTH);



#
# end of main code
################################################################################

###################################
# print a line for the summary table

sub print_summary_line($t, $altname) {

  $t = shift;
  $altname = shift;

  $posts2002 = $t->{'KosPosts2002'};
  if ($posts2002 > 0) {
    $p = $t->{'Comments2002'} / $posts2002;
    $pop2002 = sprintf("%.0f",$p);  

    $wc = $t->{'WordCount2002'} / $posts2002;
    $wc2002 = sprintf("%.0f",$wc);  
  }

  $posts2003 = $t->{'KosPosts2003'} + $t->{'OtherPosts'};
  if ($posts2003 > 0) {
    $p = $t->{'Comments2003'} / $posts2003;
    $pop2003 = sprintf("%.0f",$p);
  }
  
  $t->{'Comments'} = $t->{'Comments2002'} + $t->{'Comments2003'};

  if ($t->{'OtherPosts'} > 0) {
    $wc = $t->{'WordCount2003'} / $t->{'OtherPosts'};
    $wc2003 = sprintf("%.0f",$wc);  
  }

  if ($t->{'KosPosts2003'} > 0) {
    $wc = $t->{'WordCountKos2003'} / $t->{'KosPosts2003'};
    $wck2003 = sprintf("%.0f",$wc);  
  }

  $totalwords = $t->{'WordCountKos2003'} + $t->{'WordCount2003'} + $t->{'WordCount2002'};
  $totalposts = $posts2002 + $posts2003;
  $t->{'TotalPosts'} = $totalposts;

  $lpf = $t->{'NumLinks'} / $totalposts;
  $lp = sprintf("%.1f",$lpf);

  $lkf = 1000 * $t->{'NumLinks'} / $totalwords;
  $lk = sprintf("%.1f",$lkf);

  $RowClass = ($sumline % 2 == 1) ? "oddrow" : "";

  if ($altname eq "") {
    print "<tr class='$RowClass'><td><a href='kos_$_.html'>$t->{'description'}</a></td>";
  } else {
    print "<tr class='Total'><td>$altname</td>";
  }

  print "<td align='right'>$totalposts</td>";
  print "<td align='right'>$posts2002</td>";
  print "<td align='right'>$t->{'KosPosts2003'}</td>";
  print "<td align='right'>$t->{'OtherPosts'}</td>";
  print "<td align='right'>$t->{'Comments2002'}</td>\n";
  print "<td align='right'>$t->{'Comments2003'}</td>\n";
  print "<td align='right'>$pop2002</td>\n";
  print "<td align='right'>$pop2003</td>\n";
  print "<td align='right'>$wc2002</td>\n";
  print "<td align='right'>$wck2003</td>\n";
  print "<td align='right'>$wc2003</td>\n";
  print "<td align='right'>$lp</td>\n";
  print "<td align='right'>$lk</td>\n";
  print "<td align='right'>$t->{'ZeroLinked'}</td>\n";
  print "<td align='right'>$t->{'OneLinked'}</td>\n";
  print "<td align='right'>$t->{'ManyLinked'}</td>\n";
  print "</tr>";


  ## And if we are not a SubTotal/Total line
  return if ($altname ne "");

  $total->{'KosPosts2002'} += $t->{'KosPosts2002'};
  $total->{'KosPosts2003'} += $t->{'KosPosts2003'};
  $total->{'OtherPosts'} += $t->{'OtherPosts'};
  $total->{'Comments2002'} += $t->{'Comments2002'};
  $total->{'Comments2003'} += $t->{'Comments2003'};
  $total->{'WordCount2002'} += $t->{'WordCount2002'};
  $total->{'WordCountKos2003'} += $t->{'WordCount2003'};
  $total->{'WordCount2003'} += $t->{'WordCount2003'};
  $total->{'NumLinks'} += $t->{'NumLinks'};
  $total->{'ZeroLinked'} += $t->{'ZeroLinked'};
  $total->{'OneLinked'} += $t->{'OneLinked'};
  $total->{'ManyLinked'} += $t->{'ManyLinked'};

}

########################################
# Print a page header

sub print_header($t) {
  $t = shift;

  print <<EOT;
<html>
 <head>
 <title>Daily Kos Archives Catalog: $t</title>
 </head>
 <a href="http://campaigntrails.org/">
<img style="width: 900px; height: 39px;" border="0" 
alt="CampaignTrails..................................................." 
src="http://campaigntrails.org/CampaignTrails-banner.png">
</a>

 <h1>Daily Kos Archive Catalog: $t</h1>
 <style> 
    tr.oddrow {background-color: #dca} 
    tr.total {background-color: #ba8}
    body { font-family : verdana }
    h1 { font-size : 16pt }
    td, th   { font-family : Arial ; font-size : 9pt }
    td { border-right : 1px solid #444 }
    th { border : 1px solid #444 }
  </style>

EOT

  print "<p><a href='ByTopic.html'>By Topic</a>  <a href='ByMonth.html'>By Month</a>\n";

  print "<a href='index.html'>Explanation about the data</a></p>";

  if ($t =~ /^By /) {
    print "<p><i>Covers May 26, 2002 to October 14, 2003</i></p>";

  }

  print "<table border='0' cellspacing='0'>";
  
}

##########################################
# Print a page footer


sub print_footer() {
  $d = localtime();

  print <<EOT;

<p></p>
<hr>
<p style="font-size : 9pt">
<i>Index Generation: by $MyName on $d<br>
<i>You are free to mirror these files, or re-generate this index, provided that you
include this message (and the generation credits) and point back to the original site.</i>
</p>
</html>

EOT

}

##########################################
# Guess the article author

sub guess_author() {
  return $1 if (m|^<p><i>[Bb]y (\w+ \w+)</i></p>$|);
  return $1 if (m|^<p><i>By (\w+)</i>|);
  return $1 if (m|^<p><i>(\w+ \w+)</i></p>$|);
  return "Steve Soto" if (m|^<p><b>Soto ?</b>|);
  return "Steve Soto" if (m|<p><b>Soto:</b></p>|);
  return "Steve Soto" if (m|^<p><b>Steve Soto</b>|);
  return "Steve Soto" if (m|^<b>Soto</b>|);
  return "Billmon" if (m|^<p><b>[Bb]illmon</b>|);
  return "Billmon" if (m|^<b>[Bb]illmon</b>|);
  return "Billmon" if (m|^<p><b>[Bb]illmon  </b>|);

  return "RonK" if (m|^<p><b>RonK|);
  return "RonK" if (m|^<p><i>by RonK|);
  return "RonK" if (m|^<b>RonK, Seattle</b>|);
  return "Steve Gilliard" if (m|^<p><b>Steve Gilliard ?</b>|);
  return "Steve Gilliard" if (m|^<b>Steve Gilliard</b></p>$|);
  return "Steve Gilliard" if (m|<p><i>By Steve Gilliard</i></p>|);

  if (m|^<p><b>(\w+)</b></p>$|) {
    return "Billmon" if ($1 eq "Billmon");
  }
   
  if (m|^<p><b>(\w+ \w+):? ?</b></p>$|) {
    return "Steve Soto" if ($1 eq "Steve Soto");
    return "Jerome Armstrong" if ($1 eq "Jerome Armstrong");
    return "";
  }

  if (m|^<p><b>(\w+ \w+):?</b><|) {
    return "Steve Soto" if ($1 eq "Steve Soto");
    return "Jerome Armstrong" if ($1 eq "Jerome Armstrong");
  
  }

  return "";
}

######################################################
# Guess the date posted 
# -- using the date at the footer since it includes time

sub guess_date($_) {
  return $1 if (m/<span class="posted">Posted ([^|<]+) [\s|]/);
  return $1 if (m/<span class="posted">Posted by Kos ([^|<]+)  /);
  return $1 if (m/<span class="posted">Posted by Kos at ([^|<]+)</);
  return $1 if (m/<span class="posted">Posted ([^|<]+)</);

  return "";
}

######################################################
# Guess the subject


sub guess_subject() {
  $_ = shift;

          return "democracy" if (/ rally/);
          return "democracy" if (/Sarandon/);	# Susan

          return "democracy" if (/Diebold/);
          return "democracy" if (/[Pp]unch [Cc]ards/);
          return "democracy" if (/GOTV/);	# Get Out The Vote
          return "democracy" if (/FBI/);
          return "democracy" if (/^Mystery/);
          return "democracy" if (/press repression/);  # not here, no
          return "democracy" if (/CFR/);  # Campaign Finance Reform

	# nations other than Iraq
          return "world" if (/Korea/);
          return "world" if (/Iran/);
          return "world" if (/Liberia/);
          return "world" if (/Cuba/);
          return "world" if (/Castro/);
          return "world" if (/China/);
          return "world" if (/Salvador/);
          return "world" if (/Israel/);  # strange company to be in, nu?

          return "gwot" if (/Homeland/);
          return "gwot" if (/national security/);
          return "gwot" if (/[Tt]aliban/);
          return "gwot" if (/9[\/-]11/);
          return "gwot" if (/Osama/);          # who? where?
          return "gwot" if (/[Bb]in [Ll]aden/);
          return "gwot" if (/Lindh/);
          return "gwot" if (/Arab linguists/);  # fired
          return "gwot" if (/Gitmo/);
          return "gwot" if (/Guantanamo/);
          return "gwot" if (/WOT/);		# "War On Terror"
          return "gwot" if (/airport/);
          return "gwot" if (/Padilla/);		# Jose

          return "iraq" if (/Iraq/);      # place with oil
          return "iraq" if (/war\b/i);    # means to get it
          return "iraq" if (/Saddam/);    # guy standing in the way
          return "iraq" if (/WMD/);       # pretending he has these
          return "iraq" if (/Baghdad/i);  # living high here
          return "iraq" if (/[Dd]esert/i);# living low here
          return "iraq" if (/army/);      # vini vidi vici
          return "iraq" if (/soldiers/);
          return "iraq" if (/Slocum/);
          return "iraq" if (/Coalition/); # of the billing
          return "iraq" if (/Powell/);    # he tried
          return "iraq" if (/Rumsfeld/);  # he didn't
          return "iraq" if (/Bremer/);    # medal of freedom
          return "iraq" if (/Tenet/);     # him too
          return "iraq" if (/Perle/);     # Richard
          return "iraq" if (/Blix/);              # Swede
          return "iraq" if (/Condi/);             # Swayed
          return "iraq" if (/Turkey/);    
          return "iraq" if (/Blair/);             # poor chap
          return "iraq" if (/Shia/);
          return "iraq" if (/bombing/);
          return "iraq" if (/Sadr/);		# Moktada al-
          return "iraq" if (/UN/);
          return "iraq" if (/Europe/);
          return "iraq" if (/POW/);
          return "iraq" if (/Pollack/);
          return "iraq" if (/Geneva/);		# piece of paper
          return "iraq" if (/Republican Guard/);
          return "iraq" if (/Security Council/);
          return "iraq" if (/the draft/);
          return "iraq" if (/Pentagon/);	
          return "iraq" if (/[Qq]uagmire/);	# the Q-word
          return "iraq" if (/Judith Miller/);
          return "iraq" if (/^Victory no/);	# 
          return "iraq" if (/^What the Expert/);
          return "iraq" if (/freepers/);	# opposite of kossacks
          return "iraq" if (/looters/); 
          return "iraq" if (/^Sigh...$/);	#

          return "gwot" if (/[Aa]fghan/);
          return "gwot" if (/terror/);


          return "domestic" if (/ANWR/);	# Alaska National Wildlife Refuge
          return "domestic" if (/drilling/);
          return "domestic" if (/marijuana/);	

          return "domestic" if (/identity politics/);
          return "domestic" if (/Social Security/);
          return "domestic" if (/Karma/);
          return "domestic" if (/EPA/);		# Envi. Protection Agency
          return "domestic" if (/HMO/);		# Health Management Organization
          return "domestic" if (/Labor/);
          return "domestic" if (/The Pledge/);
          return "domestic" if (/[Aa]bortion/); # the 3rd 
          return "domestic" if (/[Gg]ay/);	# the other third rail
          return "domestic" if (/sodomy/);
          return "domestic" if (/Santorum/);	# made some comments about above
          # return "domestic" if (/justice/);
          return "domestic" if (/polution/);
          return "domestic" if (/RIAA/);	# 
          return "domestic" if (/Space shuttle/);
          return "domestic" if (/creationism/);
          return "domestic" if (/Atheist/);
          return "domestic" if (/segregationist/);
          return "domestic" if (/[Jj]udge/);

          return "misc" if (/baseball/);
          return "misc" if (/the fans/);
          return "misc" if (/Yankees/);
          return "misc" if (/Cubs/);
          return "misc" if (/Lord of the Rings/);
          return "misc" if (/Bestseller/);
          return "misc" if (/^JetBlue/);

          return "bushies" if (/job ratings/);
          return "economy" if (/[Jj]ob/);
          return "economy" if (/etailers/);
          return "economy" if (/[Cc]onsumer/);
          return "economy" if (/[Ee]conom/);
          return "economy" if (/eficit/);
          return "economy" if (/budget/);
          return "economy" if (/SEC/);
          return "economy" if (/Dow /);
          return "economy" if (/layoffs/);
          return "economy" if (/[Dd]ebt/);
          return "economy" if (/Greenspan/);
          return "economy" if (/401(k)/);
          return "economy" if (/Wall Street/);

          return "states" if (/Vilsack/);
          return "states" if (/Maine/);
          return "states" if (/ FL/);
          return "states" if (/ gov's/);


	return "bushies" if (/John Dean/);

	# 2004 Prez primary 
	return "primary" if (/Iowa/);
	return "primary" if (/Dwarves/);
	return "primary" if (/fundraising/);
	return "primary" if (/undecideds/);
	return "primary" if (/rimary/);
	return "primary" if (/Dean/);		# Howard	
	return "primary" if (/AFL-CIO/);
	return "primary" if (/Labor endorsement/);
	return "primary" if (/Clark/);		# Gen. Wesley
	return "primary" if (/Gore/);		# Al
	return "primary" if (/Joe\b/);
	return "primary" if (/Lieberman/);
	return "primary" if (/Kerry/);		# John
	return "primary" if (/Hart/);		# Gary
	return "primary" if (/Gephardt/);	# Richard
	return "primary" if (/Edwards/);
	return "primary" if (/Edward's/);  # he meant Edwards's
	return "primary" if (/Graham/);		# 
	return "primary" if (/Kucinich/);	# Dennis
	return "primary" if (/Cattle/);
	return "primary" if (/Zinni/);     # another general
	return "primary" if (/Hillary/);	# Clinton
	return "primary" if (/Biden/);		# Joe
	return "primary" if (/Lehane/);		# Kerry comm. director
	return "primary" if (/Nader/);		# spoiler of 2000
	return "primary" if (/turn guns/);
	return "primary" if (/Dem [Dd]ebate/);
	return "primary" if (/best presidential/);
	return "primary" if (/^Presidential Poll/);
	return "primary" if (/- Presidential/);
	return "primary" if (/--Presidential/);
	return "primary" if (/^Full Disclosure$/);
	return "primary" if (/tour of the horizon/); # a post from Jerome

	# Bush Administration
          return "bushies" if (/George/);	# 43
          return "bushies" if (/dubya/);
          return "bushies" if (/Shrub/);
          return "bushies" if (/ lies/);
          return "bushies" if (/white house/i);
          return "bushies" if (/lazy president/);
          return "bushies" if (/disappearing [Pp]residen/);
          return "bushies" if (/Cheney/);	# Dick
          return "bushies" if (/the president/);
          return "bushies" if (/halliburton/i);
          return "bushies" if (/Rove/);		# Dark Prince
          return "bushies" if (/Ashcroft/);	# A.G. #1
          return "bushies" if (/Pickering/);
          return "bushies" if (/Secretary/);
          return "bushies" if (/Rehnquist/);	
          return "bushies" if (/[Tt]reasury/);
          return "bushies" if (/O'Neill/);      # Paul, Sec'y of Treas. #1  
          return "bushies" if (/Snow/);        # John, Sec'y of Treas #2
          return "bushies" if (/dissatisfied/);
          return "bushies" if (/hypocr/);	
          return "bushies" if (/malaise/);	
          return "bushies" if (/DiIulio/);   # didn't stay long
          return "bushies" if (/Falwell/);
          return "bushies" if (/Estrada/);	# not Erick
          return "bushies" if (/Christian Coalition/);
          return "bushies" if (/SOTU/);		# State Of The Union
          return "bushies" if (/state of the union/);
          return "bushies" if (/^Final thoughts/);
          return "bushies" if (/[Cc]orporate/);


	# 2002 Midterm Elections
	return "congress" if (/[Cc]ongress/);	# opposite of progress
	return "congress" if (/big night/);
	return "congress" if (/ouse seat/);
	return "congress" if (/House control/);
	return "congress" if (/redistricting/);
	return "congress" if (/Young/);
	return "congress" if (/Harris/);    # 
	return "congress" if (/Minnesota/);
	return "congress" if (/Colorado/);
	return "congress" if (/Smith/);
	return "congress" if (/Taylor/);
	return "congress" if (/ will run /);
	return "congress" if (/Indiana 8th/);
	return "congress" if (/[Ss]enate/);
	return "congress" if (/Murkowski/);
	return "congress" if (/Janklow/);
	return "congress" if (/Reid/);		# Harry, Harry Harry
	return "congress" if (/senator/);
	return "congress" if (/Cleland/);	# Max of GA
	return "congress" if (/Bowles/);	# Erskine
	return "congress" if (/Dole/);		# Elizabeth of NC
	return "congress" if (/Toricelli/);	# Bob of NJ
	return "congress" if (/Lautenberg/);	# Frank of NJ
	return "congress" if (/Landrieu/);	# Mary of LA
          return "congress" if (/[Cc]omittee/);
          return "congress" if (/Lott/);
	return "congress" if (/Daschle/);	# dashing Tom of SD
	return "congress" if (/Kennedy/);   	# Big Teddy K of MA
	return "congress" if (/Hatch/);		
	return "congress" if (/Harkin/);     	# Tom of IA
	return "congress" if (/Gangske/);     
	return "congress" if (/Mondale/);     	# Fritz
	return "congress" if (/Frist/);      	# Bill of Tenn.
	return "congress" if (/Wellstone/); 	# Paul of Minn.
	return "congress" if (/Louisiana/);
	return "congress" if (/Townsend/);
	return "congress" if (/Largent/);	# Steve
	return "congress" if (/Chamblis/);
	return "congress" if (/Nickles/);
	return "congress" if (/Pryor/);
	return "congress" if (/Pelosi/);      	# Mdme speaker in 2007
	return "congress" if (/predictions/);
	return "congress" if (/ LA /);
        return "congress" if (/NJ/);  # Toricelli replacement
	return "congress" if (/Zell/);		# Yosemite Sam

          return "states" if (/Illinois/);  # #000280

	# California "Total Recall" effort
          return "calif" if (/Arnold/);		# or Ahnold
          return "calif" if (/Schwarzen/);	# the Governator
          return "calif" if (/Simon/);       	# Bill Simon
          return "calif" if (/California/);  	# here I come
          return "calif" if (/Arianna/);	# 
          return "calif" if (/Davis/);		# Grey
          return "calif" if (/Bustam/);		# Cruz
          return "calif" if (/Cruz/);
          return "calif" if (/CA poll/);
          return "calif" if (/[Rr]ecall/);
          return "calif" if (/CA Cattle Call/);

	# Goobernatorial races and other local issues
          return "states" if (/Barbour/);	# Haley, Miss. Gov.	
          return "states" if (/Loco/);		
          return "states" if (/governor/i);
          return "states" if (/gubern/i);
          return "states" if (/guv/i);
          return "states" if (/Romney/);
          return "states" if (/Jeb/);		# Bush of Fla.
          return "states" if (/Everglades/);	
          return "states" if (/Reno/);		# Janet of Fla.
          return "states" if (/McBride/);	# Bill of Fla.
          return "states" if (/Shaheen/);	# Jeanne of Fla
          return "states" if (/Pataki/);	# George of NY
          return "states" if (/Texas/);
          return "states" if (/NYC/);
          return "states" if (/Sebelius/);	# Kathleen of Kan.
          return "states" if (/Ryan/);
          return "states" if (/ LA/);
          return "states" if (/Forrester/);
          return "states" if (/Lingle/);	# Linda of HI

	# Eat the Press!
	  return "media" if (/Chris Mathews/);
          return "media" if (/Google News/);
          return "media" if (/ESPN/);
          return "media" if (/radio/);
          return "media" if (/Rush/);    # Mega-dittos to you, pill-poppoer
          return "media" if (/cartoon/);
          return "media" if (/editing/);
          return "media" if (/Mallard/);
          return "media" if (/Guardian/);
          return "media" if (/Fox prop/);
          return "media" if (/Fox sues/);
          return "media" if (/the press /);
          return "media" if (/Doonesbury/);
          return "media" if (/\bTV/);
          return "media" if (/Coulter/);   # Ann
          return "media" if (/O'Reilly/);  # Bill

          return "iraq" if (/Plame/);	
          return "iraq" if (/Expose/);
          return "iraq" if (/\bleak/);


          return "dems" if (/DINO/);		# Democrat In Name Only
          return "dems" if (/ePatriot/);
          return "dems" if (/party's/);
          return "dems" if (/D[LN]C/);
          return "dems" if (/McAuliffe/);	# Terry
          return "dems" if (/ACLU/);
          return "dems" if (/LaRouche/);   	# Lyndon, ersatz Dem 
          return "dems" if (/philosophical/);
          # return "bushies" if (/GOP/);
          return "bushies" if (/Republican/i);

	# Site Maintenance
	return "site" if (/Kos/);
	return "site" if (/Political State/);
	return "site" if (/^Link/);
	return "site" if (/^Email/);
	return "site" if (/^Jury/);
	return "site" if (/^MT com/);
	return "site" if (/ days off/);
	return "site" if (/^Status$/);
	return "site" if (/^Over and Out/);
	return "site" if (/^Things almost/);
	return "site" if (/^We are experi/);
	return "site" if (/^Blog problems/);
	return "site" if (/Upgrading site/);
	return "site" if (/[Ss]ite [Uu]pdate/);
	return "site" if (/^Good News/);
	return "site" if (/^I'm back/);
	return "site" if (/^Experiment/);
	return "site" if (/^Congrats to MyDD/);
	return "site" if (/^Day 1/);
	return "site" if (/Incubating/);
	return "site" if (/^Coming Att/);

	return "misc" if (/^Spam/);

        # some catch-all terms, in case subject isn't guessed from above

          return "bushies" if (/Bush/i);    
          return "democracy" if (/arrested/);

          return "dems" if (/[Dd]em/); # in case nothing else

          return "economy" if (/[Mm]arket/);

          return "domestic" if (/Supreme/);

          return "iraq" if (/Rice/);           

          return "gwot" if (/Security/);

          return "gwot" if (/[Ii]ntelligence/); 

          # return "election" if (/poll/i);  # will set correctly if 2002 or 2003
          return "election" if (/debate/i);  # will set correctly if 2002 or 2003
          return "election" if (/ race/i);

          return "open" if (/ Thread/i);
          return "open" if (/ Thead/i);


	return "";
}