#!/usr/bin/perl
# ------ extract-isa16.pl
# Make a list of sessions of 2016 Third ISA Forum of Sociology
# 2013/08/15 - 2013/09/19 (extract-isa14.pl)
# 2015/05/06

# Download the lists of sessions from ISA website:
# wget -N -x            http://www.isa-sociology.org/forum-2016/rc/joint-sessions.php
# wget -N -x -r -l1 -np http://www.isa-sociology.org/forum-2016/rc/
# wget -N -x -r -l1 -np http://www.isa-sociology.org/forum-2016/wg/
# wget -N -x -r -l1 -np http://www.isa-sociology.org/forum-2016/tg/

undef $/ ;
$\ ="\n";
$" = $,  ="\t";

$Count =0;

sub url {
	my $f = shift;
	$f =~ s|\@|\?| ;
	$f =~ s|\\|\/|g ;
	"http://$f" ;
}

sub output {
	my ( @p ) = @_ ;
	my $text;
	foreach $text (@p){
		$text =~ s/^[^>]+>// ;
		$text =~ s|</a>| // |g ;
		$text =~ s/<[^>]+>/ /g ;
		$text =~ s/\n/ /g ;
		$text =~ s/&#(\d+);/ chr($1) /eg ;
		$text =~ s/&hellip;/ ... /g ;
		$text =~ s/&nbsp;/ /g ;
		$text =~ s/\s+/ /g ;
		
		$text =~ s/(\[[A-Z]+\])\s*$//i;
#		$text =~ s/<h\d .*$//i;
		
		$text =~ s/\"+/\"/g;
	}

	print @p;
}

print STDERR @ARGV;
print STDERR scalar(@ARGV), " files.";

# get file name

foreach $f ( @ARGV ) {
	open IN, $f or die("Cannot open $f");
	my ($committee) = $f=~/(\w+)$/ ;
	my ($baseurl) = url($f);

	$_ = <IN>;
	s/\n/ /g ;
	s|<a\s+href|<aa href|gi ;

	($h1) = m|<h1>(.+)</h1>|i ;
#	print "\n\n";
	print "";
	output ($h1);

	my @entry = split ( /<a /  );	# *** Divide entries
	my $e;
	foreach $e(@entry){
		my ( $id) = $e=~/id\s*=\s*"([^"]*)"/i; 	# *** Extract ID
		my ( $name ) = $e=~/name\s*=\s*"([^"]*)"/i; 	# *** Extract NAME
		next if $id eq 'top';
		next unless $id;
		warn "differ: $id - $name\n"  if $id ne $name;

		my ($title) = $e=~m|<h2>(.+)</h2>|i ;
		($title) = $e=~m|<h3>(.+)</h3>|i  unless $title;
		next if $title =~ m |<aa |;

		output( $committee , $id , "$baseurl\#$id" , $title );

		++$Count;
	}
}

print STDERR $Count, " records."
	
#------------- EOF