#!/usr/bin/perl -w

=head1 NAME

Planteome Annotation Wiki - Data Import Script

=head1 VERSION

0.4

=head1 DESCRIPTION

Transform external gene annotation data into an XML document readable
by the MediaWiki extension DataTransfer (Special:ImportXML) feature.

Also generates appropriate provenance of data based on a prefixed
Source header in the import file.

Handles GAF file type option for import; accommodates ontologically-
focused annotations.

New to this revision: ... 

=head1 USAGE

paw_TransformForImport.pl -i INPUT_FILE -t TYPE -o OUTPUT_FILE -v -d

=head1 OPTIONS

    -i  Name of input file
        (if no path supplied, assumes current working directory)
    -t  Specifies input type of file
        (allowed values: 'csv', 'tab', 'gaf')
    -o  Name/template of output XML file; if no path supplied, assumes 
        current working directory. The first file will match the 
        supplied value; consequent files will be numbered in this
        format: '-#.suffix'
    -s  Seed values (comma-delimited) for Source, Annotation, and 
        Publication IDs (in that order, e.g. '3,38,15'); Defaults to 
        '1' for all seed values
    -v  View verbose information
    -d  View debugging information

=head1 DEPENDENCIES

Requires that the input file contain at least two headers: the first 
will hold the provenance information associated with the imported 
data, and all following headers will specify the annotation data to
be imported in accordance with the input type.

=head2 Source Header Format Example

    [Source]
    Source Date Time Stamp=Apr 2 2008
    Source Database=Oryzabase
    Source Version=rel. 10
    Source URI=http://www.shigen.nig.ac.jp/rice/oryzabase/
    Source File=http://www.shigen.nig.ac.jp/rice/oryzabase/genes/...

=head2 Data Header Generic Format Examples

    NOTES: The field separator may also be a tab instead, but must be 
    employed consistently across the entire file. GAF files do not 
    require a [Format] section.

    [Format]
    Template=Annotation
    Fields=Species Name,Gene Symbol,Gene Name,Chromosome,Gene Type,Accession ID
    [Data]
    val1,val2,val3,...
    "
    "
    "
    [Format]
    Template=Publication
    Fields=Publication,Publication ID,Publication Title,...,Accession ID
    [Data]
    val1,val2,val3,...
    "
    "
    "

    NOTE: One of the fields in each format MUST be named "Accession
    ID" for the tracking of provenance.

=head1 AUTHOR

Justin Preece
    Faculty Research Assistant, Bioinformatics
    Jaiswal Lab, Botany & Plant Pathology
    Oregon State University
    L<mailto:preecej@science.oregonstate.edu>

=cut

# TODO: file size check and clip
# TODO: file numbering

# ---------------------------------------------------------------------------
# modules
# ---------------------------------------------------------------------------

# general
use strict;
use Cwd;
use Switch;
use Getopt::Std;
use Data::Dumper;

# specific
use XML::Smart;
use XML::Writer;
use IO::File;
use Encode; 

# ---------------------------------------------------------------------------
# declarations
# ---------------------------------------------------------------------------

my %opts; # arg options
my $file_type; # tab, csv, gaf
my $file_del; # data delimeter
my $input_file;
my $output_file;
my $verbose = 0; # flag for verbose output
my $debug = 0; # debugging switch

# global data storage ----------

# universal 
my %source;
my $output_data;

# tab, csv
my $template_name;
my @field_names;
my @field_data;
my $xml; # represents the xml doc

# gaf
my %ontology_info; # holds ontology names and aspects, keyed by abbreviation 
my $curr_ontology_type; # flag to track the proper ontology to reference, line by line
my %annotation_universals; # holds values assumed not to vary across the file
my %annotations; # keyed on Gene Symbol

# other config constants (including temporary arrangements)

# default page id seeds, until I can figure out how to auto-increment w/in the
# an import/update script
my $SOURCE_TITLE_SEED = 1;
my $ANNOT_TITLE_SEED = 1;
my $PUB_TITLE_SEED = 1;

$Data::Dumper::Pad = "... "; 

# ---------------------------------------------------------------------------
# functions
# ---------------------------------------------------------------------------


# ---------------------------------------------------------------------------
sub init
{
    # read and set options
    getopts('i:t:o:s:vd', \%opts);
    
    foreach my $key (keys %opts) { 
        my $value = $opts{$key};
        switch ($key) {
            case "i" { 
                if ($value =~ /\//) { # assume path
                    $input_file = $value;
                } else {
                    $input_file = getcwd() . "\/$value";
                }
            }
            case "t" { $file_type = $value; }
            
            case "o" {
                if ($value =~ /\//) { # assume path
                    $output_file = $value;
                } else {
                    $output_file = getcwd() . "\/$value";
                }
            }
            case "s" {
                my @seeds = split(',',$value);
                if (scalar @seeds == 3) {
                    $SOURCE_TITLE_SEED = $seeds[0];
                    $ANNOT_TITLE_SEED = $seeds[1];
                    $PUB_TITLE_SEED = $seeds[2];
                } else {
                    die("Please supply three consecutive, comma-delimited seed" 
                        . "values for the 's' option.");
                }
            }
            case "v" { $verbose = 1; }
            
            case "d" { $debug = 1; }
        }
    }
    
    # split data on either commas or tabs
    switch ($file_type) {
        case "csv" { $file_del = ','; }
        case "tab" { $file_del = '\t'; }
        case "gaf" { $file_del = '\t'; }
        else {
            die(uc($file_type) . " is not a valid file type. Please supply "
                . "a tab, csv, or gaf file.\n");
        }
    }
        
    system "clear";
    print "\n"
        . "------------------------------------------------------------\n"
        . "-- Planteome Annotation Wiki Import Transformation Script --\n"
        . "------------------------------------------------------------\n"
        . "\n"
        . "Input File: $input_file\n"
        . "Designated input file type: $file_type\n"
        . "Output File (Template): $output_file\n"
        . "Source ID Seed: $SOURCE_TITLE_SEED\n"
        . "Annotations ID Seed: $ANNOT_TITLE_SEED\n"
        . "Publications ID Seed: $PUB_TITLE_SEED\n"
        . "Running in verbose mode? " . ($verbose ? "Yes" : "No") . "\n"
        . "Running in debug mode? " . ($debug ? "Yes" : "No") . "\n"
        . "\n"
        . "------------------------------------------------------------\n"
        . "------------------------------------------------------------\n"
        . "------------------------------------------------------------\n"
        . "\n";
}


# read, parse, and store generic CSV and tab templates and annotations
# ---------------------------------------------------------------------------
sub import_generic
{
    # read in "[Format] section...
    my $line;
    my $count = 0;
    
    while (<INPUT_FILE>)
    {
       $count++;
       $line = $_;
       chomp $line;
       my $data_val = (split('=',$line))[1];
       switch ($count) {
           case 2 { $template_name = $data_val; }
           case 3 { @field_names = split($file_del,$data_val); }
           else {;}
       }
       if ($count == 3) { last; }
    }
   
    # loop through data rows and add all data fields to an array of hashes
    while (<INPUT_FILE>)
    {
        $line = $_;
        chomp $line;
        
        my @tmp_data_ary = split($file_del, $line);
        for (@tmp_data_ary) {
            if ($_ eq '-' || $_ eq '.') {
                $_ = "";
            }
        }

        my %tmp_hash;
        
        if ($debug) { print join(',',@tmp_data_ary) . "\n"; }
        
        for (my $i=0; $i<scalar(@field_names); $i++) {
            if ($debug) { print "$i: $field_names[$i]: $tmp_data_ary[$i]\n"; }           
            $tmp_hash{$field_names[$i]} = $tmp_data_ary[$i];
        }
        push @field_data, \%tmp_hash;
    }
}

# Add new ontology reference data. Aggregates add'l evidence codes, evidence, 
# assignations, and publications for the existing term id.
# ---------------------------------------------------------------------------
sub add_ontology_ref_data($$)
{
    my %ref_data = %{$_[0]};
    my $ont_type = $_[1];
 
    my @evidence = split('\|', $ref_data{"with_or_from"});
    my @publications = split('\|', $ref_data{"db_reference"});
 
    my @tmp_pubs = @publications;
    for my $tmp_pub (@tmp_pubs)
    {
        if (substr($tmp_pub,0,5) ne "PMID:")
        {
            my @junk = splice(@publications,0,1);
        }
    }

    # aspect (assumes only one, allows overwrite)
    $annotations{$ref_data{"db_object_symbol"}}
        {"Ontological References"} 
            {$ref_data{"term_id"}}
            {"Aspect"} = $ontology_info{$ont_type}
                            {"aspects"}
                                {uc($ref_data{"aspect"})};

    # evidence codes
    $annotations{$ref_data{"db_object_symbol"}}
        {"Ontological References"}
            {$ref_data{"term_id"}}
                {"Evidence Codes"}{$ref_data{"evidence_code"}} = "";

    # evidence
    for (@evidence)
    {
        $annotations{$ref_data{"db_object_symbol"}}
            {"Ontological References"}
                {$ref_data{"term_id"}}
                    {"Evidence"}{$_} = "";
    }

    # assigned by
    $annotations{$ref_data{"db_object_symbol"}}
        {"Ontological References"}
            {$ref_data{"term_id"}}
                {"Assigned By"}{$ref_data{"assigned_by"}} = "";

    # publications
    for (@publications)
    {
        $annotations{$ref_data{"db_object_symbol"}}
            {"Ontological References"}
                {$ref_data{"term_id"}}
                    {"Publications"}{$_} = "";
    }
}

# populate a hash containing ontology types and their aspects; used as a 
# reference for aspect proper names when importing GAF data for transformation
# ---------------------------------------------------------------------------
sub initialize_ontology
{
    %ontology_info = (
        GO => {
            name => "Gene Ontology",
            aspects => {
                P => "Biological Process",
                C => "Cellular Component",
                F => "Molecular Function"
            }
        },
        PO => {
            name => "Plant Ontology",
            aspects => {
                A => "Plant Anatomy",
                G => "Plant Growth and Development Stage"
            }
        }
    );
    
    if ($debug) { print "...<DEBUG: \%ontology_info>\n"
    . Dumper(\%ontology_info) . "\n\n"; }
}

# read, parse, and store GAF annotations
# ---------------------------------------------------------------------------
sub import_gaf
{
   	#[GAF implied data structure]
    # singletons: db/Source (not really, but OK for now), taxon/Species ID, 
    #   gene_type/Gene Type (later: proteins, too)
    # lower priority: db_object_id, date
    # not needed: qualifier
    # unvaried fields (gene-level): db_object_symbol/Gene Symbol, 
    #   db_object_name/Gene Name, 
    #   db_object_synonym/Gene Locus|Source Accession ID|Chromosome|Gene Synonyms (see below), 
    # varied fields (gene synonyms): db_object_synonym/Gene Synonym
    # varied fields (ontology-level): term_id/Term ID, evidence_code/Evidence Code, assigned_by/Assigned By
    #   aspect/Aspect, db_reference (multi-, get PMID from here), with_or_from (multi-)

    # [Annotation Object Structure]
    # %annotation_universals
    #     "Source"
    #     "Species ID"
    #     "Gene Type"
    # %annotations
    #     "$Gene Symbol" => %annotation_properties
	#     	                    "Gene Name" => string
	# 		    				"Gene Locus"/"Source Accession ID" (first uc("AT.G"))  => string
	# 			    			"Chromosome" (AT#G in "Gene Locus")  => string
	# 				    		"Gene Synonyms"  => % strings
	#                           "Ontological References" => % "$Term ID" => %annotation_ontology_ref
	#             				    						        "Aspect" => string (assumes only one)
	#     				                						    "Evidence Codes" => % strings
	#                                                               "Evidence" => % strings ("with_or_from")
	#                                                               "Assigned By" => string ("assigned_by")
	#                                                               "Publications" => % PMID's from "db:reference", used to create separate Pub pages
    
	# set up a hash of ontology types and aspects to be referenced during data import
	initialize_ontology();

    # loop through data rows and build hashed annotation data structure
    my $count = 0;
    
    # regex for locating a useable accession id from a locus (species-specific)
    my $locus_finder_expr;  
    
    while (<INPUT_FILE>)
    {
        $count++;
        my $line = $_;
        chomp $line;
        
        my @curr_line_ary = split("\t", $line);
        for (@curr_line_ary) {
            if ($_ eq '-' || $_ eq '.') {
                $_ = "";
            }
        }
        if ($debug) { print "...<DEBUG: \@curr_line_ary>\n"
            . Dumper(\@curr_line_ary) . "\n\n"; }
        
        my %curr_line_hash = (
            "db"                => $curr_line_ary[0], # Source
            "db_object_symbol"  => $curr_line_ary[2], # Gene Symbol
            "term_id"           => $curr_line_ary[4], # Term ID
            "db_reference"      => $curr_line_ary[5], # inc. Publication info (PMID)
            "evidence_code"     => $curr_line_ary[6], # Evidence Code
            "with_or_from"      => $curr_line_ary[7], # Evidence (data)
            "aspect"            => $curr_line_ary[8], # Aspect
            "db_object_name"    => $curr_line_ary[9], # Gene Name
            "db_object_synonym" => $curr_line_ary[10], # Gene Locus, Source Accession ID, Chromosome, Gene Synonyms
            "db_object_type"    => $curr_line_ary[11], # Gene Type
            "taxon"             => $curr_line_ary[12], # Species ID
            "assigned_by"       => $curr_line_ary[14] # Assigned By
            );

        if ($debug) { print "...<DEBUG: \%curr_line_hash>\n"
            . Dumper(\%curr_line_hash) . "\n\n"; }
        
        # grab the unvaried values from the first line
        if ($count == 1) {
            %annotation_universals = (
                "Source"       => $curr_line_hash{"db"}, # currently not in use
                "Gene Type"    => $curr_line_hash{"db_object_type"},
                "Species ID"   => "NCBI:" . (split(':',$curr_line_hash{"taxon"}))[1]#,
                #"Species Name" => $species_name # TODO: get this from NCBI
                );
            
            # set species-specific values
            switch ($annotation_universals{"Species ID"}) {
                case "NCBI:3702" {
                    $locus_finder_expr = "^([Aa][Tt].[Gg])";
                    $annotation_universals{"Species Name"} = "Arabidopsis thaliana"; # temp; need an NCBI lookup
                }
                case "NCBI:4530" {
                    $locus_finder_expr = "^(LOC_|[Oo][Ss]|osa-)";
                    $annotation_universals{"Species Name"} = "Oryza sativa"; # temp; need an NCBI lookup
                }
                case "NCBI:29760" {
                    $locus_finder_expr = "^(Vv)";
                    $annotation_universals{"Species Name"} = "Vitis vinifera"; # temp; need an NCBI lookup
                }
                else {
                    die($annotation_universals{"Species ID"} 
                        . " is not a valid NCBI taxon ID.\n");
                }
            }

            if ($debug) { print "...<DEBUG: \%annotation_universals>\n"
                . Dumper(\%annotation_universals) . "\n\n"; }

        }
        
        # set the ontology for the current line
        $curr_ontology_type = uc((split(':',$curr_line_hash{"term_id"}))[0]);

        # check to see if Gene Symbol hash key exists (for grouping)
        # if not, add the new Gene Symbol and its associated props
        if (!exists $annotations{$curr_line_hash{"db_object_symbol"}})
        {
            
            # print "\n*** NEW SYMBOL: $curr_line_hash{'db_object_symbol'} ***\n"; # TEST

            # prepare Gene Locus, Source Accession ID, Chromosome, Gene Synonyms
            my @loci;
            my $locus = "";

            if (exists $curr_line_hash{"db_object_synonym"})
            {
                my @synonyms = split('\|',$curr_line_hash{"db_object_synonym"});
                
                if ($debug) { print "...<DEBUG: \@synonyms>\n"
                    . Dumper(\@synonyms) . "\n\n"; }
                
                # find the gene locus, if it is listed
                @loci = grep /$locus_finder_expr/, @synonyms;
    
                if ($debug) { print "...<DEBUG: \@loci>\n"
                    . Dumper(\@loci) . "\n\n"; }
            }
            
            if (scalar(@loci) > 0) # we have at least one match; use the first one
            {    
                $locus = $loci[0]; 
            }
            else # no match; attempt to use the Gene Symbol instead
            {
                if ($curr_line_hash{"db_object_symbol"} =~ $locus_finder_expr)
                {
                    # the split drops the variant/allele signifier, if present
                    if (!(split('.',$curr_line_hash{"db_object_symbol"}))[0])
                    {
                        $locus = $curr_line_hash{"db_object_symbol"};
                    }
                    else
                    {
                        $locus = (split('.',$curr_line_hash{"db_object_symbol"}))[0];
                    }
                }
                else # no match; attempt to use the Gene Name instead
                {
                    if ($curr_line_hash{"db_object_name"} =~ $locus_finder_expr)
                    {
                        $locus = (split('.',$curr_line_hash{"db_object_name"}))[0];
                    }
                }
            }
            
            my $chromosome = "";
            if ($locus) {
                if ($annotation_universals{"Species ID"} eq "NCBI:3702") {
                    # for Ath, third char in locus, if it exists
                    $chromosome = ($locus ne "" ? (split('',$locus))[2] : "");
                }
                if ($annotation_universals{"Species ID"} eq "NCBI:29760") {
                    # for Vv, third and fourth char in locus, if it exists
                    $chromosome = ($locus ne "" ? (split('',$locus))[2] . (split('',$locus))[3] : "");
                }
            }
                
            # set some sort of pseudo-unique value as the accession id, 
            # in order of succession: locus, then symbol
            # (NOTE: this is dangerous; a stable identifier is preferred)
            my $accession_id = 
                $locus ? $locus : $curr_line_hash{"db_object_symbol"};
            
            # set up props
            my $annotation_properties = {
                "Accession ID"  => $accession_id,
                "Gene Name"     => $curr_line_hash{"db_object_name"},
                "Gene Locus"    => $locus,
	 			"Chromosome"    => $chromosome
			    };
			    
            # add synonyms
			for (split('\|', $curr_line_hash{"db_object_synonym"})) {
			    $$annotation_properties{"Gene Synonyms"}{$_} = ""; }
			
            if ($debug) { print "...<DEBUG: \%annotation_properties>\n"
                . Dumper($annotation_properties) . "\n\n"; }

            # add new gene annotation and assign props
            $annotations{$curr_line_hash{"db_object_symbol"}} = $annotation_properties;

            add_ontology_ref_data(\%curr_line_hash, $curr_ontology_type); # add the first ontology reference (every time)            
        }
        # that Gene Symbol does exist, so we just need to roll-up multi-line 
        # annotation information, like gene synonyms and ontology refs
        else 
        {
            #print "\n*** EXISTING SYMBOL: $curr_line_hash{'db_object_symbol'} ***\n"; # TEST

            # add any add'l synonyms
			for (split('\|', $curr_line_hash{"db_object_synonym"})) {
			    $annotations{$curr_line_hash{"db_object_symbol"}}
			        {"Gene Synonyms"}{$_} = ""; }

            add_ontology_ref_data(\%curr_line_hash, $curr_ontology_type); # add add'l ontology reference data            
        }
    }
    if ($debug) { print "...<DEBUG: \%annotations>\n"
        . Dumper(\%annotations) . "\n\n"; }
}


# read, parse, and store source
# ---------------------------------------------------------------------------
sub import_data
{
    print "Opening input file and reading header info...\n\n";

    # open file
    open(INPUT_FILE,$input_file) or die("Could not open input file.");

    # read in the source data
    my $count = 0;
    my $line; 
    while (<INPUT_FILE>)
    {
       $count++;
       $line = $_;
       chomp $line;
       my $data_val = (split('=',$line))[1];
       switch ($count) {
           case 2 { $source{'SourceDateStamp'} = $data_val; }
           case 3 { $source{'SourceDatabase'} = $data_val; }
           case 4 { $source{'SourceVersion'} = $data_val; }
           case 5 { $source{'SourceURI'} = $data_val; }
           case 6 { $source{'SourceFile'} = $data_val; }
           else {;}
       }
       if ($count == 6) { last; }
    }

    print "Reading data...\n\n";
    $line = <INPUT_FILE>; # skip "[Data]"

    switch ($file_type) {
        case ('csv' || 'tab') { import_generic(); }
        case 'gaf'            { import_gaf(); }
    }

    close INPUT_FILE;
}


# spit out the data to make sure you've read in the files correctly
# ---------------------------------------------------------------------------
sub show_input
{
    print "\n[Source]\n";
    foreach my $key (keys %source) {
        print "$key: $source{$key}\n";
    }
    print "\n";

    switch ($file_type) {
        case ('csv' || 'tab') {
            print "[Template]\n$template_name\n\n";
            print "[Fields]\n" . join(', ',@field_names) . "\n\n";
            
            print "[Data]\n";
            foreach my $row (@field_data) {
                foreach my $key (keys %$row) {
                    print "$key => " . $row->{$key} . "\n";
                }
                print "\n";
            }
        }
        case 'gaf' {
            print "[Data]\n";

            for my $key (keys %annotation_universals) {
                print "$key: " . $annotation_universals{$key} . "\n";
            }
            print "\n";
            
            print "[Annotations]\n" . Dumper(\%annotations) . "\n\n";
        }
    }
    print "\n";
}

# xml transformation for generic tab or CSV-templated data
# (currently uses XML::Smart)
# ---------------------------------------------------------------------------
sub transform_generic
{
    my $curr_node; # placeholder for general node cursor
    my $curr_prov_node; # placeholder for node cursor in provenance pages
    my $curr_annot_node; # placeholder for node cursor in annotation pages
    my $curr_accession_id; # holds each rows accession id for provenance marking

    
    $xml = new XML::Smart($output_data, 'XML::Smart::Parser');

    # set root element, source page and elements
    # (temp set of page title until moved to import extension) 
    $xml->{Pages}{Page} = {Title => "Source:$SOURCE_TITLE_SEED"};

    $curr_node = $xml->{Pages}{Page};
    $curr_node->{Template} = {Name => 'Source'};
    $curr_node = $curr_node->{Template};    
    
    $curr_node->{Field}[0] = {Name => 'Source Date Stamp'};
    $curr_node->{Field}[0]->content(0,$source{'SourceDateStamp'});
    $curr_node->{Field}[1] = {Name => 'Source Database'};
    $curr_node->{Field}[1]->content(0,$source{'SourceDatabase'});
    $curr_node->{Field}[2] = {Name => 'Source Version'};
    $curr_node->{Field}[2]->content(0,$source{'SourceVersion'});
    $curr_node->{Field}[3] = {Name => 'Source URI'};
    $curr_node->{Field}[3]->content(0,$source{'SourceURI'});
    $curr_node->{Field}[4] = {Name => 'Source File'};
    $curr_node->{Field}[4]->content(0,$source{'SourceFile'});
    
    $curr_node = $curr_node->back->back; # return to <Pages> node
    
    if ($debug) { print "Current node: " . $curr_node->path . "\n"; }
    
    my $next_page_title_id = $ANNOT_TITLE_SEED;
    
    # iterate through the data
    foreach my $row (@field_data) {

        # set up next annotation page
        my $next_page = { Title => "Annotation:$next_page_title_id" };
        push(@{$curr_node->{Page}}, $next_page);

        $curr_annot_node = $curr_node->{Page}(
            "Title","eq","Annotation:$next_page_title_id");
        
        if ($debug) { print "Curr annot node: ".$curr_annot_node->path."\n";}

        $curr_annot_node->{Template} = {Name => "$template_name"};
        $curr_annot_node = $curr_annot_node->{Template};    
      
        # set up next provenance page
        $next_page = { Title => "Annotation:$next_page_title_id/Provenance" };
        push(@{$curr_node->{Page}}, $next_page);

        $curr_prov_node = $curr_node->{Page}(
            "Title","eq","Annotation:$next_page_title_id/Provenance");
        
        if ($debug) {print "Curr prov node: " . $curr_prov_node->path . "\n"; }

        $curr_prov_node->{Template} = {Name => 'Provenance'};
        $curr_prov_node = $curr_prov_node->{Template};    
        $curr_prov_node->{Field} = {Name => 'Annotation Page'};
        $curr_prov_node->{Field}->content(0,"Annotation:$next_page_title_id");
        $curr_prov_node = $curr_prov_node->back;

        my $field_ct = 0; # counter for field position in pages 

        # grab the Accession ID for the current row of data
        foreach my $key (keys %$row) {
            if ($key eq "Accession ID") {
                $curr_accession_id = $row->{$key};
                if ($debug) {
                    print "* Found Accession ID: $curr_accession_id *\n";
                }
            }
        }
        if (!(defined $curr_accession_id)) {
            die "Error: No Accession ID available. Ending program.\n";
        }
        
        # iterate through the annotation data and assign to <Field> elements
        foreach my $key (keys %$row) {
            if ($debug) { print "$key => " . $row->{$key} . "\n"; }

            # build the annotation page
            $curr_annot_node->{Field}[$field_ct] = {Name => $key};
            $curr_annot_node->{Field}[$field_ct]->content(0,$row->{$key});
            $field_ct++;
            
            # add a corresponding template to the annotation provenance page
            my $next_prov_node = {Name => 'Provenance_Repeater'};
            push(@{$curr_prov_node->{Template}}, $next_prov_node);
            
            # grab the last template you added
            $curr_prov_node = @{$curr_prov_node->{Template}}[-1];

            # assign the relevant provenance field data            
            $curr_prov_node->{Field}[0] = {Name => 'Source'};
            $curr_prov_node->{Field}[0]->content(0,"Source:$SOURCE_TITLE_SEED");
            $curr_prov_node->{Field}[1] = {Name => 'Source Accession ID'};
            $curr_prov_node->{Field}[1]->content(0,$curr_accession_id);
            $curr_prov_node->{Field}[2] = {Name => 'Source Template'};
            $curr_prov_node->{Field}[2]->content(0,$template_name);
            $curr_prov_node->{Field}[3] = {Name => 'Source Field'};
            $curr_prov_node->{Field}[3]->content(0,$key);
            
            $curr_prov_node = $curr_prov_node->back;
        }
        $next_page_title_id++;
    }

    # write out xml doc to a single ImportXML file
    print "Writing data to output file...\n\n";
    $xml->save($output_file);
    $output_data = $xml->data;
}

# xml transformation for GAF data
# (currently uses XML::DOM)
# ---------------------------------------------------------------------------
sub transform_gaf
{
    # define templates and their fields for Provenance-generation
    my $template_field_map = {
        Annotation => [
            'Species Name',
            'Species ID',
            'Gene Symbol',
            'Gene Name',
            'Gene Locus',
            'Gene Type',
            'Chromosome',
            'Has Phenotype'
            ]
        };
    
    my %ref_pubs; # list of non-duplicate publication references
        
    # create new xml doc, write to string
    my $writer = new XML::Writer(
        OUTPUT      => \$output_data,
        DATA_MODE   => 1,
        DATA_INDENT => 4,
        ENCODING    => 'utf-8'
        );

    # clear the $curr_ontology_type var
    $curr_ontology_type = "";
    
    # create root elements
    $writer->xmlDecl;
    $writer->startTag("Pages");

        # create source page
        $writer->startTag("Page",Title=>"Source:$SOURCE_TITLE_SEED");
            $writer->startTag("Template",Name=>"Source");

            # iterate the source hash for element name attribs and vals
            my @pretty_elements;
            foreach my $element (keys %source)
            {
                # split on CamelCase (saves a few lines and was fun to write)
                for ($element) {
                    @pretty_elements = /[A-Z](?:[A-Z]+|[a-z]*)(?=$|[A-Z])/g;
                }
                $writer->dataElement("Field", $source{$element}, Name=>"@pretty_elements");
            }

            $writer->endTag("Template");
        $writer->endTag("Page");
    
        # iterate %annotations
        my $annot_title_count = $ANNOT_TITLE_SEED;
        
        foreach my $annotation (keys %annotations)
        {
            # create annotation page
            $writer->startTag("Page",Title=>"Annotation:$annot_title_count");
                $writer->startTag("Template",Name=>"Annotation");
                    $writer->dataElement("Field", $annotation_universals{"Species Name"}, Name=>"Species Name");
                    $writer->dataElement("Field", $annotation_universals{"Species ID"}, Name=>"Species ID");
                    $writer->dataElement("Field", $annotation, Name=>"Gene Symbol");
                    if (length($annotations{$annotation}{"Gene Name"}) > 0) {
                        $writer->dataElement("Field", $annotations{$annotation}{"Gene Name"}, Name=>"Gene Name");
                    }
                    $writer->dataElement("Field", $annotations{$annotation}{"Gene Locus"}, Name=>"Gene Locus");
                    $writer->dataElement("Field", ucfirst($annotation_universals{"Gene Type"}), Name=>"Gene Type");
                    $writer->dataElement("Field", $annotations{$annotation}{"Chromosome"}, Name=>"Chromosome");
                    $writer->dataElement("Field", "No", Name=>"Has Phenotype");
                $writer->endTag("Template");
            $writer->endTag("Page");
            
            # create gene synonyms page
            if (scalar keys (%{$annotations{$annotation}{'Gene Synonyms'}}) > 0)
            {
                $writer->startTag("Page",Title=>"Annotation:$annot_title_count/Gene Synonyms");
                    $writer->startTag("Template",Name=>"Gene_Synonyms");
                        $writer->dataElement("Field", "Annotation:$annot_title_count", Name=>"Annotation Page");
                    $writer->endTag("Template");
    
                    foreach my $synonym (keys %{$annotations{$annotation}{'Gene Synonyms'}})
                    {
                        $writer->startTag("Template",Name=>"Gene_Synonym_Repeater");
                        $writer->dataElement("Field", $synonym, Name=>"Gene Synonym");    
                        $writer->endTag("Template");
                    }
                $writer->endTag("Page");
            }
            
            # create ont refs page
            if (scalar keys (%{$annotations{$annotation}{"Ontological References"}}) > 0)
            {
                $writer->startTag("Page",Title=>"Annotation:$annot_title_count/Ontologies");
                    $writer->startTag("Template",Name=>"Ontological_References");
                        $writer->dataElement("Field", "Annotation:$annot_title_count", Name=>"Annotation Page");
                    $writer->endTag("Template");
    
                    foreach my $ont_term (keys %{$annotations{$annotation}{"Ontological References"}})
                    {
                        # determine current ontology
                        $curr_ontology_type = (split(':',$ont_term))[0];
                        
                        # gather PMID's for separate Ref Publication page creation (avoid dupes)
                        if (scalar keys (%{$annotations{$annotation}{"Ontological References"}{$ont_term}{"Publications"}}) > 0) {
                            foreach my $pub_term (keys %{$annotations{$annotation}{"Ontological References"}{$ont_term}{"Publications"}})
                            {
                                if (!exists $ref_pubs{$pub_term}) {
                                    $ref_pubs{$pub_term} = { "Annotation:$annot_title_count" => "" };
                                }
                                 else
                                 {
                                     $ref_pubs{$pub_term}{"Annotation:$annot_title_count"} = "";
                                 }
                            }
                        }
                        
                        $writer->startTag("Template",Name=>"$ontology_info{$curr_ontology_type}{name} Reference Repeater");
                            $writer->dataElement("Field", $ontology_info{$curr_ontology_type}{"name"}, Name=>"Ontology");
                            $writer->dataElement("Field", $ont_term, Name=>"Term ID");
                            $writer->dataElement("Field", $annotations{$annotation}{"Ontological References"}{$ont_term}{"Aspect"}, Name=>"Aspect");

                            my @evidence_codes;
                            foreach my $evidence_code (keys %{$annotations{$annotation}{"Ontological References"}{$ont_term}{"Evidence Codes"}}) {
                               push @evidence_codes, $evidence_code; 
                            }
                            $writer->dataElement("Field", join(', ',@evidence_codes), Name=>"Evidence Code"); 

                            my @ary_evidence;
                            foreach my $evidence (keys %{$annotations{$annotation}{"Ontological References"}{$ont_term}{"Evidence"}}) {
                               push @ary_evidence, $evidence; 
                            }
                            if (scalar(@ary_evidence) > 0) {
                                $writer->dataElement("Field", join(', ',@ary_evidence), Name=>"Evidence");
                            }

                            my @ary_assignations;
                            foreach my $assignation (keys %{$annotations{$annotation}{"Ontological References"}{$ont_term}{"Assigned By"}}) {
                               push @ary_assignations, $assignation; 
                            }
                            if (scalar(@ary_assignations) > 0) {
                                $writer->dataElement("Field", join(', ',@ary_assignations), Name=>"Assigned By");
                            }

                        $writer->endTag("Template");
                    }
                $writer->endTag("Page");
            }
            
            # create provenance page
            $writer->startTag("Page",Title=>"Annotation:$annot_title_count/Provenance");
                $writer->startTag("Template",Name=>"Provenance");
                    $writer->dataElement("Field", "Annotation:$annot_title_count", Name=>"Annotation Page");
                $writer->endTag("Template");
                
                # items on the Annotation page
                foreach (@{@$template_field_map{"Annotation"}})
                {
                    # include checks for all locations of annotation data 
                    if (length($annotations{$annotation}{$_}) > 0 
                        || length($annotation_universals{$_}) > 0
                        || $_ eq "Gene Symbol" # this will always be there; it's the key for the annotations
                        || $_ eq "Has Phenotype") {
                        $writer->startTag("Template",Name=>"Provenance_Repeater");
                            $writer->dataElement("Field", $_, Name=>"Source Field or Object");
                            $writer->dataElement("Field", "Annotations", Name=>"Source Category");
                            $writer->dataElement("Field", $annotations{$annotation}{"Accession ID"}, Name=>"Source Accession ID");
                            $writer->dataElement("Field", "Source:$SOURCE_TITLE_SEED", Name=>"Source");
                        $writer->endTag("Template");
                    }
                }

                # items on the Gene Synonyms subpage
                if (scalar keys (%{$annotations{$annotation}{'Gene Synonyms'}}) > 0)
                {
                    foreach my $synonym (keys %{$annotations{$annotation}{'Gene Synonyms'}})
                    {
                        $writer->startTag("Template",Name=>"Provenance_Repeater");
                            $writer->dataElement("Field", $synonym, Name=>"Source Field or Object");    
                            $writer->dataElement("Field", "Gene Synonyms", Name=>"Source Category");
                            $writer->dataElement("Field", $annotations{$annotation}{"Accession ID"}, Name=>"Source Accession ID");
                            $writer->dataElement("Field", "Source:$SOURCE_TITLE_SEED", Name=>"Source");
                        $writer->endTag("Template");
                    }
                }

                # items on the Ontologies subpage
                if (scalar keys (%{$annotations{$annotation}{"Ontological References"}}) > 0)
                {
                    foreach my $ont_term (keys %{$annotations{$annotation}{"Ontological References"}})
                    {
                        $writer->startTag("Template",Name=>"Provenance_Repeater");
                            $writer->dataElement("Field", $ont_term, Name=>"Source Field or Object");    
                            $writer->dataElement("Field", "Ontological References", Name=>"Source Category");
                            $writer->dataElement("Field", $annotations{$annotation}{"Accession ID"}, Name=>"Source Accession ID");
                            $writer->dataElement("Field", "Source:$SOURCE_TITLE_SEED", Name=>"Source");
                        $writer->endTag("Template");
                    }
                }

                    
            $writer->endTag("Page");
            
            $annot_title_count++;
        }

    if ($debug) { print "...<DEBUG: \%ref_pubs>\n"
        . Dumper(\%ref_pubs) . "\n\n"; }
        
    # create Reference Publication pages
    my $pub_title_count = $PUB_TITLE_SEED;

    if (scalar keys (%ref_pubs) > 0) {
    
        foreach my $ref_pub (keys %ref_pubs)
        {
            $writer->startTag("Page",Title=>"Publication:$pub_title_count");
                $writer->startTag("Template",Name=>"Reference_Publication");
                    $writer->dataElement("Field", "PubMed", Name=>"Publication"); # TODO: replace hard-coded pub type w/ something dynamic
                    $writer->dataElement("Field", $ref_pub, Name=>"Publication ID");
                    my @annot_refs; # holds com-del string of annotation page references
                    foreach my $annot_ref (keys %{$ref_pubs{$ref_pub}})
                    {
                        push @annot_refs, $annot_ref;
                    }
                    $writer->dataElement("Field", join(',',@annot_refs), Name=>"Annotation References");
                $writer->endTag("Template");
            $writer->endTag("Page");
            
            $pub_title_count++;
        }
    }
        
    # close doc
    $writer->endTag("Pages");
    $writer->end();

    # write out the file
    open(OUTPUT_FILE,">$output_file");
    print OUTPUT_FILE $output_data;
    close OUTPUT_FILE;
}


# loop through the hash and build annotation data and source xml doc
# ---------------------------------------------------------------------------
sub write_xml
{
    print "Transforming " . uc($file_type) . " data to SMW/SF XML...\n\n";

    switch ($file_type) {
        case ('csv' || 'tab') { transform_generic(); }
        case 'gaf' { transform_gaf(); }
    }
}


# print the transformed data (as xml)
# ---------------------------------------------------------------------------
sub show_output
{
    print "[XML]\n";
    print $output_data;
    print "\n";
}

# ---------------------------------------------------------------------------
# main
# ---------------------------------------------------------------------------

init;
import_data;
if ($verbose) { show_input; }
write_xml();
if ($verbose) { show_output; }

exit;

