#!/usr/local/bin/perl ############################################################################### # Program : ProcessProject.cgi # Author : Eric Deutsch # $Id$ # # Description : This CGI program that allows users to submit a processing # job to process a set of experiments in a project. # ############################################################################### ############################################################################### # Get the script set up with everything it will need ############################################################################### use strict; use lib qw (../../lib/perl); use vars qw ($q $sbeams $sbeamsMA $dbh $current_contact_id $current_username $current_work_group_id $current_work_group_name $current_project_id $current_project_name $TABLE_NAME $PROGRAM_FILE_NAME $CATEGORY $DB_TABLE_NAME $PK_COLUMN_NAME @MENU_OPTIONS); use DBI; #use CGI; use CGI::Carp qw(fatalsToBrowser croak); use POSIX; use SBEAMS::Connection qw($q); use SBEAMS::Connection::Settings; use SBEAMS::Connection::Tables; use SBEAMS::Microarray; use SBEAMS::Microarray::Settings; use SBEAMS::Microarray::Tables; use lib "/net/arrays/Pipeline/tools/lib"; require "QuantitationFile.pl"; #$q = new CGI; $sbeams = new SBEAMS::Connection; $sbeamsMA = new SBEAMS::Microarray; $sbeamsMA->setSBEAMS($sbeams); ############################################################################### # Global Variables ############################################################################### main(); ############################################################################### # Main Program: #x # Call $sbeams->InterfaceEntry with pointer to the subroutine to execute if # the authentication succeeds. ############################################################################### sub main { #### Do the SBEAMS authentication and exit if a username is not returned exit unless ($current_username = $sbeams->Authenticate()); #### Read in the default input parameters my %parameters; my $n_params_found = $sbeams->parse_input_parameters(q=>$q,parameters_ref=>\%parameters); #$sbeams->printDebuggingInfo($q); #### Process generic "state" parameters before we start $sbeams->processStandardParameters(parameters_ref=>\%parameters); #### Print the header, do what the program does, and print footer $sbeamsMA->printPageHeader(); processRequests(); $sbeamsMA->printPageFooter(); } # end main ############################################################################### # Process Requests # # Test for specific form variables and process the request # based on what the user wants to do. ############################################################################### sub processRequests { $current_username = $sbeams->getCurrent_username; $current_contact_id = $sbeams->getCurrent_contact_id; $current_work_group_id = $sbeams->getCurrent_work_group_id; $current_work_group_name = $sbeams->getCurrent_work_group_name; $current_project_id = $sbeams->getCurrent_project_id; $current_project_name = $sbeams->getCurrent_project_name; $dbh = $sbeams->getDBHandle(); # Enable for debugging if (0==1) { print "Content-type: text/html\n\n"; my ($ee,$ff); foreach $ee (keys %ENV) { print "$ee =$ENV{$ee}=
\n"; } foreach $ee ( $q->param ) { $ff = $q->param($ee); print "$ee =$ff=
\n"; } } #### Decide where to go based on form values if ($q->param('PROCESS')) {createFile();} elsif($q->param('FINALIZE')) {submitJob();} else { printEntryForm();} } # end processRequests ############################################################################### # Print Entry Form ############################################################################### sub printEntryForm { ## Define Standard Variables my %parameters; my $element; my $sql_query; my (%url_cols,%hidden_cols); my $CATEGORY="Welcome to the Data Processing Pipeline!"; $parameters{project_id} = $sbeams->getCurrent_project_id(); $sbeams->printUserContext(); print qq~

$CATEGORY



~; if ($parameters{project_id} > 0) { $sql_query = qq~ SELECT A.array_id,A.array_name, ARSM1.name AS 'Sample1Name',D1.dye_name AS 'sample1_dye', ARSM2.name AS 'Sample2Name',D2.dye_name AS 'sample2_dye', AQ.array_quantitation_id,AQ.data_flag AS 'quan_flag', AQ.stage_location,AL.source_filename AS 'key_file' FROM $TBMA_ARRAY_REQUEST AR LEFT JOIN $TBMA_ARRAY_REQUEST_SLIDE ARSL ON ( AR.array_request_id = ARSL.array_request_id ) LEFT JOIN $TBMA_ARRAY_REQUEST_SAMPLE ARSM1 ON ( ARSL.array_request_slide_id = ARSM1.array_request_slide_id AND ARSM1.sample_index=0) LEFT JOIN $TBMA_LABELING_METHOD LM1 ON ( ARSM1.labeling_method_id = LM1.labeling_method_id ) LEFT JOIN $TBMA_DYE D1 ON ( LM1.dye_id = D1.dye_id ) LEFT JOIN $TBMA_ARRAY_REQUEST_SAMPLE ARSM2 ON ( ARSL.array_request_slide_id = ARSM2.array_request_slide_id AND ARSM2.sample_index=1) LEFT JOIN $TBMA_LABELING_METHOD LM2 ON ( ARSM2.labeling_method_id = LM2.labeling_method_id ) LEFT JOIN $TBMA_DYE D2 ON ( LM2.dye_id = D2.dye_id ) LEFT JOIN $TBMA_ARRAY A ON ( A.array_request_slide_id = ARSL.array_request_slide_id ) LEFT JOIN $TBMA_ARRAY_LAYOUT AL ON ( A.layout_id = AL.layout_id ) LEFT JOIN $TBMA_ARRAY_SCAN ASCAN ON ( A.array_id = ASCAN.array_id ) LEFT JOIN $TBMA_ARRAY_QUANTITATION AQ ON ( ASCAN.array_scan_id = AQ.array_scan_id ) WHERE AR.project_id=$parameters{project_id} AND AQ.array_quantitation_id IS NOT NULL AND AR.record_status != 'D' AND A.record_status != 'D' AND ASCAN.record_status != 'D' AND AQ.record_status != 'D' AND AQ.data_flag != 'BAD' ORDER BY A.array_name ~; my $base_url = "$CGI_BASE_DIR/Microarray/ManageTable.cgi?TABLE_NAME=MA_"; %url_cols = ('array_name' => "${base_url}array&array_id=%0V", 'quan_flag' => "${base_url}array_quantitation&array_quantitation_id=%6V", ); %hidden_cols = ('array_id' => 1, 'array_quantitation_id' => 1, ); } my $sth = $dbh->prepare("$sql_query") or croak $dbh->errstr; my $rv = $sth->execute or croak $dbh->errstr; my @rows; my @row; while (@row = $sth->fetchrow_array) { my @temprow = @row; push(@rows,\@temprow); } $sth->finish; my @group_names; my %group_names_hash; my @slide_group_names; my @slide_rowrefs; my @slide_directions; foreach $element (@rows) { my $sample1name = $$element[2]; my $sample2name = $$element[4]; my $forcondition = "${sample1name}_vs_${sample2name}"; my $revcondition = "${sample2name}_vs_${sample1name}"; my $thiscondition; my $direction = ""; if (defined($group_names_hash{$forcondition})) { $direction = "f"; $thiscondition = $forcondition; } if (defined($group_names_hash{$revcondition})) { $direction = "r"; $thiscondition = $revcondition; } unless ($direction) { $direction = "f"; $thiscondition = $forcondition; push(@group_names,$thiscondition); $group_names_hash{$thiscondition}=$thiscondition; } push(@slide_group_names,$thiscondition); push(@slide_rowrefs,$element); push(@slide_directions,$direction); } my $group; my $error_flag = 0; my ($quantitation_file,$qf_status); my (@ERRORS,@command_file); my (@results,@parts); my @project_outline; foreach $group (@group_names) { my $row_counter=0; my $first_flag=1; my $channel_direction = ""; foreach $element (@slide_group_names) { if ($element eq $group) { if ($first_flag) { my $cmd_line = "$group ${$slide_rowrefs[$row_counter]}[9] EXP"; push (@command_file,$cmd_line); $first_flag=0; } #### Verify that the data file is okay $quantitation_file = ${slide_rowrefs[$row_counter]}[8]; my $sample1_dye = ${slide_rowrefs[$row_counter]}[3]; my $sample2_dye = ${slide_rowrefs[$row_counter]}[5]; $qf_status = ""; #### If the data file is okay if ( -e $quantitation_file ) { $qf_status = "    --- ". "File exists"; #### Run a parse program on it to see which channel is which dye my %quantitation_data = readQuantitationFile(inputfilename=>"$quantitation_file", headeronly=>1); unless ($quantitation_data{success}) { $qf_status = "    --- ". "$quantitation_data{error_msg}"; } else { #### Pull out the channel information my @channels = @{$quantitation_data{channels}}; my $channel; ### my $number_of_channels = scalar(@channels); my $first_channel = "ch1"; my $other_channel = "ch".($number_of_channels/2 + 1); ### #### Loop over each channel foreach $channel (@channels) { @parts = ($channel->{channel_label},$channel->{fluorophor}); $parts[1] =~ /(\d+)/; my $number_part = $1; my $match_flag = 0; if ($sample1_dye =~ /$number_part/) { $match_flag = 1; if ($parts[0] eq $first_channel) { $channel_direction = "f"; } if ($parts[0] eq $other_channel) { $channel_direction = "r"; } } if ($sample2_dye =~ /$number_part/) { if ($match_flag) { print "Whoah! Double match!
\n"; } $match_flag = 2; if ($parts[0] eq $first_channel) { $channel_direction = "r"; } if ($parts[0] eq $other_channel) { $channel_direction = "f"; } } unless ($match_flag) { print "Unable to match '$parts[1]' with either dye ($sample1_dye and $sample2_dye).
\n"; } } # endforeach if ($channel_direction eq "r") { $slide_directions[$row_counter] =~ tr/fr/rf/; } else { #keep direction the same } $qf_status = "    --- ". "File verified"; } # endelse #### If the data file is not found } else { $error_flag++; $qf_status = "    --- ". "FILE MISSING"; push(@ERRORS,"Unable to find file $quantitation_file"); } #### Print out the quantitation file row my $cmd_line = "$quantitation_file ". $slide_directions[$row_counter]; push (@command_file,$cmd_line); print "$quantitation_file ". "$slide_directions[$row_counter] ". "$qf_status
\n"; } $row_counter++; } } ####################################### ### Start Pipeline Customization ### ####################################### ## Print all the javascript! printProjectJavascript(); # Table for entire form to sit in print qq~
~; print qq~ $LINESEPARATOR
Forward Files:

Reverse Files:




Available Files:
~; # Removed buttons- place within above print statement to activate. #

# #

# #

print qq~ $LINESEPARATOR
Optional Pipeline Configurations
- Default values used if not selected
- Click on red question marks for help text

Pre-process:  Help
 Use Base Value:    Help
 Saturating Intensity:    Help
 Scale to Value:    Help

Normalizing Method: Help
 Median   None
 Generate debug file

Merge Replicates:  Help
 Minimum <num> replicate measurements:     Help
 Use general list of bad genes
 Select local file of bad genes:
      Help
*must specify full path of bad gene file
 Filter Outliers  Help

VERA/SAM: Help
 Use VERA and SAM  Help
 Cease Optimization when changes per step are less than:  
 Generate file showing how parameters converge
 Generate debug file
 Use your own error model  

Merge Conditions:  Help
-only used with more than one condition
Conditions:
  

  

Unused Conditions:


   

 Lambda >= <num>  
 Ratio >= <num>  
 Standard Devation >=<num>  
 Gene represented at least <num> times  

Miscellaneous
 Create clone file (adds info from key file to .sig file)
 Email notification - Type comma-separated email addresses (\@systemsbiology is implied, unless otherwise specified):
   

$LINESEPARATOR


~; } # End of table form sits in print qq~

~; #################################### ### End Pipeline Customization ### #################################### sub createFile{ my @forward_files = $q->param('forwardSelectionList'); my @reverse_files = $q->param('reverseSelectionList'); #Preprocess values: my $base = $q->param('preprocessBase'); my $baseValue = $q->param('preprocessBaseValue'); my $sat = $q->param('preprocessSat'); my $satValue = $q->param('preprocessSatValue'); my $scale = $q->param('preprocessScale'); my $scaleValue = $q->param('preprocessScaleValue'); my $norm = $q->param('normalization'); my $preprocDebug = $q->param('preprocessDebug'); #MergeReps values: my $replicate = $q->param('errorModel'); my $replicateValue= $q->param('errorModelValue'); my $exclude = 0; my $excludeFile; my $defaultFile = "/net/arrays/Pipeline/tools/etc/excluded_gene_names"; my $temp = $q->param('excludeGenes'); if ($temp){ $exclude = $temp; $excludeFile = $defaultFile; } else{ $temp = $q->param('excludeLocalGenes'); if ($temp){ $exclude = $temp; $excludeFile = $q->param('excludeFile'); } } my $filter = $q->param('filterGenes'); #VERA values: my $useVandS = $q->param('useVERAandSAM'); my $veraFlag = $q->param('veraCrit'); my $veraValue = $q->param('veraCritValue'); my $veraEvolFile = $q->param('veraEvolFlag'); my $veraDebug = $q->param('debugFlag'); my $veraModelFlag= $q->param('modelFlag'); my $veraModelFile= $q->param('modelFile'); #mergeConds values my @merge_files = $q->param('mergeCondsList'); my $lambdaFlag = $q->param('lambdaFlag'); my $lambdaValue = $q->param('lambdaValue'); my $ratioFlag = $q->param('ratioFlag'); my $ratioValue = $q->param('ratioValue'); my $stdevFlag = $q->param('stdevFlag'); my $stdevValue = $q->param('stdevValue'); my $repFlag = $q->param('repFlag'); my $repValue = $q->param('repValue'); #Miscellaneous values my $postSam = $q->param('postSam'); my $notify = $q->param('notify'); my $addresses = $q->param('addresses'); #File Creating Variables my $project_id = $q->param('project_id'); my $printLine; my $BASE_DIR = "/net/arrays/Pipeline"; my $OUTPUT_DIR = "$BASE_DIR/output"; print qq~

This is the plan file that will be submitted.
If you care to manually alter the file, you may do so in this textbox.
[I will add a link to a page on how to manually alter files here]
Click "Submit to Pipeline" to continue


~; } ############################################################################### # submit Job ############################################################################### sub submitJob { my $command_file_content = $q->param('planFileText'); my $project_id = $q->param('id'); my ($sec,$min,$hour,$mday,$mon,$year) = localtime(time); my $timestr = strftime("%Y%m%d.%H%M%S",$sec,$min,$hour,$mday,$mon,$year); my $plan_filename = "job$timestr.planFile"; my $control_filename = "job$timestr.control"; my $log_filename = "job$timestr.log"; my $queue_dir = "/net/arrays/Pipeline/queue"; #### Verify that the plan file does not already exist if ( -e $plan_filename ) { print qq~ Wow, the job filename '$plan_filename' already exists!
Please go back and click PROCESS again. If this happens twice in a row, something is very wrong. Contact edeutsch.
\n ~; return; } #### Write the plan file print "Writing processing plan file '$plan_filename'
\n"; open(PLANFILE,">$queue_dir/$plan_filename") || croak("Unable to write to file '$queue_dir/$plan_filename'"); print PLANFILE $command_file_content; close(PLANFILE); #### Write the control file print "Writing job control file '$control_filename'
\n"; open(CONTROLFILE,">$queue_dir/$control_filename") || croak("Unable to write to file '$queue_dir/$control_filename'"); print CONTROLFILE "submitted_by=$current_username\n"; print CONTROLFILE "project_id=$project_id\n"; print CONTROLFILE "status=SUBMITTED\n"; close(CONTROLFILE); print "Done!

\n"; print qq~ The plan and job control files have been successfully written to the queue. Your job will be processed in the order received. You can see the log file of your job by clicking on the link below:

Well, theres no link yet, but paste this into a unix window:

cd /net/arrays/Pipeline/output/project_id/$project_id
if ( -e $log_filename ) tail -f $log_filename



~; } ############################################################################### # printProjectJavascript ############################################################################### sub printProjectJavascript { print qq~ ~; return; }