--- loncom/interface/statistics/lonsurveyreports.pm	2005/03/01 22:04:55	1.3
+++ loncom/interface/statistics/lonsurveyreports.pm	2006/05/30 12:46:50	1.13
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonsurveyreports.pm,v 1.3 2005/03/01 22:04:55 matthew Exp $
+# $Id: lonsurveyreports.pm,v 1.13 2006/05/30 12:46:50 www Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -27,23 +27,29 @@
 package Apache::lonsurveyreports;
 
 use strict;
-use Apache::lonnet();
+use Apache::lonnet;
 use Apache::loncommon();
 use Apache::lonhtmlcommon();
 use Apache::loncoursedata();
 use Apache::lonstatistics;
 use Apache::lonlocal;
 use Apache::lonstathelpers;
+use Spreadsheet::WriteExcel;
 use HTML::Entities();
 use Time::Local();
+use lib '/home/httpd/lib/perl/';
+use LONCAPA;
+ 
 
-my @SubmitButtons = ({ name => 'PrevProblem',
+my @SubmitButtons = (
+                     { name => 'break'},
+                     { name => 'PrevProblem',
                        text => 'Previous Survey' },
                      { name => 'NextProblem',
                        text => 'Next Survey' },
-                     { name => 'break'},
                      { name => 'SelectAnother',
-                       text => 'Choose a different Survey Problem' },
+                       text => 'Choose a different Survey' },
+                     { name => 'break'},
                      { name => 'Generate',
                        text => 'Generate Report'},
                      );
@@ -74,8 +80,9 @@ sub BuildSurveyReportsPage {
         &Apache::lonstathelpers::manage_caches($r,'Statistics','stats_status');
     $r->rflush();
     #
-    if (exists($ENV{'form.problemchoice'}) && 
-        ! exists($ENV{'form.SelectAnother'})) {
+    if (exists($env{'form.problemchoice'}) && 
+        ! exists($env{'form.SelectAnother'})) {
+        $r->print(' 'x3);
         foreach my $button (@SubmitButtons) {
             if ($button->{'name'} eq 'break') {
                 $r->print("<br />\n");
@@ -90,30 +97,34 @@ sub BuildSurveyReportsPage {
         }
         #
         $r->print('<hr />');
+        $r->print('<h4>'.
+                  &Apache::lonlocal::locallocaltime(time).','.
+                  &Apache::lonstatistics::section_and_enrollment_description().
+                  '</h4>');
         $r->rflush();
         #
         # Determine which problem we are to analyze
         my $current_problem = &Apache::lonstathelpers::get_target_from_id
-            ($ENV{'form.problemchoice'});
+            ($env{'form.problemchoice'});
         #
         my ($navmap,$prev,$curr,$next) = 
             &Apache::lonstathelpers::get_prev_curr_next($current_problem,
                                                         '.',
                                                         'part_survey',
                                                         );
-        if (exists($ENV{'form.PrevProblem'}) && defined($prev)) {
+        if (exists($env{'form.PrevProblem'}) && defined($prev)) {
             $current_problem = $prev;
-        } elsif (exists($ENV{'form.NextProblem'}) && defined($next)) {
+        } elsif (exists($env{'form.NextProblem'}) && defined($next)) {
             $current_problem = $next;
         } else {
             $current_problem = $curr;
         }
         #
         # Store the current problem choice and send it out in the form
-        $ENV{'form.problemchoice'} = 
+        $env{'form.problemchoice'} = 
             &Apache::lonstathelpers::make_target_id($current_problem);
         $r->print('<input type="hidden" name="problemchoice" value="'.
-                  $ENV{'form.problemchoice'}.'" />');
+                  $env{'form.problemchoice'}.'" />');
         #
         if (! defined($current_problem->{'resource'})) {
             $r->print('resource is undefined');
@@ -121,11 +132,21 @@ sub BuildSurveyReportsPage {
             my $resource = $current_problem->{'resource'};
             $r->print('<h1>'.$resource->compTitle.'</h1>');
             $r->print('<h3>'.$resource->src.'</h3>');
-            $r->print(&Apache::lonstathelpers::render_resource($resource));
+            if ($env{'form.renderprob'} eq 'true') {
+                $r->print(&Apache::lonstathelpers::render_resource($resource));
+            }
             $r->rflush();
             my %Data = &Apache::lonstathelpers::get_problem_data
                 ($resource->src);
-            &make_HTML_report($r,$current_problem,\%Data,\@Students);
+            &compile_student_answers($r,$current_problem,\%Data,\@Students);
+            if ($env{'form.output'} eq 'HTML' || 
+                ! defined($env{'form.output'})) {
+                &make_HTML_report($r,$current_problem,\%Data,\@Students);
+            } elsif ($env{'form.output'} eq 'Excel') {
+                &make_Excel_report($r,$current_problem,\%Data,\@Students);
+            } elsif ($env{'form.output'} eq 'TXT') {
+                &make_text_report($r,$current_problem,\%Data,\@Students);
+            }
         }
         $r->print('<hr />');
     } else {
@@ -179,16 +200,16 @@ sub SurveyProblemSelector {
               respid=>undef,
               resptype=>undef});
         my $checked = '';
-        if ($ENV{'form.problemchoice'} eq $value) {
+        if ($env{'form.problemchoice'} eq $value) {
             $checked = 'checked ';
         }
         my $link = $problem->{'res'}->src.
-            '?symb='.&Apache::lonnet::escape($problem->{'res'}->symb);
+            '?symb='.&escape($problem->{'res'}->symb);
         $Str .= '<tr><td>'.
             '<input type="radio" name="problemchoice" id="'.$id.'" '.
                    'value="'.$value.'" '.$checked.'/>'.'</td>'.
             '<td><nobr>'.
-            '<label for="'.$id.'">'.$problem->{'res'}->compTitle.'('.$problem->{'seq'}->compTitle.')'.'</lablel>'.
+            '<label for="'.$id.'">'.$problem->{'res'}->compTitle.'('.$problem->{'seq'}->compTitle.')'.'</label>'.
             ('&nbsp;'x2).
             qq{<a target="preview" href="$link">view</a>}.'</td></tr>'.$/;
     }
@@ -203,9 +224,13 @@ sub SurveyProblemSelector {
 ##
 #########################################################
 #########################################################
-sub Compile_Student_Answers {
-    my ($problem,$ProblemData,$Students) = @_;
+sub compile_student_answers {
+    my ($r,$problem,$ProblemData,$Students) = @_;
     my $resource = $problem->{'resource'};
+    my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
+        ($r,'Processing Student Submissions',
+         'Processing Student Submissions', 
+         scalar(@$Students),'inline',undef,'Statistics','stats_status');
     foreach my $student (@$Students) {
         foreach my $partid (@{$resource->parts}) {
             my @response_ids   = $resource->responseIds($partid);
@@ -227,13 +252,13 @@ sub Compile_Student_Answers {
                     foreach my $response (@responses) {
                         my ($foilid,$option) = 
                             map { 
-                                &Apache::lonnet::unescape($_); 
+                                &unescape($_); 
                             } split('=',$response);
                         $data->{'foil_count'}->{$foilid}++;
                         $data->{'foil_responses'}->{$foilid}->{$option}++;
                     }
                 } elsif ($resptype =~ /^(radiobutton)$/) {
-                    my ($foil,$value) = map { &Apache::lonnet::unescape($_); } split('=',$student_response);
+                    my ($foil,$value) = map { &unescape($_); } split('=',$student_response);
                     $value += 1;  # explicitly increment it...
                     $data->{'foil_responses'}->{$foil}++;
                     $data->{'foil_values'}->{$value}++;
@@ -246,7 +271,188 @@ sub Compile_Student_Answers {
                 }
             }
         }
+        &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+                                                 'last student');
+    }
+    &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
+    return;
+}
+
+
+#########################################################
+#########################################################
+##
+## make_text_report
+##
+#########################################################
+#########################################################
+sub make_text_report {
+    my ($r,$problem,$problem_data,$students) = @_;
+    my ($file,$filename) = &Apache::loncommon::create_text_file($r,'txt');
+    if (! defined($file)) { return '';}
+    $r->print('<script>'.
+              'window.document.Statistics.stats_status.value="'.
+              &mt('Building text document.').
+              '";</script>');
+    my $resource = $problem->{'resource'};
+    print $file $resource->compTitle.$/;
+    print $file &Apache::lonstatistics::section_and_enrollment_description().
+        ' '.&mt('Generated on [_1]',&Apache::lonlocal::locallocaltime(time)).
+        $/;
+    my $something_has_been_output = 0;
+    foreach my $partid (@{$resource->parts}) {
+        my @response_ids   = $resource->responseIds($partid);
+        my @response_types = $resource->responseType($partid);
+        for (my $i=0;$i<=$#response_ids;$i++) {
+            my $respid   = $response_ids[$i];
+            my $resptype = $response_types[$i];
+            my $data = $problem->{'responsedata'}->{$partid}->{$respid};
+            if (exists($data->{'responses'}) && 
+                ref($data->{'responses'}) eq 'ARRAY') {
+                # Essay type response
+                print $file ('-'x40).$/;
+                print $file 
+                    $resource->part_display($partid).', '.$respid.':'.$resptype.$/;
+                foreach my $submission (@{$data->{'responses'}}) {
+                    print $file ('-'x20).$/;
+                    $submission =~ s/(\\r\\n|\\n)/\n/g;
+                    $submission =~ s/\\(\'|\"|\`)/$1/g;
+                    print $file $submission.$/.$/;
+                    $something_has_been_output=1;
+                }
+            }
+        }
     }
+    close($file);
+    if($something_has_been_output) {
+        $r->print('<p><a href="'.$filename.'">'.
+                  &mt('Your text file.').
+                  '</a></p>'."\n");
+        $r->print('<script>'.
+                  'window.document.Statistics.stats_status.value="'.
+               &mt('Done compiling text file.  See link below to download.').
+                  '";</script>');
+    } else {
+        $r->print('<h2>'.
+                  &mt("There is no essay or string response data to output for this survey.").'</h2>');
+    }
+    $r->rflush();
+    return;
+}
+
+
+#########################################################
+#########################################################
+##
+## make_Excel_report
+##
+#########################################################
+#########################################################
+sub make_Excel_report {
+    my ($r,$problem,$problem_data,$students) = @_;
+    my ($workbook,$filename,$format) = &Apache::loncommon::create_workbook($r);
+    if (! defined($workbook)) { return '';}
+    $r->print('<script>'.
+              'window.document.Statistics.stats_status.value="'.
+              &mt('Building spreadsheet.').
+              '";</script>');
+    my $worksheet  = $workbook->addworksheet('Survey Reports');
+    #
+    my $rows_output=0;
+    $worksheet->write($rows_output++,0,
+                    $env{'course.'.$env{'request.course.id'}.'.description'},
+                      $format->{'h1'});
+    $rows_output++;
+    #
+    my $resource = $problem->{'resource'};
+    $worksheet->write($rows_output++,0,$resource->compTitle,$format->{'h2'});
+    foreach my $partid (@{$resource->parts}) {
+        my @response_ids   = $resource->responseIds($partid);
+        my @response_types = $resource->responseType($partid);
+        for (my $i=0;$i<=$#response_ids;$i++) {
+            my $respid   = $response_ids[$i];
+            my $resptype = $response_types[$i];
+            my $data = $problem->{'responsedata'}->{$partid}->{$respid};
+            my $cols_output=0;
+            $worksheet->write($rows_output,$cols_output++,
+                              $resource->part_display($partid),$format->{'h3'});
+            $worksheet->write($rows_output,$cols_output++,
+                              'Response '.$respid.', '.$resptype,
+                              $format->{'h3'});
+            $rows_output++;
+            if (exists($data->{'responses'}) && 
+                ref($data->{'responses'}) eq 'ARRAY') {
+                my $warned_about_size = 0;
+                foreach my $data (@{$data->{'responses'}}) {
+                    if (length($data) > 255 && ! $warned_about_size) {
+                        $r->print('<p>'.
+                                  &mt('[_1]:[_2] responses to  [_3] may be too long to fit Excel spreadsheet.',
+                                      $resource->compTitle,
+                                      $resource->part_display($partid),
+                                      $respid).
+                                  '</p>');
+                        $r->rflush();
+                        $warned_about_size=1;
+                    }
+                    $worksheet->write($rows_output++,0,$data);
+                }
+            } elsif (exists($data->{'foil_count'}) && 
+                     exists($data->{'foil_responses'})) {
+                my $respdata = $problem_data->{$partid.'.'.$respid};
+                my @rowdata = ('Foil Name','Foil Text','Option',
+                               'Frequency');
+                $worksheet->write_row($rows_output++,0,
+                                      \@rowdata,$format->{'h4'});
+                #
+                my @foils = sort(keys(%{$data->{'foil_responses'}}));
+                foreach my $foilid (@foils) {
+                    my $foil_count = $data->{'foil_count'}->{$foilid};
+                    my $foiltext = $respdata->{'_Foils'}->{$foilid}->{'text'};
+                    my $foilname = $respdata->{'_Foils'}->{$foilid}->{'name'};
+                    $foiltext = &HTML::Entities::decode($foilname);
+                    my $cols_output=0;
+                    $worksheet->write($rows_output,$cols_output++,$foilname);
+                    $worksheet->write($rows_output,$cols_output++,$foiltext);  
+                    my $option_start_col = $cols_output;
+                    #
+                    foreach my $option (sort(@{$respdata->{'_Options'}})){
+                        $cols_output= $option_start_col;
+                        $worksheet->write($rows_output,$cols_output++,
+                                          $option);
+                        my $count=
+                            $data->{'foil_responses'}->{$foilid}->{$option};
+                        $worksheet->write($rows_output,$cols_output++,$count);
+                        $rows_output++;
+                    }
+                }
+            } elsif (exists($data->{'_count'})      && 
+                     exists($data->{'foil_values'}) && 
+                     exists($data->{'map'})) {
+                my $respdata = $problem_data->{$partid.'.'.$respid};
+                my @rowdata = ('Foil Name','Foil Text','Frequency');
+                $worksheet->write_row($rows_output++,0,
+                                      \@rowdata,$format->{'h4'});
+                foreach my $value (sort(keys(%{$data->{'foil_values'}}))) {
+                    undef(@rowdata);
+                    my $foilid = $data->{'map'}->{$value};
+                    push(@rowdata,$respdata->{'_Foils'}->{$foilid}->{'name'});
+                    push(@rowdata,$respdata->{'_Foils'}->{$foilid}->{'text'});
+                    push(@rowdata,$data->{'foil_values'}->{$value});
+                    $worksheet->write_row($rows_output++,0,\@rowdata);
+                }
+            }  
+            $rows_output++;
+        }    #response ids
+    }    # partids
+    $workbook->close();
+    $r->print('<p><a href="'.$filename.'">'.
+              &mt('Your Excel spreadsheet.').
+              '</a></p>'."\n");
+    $r->print('<script>'.
+              'window.document.Statistics.stats_status.value="'.
+              &mt('Done compiling spreadsheet.  See link below to download.').
+              '";</script>');
+    $r->rflush();
     return;
 }
 
@@ -259,8 +465,6 @@ sub Compile_Student_Answers {
 #########################################################
 sub make_HTML_report {
     my ($r,$problem,$ProblemData,$Students) = @_;
-    &Compile_Student_Answers($problem,$ProblemData,$Students);
-    # &output_hash('',$ProblemData);
     my $resource = $problem->{'resource'};
     foreach my $partid (@{$resource->parts}) {
         my @response_ids   = $resource->responseIds($partid);
@@ -282,7 +486,8 @@ sub make_HTML_report {
             $Str .= '<tr>'.
                 '<td><b>'.&mt('Total').'</b></td>'.
                 '<td>'.$data->{'_count'}.'</td>'.
-                '<td>'.&mt('Part [_1], Response [_2]',$partid,$respid).'</td>'.
+                '<td>'.&mt('Part [_1], Response [_2]',
+			   $resource->part_display($partid),$respid).'</td>'.
                 '</tr>';
             if (exists($data->{'responses'}) && 
                 ref($data->{'responses'}) eq 'ARRAY') {
@@ -344,7 +549,6 @@ sub make_HTML_report {
                     my $foilid = $data->{'map'}->{$value};
                     my $foiltext = $ProblemData->{$partid.'.'.$respid}->{'_Foils'}->{$foilid}->{'text'};
                     my $foilname = $ProblemData->{$partid.'.'.$respid}->{'_Foils'}->{$foilid}->{'name'};
-                    $sum = $value * $data->{'foil_values'}->{$value};
                     $tmp .= '<tr>'.
                         '<td>'.$foilname.'</td>'.
                         '<td>'.$foiltext.'</td>'.
@@ -390,23 +594,58 @@ sub CreateInterface {
     ##
     ## Environment variable initialization
     my $Str = '';
-    $Str .= &Apache::lonhtmlcommon::breadcrumbs
-        (undef,'Student Submission Reports');
+    my $output_selector = '<select name="output" size="5">'.$/;
+    if (! exists($env{'form.output'})) {
+        $env{'form.output'} = 'HTML';
+    }
+    foreach my $output_format ( {name=>'HTML',text=>&mt("HTML") },
+                                {name=>'Excel',text=>&mt("Excel") },
+                                {name=>'TXT',text=>&mt("Text (essays only)") },
+                                ) {
+        $output_selector.='<option value="'.$output_format->{'name'}.'"';
+        if ($env{'form.output'} eq $output_format->{'name'}) {
+            $output_selector.=' selected';
+        }
+        $output_selector.= '>'.$output_format->{'text'}.'</option>'.$/;
+    }
+    $output_selector .= '</select>'.$/;
+    $Str .= &Apache::lonhtmlcommon::breadcrumbs('Student Submission Reports');
     $Str .= '<p>';
     $Str .= '<table cellspacing="5">'."\n";
     $Str .= '<tr>';
-    $Str .= '<td align="center"><b>'.&mt('Sections').'</b></td>';
-    $Str .= '<td align="center"><b>'.&mt('Enrollment Status').'</b></td>';
+    $Str .= '<th>'.&mt('Sections').'</th>';
+    $Str .= '<th>'.&mt('Groups').'</th>';
+    $Str .= '<th>'.&mt('Enrollment Status').'</th>';
+    $Str .= '<th>'.&mt('Output Format').'</th>';
+    $Str .= '<th>'.'&nbsp;'.'</th>';
     $Str .= '</tr>'."\n";
     #
     $Str .= '<tr><td align="center">'."\n";
     $Str .= &Apache::lonstatistics::SectionSelect('Section','multiple',5);
     $Str .= '</td>';
     #
+    $Str .= '<td align="center">'."\n";
+    $Str .= &Apache::lonstatistics::GroupSelect('Group','multiple',5);
+    $Str .= '</td>';
+    #
     $Str .= '<td align="center">';
     $Str .= &Apache::lonhtmlcommon::StatusOptions(undef,undef,5);
     $Str .= '</td>';
     #
+    $Str .= '<td align="center">'.$output_selector.'</td>';
+    #
+    # Render problem checkbox
+    my $prob_checkbox = '<input type="checkbox" name="renderprob" ';
+    if (exists($env{'form.renderprob'}) && $env{'form.renderprob'} eq 'true') {
+        $prob_checkbox .= 'checked ';
+    }
+    $prob_checkbox .= 'value="true" />';
+    $Str .= '<td align="right" valign="top">'.
+        '<label><b>'.
+        &mt('Show problem [_1]',$prob_checkbox).'</b></label><br />'.
+        '<label><b>'.
+        '</td>';
+    #
     $Str .= '</tr>'."\n";
     $Str .= '</table>'."\n";
     #