#!/usr/local/bin/perl # # Author: Michael H. Goldwasser (mhg@cs.luc.edu) # Version: 2.0 (27 Feb 2002) # Version: 2.0.1 ( 9 Sep 2002) # -- added needed default for $extracreditstart # # 'autograde' is a Perl script written for a unix/linux system. # It is designed to compile student programs, and to execute those # programs on test input provided either by the instructor or by other # students. A users guide is distributed with this package. # # It has been discussed in the paper "A gimmick to integrate software # testing throughout the curriculum," which appears in the Proceedings # of the 33rd Annual SIGCSE Technical Symposium on Computer Science # Education, Feb. 27 - Mar. 3, 2002. # DISTRIBUTION # # This software is public domain. It may freely be copied and used in # non-commercial products, assuming proper credit is given to the # author and this message remains, but it should not be resold. If # you want to use the software for commercial products, contact the # author. # # This software is provided ``as is'' without warranty of any kind, # either expressed or implied. The entire risk as to the quality and # performance of the software is with you. Should the software prove # defective, you assume the cost of all necessary servicing, repair or # correction. # # Please report any bugs in this package. Although there is no # guarantee, such bugs may be fixed in a future release of the software. # Remark: to make this script self-sufficient, the getopts.pl package # from perl5 distribution has been included at the end. If this # package is installed on your site and if you prefer to use it, you # can uncomment the following 'require' statement and remove the last # portion of this file # #require "getopts.pl"; # these lines are hopefully redifined in the config file $sourcesuffix =""; $compile_file =""; $compile_final =""; $asgn_name = ""; @required_files = (); @excluded_files = (); @our_files = (); @tests = (); @samples = (); @HTHinput = (); $HTHcommand = ""; @readme = (); $extracreditstart = 99999; @people = (); $config = "autograde.config"; $ourfilesDIR = "ourfiles"; $submitDIR = "submit"; $workingDIR = "working"; $modelDIR = "model"; $answerDIR = "answers"; $inputfileDIR = "input"; @resultsDIR = ("results"); $HTHDIR = "competition"; $timeout = 10; $randsamp = 1.0; # ========================================================================== # Print the usage information if help requested (-h) or a bad option given. # sub usage { die <<"EndUsage"; usage: Options: -h displays this message -H use head-to-head competition mode (Default: instructor tests) -g gather inputs (implies competition mode) -m make model solutions -p process student submissions (copy/compile) -e execute student tests -s output statistics (implies competition mode) -a do all relevant phases (among '-g -m -e -s') -l login comma-separated list of students (Default: everyone) -t sec length of timeout in seconds (Default: 10) -r prob random sample with probability (prob<1.0) -d run in debug mode -q run in quiet mode [ The remaining options are to modify default directory hierarchy ] -F config Configuration file (Default: "autograde.config") -S dir directory of student submissions (Default: "submit") -O dir directory containing "our" files (Default: "ourfiles") -M dir source for model solution program (Default: "model") -C dir competition subdirectory (Default: "competition") -W dir original working directory (Default: "working") -A dir directory containing model answers (Default: "answers") -I dir directory for test input files (Default: "input") -R dir comma separated list of directories with competition results (Default: "results") EndUsage } # ========================================================================== # Get the command-line options if (!(&Getopts('aA:C:deF:ghHI:l:mM:O:pqr:R:sS:t:W:')) || $opt_h) { &usage; } if ($opt_H) { $HTHmode = 1;} if ($opt_g) { $doGather = 1; $HTHmode=1;} if ($opt_m) { $doModel = 1; } if ($opt_p) { $doProcess = 1;} if ($opt_e) { $doTests = 1; $doProcess=1;} if ($opt_s) { $doStats = 1; $HTHmode=1;} if ($opt_a) { $doGather=1; $doModel=1; $doTests=1; $doStats=1; $doProcess=1} if ($opt_l) { @people = split(/,/,$opt_l); } if ($opt_r) { $randsamp = $opt_r;} if ($opt_t) { $timeout = $opt_t;} if ($opt_d) { $debug = 1;} if ($opt_q) { $quiet = 1;} if ($opt_A) { $answerDIR = $opt_A; } if ($opt_C) { $HTHDIR = $opt_C; } if ($opt_F) { $config = $opt_F; } if ($opt_I) { $inputfileDIR = $opt_I; } if ($opt_M) { $modelDIR = $opt_M; } if ($opt_O) { $ourfilesDIR = $opt_O; } if ($opt_R) { @resultsDIR = split(/,/,$opt_R);} if ($opt_S) { $submitDIR = $opt_S; } if ($opt_W) { $workingDIR = $opt_W; } # ========================================================================== printf("DEBUG: Running in %s mode.\n",($HTHmode ? "head-to-head competition" : "standard")) if $debug; if (&verify($config,0,0,0)) { unless ($return = do $config) { warn "couldn't parse file: $config" if $config; warn "couldn't do file: $!" unless defined $return; warn "couldn't run file" unless $return; } else { chop($baseDIR = `pwd`); printf("baseDIR=%s\n",$baseDIR) if $debug; $subDIR = "."; if ($HTHmode) { &verify($HTHDIR,1,1,1); $subDIR = "$HTHDIR"; } if (($doGather || $doProcess) && ($#people==-1)) { # use everyone &verify($submitDIR,0,1,0); if (!opendir(SUBMIT,$submitDIR)) { print "Error: could not open submit directory $submitDIR\n"; } else { @temppeople = grep {/^[^\.]/ && -d "$submitDIR/$_"} readdir(SUBMIT); @people = sort @temppeople; closedir SUBMIT; } } # gather inputfiles from students, if desired if ($HTHmode && $doGather && ($#HTHinput >-1) && &verify("$HTHDIR/$inputfileDIR",1,1,1) && &verify($submitDIR,0,1,0)) { printf("Gathering input from students\n") if (!$quiet); foreach $person (@people) { printf("%-10s input ",$person) if (!$quiet); if (!opendir(STUDENT,"$submitDIR/$person")) { print "Error: could not open student directory $submitDIR/$person\n"; } else { @submittedfull = readdir(STUDENT); closedir STUDENT; @submitteddirectory = grep {/^[^\.]/ && -d $_} @submittedfull; @submittedfiles = grep {/^[^\.]/ && !$excl{$_}} @submittedfull; $HTHinputfound = 0; foreach $file (@submittedfiles) { printf(" considering file $file.\n") if $debug; foreach $template (@HTHinput) { if ($file =~ m/$template/i) { # found a match $HTHinputfound = 1; if (!-e "$baseDIR/$HTHDIR/$inputfileDIR/$person"){ ©file("$baseDIR/$submitDIR/$person/$file","$baseDIR/$HTHDIR/$inputfileDIR/$person"); } } } } print("NOT ") if (!$HTHinputfound); print("found.\n"); } } } # generate list of test/name pairs if ($doTests || $doModel) { if ($HTHmode) { if (!&verify("$HTHDIR/$inputfileDIR",0,1,0) || !opendir(HTHTESTS,"$HTHDIR/$inputfileDIR")) { print "Error: could not open directory $HTHDIR/$inputfileDIR\n"; } else { @hthtestsfull = readdir(HTHTESTS); closedir HTHTESTS; @temphthtests = grep {/^[^\.]/} @hthtestsfull; @hthtests = sort @temphthtests; foreach $name (@hthtests) { $localcommand = $HTHcommand; $localcommand =~ s|FILE|$baseDIR/$HTHDIR/$inputfileDIR/$name|g; push @testpairs, [$localcommand,$name]; } } } else { $testnum=1; foreach $t (@tests) { if ($testnum < $extracreditstart) { push @testpairs, [$t,"test".$testnum]; } else { push @testpairs, [$t,"extra".(1+$testnum-$extracreditstart)]; } $testnum++; } } } # mark excluded file names %excl=(); foreach $file (@excluded_files) { $excl{$file}++; } foreach $file (@our_files) { $excl{$file}++; } # create model solutions, if desired if ($doModel && &verify($modelDIR,0,1,0) && &verify("$subDIR/$answerDIR",1,1,1) && &verify("$subDIR/$workingDIR",1,1,1) && ($#our_files<0 || &verify($ourfilesDIR,0,1,0))) { # uses first (and only) person in model directory if (!opendir(MODEL,$modelDIR)) { print "Error: could not open model directory: $modelDIR\n"; } else { printf("Processing model program\n") if (!$quiet); @model = grep {/^[^\.]/ && -d "$modelDIR/$_"} readdir(MODEL); closedir MODEL; $person = @model[0]; &singleperson("$baseDIR/$subDIR","$baseDIR/$modelDIR",$person, 1,$HTHmode,1,@testpairs,); } } # process student submissions, executing tests if desired. if ($doProcess && &verify($submitDIR,0,1,0) && &verify("$subDIR/$workingDIR",1,1,1) && (!$HTHmode || !$doTests || &verify("$subDIR/$resultsDIR[0]",1,1,1)) && (!$doTests || &verify("$subDIR/$answerDIR",0,1,0)) && ($#our_files<0 || &verify($ourfilesDIR,0,1,0))) { foreach $person (@people) { printf("Processing $person...\n") if (!$quiet); &singleperson("$baseDIR/$subDIR","$baseDIR/$submitDIR",$person, 0,$HTHmode,$doTests,@testpairs,); } } # report performance statistics for competition, if desired. if ($HTHmode && $doStats) { foreach $result (@resultsDIR) { if (&verify("$subDIR/$result",0,1,0) && opendir(DIR,"$subDIR/$result")) { @resultpeople = grep {-f "$subDIR/$result/$_"} readdir(DIR); closedir DIR; foreach $person (@resultpeople) { $all{$person}=1; open(FILE,"$subDIR/$result/$person"); while() { if (m/^(\S+)\s+(\S+)\s/) { if ($2 eq "succeeded") { # print "$person succeeded on $1\n"; $implsuccess{$person}{$1} = 1; } else { $implfailure{$person}{$1} = 1; } } else { print "Error: reading line '$_'\n"; } } } } } # which implementations appear perfect foreach $person (keys %all) { if (!$implfailure{$person}) { $perfect{$person}++; $countperfectoutput++; } } # calculate inputsuccess based only on the underlying flawed programs foreach $person (keys %all) { if (!$perfect{$person}) { foreach $test (keys %{$implsuccess{$person}}) { $testfailure{$test}++; } foreach $test (keys %{$implfailure{$person}}) { $testsuccess{$test}++; if ($test eq $person) { ($selfflaw)++; } } } } # time to output stats printf("login input%s code%s\n","%","%"); foreach $person (sort (keys %all)) { $testsuccesscount = $testsuccess{$person}; $testfailurecount = $testfailure{$person}; $testattemptcount = $testsuccesscount + $testfailurecount; $implsuccesscount = keys %{$implsuccess{$person}}; $implfailurecount = keys %{$implfailure{$person}}; $implattemptcount = $implsuccesscount + $implfailurecount; printf("%-10s%4s %4s\n",$person, (($testattemptcount>0) ? sprintf("%4.2f",$testsuccesscount/$testattemptcount) : "----"), (($implattemptcount>0) ? sprintf("%4.2f",$implsuccesscount/$implattemptcount) : "----")); # compile some class statistics if ($testattemptcount>0) { $countinput++; $avg += $testsuccesscount/$testattemptcount; if ($testsuccesscount==$testattemptcount) { $countperfectinput++; } } } printf("\n%d of %d implementations appear perfect\n",$countperfectoutput,$#resultpeople+1); printf("%d of %d input sets exposed all flawed programs\n",$countperfectinput,$countinput); printf("a typical input set exposed %5.3f of flawed programs\n",$avg/$countinput); printf("%d students submitted tests which expose their own flaws\n",$selfflaw); } } } # What is proper/portable way to copy a file # system "cp $from $to" # does not seem portable and # link $from,$to # may create a hard link, and seems to fail when spaces in filename. # doing line-by-line copy seems like overkill sub copyfile { local($from,$to) = @_; system "cp \"$from\" \"$to\""; } ###################### # There MUST be a better way than this! # I'm particularly concerned by portability of reliance on 'ps' # (is there a cleaner way to make sure all children get found/killed) ###################### sub mykill { local($thispid) = @_; local($childpid) = 0; local($pgid) = 0; local(*PS); printf("within mykill($thispid)...\n") if $debug; open(PS, "ps -A -o pid,ppid |"); while () { if (m/([0-9]+)\s+$thispid/) { $childpid = $1; printf("childpid = $childpid\n") if $debug; mykill($childpid); } } close PS; kill 9,$thispid; } sub singleperson { local($rootDIR,$sourceDIR,$person,$model,$hth,$dothetests,@Tpairs) = @_; local($readmefound); local($HTHinputfound); local($submittedfull); local($submitteddirectory); local($submittedfiles); local($dir); local($file); local($template); local($compile); local($compilestatus); local($t); local($outcome); local($time); local($CHDIR); if (&verify("$rootDIR/$workingDIR/$person",1,1,1)) { if (!($CHDIR=(chdir "$rootDIR/$workingDIR/$person"))) { printf("ERROR: could not chdir to $rootDIR/$workingDIR/$person\n"); printf("%s\n",`pwd`) if $debug; } else { # reset flags $readmefound = 0; $HTHinputfound = 0; open (REPORT,">$person.autograde"); print REPORT "----------------------------------------------\n"; print REPORT "Report for: $person, $asgn_name\n"; print REPORT "----------------------------------------------\n\n"; if (!opendir(PERSON,"$sourceDIR/$person")) { print "Error: could not open source directory $sourceDIR/$person\n"; } else { @submittedfull = readdir(PERSON); closedir PERSON; @submitteddirectory = grep {/^[^\.]/ && -d $_} @submittedfull; @submittedfiles = grep {/^[^\.]/ && !$excl{$_}} @submittedfull; foreach $dir (@submitteddirrectory) { print REPORT "NOTE: Folder $dir has been submitted.\n"; } foreach $file (@submittedfiles) { printf("considering file $file.\n") if $debug; if (!-e "$file") { ©file("$sourceDIR/$person/$file","$file"); chmod 0600, "$file"; } if ($#HTHinput > -1) { foreach $template (@HTHinput) { if ($file =~ m/$template/i) { # found a match $HTHinputfound = 1; } } } if ($#readme > -1) { foreach $template (@readme) { if ($file =~ m/$template/i) { # found a match $readmefound = 1; if (!-e "readme") { symlink $file,"readme"; } } } } } } if (($#HTHinput>-1) && !$HTHinputfound) { print REPORT "--------------------------------------------------------------\nWARNING: no test input file was found.\n"; } if (($#readme>-1) && !$readmefound) { print REPORT "--------------------------------------------------------------\nWARNING: no readme was found.\n"; } foreach $file (@required_files) { if (!-e "$file"){ print REPORT "--------------------------------------------------------------\nWARNING: Required file $file NOT submitted.\n"; } } foreach $file (@our_files) { if (-e "$file" || -l "$file") { unlink "$file"; } symlink "$baseDIR/$ourfilesDIR/$file",$file; } print REPORT "--------------------------------------------------------------\n"; # do file-by-file compilations if ($sourcesuffix && $compile_file) { foreach $file (@submittedfiles) { if ($file =~ m/$sourcesuffix$/) { printf("\nabout to compile $file.\n") if $debug; $file =~ s/\s/\\ /; print REPORT "$file: "; $compile = $compile_file; $compile =~ s|FILE|$file|g; $compilestatus = `$compile 2>&1`; # captures STDOUT and STDERR if ($compilestatus) { printf("compilation failed.\n") if $debug; print REPORT "compilation failed\n"; print REPORT $compilestatus; print REPORT "--------------------------------------------------------------\n"; } else { printf("compilation succeeded.\n") if $debug; print REPORT "successfully compiled.\n"; print REPORT "--------------------------------------------------------------\n"; } } } } # choose our provided files over those submitted by students # (just in case they redefine any of our classes!) foreach $file (@our_files) { if (-e "$file" || -l "$file") { unlink "$file"; } symlink "$baseDIR/$ourfilesDIR/$file",$file; } # do final-compile command if ($compile_final) { printf("\nabout to do final compilation.\n") if $debug; $compilestatus = `$compile_final 2>&1`; # captures STDOUT and STDERR if ($compilestatus) { printf("final compilation failed.\n") if $debug; print REPORT "final compilation failed\n"; print REPORT $compilestatus; print REPORT "--------------------------------------------------------------\n"; } else { printf("final compilation succeeded.\n") if $debug; print REPORT "final successfully compiled.\n"; print REPORT "--------------------------------------------------------------\n"; } } if ($dothetests && &verify("$rootDIR/$workingDIR/$person/testing",1,1,1)) { printf REPORT ("beginning sequence of %d tests:\n",$#Tpairs+1); open (RESULT,">$person.results") if $hth; foreach $t (@Tpairs) { # random sample, if desired if ($model || !$hth || $randsamp == 1.0 || (($rand=rand) <= $randsamp)) { printf REPORT ("%-10s ",$t->[1]); printf RESULT ("%-10s ",$t->[1]) if $hth; ($outcome,$time) = &singletest($t->[0],$t->[1],"$rootDIR/$answerDIR",1,$model); printf REPORT ("%-10s (%0d seconds)\n",$outcome,$time); printf RESULT ("$outcome\n") if $hth; } } if ($hth) { close RESULT; if (!$model) { ©file("$person.results","$rootDIR/$resultsDIR[0]/$person"); } } else { $sampnum=1; foreach $s (@samples) { printf REPORT ("%-10s ","sample".$sampnum); ($outcome,$time) = &singletest($s,"sample".$sampnum,"$rootDIR/$answerDIR",0,$model); printf REPORT ("%-10s (%0d seconds)\n",$outcome,$time); $sampnum++; } } } close REPORT; chdir "$baseDIR"; } } } # end of singleperson sub singletest { local($commandline,$testname,$answerD,$doDiff,$isModel) = @_; local($starttime); local($pid); local($result); local($time); local($elapsed); print "\n-----------------------------\nstarting test: $testname ($commandline)\n" if $debug; print " (answerdir: $answerD)\n" if $debug; $startime = time(); eval { $pid = open(PIPE, "$commandline 1>testing/$testname.output 2>&1 |"); # captures STDOUT and STDERR print "started pid = $pid\n" if $debug; print "command: $commandline\n" if $debug; print `date` if $debug; local $SIG{ALRM} = sub {die "alarm\n"}; alarm $timeout; print "setting alarm for $timeout seconds\n" if $debug; waitpid($pid,0); print " ending pid = $pid\n" if $debug; print `date` if $debug; close PIPE; alarm 0; }; die if $@ && $@ ne "alarm\n"; if ($@) { # timed out print "timeout reached\n" if $debug; &mykill($pid); print "killed pid = $pid\n" if $debug; print `date` if $debug; close PIPE; print "closed PIPE\n" if $debug; print `date` if $debug; $result = "Timed out"; $time = $timeout; } else { # completed $elapsed = time() - $startime; if ($doDiff && !$isModel) { $diff = `diff -i -b -w testing/$testname.output $answerD/$testname.key`; if ($diff) { $result = "failed"; open (TEMP,">testing/$testname.diff"); print TEMP $diff; close TEMP; } else { $result = "succeeded"; $time = $elapsed; } } else { if ($isModel) { if (-e "$answerD/$testname.key") { unlink "$answerD/$testname.key"; } ©file("testing/$testname.output", "$answerD/$testname.key"); } $result = "completed"; $time = $elapsed; } } return ($result,$time); } # end of singletest sub verify { local($name,$writable,$dir,$create) = @_; local($success); if (!-e "$name") { if ($create && $dir) { if (mkdir $name,0700) { $success = 1; } else { $success = 0; print "Error: unable to create directory '$name'\n"; } } else { $success=0; print "Error: '$name' does not exist\n"; } } else { if (!-r "$name") { $success=0; print "Error: '$name' exists, but unreadable\n"; } else { if ($writeable && (!-w "$name")) { $success=0; print "Error: '$name' exists, but not writable\n"; } else { if ($dir) { if (!-d "$name") { $success = 0; print "Error: '$name' not a directory\n"; } elsif (!-x "$name") { $success = 0; print "Error: directory '$name' not executable\n"; } else { $success = 1; } } else { $success = (-f "$name"); print "Error: '$name' exists, but not a file\n" if (!$success); } } } } return ($success); } ######################################################################## # /usr/lib/perl5/getopts.pl ######################################################################## # getopts.pl - a better getopt.pl # Usage: # do Getopts('a:bc'); # -a takes arg. -b & -c not. Sets opt_* as a # # side effect. sub Getopts { local($argumentative) = @_; local(@args,$_,$first,$rest); local($errs) = 0; @args = split( / */, $argumentative ); while(@ARGV && ($_ = $ARGV[0]) =~ /^-(.)(.*)/) { ($first,$rest) = ($1,$2); $pos = index($argumentative,$first); if($pos >= 0) { if($pos < $#args && $args[$pos+1] eq ':') { shift(@ARGV); if($rest eq '') { ++$errs unless @ARGV; $rest = shift(@ARGV); } ${"opt_$first"} = $rest; } else { ${"opt_$first"} = 1; if($rest eq '') { shift(@ARGV); } else { $ARGV[0] = "-$rest"; } } } else { print STDERR "Unknown option: $first\n"; ++$errs; if($rest ne '') { $ARGV[0] = "-$rest"; } else { shift(@ARGV); } } } $errs == 0; } ####### end of getopts.pl #######