1 #!/usr/bin/perl -w
   2 #
   3 # $RCSfile: SplitTextFiles.pl,v $
   4 # $Date: 2008/01/30 21:45:03 $
   5 # $Revision: 1.19 $
   6 #
   7 # Author: Manish Sud <msud@san.rr.com>
   8 #
   9 # Copyright (C) 2004-2008 Manish Sud. All rights reserved.
  10 #
  11 # This file is part of MayaChemTools.
  12 #
  13 # MayaChemTools is free software; you can redistribute it and/or modify it under
  14 # the terms of the GNU Lesser General Public License as published by the Free
  15 # Software Foundation; either version 3 of the License, or (at your option) any
  16 # later version.
  17 #
  18 # MayaChemTools is distributed in the hope that it will be useful, but without
  19 # any warranty; without even the implied warranty of merchantability of fitness
  20 # for a particular purpose.  See the GNU Lesser General Public License for more
  21 # details.
  22 #
  23 # You should have received a copy of the GNU Lesser General Public License
  24 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
  25 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
  26 # Boston, MA, 02111-1307, USA.
  27 #
  28 
  29 use 5.006;
  30 use strict;
  31 use FindBin; use lib "$FindBin::Bin/../lib";
  32 use Getopt::Long;
  33 use File::Basename;
  34 use Text::ParseWords;
  35 use Benchmark;
  36 use FileUtil;
  37 use TextUtil;
  38 
  39 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
  40 my($TextFile, @TextFilesList, $Index, $OutQuote, $InDelim, $OutDelim, $LineCount, $NewFileIndex, @NewTextFilesList, $IgnoreFile, $FileDir, $FileName, $FileExt, $NewFullFileName, $MaxLinesCount, $MaxLinesPerFile, $Line, @Words, @ColLabels, $ColLabelsLine);
  41 
  42 # Autoflush STDOUT
  43 $| = 1;
  44 
  45 # Starting message...
  46 $ScriptName = basename $0;
  47 print "\n$ScriptName:Starting...\n\n";
  48 $StartTime = new Benchmark;
  49 
  50 # Get the options and setup script...
  51 SetupScriptUsage();
  52 if ($Options{help} || @ARGV < 1) {
  53   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
  54 }
  55 
  56 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
  57 
  58 if (@TextFilesList > 1) {
  59   print "Processing Text files...\n";
  60 }
  61 FILELIST: for $Index (0 .. $#TextFilesList) {
  62   $TextFile = $TextFilesList[$Index];
  63   if (@TextFilesList > 1) {
  64     print "\nProcessing file $TextFile...\n";
  65   }
  66   else {
  67     print "Processing file $TextFile...\n"
  68   }
  69   if (!(-e $TextFile)) {
  70     warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
  71     next FILELIST;
  72   }
  73   if (!$Options{fast}) {
  74     if (!CheckFileType($TextFile, "csv tsv")) {
  75       warn "Warning: Ignoring file $TextFile: It's not a text file\n";
  76       next FILELIST;
  77     }
  78     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
  79     if ($FileExt =~ /^tsv$/i) {
  80       $InDelim = "\t";
  81     }
  82     else {
  83       $InDelim = "\,";
  84       if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
  85 	warn "Warning: Ignoring file $TextFile: The value specified, $Options{indelim}, for option \"--indelim\" is not valid for csv files\n";
  86 	next FILELIST;
  87       }
  88       if ($Options{indelim} =~ /^semicolon$/i) {
  89 	$InDelim = "\;";
  90       }
  91     }
  92   }
  93   if (!open TEXTFILE, "$TextFile") {
  94     warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
  95     next FILELIST;
  96   }
  97   $LineCount = 0;
  98   while (<TEXTFILE>) {
  99       $LineCount++;
 100   }
 101   close TEXTFILE;
 102   if ($LineCount  < $Options{numfiles}) {
 103     warn "Warning: Ignoring file $TextFile: Total number of lines, $LineCount, is smaller than\nnumber of new files, $Options{numfiles}\n";
 104     next FILELIST;
 105   }
 106   $FileDir = ""; $FileName = ""; $FileExt = "";
 107   ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
 108   if (!$Options{fast}) {
 109     $FileExt = "csv";
 110     if ($Options{outdelim} =~ /^tab$/i) {
 111       $FileExt = "tsv";
 112     }
 113   }
 114   $IgnoreFile = 0;
 115   @NewTextFilesList = ();
 116  NEWFILELIST: for $NewFileIndex (1 .. $Options{numfiles}) {
 117     $NewFullFileName = $FileName;
 118     if ($Options{root} && (@TextFilesList == 1)) {
 119       my ($RootFileDir, $RootFileName, $RootFileExt) = ParseFileName($Options{root});
 120       if ($RootFileName && $RootFileExt) {
 121 	$NewFullFileName = $RootFileName;
 122       }
 123       else {
 124 	$NewFullFileName = $Options{root};
 125       }
 126     }
 127     $NewFullFileName .= "Part" . "$NewFileIndex" . ".$FileExt";
 128     push @NewTextFilesList, $NewFullFileName;
 129     if (!$Options{overwrite}) {
 130       if (-e $NewFullFileName) {
 131 	$IgnoreFile = 1;
 132 	warn "Warning: Ignoring file $TextFile: New Text file, $NewFullFileName, already exists\n";
 133 	last NEWFILELIST;
 134       }
 135     }
 136   }
 137   if ($IgnoreFile) {
 138     next FILELIST;
 139   }
 140   $MaxLinesPerFile = int $LineCount  / $Options{numfiles};
 141   $MaxLinesCount = $MaxLinesPerFile;
 142   $LineCount = 0;
 143   $NewFileIndex = 1;
 144 
 145   open NEWTEXTFILE, ">$NewTextFilesList[$NewFileIndex - 1]" or die "Error: Can't open $NewTextFilesList[$NewFileIndex -1]: $! \n";
 146   print "Generating $NewTextFilesList[$NewFileIndex - 1] file\n";
 147   open TEXTFILE, "$TextFile" or die "Error: Can't open $TextFile: $! \n";
 148 
 149   if ($Options{label} =~ /^yes$/i) {
 150     if ($Options{fast}) {
 151       $ColLabelsLine = GetTextLine(\*TEXTFILE);
 152     }
 153     else {
 154       $Line = GetTextLine(\*TEXTFILE);
 155       @ColLabels = quotewords($InDelim, 0, $Line);
 156       $ColLabelsLine = JoinWords(\@ColLabels, $OutDelim, $OutQuote);
 157     }
 158     print NEWTEXTFILE "$ColLabelsLine\n";
 159   }
 160   while ($Line = GetTextLine(\*TEXTFILE)) {
 161     if (!$Options{fast}) {
 162       @Words = quotewords($InDelim, 0, $Line);
 163       $Line = JoinWords(\@Words, $OutDelim, $OutQuote);
 164     }
 165     print NEWTEXTFILE "$Line\n";
 166     $LineCount++;
 167     if ($NewFileIndex <= $Options{numfiles}) {
 168       if ($LineCount >= $MaxLinesCount) {
 169 	if ($NewFileIndex < $Options{numfiles}) {
 170 	  close NEWTEXTFILE;
 171 	}
 172 	$NewFileIndex++;
 173 	$MaxLinesCount = $MaxLinesPerFile * $NewFileIndex;
 174 	if ($NewFileIndex <= $Options{numfiles}) {
 175 	  open NEWTEXTFILE, ">$NewTextFilesList[$NewFileIndex - 1]" or die "Error: Can't open $NewTextFilesList[$NewFileIndex - 1]: $! \n";
 176 	  print "Generating $NewTextFilesList[$NewFileIndex - 1] file\n";
 177 	  if ($Options{label} =~ /^yes$/i) {
 178 	    print NEWTEXTFILE "$ColLabelsLine\n";
 179 	  }
 180 	}
 181       }
 182     }
 183   }
 184   close NEWTEXTFILE;
 185   close TEXTFILE;
 186 }
 187 
 188 print "$ScriptName:Done...\n\n";
 189 
 190 $EndTime = new Benchmark;
 191 $TotalTime = timediff ($EndTime, $StartTime);
 192 print "Total time: ", timestr($TotalTime), "\n";
 193 
 194 ###############################################################################
 195 
 196 # Setup script usage  and retrieve command line arguments specified using various options...
 197 sub SetupScriptUsage {
 198 
 199   # Retrieve all the options...
 200   %Options = ();
 201   $Options{label} = "yes";
 202   $Options{numfiles} = 2;
 203   $Options{indelim} = "comma";
 204   $Options{outdelim} = "comma";
 205   $Options{quote} = "yes";
 206   if (!GetOptions(\%Options, "fast|f", "help|h", "indelim=s", "label|l=s", "numfiles|n=i", "outdelim=s", "overwrite|o", "quote|q=s", "root|r=s", "workingdir|w=s")) {
 207     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
 208   }
 209   if ($Options{workingdir}) {
 210     if (! -d $Options{workingdir}) {
 211       die "Error: The value specified, $Options{workingdir},  for option \"-w --workingdir\" is not a directory name.\n";
 212     }
 213     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
 214   }
 215   if ($Options{numfiles} < 2) {
 216     die "Error: The value specified, $Options{numfiles},  for option \"-n --numfiles\" is not valid. Allowed values: >= 2 \n";
 217   }
 218   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
 219     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
 220   }
 221   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
 222     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
 223   }
 224   if ($Options{quote} !~ /^(yes|no)$/i) {
 225     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
 226   }
 227   if ($Options{label} !~ /^(yes|no)$/i) {
 228     die "Error: The value specified, $Options{label}, for option \"-l --label\" is not valid. Allowed values: yes or no\n";
 229   }
 230   $OutDelim = ($Options{outdelim} =~ /^tab$/i ) ? "\t" : (($Options{outdelim} =~ /^semicolon$/i) ? "\;" : "\,");
 231   $OutQuote = ($Options{quote} =~ /^yes$/i) ? 1 : 0;
 232 }
 233