1 #!/usr/bin/perl -w
   2 #
   3 # $RCSfile: JoinTextFiles.pl,v $
   4 # $Date: 2008/01/30 21:44:48 $
   5 # $Revision: 1.18 $
   6 #
   7 # Author: Manish Sud <msud@san.rr.com>
   8 #
   9 # Copyright (C) 2004-2008 Manish Sud. All rights reserved.
  10 #
  11 # This file is part of MayaChemTools.
  12 #
  13 # MayaChemTools is free software; you can redistribute it and/or modify it under
  14 # the terms of the GNU Lesser General Public License as published by the Free
  15 # Software Foundation; either version 3 of the License, or (at your option) any
  16 # later version.
  17 #
  18 # MayaChemTools is distributed in the hope that it will be useful, but without
  19 # any warranty; without even the implied warranty of merchantability of fitness
  20 # for a particular purpose.  See the GNU Lesser General Public License for more
  21 # details.
  22 #
  23 # You should have received a copy of the GNU Lesser General Public License
  24 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
  25 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
  26 # Boston, MA, 02111-1307, USA.
  27 #
  28 
  29 use 5.006;
  30 use strict;
  31 use FindBin; use lib "$FindBin::Bin/../lib";
  32 use Getopt::Long;
  33 use File::Basename;
  34 use Text::ParseWords;
  35 use Benchmark;
  36 use SDFileUtil;
  37 use FileUtil;
  38 use TextUtil;
  39 
  40 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
  41 my($TextFile, @TextFilesList, $Index, $NewTextFile, $OutQuote, $InDelim, $OutDelim, $FileDir, $FileName, $FileExt, $Line, @Words, $ColLabelsLine, @ColLabels, $NewColLabelsLine, @NewColLabels);
  42 
  43 # Autoflush STDOUT
  44 $| = 1;
  45 
  46 # Starting message...
  47 $ScriptName = basename $0;
  48 print "\n$ScriptName:Starting...\n\n";
  49 $StartTime = new Benchmark;
  50 
  51 # Get the options and setup script...
  52 SetupScriptUsage();
  53 if ($Options{help} || @ARGV < 1) {
  54   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
  55 }
  56 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
  57 
  58 if (@TextFilesList == 1) {
  59   die "Error: Specify more than one Text file.\n";
  60 }
  61 
  62 if ($Options{root}) {
  63   $FileDir = ""; $FileName = ""; $FileExt = "";
  64   ($FileDir, $FileName, $FileExt) = ParseFileName($Options{root});
  65   if ($FileName && $FileExt) {
  66     $NewTextFile = $FileName;
  67   }
  68   else {
  69       $NewTextFile =  $Options{root};
  70   }
  71 }
  72 else {
  73   $FileDir = ""; $FileName = ""; $FileExt = "";
  74   ($FileDir, $FileName, $FileExt) = ParseFileName($TextFilesList[0]);
  75   $NewTextFile = $FileName . "1To" . @TextFilesList . "Joined";
  76 }
  77 if ($Options{outdelim} =~ /^tab$/i) {
  78   $NewTextFile .= ".tsv";
  79 }
  80 else {
  81   $NewTextFile .= ".csv";
  82 }
  83 if (!$Options{overwrite}) {
  84   if (-e $NewTextFile) {
  85     die "Error: The file $NewTextFile already exists.\n";
  86   }
  87 }
  88 if ($Options{root}) {
  89   for $Index (0 .. $#TextFilesList) {
  90     if (lc($NewTextFile) eq lc($TextFilesList[$Index])) {
  91       die "Error: Output filename, $NewTextFile, is similar to a input file name.\nSpecify a different name using \"-r --root\" option or use default name.\n";
  92     }
  93   }
  94 }
  95 print "Generating new Text file $NewTextFile...\n";
  96 
  97 # Go over each text file and check consistency of data: same number of columns,
  98 # presenece/absence of column labels, and input delimiter which makes sense for
  99 # the file extension. And afterwards, join these files appropriately.
 100 
 101 open NEWTEXTFILE, ">$NewTextFile" or die "Error: Couldn't open $NewTextFile: $! \n";
 102 $NewColLabelsLine = "";
 103 @NewColLabels = ();
 104 FILELIST: for $Index (0 .. $#TextFilesList) {
 105   $TextFile = $TextFilesList[$Index];
 106   print "\nProcessing file $TextFile...\n";
 107   if (!(-e $TextFile)) {
 108     warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
 109     next FILELIST;
 110   }
 111   if (!$Options{fast}) {
 112     if (!CheckFileType($TextFile, "csv tsv")) {
 113       warn "Warning: Ignoring file $TextFile: It's not a text file\n";
 114       next FILELIST;
 115     }
 116     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
 117     if ($FileExt =~ /^tsv$/i) {
 118       $InDelim = "\t";
 119     }
 120     else {
 121       $InDelim = "\,";
 122       if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
 123 	warn "Warning: Ignoring file $TextFile: The value specified, $Options{indelim}, for option \"--indelim\" is not valid for csv files\n";
 124 	next FILELIST;
 125       }
 126       if ($Options{indelim} =~ /^semicolon$/i) {
 127 	$InDelim = "\;";
 128       }
 129     }
 130   }
 131   if (!open TEXTFILE, "$TextFile") {
 132     warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
 133     next FILELIST;
 134   }
 135   if ($Options{label} =~ /^yes$/i) {
 136     if ($Options{fast}) {
 137       $ColLabelsLine = GetTextLine(\*TEXTFILE);
 138       if (!$NewColLabelsLine) {
 139 	$NewColLabelsLine = $ColLabelsLine;
 140 	print NEWTEXTFILE "$NewColLabelsLine\n";
 141       }
 142     }
 143     else {
 144       $Line = GetTextLine(\*TEXTFILE);
 145       if ($NewColLabelsLine) {
 146 	@ColLabels = quotewords($InDelim, 0, $Line);
 147 	if (@ColLabels != @NewColLabels) {
 148 	  warn "Warning: Ignoring file $TextFile: The number of columns in this file, ", scalar(@ColLabels), ", is different from the number of columns, ", scalar(@NewColLabels), ", in the first valid text file. \n";
 149 	  next FILELIST;
 150 	}
 151 	$ColLabelsLine = JoinWords(\@ColLabels, $OutDelim, $OutQuote);
 152 	if ($ColLabelsLine ne $NewColLabelsLine) {
 153 	  warn "Warning: Ignoring file $TextFile: The column names in this file are different from those in first valid text file.\nColumnlabels in first valid text file: $NewColLabelsLine \nColumnlabels in current text file: $ColLabelsLine\n";
 154 	  next FILELIST;
 155 	}
 156       }
 157       else {
 158 	@NewColLabels = quotewords($InDelim, 0, $Line);
 159 	$NewColLabelsLine = JoinWords(\@NewColLabels, $OutDelim, $OutQuote);
 160 	print NEWTEXTFILE "$NewColLabelsLine\n";
 161       }
 162     }
 163   }
 164   while ($Line = GetTextLine(\*TEXTFILE)) {
 165     if (!$Options{fast}) {
 166       @Words = quotewords($InDelim, 0, $Line);
 167       $Line = JoinWords(\@Words, $OutDelim, $OutQuote);
 168     }
 169     print NEWTEXTFILE "$Line\n";
 170   }
 171   close TEXTFILE;
 172 }
 173 close NEWTEXTFILE;
 174 
 175 print "$ScriptName:Done...\n\n";
 176 
 177 $EndTime = new Benchmark;
 178 $TotalTime = timediff ($EndTime, $StartTime);
 179 print "Total time: ", timestr($TotalTime), "\n";
 180 
 181 ###############################################################################
 182 
 183 # Setup script usage  and retrieve command line arguments specified using various options...
 184 sub SetupScriptUsage {
 185 
 186   # Retrieve all the options...
 187   %Options = ();
 188   $Options{label} = "yes";
 189   $Options{indelim} = "comma";
 190   $Options{outdelim} = "comma";
 191   $Options{quote} = "yes";
 192   if (!GetOptions(\%Options, "fast|f", "help|h", "indelim=s", "label|l=s", "outdelim=s", "overwrite|o", "quote|q=s", "root|r=s", "workingdir|w=s")) {
 193     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
 194   }
 195   if ($Options{workingdir}) {
 196     if (! -d $Options{workingdir}) {
 197       die "Error: The value specified, $Options{workingdir},  for option \"-w --workingdir\" is not a directory name.\n";
 198     }
 199     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
 200   }
 201   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
 202     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
 203   }
 204   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
 205     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
 206   }
 207   if ($Options{quote} !~ /^(yes|no)$/i) {
 208     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
 209   }
 210   if ($Options{label} !~ /^(yes|no)$/i) {
 211     die "Error: The value specified, $Options{label}, for option \"-l --label\" is not valid. Allowed values: yes or no\n";
 212   }
 213   $OutDelim = ($Options{outdelim} =~ /^tab$/i ) ? "\t" : (($Options{outdelim} =~ /^semicolon$/i) ? "\;" : "\,");
 214   $OutQuote = ($Options{quote} =~ /^yes$/i) ? 1 : 0;
 215 }
 216