1 #!/usr/bin/perl -w
   2 #
   3 # $RCSfile: JoinTextFiles.pl,v $
   4 # $Date: 2010/01/03 00:59:52 $
   5 # $Revision: 1.22 $
   6 #
   7 # Author: Manish Sud <msud@san.rr.com>
   8 #
   9 # Copyright (C) 2004-2010 Manish Sud. All rights reserved.
  10 #
  11 # This file is part of MayaChemTools.
  12 #
  13 # MayaChemTools is free software; you can redistribute it and/or modify it under
  14 # the terms of the GNU Lesser General Public License as published by the Free
  15 # Software Foundation; either version 3 of the License, or (at your option) any
  16 # later version.
  17 #
  18 # MayaChemTools is distributed in the hope that it will be useful, but without
  19 # any warranty; without even the implied warranty of merchantability of fitness
  20 # for a particular purpose.  See the GNU Lesser General Public License for more
  21 # details.
  22 #
  23 # You should have received a copy of the GNU Lesser General Public License
  24 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
  25 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
  26 # Boston, MA, 02111-1307, USA.
  27 #
  28 
  29 use strict;
  30 use FindBin; use lib "$FindBin::Bin/../lib";
  31 use Getopt::Long;
  32 use File::Basename;
  33 use Text::ParseWords;
  34 use Benchmark;
  35 use SDFileUtil;
  36 use FileUtil;
  37 use TextUtil;
  38 
  39 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
  40 my($TextFile, @TextFilesList, $Index, $NewTextFile, $OutQuote, $InDelim, $OutDelim, $FileDir, $FileName, $FileExt, $Line, @Words, $ColLabelsLine, @ColLabels, $NewColLabelsLine, @NewColLabels);
  41 
  42 # Autoflush STDOUT
  43 $| = 1;
  44 
  45 # Starting message...
  46 $ScriptName = basename $0;
  47 print "\n$ScriptName:Starting...\n\n";
  48 $StartTime = new Benchmark;
  49 
  50 # Get the options and setup script...
  51 SetupScriptUsage();
  52 if ($Options{help} || @ARGV < 1) {
  53   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
  54 }
  55 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
  56 
  57 if (@TextFilesList == 1) {
  58   die "Error: Specify more than one Text file.\n";
  59 }
  60 
  61 if ($Options{root}) {
  62   $FileDir = ""; $FileName = ""; $FileExt = "";
  63   ($FileDir, $FileName, $FileExt) = ParseFileName($Options{root});
  64   if ($FileName && $FileExt) {
  65     $NewTextFile = $FileName;
  66   }
  67   else {
  68       $NewTextFile =  $Options{root};
  69   }
  70 }
  71 else {
  72   $FileDir = ""; $FileName = ""; $FileExt = "";
  73   ($FileDir, $FileName, $FileExt) = ParseFileName($TextFilesList[0]);
  74   $NewTextFile = $FileName . "1To" . @TextFilesList . "Joined";
  75 }
  76 if ($Options{outdelim} =~ /^tab$/i) {
  77   $NewTextFile .= ".tsv";
  78 }
  79 else {
  80   $NewTextFile .= ".csv";
  81 }
  82 if (!$Options{overwrite}) {
  83   if (-e $NewTextFile) {
  84     die "Error: The file $NewTextFile already exists.\n";
  85   }
  86 }
  87 if ($Options{root}) {
  88   for $Index (0 .. $#TextFilesList) {
  89     if (lc($NewTextFile) eq lc($TextFilesList[$Index])) {
  90       die "Error: Output filename, $NewTextFile, is similar to a input file name.\nSpecify a different name using \"-r --root\" option or use default name.\n";
  91     }
  92   }
  93 }
  94 print "Generating new Text file $NewTextFile...\n";
  95 
  96 # Go over each text file and check consistency of data: same number of columns,
  97 # presenece/absence of column labels, and input delimiter which makes sense for
  98 # the file extension. And afterwards, join these files appropriately.
  99 
 100 open NEWTEXTFILE, ">$NewTextFile" or die "Error: Couldn't open $NewTextFile: $! \n";
 101 $NewColLabelsLine = "";
 102 @NewColLabels = ();
 103 FILELIST: for $Index (0 .. $#TextFilesList) {
 104   $TextFile = $TextFilesList[$Index];
 105   print "\nProcessing file $TextFile...\n";
 106   if (!(-e $TextFile)) {
 107     warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
 108     next FILELIST;
 109   }
 110   if (!$Options{fast}) {
 111     if (!CheckFileType($TextFile, "csv tsv")) {
 112       warn "Warning: Ignoring file $TextFile: It's not a text file\n";
 113       next FILELIST;
 114     }
 115     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
 116     if ($FileExt =~ /^tsv$/i) {
 117       $InDelim = "\t";
 118     }
 119     else {
 120       $InDelim = "\,";
 121       if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
 122 	warn "Warning: Ignoring file $TextFile: The value specified, $Options{indelim}, for option \"--indelim\" is not valid for csv files\n";
 123 	next FILELIST;
 124       }
 125       if ($Options{indelim} =~ /^semicolon$/i) {
 126 	$InDelim = "\;";
 127       }
 128     }
 129   }
 130   if (!open TEXTFILE, "$TextFile") {
 131     warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
 132     next FILELIST;
 133   }
 134   if ($Options{label} =~ /^yes$/i) {
 135     if ($Options{fast}) {
 136       $ColLabelsLine = GetTextLine(\*TEXTFILE);
 137       if (!$NewColLabelsLine) {
 138 	$NewColLabelsLine = $ColLabelsLine;
 139 	print NEWTEXTFILE "$NewColLabelsLine\n";
 140       }
 141     }
 142     else {
 143       $Line = GetTextLine(\*TEXTFILE);
 144       if ($NewColLabelsLine) {
 145 	@ColLabels = quotewords($InDelim, 0, $Line);
 146 	if (@ColLabels != @NewColLabels) {
 147 	  warn "Warning: Ignoring file $TextFile: The number of columns in this file, ", scalar(@ColLabels), ", is different from the number of columns, ", scalar(@NewColLabels), ", in the first valid text file. \n";
 148 	  next FILELIST;
 149 	}
 150 	$ColLabelsLine = JoinWords(\@ColLabels, $OutDelim, $OutQuote);
 151 	if ($ColLabelsLine ne $NewColLabelsLine) {
 152 	  warn "Warning: Ignoring file $TextFile: The column names in this file are different from those in first valid text file.\nColumnlabels in first valid text file: $NewColLabelsLine \nColumnlabels in current text file: $ColLabelsLine\n";
 153 	  next FILELIST;
 154 	}
 155       }
 156       else {
 157 	@NewColLabels = quotewords($InDelim, 0, $Line);
 158 	$NewColLabelsLine = JoinWords(\@NewColLabels, $OutDelim, $OutQuote);
 159 	print NEWTEXTFILE "$NewColLabelsLine\n";
 160       }
 161     }
 162   }
 163   while ($Line = GetTextLine(\*TEXTFILE)) {
 164     if (!$Options{fast}) {
 165       @Words = quotewords($InDelim, 0, $Line);
 166       $Line = JoinWords(\@Words, $OutDelim, $OutQuote);
 167     }
 168     print NEWTEXTFILE "$Line\n";
 169   }
 170   close TEXTFILE;
 171 }
 172 close NEWTEXTFILE;
 173 
 174 print "$ScriptName:Done...\n\n";
 175 
 176 $EndTime = new Benchmark;
 177 $TotalTime = timediff ($EndTime, $StartTime);
 178 print "Total time: ", timestr($TotalTime), "\n";
 179 
 180 ###############################################################################
 181 
 182 # Setup script usage  and retrieve command line arguments specified using various options...
 183 sub SetupScriptUsage {
 184 
 185   # Retrieve all the options...
 186   %Options = ();
 187   $Options{label} = "yes";
 188   $Options{indelim} = "comma";
 189   $Options{outdelim} = "comma";
 190   $Options{quote} = "yes";
 191   if (!GetOptions(\%Options, "fast|f", "help|h", "indelim=s", "label|l=s", "outdelim=s", "overwrite|o", "quote|q=s", "root|r=s", "workingdir|w=s")) {
 192     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
 193   }
 194   if ($Options{workingdir}) {
 195     if (! -d $Options{workingdir}) {
 196       die "Error: The value specified, $Options{workingdir},  for option \"-w --workingdir\" is not a directory name.\n";
 197     }
 198     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
 199   }
 200   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
 201     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
 202   }
 203   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
 204     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
 205   }
 206   if ($Options{quote} !~ /^(yes|no)$/i) {
 207     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
 208   }
 209   if ($Options{label} !~ /^(yes|no)$/i) {
 210     die "Error: The value specified, $Options{label}, for option \"-l --label\" is not valid. Allowed values: yes or no\n";
 211   }
 212   $OutDelim = ($Options{outdelim} =~ /^tab$/i ) ? "\t" : (($Options{outdelim} =~ /^semicolon$/i) ? "\;" : "\,");
 213   $OutQuote = ($Options{quote} =~ /^yes$/i) ? 1 : 0;
 214 }
 215