#!/usr/local/bin/ruby

##
# @file      charAnyl.rb
# @author    Mitch Richling <https://www.mitchr.me/>
# @Copyright Copyright 2008 by Mitch Richling.  All rights reserved.
# @Revision  $Revision: 1.4 $ 
# @SCMdate   $Date: 2011/06/23 19:49:29 $
# @brief     Count chars in STDIN and summarize.@EOL
# @Keywords  
# @Std       Ruby 1.8
#
#            Read data from a file or from STDIN, and print out
#            various bits of statistical information.
#
#            For the options, a - turns it off and a + turns it
#            on. The sign given in the table is the opposite of the
#            default. Options must appear before any file names on the
#            command line.
#              -counts Character count table
#              -color   Color in character count table
#              -wide    Wide format for character count table
#              -high    Suppress the upper 128 counts in the table for ASCII input
#              -sum    Character count summary
#
#            TODO
#
#              * Add line count
#              * Add line length stats (min, max, mean, sd)
#              * Add ws/non-ws char counts per line



# Set defaults and process command line arguments
prtAllCnts = true
prtSumm    = true
doColor    = true
noHiIfPos  = true
wideFmt    = true
ARGV.each do |cArg|
  if(cArg == '-counts') then
    prtAllCnts = false
  elsif(cArg == '+counts') then
    prtAllCnts = true
  elsif(cArg == '-sum') then
    prtSumm = false
  elsif(cArg == '+sum') then
    prtSumm = true
  elsif(cArg == '-color') then
    doColor = false
  elsif(cArg == '+color') then
    doColor = true
  elsif(cArg == '-wide') then
    wideFmt = false
  elsif(cArg == '+wide') then
    wideFmt = true
  elsif(cArg == '-high') then
    noHiIfPos = false
  elsif(cArg == '+high') then
    noHiIfPos = true
  end
end

inFile = $stdin
if ( !(ARGV[0].nil?)) then
  inFile = open(ARGV[0], 'r')
end

# Read the data
charCounts = Array.new
lastChar = nil
cntCrLf = 0
inFile.each_byte do |b|
  charCounts[b] = (charCounts[b] || 0) + 1
  if ((lastChar == 13) && (b == 10)) then
    cntCrLf += 1
  end
  lastChar = b
end

# Compute some stats...
highCnt   = highUnq   = 0
chrCnt    = chrUnq    = 0
ucCnt     = ucUnq     = 0
lcCnt     = lcUnq     = 0
digCnt    = digUnq    = 0
puncCnt   = puncUnq   = 0
wsLowCnt  = wsLowUnq  = 0
spcCnt    = spcUnq    = 0
nprLowCnt = nprLowUnq = 0
pLenMax = 4
chIdxCnt = Array.new
0.upto(255) do |idx|
  cnt = charCounts[idx] || 0
  charCounts[idx] = cnt
  chIdxCnt.push([idx, cnt])
  pLenMax = [ sprintf("%d", cnt).length, pLenMax].max
  if (idx <= 32) then
    if ((idx >= 9) && (idx <= 13)) then
      wsLowCnt += cnt
      wsLowUnq += ( cnt>0 ? 1 : 0)
    elsif (idx == 32) then
      spcCnt += cnt
      spcUnq += ( cnt>0 ? 1 : 0)
    else
      nprLowCnt += cnt
      nprLowUnq += ( cnt>0 ? 1 : 0)
    end
  elsif (idx > 126)
    highCnt += cnt
    highUnq += ( cnt>0 ? 1 : 0)
  else
    if ( (idx >= 97) && (idx <= 122) ) then
      lcCnt += cnt
      lcUnq += ( cnt>0 ? 1 : 0)
    elsif ( (idx >= 65) && (idx <= 90) ) then
      ucCnt += cnt
      ucUnq += ( cnt>0 ? 1 : 0)
    elsif ( (idx >= 48) && (idx <= 57) ) then
      digCnt += cnt
      digUnq += ( cnt>0 ? 1 : 0)
    else
      puncCnt += cnt
      puncUnq += ( cnt>0 ? 1 : 0)
    end
  end
  chrCnt += cnt
  chrUnq += ( cnt>0 ? 1 : 0)
end
wsCnt     = wsLowCnt + spcCnt;
anCnt     = ucCnt + lcCnt + digCnt
prCnt     = anCnt + puncCnt + wsCnt
nonPrtCnt = nprLowCnt + highCnt
wsUnq     = wsLowUnq + spcUnq;
anUnq     = ucUnq + lcUnq + digUnq
prUnq     = anUnq + puncUnq + wsUnq
nonPrtUnq = nprLowUnq + highUnq

manPrt = prCnt + charCounts[8]

# Find the smallest group of chars that make up 50% of the total char count
target  = chrCnt/2
curSum  = 0
topList = Hash.new
(chIdxCnt.sort { |x,y| [y[1], y[0]] <=> [x[1],x[0]] }).each do |idx, cnt|
  curSum += cnt
  topList[idx] = 1
  if (curSum >= target) then
    break
  end
end

# Print the char counts
if (prtAllCnts) then
  desc = [ "NUL", "SOH", "STX", "ETX", "EOT", "NEQ", "ACK", "BEL", "BS ",
           "HT ", "NL ", "VT ", "NP ", "CR ", "SO ", "SI ", "DLE", "DC1",
           "DC2", "DC3", "DC4", "NAK", "SYN", "ETB", "CAN", "EM ", "SUB",
           "ESC", "FS ", "GS ", "RS ", "US ", "SP " ]
  desc[127] = 'DEL'  
  128.upto(255) { |i| desc[i] = 'N/A' }
  0.upto(127) { |i| desc[i] = ( desc[i] ? desc[i] : i.chr ) }

  numRows = numCols = 0
  if (wideFmt) then
    numRows = 32
    numCols = 8
  else
    numRows = 64
    numCols = 4
  end

  # Suppress the display of the non-ASCII part of the table if it is empty
  chrCappa = 255
  if (noHiIfPos && (highCnt == 0)) then
    chrCappa = 127
  end
  
  # Print titles
  0.upto(numCols-1) do |colNum|
    idx = colNum*numRows;
    if (idx <= chrCappa) then
      if (idx <= 127) then
        printf("| %3s %3s %2s %-3s : %#{pLenMax}s ", 'Oct', 'Dec', 'Hx', 'Chr', 'Cnt')
      else
        printf("| %3s %3s %2s : %#{pLenMax}s ", 'Oct', 'Dec', 'Hx', 'Cnt')
      end
    end
  end
  print "|\n";

  # Print table
  0.upto(numRows-1) do |rowNum|
    printf("|");
    0.upto(numCols-1) do |colNum|
      idx = rowNum+colNum*numRows
      if (idx <= chrCappa) then
        cStart = cEnd = ''
        if (doColor && (charCounts[idx] > 0)) then
          if (topList.member?(idx)) then
            cStart = "\e[0;43m"
            cEnd = "\e[0m"
          else
            cStart = "\e[0;46m"
            cEnd = "\e[0m"
          end
        end
        if (idx <= 127) then
          printf(" %s%03o %3d %02x %-3s : %#{pLenMax}d%s |", cStart, idx, idx, idx, desc[idx], charCounts[idx], cEnd)
        else
          printf(" %s%03o %3d %02x : %#{pLenMax}d%s |", cStart, idx, idx, idx, charCounts[idx], cEnd)
        end
      end
    end
    print "\n"
  end
  print "\n"
end

# Guess at the line ending mode
eolCnt = charCounts[10] + charCounts[13]
eolMode = "UNKNOWN"
if (eolCnt > 0) then
  if ((charCounts[10] > 0) && (charCounts[10] * 0.1 > charCounts[13])) then
    if ((charCounts[10] > 0) && (charCounts[13] == 0)) then
      eolMode = "UNIX"
    else
      eolMode = "UNIX -- ALMOST"
    end
  elsif ((charCounts[13] > 0) && (charCounts[13] * 0.1 > charCounts[10])) then
    if ((charCounts[13] > 0) && (charCounts[10] == 0)) then
      eolMode = "MacOS 9"
    else
      eolMode = "MacOS 9 -- ALMOST"
    end
  elsif ((cntCrLf > 0) && (cntCrLf * 0.1 > (charCounts[10]-cntCrLf)) && (cntCrLf * 0.1 > (charCounts[13]-cntCrLf))) then
    if ((cntCrLf > 0) && (charCounts[10] == charCounts[13]) && (cntCrLf == charCounts[10])) then
      eolMode = "MSDOS"
    else
      eolMode = "MSDOS -- ALMOST"
    end
  end
else
  eolMode = "NONE"
end

# Print out some stats
if (prtSumm && (chrCnt > 0)) then
  prWid = sprintf("%d", chrCnt).length + 1
  (0<chrCnt)            && printf("Character Class Breakdown\n")
  (0<chrCnt)            && printf("  Chars .............. %#{prWid}d  Chars (%d/256)                                   \n", chrCnt,                                         chrUnq)
  (0<prCnt)             && printf("    Printable ........ %#{prWid}d  | %5.1f%% --- Printable (%d/100)                 \n", prCnt,           100*prCnt/chrCnt,              prUnq)
  (0<puncCnt)           && printf("      Punct .......... %#{prWid}d  |            | %5.1f%% --- Punctuation (%d/32)   \n", puncCnt,         100*puncCnt/prCnt,             puncUnq)
  (0<charCounts[40])    && printf("        Par ( ........ %#{prWid}d  |            |            | %5.1f%%              \n", charCounts[40],  100*charCounts[40]/puncCnt)
  (0<charCounts[41])    && printf("        Par ) ........ %#{prWid}d  |            |            | %5.1f%%              \n", charCounts[41],  100*charCounts[41]/puncCnt)
  (0<charCounts[91])    && printf("        Sqr [ ........ %#{prWid}d  |            |            | %5.1f%%              \n", charCounts[91],  100*charCounts[91]/puncCnt)
  (0<charCounts[93])    && printf("        Sqr ] ........ %#{prWid}d  |            |            | %5.1f%%              \n", charCounts[93],  100*charCounts[93]/puncCnt)
  (0<charCounts[123])   && printf("        Cur { ........ %#{prWid}d  |            |            | %5.1f%%              \n", charCounts[123], 100*charCounts[123]/puncCnt)
  (0<charCounts[125])   && printf("        Cur } ........ %#{prWid}d  |            |            | %5.1f%%              \n", charCounts[125], 100*charCounts[125]/puncCnt)
  (0<anCnt)             && printf("      Alphanumeric ... %#{prWid}d  |            | %5.1f%% --- Alphanumeric (%d/36)  \n", anCnt,           100*anCnt/prCnt,               anUnq)
  (0<ucCnt)             && printf("        Upper ........ %#{prWid}d  |            |            | %5.1f%%              \n", ucCnt,           100*ucCnt/anCnt)
  (0<lcCnt)             && printf("        Lower ........ %#{prWid}d  |            |            | %5.1f%%              \n", lcCnt,           100*lcCnt/anCnt)
  (0<digCnt)            && printf("        Digit ........ %#{prWid}d  |            |            | %5.1f%%              \n", digCnt,          100*digCnt/anCnt)
  (0<wsCnt)             && printf("      Whitespace ..... %#{prWid}d  |            | %5.1f%% --- Whitespace (%d/6)     \n", wsCnt,           100*wsCnt/prCnt,               wsUnq)
  (0<spcCnt)            && printf("        Spaces ....... %#{prWid}d  |                         | %5.1f%%              \n", spcCnt,          100*spcCnt/wsCnt)
  (0<wsLowCnt)          && printf("        Below 32 ..... %#{prWid}d  |                         | %5.1f%% --- Below 32 \n", wsLowCnt,        100*wsLowCnt/wsCnt)
  (0<charCounts[9])     && printf("          H Tabs ..... %#{prWid}d  |                                      | %5.1f%% \n", charCounts[9],   100*charCounts[9]/wsLowCnt)
  (0<charCounts[11])    && printf("          V Tabs ..... %#{prWid}d  |                                      | %5.1f%% \n", charCounts[11],  100*charCounts[11]/wsLowCnt)
  (0<charCounts[10])    && printf("          Newlines ... %#{prWid}d  |                                      | %5.1f%% \n", charCounts[10],  100*charCounts[10]/wsLowCnt)
  (0<charCounts[13])    && printf("          C Returns .. %#{prWid}d  |                                      | %5.1f%% \n", charCounts[13],  100*charCounts[13]/wsLowCnt)
  (0<charCounts[12])    && printf("          New Page ... %#{prWid}d  |                                      | %5.1f%% \n", charCounts[12],  100*charCounts[12]/wsLowCnt)
  (0<nprLowCnt+highCnt) && printf("    Non-printable .... %#{prWid}d  | %5.1f%% --- Non-printable (%d/156)             \n", nonPrtCnt,       100*nonPrtCnt/chrCnt,          nonPrtUnq)
  (0<nprLowCnt)         && printf("      Below 32 ....... %#{prWid}d               | %5.1f%% --- Below 32 (%d/27)      \n", nprLowCnt,       100*nprLowCnt/nonPrtCnt,       nprLowUnq)
  (0<charCounts[8])     && printf("        BSP .......... %#{prWid}d               |            | %5.1f%%              \n", charCounts[8],   100*charCounts[8]/nprLowCnt)
  (0<highCnt)           && printf("      Above 126 ...... %#{prWid}d               | %5.1f%% --- Above 32 (%d/129)     \n", highCnt,         100*highCnt/nonPrtCnt,         highUnq)
  (0<charCounts[127])   && printf("        DEL .......... %#{prWid}d                            | %5.1f%%              \n", charCounts[127], 100*charCounts[127]/highCnt)


  (0<eolCnt)            && printf("Line Ending Analysis\n")
  (0<charCounts[10])    && printf("  LF ............ %d\n", charCounts[10])
  (0<charCounts[13])    && printf("  CR ............ %d\n", charCounts[13])
  (0<eolCnt)            && printf("  CR/LF Pairs ... %d\n", cntCrLf)
  (0<eolCnt)            && printf("  EOL Mode ...... %s\n", eolMode)

  (0<chrCnt)            && printf("Other statistics\n")
  (0<manPrt)            && printf("  Man Printable ..... %5.1f%%\n", 100.0*manPrt/chrCnt)
end