Open Computing ``Hands-On'': ``Wizard's Grabbag'' Column: June, 1994: Listings

Listing 1: The shuffle script processes line-oriented data, catenating it, then extracting selected lines into specified files with possible ordering.

A. Listing of the shuffle Korn shell script:

  1  #!/usr/bin/ksh
  2  # @(#) shuffle Version 5  A rule-based list processor
  3  # Author: Thomas Baker <tbaker@unix.amherst.edu>
  4  # Modified by: Becca Thomas, February 1994
  5  $DBG_SH                         # Dormant debugging directive
  6  
  7  trap 'rm -f $Tmpfile $Targetfilenames >|$Devnull 2>&1; \
  8      exit $Stat' 0
  9  trap 'print -u2 "$(basename $0): Interrupted!"; exit' 1 2 3 15
 10  
 11  # CONFIGURATION
 12  Allfiles=combined.dat               # File for all catenated input files
 13  Bkupdir=.backup                     # Unix input-files backup directory
 14  #Bkupdir=backup                     # MKS  input-files backup directory
 15  Devnull="/dev/null"                 # Unix bit-bucket file
 16  Rulefile=.rules                     # Unix rule file
 17  #Rulefile=rules                     # MKS  rule file
 18  Usage="Usage: $(basename $0) datafile [datafile ...]" # Correct usage
 19  # Temporary directory-dependent variables:
 20  Tmpdir=/tmp                         # MKS/Unix temporary directory
 21  #Devnull=$Tmpdir/null               # MKS  bit-bucket file
 22  Targetfilenames=$Tmpdir/sht$$.tmp   # MKS/Unix target-names file
 23  Tmpfile=$Tmpdir/shf$$.tmp           # MKS/Unix temporary work file
 24  
 25  # FUNCTION DEFINITIONS:
 26  function usage_exit {
 27      print -u2 "$Usage"; Stat=1 ; exit
 28  }
 29  function movelines { # Args: $Searchkey $Source $Target $Sortcmd
 30      print -n "Lines with [$1] moved from \""$2"\" to \""$3"\""
 31      egrep "$1" $2 >>$3; egrep -v "$1" $2 >|$Tmpfile; mv $Tmpfile $2
 32      [ "$4" ] && print ", ${4}." || print "." # Print sort command
 33      [ "$4" ] && { eval $4 -o $3 $3 ||
 34          { print "\aBad rule-file sort command: $4"; Stat=2; exit;};}
 35  }
 36  
 37  # PROCESS COMMAND-LINE ARGUMENTS:
 38  case $# in      # User must specify at least one file-name argument
 39      0)  usage_exit ;;
 40  esac
 41  
 42  # SANITY CHECK: Rule file:
 43  [ -r $Rulefile ] ||
 44      { print -u2 "\aCannot read \"$Rulefile\" file!"; Stat=4; exit;}
 45  sed 's/#.*$//' $Rulefile |          # Remove comments.
 46  egrep -v '^$' |                     # Remove blank lines.
 47  nawk -F\| '                         # Rules separated by vertical bar
 48  NR == 1 && ($1 != "." || $2 != "$Allfiles") {   # Check first rule
 49      print $0, ": rule 1 is illegal!" }
 50  NF != 3 && NF != 4 {                # All rules have 3 or 4 fields.
 51      print $0, ": must have 3 or 4 fields!" }
 52  $2 == $3 {                          # Source different from target.
 53      print $0, ": source cannot equal target!" }
 54  $4 != "" && $4 !~ /^sort/ {         # Field 4 is for sort commands.
 55      print $0, ": field 4 is only for sort!" }
 56  $1 == "" || $2 == "" || $3 == "" {  # First three fields are non-empty.
 57      print $0, ": 1 of first 3 fields is empty!" }
 58  { target[$3] = 1 }                  # Note names of target files
 59  NR > 1 {                            # For all lines after the first
 60      if ($2 in target)               # If source file is also a target
 61          next;                       # No problem, fetch next input line
 62      else print $0, ": ", $2, "has no precedent!"
 63  }' >| $Tmpfile                      # Save unique lines and display
 64  [ -s $Tmpfile ] &&
 65      { print -u2 "Bad rule format:\n$(cat $Tmpfile)"; Stat=5; exit;}
 66  
 67  # SANITY CHECKS: Current directory, combined data, backup directory:
 68  [ -w "." ] ||                       # Current (data) directory
 69      { print -u2 "\aCannot write to current directory!"; Stat=6; exit;}
 70  [ -f $Allfiles ] &&                 # Combined data file
 71      { print -u2 "\a\"$Allfiles\" should not yet exist!"; Stat=7; exit;}
 72  [ -d $Bkupdir ] || mkdir $Bkupdir 2>|$Devnull ||
 73      { print -u2 "\aCannot make directory \"$Bkupdir\"!"; Stat=8; exit;}
 74  [ "$(ls $Bkupdir)" ] && {           # if there are files in backup dir
 75  print -n "Okay to erase files in $Bkupdir (y*|Y*/n)? "; read ans
 76  case $ans in
 77      y*|Y*)  rm -f $Bkupdir/* >|$Devnull 2>&1 ;; # Remove old backups
 78      *)      print "Exiting, check $Bkupdir directory."; Stat=0; exit ;;
 79  esac;}
 80  
 81  # CHECK DATA FILES, BACK UP, THEN COMBINE INTO A COMMON FILE:
 82  for File in "$@"; do
 83      [ -d $File ] && continue                # Ignore directories.
 84      [ "$File" = "$Rulefile" ] && continue   # Ignore rules (just data).
 85      [ "$(dirname $File)" = "." ] || [ "$(dirname $File)" = "$PWD" ] ||
 86          { print -u2 "\aData files must be in current directory!"
 87          Stat=9; exit;}
 88      [ -r $File ] ||
 89          { print -u2 "\a\"$File\" file not readable."; Stat=10; exit;}
 90      { file $File | egrep 'text|empty' >|$Devnull 2>&1;} ||
 91          { print -u2 "\a\"$File\" not text nor empty."; Stat=11; exit;}
 92      egrep '^[   ]*$' $File >|$Devnull 2>&1 &&
 93          { print -u2 "\a\"$File\" has blank lines!"; Stat=12; exit;}
 94      cp $File $Bkupdir ||    # Copy to backup directory.
 95          { print -u2 "\aCannot back up $File!"; Stat=13; exit;}
 96      cat $File >> $Allfiles; rm $File   # Combine into common file.
 97  done
 98  
 99  # CHECK COMBINED DATA FILE:
100  [ -s $Allfiles ] || { print -u2 "\aNo data to process!"; Stat=14; exit;}
101  Beforesize=$(wc -c <$Allfiles | awk '{ print $1 }') # Data size before
102  print "Data backed up to \"$Bkupdir\", concatenated in \"$Allfiles\"."
103  
104  # PROCESS DATA FILES under direction of rule file:
105  OldIFS="$IFS"               # Save old internal field separator char(s)
106  IFS="|"                     # Rule-file field separator for "read"
107  sed 's/#.*$//' $Rulefile |          # Remove rule-file comments
108  egrep -v '^$' |                     # Remove blank lines
109  while read Searchkey From To Sortcmd ; do   # put fields into variables
110      eval Source=$From; eval Target=$To      # interpolate these var.
111      movelines $Searchkey $Source $Target $Sortcmd # Do the shuffle
112      print -u3 "$Target"             # Output goes to fd 3.
113  done 3>| $Targetfilenames           # Store fd3 output in a file.
114  IFS="$OldIFS"                       # Restore original IFS values.
115  Targetnames=$(sort -u $Targetfilenames) # Place unique list in variable.
116  
117  # CONCLUSION: Cleanup and exit message:
118  for File in $Targetnames $Allfiles; do
119      [ -s $File ] || rm $File        # Erase data files if empty
120  done
121  if [ $Beforesize -ne $(cat $Targetnames 2>|$Devnull | wc -c) ]; then
122      print -u2 "Warning: data may have been lost--use backup!\a\a\a"
123  else
124      print -u2 "Done: data shuffled and intact!"
125  fi

B. A sample data file:

- 1994 Feb 23 Smith 01 John Lunch at Panda East.
- 1994 Jan 23 Smith 02 Not coming to session, but writing paper.
- 1994 Feb 23 Smith 03 FOLLOWUP Read Sep 1993 SCILS article on SGML
Smith John 432 E43rd St, New York NY 01002 212-555-5555, fax 666-6666
Feb 10 BDAY Sarah (1956)
LATER Read SCILS article on SGML.
NOW Renew passport!
Beans stock and info 800-221-4221, customer service 800-341-4341
Clothes Shoes Timberland "Blucher" size W12
Convert US Ounces to Grams: 1 oz = 28.35 gm
Wallet [07 Sep 93] NY Drivers' # A01234 56789 123456 78, exp 7/96
- 1993 Dec 20 10am Called John Smith, set appt and faxed letter.
Wallet [07 Sep 93] Visa 1234-5678-1234-5678, lost: 1-800-423-3823
Fastback differential backup of C: c:/fastback/fb ')c)b)d)s))'
Clothes Shoes Adidas Marath.Train.II 1CA, size 12.5(D) 48(F) 13(USA)

C. A sample rule file:

# Rule file for "Shuffle: a rule-based list processor"
# 1. Rules contain: searchkey|source|target|optional_sort_command
# 2. First rule must have "." in first field, "$Allfiles" in second.
# 3. Common sort types:
#    sort                        Straight alphabetic.
#    sort +0M -1 +1n -2          Data format: Jun 25
#    sort +1n -2 +2M -3 +3n -4   Data format: - 1992 Jun 25
.|$Allfiles|phone|sort
^- |phone|1993|sort +1n -2 +2M -3 +3n -4
^- 1994 |1993|1994|sort +1n -2 +2M -3 +3n -4
^Jan |phone|calendar
^Feb |phone|calendar
^Dec |phone|calendar|sort +0M -1 +1n -2
BDAY|calendar|bday|sort +0M -1 +1n -2
^NOW |phone|now|sort
^LATER |phone|later|sort

D. Another example of a data-file line:

Jan 23 Smith John Lunch at Panda East.

E. Some transformations of the data-file line shown above in Part D:

- 1994 Jan 23 Smith 01 John Lunch at Panda East.
- 1994 Jan 23 Smith 02 Not coming to session, but writing paper.
- 1994 Jan 23 Smith 03 FOLLOWUP Sep 1993 SCILS article on SGML
- 1994 Jan 23 Smith 04 FOLLOWUP Call Joachim Mann 321-4567
Mann Joachim, tel 321-4567
LATER Read Sep 1993 SCILS article on SGML.

F. Another example of a rule-file line:

FOLLOWUP|1994|followup|sort +1n -2 +2M -3 +3n -4

G. A version of shuffle written for Coherent that runs under the Bourne shell with the ``old'' awk.

  1  #!/usr/bin/sh
  2  # @(#) shuffle Version 5  A rule-based list processor
  3  # Author: Thomas Baker <tbaker@unix.amherst.edu>
  4  # Modified by: Becca Thomas, February 1994
  5  # Modified by: Ga'bor Zahemszky, March 1994 to use sh and "old" awk
  6  $DBG_SH                             # Dormant debugging directive
  7  
  8  trap 'rm -f $Tmpfile $Targetfilenames >$Devnull 2>&1; exit $Stat' 0
  9  trap 'echo "`basename $0`: Interrupted!" >&2 ; exit' 1 2 3 15
 10  
 11  # CONFIGURATION
 12  Allfiles=combined.dat               # File for all catenated input files
 13  Bkupdir=.backup                     # Unix input-files backup directory
 14  #Bkupdir=backup                     # MKS  input-files backup directory
 15  Devnull="/dev/null"                 # Unix bit-bucket file
 16  Rulefile=.rules                     # Unix rule file
 17  #Rulefile=rules                     # MKS  rule file
 18  Usage="Usage: `basename $0` datafile [datafile ...]" # Correct usage
 19  # Temporary directory-dependent variables:
 20  Tmpdir=/tmp                         # MKS/Unix temporary directory
 21  #Devnull=$Tmpdir/null               # MKS  bit-bucket file
 22  Targetfilenames=$Tmpdir/sht$$.tmp   # MKS/Unix target-names file
 23  Tmpfile=$Tmpdir/shf$$.tmp           # MKS/Unix temporary work file
 24  
 25  # FUNCTION DEFINITIONS:
 26  usage_exit() {
 27      echo "$Usage" >&2 ; Stat=1 ; exit
 28  }
 29  movelines() { # Args: $Searchkey $Source $Target $Sortcmd
 30      echo "Lines with [$1] moved from \""$2"\" to \""$3"\""
 31      egrep "$1" $2 >>$3; egrep -v "$1" $2 >$Tmpfile; mv $Tmpfile $2
 32      [ "$4" ] && echo ", ${4}." || echo "." # Print sort command
 33      [ "$4" ] && { eval $4 -o $3 $3 ||
 34          { echo "\007Bad rule-file sort command: $4"; Stat=2; exit;};}
 35  }
 36  
 37  # PROCESS COMMAND-LINE ARGUMENTS:
 38  case $# in      # User must specify at least one file-name argument
 39      0)  usage_exit ;;
 40  esac
 41  
 42  # SANITY CHECK: Rule file:
 43  [ -r $Rulefile ] ||
 44      { echo "\007Cannot read \"$Rulefile\" file!" >&2 ; Stat=4; exit;}
 45  sed 's/#.*$//' $Rulefile |          # Remove comments.
 46  egrep -v '^$' |                     # Remove blank lines.
 47  oawk -F\| '                         # Rules separated by vertical bar
 48  NR == 1 && ($1 != "." || $2 != "$Allfiles") {   # Check first rule
 49      print $0, ": rule 1 is illegal!" }
 50  NF != 3 && NF != 4 {                # All rules have 3 or 4 fields.
 51      print $0, ": must have 3 or 4 fields!" }
 52  $2 == $3 {                          # Source different from target.
 53      print $0, ": source cannot equal target!" }
 54  $4 != "" && $4 !~ /^sort/ {         # Field 4 is for sort commands.
 55      print $0, ": field 4 is only for sort!" }
 56  $1 == "" || $2 == "" || $3 == "" {  # First three fields are non-empty.
 57      print $0, ": 1 of first 3 fields is empty!" }
 58  { target[$3] = 1 }                  # Note names of target files
 59  NR > 1 {                            # For all lines after the first
 60      ZGvar2 = 0
 61      for (ZGvar1 in target) {
 62          if (ZGvar1 == $2) {
 63              next
 64          } else {
 65              ZGvar2 = 1
 66          }
 67       }
 68      if (ZGvar2 == 1) {
 69          print $0, ": ", $2, "has no precedent!"
 70      }
 71  }' > $Tmpfile                       # Save unique lines and display
 72  [ -s $Tmpfile ] &&
 73      { echo "Bad rule format:\n`cat $Tmpfile`" >&2 ; Stat=5; exit;}
 74  
 75  # SANITY CHECKS: Current directory, combined data, backup directory:
 76  [ -w "." ] ||                       # Current (data) directory
 77      { echo "\007Cannot write to current directory!" >&2 ; Stat=6; exit;}
 78  [ -f $Allfiles ] &&                 # Combined data file
 79      { echo "\007\"$Allfiles\" shouldn't exist!" >&2 ; Stat=7; exit;}
 80  [ -d $Bkupdir ] || mkdir $Bkupdir 2>$Devnull ||
 81      { echo "\007Can't make directory \"$Bkupdir\"!" >&2 ; Stat=8; exit;}
 82  [ "`ls $Bkupdir`" ] && {            # if there are files in backup dir
 83  echo "Okay to erase files in $Bkupdir (y*|Y*/n)? \c"; read ans
 84  case $ans in
 85      y*|Y*)  rm -f $Bkupdir/* >$Devnull 2>&1 ;;  # Remove old backups
 86      *)      echo "Exiting, check $Bkupdir directory."; Stat=0; exit ;;
 87  esac;}
 88  
 89  # CHECK DATA FILES, BACK UP, THEN COMBINE INTO A COMMON FILE:
 90  for File in $*; do
 91      [ -d $File ] && continue                # Ignore directories.
 92      [ "$File" = "$Rulefile" ] && continue   # Ignore rules (just data).
 93      [ "`dirname $File`" = "." ] || [ "`dirname $File`" = "`pwd`" ] ||
 94          { echo "\007Data files must be in current directory!" >&2
 95          Stat=9; exit;}
 96      [ -r $File ] ||
 97          { echo "\007\"$File\" file not readable." >&2 ; Stat=10; exit;}
 98      { file $File | egrep 'text|empty' >$Devnull 2>&1;} ||
 99          { echo "\007\"$File\" not text nor empty." >&2 ; Stat=11; exit;}
100      egrep '^[   ]*$' $File >$Devnull 2>&1 &&
101          { echo "\007\"$File\" has blank lines!" >&2 ; Stat=12; exit;}
102      cp $File $Bkupdir ||    # Copy to backup directory.
103          { echo "\007Cannot back up $File!" >&2 ; Stat=13; exit;}
104      cat $File >> $Allfiles; rm $File   # Combine into common file.
105  done
106  
107  # CHECK COMBINED DATA FILE:
108  [ -s $Allfiles ] || { echo "\007No data to process!">&2; Stat=14; exit;}
109  Beforesize=`wc -c <$Allfiles | oawk '{ print $1 }'` # Data size before
110  echo "Data backed up to \"$Bkupdir\", concatenated in \"$Allfiles\"."
111  
112  # PROCESS DATA FILES under direction of rule file:
113  OldIFS="$IFS"               # Save old internal field separator char(s)
114  IFS="|"                     # Rule-file field separator for "read"
115  sed 's/#.*$//' $Rulefile |          # Remove rule-file comments
116  egrep -v '^$' |                     # Remove blank lines
117  while read Searchkey From To Sortcmd ; do   # put fields into variables
118      eval Source=$From; eval Target=$To      # interpolate these var.
119      movelines $Searchkey $Source $Target $Sortcmd # Do the shuffle
120      echo "$Target" >&3              # Output goes to fd 3.
121  done 3> $Targetfilenames            # Store fd3 output in a file.
122  IFS="$OldIFS"                       # Restore original IFS values.
123  Targetnames=`sort -u $Targetfilenames`  # Place unique list in variable.
124  
125  # CONCLUSION: Cleanup and exit message:
126  for File in $Targetnames $Allfiles; do
127      [ -s $File ] || rm $File        # Erase data files if empty
128  done
129  if [ $Beforesize -ne `cat $Targetnames 2>$Devnull | wc -c` ]; then
130      echo "Warning: data may have been lost--use backup!\007" >&2
131  else
132      echo "Done: data shuffled and intact!" >&2
133  fi

Figure 1: A data-flow diagram for the example discussed in Tom Baker's introductory letter.

$Allfiles
   |
   V                                      Sorted by year:
 phone ---> 1993 [^- ] -----\-----------> 1994 [^- 1994 ]
   |                         \----------> 1993 (everything else) 
   |
   V                                      Sorted by month:
 phone ---> calendar [^Jan,^Feb..] \----> bday [BDAY]
   |                                \---> calendar (everything else)
   |
   V        Sorted alphabetically:
 phone ---> now [^NOW ]
   |
 phone ---> later [^LATER ]
   |
   \------> phone (everything else)