%\documentclass[colorBG,slideColor,troispoints,pdf]{prosper}
%\documentclass[colorBG,total,slideColor,pdf]{prosper}
\documentclass[pdf]{prosper}
%\documentclass[colorBG,slideColor,ps]{prosper}

\usepackage[toc,highlight,Tycja]{HA-prosper}

\usepackage{alltt,key,xr,cols,rcs,acro,nick,%
  graphicx,varioref,explanation,booktabs}

\RCS $Revision: 1.9 $

% Copyright (c) 2004 by Nick Urbanik <nicku(at)nicku.org>.
% This material may be distributed only subject to the terms and
% conditions set forth in the Open Publication License, v1.0 or later
% (the latest version is presently available at
% http://www.opencontent.org/openpub/).
                                                                                % For Prosper sizing: see
% http://sourceforge.net/mailarchive/forum.php?thread_id=1373415&forum_id=4400
% export GS_OPTIONS="-sPAPERSIZE=a4"

\externaldocument[lt-]%
{../../linux_training-plus-config-files-ossi/build/masterfile}

% \usepackage[pdfpagemode=None,pdfauthor={Nick Urbanik}]{hyperref}

\renewcommand*{\bs}{\texttt{\char '134}} % Backslash `\'
\newcommand*{\labTitle}{Shell Programming---an Introduction}
\newcommand*{\subject}{Operating Systems and Systems Integration}

\providecommand*{\RPM}{\acro{RPM}\xspace}
\providecommand*{\CD}{\acro{CD}\xspace}

\title{Shell Programming---an Introduction}
%% \author{Nick Urbanik}
%% \email{nicku@vtc.edu.hk}
%% \institution{A computing department}
%% \slideCaption{OSSI --- Shell Programming --- ver. \RCSRevision}
\author{Nick Urbanik\\
\email{nicku@vtc.edu.hk}\\
\institution{A computing department}}
\slideCaption{OSSI --- Shell Programming --- ver. \RCSRevision}
\subtitle{\footnotesize{}Copyright Conditions: Open Publication License (see
  \url{http://www.opencontent.org/openpub/})}%
%% \subtitle{\footnotesize{}Copyright Conditions: GNU FDL (see
%% \url{http://www.gnu.org/licenses/fdl.html})}
%%\Logo{\includegraphics[width=15mm]{ict-logo-smaller}}
\setlength{\extrarowheight}{0pt}

\DefaultTransition{Wipe}
\TitleSlideNav{FullScreen}
\NormalSlideNav{ShowBookmarks}
\LeftFoot{OSSI --- ver. \RCSRevision}
\RightFoot{Shell Programming}

\begin{document}
\maketitle

%\tableofcontents

\begin{slide}{Aim}
\small
After successfully working through this exercise, You will:
\begin{itemize}
\item write simple shell scripts using \texttt{for}, \texttt{if},
  \texttt{while}, \texttt{case}, \texttt{getopts} statements;
\item write shell script functions, and be able to handle parameters;
\item understand basic regular expressions, and be able to create your
  own regular expressions;
\item understand how to execute and debug these scripts;
\item understand some simple shell scripts written by others.
\end{itemize}
\end{slide}

\begin{slide}{Why Shell Scripting?}
  
\begin{itemize}
\item Basic startup, shutdown of Linux, \UNIX systems uses large
  number of shell scripts
  \begin{itemize}
  \item understanding shell scripting important to understand and perhaps
    modify behaviour of system
  \end{itemize}
  
\item Very high level: powerful script can be very short
\item Can build, test script incrementally
\item Useful on the command line: ``one liners''
\end{itemize}
\end{slide}

\begin{slide}{Where to get more information}
  \label{sld:references}
  \begin{itemize}
  \item the Libarary has two copies of the book, \emph{Learning the Bash
    Shell}, second edition, by Cameron Newham \& Bill Rosenblatt,
    O'Reilly, 1998. 

  \item There is a free on-line book about shell programming at:
    \url{http://tldp.org/LDP/abs/html/index.html} and
    \url{http://tldp.org/LDP/abs/abs-guide.pdf}.  It has hundreds of
    pages, and is packed with \emph{examples}.

  \item The handy
    reference to shell programming is:
\begin{verbatim}
$ pinfo bash
\end{verbatim}%$
    or
\begin{verbatim}
$ man bash
\end{verbatim}%$
  \end{itemize}
\end{slide}

\begin{slide}{The Shell is an Interpreter}
  \begin{itemize}
  \item Some languages are compiled: C, \C++, Java,\dots
  \item Some languages are interpreted: Java bytecode, Shell
  \item Shell is an interpreter: kernel does not run shell program
    directly:
    \begin{itemize}
    \item kernel runs the shell program \texttt{/bin/sh} with script
      file name as a parameter
    \item the kernel cannot execute the shell script directly, as it
      can a binary executable file that results from compiling a C program
    \end{itemize}
  \end{itemize}
\end{slide}


\begin{slide}{The Shebang}
  \begin{itemize}
  \item Linux kernel starts shell script
  \item kernel reads first two characters of each executable
    \begin{itemize}
    \item If first 2 chars are ``\#!'' then
    \item kernel executes the name that follows, with the file name of
    the script as a parameter
    \end{itemize}
  \item Example: a file called \texttt{find.sh} has this as the first
    line:
\begin{alltt}
#! /bin/sh
\end{alltt}
  \item then kernel executes this:
\begin{alltt}
/bin/sh find.sh
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Making the script executable}
  \label{sld:making-script-executable}
  To easily execute a script, it should:
  \begin{itemize}
  \item be on the \texttt{PATH}
  \item have execute permission.
  \end{itemize}
  How to do each of these?
  \begin{itemize}
  \item Red Hat Linux by default, includes the directory
    $\sim$\texttt{/bin} on the \texttt{PATH}, so create this
    directory, and put your scripts there:
\begin{alltt}
$ \textbf{mkdir \(\sim\)/bin}
\end{alltt}%$
  \item If your script is called \texttt{script}, then this command
    will make it executable:
\begin{alltt}
$ \textbf{chmod +x script}
\end{alltt}%$
  \end{itemize}
\end{slide}

\begin{slide}{Special Characters}
  \begin{itemize}
  \item Many characters are \emph{special} to the shell, and have a
    particular meaning to the shell.
  \end{itemize}
  \vspace*{-1ex}
\scriptsize\setlength{\extrarowheight}{0pt}
  \begin{tabular}[t]{@{}>{\ttfamily}cll@{}}
    \toprule%
    \multicolumn{1}{c}{\textbf{Character}} & \textbf{Meaning} &
    \multicolumn{1}{c@{}}{\textbf{See slide}}\\
    \midrule%
    $\sim$ & Home directory & \S\,\pageref{sld:making-script-executable}\\
    ` & Command substitution. Better: \texttt{\$(\ldots)} &
    \S\,\pageref{sld:command-substitution} \\
    \# & Comment &  \\
    \$ & Variable expression & \S\,\pageref{sld:variables} \\
    \& & Background Job &
    \vref{lt-sec:basic-shell-and-tools-approach-background-processes} \\
    \texttt{*} & File name matching wildcard &
    \vref{lt-sec:basic-shell-and-tools-approach-filename-generation} \\
    \verb+|+ & Pipe & \vref{lt-sec:basic-shell-and-tools-approach-pipelines} \\
    \bottomrule
  \end{tabular}
\end{slide}

\begin{slide}{Special Characters---continued: 2}
  \scriptsize\setlength{\extrarowheight}{0pt}
  \begin{tabular}[t]{@{}>{\ttfamily}cll@{}}
    \toprule%
    \multicolumn{1}{c}{\textbf{Character}} & \textbf{Meaning} &
    \multicolumn{1}{c@{}}{\textbf{See slide}}\\
    \midrule%
    ( & Start subshell & \S\,\pageref{sld:change-IFS},
    \pageref{sld:vars-in-subshell}, \pageref{sld:blocks} \\
    ) & End subshell &
    \S\,\pageref{sld:change-IFS}, \pageref{sld:vars-in-subshell}, \pageref{sld:blocks} \\
    {[} & Start character set file name matching &
    \vref{lt-sec:basic-shell-and-tools-approach-pipelines} \\
    {]} & End character set file name matching &
    \vref{lt-sec:basic-shell-and-tools-approach-pipelines} \\
    \{ & Start command block & \S\,\pageref{sld:blocks} \\
    ; & Command separator & \S\,\pageref{sld:error-handling} \\
    \bs & Quote next character & \S\,\pageref{sld:more-about-quoting} \\
    ' & Strong quote & \S\,\pageref{sld:more-about-quoting} \\
    " & Weak quote & \S\,\pageref{sld:more-about-quoting} \\
    \bottomrule
  \end{tabular}
\end{slide}

\begin{slide}{Special Characters---continued: 3}
  \scriptsize\setlength{\extrarowheight}{0pt}
  \begin{tabular}[t]{@{}>{\ttfamily}cll@{}}
    \toprule%
    \multicolumn{1}{c}{\textbf{Character}} & \textbf{Meaning} &
    \multicolumn{1}{c@{}}{\textbf{See slide}}\\
    \midrule%
    \texttt{<} & Input redirect &
    \vref{lt-sec:basic-shell-and-tools-approach-input-redirection} \\
    \texttt{>} & Output redirect &
    \vref{lt-sec:basic-shell-and-tools-approach-output-redirection} \\
    \texttt{/} & Pathname directory separator & \\
    ? & Single-character match in filenames &
    \vref{lt-sec:basic-shell-and-tools-approach-filename-generation} \\
    ! & Pipline logical NOT & \S\,\pageref{sld:not-operator}\\
    \bottomrule
  \end{tabular}
  \begin{itemize}
  \item Note that references to pages in the tables above refer to the
    modules in the workshop notes
  \end{itemize}
\end{slide}

\begin{slide}{Quoting}
\label{sld:quoting}
\begin{itemize}
\item Sometimes you want to use a special character
  \texttt{literally}; i.e., without its special meaning.
\item Called \texttt{quoting}
\item Suppose you want to print the string: \texttt{2 * 3 > 5 is a valid
    inequality}?
\item If you did this:
\begin{alltt}
$ \textbf{echo 2 * 3 > 5 is a valid inequality}
\end{alltt}%$
the new file `\texttt{5}' is created, containing the character
`\texttt{2}', then the names of all the files in the current
directory, then the string ``\texttt{3 is a valid inequality}''.
\end{itemize}
\end{slide}

\begin{slide}{Quoting---2}
\label{sld:quoting2}
\begin{itemize}
\item To make it work, you need to protect the special characters
  `\texttt{*}' and `\texttt{>}' from the shell by quoting them.  There
  are three methods of quoting:
  \begin{itemize}
  \item Using double quotes (``weak quotes'')
  \item Using single quotes (``strong quotes'')
  \item Using a backslash in front of each special character you want
    to quote
  \end{itemize}
\item This example shows all three:
\begin{alltt}
$ \textbf{echo "2 * 3 > 5 is a valid inequality"}
$ \textbf{echo '2 * 3 > 5 is a valid inequality'}
$ \textbf{echo 2 \bs* 3 \bs> 5 is a valid inequality}
\end{alltt}%$
\end{itemize}
\end{slide}

\begin{slide}{Quoting---When to use it?}
  \begin{itemize}
  \item Use quoting when you want to pass special characters to
    another program.
  \item Examples of programs that often use special characters:
    \begin{itemize}
    \item \texttt{find}, \texttt{locate}, \texttt{grep},
      \texttt{expr}, \texttt{sed} and \texttt{echo}
    \end{itemize}
  \item Here are examples where quoting is required for the program to
  work properly:
\begin{alltt}
$ \textbf{find . -name \bs*.jpg}
$ \textbf{locate '/usr/bin/c*'}
$ \textbf{grep 'main.*(' *.c}
$ \textbf{i=`expr i \bs* 5`}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{True and False}
  \begin{itemize}
  \item Shell programs depend on executing external programs
  \item When any external program execution is successful, the exit
    status is zero, 0
  \item An error results in a non-zero error code
  \item To match this, in shell programming:
    \begin{itemize}
    \item The value 0 is true
    \item any non-zero value is false
    \end{itemize}
  \item This is opposite from other programming languages
  \end{itemize}
\end{slide}

\begin{slide}{Variables---1}
  \label{sld:variables}
  \begin{itemize}
  \item Variables not declared; they just appear when assigned to
  \item Assignment:
    \begin{itemize}
    \item no dollar sign
    \item no space around equals sign
    \item examples:
\begin{alltt}
$ \textbf{x=10}   # \textnormal{correct}
$ \textbf{x = 10} # \textnormal{wrong: try to execute program called ``\texttt{x}''}
\end{alltt}
    \end{itemize}
  \item Read value of variable:
    \begin{itemize}
    \item put a `\texttt{\$}' in front of variable name
    \item example:
\begin{alltt}
$ \textbf{echo "The value of x is $x"}
\end{alltt}
    \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{Variables---Assignments}
  \begin{itemize}
  \item You can put multiple assignments on one line:
\begin{alltt}
i=0 j=10 k=100
\end{alltt}

  \item You can set a variable temporarily while executing a program:
\begin{alltt}
$ \textbf{echo $EDITOR}
emacsclient
$ \textbf{EDITOR=gedit crontab -e}
$ \textbf{echo $EDITOR}
emacsclient
\end{alltt}%$
  \end{itemize}
\end{slide}

\begin{slide}{Variables---Local to Script}
  \label{sld:vars-in-subshell}
  \begin{itemize}
  \item Variables disappear after a script finishes
  \item Variables created in a sub shell disappear
    \begin{itemize}
    \item parent shell cannot read variables in a sub shell
    \item example:
\begin{alltt}
$ \textbf{cat variables}
#! /bin/sh
echo $HOME
HOME=happy
echo $HOME
$ \textbf{./variables}
/home/nicku
happy
$ \textbf{echo $HOME}
/home/nicku
\end{alltt}
    \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{Variables---Unsetting Them}
  \begin{itemize}
  \item You can make a variable hold the null string by assigning it
    to nothing, but it does not disappear totally:
\begin{alltt}
$ \textbf{VAR=}
$ \textbf{set | grep '^VAR'}
VAR=
\end{alltt}
  \item You can make it disappear totaly using \texttt{unset}:
\begin{alltt}
$ \textbf{unset VAR}
$ \textbf{set | grep '^VAR'}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Command-line Parameters}
  \begin{itemize}
  \item Command-line parameters are called \texttt{\$0}, \texttt{\$1},
    \texttt{\$2}, \ldots
  
  \item Example: when call a shell script called
  ``\texttt{shell-script}'' like this:
\begin{alltt}
$ \textbf{shell-script param1 param2 param3 param4}
\end{alltt}%$
    \vspace*{-1ex}
    {\scriptsize
    \begin{tabular}[t]{@{}ll@{}}
      \toprule%
      \emph{variable} & \emph{value}\\
      \midrule%
      \texttt{\$0} & \texttt{shell-script}\\
      \texttt{\$1} & \texttt{param1}\\
      \texttt{\$2} & \texttt{param2}\\
      \texttt{\$3} & \texttt{param3}\\
      \texttt{\$4} & \texttt{param4}\\
      \texttt{\$\#} & number of parameters to the program, e.g., 4\\
      \bottomrule
    \end{tabular}}
  \begin{itemize}
  \item Note: these variables are read-only.
  \end{itemize}
\end{itemize}
\end{slide}

\begin{slide}{Special Built-in Variables}
  \begin{itemize}
  \item Both \texttt{\$@} and \texttt{\$*} are a list of all the
    parameters.
  \item The only difference between them is when they are quoted in
    quotes---see manual page for \texttt{bash}
  \item \texttt{\$?} is exit status of last command
  \item \texttt{\$\$} is the process ID of the current shell
  \item Example shell script:
\begin{alltt}
#! /bin/sh
echo $0 is the full name of this shell script
echo first parameter is $1
echo first parameter is $2
echo first parameter is $3
echo total number of parameters is $#
echo process ID is $$
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Variables: use Braces \texttt{\$\{\ldots\}}}
  \begin{itemize}
  \item It's good to put braces round a variable name when getting its value
  \item Then no problem to join its value with other text:
\begin{alltt}
$ \textbf{test=123}
$ \textbf{echo $\{test\}}
123
\textbf{# No good, variable $test456 is undefined:}
$ \textbf{echo $test456}
 
$ \textbf{echo $\{test\}456}
123456
\end{alltt}%
\end{itemize}
\end{slide}

\begin{slide}{Braces and Parameters after \textbf{\$9}}
  \begin{itemize}
  \item Need braces to access parameters after \texttt{\$9}:
\begin{alltt}
$ \textbf{cat paramten}
#! /bin/sh
echo $10
echo $\{10\}
$ \textbf{./paramten a b c d e f g h i j}
a0
j
\end{alltt}

\item Notice that \texttt{\$10} is the same as \texttt{\$\{1\}0}, i.e., the
first parameter ``\texttt{a}'' then the literal character zero
``\texttt{0}''
  \end{itemize}
\end{slide}

\begin{slide}{More about Quoting}
  \label{sld:more-about-quoting}
  \begin{itemize}
  \item Double quotes: \texttt{"..."} stops the special behaviour of
    all special characters, except for:
    \begin{itemize}
    \item variable interpretation (\texttt{\$})
    \item backticks (\texttt{`}) --- see slide~\ref{sld:command-substitution}
    \item the backslash (\bs)
    \end{itemize}
  \item Single quotes \texttt{'...'}:
    \begin{itemize}
    \item stop the special behaviour of \emph{all} special
      characters
    \end{itemize}
  \item Backslash:
    \begin{itemize}
    \item preserves literal behaviour of character, except for
      newline; see
      slides~\S\pageref{sld:conditions-combining-comparisons},
      \S\pageref{sld:arithmetic-expressions-with-((...))},
      \S\pageref{sld:for-statement-1}
    \item Putting ``\bs'' at the end of the line lets you continue a
    long line on more than one physical line, but the shell will treat
    it as if it were all on one line.
    \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{Command Substitution}
  \label{sld:command-substitution}
  \begin{itemize}
  \item Enclose command in \texttt{\$(...)} or backticks:%$
    \texttt{`...`}
  \item Means, ``Execute the command called within these quotes and
    put the output back here.''
  \item Here is an example using \texttt{expr}:
\begin{alltt}
$ \textbf{expr 3 + 2}
5
$ \textbf{i=expr 3 + 2}    # \textnormal{error: try execute command `\texttt{3}'}
$ \textbf{i=$(expr 3 + 2)} # \textnormal{correct}
$ \textbf{i=`expr 3 + 2`}  # \textnormal{also correct}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Command Substitution---Example}
  \begin{itemize}
  \item 
\begin{alltt}
$ \textbf{hostname}
nickpc.tyict.vtc.edu.hk
$ \textbf{h=hostname}
$ \textbf{echo $h}
hostname
$ \textbf{h=$(hostname)}
$ \textbf{echo $h}
nickpc.tyict.vtc.edu.hk
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Conditions---String Comparisons}
  \begin{itemize}
  \item All programming languages depend on \emph{conditions} for
    \texttt{if} statements and for \texttt{while} loops
  \item Shell programming uses a built-in command which is either
    \texttt{test} or \texttt{[...]}
  \item Examples of string comparisons:
    {\tiny
\begin{alltt}
[ "$USER" = root ]    # true if the value of $USER is "root"
[ "$USER" != root ]   # true if the value of $USER is not "root"
[ -z "$USER" ]        # true if the string "$USER" has zero length
[ string1 < string2 ] # true if string1 sorts less than string2
[ string1 > string2 ] # true if string1 sorts greater than string2
\end{alltt}
  }
  \item Note: the spaces after the ``[`` and before the ``]'' are
    essential.
  \end{itemize}
\end{slide}

\begin{slide}{Conditions---Integer Comparisons}
  \label{sld:test-arith}
  \begin{itemize}
  \item Examples of numeric integer comparisons:
    {\scriptsize
\begin{alltt}
[ "$x" -eq 5 ]  # true if the value of $x is 5
[ "$x" -ne 5 ]  # true if integer $x is \emph{not} 5
[ "$x" -lt 5 ]  # true if integer $x is \(<\) 5
[ "$x" -gt 5 ]  # true if integer $x is \(>\) 5
[ "$x" -le 5 ]  # true if integer $x is \(\le\) 5
[ "$x" -ge 5 ]  # true if integer $x is \(\ge\) 5
\end{alltt}%
}
\item Note again that the spaces after the ``[`` and before the~``]''
  are essential.
  \end{itemize}
\end{slide}

\begin{slide}{Conditions---File Tests, NOT Operator}
  \label{sld:not-operator}
  \begin{itemize}
  \item The shell provides many tests of information about files.
  \item Do \texttt{man test} to see the complete list.
  \item Some examples:
  \end{itemize}
  {\scriptsize%
\begin{alltt}
$ \textbf{[ -f file ]}   # true if file is an ordinary file
$ \textbf{[ ! -f file ]} # true if file is NOT an ordinary file
$ \textbf{[ -d file ]}   # true if file is a directory
$ \textbf{[ -u file ]}   # true if file has SUID permission
$ \textbf{[ -g file ]}   # true if file has SGID permission
$ \textbf{[ -x file ]}   # true if file exists and is executable
$ \textbf{[ -r file ]}   # true if file exists and is readable
$ \textbf{[ -w file ]}   # true if file exists and is writeable
$ \textbf{[ file1 -nt file2 ]} # true if file1 is newer than file2
\end{alltt}
    }
\end{slide}

\begin{slide}{Conditions---Combining Comparisons}
  \label{sld:conditions-combining-comparisons}
  \begin{itemize}
  \item Examples of combining comparisons with AND: \texttt{-a} and
    OR: \texttt{-o}, and grouping with \texttt{\bs(\ldots\bs)}
    {\tiny
\begin{alltt}
# true if the value of $x is 5 AND $USER is not equal to root:
[ "$x" -eq 5 -a "$USER" != root ]
# true if the value of $x is 5 OR $USER is not equal to root:
[ "$x" -eq 5 -o "$USER" != root ]
# true if ( the value of $x is 5 OR $USER is not equal to root ) AND
# ( $y > 7 OR $HOME has the value happy )
[ \bs( "$x" -eq 5 -o "$USER" != root \bs) -a \bs
\bs( "$y" -gt 7 -o "$HOME" = happy \bs) ]
\end{alltt}
    }
  \item Note again that the spaces after the ``[`` and before the
    ``]'' are essential.
  \item Do \texttt{man test} to see the information about all the operators.
  \end{itemize}
\end{slide}

\begin{slide}{Arithmetic Assignments}
  \footnotesize
  \begin{itemize}
  \item Can do with the external program \texttt{expr}
    \begin{itemize}
    \item \ldots but \texttt{expr} is not so easy to use, although it
      is very standard and \emph{portable}
    \item Easier is to use the built in \texttt{let} command
    \item Examples:
      {\tiny
\begin{alltt}
$ \textbf{let x=1+4}
$ \textbf{let x='1 + 4'}
$ \textbf{let 'x = 1 + 4'}
$ \textbf{let x="(2 + 3) * 5"}    # now x is 25
$ \textbf{let "x = 2 + 3 * 5"}    # now x is 17
$ \textbf{let "x += 5"}           # now x is 22
$ \textbf{let "x = x + 5"}        # now x is 27; note do not need $ on rhs
\end{alltt}%
    }
    %\vspace*{-2ex}
  \item Notice that you do not need to quote the special characters
    with \texttt{let}.
  \item Quote if you want to use white space.
    \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{Arithmetic Expressions with \texttt{\$((\ldots))}}
  \label{sld:arithmetic-expressions-with-((...))}
  \begin{itemize}
  \item The shell interprets anything inside \texttt{\$((\ldots))} as
    an arithmetic expression
  \item You could calculate the number of days left in the year like
    this:
\begin{alltt}
$ \textbf{echo "There are \bs
$(( (365-$(date +%j)) / 7 )) weeks \bs
left till December 31"}
\end{alltt}%$
  \item You don't need to put a dollar sign in front of variables in
    arithmetic expressions, but it is okay to put them there.
  \end{itemize}
\end{slide}


\begin{slide}{Arithmetic Conditions with \texttt{((\ldots))}}
  \label{sld:arithmetic-conditions-new}
  \begin{itemize}
  \item A (less portable) alternative to the arithmetic conditions in
    slide~\pageref{sld:test-arith} is putting the expression in
    \texttt{((\ldots))}
  \item So you can do
\begin{alltt}
(( (3>2) && (4<=1) ))
\end{alltt}
    %\vspace*{-2ex}
    instead of
\begin{alltt}
[ \bs( 3 -gt 2 \bs) -a \bs( 4 -le 1 \bs) ]
\end{alltt}
    %\vspace*{-2.5ex}
  \item Operators that work with \texttt{let}, \texttt{\$((\ldots))}
  and \texttt{((\ldots))} are:
\begin{alltt}
+ - * / % << >> & | ~ ! ^
< > <= >= == !=
\end{alltt}
  %\vspace*{-2ex}%
  which have \emph{exactly} the same effect as in the C
  programming language.
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{if} Statement}
  \begin{itemize}
  \item Syntax:
    {\tiny
\begin{alltt}
if \emph{test-commands}
then
    \emph{statements-if-test-commands-1-true}
elif \emph{test-commands-2}
then
    \emph{statements-if-test-commands-2-true}
else
    \emph{statements-if-all-test-commands-false}
fi
\end{alltt}
}
  \item Example:
    {\tiny
\begin{verbatim}
if grep nick /etc/passwd > /dev/null 2>&1
then
    echo Nick has a local account here
else
    echo Nick has no local account here
fi
\end{verbatim}
    }
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{while} Statement}
  \begin{itemize}
  \item Syntax:
    {\tiny
\begin{alltt}
while \emph{test-commands}
do
    \emph{loop-body-statements}
done
\end{alltt}
}
  \item Example:
    {\tiny
\begin{verbatim}
i=0
while [ "$i" -lt 10 ]
do
    echo -n "$i "       # -n suppresses newline.
    i=`expr $i + 1`     # let "i = i + 1" also works.
done
\end{verbatim}
    }
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{for} Statement}
  \label{sld:for-statement-1}
  \begin{itemize}
  \item Syntax:
    {\scriptsize
\begin{alltt}
for \emph{name} in \emph{words}
do
    \emph{loop-body-statements}
done
\end{alltt}
}
  \item Example:
{\scriptsize
\begin{verbatim}
for planet in Mercury Venus Earth Mars \
    Jupiter Saturn Uranus Neptune Pluto
do
    echo $planet
done
\end{verbatim}
  }
  \begin{itemize}
  \item The backslash ``\texttt{\bs}'' quotes the newline.  It's just
    a way of folding a long line in a shell script over two or more lines.
  \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{for} Loops: Another Example}
  \small%
  \begin{itemize}
  \item Here the shell turns \texttt{*.txt} into a list of file names
  ending in ``\texttt{.txt}'':%
  {\scriptsize%
\begin{verbatim}
for i in *.txt
do
    echo $i
    grep 'lost treasure' $i
done
\end{verbatim}
  }
\item You can leave the \texttt{in \emph{words}} out; in that case,
  \texttt{\emph{name}} is set to each parameter in turn:%
  {\tiny%
  \begin{verbatim}
i=0
for parameter
do
    let 'i = i + 1'
    echo "parameter $i is $parameter"
done
  \end{verbatim}
  }
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{for} Loops: second, C-like syntax}
  \begin{itemize}
  \item There is a second (less frequently used, and less portable)
    C-like \texttt{for} loop syntax:%
    {\footnotesize%
\begin{alltt}
for (( \emph{expr1} ; \emph{expr2} ; \emph{expr3} ))
do
    \emph{loop-body-statements}
done
\end{alltt}
    }
    %\vspace*{-3ex}
  \item Rules: same as for arithmetic conditions---see
    slide~\pageref{sld:arithmetic-conditions-new}
  \item Example:
{\footnotesize
\begin{verbatim}
for (( i = 0; i < 10; ++i ))
do
    echo $i
done
\end{verbatim}%$
  }
    \end{itemize}
\end{slide}

\begin{slide}{\texttt{break} and \texttt{continue}}
  \begin{itemize}
  \item Use inside a loop
  \item Work like they do in C
  \item \texttt{break} terminates the innermost loop; execution
    goes on after the loop
  \item \texttt{continue} will skip the rest of the body of the loop,
    and resume execution on the next itteration of the loop.
  \end{itemize}
\end{slide}

\begin{slide}{Blocks: \texttt{\{\ldots\}}}
  \label{sld:blocks}
  \begin{itemize}
  \item A subshell is one way of grouping commands together, but it
    starts a new process, and any variable changes are localised
  \item An alternative is to group commands into a block, enclosing a
    set of commands in braces: \texttt{\{\ldots\}}
  \item Useful for grouping commands for file input or output %(see
    %slide~\pageref{sld:grouping-for-file-io}).
  \item See next slide for another application.
  \end{itemize}
\end{slide}

\begin{slide}{Error Handling: \texttt{\textbar\textbar}, \texttt{\&\&}
    and \texttt{exit}}
  \label{sld:error-handling}
  \begin{itemize}
  \item Suppose we want the user to provide exactly two parameters, and
    exit otherwise
  \item A common method of handling this is something like:%
    {\scriptsize%
\begin{alltt}
[ $# -eq 2 ] || \{ echo "Need two parameters"; exit 1; \}
\end{alltt}%$
    }%
    %\vspace*{-2.5ex}
  \item Read this as ``the number of parameters is two OR exit''
  \item Works because this logical OR uses short-circuit Boolean
    evaluation; the second statement is executed only if the first
    fails (is false)
  \item Logical AND ``\texttt{\&\&}'' can be used in the same way; the
    second statement will be executed only if the first is successful
    (true)
  \item A note about blocks: must have semicolon ``\texttt{;}'' or
    newline at end of last statement before closing brace
  \end{itemize}
\end{slide}

\begin{slide}{Output: \texttt{echo} and \texttt{printf}}
  \begin{itemize}
  \item To perform output, use \texttt{echo}, or for more formatting,
    \texttt{printf}.
  \item Use \texttt{echo -n} to print no newline at end.
  \item Just \texttt{echo} by itself prints a newline
  \item \texttt{printf} works the same as in the C programming
    language, except no parentheses or commas:
\begin{alltt}
$ \textbf{printf "%16s\bs{}t%8d\bs{}n" $my_string $my_number}
\end{alltt}%$
  \item Do \texttt{man printf} (or look it up in the \texttt{bash}
    manual page) to read all about it.
  \end{itemize}
\end{slide}

\begin{slide}{Input: the \texttt{read} Command}
  \begin{itemize}
  \item For input, use the built-in shell command \texttt{read}
  \item \texttt{read} reads standard input and puts the result into
    one or more variables
  \item If use one variable, variable holds the whole line
  \item Syntax:
\begin{alltt}
read \emph{var1} \emph{var2}\ldots
\end{alltt}
    %\vspace*{-3ex}
  \item Often used with a \texttt{while} loop like this:
\begin{alltt}
while read var1 var2
do
    # do something with var1 and var2
done
\end{alltt}
    %\vspace*{-3ex}
  \item Loop terminates when reach end of file
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{set}: Splitting a Multi-Word Variable}
  \begin{itemize}
  \item Sometimes may want to split a multi-word variable into
    single-word variables
  \item \texttt{read} won't work like this:
\begin{alltt}
MY_FILE_INFO=$(ls -lR | grep $file)
# \ldots
echo ${MY_FILE_INFO} | read perms links \bs 
user group size month day time filename
\end{alltt}%$
%\vspace*{-2ex}
  \item Use the builtin command \texttt{set} instead:
\begin{alltt}
MY_FILE_INFO=$(ls -lR | grep $file)
# \ldots
set ${MY_FILE_INFO}
perms=$1 links=$2 user=$3 group=$4 size=$5
month=$6 day=$7 time=$8 filename=$9
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{More about \texttt{set}, and \texttt{IFS}}
  \begin{itemize}
  \item \texttt{set} splits its arguments into pieces (usually) at
    whitespace
  \item It sets the first value as \texttt{\$1}, the second as
    \texttt{\$2}, and so on.
  \item Note that you can change how \texttt{set} and the shell splits
    things up by changing the value of a special variable called
    \texttt{IFS}
  \item \texttt{IFS} stands for \emph{Internal Field Separator}
  \item Normally the value of \texttt{IFS} is the string
    ``\meta{space}\meta{tab}\meta{newline}''
  \item Next slide shows how changing \texttt{IFS} to a colon let us
    easily split the \texttt{PATH} into separate
    directories:~$\rightsquigarrow$~{\tiny next~slide}
  \end{itemize}
\end{slide}

\begin{slide}{Example: Changing \texttt{IFS}}
  \label{sld:change-IFS}
    \begin{itemize}
    \item Notice that here, I make the change to \texttt{IFS} in a
    subshell.  I have simply typed the loop at the prompt.
  \item As I said in slide~\pageref{sld:vars-in-subshell}, changes in
    a subshell are local to the subshell:
    {\tiny%
\begin{alltt}
 $ echo $PATH
/usr/bin:/bin:/usr/X11R6/bin:/home/nicku/bin
$ (IFS=:
> for dir in $PATH
> do
>     echo $dir
> done
> )
/usr/bin
/bin
/usr/X11R6/bin
/home/nicku/bin
\end{alltt}%$
  }
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{case} Statement}
  \begin{itemize}
  \item Similar to the \texttt{switch} statement in C, but more
    useful and more general
  \item Uses pattern matching against a string to decide on an
    action to take
  \item Syntax:
\begin{alltt}
case \emph{expression} in
    \emph{pattern1} )
        \emph{statements} ;;
    \emph{pattern2} )
        \emph{statements} ;;
    \ldots
esac
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{case} Statement: Example}
  \begin{itemize}
  \item This example code runs the appropriate program on a graphics
    file, depending on the file extension, to convert the file
    to another format:
  \end{itemize}
  %\vspace*{-2.5ex}
\begin{alltt}
case $filename in
    *.tif)
        tifftopnm $filename > $ppmfile
        ;;
    *.jpg)
        tjpeg $filename > $ppmfile
        ;;
    *)
        echo -n "Sorry, cannot handle this "
        echo "graphics format"
        ;;
esac
\end{alltt}%$
\end{slide}

\begin{slide}{\texttt{shift}: Move all Parameters Up}
  \begin{itemize}
  \item Sometimes we want to process command-line parameters in a loop
  \item The \texttt{shift} statement is made for this
  \item Say that we have four parameters: \texttt{\$1} has the value
    \texttt{one}, \texttt{\$2} has the value \texttt{two},
    \texttt{\$3} has the value \texttt{three} and \texttt{\$4} has the
    value \texttt{four}
  \item Then after executing the \texttt{shift} statement, the values
    are now:
    
    \texttt{\$1} has the value \texttt{two}, \texttt{\$2} has the
    value \texttt{three} and \texttt{\$3} has the value \texttt{four},
    and \texttt{\$4} no longer exists.
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{shift}: Many Places}
  \begin{itemize}
  \item You can give a number argument to \texttt{shift}:
    \begin{itemize}
    \item If before, we have four parameters: \texttt{\$1} has the
      value \texttt{one}, \texttt{\$2} has the value \texttt{two},
      \texttt{\$3} has the value \texttt{three} and \texttt{\$4} has
      the value \texttt{four}
  \item After executing the statement:
\begin{alltt}
$ \textbf{shift 2}
\end{alltt}%$
    we have two parameters left: \texttt{\$1} has the value
    \texttt{three} and \texttt{\$2} has the value \texttt{four};
    \texttt{\$3} and \texttt{\$4} no longer exist.
    \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{Command-Line Options---1}
  \begin{itemize}
  \item Sometimes we want to modify the behaviour of a shell script
    \begin{itemize}
    \item For example, want an option to show more information on
      request
   \item could use an option ``\texttt{-v}'' (for ``verbose'') to tell
     the shell script that we want it to tell us more information about
     what it is doing
   \item If script is called \texttt{showme}, then we could use our
     \texttt{-v} option like this:
\begin{alltt}
$ \textbf{showme -v}
\end{alltt}%$
   \item the script then shows more information.
    \end{itemize}
   \item 
  \end{itemize}
\end{slide}

\begin{slide}{Command-Line Options---2}
  \begin{itemize}
  \item For example, We might provide an option to give a starting
    point for a script to search for \SUID programs
  \item Could make the option \texttt{-d \emph{directory}}
  \item If script is called \texttt{findsuid}, could call it like
    this:
\begin{alltt}
$ \textbf{findsuid -d /usr}
\end{alltt}%$
    to tell the script to start searching in the directory
    \texttt{/usr} instead of the current directory
  \end{itemize}
\end{slide}

\begin{slide}{Command-Line Options---3}
  \begin{itemize}
  \item We could do this using \texttt{shift}, a \texttt{while} loop,
  and a \texttt{case} statement, like this:
\begin{alltt}
while [ -n "$(echo $1 | grep '-')" ]
do
    case $1 in
        -v) VERBOSE=1 ;;
        -d)
            shift
            DIRECTORY=$1
            ;;
        *)  echo "usage: $0 [-v] [-d dir]"
            exit 1 ;;
    esac
    shift
done
\end{alltt}%$
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{getopts}: Command-Line Options---4}
  \begin{itemize}
  \item Problems with above solution: inflexibility:
    \begin{itemize}
    \item Does not allow options to be ``bundled'' together like
      \texttt{-abc} instead of \texttt{-a -b -c}
    \item Requires a space between option and its argument, i.e.,
      doesn't let you do \texttt{-d/etc} as well as \texttt{-d /etc}
%%     \item Other parameters must come last; they cannot be mixed up
%%       with the options
    \item Better method: use the built-in command \texttt{getopts}:
      {\tiny%
\begin{alltt}
while getopts ":vd:" opt
do
    case opt in
        v) VERBOSE=1 ;;
        d) DIRECTORY=$OPTARG ;;
        *)  echo "usage: $0 [-v] [-d dir]"
            exit 1 ;;
    esac
done
shift $((OPTIND - 1))
\end{alltt}%$
  }
    \end{itemize}
  \end{itemize}
\end{slide}

% This is what emacs' shell script mode does for insert->options:
%% while getopts ::vd: OPT; do
%% case $OPT in
%% :|+:)

%% ;;v|+v)

%% ;;d|+d)
%% "$OPTARG"
%% ;;*)
%% echo "usage: ${0##*/} [+- ARG] [+-vd ARG} [--] ARGS..."
%% exit 2
%% esac
%% done
%% shift $[ OPTIND - 1 ]
%$

\begin{slide}{\texttt{getopts}: Command-Line Options---5}
  \begin{itemize}
  \item \texttt{getopts} takes two arguments:
    \begin{itemize}
    \item first comes the string that can contain letters and colons.
      \begin{itemize}
      \item Each letter represents one option
      \item A colon comes after a letter to indicate that option takes
        an arguement, like \texttt{-d \texttt{directory}}
      \item A colon at the beginning makes \texttt{getopts} less
        noisy, so you can provide your own error message, as shown in
        the example.
      \end{itemize}
    \item The second is a variable that will hold the option (without
      the hyphen ``\texttt{-}'')
    \end{itemize}
  \item Shift out all processed options using the variable
  \texttt{OPTIND}, leaving any other arguments accessible
  \item Search for \texttt{getopts} in the \texttt{bash} man page
  \end{itemize}
\end{slide}

\begin{slide}{Temporary Files: \texttt{mktemp}}
  \label{sld:mktemp}
  \begin{itemize}
  \item Sometimes it is convenient to store temporary data in a
    temporary file
  \item The \texttt{mktemp} program is designed for this
  \item We use it something like this:
\begin{alltt}
TMPFILE=$(mktemp /tmp/temp.XXXXXX) || exit 1
\end{alltt}%$
  \item \texttt{mktemp} will create a new file, replacing the
    ``\texttt{XXXXXX}'' with a random string
  \item Do \texttt{man mktemp} for the complete manual.
  \end{itemize}
\end{slide}

\begin{slide}{Signals that may Terminate your Script}
  \begin{itemize}
  \item Many key strokes will send a \texttt{signal} to a process
  \item Examples:
    \begin{itemize}
    \item \key{Control-C} sends a \texttt{SIGINT} signal to the
      current process running in the foreground
    \item \key{Control-\bs} sends a \texttt{SIGQUIT} signal
    \end{itemize}
  \item When you log out, all your processes are sent a
    \texttt{SIGHUP} (hangup) signal
  \item If your script is connected to another process that terminates
    unexpectedly, it will receive a \texttt{SIGPIPE} signal
  \item If anyone terminates the program with the \texttt{kill}
  program, the default signal is \texttt{SIGTERM}
\end{itemize}
\end{slide}


\begin{slide}{Signals: \texttt{trap}}
  \begin{itemize}
\item Sometimes you want your script to clean up after itself
    nicely, and remove temporary files
  \item Do this using \texttt{trap}
  \end{itemize}
\end{slide}

\begin{slide}{Signals: \texttt{trap} Example}
  \begin{itemize}
  \item Supose your script creates some temporary files, and you want
    to remove them if your script recieves any of these signals
  \item You can ``catch'' the signal, and remove the files when the
    signals are received before the program terminates
  \item Suppose the temporary files have names stored in the variables
    \texttt{TEMP1} and \texttt{TEMP2}
  \item Then you would trap these signals like this:
\begin{alltt}
trap "rm $TEMP1 $TEMP2" HUP INT QUIT PIPE TERM
\end{alltt}
    %\vspace*{-2.5ex}
  \item Conveniently, (but not very portably), \texttt{bash} provides
    a ``pretend'' signal called \texttt{EXIT}; can add this to the
    list of signals you trap, so that the temporary files will be
    removed when the program exits normally.
  \end{itemize}
\end{slide}


\begin{slide}{Functions}
  \begin{itemize}
  \item The shell supports functions and function calls
  \item A function works like an external command, except that it does
    not start another process
  \item Syntax:
\begin{alltt}
function \emph{functname}
\{
    \emph{shell commands}
\}
\end{alltt}
    %\vspace*{-2.5ex}
    Or:
\begin{alltt}
\emph{functname} ()
\{
    \emph{shell commands}
\}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Parameters in Functions}
  \begin{itemize}
  \item Work the same as parameters to entire shell script
  \item First parameter is \texttt{\$1}, second is \texttt{\$2},\ldots,
    the tenth parameter is \texttt{\$\{10\}}, and so on.
  \item \texttt{\$\#} is the number of parameters passed to the
    function
  \item As with command line parameters, they are read-only
  \item Assign to meaningful names to make your program more
    understandable
  \end{itemize}
\end{slide}

\begin{slide}{Example, Calling a Function}
  \begin{itemize}
  \item This is a simple example program:
\begin{verbatim}
#! /bin/sh

function cube {
    echo $(($1 * $1 * $1))
}

j=$(cube 5)
echo $j     # Output is 125
\end{verbatim}
  \item Note the use of command substitution to get a return value.
  \item The function prints result to standard output.
  \end{itemize}
\end{slide}

\begin{slide}{Debugging Shell Scripts---1}
  \begin{itemize}
  \item If you run the script with:
\begin{alltt}
$ \textbf{sh -v \emph{script}}
\end{alltt}%$
    then each statement will be printed as it is executed
  \item If you run the script with:
\begin{alltt}
$ \textbf{sh -x \emph{script}}
\end{alltt}%$
    then an execution trace will show the value of all variables as
    the script executes.
  \end{itemize}
\end{slide}


\begin{slide}{Debugging Shell Scripts---2}
  \begin{itemize}
  \item Use \texttt{echo} to display the value of variables as the
    program executes
  \item You can turn the \texttt{-x} shell option on in any part of
    your script with the line:
\begin{alltt}
set -x
\end{alltt}
    and turn it off with:
\begin{alltt}
set +x
\end{alltt}
  \item The book \emph{Learning the bash Shell} includes a \texttt{bash}
    shell debugger if you get desperate
  \end{itemize}
\end{slide}

\begin{slide}{Writing Shell Scripts}
  \begin{itemize}
  \item Build your shell script \emph{incrementally}:
    \begin{itemize}
    \item Open the editor in one window, have a terminal window open
    in which to run your program as you write it
  \item Test as you implement: this makes shell script development
    easy
  \item Do not write a very complex script, and \emph{then}  begin
    testing it!
    \end{itemize}
  \item Use the standard software engineering practice you know:
    \begin{itemize}
    \item Use meaningful variable names, function names
    \item Make your program self-documenting
    \item Add comment blocks to explain obscure or difficult parts of
      your program
    \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{Useful External Programs---1}
  Each of these has a manual page, and many have info manuals.  Read
  their online documentation for more information.
  \begin{itemize}
  \item \texttt{awk} --- powerful tool for processing columns of data
  \item \texttt{basename} --- remove directory and (optionally)
    extension from file name
  \item \texttt{cat} --- copy to standard output
  \item \texttt{cut} --- process columns of data
  \item \texttt{du} --- show disk space used by directories and files
  \item \texttt{egrep}, \texttt{grep} --- find lines containing
    patterns in files
  \item \texttt{find} --- find files using many criteria
  \end{itemize}
\end{slide}


\begin{slide}{Useful External Programs---2}
  \begin{itemize}
  \item \texttt{last} --- show the last time a user was logged in
  \item \texttt{lastb} --- show last bad log in attempt by a user
  \item \texttt{rpm} --- \RPM package manager: manage software package
    database
  \item \texttt{sed} --- stream editor: edit files automatically
  \item \texttt{sort} --- sort lines of files by many different
    criteria
  \item \texttt{tr} --- translate one set of characters to another set
  \item \texttt{uniq} --- replace repeated lines with just one line,
    optionally with a count of the number of repeated lines
  \end{itemize}
\end{slide}

\begin{slide}{Regular Expressions}
  \begin{itemize}
  \item Many programs and programming languages use \emph{regular
      expressions}, including Java 1.4, Perl and VB.NET (plus many
    others; even MS Word uses regular expressions under Find $\to$
    Advanced)
  \item These programs use regular expressions:
    \begin{itemize}
    \item \texttt{grep}, \texttt{egrep}, \texttt{sed}, \texttt{awk} 
    \end{itemize}
  \item \emph{Regular expressions} provide a powerful language for
    manipulating data and extracting important information from masses
    of data
  \end{itemize}
\end{slide}

\begin{slide}{What is in a Regular Expression?}
  \begin{itemize}
  \item There are two types of character in a regular expression:
    \begin{itemize}
    \item Metacharacters
      \begin{itemize}
      \item These include:
      \item \texttt{*} \texttt{\bs} \texttt{.} \texttt{+}
      \texttt{?} \texttt{\textasciicircum} \texttt{(} \texttt{)}
      \texttt{[} \texttt{\{} \texttt{\textbar}
      \end{itemize}
    \item Ordinary, literal characters:
      \begin{itemize}
      \item  all the other characters
      \end{itemize}
    \end{itemize}
  \end{itemize}
\end{slide}

\begin{slide}{Literal characters}
  \begin{itemize}
  \item Find all lines containing "\texttt{chan}" in the password
    file:
\begin{alltt}
$ \textbf{grep chan /etc/passwd}
\end{alltt}%$
  \item The regular expressions is "\texttt{chan}"
  \item It is made entirely of literal characters
  \item It matches only lines that contain the exact string
  \item It will match lines containing the words \texttt{chan},
    \texttt{changed}, \texttt{merchant}, \texttt{mechanism},\ldots
  \end{itemize}
\end{slide}

\begin{slide}{Character Classes: \texttt{[\ldots]}}
  \begin{itemize}
  \item A character class represents \emph{one} character
  \item Examples:%
    {\scriptsize%
\begin{alltt}
# Find all words in the dictionary that contain a vowel:
$ \textbf{grep "[aeiou]" /usr/share/dict/words}
# Find all lines that contain a digit:
$ \textbf{grep "[0123456789]" /usr/share/dict/words}
# Find all lines that contain a digit:
$ \textbf{grep "[0-9]" /usr/share/dict/words}
# Find all lines that contain a capital letter:
$ \textbf{grep "[A-Z]" /usr/share/dict/words}
\end{alltt}%
    }
  \end{itemize}
\end{slide}

\begin{slide}{Negated Character Classes: \texttt{[\textasciicircum\ldots]}}
  \begin{itemize}
\item Examples of negated character classes:
    {\scriptsize%
\begin{alltt}
# Find all words in the dictionary
# that contain a character that is not a vowel:
$ \textbf{grep "[^aeiou]" /usr/share/dict/words}
# Two ways of finding all lines that contain
# a character that is not a digit:
$ \textbf{grep "[^0123456789]" /usr/share/dict/words}
$ \textbf{grep "[^0-9]" /usr/share/dict/words}
# Find all lines that contain a character
# that is not a digit, or a letter
$ \textbf{grep "[^0-9a-zA-Z]" /usr/share/dict/words}
\end{alltt}%
    }
  \item Remember: each set of square brackets represents exactly
  \emph{one} character.
  \end{itemize}
\end{slide}

\begin{slide}{Match Any Character}
  \begin{itemize}
  \item The dot ``\texttt{.}'' matches any single character, except a
    newline.
  \item The pattern '.....' matches all lines that contain at least
  five characters
  \end{itemize}
\end{slide}


\begin{slide}{Matching the Beginning or End of Line}
  \begin{itemize}
  \item To match a line that contains exacly five characters:
\begin{alltt}
$ \textbf{grep '^.....$' /usr/share/dict/words}
\end{alltt}
  \item The hat, \verb+^+ represents the position right at the start
    of the line
  \item The dollar \texttt{\$} represents the position right at the
    end of the line.
  \item Niether \verb+^+ nor \texttt{\$} represents a character
  \item They represent a position
  \item Sometimes called \emph{anchors}, since they anchor the other
    characters to a specific part of the string
  \end{itemize}
\end{slide}

\begin{slide}{Match Repetitions: \texttt{*}, \texttt{?},
    \texttt{+}, \texttt{\{n\}}, \texttt{\{n,m\}}}
  \begin{itemize}
  \item To match \emph{zero} or more:
    
  \item \texttt{a*} represents zero or more of the lower case letter
    \texttt{a}, so the pattern will match \texttt{""} (the empty
    string), ``\texttt{a}'', ``\texttt{aa}'',
    ``\texttt{aaaaaaaaaaaaaaa}'', ``\texttt{qwewtrryu}'' or the
    ``nothing'' in front of any string!
  \item To match \texttt{one} or more:
  \item `\texttt{a+}' matches one or more ``\texttt{a}''s
  \item `\texttt{a?}' matches zero or one ``\texttt{a}''
  \item `\texttt{a\{10\}}' matches exactly 10  ``\texttt{a}''s
  \item `\texttt{a\{5,10\}}' matches between 5 and 10 (inclusive) ``\texttt{a}''s
  \end{itemize}
\end{slide}

\begin{slide}{Matching Alternatives: ``\texttt{\textbar}''}
  \begin{itemize}
  \item the vertical bar represents alternatives:
  \item The regular expresssion `\texttt{nick|albert|alex}' will match
    either the string ``\texttt{nick}'' or the string
    ``\texttt{albert}'' or the string ``\texttt{alex}''
  \item Note that the vertical bar has very low precedence:
    
  \item the pattern `\texttt{\textasciicircum{}fred\textbar{}nurk}'
    matches ``\texttt{fred}'' only if it occurs at the start of the
    line, while it will match ``\texttt{nurk}'' at any position in the
    line
  \end{itemize}
\end{slide}

\begin{slide}{Putting it All Together: Examples}
  \begin{itemize}
  \item Find all words that contain at least two vowels:
\begin{alltt}\small
$ \textbf{grep '[aeiou].*[aeiou]' /usr/share/dict/words}
\end{alltt}%$
  \item Find all words that contain \emph{exactly} two vowels:
\begin{alltt}
$ \textbf{egrep \bs
'[^aeiou]*[aeiou][^aeiou]*[aeiou][^aeiou]*' \bs
/usr/share/dict/words}
\end{alltt}%$
  \item Find all lines that are empty, or contain only white space:
\begin{alltt}
$ \textbf{grep '^\bs{}s*$' file}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Basic \texttt{awk}}
  \begin{itemize}
  \item \texttt{awk} is a complete programming language
  \item Mostly used for one-line solutions to problems of extracting
    columns of data from text, and processing it
  \item A complete book is available on \texttt{awk} at
    \url{http://www.ssc.com/ssc/eap/}.
  \end{itemize}
\end{slide}

\begin{slide}{What Does \texttt{awk} Do?}
  \begin{itemize}
  \item \texttt{awk} reads file(s) or standard input one line at a
    time, and
  \item automatically splits the line into fields, and calls them
    \texttt{\$1}, \texttt{\$2},\ldots, \texttt{\$NF}
  \item \texttt{NF} is equal to the number of fields the line was
    split into
  \item \texttt{\$0} contains the whole line
  \item \texttt{awk} has an option \texttt{-F} that allows you to
    select another pattern as the field separator
    \begin{itemize}
    \item Normally \texttt{awk} splits columns by white space
    \end{itemize}
  \item To execute code after all lines are processed, create an
  \texttt{END} block.
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{awk} Examples}
  \begin{itemize}
  \item Print the sizes of all files in current directory:
\begin{alltt}
 \textbf{ls -l | awk '\{print $5\}'}
\end{alltt}
  \item Add the sizes of all files in current directory:
\begin{alltt}
 \textbf{ls -l | awk '\{sum += $5\} END\{print sum\}'}
\end{alltt}
  \item Print only the permissions, user, group and file names of
  files in current directory:
\begin{alltt}
 \textbf{ls -l | awk '\{print $1, $3, $4, $NF\}'}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{sed}---the Stream Editor}
  \begin{itemize}
  \item \texttt{sed} provides many facilities for editing files
  \item The \emph{substitute} command, \texttt{s///}, is the most
    important
  \item The syntax (using \texttt{sed} as an editor of stanard input),
    is:
\begin{alltt}
$ \textbf{sed 's/\emph{original}/\emph{replacement}/'}
\end{alltt}%$
  \item Example: replace the first instance of \texttt{Windows} with
    \texttt{Linux} on each line of the input:
\begin{alltt}
 \textbf{sed 's/Windows/Linux/'}
\end{alltt}
  \item Example: replace \emph{all} instances of \texttt{Windows} with
    \texttt{Linux} on each line of the input:
\begin{alltt}
 \textbf{sed 's/Windows/Linux/g'}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{sed}---Backreferencees}
  \begin{itemize}
  \item You can match part of the \texttt{\emph{original}} in a
    \texttt{sed} substitute command, and put that part back into the
    replacement part.
  \item You enclose the part you want to refer to later in
    \texttt{\bs(\ldots\bs)}
  \item You can get the first value in the replacement part by
    \texttt{\bs1}, the second opening parenthesis of
    \texttt{\bs(\ldots\bs)} by \texttt{\bs2}, and so on.
  \end{itemize}
\end{slide}  


\begin{slide}{\texttt{sed}---Backreferencees: Example}
  \begin{itemize}
  \item If you do \texttt{find /etc | xargs file -b}, you will get a
    lot of output like this:
\begin{alltt}
symbolic link to bg5ps.conf.zh_TW.Big5
symbolic link to rc.d/rc.local
symbolic link to rc.d/rc
symbolic link to rc.d/rc.sysinit
symbolic link to ../../X11/xdm/Xservers
\end{alltt}
  %\vspace*{-2.5ex}
\item If you want to edit each line to remove everything after
  ``\texttt{symbolic link}'', then you could pipe the data through
  \texttt{sed} like this:
\begin{alltt}
$ \textbf{find /etc | xargs file -b \bs
| sed 's/\bs(symbolic link\bs).*/\bs1/'}
\end{alltt}%$
  %\vspace*{-2.5ex}
\item See slide~\pageref{sld:find-examples-1} for an application
  \end{itemize}
\end{slide}

\begin{slide}{\texttt{find} Examples}
  \label{sld:find-examples-1}
  \begin{itemize}
  \item Count the number of unique manual pages on the computer:
\begin{alltt}
$ \textbf{find /usr/share/man -type f | wc -l}
\end{alltt}
  \item Print a table of types of file under the \texttt{/etc}
    directory, with the most common file type down at the bottom:
\begin{alltt}
$ \textbf{find /etc | xargs file -b \bs
  | sed 's/\bs(symbolic link\bs).*/\bs1/' \bs
  | sort \bs
  | uniq -c \bs
  | sort -n}
\end{alltt}
  \end{itemize}
\end{slide}

\begin{slide}{Finding SUID Programs}
  \begin{itemize}
  \item Finding \SUID or \SGID files:
\begin{alltt}
$ \textbf{sudo find / -type f \bs( perm -2000 -o -perm -4000 \bs) \bs
> files.secure}
\end{alltt}%$
\item Let's compare with a list of \SUID and \SGID files to see if
  there are any changes, since \SUID and \SGID programs can be a
  security risk:
\begin{alltt}
$ \textbf{sudo find / -type f \bs( perm -2000 -o -perm -4000 \bs) |
diff - files.secure}
\end{alltt}%$
  \end{itemize}
\end{slide}

\begin{slide}{A \texttt{find} Example with Many Options}
  \begin{itemize}
  \item Set all directories to have the access mode 771, set all
    backup files (*.BAK) to mode 600, all shell scripts (*.sh) to mode
    755, and all text files (*.txt) to mode 644:
  \end{itemize}
  {\scriptsize%
\begin{alltt}
$ \textbf{find . \bs( -type d       -a exec chmod 771 \{\} \bs; \bs) -o \bs
         \bs( -name "*.BAK" -a exec chmod 600 \{\} \bs; \bs) -o \bs
         \bs( -name "*.sh"  -a exec chmod 755 \{\} \bs; \bs) -o \bs
         \bs( -name "*.txt" -a exec chmod 644 \{\} \bs; \bs)}
\end{alltt}%$
}
\end{slide}

\begin{slide}{\texttt{rpm} Database Query Commands}
  \begin{itemize}
  \item The \texttt{rpm} software package management system includes a
    database with very detailed information about every file of every
    software package that is installed on the computer.
  \item You can query this database using the \texttt{rpm} command.
  \item The manual page does not give the complete picture, but there
    is a book called \emph{Maximum RPM} that comes on the Red Hat
    documentation \CD
  \item This package is installed on \texttt{ictlab}
  \item \sloppypar{}You can see the appropriate section at this \URL:
    {\tiny\url{http://nicku.org/doc/maximum-rpm-1.0/html/s1-rpm-query-parts.html}}
  \end{itemize}
\end{slide}


%% Single back quotes \texttt{`...`} mean: ``execute the external program
%% called within these quotes and put the output back here.''  This is
%% called \emph{command substitution} in the bash manual.  Command
%% expansion is really quite different from the other three quoting
%% methods.  Here is an example using the \texttt{hostname} command,
%% which prints the hostname on standard output:
%% \begin{verbatim}
%% $ hostname
%% nickpc.tyict.vtc.edu.hk
%% $ h=hostname
%% $ echo $h
%% hostname
%% $ h=`hostname`
%% $ echo $h
%% nickpc.tyict.vtc.edu.hk
%% \end{verbatim}%$

%% \subsection{When to use quoting}
%% \label{sec:when-to-quote}

%% Many programs, such as \texttt{grep} or \texttt{find} need some
%% special characters that they themselves will interpret.  We need to be
%% able to send these characters unchanged to the program.  In this case,
%% quote them.  Examples:
%% \begin{verbatim}
%% $ find . -name "*.rpm"
%% \end{verbatim}%$
%% If we do not quote the asterisk, the shell will expand \texttt{*.rpm}
%% to match only the \texttt{rpm} files in the current directory, but we
%% want find to locate all the \texttt{.rpm} files in the directories
%% \emph{below} the current directory also.

%% If you want a variable value that contains spaces to not be
%% automatically split my the shell, then quote it.  Here,
%% \texttt{testquote} is a short shell script that prints information
%% about its parameters:
%% \begin{verbatim}
%% $ test="one two"
%% $ testquote $test
%% You have 2 parameters.  They are:
%% parameter 1: one
%% parameter 2: two
%% $ testquote "$test"
%% You have 1 parameters.  They are:
%% parameter 1: one two
%% \end{verbatim}%$

%% \label{pag:dollar-star-quoting}%
%% Note that \texttt{"\$*"} is one value (not split up), while
%% \texttt{"\$@"} is split into the original parameters.  So if
%% `\texttt{\$\#}' had the value 4, then there are four separately quoted
%% values in \texttt{"\$@"}.  See the beginning of
%% section~\pageref{sec:special-variables}.

%% Here is a little example showing the difference between \texttt{"\$@"}
%% and \texttt{"\$*"}:
%% \begin{verbatim}
%% $ cat test_at_star
%% #! /bin/sh
%% testquote "$@"
%% testquote "$*"
%% $ test_at_star one two three
%% You have 3 parameters.  They are:
%% parameter 1: one
%% parameter 2: two
%% parameter 3: three
%% You have 1 parameters.  They are:
%% parameter 1: one two three
%% \end{verbatim}
%% Notice how  \texttt{"\$*"} just turned into one long parameter that
%% contains spaces.
%% \subsection{Printing Output}
%% \label{sec:output}

%% We use \texttt{echo} to print things.  By default, it puts a new line
%% at the end.  To avoid printing a newline, use \texttt{echo -n}:
%% \begin{verbatim}
%% $ cat echo-n
%% #! /bin/sh
%% echo "Hello "
%% echo World
%% echo -n "Hello "
%% echo World
%% $ ./echo-n 
%% Hello 
%% World
%% Hello World
%% \end{verbatim}

%% \subsection{Reading Input}
%% \label{sec:input}

%% There are many ways of reading input, but one simple way is to use
%% \texttt{read};
%% \begin{verbatim}
%% $ read answer
%% yes
%% $ echo $answer
%% yes
%% \end{verbatim}%$

%% \section{The Basic Statements}
%% \label{sec:statements}

%% The shell is a complete programming language, and supports
%% \texttt{for} loops, \texttt{while} loops, \texttt{if} statements,
%% \texttt{case} statements (like \texttt{switch} in C), as well as
%% function calls.  We look at only a small subset of these.


%% \subsection{The \texttt{if} statement}
%% \label{sec:if}

%% The syntax of the \texttt{if} statement is:
%% \begin{alltt}
%% if \emph{test-commands}
%% then
%%     \emph{statements}
%% fi
%% \end{alltt}
%% We can add an \emph{else}:
%% \begin{alltt}
%% if \emph{test-commands}
%% then
%%     \emph{statements-if-true}
%% else
%%     \emph{statements-if-false}
%% fi
%% \end{alltt}
%% and we can have other \texttt{if} conditions nested inside, but they
%% are introduced with a new keyword: \texttt{elif}:
%% \begin{alltt}
%% if \emph{test-commands}
%% then
%%     \emph{statements-if-test-commands-1-true}
%% elif \emph{test-commands-2}
%%     \emph{statements-if-test-commands-2-true}
%% else
%%     \emph{statements-if-all-test-commands-false}
%% fi
%% \end{alltt}

%% The \texttt{test-commands} is either:
%% \begin{itemize}
%% \item a program being executed, or
%% \item a test made using the program \texttt{test}; see \texttt{man
%%     test} for all the tests you can make using \texttt{test}.  Also
%%     see section~\pageref{sec:test}.
%% \end{itemize}

%% A simple example:
%% \begin{verbatim}
%% if grep nick /etc/passwd > /dev/null 2>&1
%% then
%%     echo Nick has a local account here
%% else
%%     echo Nick has no local account here
%% fi
%% \end{verbatim}
%% We redirect all output from grep to avoid the side effect of printing
%% the line grep found.

%% If you want to put the \texttt{then} on the same line as the
%% \texttt{if}, you need to put a semicolon before the \texttt{then}.
%% Here is another example that adds the user \texttt{nicku} to the
%% sudoers file if that user is not there already:
%% \begin{verbatim}
%% if ! grep nicku /etc/sudoers > /dev/null 2>&1; then
%%     echo "nicku   ALL=(ALL) ALL" >> /etc/sudoers
%% fi
%% \end{verbatim}

%% \subsection{The \texttt{while} statement}
%% \label{sec:while}

%% The format of the \texttt{while} statement is:
%% \begin{alltt}
%% while \emph{test-commands}
%% do
%%     \emph{loop-body-statements}
%% done
%% \end{alltt}

%% Again, if you want to put the \texttt{do} on the same line as the
%% \texttt{while}, then you need an extra semicolon before the
%% \texttt{do}.  A simple example:
%% \begin{verbatim}
%% i=0
%% while [ "$i" -lt 10 ]; do
%%     echo -n "$i "       # -n suppresses newline.
%%     i=`expr $i + 1`     # i=$(($i+1)) also works.
%% done
%% \end{verbatim}%$
%% The square brackets are an example of the \texttt{test} program.  See
%% section~\pageref{sec:test}.

%% \subsubsection{\texttt{expr}}

%% In the last example using a \texttt{while} loop, we used the program
%% \texttt{expr} to do arithmetic.  This is the portable way to do
%% arithmetic in shell programming.  Note that since \texttt{expr} prints
%% its output on standard output, we use command substitution to assign
%% the program output to the variable \texttt{i}.  See the manual page
%% for \texttt{expr} for more information.
%% \subsection{The \texttt{for} statement}
%% \label{sec:for}

%% The format of the \texttt{for} statement is:
%% \begin{alltt}
%% for \emph{name} in \emph{words}
%% do
%%     \emph{loop-body-statements}
%% done
%% \end{alltt}

%% Here is a simple example:
%% \begin{verbatim}
%% for planet in Mercury Venus Earth Mars Jupiter Saturn Uranus Neptune Pluto
%% do
%%     echo $planet
%% done
%% \end{verbatim}%$
%% You can leave the \texttt{in \emph{words}} out; in that case,
%% \texttt{\emph{name}} is set to each parameter in turn.

%% Here is another example:
%% \begin{verbatim}
%% for i in *.txt
%% do
%%     echo $i
%%     grep 'lost treasure' $i
%% done
%% \end{verbatim}
%% Note that the shell will expand the wildcard characters into a list of
%% file names that you can process one by one in the loop.

%% \subsection{\texttt{break} and \texttt{continue}}
%% \label{sec:break-and-continue}

%% Inside loops you can the \texttt{break} and \texttt{continue}
%% statements.  They work like they do in C\@.

%% \subsection{The \texttt{test} program}
%% \label{sec:test}

%% The \texttt{test} program is used to perform comparisons of strings,
%% numbers and files, often used with the \texttt{if} and \texttt{while}
%% statements.  I will not waste space by copying the manual page here:
%% do \texttt{man test} to read all about \texttt{test}.

%% You can call the test program two ways: one as the name \texttt{test},
%% the other (more common way) as \texttt{[ ... ]}.  If we look, there is
%% a program called ``\texttt{[}'':%]
%% \begin{verbatim}
%% $ which [
%% /usr/bin/[
%% $ ls -l /usr/bin/[
%% lrwxrwxrwx    1 root     root            4 Nov 25 13:36 /usr/bin/[ -> test
%% \end{verbatim}%]]]

%% \paragraph{Important Note:}

%% there must be white space before and after the ``\texttt{[}'':%]
%% \begin{verbatim}
%% i=0
%% while ["$i" -lt 10]; do
%%     echo -n "$i "
%%     i=`expr $i + 1`
%% done
%% bash: [0: command not found
%% \end{verbatim}%$]

%% \subsection{Using the ``\texttt{\&\&}'' and
%%   ``\texttt{\textbar\textbar}'' Operators for Flow Control}
%% \label{sec:operators}

%% The shell has many operators; see the manual for a complete list.  But
%% here we look at two familiar operators that are surprisingly useful,
%% and yet which may have a use that is unfamiliar to you.

%% They are used for logical operations, and are shortcut logical
%% operators, just as you are familiar with in the C and Java programming
%% languages.  However, in shell programming, they are used for flow
%% control, rather like an \texttt{if} statement.

%% Suppose we have a shell script that we must call with two parameters,
%% and that it should fail if there are fewer or more parameters.  We can
%% use the `\texttt{\&\&}' operator after a test and exit.  Here is a
%% little shell script that will only accept two parameters and will exit
%% with a help message otherwise:
%% \begin{verbatim}
%% #! /bin/sh

%% [ $# -ne 2 ] && echo $0 parameter1 parameter2 && exit

%% echo parameter1 is $1, and parameter2 is $2.
%% \end{verbatim}
%% So let's run it, first with no parameters, then with two:
%% \begin{verbatim}
%% $ ./two-parameters 
%% ./two-parameters parameter1 parameter2
%% $ ./two-parameters p q
%% parameter1 is p, and parameter2 is q.
%% \end{verbatim}
%% The syntax is like this:
%% \begin{alltt}
%% \emph{command1} && \emph{command2}
%% \end{alltt}
%% \texttt{\emph{command2}} will execute only if \texttt{\emph{command1}}
%% is successful.

%% Similarly, the syntax for the `\verb!||!' operator is:
%% \begin{alltt}
%% \emph{command1} || \emph{command2}
%% \end{alltt}
%% \texttt{\emph{command2}} will execute only if \texttt{\emph{command1}}
%% is \emph{not} successful.

%% \section{Regular Expressions}
%% \label{sec:regexp}

%% Much of what a system administrator does is editing configuration
%% files.  There are tools to help with this; one such tool is the
%% program \texttt{sed}; another is the programming language Perl\@.  The
%% one thing that comes in useful in both cases are \texttt{regular
%%   expressions}.  The \texttt{grep} command also uses regular
%% expressions.  Regular expressions provide a way of matching patterns
%% in a text file; they can also provide a way of altering the text that
%% matches the pattern.  Getting started with regular expressions is our
%% aim today.


%% A regular expression is a string of characters.  Some of these
%% characters have a special meaning; most do not.  The characters with a
%% special meaning are called \emph{metacharacters}.  Here are some example
%% regular expressions without metacharacters:
%% \begin{verbatim}
%% /nicku/      # simply matches the string "nicku"
%% /hacker/     # simply matches the string "hacker"
%% \end{verbatim}

%% \subsection{Some Funny Characters (metacharacters)}
%% \label{sec:metachars}

%% \begin{description}
%% \item[Asterisk: \texttt{*}] matches zero or more of the thing that
%%   came just before.  Example:

%% \texttt{1133*} matches 11 followed by one or more 3's, so it will
%% match: 113 or 1133 or 11333 or 1133333333333333\ldots

%% \item[Dot: \texttt{.}] matches any single character, except newline.
%%   So \texttt{".*"} matches zero or more of any character.

%% \item[{Caret: \textasciicircum}] matches beginning of a line, or inside
%%   backets means something different (see below).

%% \item[Dollar sign: \texttt{\$}] matches the end of a line.  For
%%   example, ``\textasciicircum\texttt{\$}'' matches blank lines.

%% \item[Brackets: \texttt{[...]}] matches one character from the set in
%%   the brackets.  Examples:
  
%%   \texttt{"[xyz]"} matches the characters \texttt{x}, \texttt{y}, or
%%   \texttt{z}.
  
%%   \texttt{"[c-n]"} matches any of the characters in the range
%%   \texttt{c} to \texttt{n}.
  
%%   \texttt{"[B-Pk-y]"} matches any of the characters in the ranges
%%   \texttt{B} to \texttt{P} and \texttt{k} to \texttt{y}.

%%   \texttt{"[a-z0-9]"} matches any lowercase letter or any digit.
  
%%   \texttt{"[\textasciicircum{}b-d]"} matches all characters
%%   \emph{except} those in the range \texttt{b} to \texttt{d}.
  
%%   Combined sequences of bracketed characters match common word
%%   patterns. \texttt{"[Yy][Ee][Ss]"} matches yes, Yes, YES, yEs,\ldots

%%   \texttt{"[0-9][0-9][0-9][0-9][0-9][0-9]\allowbreak[0-9]\allowbreak
%%     [0-9]\allowbreak[0-9]"}
%%   matches any \IVE student number.
%% \item[Backslash: \texttt{\bs}] quotes a metacharacter to take away its
%%   special meaning.  You can match a literal \texttt{"\$"} with
%%   \texttt{"\bs\$"}, or a backslash with \texttt{"\bs\bs"}.

%% \item[Ampersand: \texttt{\&}] means, in a replacement string, the
%%   string to be replaced.  See below.
%% \end{description}


%% \subsection{Sed}
%% \label{sec:sed}

%% The \texttt{sed} program (\textbf{s}tream \textbf{ed}itor) is a
%% non-interactive editing program.  We will look only at a subset of its
%% behaviour today: substitutions.

%% Let's start with an example:
%% \begin{verbatim}
%% sed '/nicku/s//nickl/' /tmp/sudoers-orig > /tmp/sudoers
%% \end{verbatim}
%% On each line of the input file \texttt{/tmp/sudoers-orig},
%% \texttt{sed} will replace the first instance of \texttt{nicku} with
%% \texttt{nickl} and send the result to the output.

%% Let's pull that expression \texttt{/nicku/s//nickl/} apart to see how
%% it works:

%% It begins with an \emph{address}, which is a simple regular expression
%% without any metacharacters:
%% \begin{verbatim}
%% /nicku/
%% \end{verbatim}
%% This \texttt{sed} address matches all line on the input file that
%% contain the string \texttt{nicku}.  It will apply the substitute
%% operation to them.

%% The next part is a \emph{substitution expression}:
%% \begin{verbatim}
%% s//nickl/
%% \end{verbatim}
%% The syntax of a sustitution expression is:
%% \begin{alltt}
%% s/\emph{pattern to replace}/\emph{replacement}/
%% \end{alltt}

%% Here, the \emph{pattern to replace} is empty: that means that we use
%% the value from the address pattern, so we are replacing the string
%% \texttt{nicku} with the \emph{replacement}, \texttt{nickl}.

%% So if the input file \texttt{/tmp/sudoers-orig} contains this line:
%% \begin{verbatim}
%% nicku   ALL=(ALL) ALL
%% \end{verbatim}
%% then the output file will contain:
%% \begin{verbatim}
%% nickl   ALL=(ALL) ALL
%% \end{verbatim}
%% instead.

%% Here is another example:
%% \begin{verbatim}
%% sed '/^\/misc/s//#&/' /tmp/auto.master-orig > /tmp/auto.master
%% \end{verbatim}
%% What does this do?  It takes as input the file
%% \texttt{/tmp/auto.master-orig}, then finds a line starting with
%% \texttt{/misc}, and puts a comment character before it.  The edited
%% output file is \texttt{/tmp/auto.master}.

%% Again, let us examine this expression \verb!'/^\/misc/s//#&/'!
%% part-by-part:

%% It begins with an \emph{address}, which (in this case) is a regular
%% expression:
%% \begin{verbatim}
%% /^\/misc/
%% \end{verbatim}
%% This means: apply the command to lines that start with (the
%% \texttt{"\textasciicircum"} metacharacter) \texttt{/misc}.  We have to use a
%% backslash to quote the forward slash, because otherwise the forward
%% slash would mark the end of the regular expression, rather than a
%% literal forward slash.

%% The rest is a \texttt{substitution expression}:
%% \begin{verbatim}
%% s//#&/
%% \end{verbatim}
%% Again the \emph{pattern to replace} is empty: that means that we use
%% the value from the address pattern, so we are replacing the string
%% \texttt{/misc} with the \emph{replacement}.

%% The hash symbol \texttt{"\#"} in the replacement expression is a
%% literal hash, i.e., the comment character that we are inserting.

%% The special metacharacter \texttt{"\&"} has the value of the entire
%% \emph{pattern to replace}.  So we are replacing a line on the input
%% file like this:
%% \begin{verbatim}
%% /misc   /etc/auto.misc  --timeout 60
%% \end{verbatim}
%% with this in the output:q
%% \begin{verbatim}
%% #/misc  /etc/auto.misc  --timeout 60
%% \end{verbatim}

%% \subsection{Where can I find out more about sed?}
%% \label{where-can-i-find-out-more-about-sed}

%% The book \url{http://www.linuxdoc.org/LDP/abs/html/index.html} has an
%% appendix about \texttt{sed}.  It has a rather limited manual page, but
%% there is an \acro{FAQ} at \url{http://www.ptug.org/sed/sedfaq.htm}.


%% \section{Finding examples of shell scripts on your computer}
%% \label{sec:examples}

%% Your Linux system has a large number of shell scripts that you can
%% refer to as examples.  I counted about 1400.  Here is one way of
%% listing their file names:
%% \begin{verbatim}
%% $ file /bin/* /usr/bin/* /usr/sbin/* /sbin/* /etc/rc.d/* /usr/X11R6/bin/* \
%% | grep -i "shell script" | awk -F: '{print $1}'
%% \end{verbatim}%$
%% Let's see how this works.  I suggest executing the commands separately
%% to see what they do:
%% \begin{verbatim}
%% $ file /bin/* /usr/bin/*
%% $ file /bin/* /usr/bin/* | grep -i "shell script"
%% $ file /bin/* /usr/bin/* | grep -i "shell script" | awk -F: '{print $1}'
%% \end{verbatim}
%% The \texttt{awk} program is actually a complete programming language.
%% It is mainly useful for selecting columns of data from text.

%% \texttt{awk} automatically loops through the input, and divides the
%% input lines into fields.  It calls these fields \texttt{\$1},
%% \texttt{\$2},\dots\texttt{\$NF}\@.  \texttt{\$0} contains the whole
%% line.  Here the option \texttt{-F:} sets the \emph{field separator} to
%% the colon character.  Normally it is any white space.  So printing
%% \texttt{\$1} here prints what comes before the colon, which is the
%% file name.

%% Suppose you want to look for all shell scripts containing a particular
%% command or statement?  Looking for example shell scripts that use the
%% \texttt{mktemp} command:
%% \begin{verbatim}
%% $ file /bin/* /usr/bin/* /usr/sbin/* /sbin/* /etc/rc.d/* /usr/X11R6/bin/* \
%% | grep -i 'shell script'| awk -F: '{print $1}' | xargs grep mktemp
%% \end{verbatim}%$

%% Here is a useful little shell script that does this:
%% \begin{verbatim}
%% #! /bin/sh

%% if [ $# -eq 0 ]
%% then
%%     cmd=`basename $0`
%%     echo $cmd: search all Bourne shell scripts for a command
%%     echo usage: $cmd [grep-options] command-to-grep-for
%%     echo the grep-option -l is useful
%%     exit 1
%% fi

%% (
%% IFS=:
%% for d in $PATH
%% do
%%     file $d/*
%% done
%% find /etc/rc.d -type f | xargs file
%% ) \
%% | grep 'Bourne.* shell script' \
%% | awk -F: '{print $1}' \
%% | xargs grep "$@"
%% \end{verbatim}%$
%% We run the \texttt{for} loop in a sub shell to make the change to
%% \texttt{IFS} local.  \texttt{IFS} is the ``inter field separator''.
%% The shell will automatically split lines into fields separated by the
%% \texttt{IFS}\@.

%% \subsection{Where can I find out more about awk?}
%% \label{Where can I find out more about awk?}

%% There is a whole book about \texttt{awk}; you can buy it from O'Reilly
%% for about \$300 HK, or you can read it online at
%% \url{http://www.ssc.com/ssc/eap/}.

%% %
%% % perl -e '@path=split /:/, $ENV{PATH};for $d ( @path ) {print "$d\n"}'
%% % find /etc/rc.d -type f | xargs file | grep -i 'shell script'\
%% % | awk -F: '{print $1}' | wc -l
%% %     117
%% % perl -e '@path=split /:/, $ENV{PATH};for $d ( @path ) {print "$d\n"}' \
%% % | while read d;do file $d/*|grep -i 'shell script' \
%% % | awk -F: '{print $1}'; done | wc -l
%% %    1275
%% %
%% % Better than using perl, use word splitting in shell:
%% % (IFS=:;for d in $PATH;do echo $d;done)
%% %$

%% \section{Debugging Shell Scripts}
%% \label{sec:debugging}

%% It is best to write shell scripts incrementally: write part, test that
%% it works, and continue until your script does what is required.

%% You can use \emph{echo} statements to print the values of variables.

%% You can run the script with the \emph{verbose} option to bash.  For a
%% script called \texttt{script}, you could run it in verbose mode like
%% this:
%% \begin{verbatim}
%% $ sh -v script
%% \end{verbatim}%$
%% You can see each command after it has been expanded by using the
%% \texttt{-x} option:
%% \begin{verbatim}
%% $ sh -x script
%% \end{verbatim}%$

%% \section{Common Mistakes}
%% \label{sec:common-mistakes}

%% I see many people making the same mistakes.  This is due partly to the
%% difference in the shell from other programming languages, and partly
%% due to missing lectures or being late in the laboratory! \verb!:-)!

%% \begin{description}
%% \item[Spaces are important!] The shell cares about spaces much more
%%   than other programming languages.  This is because it does so many
%%   different things; if you put
%% \begin{verbatim}
%% i;
%% \end{verbatim}
%% in a C program, it is just an expression that is evaluated, the result
%% is thrown away, and nothing happens.  The shell, on the other hand,
%% will look for an program by the name \texttt{i}, and execute it.

%% The shell breaks things up into separate tokens at white space.  Where
%% you put spaces does matter.
%% \begin{description}
%% \item[Don't put spaces in assignments:] An assignment is a single
%%   thing.  If you put spaces in it, the shell will try to execute a
%%   program by the name of the variable you are trying to assign to!
%% \begin{verbatim}
%% i =20
%% bash: i: command not found
%% \end{verbatim}
  
%% \item[\texttt{eval} needs spaces:] You need to put spaces between the
%%   operands of the external program \texttt{eval}:
%% \begin{verbatim}
%% i=0
%% i=`eval i+1`
%% bash: i+1: command not found
%% \end{verbatim}

%% \item[Put spaces around the \texttt{[ ... ]}] See the notes in the
%%   section\pageref{sec:test}.
%% \end{description}
%% \item[Use meaningful variable names:] I saw people get confused about
%%   variables such as \texttt{\$1}, \texttt{\$2}.  Assign them to
%%   meaningful names, and you won't get so confused.  Use what your
%%   other lecturers taught you about good programming practice!
%% \end{description}

%% \section{Questions}
%% \label{sec:questions}

%% Make all these scripts executable programs on your \texttt{PATH}.
%% \begin{enumerate}
%% \item Write a simple shell script that takes any number of arguments on the
%%   command line, and prints the arguments with ``Hello '' in front.  For
%%   example, if the name of the script is \texttt{hello}, then you
%%   should be able to run it like this:
%% \begin{verbatim}
%% $ hello Nick Urbanik
%% Hello Nick Urbanik
%% $ hello Edmund
%% Hello Edmund
%% \end{verbatim}

%% \item Write a simple shell script that takes two numbers as parameters
%%   and uses a \texttt{while} loop to print all the numbers from the
%%   first to the second inclusive, each number separated only by a space
%%   from the previous number.  Example, if the script is called
%%   \texttt{jot}, then
%% \begin{verbatim}
%% $ jot 2 8
%% 2 3 4 5 6 7 8
%% \end{verbatim}%$

%% \item Suppose that the script you wrote for the previous question is
%%   called \texttt{jot}.  Then run it calling \texttt{sh} yourself.
%%   Notice the difference:
%% \begin{verbatim}
%% sh jot 2 5
%% sh -v jot 2 5
%% sh -x jot 2 5
%% \end{verbatim}
%% Do you notice any difference in the output from last two?
  
%% \item Write a shell script that, for each \texttt{.rpm} file in the
%%   current directory, prints the name of the package on a line by
%%   itself, then runs \texttt{rpm -K} on the package, then prints a
%%   blank line, using a \texttt{for} loop.
  
%%   Mount the server
%%   \texttt{ictlab\allowbreak.tyict\allowbreak.vtc\allowbreak.edu\allowbreak%
%%     .hk:\allowbreak/var\allowbreak/ftp\allowbreak/pub}
%%   on a convenient directory on your machine, such as
%%   \texttt{/mnt/ftp}.  Test your script on the files in
%%   \texttt{/mnt\allowbreak/ftp\allowbreak/rh-7.2-updated\allowbreak%
%%     /RedHat\allowbreak/RPMS}\@.
  
%%   \begin{explanation}
%%     The option \texttt{rpm -K} chec\textbf{k}s that the software
%%     package is not corrupted, and is signed by the author (if you have
%%     imported the author's public key in your \texttt{gpg} setup)
%%   \end{explanation}
  
%% \item Modify the script you wrote for the previous question to print
%%   the output of \texttt{rpm -K} \emph{only} for \emph{all} the files
%%   that fail the test.  In particular, if the package's \acro{GPG}
%%   signature fails, then your script should display the output of
%%   \texttt{rpm -K}\@.  There are at least two packages in this
%%   directory which do not have a valid \acro{GPG} signature; one of
%%   them is \texttt{redhat-release-7.2-1.noarch.rpm}; what is the other?

%%   Here is output from \texttt{rpm -K} for two packages, one with no
%%   \acro{GPG} signature, the other with:
%% \begin{verbatim}
%% $ rpm -K redhat-release-7.2-1.noarch.rpm bash-2.05-8.i386.rpm
%% redhat-release-7.2-1.noarch.rpm: md5 OK
%% bash-2.05-8.i386.rpm: md5 gpg OK
%% \end{verbatim}%$

%%   Test it in the same network directory as for
%%   the previous question.

%% \item Write a shell script to add a local group called \texttt{admins}
%%   if it does not already exist.  Do not execute any external program
%%   if the \texttt{admins} group already exists.

%% \item Download a copy of the bogus student registration data from
%%   \url{http://nicku.org/snm/lab/regular-expressions/artificial-student-data.txt}.
%%   Use this for the following exercises, together with the
%%   \texttt{grep} program:
%%   \begin{enumerate}
%%   \item Search for all students with the name ``Chan''

%%   \item Search for all students whose student number begins and ends
%%     with 9, and with any other digits in between.

%%   \item Search for all student records where the Hong Kong ID has a
%%     letter, not a number, in the parentheses.

%%   \item If you have time, you may do the same exercises, but display
%%     only the students' names, or student number.
%%   \end{enumerate}
  
%% \item Write a shell script to take a file name on its command line,
%%   and edit it with \texttt{sed} so that every instance of
%%   ``\texttt{/usr/local/bin}'' is changed to ``\texttt{/usr/bin}''
  
%% \item Write a shell script to take a file name on its command line,
%%   and edit it using \texttt{sed} so that every line that begins with
%%   the string \texttt{server}:
%% \begin{alltt}
%% server \emph{other text}
%% \end{alltt}
%% is edited so that averything after ``\texttt{server~}'' (i.e., the
%% ``\texttt{\emph{other text}}'') is replaced with the string
%% ``\texttt{clock.tyict.vtc.edu.hk}'', so that the line above looks like this:
%% \begin{alltt}
%% server clock.tyict.vtc.edu.hk
%% \end{alltt}
%% Test this on a copy of the file \texttt{/etc/ntp.conf} that is on your
%% computer.  (Install the package \texttt{ntp} if it is not there).
%% \end{enumerate}
\end{document}