#!/bin/sh

# ================================================================
set -e

# For building with autoconf:
# * in-directory build:
#   pwd                  is /path/to/the/tree/c/reg_test
#   path to mlr          is /path/to/the/tree/c/mlr
#   path to reg_test/run is /path/to/the/tree/c/reg_test/run
#
# * out-of-directory ("VPATH") build:
#   pwd                  is /path/to/build/tree/c/reg_test
#   path to mlr          is /path/to/build/tree/c/mlr
#   path to reg_test/run is /path/to/source/tree/c/reg_test/run

# For building without autoconf:
#   pwd                  is /does/not/matter
#   path to mlr          is /path/to/the/tree/c/mlr
#   path to reg_test/run is /path/to/the/tree/c/reg_test/run

ourdir=`dirname $0`
srcdir=$ourdir/../..
pwd=`pwd`

try1=$pwd/../mlr    # For autoconf builds, in-tree or out-of-tree
try2=$srcdir/c/mlr  # For non-autoconf builds
if [ -x "$try1" ]; then
  path_to_mlr=$try1
elif [ -x "$try2" ]; then
  path_to_mlr=$try2
else
  echo "$0: Could not find path to mlr executable." 1>&2
  echo "Try 1: $try1" 1>&2
  echo "Try 2: $try2" 1>&2
  exit 1
fi

if [ "$1" = "--valgrind" ]; then
  # Leak-check the test suite. Needs 'make mlrg' first.
  # ../tools/clean-valg can be used to filter the output.
  path_to_mlr="valgrind --leak-check=full ${path_to_mlr}g"
fi
echo Using mlr executable $path_to_mlr

indir=$ourdir/input
expdir=$ourdir/expected
outdir=$pwd/output
outfile=$outdir/out
expfile=$expdir/out
mkdir -p $outdir

rm -f $outfile
touch $outfile
echo

num_passed=0

announce() {
	echo >> $outfile
	echo "================================================================" >> $outfile
	echo "$@" >> $outfile
	echo >> $outfile
}

# A key feature of this regression script is that it can be invoked from any
# directory. Depending on the directory it's invoked from, the path to the mlr
# executable may vary.  Nonetheless for debugging it's crucial that we echo out
# each command being executed. See also diff -I below.
run_mlr() {
  # Use just "mlr" for info messages
	echo mlr "$@"
	echo mlr "$@" >> $outfile
  # Use path to mlr for invoking the command
	$path_to_mlr "$@" >> $outfile
	echo >> $outfile
	# since set -e
	num_passed=`expr $num_passed + 1`
}

# ================================================================
announce STATELESS MAPPERS

run_mlr cat $indir/abixy
run_mlr cat /dev/null

run_mlr cat -n $indir/abixy
run_mlr cat -N counter $indir/abixy

run_mlr cut -f a,x $indir/abixy
run_mlr cut --complement -f a,x $indir/abixy

run_mlr cut -r    -f c,e         $indir/having-fields-regex.dkvp
run_mlr cut -r    -f '"C","E"'   $indir/having-fields-regex.dkvp
run_mlr cut -r    -f '"c"i,"e"'  $indir/having-fields-regex.dkvp
run_mlr cut -r    -f '"C"i,"E"'  $indir/having-fields-regex.dkvp
run_mlr cut -r -x -f c,e         $indir/having-fields-regex.dkvp
run_mlr cut -r -x -f '"C","E"'   $indir/having-fields-regex.dkvp
run_mlr cut -r -x -f '"C","E"i'  $indir/having-fields-regex.dkvp
run_mlr cut -r -x -f '"c","e"i'  $indir/having-fields-regex.dkvp

run_mlr --csvlite cut -r -f '^Name$,^Date_[0-9].*$' $indir/date1.csv $indir/date2.csv

run_mlr having-fields --at-least  a,b         $indir/abixy
run_mlr having-fields --at-least  a,c         $indir/abixy
run_mlr having-fields --at-least  a,b,i,x,y   $indir/abixy
run_mlr having-fields --which-are a,b,i,x     $indir/abixy
run_mlr having-fields --which-are a,b,i,x,y   $indir/abixy
run_mlr having-fields --which-are a,b,i,y,x   $indir/abixy
run_mlr having-fields --which-are a,b,i,x,w   $indir/abixy
run_mlr having-fields --which-are a,b,i,x,y,z $indir/abixy
run_mlr having-fields --at-most   a,c         $indir/abixy
run_mlr having-fields --at-most   a,b,i,x,y   $indir/abixy
run_mlr having-fields --at-most   a,b,i,x,y,z $indir/abixy

run_mlr having-fields --all-matching  '"^[a-z][a-z][a-z]$"'  $indir/having-fields-regex.dkvp
run_mlr having-fields --any-matching  '"^[a-z][a-z][a-z]$"'  $indir/having-fields-regex.dkvp
run_mlr having-fields --none-matching '"^[a-z][a-z][a-z]$"'  $indir/having-fields-regex.dkvp
run_mlr having-fields --all-matching  '"^[a-z][a-z][a-z]$"i' $indir/having-fields-regex.dkvp
run_mlr having-fields --any-matching  '"^[a-z][a-z][a-z]$"i' $indir/having-fields-regex.dkvp
run_mlr having-fields --none-matching '"^[a-z][a-z][a-z]$"i' $indir/having-fields-regex.dkvp

run_mlr rename b,BEE,x,EKS $indir/abixy
run_mlr rename nonesuch,nonesuch,x,EKS $indir/abixy

run_mlr --csvlite rename -r    '^Date_[0-9].*$,Date'  $indir/date1.csv $indir/date2.csv
run_mlr --csvlite rename -r    '(.*)e(.*),\1EEE\2'    $indir/date1.csv $indir/date2.csv
run_mlr --csvlite rename -r    '"(.*)e(.*)"i,\1EEE\2' $indir/date1.csv $indir/date2.csv
run_mlr --csvlite rename -r -g '"(.*)e(.*)"i,\1EEE\2' $indir/date1.csv $indir/date2.csv
run_mlr --csvlite rename -r    '"e",EEE'              $indir/date1.csv $indir/date2.csv
run_mlr --csvlite rename -r -g '"e",EEE'              $indir/date1.csv $indir/date2.csv
run_mlr --csvlite rename -r    '"e"i,EEE'             $indir/date1.csv $indir/date2.csv
run_mlr --csvlite rename -r -g '"e"i,EEE'             $indir/date1.csv $indir/date2.csv

run_mlr regularize $indir/regularize.dkvp

run_mlr --opprint bar -f x,y --lo 0 --hi 1 $indir/abixy
run_mlr --opprint bar -f x,y -c c -x x -b b --lo 0.1 --hi 0.9 -w 20 $indir/abixy
run_mlr --opprint bar --auto -f x,y -w 10 $indir/abixy

# ----------------------------------------------------------------
announce TRIVIAL RETAINERS

run_mlr group-by a   $indir/abixy
run_mlr group-by a,b $indir/abixy

run_mlr group-like $indir/het.dkvp

run_mlr tac $indir/abixy
run_mlr tac /dev/null

# ----------------------------------------------------------------
announce HEAD/TAIL/ETC.

run_mlr head -n 2        $indir/abixy-het
run_mlr head -n 2 -g a   $indir/abixy-het
run_mlr head -n 2 -g a,b $indir/abixy-het

run_mlr tail -n 2        $indir/abixy-het
run_mlr tail -n 2 -g a   $indir/abixy-het
run_mlr tail -n 2 -g a,b $indir/abixy-het

run_mlr top -f x,y -n 2        $indir/abixy-het
run_mlr top -f x,y -n 2 -g a   $indir/abixy-het
run_mlr top -f x,y -n 2 -g a,b $indir/abixy-het

run_mlr top -f x,y -n 2        $indir/ints.dkvp
run_mlr top -f x,y -n 2 -F     $indir/ints.dkvp

run_mlr top    -n 4 -f x        $indir/abixy-wide
run_mlr top    -n 1 -f x,y      $indir/abixy-wide
run_mlr top    -n 4 -f x   -g a $indir/abixy-wide
run_mlr top    -n 1 -f x,y -g a $indir/abixy-wide
run_mlr top -a -n 4 -f x        $indir/abixy-wide
run_mlr top -a -n 4 -f x   -g a $indir/abixy-wide

run_mlr top    -n 3 -f x,y       $indir/near-ovf.dkvp
run_mlr top    -n 3 -f x,y --min $indir/near-ovf.dkvp
run_mlr top -F -n 3 -f x,y       $indir/near-ovf.dkvp
run_mlr top -F -n 3 -f x,y --min $indir/near-ovf.dkvp

run_mlr --seed 12345 bootstrap       $indir/abixy-het
run_mlr --seed 12345 bootstrap -n  2 $indir/abixy-het
run_mlr --seed 12345 bootstrap -n 20 $indir/abixy-het

run_mlr --seed 12345 sample -k 2        $indir/abixy-het
run_mlr --seed 12345 sample -k 2 -g a   $indir/abixy-het
run_mlr --seed 12345 sample -k 2 -g a,b $indir/abixy-het

run_mlr uniq    -g a   $indir/abixy-het
run_mlr uniq    -g a,b $indir/abixy-het

run_mlr uniq    -f a   $indir/abixy-het
run_mlr uniq    -f a,b $indir/abixy-het

run_mlr uniq -c -g a   $indir/abixy-het
run_mlr uniq -c -g a,b $indir/abixy-het

run_mlr count-distinct -f a   $indir/small $indir/abixy
run_mlr count-distinct -f a,b $indir/small $indir/abixy

run_mlr count-distinct -f a   -n $indir/small $indir/abixy
run_mlr count-distinct -f a,b -n $indir/small $indir/abixy

run_mlr grep    pan $indir/abixy-het
run_mlr grep -v pan $indir/abixy-het

run_mlr decimate         -n 4 $indir/abixy
run_mlr decimate      -b -n 4 $indir/abixy
run_mlr decimate      -e -n 4 $indir/abixy
run_mlr decimate -g a    -n 2 $indir/abixy
run_mlr decimate -g a -b -n 2 $indir/abixy
run_mlr decimate -g a -e -n 2 $indir/abixy

# ----------------------------------------------------------------
announce SORT

run_mlr sort -f a   $indir/abixy
run_mlr sort -r a   $indir/abixy
run_mlr sort -f x   $indir/abixy
run_mlr sort -r x   $indir/abixy
run_mlr sort -nf x  $indir/abixy
run_mlr sort -nr x  $indir/abixy

run_mlr sort -f a,b   $indir/abixy
run_mlr sort -r a,b   $indir/abixy
run_mlr sort -f x,y   $indir/abixy
run_mlr sort -r x,y   $indir/abixy
run_mlr sort -nf x,y  $indir/abixy
run_mlr sort -nr x,y  $indir/abixy

run_mlr sort -f a -nr x $indir/abixy
run_mlr sort -nr y -f a $indir/abixy
run_mlr sort -f a -r b -nf x -nr y $indir/abixy

run_mlr sort -f x $indir/sort-het.dkvp
run_mlr sort -r x $indir/sort-het.dkvp

# ----------------------------------------------------------------
announce JOIN

run_mlr --opprint join                   -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u                -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join         --ul      -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u      --ul      -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join              --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u           --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join    --ul      --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u      --ul --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join    --np --ul      -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u --np --ul      -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join    --np      --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u --np      --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join    --np --ul --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u --np --ul --ur -f $indir/joina.dkvp -l l -r r -j o $indir/joinb.dkvp


run_mlr --opprint join                   -f /dev/null -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u                -f /dev/null -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join         --ul      -f /dev/null -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u      --ul      -f /dev/null -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join              --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u           --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join         --ul --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u      --ul --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join    --np --ul      -f /dev/null -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u --np --ul      -f /dev/null -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join    --np      --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u --np      --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp

run_mlr --opprint join    --np --ul --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp
run_mlr --opprint join -u --np --ul --ur -f /dev/null -l l -r r -j o $indir/joinb.dkvp


run_mlr --opprint join                   -f $indir/joina.dkvp -l l -r r -j o /dev/null
run_mlr --opprint join -u                -f $indir/joina.dkvp -l l -r r -j o /dev/null

run_mlr --opprint join         --ul      -f $indir/joina.dkvp -l l -r r -j o /dev/null
run_mlr --opprint join -u      --ul      -f $indir/joina.dkvp -l l -r r -j o /dev/null

run_mlr --opprint join              --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null
run_mlr --opprint join -u           --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null

run_mlr --opprint join         --ul --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null
run_mlr --opprint join -u      --ul --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null

run_mlr --opprint join    --np --ul      -f $indir/joina.dkvp -l l -r r -j o /dev/null
run_mlr --opprint join -u --np --ul      -f $indir/joina.dkvp -l l -r r -j o /dev/null

run_mlr --opprint join    --np      --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null
run_mlr --opprint join -u --np      --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null

run_mlr --opprint join    --np --ul --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null
run_mlr --opprint join -u --np --ul --ur -f $indir/joina.dkvp -l l -r r -j o /dev/null

run_mlr --odkvp join -u -j a -f $indir/join-het.dkvp $indir/abixy-het
run_mlr --odkvp join -u -j a -f $indir/abixy-het     $indir/join-het.dkvp
run_mlr --odkvp join -u --np --ul --ur -j a -f $indir/join-het.dkvp $indir/abixy-het
run_mlr --odkvp join -u --np --ul --ur -j a -f $indir/abixy-het     $indir/join-het.dkvp

for sorted_flag in "" "-u"; do
  for pairing_flags in "" "--np --ul" "--np --ur"; do
    for i in 1 2 3 4 5 6; do
      run_mlr join $sorted_flag $pairing_flags -l l -r r -j j -f $indir/het-join-left $indir/het-join-right-r$i
      for j in 1 2 3 4 5 6; do
        if [ "$i" -le "$j" ]; then
          run_mlr join $sorted_flag $pairing_flags -l l -r r -j j -f $indir/het-join-left $indir/het-join-right-r$i$j
        fi
      done
    done
  done
done

# ----------------------------------------------------------------
announce JOIN PREPIPE

run_mlr --prepipe cat --odkvp join -u -j a -f $indir/join-het.dkvp $indir/abixy-het
run_mlr --odkvp join --prepipe cat -u -j a -f $indir/join-het.dkvp $indir/abixy-het
run_mlr --prepipe cat --odkvp join --prepipe cat -u -j a -f $indir/join-het.dkvp $indir/abixy-het

# ----------------------------------------------------------------
announce JOIN MIXED-FORMAT

# in1, in2, out all same formats
run_mlr --json    join -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.json
run_mlr --dkvp    join -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.dkvp
run_mlr --csvlite join -j x -f $indir/multi-format-join-a.csv  $indir/multi-format-join-b.csv

# in2 different format
run_mlr --json    join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.json
run_mlr --dkvp    join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.dkvp
run_mlr --csvlite join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.csv

run_mlr --json    join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.json
run_mlr --dkvp    join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.dkvp
run_mlr --csvlite join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.csv

run_mlr --json    join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.json
run_mlr --dkvp    join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.dkvp
run_mlr --csvlite join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.csv

# vary all three
run_mlr --ijson    --ojson join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.json
run_mlr --idkvp    --ojson join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --ojson join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.csv

run_mlr --ijson    --ojson join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.json
run_mlr --idkvp    --ojson join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --ojson join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.csv

run_mlr --ijson    --ojson join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.json
run_mlr --idkvp    --ojson join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --ojson join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.csv


run_mlr --ijson    --odkvp join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.json
run_mlr --idkvp    --odkvp join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --odkvp join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.csv

run_mlr --ijson    --odkvp join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.json
run_mlr --idkvp    --odkvp join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --odkvp join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.csv

run_mlr --ijson    --odkvp join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.json
run_mlr --idkvp    --odkvp join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --odkvp join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.csv


run_mlr --ijson    --ocsvlite join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.json
run_mlr --idkvp    --ocsvlite join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --ocsvlite join -i csvlite -j x -f $indir/multi-format-join-a.csv $indir/multi-format-join-b.csv

run_mlr --ijson    --ocsvlite join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.json
run_mlr --idkvp    --ocsvlite join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --ocsvlite join -i dkvp     -j x -f $indir/multi-format-join-a.dkvp $indir/multi-format-join-b.csv

run_mlr --ijson    --ocsvlite join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.json
run_mlr --idkvp    --ocsvlite join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.dkvp
run_mlr --icsvlite --ocsvlite join -i json     -j x -f $indir/multi-format-join-a.json $indir/multi-format-join-b.csv

# ----------------------------------------------------------------
announce RESHAPE

run_mlr --pprint reshape -i X,Y,Z   -o item,price $indir/reshape-wide.tbl
run_mlr --pprint reshape -i X,Z     -o item,price $indir/reshape-wide.tbl
run_mlr --pprint reshape -r '[X-Z]' -o item,price $indir/reshape-wide.tbl
run_mlr --pprint reshape -r '[XZ]'  -o item,price $indir/reshape-wide.tbl

run_mlr --pprint reshape -s item,price $indir/reshape-long.tbl

run_mlr --pprint reshape -i X,Y,Z -o item,price then reshape -s item,price $indir/reshape-wide.tbl
run_mlr --pprint reshape -s item,price then reshape -i X,Y,Z -o item,price $indir/reshape-long.tbl

run_mlr reshape -i X,Y,Z   -o item,price $indir/reshape-wide-ragged.dkvp
run_mlr reshape -i X,Z     -o item,price $indir/reshape-wide-ragged.dkvp
run_mlr reshape -r '[X-Z]' -o item,price $indir/reshape-wide-ragged.dkvp
run_mlr reshape -r '[XZ]'  -o item,price $indir/reshape-wide-ragged.dkvp

run_mlr reshape -s item,price $indir/reshape-long-ragged.dkvp

run_mlr --json reshape -i x,y -o item,value $indir/small-non-nested.json

# ----------------------------------------------------------------
announce STATS

run_mlr --opprint stats1    -a mean,sum,count,min,max,mode -f i,x,y $indir/abixy
run_mlr --opprint stats1    -a min,p10,p50,mode,p90,max    -f i,x,y $indir/abixy
run_mlr --opprint stats1    -a mean,meaneb,stddev          -f i,x,y $indir/abixy
run_mlr --oxtab   stats1 -s -a mean,sum,count,min,max,mode -f i,x,y $indir/abixy

run_mlr --opprint stats1    -a mean,sum,count,min,max,mode -f i,x,y -g a $indir/abixy
run_mlr --opprint stats1    -a min,p10,p50,mode,p90,max    -f i,x,y -g a $indir/abixy
run_mlr --opprint stats1    -a mean,meaneb,stddev          -f i,x,y -g a $indir/abixy
run_mlr --oxtab   stats1 -s -a mean,sum,count,min,max,mode -f i,x,y -g a $indir/abixy

run_mlr --opprint stats1    -a mean,sum,count,min,max,mode -f i,x,y -g a,b $indir/abixy
run_mlr --opprint stats1    -a min,p10,p50,mode,p90,max    -f i,x,y -g a,b $indir/abixy
run_mlr --opprint stats1    -a mean,meaneb,stddev          -f i,x,y -g a,b $indir/abixy
run_mlr --oxtab   stats1 -s -a mean,sum,count,min,max,mode -f i,x,y -g a,b $indir/abixy

run_mlr --oxtab   stats1 -a mean -f x      $indir/abixy-het
run_mlr --oxtab   stats1 -a mean -f x -g a $indir/abixy-het

run_mlr --oxtab   stats1 -a p0,p50,p100 -f x,y    $indir/near-ovf.dkvp
run_mlr --oxtab   stats1 -a p0,p50,p100 -f x,y -F $indir/near-ovf.dkvp

run_mlr --opprint stats2       -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2        $indir/abixy-wide
run_mlr --opprint stats2       -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2 -g a,b $indir/abixy-wide
run_mlr --oxtab   stats2 -s    -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2        $indir/abixy-wide-short
run_mlr --oxtab   stats2 -s    -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2 -g a,b $indir/abixy-wide-short
run_mlr --opprint stats2 --fit -a linreg-ols,linreg-pca             -f x,y,xy,y2        $indir/abixy-wide-short
run_mlr --opprint stats2 --fit -a linreg-ols,linreg-pca             -f x,y,xy,y2 -g a   $indir/abixy-wide-short

run_mlr --opprint stats2    -a logireg -f x,y      $indir/logi.dkvp
run_mlr --opprint stats2    -a logireg -f x,y -g g $indir/logi.dkvp

run_mlr --oxtab   stats2 -a cov -f x,y      $indir/abixy-het
run_mlr --oxtab   stats2 -a cov -f x,y -g a $indir/abixy-het

run_mlr --opprint step -a rsum,delta,counter -f x,y $indir/abixy
run_mlr --opprint step -a rsum,delta,counter -f x,y -g a $indir/abixy
run_mlr --opprint step -a ewma -d 0.1,0.9 -f x,y -g a $indir/abixy
run_mlr --opprint step -a ewma -d 0.1,0.9 -o smooth,rough -f x,y -g a $indir/abixy

run_mlr --odkvp   step -a rsum,delta,counter -f x,y      $indir/abixy-het
run_mlr --odkvp   step -a rsum,delta,counter -f x,y -g a $indir/abixy-het
run_mlr --opprint step -a ewma -d 0.1,0.9 -f x,y -g a $indir/abixy-het
run_mlr --opprint step -a ewma -d 0.1,0.9 -o smooth,rough -f x,y -g a $indir/abixy-het

run_mlr --icsvlite --opprint step -a from-first -f x      $indir/from-first.csv
run_mlr --icsvlite --opprint step -a from-first -f x -g g $indir/from-first.csv

run_mlr --opprint histogram -f x,y --lo 0 --hi 1 --nbins 20 $indir/small

run_mlr --opprint histogram --nbins 9 --auto -f x,y $indir/ints.dkvp

run_mlr --csvlite --opprint merge-fields    -a min,max,sum -c _in,_out $indir/merge-fields-in-out.csv
run_mlr --csvlite --opprint merge-fields -k -a sum -c _in,_out $indir/merge-fields-in-out.csv

run_mlr --oxtab merge-fields -k -a sum,count -f a_in_x,a_out_x -o foo $indir/merge-fields-abxy.dkvp
run_mlr --oxtab merge-fields -k -a sum,count -r in_,out_       -o bar $indir/merge-fields-abxy.dkvp
run_mlr --oxtab merge-fields -k -a sum,count -c in_,out_              $indir/merge-fields-abxy.dkvp

# ----------------------------------------------------------------
announce DSL OPERATOR ASSOCIATIVITY
# Note: filter -v and put -v print the AST.

run_mlr put    -v '$x = 1 || 2 || 3'   /dev/null
run_mlr filter -v '     1 || 2 || 3'   /dev/null
run_mlr put    -v '$x = 1 ^^ 2 ^^ 3'   /dev/null
run_mlr filter -v '     1 ^^ 2 ^^ 3'   /dev/null
run_mlr put    -v '$x = 1 && 2 && 3'   /dev/null
run_mlr filter -v '     1 && 2 && 3'   /dev/null

run_mlr put    -v '$x = 1  == 2  == 3' /dev/null
run_mlr filter -v '     1  == 2  == 3' /dev/null
run_mlr put    -v '$x = 1  != 2  != 3' /dev/null
run_mlr filter -v '     1  != 2  != 3' /dev/null
run_mlr put    -v '$x = 1  =~ 2  =~ 3' /dev/null
run_mlr filter -v '     1  =~ 2  =~ 3' /dev/null
run_mlr put    -v '$x = 1 !=~ 2 !=~ 3' /dev/null
run_mlr filter -v '     1 !=~ 2 !=~ 3' /dev/null
run_mlr put    -v '$x = 1  == 2  != 3' /dev/null
run_mlr filter -v '     1  == 2  != 3' /dev/null
run_mlr put    -v '$x = 1  != 2  == 3' /dev/null
run_mlr filter -v '     1  != 2  == 3' /dev/null

run_mlr put    -v '$x = 1  <  2  <  3' /dev/null
run_mlr filter -v '     1  <  2  <  3' /dev/null
run_mlr put    -v '$x = 1  <= 2  <= 3' /dev/null
run_mlr filter -v '     1  <= 2  <= 3' /dev/null
run_mlr put    -v '$x = 1  >  2  >  3' /dev/null
run_mlr filter -v '     1  >  2  >  3' /dev/null
run_mlr put    -v '$x = 1  >= 2  >= 3' /dev/null
run_mlr filter -v '     1  >= 2  >= 3' /dev/null
run_mlr put    -v '$x = 1  <  2  <= 3' /dev/null
run_mlr filter -v '     1  <  2  <= 3' /dev/null
run_mlr put    -v '$x = 1  <= 2  <  3' /dev/null
run_mlr filter -v '     1  <= 2  <  3' /dev/null

run_mlr put    -v '$x = 1 |  2 |  3'   /dev/null
run_mlr filter -v '     1 |  2 |  3'   /dev/null
run_mlr put    -v '$x = 1 ^  2 ^  3'   /dev/null
run_mlr filter -v '     1 ^  2 ^  3'   /dev/null
run_mlr put    -v '$x = 1 &  2 &  3'   /dev/null
run_mlr filter -v '     1 &  2 &  3'   /dev/null
run_mlr put    -v '$x = 1 |  2 &  3'   /dev/null
run_mlr filter -v '     1 |  2 &  3'   /dev/null
run_mlr put    -v '$x = 1 |  2 ^  3'   /dev/null
run_mlr filter -v '     1 |  2 ^  3'   /dev/null
run_mlr put    -v '$x = 1 ^  2 |  3'   /dev/null
run_mlr filter -v '     1 ^  2 |  3'   /dev/null
run_mlr put    -v '$x = 1 ^  2 &  3'   /dev/null
run_mlr filter -v '     1 ^  2 &  3'   /dev/null
run_mlr put    -v '$x = 1 &  2 |  3'   /dev/null
run_mlr filter -v '     1 &  2 |  3'   /dev/null
run_mlr put    -v '$x = 1 &  2 ^  3'   /dev/null
run_mlr filter -v '     1 &  2 ^  3'   /dev/null

run_mlr put    -v '$x = 1  << 2  << 3' /dev/null
run_mlr filter -v '     1  << 2  << 3' /dev/null
run_mlr put    -v '$x = 1  >> 2  >> 3' /dev/null
run_mlr filter -v '     1  >> 2  >> 3' /dev/null
run_mlr put    -v '$x = 1  << 2  >> 3' /dev/null
run_mlr filter -v '     1  << 2  >> 3' /dev/null
run_mlr put    -v '$x = 1  >> 2  << 3' /dev/null
run_mlr filter -v '     1  >> 2  << 3' /dev/null

run_mlr put    -v '$x = 1 + 2 + 3'   /dev/null
run_mlr filter -v '     1 + 2 + 3'   /dev/null
run_mlr put    -v '$x = 1 - 2 - 3'   /dev/null
run_mlr filter -v '     1 - 2 - 3'   /dev/null
run_mlr put    -v '$x = 1 + 2 - 3'   /dev/null
run_mlr filter -v '     1 + 2 - 3'   /dev/null
run_mlr put    -v '$x = 1 - 2 + 3'   /dev/null
run_mlr filter -v '     1 - 2 + 3'   /dev/null
run_mlr put    -v '$x = 1 . 2 . 3'   /dev/null
run_mlr filter -v '     1 . 2 . 3'   /dev/null

run_mlr put    -v '$x = 1 * 2 * 3'   /dev/null
run_mlr filter -v '     1 * 2 * 3'   /dev/null
run_mlr put    -v '$x = 1 / 2 / 3'   /dev/null
run_mlr filter -v '     1 / 2 / 3'   /dev/null
run_mlr put    -v '$x = 1 // 2 // 3' /dev/null
run_mlr filter -v '     1 // 2 // 3' /dev/null
run_mlr put    -v '$x = 1 % 2 % 3'   /dev/null
run_mlr filter -v '     1 % 2 % 3'   /dev/null
run_mlr put    -v '$x = 1 ** 2 ** 3' /dev/null
run_mlr filter -v '     1 ** 2 ** 3' /dev/null


run_mlr put    -v '$x = 1 *  2 /  3'   /dev/null
run_mlr filter -v '     1 *  2 /  3'   /dev/null
run_mlr put    -v '$x = 1 *  2 // 3'   /dev/null
run_mlr filter -v '     1 *  2 // 3'   /dev/null
run_mlr put    -v '$x = 1 *  2 %  3'   /dev/null
run_mlr filter -v '     1 *  2 %  3'   /dev/null
run_mlr put    -v '$x = 1 *  2 ** 3'   /dev/null
run_mlr filter -v '     1 *  2 ** 3'   /dev/null

run_mlr put    -v '$x = 1 /  2 *  3'   /dev/null
run_mlr filter -v '     1 /  2 *  3'   /dev/null
run_mlr put    -v '$x = 1 /  2 // 3'   /dev/null
run_mlr filter -v '     1 /  2 // 3'   /dev/null
run_mlr put    -v '$x = 1 /  2 %  3'   /dev/null
run_mlr filter -v '     1 /  2 %  3'   /dev/null
run_mlr put    -v '$x = 1 /  2 ** 3'   /dev/null
run_mlr filter -v '     1 /  2 ** 3'   /dev/null

run_mlr put    -v '$x = 1 // 2 *  3'   /dev/null
run_mlr filter -v '     1 // 2 *  3'   /dev/null
run_mlr put    -v '$x = 1 // 2 /  3'   /dev/null
run_mlr filter -v '     1 // 2 /  3'   /dev/null
run_mlr put    -v '$x = 1 // 2 %  3'   /dev/null
run_mlr filter -v '     1 // 2 %  3'   /dev/null
run_mlr put    -v '$x = 1 // 2 ** 3'   /dev/null
run_mlr filter -v '     1 // 2 ** 3'   /dev/null

run_mlr put    -v '$x = 1 %  2 *  3'   /dev/null
run_mlr filter -v '     1 %  2 *  3'   /dev/null
run_mlr put    -v '$x = 1 %  2 /  3'   /dev/null
run_mlr filter -v '     1 %  2 /  3'   /dev/null
run_mlr put    -v '$x = 1 %  2 // 3'   /dev/null
run_mlr filter -v '     1 %  2 // 3'   /dev/null
run_mlr put    -v '$x = 1 %  2 ** 3'   /dev/null
run_mlr filter -v '     1 %  2 ** 3'   /dev/null

run_mlr put    -v '$x = 1 ** 2 *  3'   /dev/null
run_mlr filter -v '     1 ** 2 *  3'   /dev/null
run_mlr put    -v '$x = 1 ** 2 /  3'   /dev/null
run_mlr filter -v '     1 ** 2 /  3'   /dev/null
run_mlr put    -v '$x = 1 ** 2 // 3'   /dev/null
run_mlr filter -v '     1 ** 2 // 3'   /dev/null
run_mlr put    -v '$x = 1 ** 2 %  3'   /dev/null
run_mlr filter -v '     1 ** 2 %  3'   /dev/null

run_mlr put    -v '$x = ++1'   /dev/null
run_mlr filter -v '     ++1'   /dev/null
run_mlr put    -v '$x = --1'   /dev/null
run_mlr filter -v '     --1'   /dev/null
run_mlr put    -v '$x = !!1'   /dev/null
run_mlr filter -v '     !!1'   /dev/null
run_mlr put    -v '$x = ~~1'   /dev/null
run_mlr filter -v '     ~~1'   /dev/null

run_mlr put    -v '$x = 1 ? 2 : 3'         /dev/null
run_mlr filter -v '     1 ? 2 : 3'         /dev/null
run_mlr put    -v '$x = 1 ? 2 ? 3 : 4 : 5' /dev/null
run_mlr filter -v '     1 ? 2 ? 3 : 4 : 5' /dev/null
run_mlr put    -v '$x = 1 ? 2 : 3 ? 4 : 5' /dev/null
run_mlr filter -v '     1 ? 2 : 3 ? 4 : 5' /dev/null

# ----------------------------------------------------------------
announce DSL OPERATOR PRECEDENCE
# Note: filter -v and put -v print the AST.

run_mlr put    -v '$x = 1 || 2 ^^ 3'   /dev/null
run_mlr filter -v '     1 || 2 ^^ 3'   /dev/null
run_mlr put    -v '$x = 1 || 2 && 3'   /dev/null
run_mlr filter -v '     1 || 2 && 3'   /dev/null

run_mlr put    -v '$x = 1 ^^ 2 || 3'   /dev/null
run_mlr filter -v '     1 ^^ 2 || 3'   /dev/null
run_mlr put    -v '$x = 1 ^^ 2 && 3'   /dev/null
run_mlr filter -v '     1 ^^ 2 && 3'   /dev/null

run_mlr put    -v '$x = 1 && 2 || 3'   /dev/null
run_mlr filter -v '     1 && 2 || 3'   /dev/null
run_mlr put    -v '$x = 1 && 2 ^^ 3'   /dev/null
run_mlr filter -v '     1 && 2 ^^ 3'   /dev/null

run_mlr put    -v '$x =  1 == 2 <= 3'  /dev/null
run_mlr filter -v '      1 == 2 <= 3'  /dev/null
run_mlr put    -v '$x =  1 <= 2 == 3'  /dev/null
run_mlr filter -v '      1 <= 2 == 3'  /dev/null

run_mlr put    -v '$x =  1 <= 2 |  3'  /dev/null
run_mlr filter -v '      1 <= 2 |  3'  /dev/null
run_mlr put    -v '$x =  1 |  2 <= 3'  /dev/null
run_mlr filter -v '      1 |  2 <= 3'  /dev/null

run_mlr put    -v '$x =  1 |  2 ^  3'  /dev/null
run_mlr filter -v '      1 |  2 ^  3'  /dev/null
run_mlr put    -v '$x =  1 ^  2 |  3'  /dev/null
run_mlr filter -v '      1 ^  2 |  3'  /dev/null

run_mlr put    -v '$x =  1 ^  2 &  3'  /dev/null
run_mlr filter -v '      1 ^  2 &  3'  /dev/null
run_mlr put    -v '$x =  1 &  2 ^  3'  /dev/null
run_mlr filter -v '      1 &  2 ^  3'  /dev/null

run_mlr put    -v '$x =  1 &  2 << 3'  /dev/null
run_mlr filter -v '      1 &  2 << 3'  /dev/null
run_mlr put    -v '$x =  1 << 2 &  3'  /dev/null
run_mlr filter -v '      1 << 2 &  3'  /dev/null

run_mlr put    -v '$x =  1 +  2 * 3'   /dev/null
run_mlr filter -v '      1 +  2 * 3'   /dev/null
run_mlr put    -v '$x =  1 *  2 + 3'   /dev/null
run_mlr filter -v '      1 *  2 + 3'   /dev/null
run_mlr put    -v '$x =  1 + (2 * 3)'  /dev/null
run_mlr filter -v '      1 + (2 * 3)'  /dev/null
run_mlr put    -v '$x =  1 * (2 + 3)'  /dev/null
run_mlr filter -v '      1 * (2 + 3)'  /dev/null
run_mlr put    -v '$x = (1 + 2) * 3'   /dev/null
run_mlr filter -v '     (1 + 2) * 3'   /dev/null
run_mlr put    -v '$x = (1 * 2) + 3'   /dev/null
run_mlr filter -v '     (1 * 2) + 3'   /dev/null

run_mlr put    -v '$x =  1 +   2 ** 3'  /dev/null
run_mlr filter -v '      1 +   2 ** 3'  /dev/null
run_mlr put    -v '$x =  1 **  2 +  3'  /dev/null
run_mlr filter -v '      1 **  2 +  3'  /dev/null
run_mlr put    -v '$x =  1 +  (2 ** 3)' /dev/null
run_mlr filter -v '      1 +  (2 ** 3)' /dev/null
run_mlr put    -v '$x =  1 ** (2 +  3)' /dev/null
run_mlr filter -v '      1 ** (2 +  3)' /dev/null
run_mlr put    -v '$x = (1 +  2) ** 3'  /dev/null
run_mlr filter -v '     (1 +  2) ** 3'  /dev/null
run_mlr put    -v '$x = (1 ** 2) +  3'  /dev/null
run_mlr filter -v '     (1 ** 2) +  3'  /dev/null

run_mlr put    -v '$x =  1 *   2 ** 3'  /dev/null
run_mlr filter -v '      1 *   2 ** 3'  /dev/null
run_mlr put    -v '$x =  1 **  2 *  3'  /dev/null
run_mlr filter -v '      1 **  2 *  3'  /dev/null
run_mlr put    -v '$x =  1 *  (2 ** 3)' /dev/null
run_mlr filter -v '      1 *  (2 ** 3)' /dev/null
run_mlr put    -v '$x =  1 ** (2 *  3)' /dev/null
run_mlr filter -v '      1 ** (2 *  3)' /dev/null
run_mlr put    -v '$x = (1 *  2) ** 3'  /dev/null
run_mlr filter -v '     (1 *  2) ** 3'  /dev/null
run_mlr put    -v '$x = (1 ** 2) *  3'  /dev/null
run_mlr filter -v '     (1 ** 2) *  3'  /dev/null

run_mlr put    -v '$x = -1 +  2 *  3'  /dev/null
run_mlr filter -v '     -1 +  2 *  3'  /dev/null
run_mlr put    -v '$x = -1 *  2 +  3'  /dev/null
run_mlr filter -v '     -1 *  2 +  3'  /dev/null
run_mlr put    -v '$x =  1 + -2 *  3'  /dev/null
run_mlr filter -v '      1 + -2 *  3'  /dev/null
run_mlr put    -v '$x =  1 * -2 +  3'  /dev/null
run_mlr filter -v '      1 * -2 +  3'  /dev/null
run_mlr put    -v '$x =  1 +  2 * -3'  /dev/null
run_mlr filter -v '      1 +  2 * -3'  /dev/null
run_mlr put    -v '$x =  1 *  2 + -3'  /dev/null
run_mlr filter -v '      1 *  2 + -3'  /dev/null

run_mlr put    -v '$x = ~1 |  2 &  3'  /dev/null
run_mlr filter -v '     ~1 |  2 &  3'  /dev/null
run_mlr put    -v '$x = ~1 &  2 |  3'  /dev/null
run_mlr filter -v '     ~1 &  2 |  3'  /dev/null
run_mlr put    -v '$x =  1 | ~2 &  3'  /dev/null
run_mlr filter -v '      1 | ~2 &  3'  /dev/null
run_mlr put    -v '$x =  1 & ~2 |  3'  /dev/null
run_mlr filter -v '      1 & ~2 |  3'  /dev/null
run_mlr put    -v '$x =  1 |  2 & ~3'  /dev/null
run_mlr filter -v '      1 |  2 & ~3'  /dev/null
run_mlr put    -v '$x =  1 &  2 | ~3'  /dev/null
run_mlr filter -v '      1 &  2 | ~3'  /dev/null

run_mlr put    -v '$x = $a==1 && $b == 1 && $c == 1' /dev/null
run_mlr filter -v '     $a==1 && $b == 1 && $c == 1' /dev/null
run_mlr put    -v '$x = $a==1 || $b == 1 && $c == 1' /dev/null
run_mlr filter -v '     $a==1 || $b == 1 && $c == 1' /dev/null
run_mlr put    -v '$x = $a==1 || $b == 1 || $c == 1' /dev/null
run_mlr filter -v '     $a==1 || $b == 1 || $c == 1' /dev/null
run_mlr put    -v '$x = $a==1 && $b == 1 || $c == 1' /dev/null
run_mlr filter -v '     $a==1 && $b == 1 || $c == 1' /dev/null

run_mlr put    -v '$x = $a==1 ? $b == 2 : $c == 3' /dev/null
run_mlr filter -v '     $a==1 ? $b == 2 : $c == 3' /dev/null

run_mlr put    -v '$x = true' /dev/null
run_mlr filter -v '     true' /dev/null

run_mlr put    -v 'true || 1==0; $x = 3' /dev/null
run_mlr filter -v '        true || 1==0' /dev/null

run_mlr put    -v '1==0 || false; $x = 3' /dev/null
run_mlr filter -v '        1==0 || false' /dev/null

run_mlr put    -v 'true && false; $x = 3' /dev/null
run_mlr filter -v '        true && false' /dev/null

run_mlr put    -v 'true && false && true; $x = 3' /dev/null
run_mlr filter -v '        true && false && true' /dev/null

# ----------------------------------------------------------------
announce DSL FUNCTIONAL TESTS

run_mlr filter '$x>.3'    $indir/abixy
run_mlr filter '$x>.3;'   $indir/abixy
run_mlr filter '$x>0.3'   $indir/abixy
run_mlr filter '$x>0.3 && $y>0.3'   $indir/abixy
run_mlr filter '$x>0.3 || $y>0.3'   $indir/abixy
run_mlr filter 'NR>=4 && NR <= 7'   $indir/abixy

run_mlr filter -x '$x>.3'    $indir/abixy
run_mlr filter -x '$x>0.3'   $indir/abixy
run_mlr filter -x '$x>0.3 && $y>0.3'   $indir/abixy
run_mlr filter -x '$x>0.3 || $y>0.3'   $indir/abixy
run_mlr filter -x 'NR>=4 && NR <= 7'   $indir/abixy

run_mlr filter '$nosuchfield>.3'    $indir/abixy

run_mlr put '$x2 = $x**2'  $indir/abixy
run_mlr put '$x2 = $x**2;' $indir/abixy
run_mlr put '$z = -0.024*$x+0.13' $indir/abixy
run_mlr put '$c = $a . $b' $indir/abixy
run_mlr put '$ii = $i + $i' $indir/abixy
run_mlr put '$emptytest = $i + $nosuch' $indir/abixy

run_mlr --opprint put '$nr=NR;$fnr=FNR;$nf=NF;$filenum=FILENUM' $indir/abixy $indir/abixy

run_mlr --opprint put '$y=madd($x,10,37)' then put '$z=msub($x,10,37)' $indir/modarith.dat
run_mlr --opprint put '$y=mexp($x,35,37)' then put '$z=mmul($x,$y,37)' $indir/modarith.dat

run_mlr put '$z=min($x, $y)' $indir/minmax.dkvp
run_mlr put '$z=max($x, $y)' $indir/minmax.dkvp

run_mlr --icsvlite --oxtab put '${x+y} = ${name.x} + ${name.y}; ${x*y} = ${name.x} * ${name.y}' $indir/braced.csv
run_mlr --icsvlite --oxtab filter '${name.y} < ${z}' $indir/braced.csv

run_mlr --opprint put '$z = $x < 0.5 ? 0 : 1' $indir/abixy

run_mlr --csvlite filter 'true  && true'  $indir/b.csv
run_mlr --csvlite filter 'true  && false' $indir/b.csv
run_mlr --csvlite filter 'false && true'  $indir/b.csv
run_mlr --csvlite filter 'false && false' $indir/b.csv

run_mlr --csvlite filter 'true  || true'  $indir/b.csv
run_mlr --csvlite filter 'true  || false' $indir/b.csv
run_mlr --csvlite filter 'false || true'  $indir/b.csv
run_mlr --csvlite filter 'false || false' $indir/b.csv

run_mlr --csvlite filter 'true  ^^ true'  $indir/b.csv
run_mlr --csvlite filter 'true  ^^ false' $indir/b.csv
run_mlr --csvlite filter 'false ^^ true'  $indir/b.csv
run_mlr --csvlite filter 'false ^^ false' $indir/b.csv

# This tests boolean short-circuiting
run_mlr put 'gate $x==2 && $a =~ "....."; $y=4'  $indir/short-circuit.dkvp

export X=97
export Y=98
run_mlr put '$x = ENV["X"]; $y = ENV[$name]' $indir/env-var.dkvp
export X=
export Y=
run_mlr put '$x = ENV["X"]; $y = ENV[$name]' $indir/env-var.dkvp

# ----------------------------------------------------------------
announce DSL NULL/EMPTY HANDLING

run_mlr put '$z = $x + $y'      $indir/null-vs-empty.dkvp
run_mlr put '$z = $y + $y'      $indir/null-vs-empty.dkvp
run_mlr put '$z = $x + $nosuch' $indir/null-vs-empty.dkvp
run_mlr put '$t = sub($s,       "ell", "X")' $indir/null-vs-empty.dkvp
run_mlr put '$t = sub($s,       "ell", "")'  $indir/null-vs-empty.dkvp
run_mlr put '$t = sub($nosuch,  "ell", "X")' $indir/null-vs-empty.dkvp
run_mlr put '$t = gsub($s,      "l",   "X")' $indir/null-vs-empty.dkvp
run_mlr put '$t = gsub($s,      "l",   "")'  $indir/null-vs-empty.dkvp
run_mlr put '$t = gsub($nosuch, "l",   "X")' $indir/null-vs-empty.dkvp

run_mlr --opprint filter 'isnull($x)'         $indir/nullvals.dkvp
run_mlr --opprint filter 'isnull($y)'         $indir/nullvals.dkvp
run_mlr --opprint filter 'isnull($z)'         $indir/nullvals.dkvp
run_mlr --opprint filter 'isnull($nosuch)'    $indir/nullvals.dkvp
run_mlr --opprint filter 'isnotnull($x)'      $indir/nullvals.dkvp
run_mlr --opprint filter 'isnotnull($y)'      $indir/nullvals.dkvp
run_mlr --opprint filter 'isnotnull($z)'      $indir/nullvals.dkvp
run_mlr --opprint filter 'isnotnull($nosuch)' $indir/nullvals.dkvp

run_mlr --opprint put '$f=isnull($x)'         $indir/nullvals.dkvp
run_mlr --opprint put '$f=isnull($y)'         $indir/nullvals.dkvp
run_mlr --opprint put '$f=isnull($z)'         $indir/nullvals.dkvp
run_mlr --opprint put '$f=isnull($nosuch)'    $indir/nullvals.dkvp
run_mlr --opprint put '$f=isnotnull($x)'      $indir/nullvals.dkvp
run_mlr --opprint put '$f=isnotnull($y)'      $indir/nullvals.dkvp
run_mlr --opprint put '$f=isnotnull($z)'      $indir/nullvals.dkvp
run_mlr --opprint put '$f=isnotnull($nosuch)' $indir/nullvals.dkvp

# ----------------------------------------------------------------
announce DSL TYPE-INFERENCE

run_mlr --xtab put       '$y     = $pi1 + $pi2' $indir/mixed-types.xtab
run_mlr --xtab put    -F '$y     = $pi1 + $pi2' $indir/mixed-types.xtab
run_mlr --xtab put    -S '$y     = $pi1 . $pi2' $indir/mixed-types.xtab
run_mlr --xtab filter    '999   != $pi1 + $pi2' $indir/mixed-types.xtab
run_mlr --xtab filter -F '999   != $pi1 + $pi2' $indir/mixed-types.xtab
run_mlr --xtab filter -S '"999" != $pi1 . $pi2' $indir/mixed-types.xtab

run_mlr --xtab put    '$y=abs($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=abs($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=abs($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put    '$y=abs($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=abs($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=abs($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put -F '$y=abs($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=abs($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=abs($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=abs($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=abs($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=abs($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put    '$y=ceil($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=ceil($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=ceil($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put    '$y=ceil($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=ceil($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=ceil($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put -F '$y=floor($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=floor($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=floor($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=floor($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=floor($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=floor($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put    '$y=round($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=round($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=round($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put    '$y=round($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=round($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=round($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put -F '$y=round($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=round($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=round($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=round($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=round($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=round($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put    '$y=sgn($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=sgn($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=sgn($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put    '$y=sgn($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=sgn($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put    '$y=sgn($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put -F '$y=sgn($pf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=sgn($nf1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=sgn($zf)'  $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=sgn($pi1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=sgn($ni1)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$y=sgn($zi)'  $indir/mixed-types.xtab

run_mlr --xtab put    '$min=min($pf1,$pf2);$max=max($pf1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put    '$min=min($pf1,$pi2);$max=max($pf1,$pi2)' $indir/mixed-types.xtab
run_mlr --xtab put    '$min=min($pi1,$pf2);$max=max($pi1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put    '$min=min($pi1,$pi2);$max=max($pi1,$pi2)' $indir/mixed-types.xtab

run_mlr --xtab put -F '$min=min($pf1,$pf2);$max=max($pf1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$min=min($pf1,$pi2);$max=max($pf1,$pi2)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$min=min($pi1,$pf2);$max=max($pi1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$min=min($pi1,$pi2);$max=max($pi1,$pi2)' $indir/mixed-types.xtab

run_mlr --xtab put    '$min=min($pf1,$pf2);$max=max($pf1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put    '$min=min($pf1,$pi2);$max=max($pf1,$pi2)' $indir/mixed-types.xtab
run_mlr --xtab put    '$min=min($pi1,$pf2);$max=max($pi1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put    '$min=min($pi1,$pi2);$max=max($pi1,$pi2)' $indir/mixed-types.xtab

run_mlr --xtab put -F '$min=min($pf1,$pf2);$max=max($pf1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$min=min($pf1,$pi2);$max=max($pf1,$pi2)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$min=min($pi1,$pf2);$max=max($pi1,$pf2)' $indir/mixed-types.xtab
run_mlr --xtab put -F '$min=min($pi1,$pi2);$max=max($pi1,$pi2)' $indir/mixed-types.xtab

run_mlr --xtab put    '$sum=$pf1+$pf2;$diff=$pf1-$pf2' $indir/mixed-types.xtab
run_mlr --xtab put    '$sum=$pf1+$pi2;$diff=$pf1-$pi2' $indir/mixed-types.xtab
run_mlr --xtab put    '$sum=$pi1+$pf2;$diff=$pi1-$pf2' $indir/mixed-types.xtab
run_mlr --xtab put    '$sum=$pi1+$pi2;$diff=$pi1-$pi2' $indir/mixed-types.xtab

run_mlr --xtab put -F '$sum=$pf1+$pf2;$diff=$pf1-$pf2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$sum=$pf1+$pi2;$diff=$pf1-$pi2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$sum=$pi1+$pf2;$diff=$pi1-$pf2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$sum=$pi1+$pi2;$diff=$pi1-$pi2' $indir/mixed-types.xtab

run_mlr --xtab put    '$prod=$pf1*$pf2;$quot=$pf1/$pf2' $indir/mixed-types.xtab
run_mlr --xtab put    '$prod=$pf1*$pi2;$quot=$pf1/$pi2' $indir/mixed-types.xtab
run_mlr --xtab put    '$prod=$pi1*$pf2;$quot=$pi1/$pf2' $indir/mixed-types.xtab
run_mlr --xtab put    '$prod=$pi1*$pi2;$quot=$pi1/$pi2' $indir/mixed-types.xtab

run_mlr --xtab put -F '$prod=$pf1*$pf2;$quot=$pf1/$pf2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$prod=$pf1*$pi2;$quot=$pf1/$pi2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$prod=$pi1*$pf2;$quot=$pi1/$pf2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$prod=$pi1*$pi2;$quot=$pi1/$pi2' $indir/mixed-types.xtab

run_mlr --xtab put    '$iquot=$pf1//$pf2;$mod=$pf1%$pf2' $indir/mixed-types.xtab
run_mlr --xtab put    '$iquot=$pf1//$pi2;$mod=$pf1%$pi2' $indir/mixed-types.xtab
run_mlr --xtab put    '$iquot=$pi1//$pf2;$mod=$pi1%$pf2' $indir/mixed-types.xtab
run_mlr --xtab put    '$iquot=$pi1//$pi2;$mod=$pi1%$pi2' $indir/mixed-types.xtab

run_mlr --xtab put -F '$iquot=$pf1//$pf2;$mod=$pf1%$pf2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$iquot=$pf1//$pi2;$mod=$pf1%$pi2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$iquot=$pi1//$pf2;$mod=$pi1%$pf2' $indir/mixed-types.xtab
run_mlr --xtab put -F '$iquot=$pi1//$pi2;$mod=$pi1%$pi2' $indir/mixed-types.xtab

run_mlr --xtab put    '$a=roundm($pf1,10.0);$b=roundm($pf1,-10.0)' $indir/mixed-types.xtab
run_mlr --xtab put    '$a=roundm($pf1,10)  ;$b=roundm($pf1,-10)  ' $indir/mixed-types.xtab
run_mlr --xtab put    '$a=roundm($pi1,10.0);$b=roundm($pi1,-10.0)' $indir/mixed-types.xtab
run_mlr --xtab put    '$a=roundm($pi1,10)  ;$b=roundm($pi1,-10)  ' $indir/mixed-types.xtab

# ----------------------------------------------------------------
announce DSL SCIENTIFIC NOTATION IN FIELD VALUES

run_mlr --opprint put '$y=$x+1' $indir/scinot.dkvp

# ----------------------------------------------------------------
announce DSL SCIENTIFIC NOTATION IN EXPRESSION LITERALS

run_mlr --opprint put '$y = 123     + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 123.    + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 123.4   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = .234    + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 1e2     + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 1e-2    + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 1.2e3   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 1.e3    + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 1.2e-3  + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 1.e-3   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = .2e3    + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = .2e-3   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = 1.e-3   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -123    + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -123.   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -123.4  + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -.234   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -1e2    + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -1e-2   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -1.2e3  + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -1.e3   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -1.2e-3 + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -1.e-3  + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -.2e3   + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -.2e-3  + $i' $indir/scinot1.dkvp
run_mlr --opprint put '$y = -1.e-3  + $i' $indir/scinot1.dkvp

# ----------------------------------------------------------------
announce UTF-8 STRLEN

run_mlr --inidx --ifs comma --oxtab put '$s1=strlen($1);$s2=strlen($2);$s3=strlen($3);$s4=strlen($4)' $indir/utf8-align.dkvp

# ----------------------------------------------------------------
announce STATS1/STEP INT/FLOAT

run_mlr --opprint step      -a rsum,delta,counter  -f x,y,z $indir/int-float.dkvp
run_mlr --opprint step   -F -a rsum,delta,counter  -f x,y,z $indir/int-float.dkvp
run_mlr --oxtab   stats1    -a min,max,sum,count   -f x,y,z $indir/int-float.dkvp
run_mlr --oxtab   stats1 -F -a min,max,sum,count   -f x,y,z $indir/int-float.dkvp

# ----------------------------------------------------------------
announce DSL PYTHONIC DIVISION

run_mlr --xtab put    '$quot=$pf1/10.0;$iquot=$pf1//10.0;$mod=$pf1%10.0' $indir/mixed-types.xtab
run_mlr --xtab put    '$quot=$pi1/10  ;$iquot=$pi1//10  ;$mod=$pi1%10  ' $indir/mixed-types.xtab
run_mlr --xtab put    '$quot=$nf1/10.0;$iquot=$nf1//10.0;$mod=$nf1%10.0' $indir/mixed-types.xtab
run_mlr --xtab put    '$quot=$ni1/10  ;$iquot=$ni1//10  ;$mod=$ni1%10  ' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$pf1/10.0;$iquot=$pf1//10.0;$mod=$pf1%10.0' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$pi1/10  ;$iquot=$pi1//10  ;$mod=$pi1%10  ' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$nf1/10.0;$iquot=$nf1//10.0;$mod=$nf1%10.0' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$ni1/10  ;$iquot=$ni1//10  ;$mod=$ni1%10  ' $indir/mixed-types.xtab

run_mlr --xtab put    '$quot=$pf1/-10.0;$iquot=$pf1//-10.0;$mod=$pf1%-10.0' $indir/mixed-types.xtab
run_mlr --xtab put    '$quot=$pi1/-10  ;$iquot=$pi1//-10  ;$mod=$pi1%-10  ' $indir/mixed-types.xtab
run_mlr --xtab put    '$quot=$nf1/-10.0;$iquot=$nf1//-10.0;$mod=$nf1%-10.0' $indir/mixed-types.xtab
run_mlr --xtab put    '$quot=$ni1/-10  ;$iquot=$ni1//-10  ;$mod=$ni1%-10  ' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$pf1/-10.0;$iquot=$pf1//-10.0;$mod=$pf1%-10.0' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$pi1/-10  ;$iquot=$pi1//-10  ;$mod=$pi1%-10  ' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$nf1/-10.0;$iquot=$nf1//-10.0;$mod=$nf1%-10.0' $indir/mixed-types.xtab
run_mlr --xtab put -F '$quot=$ni1/-10  ;$iquot=$ni1//-10  ;$mod=$ni1%-10  ' $indir/mixed-types.xtab

# ----------------------------------------------------------------
announce DSL REGEX MATCHING

run_mlr filter -v '$x =~ "bcd"'       $indir/regex.dkvp
run_mlr filter -v '$x =~ "^bcd"'      $indir/regex.dkvp
run_mlr filter -v '$x =~ "^abc"'      $indir/regex.dkvp
run_mlr filter -v '$x =~ "^abc$"'     $indir/regex.dkvp
run_mlr filter -v '$x =~ "^a.*d$"'    $indir/regex.dkvp
run_mlr filter -v '$x =~ "^a.*"."d$"' $indir/regex.dkvp
run_mlr filter -v '$y =~ "\".."'      $indir/regex.dkvp

run_mlr filter -v '$x =~ "bcd"i'       $indir/regex.dkvp
run_mlr filter -v '$x =~ "^bcd"i'      $indir/regex.dkvp
run_mlr filter -v '$x =~ "^abc"i'      $indir/regex.dkvp
run_mlr filter -v '$x =~ "^abc$"i'     $indir/regex.dkvp
run_mlr filter -v '$x =~ "^a.*d$"i'    $indir/regex.dkvp
run_mlr filter -v '$x =~ "^a.*"."d$"i' $indir/regex.dkvp

# ----------------------------------------------------------------
announce DSL TYPED OVERLAY

run_mlr put '$y = string($x); $z=$y.$y' $indir/int-float.dkvp
run_mlr put '$z=string($x).string($x)' $indir/int-float.dkvp
run_mlr put '$y = string($x)' then put '$z=$y.$y' $indir/int-float.dkvp
run_mlr put '$a="hello"' then put '$b=$a." world";$z=$x+$y;$c=$b;$a=sub($b,"hello","farewell")' $indir/int-float.dkvp

# ----------------------------------------------------------------
announce DSL REGEX CAPTURES

run_mlr --opprint put '$FIELD =~ "([A-Z]+)([0-9]+)";         $F1="\1"; $F2="\2"; $F3="\3"' $indir/capture.dkvp
run_mlr --opprint put '$FIELD =~ "([A-Z]+)[^0-9]*([0-9]+)";  $F1="\1"; $F2="\2"; $F3="\3"' $indir/capture.dkvp

run_mlr --opprint put '$FIELD =~ "([A-Z]+)([0-9]+)"'         then put '$F1="\1"; $F2="\2"; $F3="\3"' $indir/capture.dkvp
run_mlr --opprint put '$FIELD =~ "([A-Z]+)[^0-9]*([0-9]+)"'  then put '$F1="\1"; $F2="\2"; $F3="\3"' $indir/capture.dkvp

run_mlr --opprint put '       $FIELD =~ "....."; $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate   $FIELD =~ "....."; $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "....."; $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp

run_mlr --opprint put '$FIELD =~ "(.)";                            $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)";                         $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)";                      $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)(.)";                   $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)(.)(.)";                $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)(.)(.)(.)";             $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)(.)(.)(.)(.)";          $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)";       $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)(.)";    $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put '$FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)"; $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp

run_mlr --opprint put 'gate $FIELD =~ "(.)";                            $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)";                         $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)";                      $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)(.)";                   $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)(.)(.)";                $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)(.)(.)(.)";             $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)";          $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)";       $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)(.)";    $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'gate $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)"; $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp

run_mlr --opprint put 'filter $FIELD =~ "(.)";                            $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)";                         $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)";                      $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)(.)";                   $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)(.)(.)";                $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)(.)(.)(.)";             $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)";          $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)";       $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)(.)";    $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp
run_mlr --opprint put 'filter $FIELD =~ "(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)"; $F0="\0";$F1="\1";$F2="\2";$F3="\3";$F4="\4";$F5="\5";$F6="\6";$F7="\7";$F8="\8";$F9="\9"' $indir/capture-lengths.dkvp

# ----------------------------------------------------------------
announce DSL FILTER/GATE

run_mlr --opprint put '         $x > 0.5;  $z = "flag"' $indir/abixy
run_mlr --opprint put '       !($x > 0.5); $z = "flag"' $indir/abixy
run_mlr --opprint put 'filter   $x > 0.5;  $z = "flag"' $indir/abixy
run_mlr --opprint put 'gate     $x > 0.5;  $z = "flag"' $indir/abixy
run_mlr --opprint put 'filter !($x > 0.5); $z = "flag"' $indir/abixy
run_mlr --opprint put 'gate   !($x > 0.5); $z = "flag"' $indir/abixy

# ----------------------------------------------------------------
announce DSL DATETIME FUNCTIONS

run_mlr --csvlite put '$gmt = sec2gmt($sec)' $indir/sec2gmt
run_mlr --csvlite put '$sec = gmt2sec($gmt)' $indir/gmt2sec

run_mlr --csvlite put '$gmt = strftime($sec, "%Y-%m-%dT%H:%M:%SZ")' $indir/sec2gmt
run_mlr --csvlite put '$sec = strptime($gmt, "%Y-%m-%dT%H:%M:%SZ")' $indir/gmt2sec

run_mlr --csvlite sec2gmt sec $indir/sec2gmt

run_mlr --opprint put '$hms=sec2hms($sec);   $resec=hms2sec($hms);   $diff=$resec-$sec' $indir/sec2xhms
run_mlr --opprint put '$hms=fsec2hms($sec);  $resec=hms2fsec($hms);  $diff=$resec-$sec' $indir/fsec2xhms
run_mlr --opprint put '$hms=sec2dhms($sec);  $resec=dhms2sec($hms);  $diff=$resec-$sec' $indir/sec2xhms
run_mlr --opprint put '$hms=fsec2dhms($sec); $resec=dhms2fsec($hms); $diff=$resec-$sec' $indir/fsec2xhms

# ----------------------------------------------------------------
announce DSL SUB/GSUB

run_mlr --opprint put '$y = sub($x, "e.*l",        "")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "e.*l"i,       "")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "e.*"."l",     "")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "e.*l",        "y123y")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "e.*l"i,       "y123y")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "e.*"."l",     "y123y")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "([hg])e.*l(.)", "y\1y123\2y")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "([hg])e.*l.",   "y\1y123\2y")' $indir/sub.dat
run_mlr --opprint put '$y = sub($x, "([hg])e.*l(.)", "y\1y123.y")'  $indir/sub.dat

run_mlr --opprint put '$y = sub($x,  "a",    "aa")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "a",    "aa")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "A",    "Aa")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "a"i,   "Aa")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "A"i,   "Aa")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "a(.)", "aa\1\1\1")' $indir/gsub.dat

run_mlr --opprint put '$y = sub($x,  "a",    "")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "a",    "")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "A",    "")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "a"i,   "")'   $indir/gsub.dat
run_mlr --opprint put '$y = gsub($x, "A"i,   "")'   $indir/gsub.dat

# ----------------------------------------------------------------
announce DSL EXPERIMENTAL OOSVARS

run_mlr --opprint put -v 'begin @ox=0; $d=$x-@ox; @ox=$x' $indir/abixy
run_mlr --opprint put -v 'begin @ox="no"; $d=@ox == "no" ? 1.0 : $x/@ox; @ox=$x' then step -a ratio -f x $indir/abixy
run_mlr --opprint put -v '$d=$x/@ox; @ox=$x' then step -a ratio -f x $indir/abixy
run_mlr --opprint put -v 'begin @ox="no"; $d=@ox == "no" ? 1.0 : $x/@ox; @ox=$x' then step -a ratio -f x $indir/abixy
run_mlr --opprint put -v 'begin @rsum = 0; @rsum = @rsum + $x; $rsum = @rsum' $indir/abixy
run_mlr --opprint put -v 'begin @a=0; begin @b=0; begin @c=0; $za=@a; $zb=@b; $zc=@c; $d=@a+@b+@c; @a=@b; @b=@c; @c=$i' $indir/abixy
run_mlr --opprint put -v 'begin {@a=0; @b=0; @c=0}; $za=@a; $zb=@b; $zc=@c; $d=@a+@b+@c; @a=@b; @b=@c; @c=$i' $indir/abixy
run_mlr --opprint put -v 'begin @ox=0; $d=$x-@ox; @ox=$x' $indir/abixy

run_mlr put -v '@a=$a; @b=$b; @c=$x; end {emit @a; emit @b; emit @c}' $indir/abixy
run_mlr put -v '@a=$a; @b=$b; @c=$x; end emit @a, @b, @c' $indir/abixy

run_mlr --opprint put -v 'begin {@count=0; @sum=0.0}; @count=@count+1; @sum=@sum+$x; end{@mean=@sum/@count; emit @mean}' $indir/abixy
run_mlr --opprint put -v 'end{@mean=@sum/@count; emit @mean}; begin {@count=0; @sum=0.0}; @count=@count+1; @sum=@sum+$x' $indir/abixy

run_mlr put -v 'begin{ @a = @b[1] }; $c = @d; @e[$i][2+$j][3] = $4; end @f[@g[5][@h]] = 6' /dev/null

run_mlr put '@y[$a]=$y; end dump' $indir/abixy

run_mlr stats1 -a sum -f y -g a $indir/abixy
run_mlr put '@y_sum[$a] = $y; end dump' $indir/abixy

# ----------------------------------------------------------------
announce CHAINING

run_mlr cat then cat $indir/short
run_mlr cat then tac $indir/short
run_mlr tac then cat $indir/short
run_mlr tac then tac $indir/short

run_mlr cat then cat then cat $indir/short
run_mlr cat then cat then tac $indir/short
run_mlr cat then tac then cat $indir/short
run_mlr cat then tac then tac $indir/short
run_mlr tac then cat then cat $indir/short
run_mlr tac then cat then tac $indir/short
run_mlr tac then tac then cat $indir/short
run_mlr tac then tac then tac $indir/short

# ----------------------------------------------------------------
announce NUMBER FORMATTING

run_mlr                --opprint stats1 -a sum -f x $indir/ofmt.dat
run_mlr --ofmt '%.3lf' --opprint stats1 -a sum -f x $indir/ofmt.dat
run_mlr --opprint --ofmt '%.3lf' stats1 -a sum -f x $indir/ofmt.dat

# ----------------------------------------------------------------
announce IMPLICIT-HEADER-CSV INPUT

run_mlr --irs crlf                       --no-mmap --icsvlite --ifs ,  --opprint cut -x -f b/ $indir/multi-sep.csv
run_mlr --irs crlf --implicit-csv-header --no-mmap --icsvlite --ifs ,  --opprint cut -x -f 2  $indir/multi-sep.csv

run_mlr --irs crlf                       --no-mmap --icsvlite --ifs /, --opprint cut -x -f b  $indir/multi-sep.csv
run_mlr --irs crlf --implicit-csv-header --no-mmap --icsvlite --ifs /, --opprint cut -x -f 2  $indir/multi-sep.csv

run_mlr --irs crlf                       --mmap    --icsvlite --ifs ,  --opprint cut -x -f b/ $indir/multi-sep.csv
run_mlr --irs crlf --implicit-csv-header --mmap    --icsvlite --ifs ,  --opprint cut -x -f 2  $indir/multi-sep.csv

run_mlr --irs crlf                       --mmap    --icsvlite --ifs /, --opprint cut -x -f b  $indir/multi-sep.csv
run_mlr --irs crlf --implicit-csv-header --mmap    --icsvlite --ifs /, --opprint cut -x -f 2  $indir/multi-sep.csv

run_mlr                       --icsv --ifs ,  --opprint cut -x -f b/ $indir/multi-sep.csv
run_mlr --implicit-csv-header --icsv --ifs ,  --opprint cut -x -f 2  $indir/multi-sep.csv

run_mlr                       --icsv --ifs /, --opprint cut -x -f b  $indir/multi-sep.csv
run_mlr --implicit-csv-header --icsv --ifs /, --opprint cut -x -f 2  $indir/multi-sep.csv


# ----------------------------------------------------------------
announce HET-CSV INPUT

run_mlr --icsvlite --odkvp cat $indir/a.csv
run_mlr --icsvlite --odkvp cat $indir/b.csv
run_mlr --icsvlite --odkvp cat $indir/c.csv
run_mlr --icsvlite --odkvp cat $indir/d.csv
run_mlr --icsvlite --odkvp cat $indir/e.csv
run_mlr --icsvlite --odkvp cat $indir/f.csv
run_mlr --icsvlite --odkvp cat $indir/g.csv

run_mlr --icsvlite --odkvp cat $indir/a.csv $indir/a.csv
run_mlr --icsvlite --odkvp cat $indir/b.csv $indir/b.csv
run_mlr --icsvlite --odkvp cat $indir/c.csv $indir/c.csv
run_mlr --icsvlite --odkvp cat $indir/d.csv $indir/d.csv
run_mlr --icsvlite --odkvp cat $indir/e.csv $indir/e.csv
run_mlr --icsvlite --odkvp cat $indir/f.csv $indir/f.csv
run_mlr --icsvlite --odkvp cat $indir/g.csv $indir/g.csv

run_mlr --icsvlite --odkvp cat $indir/a.csv $indir/b.csv
run_mlr --icsvlite --odkvp cat $indir/b.csv $indir/c.csv
run_mlr --icsvlite --odkvp cat $indir/c.csv $indir/d.csv
run_mlr --icsvlite --odkvp cat $indir/d.csv $indir/e.csv
run_mlr --icsvlite --odkvp cat $indir/e.csv $indir/f.csv
run_mlr --icsvlite --odkvp cat $indir/f.csv $indir/g.csv

run_mlr --icsvlite --odkvp cat $indir/a.csv $indir/b.csv \
  $indir/c.csv $indir/d.csv $indir/e.csv $indir/f.csv $indir/g.csv

run_mlr --icsvlite --odkvp tac $indir/het.csv

run_mlr --headerless-csv-output --csvlite tac $indir/a.csv
run_mlr --headerless-csv-output --csvlite tac $indir/c.csv
run_mlr --headerless-csv-output --csvlite tac $indir/a.csv $indir/c.csv
run_mlr --headerless-csv-output --csvlite tac $indir/het.csv
run_mlr --headerless-csv-output --csvlite group-like $indir/het.csv

# ----------------------------------------------------------------
announce HET-PPRINT INPUT

run_mlr --ipprint --odkvp cat $indir/a.pprint
run_mlr --ipprint --odkvp cat $indir/b.pprint
run_mlr --ipprint --odkvp cat $indir/c.pprint
run_mlr --ipprint --odkvp cat $indir/d.pprint
run_mlr --ipprint --odkvp cat $indir/e.pprint
run_mlr --ipprint --odkvp cat $indir/f.pprint
run_mlr --ipprint --odkvp cat $indir/g.pprint

run_mlr --ipprint --odkvp cat $indir/a.pprint $indir/a.pprint
run_mlr --ipprint --odkvp cat $indir/b.pprint $indir/b.pprint
run_mlr --ipprint --odkvp cat $indir/c.pprint $indir/c.pprint
run_mlr --ipprint --odkvp cat $indir/d.pprint $indir/d.pprint
run_mlr --ipprint --odkvp cat $indir/e.pprint $indir/e.pprint
run_mlr --ipprint --odkvp cat $indir/f.pprint $indir/f.pprint
run_mlr --ipprint --odkvp cat $indir/g.pprint $indir/g.pprint

run_mlr --ipprint --odkvp cat $indir/a.pprint $indir/b.pprint
run_mlr --ipprint --odkvp cat $indir/b.pprint $indir/c.pprint
run_mlr --ipprint --odkvp cat $indir/c.pprint $indir/d.pprint
run_mlr --ipprint --odkvp cat $indir/d.pprint $indir/e.pprint
run_mlr --ipprint --odkvp cat $indir/e.pprint $indir/f.pprint
run_mlr --ipprint --odkvp cat $indir/f.pprint $indir/g.pprint

run_mlr --ipprint --odkvp cat $indir/a.pprint $indir/b.pprint \
  $indir/c.pprint $indir/d.pprint $indir/e.pprint $indir/f.pprint $indir/g.pprint

# ================================================================
announce NULL-FIELD INPUT

run_mlr --icsvlite --odkvp cat $indir/null-fields.csv
run_mlr --inidx --ifs comma --odkvp cat $indir/null-fields.nidx
run_mlr --idkvp --oxtab cat $indir/missings.dkvp

run_mlr --oxtab stats1 -a sum,min,max,mode -f x          $indir/nullvals.dkvp
run_mlr --oxtab stats1 -a sum,min,max,mode -f y          $indir/nullvals.dkvp
run_mlr --oxtab stats1 -a sum,min,max,mode -f z          $indir/nullvals.dkvp
run_mlr --oxtab stats1 -a sum,min,max,mode -f x,y,z      $indir/nullvals.dkvp
run_mlr --oxtab stats1 -a sum,min,max,mode -f x     -g a $indir/nullvals.dkvp
run_mlr --oxtab stats1 -a sum,min,max,mode -f y     -g a $indir/nullvals.dkvp
run_mlr --oxtab stats1 -a sum,min,max,mode -f z     -g a $indir/nullvals.dkvp
run_mlr --oxtab stats1 -a sum,min,max,mode -f x,y,z -g a $indir/nullvals.dkvp

run_mlr --opprint merge-fields -a sum,min,max,mode -f x,y,z -o xyz $indir/nullvals.dkvp
run_mlr --opprint merge-fields -a sum,min,max,mode -r x,y,z -o xyz $indir/nullvals.dkvp
run_mlr --opprint merge-fields -a sum,min,max,mode -c x,y,z        $indir/nullvals.dkvp

run_mlr --oxtab stats2 -a cov -f x,y        $indir/nullvals.dkvp
run_mlr --oxtab stats2 -a cov -f x,z        $indir/nullvals.dkvp
run_mlr --oxtab stats2 -a cov -f y,z        $indir/nullvals.dkvp
run_mlr --oxtab stats2 -a cov -f x,y   -g a $indir/nullvals.dkvp
run_mlr --oxtab stats2 -a cov -f x,z   -g a $indir/nullvals.dkvp
run_mlr --oxtab stats2 -a cov -f y,z   -g a $indir/nullvals.dkvp

run_mlr --opprint top    -n 5 -f x          $indir/nullvals.dkvp
run_mlr --opprint top    -n 5 -f y          $indir/nullvals.dkvp
run_mlr --opprint top    -n 5 -f z          $indir/nullvals.dkvp
run_mlr --opprint top    -n 5 -f x,y,z      $indir/nullvals.dkvp
run_mlr --opprint top    -n 5 -f x     -g a $indir/nullvals.dkvp
run_mlr --opprint top    -n 5 -f y     -g a $indir/nullvals.dkvp
run_mlr --opprint top    -n 5 -f z     -g a $indir/nullvals.dkvp
run_mlr --opprint top    -n 5 -f x,y,z -g a $indir/nullvals.dkvp

run_mlr --opprint top -a -n 5 -f x          $indir/nullvals.dkvp
run_mlr --opprint top -a -n 5 -f y          $indir/nullvals.dkvp
run_mlr --opprint top -a -n 5 -f z          $indir/nullvals.dkvp
run_mlr --opprint top -a -n 5 -f x     -g a $indir/nullvals.dkvp
run_mlr --opprint top -a -n 5 -f y     -g a $indir/nullvals.dkvp
run_mlr --opprint top -a -n 5 -f z     -g a $indir/nullvals.dkvp

run_mlr --opprint step -a counter,rsum -f x          $indir/nullvals.dkvp
run_mlr --opprint step -a counter,rsum -f y          $indir/nullvals.dkvp
run_mlr --opprint step -a counter,rsum -f z          $indir/nullvals.dkvp
run_mlr --opprint step -a counter,rsum -f x,y,z      $indir/nullvals.dkvp

run_mlr --opprint step -a counter,rsum -f x     -g a $indir/nullvals.dkvp
run_mlr --opprint step -a counter,rsum -f y     -g a $indir/nullvals.dkvp
run_mlr --opprint step -a counter,rsum -f z     -g a $indir/nullvals.dkvp
run_mlr --opprint step -a counter,rsum -f x,y,z -g a $indir/nullvals.dkvp

# ================================================================
announce SPACE-PADDING

run_mlr --mmap    --idkvp    --odkvp --ifs space --repifs cat $indir/space-pad.dkvp
run_mlr --no-mmap --idkvp    --odkvp --ifs space --repifs cat $indir/space-pad.dkvp
run_mlr --mmap    --inidx    --odkvp --ifs space --repifs cat $indir/space-pad.nidx
run_mlr --no-mmap --inidx    --odkvp --ifs space --repifs cat $indir/space-pad.nidx
run_mlr --mmap    --icsvlite --odkvp --ifs space --repifs cat $indir/space-pad.pprint
run_mlr --no-mmap --icsvlite --odkvp --ifs space --repifs cat $indir/space-pad.pprint

# ================================================================
announce DOUBLE PS

run_mlr --no-mmap --opprint cat $indir/double-ps.dkvp
run_mlr    --mmap --opprint cat $indir/double-ps.dkvp

# ================================================================
announce MISSING FINAL LF

run_mlr --no-mmap --csvlite cat $indir/truncated.csv
run_mlr           --csvlite cat $indir/truncated.csv
run_mlr --no-mmap --dkvp    cat $indir/truncated.dkvp
run_mlr           --dkvp    cat $indir/truncated.dkvp
run_mlr --no-mmap --nidx    cat $indir/truncated.nidx
run_mlr           --nidx    cat $indir/truncated.nidx
run_mlr --no-mmap --pprint  cat $indir/truncated.pprint
run_mlr           --pprint  cat $indir/truncated.pprint
run_mlr --no-mmap --xtab    cat $indir/truncated.xtab
run_mlr           --xtab    cat $indir/truncated.xtab

# ================================================================
announce UTF-8 alignment

run_mlr --icsvlite --opprint cat $indir/utf8-1.csv
run_mlr --icsvlite --opprint cat $indir/utf8-2.csv
run_mlr --icsvlite --oxtab   cat $indir/utf8-1.csv
run_mlr --icsvlite --oxtab   cat $indir/utf8-2.csv

run_mlr --inidx --ifs space --opprint         cat $indir/utf8-align.nidx
run_mlr --inidx --ifs space --opprint --right cat $indir/utf8-align.nidx
run_mlr --oxtab cat $indir/utf8-align.dkvp

run_mlr --inidx --ifs space --oxtab --xvright cat $indir/utf8-align.nidx

# ================================================================
announce MMAP AT PAGE BOUNDARIES

run_mlr --dkvp        tail -n 4 $indir/page-aligned-final-ifs.dkvp
run_mlr --dkvp        tail -n 4 $indir/page-aligned-final-irs.dkvp
run_mlr --dkvp        tail -n 4 $indir/page-aligned-final-no-ifs.dkvp
run_mlr --dkvp        tail -n 4 $indir/page-aligned-no-final-irs.dkvp
run_mlr --nidx        tail -n 4 $indir/page-aligned-no-final-irs.nidx
run_mlr --csvlite     tail -n 4 $indir/page-aligned-no-final-irs.csvl
run_mlr --csv --rs lf tail -n 4 $indir/page-aligned-no-final-irs.csvl
run_mlr --xtab        tail -n 4 $indir/page-aligned-no-final-eol.xtab

# ================================================================
announce INT64 I/O

run_mlr --oxtab put '$c=$a;$d=$b;$e=hexfmt($a);$f=hexfmt($b)' $indir/int64io.dkvp

# There is different rounding on i386 vs. x86_64 but the essential check is
# that overflow has been avoided. Hence the %g output format here.
run_mlr --oxtab --ofmt '%.8g' put '$p0=$p+0;$p1=$p+1;$p2=$p+2;$p3=$p+3' $indir/int64arith.dkvp
run_mlr --oxtab --ofmt '%.8g' put '$p0=$p-0;$p1=$p-1;$p2=$p-2;$p3=$p-3' $indir/int64arith.dkvp
run_mlr --oxtab --ofmt '%.8g' put '$p0=$p*0;$p1=$p*1;$p2=$p*2;$p3=$p*3' $indir/int64arith.dkvp
run_mlr --oxtab --ofmt '%.8g' put '$n0=$n+0;$n1=$n+1;$n2=$n+2;$n3=$n+3' $indir/int64arith.dkvp
run_mlr --oxtab --ofmt '%.8g' put '$n0=$n-0;$n1=$n-1;$n2=$n-2;$n3=$n-3' $indir/int64arith.dkvp
run_mlr --oxtab --ofmt '%.8g' put '$n0=$n*0;$n1=$n*1;$n2=$n*2;$n3=$n*3' $indir/int64arith.dkvp

# ================================================================
announce COMPRESSED INPUT

run_mlr --csv  --prepipe 'cat'   cat   $indir/rfc-csv/simple.csv
run_mlr --dkvp --prepipe 'cat'   cat   $indir/abixy
run_mlr --csv  --prepipe 'cat'   cat < $indir/rfc-csv/simple.csv
run_mlr --dkvp --prepipe 'cat'   cat < $indir/abixy

# ================================================================
announce STDIN

run_mlr --csv cat < $indir/rfc-csv/simple.csv

# ================================================================
announce RFC-CSV

run_mlr --no-mmap --csv cat $indir/rfc-csv/simple.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/simple.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/simple-truncated.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/simple-truncated.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/narrow.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/narrow.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/narrow-truncated.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/narrow-truncated.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/quoted-comma.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/quoted-comma.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/quoted-comma-truncated.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/quoted-comma-truncated.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/quoted-crlf.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/quoted-crlf.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/quoted-crlf-truncated.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/quoted-crlf-truncated.csv

run_mlr --no-mmap --csv cat $indir/rfc-csv/simple-truncated.csv $indir/rfc-csv/simple.csv
run_mlr --mmap    --csv cat $indir/rfc-csv/simple-truncated.csv $indir/rfc-csv/simple.csv

run_mlr --no-mmap --csv --ifs semicolon --ofs pipe --irs lf --ors lflf cut -x -f b $indir/rfc-csv/modify-defaults.csv
run_mlr --mmap    --csv --ifs semicolon --ofs pipe --irs lf --ors lflf cut -x -f b $indir/rfc-csv/modify-defaults.csv

# ================================================================
announce MULTI-CHARACTER IRS/IFS/IPS FOR DKVP

run_mlr --oxtab --idkvp --mmap    --irs lf   --ifs ,  --ips =  cut -o -f x,a,i $indir/multi-sep.dkvp
run_mlr --oxtab --idkvp --mmap    --irs lf   --ifs /, --ips =: cut -o -f x,a,i $indir/multi-sep.dkvp
run_mlr --oxtab --idkvp --mmap    --irs crlf --ifs ,  --ips =  cut -o -f x,a,i $indir/multi-sep.dkvp
run_mlr --oxtab --idkvp --mmap    --irs crlf --ifs /, --ips =: cut -o -f x,a,i $indir/multi-sep.dkvp

run_mlr --oxtab --no-mmap --irs lf   --ifs ,  --ips =  cut -o -f x,a,i $indir/multi-sep.dkvp
run_mlr --oxtab --no-mmap --irs lf   --ifs /, --ips =: cut -o -f x,a,i $indir/multi-sep.dkvp
run_mlr --oxtab --no-mmap --irs crlf --ifs ,  --ips =  cut -o -f x,a,i $indir/multi-sep.dkvp
run_mlr --oxtab --no-mmap --irs crlf --ifs /, --ips =: cut -o -f x,a,i $indir/multi-sep.dkvp

# ================================================================
announce MULTI-CHARACTER IRS/IFS FOR NIDX

run_mlr --oxtab --inidx --mmap    --irs lf   --ifs ,  cut -o -f 4,1,3 $indir/multi-sep.dkvp
run_mlr --oxtab --inidx --mmap    --irs lf   --ifs /, cut -o -f 4,1,3 $indir/multi-sep.dkvp
run_mlr --oxtab --inidx --mmap    --irs crlf --ifs ,  cut -o -f 4,1,3 $indir/multi-sep.dkvp
run_mlr --oxtab --inidx --mmap    --irs crlf --ifs /, cut -o -f 4,1,3 $indir/multi-sep.dkvp

run_mlr --oxtab --inidx --no-mmap --irs lf   --ifs ,  cut -o -f 4,1,3 $indir/multi-sep.dkvp
run_mlr --oxtab --inidx --no-mmap --irs lf   --ifs /, cut -o -f 4,1,3 $indir/multi-sep.dkvp
run_mlr --oxtab --inidx --no-mmap --irs crlf --ifs ,  cut -o -f 4,1,3 $indir/multi-sep.dkvp
run_mlr --oxtab --inidx --no-mmap --irs crlf --ifs /, cut -o -f 4,1,3 $indir/multi-sep.dkvp

# ================================================================
announce MULTI-CHARACTER IRS/IFS FOR CSVLITE

run_mlr --oxtab --icsvlite --mmap    --irs lf   --ifs ,  cut -o -f x/,a/,i/ $indir/multi-sep.csv
run_mlr --oxtab --icsvlite --mmap    --irs lf   --ifs /, cut -o -f x,a,i    $indir/multi-sep.csv
run_mlr --oxtab --icsvlite --mmap    --irs crlf --ifs ,  cut -o -f x/,a/,i/ $indir/multi-sep.csv
run_mlr --oxtab --icsvlite --mmap    --irs crlf --ifs /, cut -o -f x,a,i    $indir/multi-sep.csv

run_mlr --oxtab --icsvlite --no-mmap --irs lf   --ifs ,  cut -o -f x/,a/,i/ $indir/multi-sep.csv
run_mlr --oxtab --icsvlite --no-mmap --irs lf   --ifs /, cut -o -f x,a,i    $indir/multi-sep.csv
run_mlr --oxtab --icsvlite --no-mmap --irs crlf --ifs ,  cut -o -f x/,a/,i/ $indir/multi-sep.csv
run_mlr --oxtab --icsvlite --no-mmap --irs crlf --ifs /, cut -o -f x,a,i    $indir/multi-sep.csv

# ================================================================
announce MULTI-CHARACTER SEPARATORS FOR XTAB

run_mlr --mmap    --xtab --ifs crlf --ofs Z cut -x -f b $indir/truncated.xtab-crlf
run_mlr --no-mmap --xtab --ifs crlf --ofs Z cut -x -f b $indir/truncated.xtab-crlf
run_mlr --mmap    --xtab --ips . --ops @ cut -x -f b $indir/dots.xtab
run_mlr --no-mmap --xtab --ips . --ops @ cut -x -f b $indir/dots.xtab
run_mlr --no-mmap --xtab --ips . --ops := cut -x -f b $indir/dots.xtab
run_mlr --xtab --ips ": " --ops '@@@@' put '$sum=int($a+$b)' $indir/multi-ips.dkvp

# ================================================================
announce MULTI-CHARACTER IRS FOR PPRINT

run_mlr --mmap    --pprint --irs crlf --ifs / --ofs @ cut -x -f b $indir/dots.pprint
run_mlr --no-mmap --pprint --irs crlf --ifs / --ofs @ cut -x -f b $indir/dots.pprint

# ================================================================
announce MULTI-CHARACTER IXS SPECIFIERS

run_mlr --oxtab --idkvp --mmap  --irs lf   --ifs '\x2c'  --ips '\075'  cut -o -f x,a,i $indir/multi-sep.dkvp
run_mlr --oxtab --idkvp --mmap  --irs lf   --ifs /, --ips '\x3d\x3a' cut -o -f x,a,i $indir/multi-sep.dkvp

# ================================================================
announce JSON I/O

run_mlr --ijson --opprint cat $indir/small-non-nested.json
run_mlr --ijson --opprint cat $indir/small-non-nested-wrapped.json
run_mlr --ijson --oxtab   cat $indir/small-nested.json

run_mlr --ojson                                     cat $indir/json-output-options.dkvp
run_mlr --ojson --jvstack                           cat $indir/json-output-options.dkvp
run_mlr --ojson             --jlistwrap             cat $indir/json-output-options.dkvp
run_mlr --ojson --jvstack   --jlistwrap             cat $indir/json-output-options.dkvp
run_mlr --ojson                         --jquoteall cat $indir/json-output-options.dkvp
run_mlr --ojson --jvstack               --jquoteall cat $indir/json-output-options.dkvp
run_mlr --ojson             --jlistwrap --jquoteall cat $indir/json-output-options.dkvp
run_mlr --ojson --jvstack   --jlistwrap --jquoteall cat $indir/json-output-options.dkvp

# ================================================================
# A key feature of this regression script is that it can be invoked from any
# directory. Depending on the directory it's invoked from, the path to $outdir
# may vary. Nonetheless for debugging it's crucial that we echo out each
# command being executed -- here we use diff -I to skip the info lines and
# focus the test on program output per se.
diff -I '^mlr ' -C5 $expfile $outfile

# ================================================================
echo ALL REGRESSION TESTS PASSED
echo Tests completed: $num_passed
