#!/bin/sh
# $FreeBSD: ports/Tools/portbuild/scripts/dopackagestats,v 1.26 2010/02/19 16:15:58 linimon Exp $
#
# create HTML showing numbers of packages vs errors.  Run this in a directory
# accessible to the web server.
#

# alpha is obsolete
SUPPORTED_ARCHS="amd64 i386 ia64 powerpc sparc64"
ROOT_DIRECTORY=/var/portbuild

OUTFILE=`basename $0 | sed -e "s/^do//"`".html"
TMPFILE=.${OUTFILE}

# stylesheet seems like overkill for something this simple
TABLEBGCOLOR="#F0F0F0"
THCOLOR="#E0E0FF"
TDCOLOR_DONE="lightgreen"
TDCOLOR_NOT_DONE="lightyellow"

# subroutines

write_header () {
  echo "<html>" > ${TMPFILE}
  echo "<head>" >> ${TMPFILE}
  echo "<title>FreeBSD package building statistics</title>" >> ${TMPFILE}
  echo "</head>" >> ${TMPFILE}

  echo "<body>" >> ${TMPFILE}
  echo "<h1>FreeBSD package building statistics</h1>" >> ${TMPFILE}
  echo "<p>as of `date`</p>" >> ${TMPFILE}
}

write_table_begin () {
  echo "<table border='1' cellpadding='4' cellspacing='1' bgcolor='$TABLEBGCOLOR'>" >> ${TMPFILE}
  echo "<tr>" >> ${TMPFILE}
  echo "<td align='left' width='120' bgcolor='$TABLEBGCOLOR'>&nbsp;</td>" >> ${TMPFILE}
  echo "<th width='60' bgcolor='$THCOLOR'>cvs date</th>" >> ${TMPFILE}
  # MCL removed 20090808 -- this takes way too long
  # echo "<th width='60' bgcolor='$THCOLOR'>latest log</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>INDEX</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>build logs</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>packages</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>errors</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>skipped</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>not yet built</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>queue length</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>running?</th>" >> ${TMPFILE}
  echo "<th bgcolor='$THCOLOR'>completed?</th>" >> ${TMPFILE}
  echo "</tr>" >> ${TMPFILE}
}

write_row () {
    # first, gather data

    arch=$1
    build=$2
    # MCL 20080916
    directory=${ROOT_DIRECTORY}/${arch}/${build}/builds/latest

    branch=`echo $build | sed -e "s/-exp//"`
    if [ "$branch" = "4" ]; then
      indexfile=$directory/ports/INDEX
    else
      indexfile=$directory/ports/INDEX-$branch
    fi
    # work around the fact that 5-exp is really 6-exp-prime
    if [ ! -f $indexfile ]; then
      if [ -d $directory/ports ]; then
          indexfile=$directory/ports/`cd $directory/ports 2> /dev/null && ls INDEX* 2> /dev/null | head -1`
        else
          # work around the fact that 4 is EOL and thus has no ports/ directory
         indexfile=$directory/logs/`cd $directory/logs 2> /dev/null && ls INDEX* 2> /dev/null | head -1`
        fi
    fi

    # column: date of CVS checkout
    have_cvsdone=""
    cvsdone=""
    if [ -f $directory/cvsdone ]; then
      cvsdone="$(cat $directory/cvsdone | awk '{printf("%s %s\n",$2,$3)}')"
      if [ ! -z "$cvsdone" ]; then
        have_cvsdone="yes"
      fi
    fi

    # column: datestamp and URL of latest log
    have_latest=""
    latest=""
# MCL removed 20090808 -- this takes way too long
#    if [ -d $directory/logs ]; then
#      latest_suffix="$(cd $directory/logs 2> /dev/null && ls -rtTl | grep '\.log' | tail -1 | awk '{printf("%s\">%s %s</a>\n",$10,$6,$7)}')"
#      if [ -z "$latest_suffix" ]; then
#        latest="<a href=\"http://pointyhat.freebsd.org/errorlogs/$arch-$build-latest-logs/$latest_suffix"
#        have_latest="yes"
#      fi
#    fi

    # column: INDEX count
    have_index=""
    n_index=0
    if [ -f $indexfile ]; then
      n_index=`cat $indexfile | wc -l`
      have_index="yes"
    fi

    # column: buildlog count
    have_logs=""
    n_logs=0
    if [ -d $directory/logs ]; then
      n_logs=`ls $directory/logs | grep '\.log' | wc -l`
      have_logs="yes"
    fi

    # column: package count
    have_packages=""
    n_packages=0
    if [ -d $directory/packages/All ]; then
      # MCL removed 20090808 -- this takes way too long
      # n_packages=`find $directory/packages/All -name \*.tbz -or -name \*.tgz |wc -l`
      n_packages=`ls $directory/packages/All | grep -v CHECKSUM.MD5 | wc -l`
      have_packages="yes"
    fi

    # column: error count
    have_errors=""
    n_errors=0
    if [ -d $directory/errors ]; then
      n_errors=`ls $directory/errors | grep '\.log' | wc -l`
      have_errors="yes"
    fi

    # column: duds count
    have_duds=""
    n_duds=0
    if [ -f $directory/duds ]; then
      n_duds=`cat $directory/duds | wc -l`
      have_duds="yes"
    fi

    # if do not have any files, skip the row
    if [ -z "$have_cvsdone" -a \
         -z "$have_latest" -a \
         -z "$have_index" -a \
         -z "$have_logs" -a \
         -z "$have_packages" -a \
         -z "$have_errors" -a \
         -z "$have_duds" ]; then
      return
    fi

    # column: not yet built count
    # MCL 20080916 use n_packages, not n_logs; individual logs can be stale.
    # (OTOH, so can packages (see sparc64-5) but this is possibly obsolete)
    have_not_yet_built=""
    n_not_yet_built=0
    if [ ! -z "$have_index" -a \
         ! -z "$have_packages" -a \
         ! -z "$have_errors" -a \
         ! -z "$have_duds" ]; then
      if [ $n_index -ne 0 ]; then
        n_not_yet_built=`expr $n_index - $n_packages - $n_errors - $n_duds`
        have_not_yet_built="yes"
      # else index currently being rebuilt
      fi
    fi

    # column: running flag
    running_flag="N"
    # the last grep eliminates false positive of i386-6-exp for i386-6;
    # if we are still running FreeBSD in the year 3000, s/2/3/
    running_processes_for_build=`ps axww | \
      grep "pdispatch $arch $build " | \
      grep -v grep | \
      sed -e "s@.*pdispatch @@;s@ /var/portbuild/scripts/.*@@;s@ @-@g" | \
      grep "^$arch-$build-2"`
    if [ ! -z "$running_processes_for_build" ]; then
      running_flag="Y"
    fi

    # column: completed flag
    completed_flag="N"
    if [ -f $directory/build.log ]; then
      if [ ! -z "`grep 'all done at ' $directory/build.log`" ]; then
        completed_flag="Y"
      fi
    fi

    # decorate the row to make everything less "gray"
    if [ "$running_flag" = "N" -a "$completed_flag" = "Y" ]; then
      cellcolor=$TDCOLOR_DONE
    else
      cellcolor=$TDCOLOR_NOT_DONE
    fi

    # queue length -PAV-
    m_not_yet_built=""
    queue_length=""
    if [ "$completed_flag" = "N" ]; then
      m_not_yet_built=`tail -n 1000 /var/portbuild/$arch/$build/builds/latest/make|grep MASTER|grep Queue|tail -1|sed 's|.*remaining=|| ; s|, Queue.*||'`
      queue_length=`tail -n 1000 /var/portbuild/$arch/$build/builds/latest/make|grep MASTER|grep Queue|tail -1|sed 's|.*length=||'`
    fi;

    # now write the row
    echo "<tr>" >> ${TMPFILE}

    echo "<th align='left' bgcolor='$THCOLOR'>$arch-$build</th>" >> ${TMPFILE}

    echo "<td align='left' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$have_cvsdone" ]; then
      echo "<a href='http://pointyhat.freebsd.org/errorlogs/$arch-$build-latest/cvsdone'>" >> ${TMPFILE}
      echo "$cvsdone</a>" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

# MCL removed 20090808 -- this takes way too long
#    echo "<td align='left' bgcolor='$cellcolor'>" >> ${TMPFILE}
#    if [ ! -z "$have_latest" ]; then
#      echo "$latest" >> ${TMPFILE}
#    else
#      echo "&nbsp;" >> ${TMPFILE}
#    fi
#    echo "</td>" >> ${TMPFILE}

    # note: ports/INDEX-n is copied to a file called errorlogs/INDEX
    echo "<td align='left' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$have_index" ]; then
      echo "<a href='http://pointyhat.freebsd.org/errorlogs/$arch-$build-latest/INDEX'>" >> ${TMPFILE}
      echo "$n_index</a>" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

    echo "<td align='right' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$have_logs" ]; then
      echo "<a href='http://pointyhat.freebsd.org/errorlogs/$arch-$build-latest-logs'>" >> ${TMPFILE}
      echo "$n_logs</a>" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

    echo "<td align='right' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$have_packages" ]; then
      echo "<a href='http://pointyhat.freebsd.org/errorlogs/$arch-$build-packages-latest/All'>" >> ${TMPFILE}
      echo "$n_packages</a>" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

    echo "<td align='right' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$have_errors" ]; then
      echo "<a href='http://pointyhat.freebsd.org/errorlogs/$arch-$build-latest'>" >> ${TMPFILE}
      echo "$n_errors</a>" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

    echo "<td align='right' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$have_duds" ]; then
      echo "<a href='http://pointyhat.freebsd.org/errorlogs/$arch-$build-latest/duds.verbose'>" >> ${TMPFILE}
      echo "$n_duds</a>" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

    echo "<td align='right' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$m_not_yet_built" ]; then
      echo "$n_not_yet_built" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

    echo "<td align='right' bgcolor='$cellcolor'>" >> ${TMPFILE}
    if [ ! -z "$queue_length" ]; then
      echo "$queue_length" >> ${TMPFILE}
    else
      echo "&nbsp;" >> ${TMPFILE}
    fi
    echo "</td>" >> ${TMPFILE}

    echo "<td align='center' bgcolor='$cellcolor'>$running_flag</td>" >> ${TMPFILE}
    echo "<td align='center' bgcolor='$cellcolor'>$completed_flag</td>" >> ${TMPFILE}

    echo "</tr>" >> ${TMPFILE}
}

write_table_end () {
  echo "</table>" >> ${TMPFILE}
  echo "<br>" >> ${TMPFILE}
}

write_footer () {
  echo "<p>explanation of columns:</p>" >> ${TMPFILE}
  echo "<ul>" >> ${TMPFILE}
  # MCL removed 20090808 -- this takes way too long
  #echo "<li><b>latest log</b> is the date of the latest logfile.</li>" >> ${TMPFILE}
  echo "<li><b>cvs date</b> is the date of the latest CVS checkout done by the script.  It may be inaccurate if a manual checkout was done later.</li>" >> ${TMPFILE}
  echo "<li><b>INDEX</b> is number of ports in the INDEX file built from the latest cvs checkout.</li>" >> ${TMPFILE}
  echo "<li><b>build logs</b> is number of packages attempted.  Note: if a run was restarted, you may see duplicates here.</li>" >> ${TMPFILE}
  echo "<li><b>packages</b> is number of packages successfully built.  Note: if a run was restarted, you may see duplicates here.</li>" >> ${TMPFILE}
  echo "<li><b>errors</b> is number of packages that failed.  Note: if a run was restarted, you may see duplicates here.</li>" >> ${TMPFILE}
  echo "<li><b>skipped</b> is number of packages that were skipped due to NO_PACKAGE, IGNORE, BROKEN, FORBIDDEN, and so forth (\"duds\" file).</li>" >> ${TMPFILE}
  echo "<li><b>not yet built</b> is the INDEX column minus the build logs plus the errors plus the skipped.  These are packages that have not been built for one reason or another.  Note: interrupted and/or restarted builds can make this number inaccurate because of the duplicates, above.</li>" >> ${TMPFILE}
  echo "<li><b>running</b> is whether there are still processes running.</li>" >> ${TMPFILE}
  echo "<li><b>completed</b> is whether that build terminated normally or not, as seen from the logfile.</li>" >> ${TMPFILE}
  echo "</ul>" >> ${TMPFILE}

#  no longer true 20080917
#  echo "<p>notes:</p>" >> ${TMPFILE}
#  echo "<ul>" >> ${TMPFILE}
#  echo "<li>on the -exp builds, editors/openoffice.org* are skipped to save time.</li>" >> ${TMPFILE}
#  echo "</ul>" >> ${TMPFILE}

  echo "</body>" >> ${TMPFILE}
  echo "</html>" >> ${TMPFILE}
}

# main

write_header

# display all the mainstream builds first
for arch in ${SUPPORTED_ARCHS}; do

  # drop 4 and 5 as obsolete
  builds=`ls ${ROOT_DIRECTORY}/${arch} | grep '^[67891]$' | sort`
  if [ ! -z "$builds" ]; then
    write_table_begin

      for build in ${builds}; do
        write_row ${arch} ${build}
      done

    write_table_end
  fi
done

# then display all -exp builds (probably only of interest to portmgr;
#   would break up the logical flow of the above)
for arch in ${SUPPORTED_ARCHS}; do

  builds=`ls ${ROOT_DIRECTORY}/${arch} | grep '^[67891]-exp$' | sort`
  if [ ! -z "$builds" ]; then
    write_table_begin

      for build in ${builds}; do
        write_row ${arch} ${build}
      done

    write_table_end
  fi
done

write_footer

mv -f ${TMPFILE} ${OUTFILE}
