Made more informative

Output to setup log
Easier to read main log function
Add LED flashes for stage setup / failure
Move all script downloads to frontend setup script

headless-scripts/Readme.md

@@ -10,12 +10,16 @@ For now the image creation work is at [rtgoodwin's fork of pi-gen](https://githu
 
 1. Flash the image
 1. Mount the card again, and in the `boot` directory create a `teslausb_setup_variables.conf` file to export the same environment varibles normally needed for setup (including archive, wifi, and push notifications if desired.)
-1. Run the `setup-piForHeadlessBuild.sh` (note: **not** `setup-piForHeadlessSetup.sh`)
+1. Run the `setup-piForHeadlessBuild.sh` (note: **not** `setup-piForHeadlessSetup.sh`):
+`curl https://raw.githubusercontent.com/rtgoodwin/teslausb/headless-patch/headless-scripts/setup-piForHeadlessBuild.sh -o setup-piForHeadlessBuild.sh`
+`chmod +x setup-piForHeadlessBuild.sh`
+`./setup-piForHeadlessBuild.sh .`
 1. If all goes well, put card into Pi and boot. 
 
 * A `/boot/teslausb-headless-setup.log` file will be created and stages logged. This takes the place of the "STOP" commands 
 * Marker files will be created in `boot` like `TESLA_USB_SETUP_STARTED` and `TESLA_USB_SETUP_FINISHED` to track progress. May use a progress system so the script can pick back up if needed. (This is probably useful for the general/old way of setup too.)
-
+* The Pi LED will flash patterns as it gets to each stage (labeled in the setup-teslausb-headless script). 
+  * 10 flashes means setup failed!
 
 
 #### Modifications to pi-gen builder from master
@@ -31,6 +35,7 @@ Built image on a Raspi running Stretch, for maximum Pi-ception.
 1. Add a file called `series` in the patches directory with the name of each `.diff` file in the order you want them applied.
 1. Add a `files` folder in stage6 with modified `rc.local`. The modified `rc.local` will handle pulling down the `setup-teslausb-headless` file the first time. (Still working on build logic here.) Files are moved into final locations in a `00-run.sh` script and the `install` command. See the script for details. 
 1. (Yes at this point you could suggest that just putting the end state files in place instead of patching would be good, but why not be idiomatic? :)  )
+1. Add a script to flash LEDs
 1. Run `sudo ./build.sh` from the `pi-gen` directory.
 1. If you get a failure, it's almost certainly after stage2, so you can add SKIP files in stage2-stage5 present) and rerun `sudo CLEAN=1 ./build.sh`
 

headless-scripts/setup-piForHeadlessBuild.sh

@@ -21,6 +21,9 @@ BOOT_DIR="$1"
 RED='\033[0;31m' # Red for warning
 NC='\033[0m' # No Color
 GREEN='\033[0;32m'
+scripts_downloaded=false
+REPO=rtgoodwin
+BRANCH=headless-patch
 
 function stop_message () {
   echo -e "${RED}${1} ${NC}"
@@ -56,6 +59,17 @@ function verify_file_exists () {
   fi
 }
 
+function verify_setup_file_exists () {
+  local file_name="$1"
+  local expected_path="$2"
+
+  if [ ! -e "$expected_path/$file_name" ]
+    then
+      stop_message "STOP: Didn't find setup script $file_name at $expected_path. Try running the setup script again."
+      exit 1
+  fi
+}
+
 function verify_wifi_variables () {
   if [ ! -n "${SSID+x}" ] || [ ! -n "${WIFIPASS+x}"  ]
   then
@@ -112,6 +126,58 @@ function verify_pushover_variables () {
   fi
 }
 
+function download_scripts () {
+  mkdir "${BOOT_DIR}/teslausb_script_scripts"
+  if [ ! -d "${BOOT_DIR}/teslausb_script_scripts" ]
+  then
+     stop_message "ERROR: Failed to make teslausb_setup_scripts and download setup scripts. Ensure you have internet access and run this script again."
+  else
+    pushd "${BOOT_DIR}/teslausb_script_scripts"
+    
+    wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/headless-patch/setup-teslausb-headless -O setup-teslausb-headless
+    verify_setup_file_exists "setup-teslausb-headless" "${BOOT_DIR}/teslausb_script_scripts"
+    chmod +x setup-teslausb-headless
+    good_message "Downloaded main setup script."
+    
+    wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/archiveloop -O archiveloop
+    # sed s/ARCHIVE_HOST_NAME=archiveserver/ARCHIVE_HOST_NAME=$archiveserver/ ~/archiveloop > /root/bin/archiveloop
+    sed -i'.bak' -e "s/ARCHIVE_HOST_NAME=archiveserver/ARCHIVE_HOST_NAME=$archiveserver/" archiveloop
+    verify_setup_file_exists "archiveloop" "${BOOT_DIR}/teslausb_script_scripts"
+    chmod +x archiveloop
+
+    wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/archive-teslacam-clips -O archive-teslacam-clips
+    verify_setup_file_exists "archive-teslacam-clips" "${BOOT_DIR}/teslausb_script_scripts"
+    chmod +x archive-teslacam-clips
+    good_message "Configured the archive scripts."
+
+    wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/remountfs_rw -O remountfs_rw
+    verify_setup_file_exists "remountfs_rw" "${BOOT_DIR}/teslausb_script_scripts"
+    chmod +x remountfs_rw
+    good_message "Downloaded script to remount filesystems read/write if needed (/root/bin/remountfs_rw)."
+
+    if [ ${user_enabled_pushover} = "true" ]
+    then
+      wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/send-pushover
+      verify_setup_file_exists "remountfs_rw" "${BOOT_DIR}/teslausb_script_scripts"
+      chmod +x send-pushover
+      good_message "Downloaded Pushover notification script."
+    fi
+
+    good_message "Downloading ancillary setup scripts."
+    wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/create-backingfiles-partition.sh -O create-backingfiles-partition.sh
+    verify_setup_file_exists "create-backingfiles-partition.sh" "${BOOT_DIR}/teslausb_script_scripts"
+    chmod +x create-backingfiles-partition.sh
+    wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/create-backingfiles.sh -O create-backingfiles.sh
+    verify_setup_file_exists "create-backingfiles.sh" "${BOOT_DIR}/teslausb_script_scripts"
+    chmod +x create-backingfiles.sh
+    wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/make-root-fs-readonly.sh -O make-root-fs-readonly.sh
+    verify_setup_file_exists "make-root-fs-readonly.sh" "${BOOT_DIR}/teslausb_script_scripts"
+    chmod +x make-root-fs-readonly.sh
+    popd
+    good_message "All scripts downloaded and configured."
+  fi
+
+}
 
 verify_file_exists "cmdline.txt" "$BOOT_DIR"
 verify_file_exists "config.txt" "$BOOT_DIR"
@@ -171,9 +237,16 @@ network={
 }
 EOF
 
+good_message "Downloading setup scripts. They will be downloaded to ${BOOT_DIR}/teslausb_setup_scripts,"
+good_message "and moved to /root/teslausb_script_scripts during first boot and install."
+
+download_scripts
+
 echo ""
 good_message '-- Files updated and ready for headless setup --'
 echo ''
 echo 'You can now insert your SD card into the Pi for headless setup. Plug in power to the Pi and it will boot and run.'
-echo "When done (this may take a vew minutes), the Pi should be available over SSH as pi@teslausb.local"
+echo "When done (this may take a vew minutes), the Pi should be available over SSH as pi@teslausb.local."
+echo "It's recommended you have your Pi plugged into a PC USB port for power, and connected to the port labeled USB on the Pi."
+echo "That way, when setup is complete, you should see your CAM and/or MUSIC drives appear as confirmation."
 echo ""

headless-scripts/setup-teslausb-headless

@@ -5,6 +5,7 @@ BRANCH=master
 user_enabled_pushover=false
 LOGFILE=/boot/teslausb-headless-setup.log
 setup_complete=false
+stage_in_progress=false
 
 # if ! [ $(id -u) = 0 ]
 # then
@@ -42,7 +43,8 @@ function check_variable () {
   if [ -z "${!var_name+x}" ]
   then
     echo "STOP: Define the variable $var_name like this: export $var_name=value"
-    setup_log "SETUP FAILED: Define the variable $var_name as export $var_name=value in /boot/teslausb_setup_variables"
+    setup_log "SETUP FAILED: Define the variable $var_name as export $var_name=value in /boot/teslausb_setup_variables.conf"
+    /tmp/stage_flash 10 &>/dev/null
     exit 1
   fi
 }
@@ -50,73 +52,60 @@ function check_variable () {
 function check_pushover_enabled () {
   if [ ! -z "${pushover_enabled+x}" ]
   then
-    if [ ! -n "${pushover_user_key+x}" ] || [ ! -n "${pushover_app_key+x}"  ]
+      setup_log "Adding Pushover variables into /root/.teslaCamPushoverCredentials"
-    then
-      setup_log "ERROR: "
-      echo "STOP: You're trying to setup Pushover but didn't provide your User and/or App key."
-      echo "Define the variables like this:"
-      echo "export pushover_user_key=put_your_userkey_here"
-      echo "export pushover_app_key=put_your_appkey_here"
-      exit 1
-    elif [ "${pushover_user_key}" = "put_your_userkey_here" ] || [  "${pushover_app_key}" = "put_your_appkey_here" ]
-    then
-      echo "STOP: You're trying to setup Pushover, but didn't replace the default User and App key values."
-      exit 1
-    else
-      user_enabled_pushover=true
       echo "export pushover_enabled=true" > /root/.teslaCamPushoverCredentials
       echo "export pushover_user_key=$pushover_user_key" >> /root/.teslaCamPushoverCredentials
       echo "export pushover_app_key=$pushover_app_key" >> /root/.teslaCamPushoverCredentials
     fi
-  fi
 }
 
 function check_archive_server_reachable () {
-  echo "Verifying that the archive server $archiveserver is reachable..."
+  setup_log "Verifying that the archive server $archiveserver is reachable..."
   local serverunreachable=false
   ping -c 1 -w 1 "$archiveserver" 1>/dev/null 2>&1 || serverunreachable=true
 
   if [ "$serverunreachable" = true ]
   then
-    echo "STOP: The archive server $archiveserver is unreachable. Try specifying its IP address instead."
+    setup_log "WARNING: The archive server $archiveserver is unreachable. Try specifying its IP address instead."
-    exit 1
+    setup_log "Continuing setup, but you may need to edit /etc/fstab to verify your archive mount entry is correct"
   fi
 
-  echo "The archive server is reachable."
+  # echo "The archive server is reachable."
 }
 
 function check_available_space () {
-  echo "Verifying that there is sufficient space available on the MicroSD card..."
+  setup_log "Verifying that there is sufficient space available on the MicroSD card..."
 
   local available_space="$( parted -m /dev/mmcblk0 u b print free | tail -1 | cut -d ":" -f 4 | sed 's/B//g' )"
 
   if [ "$available_space" -lt  4294967296 ]
   then
-    echo "STOP: The MicroSD card is too small."
+    setup_log "ERROR: The MicroSD card is too small. Stopping setup."
+    /tmp/stage_flash 10 &>/dev/null
     exit 1
   fi
 
-  echo "There is sufficient space available."
+  setup_log "There is sufficient space available."
 }
 
-function get_ancillary_setup_scripts () {
+# function get_ancillary_setup_scripts () {
-  pushd /tmp
+#   pushd /tmp
-  wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/create-backingfiles-partition.sh
+#   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/create-backingfiles-partition.sh
-  chmod +x ./create-backingfiles-partition.sh
+#   chmod +x ./create-backingfiles-partition.sh
-  wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/create-backingfiles.sh
+#   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/create-backingfiles.sh
-  chmod +x ./create-backingfiles.sh
+#   chmod +x ./create-backingfiles.sh
-  wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/make-root-fs-readonly.sh
+#   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/make-root-fs-readonly.sh
-  chmod +x ./make-root-fs-readonly.sh
+#   chmod +x ./make-root-fs-readonly.sh
-  popd
+#   popd
-}
+# }
 
 function fix_cmdline_txt_modules_load ()
 {
-  echo "Fixing the modules-load parameter in /boot/cmdline.txt..."
+  setup_log "Fixing the modules-load parameter in /boot/cmdline.txt..."
   cp /boot/cmdline.txt ~
   cat ~/cmdline.txt | sed 's/[[:space:]]\+modules-load=[^ [:space:]]\+//' | sed 's/rootwait/rootwait modules-load=dwc2/' > /boot/cmdline.txt
   rm ~/cmdline.txt
-  echo "Fixed cmdline.txt."
+  setup_log "Fixed cmdline.txt."
 }
 
 BACKINGFILES_MOUNTPOINT=/backingfiles
@@ -125,61 +114,61 @@ function create_usb_drive_backing_files () {
   mkdir "$BACKINGFILES_MOUNTPOINT"
   /tmp/create-backingfiles-partition.sh "$BACKINGFILES_MOUNTPOINT"
 
-  echo "Mounting the partition for the backing files..."
+  setup_log "Mounting the partition for the backing files..."
   mount /backingfiles
-  echo "Mounted the partition for the backing files."
+  setup_log "Mounted the partition for the backing files."
 
   /tmp/create-backingfiles.sh "$campercent" "$BACKINGFILES_MOUNTPOINT"
 }
 
 function configure_archive () {
-  echo "Configuring the archive..."
+  setup_log "Configuring the archive..."
   mkdir /mnt/archive
   local archive_server_ip_address="$(ping -c 1 -w 1 $archiveserver 2>/dev/null | head -n 1 | grep -o -e "(\([[:digit:]]\{1,3\}\.\)\{3\}[[:digit:]]\{1,3\})" | tr -d '()')"
   echo "//$archive_server_ip_address/$sharename /mnt/archive cifs vers=3,credentials=/root/.teslaCamArchiveCredentials,iocharset=utf8,file_mode=0777,dir_mode=0777 0" >> /etc/fstab
 
   echo "username=$shareuser" > /root/.teslaCamArchiveCredentials
   echo "password=$sharepassword" >> /root/.teslaCamArchiveCredentials
-  echo "Configured the archive."
+  setup_log "Configured the archive."
 }
 
-function configure_archive_scripts () {
+# function configure_archive_scripts () {
-  echo "Configuring the archive scripts..."
+#   echo "Configuring the archive scripts..."
-  mkdir /root/bin
+#   mkdir /root/bin
 
-  pushd ~
+#   pushd ~
-  wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/archiveloop
+#   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/archiveloop
-  sed s/ARCHIVE_HOST_NAME=archiveserver/ARCHIVE_HOST_NAME=$archiveserver/ ~/archiveloop > /root/bin/archiveloop
+#   sed s/ARCHIVE_HOST_NAME=archiveserver/ARCHIVE_HOST_NAME=$archiveserver/ ~/archiveloop > /root/bin/archiveloop
-  rm ~/archiveloop
+#   rm ~/archiveloop
-  chmod +x /root/bin/archiveloop
+#   chmod +x /root/bin/archiveloop
-  popd
+#   popd
 
-  pushd /root/bin
+#   pushd /root/bin
-  wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/archive-teslacam-clips
+#   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/archive-teslacam-clips
-  chmod +x archive-teslacam-clips
+#   chmod +x archive-teslacam-clips
-  popd
+#   popd
-  echo "Configured the archive scripts."
+#   echo "Configured the archive scripts."
 
-  pushd /root
+#   pushd /root
-  wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/remountfs_rw
+#   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/remountfs_rw
-  chmod +x remountfs_rw
+#   chmod +x remountfs_rw
-  popd
+#   popd
-  echo "Downloaded script to remount filesystems read/write if needed (/root/remountfs_rw)."
+#   echo "Downloaded script to remount filesystems read/write if needed (/root/remountfs_rw)."
-}
+# }
 
 
-function configure_pushover_scripts() {
+# function configure_pushover_scripts() {
-if [ ${user_enabled_pushover} = "true" ]
+# if [ ${user_enabled_pushover} = "true" ]
-then
+# then
-  pushd /root/bin
+#   pushd /root/bin
-  wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/send-pushover
+#   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/send-pushover
-  chmod +x send-pushover
+#   chmod +x send-pushover
-  popd
+#   popd
-fi
+# fi
-}
+# }
 
 function configure_rc_local () {
-  echo "Configuring /etc/rc.local to run the archive scripts at startup..."
+  setup_log "Configuring /etc/rc.local to run the archive scripts at startup..."
   echo "#!/bin/bash -eu" > ~/rc.local
   tail -n +2 /etc/rc.local | sed '$d' >> ~/rc.local
 cat << 'EOF' >> ~/rc.local
@@ -198,11 +187,11 @@ EOF
 
   cat ~/rc.local > /etc/rc.local
   rm ~/rc.local
-  echo "Configured rc.local."
+  setup_log "Configured rc.local."
 }
 
 function configure_hostname () {
-  echo "Configuring the hostname..."
+  setup_log "Configuring the hostname..."
 
   local new_host_name="teslausb"
   cp /etc/hosts ~
@@ -210,22 +199,35 @@ function configure_hostname () {
 
   cp /etc/hostname ~
   sed "s/raspberrypi/$new_host_name/g" ~/hostname > /etc/hostname
-  echo "Configured the hostname."
+  setup_log "Configured the hostname."
 }
 
 function make_root_fs_readonly () {
   /tmp/make-root-fs-readonly.sh
 }
 
+# TURN OFF LED so we can start watching progress
+echo 1 | sudo tee /sys/class/leds/led0/brightness
+
+sleep 5
+
+# SETUP STAGE 1 - Check variables file
+
+/tmp/stage_flash 1 &>/dev/null
+setup_log "SETUP STAGE 1: Check variables file."
 if [ ! -e /boot/teslausb_setup_variables.conf ]
 then
   setup_log "ERROR: teslausb_setup_variables.conf file not found. Can't continue setup."
+  /tmp/stage_flash 10 &>/dev/null
   exit 1
 fi
 
 source /boot/teslausb_setup_variables.conf
 
-echo "Verifying environment variables..."
+# SETUP STAGE 2 - Validate variables. This should never fail but just in case.
+/tmp/stage_flash 2 &>/dev/null
+
+setup_log "SETUP STAGE 2: Verifying environment variables..."
 
 check_variable "archiveserver"
 check_variable "sharename"
@@ -237,20 +239,28 @@ check_pushover_enabled
 
 check_archive_server_reachable
 
+# SETUP STAGE 3 
+
+setup_log "SETUP STAGE 3: Check available space."
+/tmp/stage_flash 3 &>/dev/null
 check_available_space
 
-get_ancillary_setup_scripts
+# get_ancillary_setup_scripts
 
-pushd ~
+# pushd ~
 
-configure_archive_scripts
+# configure_archive_scripts
 
-configure_pushover_scripts
+# configure_pushover_scripts
 
 fix_cmdline_txt_modules_load
 
 echo "" >> /etc/fstab
 
+# SETUP STAGE 4 
+setup_log "SETUP STAGE 4: Create backing files and final config changes."
+/tmp/stage_flash 4 &>/dev/null
+
 create_usb_drive_backing_files
 
 configure_archive
@@ -259,6 +269,10 @@ configure_rc_local
 
 configure_hostname
 
+# SETUP STAGE 5 and reboot
+# If you see 5 flashes we are probably good!
+
+/tmp/stage_flash 5 &>/dev/null
 mark_setup_success
 
 make_root_fs_readonly

windows_archive/archive-teslacam-clips

@@ -5,8 +5,7 @@ CAM_MOUNT=/mnt/cam
 ARCHIVE_MOUNT=/mnt/archive
 
 function log () {
-  echo "$( date )" >> "$LOG_FILE"
+  echo "$( date ) : $1" >> "$LOG_FILE"
-  echo "$1" >> "$LOG_FILE"
 }
 
 function retry () {

windows_archive/archiveloop

@@ -5,8 +5,7 @@ ARCHIVE_HOST_NAME=archiveserver
 LOGFILE=/tmp/archiveloop.log
 
 function log () {
-  echo "$( date )" >> "$LOGFILE"
+  echo "$( date ) : $1" >> "$LOG_FILE"
-  echo "$1" >> "$LOGFILE"
 }
 
 function archive_is_reachable () {

windows_archive/send-pushover

@@ -1,14 +1,18 @@
 #!/bin/bash -eu
 
+LOGFILE=/tmp/archiveloop.log
+
 function log () {
-  echo "$( date )" >> "$LOG_FILE"
+  echo "$( date ) : $1" >> "$LOG_FILE"
-  echo "$1" >> "$LOG_FILE"
 }
 
 source /root/.teslaCamPushoverCredentials
 
+if ping -c 1 api.pushover.net &> /dev/null
+then
   curl -F "token=$pushover_app_key" \
   -F "user=$pushover_user_key" \
   -F "title=Dashcam Copy Complete" \
   -F "message=$1 file(s) were copied." \
   https://api.pushover.net/1/messages
+fi