Merge branch 'master' into u/gocnak/rsync_archive

# Conflicts:
#	windows_archive/setup-teslausb

GetShellWithoutMonitorOnLinux.md

@@ -20,7 +20,7 @@ If you prefer not to run the script, it's also a useful reference for the steps
 1. Run the following commands:
     ```
     wget https://raw.githubusercontent.com/cimryan/teslausb/master/mac_linux_archive/setup-piForHeadlessConfig.sh
-    chmod +x update-rpi-mac-linux.sh
+    chmod +x setup-piForHeadlessConfig.sh
     ```
 1. Set your SSID (Wifi network name) and WIFIPASS environment variables. The script will insert them into the `wpa_supplicant.conf` when creating it:
 
@@ -29,7 +29,7 @@ If you prefer not to run the script, it's also a useful reference for the steps
     export WIFIPASS=your_wifi_password_here
     ```
 1. Run the script: 
-    `./update-rpi-mac-linux.sh`
+    `./setup-piForHeadlessConfig.sh`
 1. If all goes well, the script will report: 
     `-- Files updated and ready for Wifi and SSH over USB --`
 1. Eject the SD card safely, insert into your Pi, and reboot. If the Pi is connected over USB to your host, and/or if the Wifi setup went correctly, you should be able to `ssh pi@raspberrypi.local`. The default password is `raspberry`. 

README.md

@@ -61,11 +61,10 @@ Since sftp/rsync is accessing a computer through SSH, the only requirement for h
 ### ***TODO: Other hosting solutions***
 
 ## Set up the Raspberry Pi
-There are four phases to setting up the Pi:
+There are three phases to setting up the Pi:
 1. Get the OS onto the micro sd card.
 1. Get a shell on the Pi.
 1. Set up the USB storage functionality.
-1. Get the Pi set up for your Tesla.
 
 ### Get the OS onto the micro SD card
 
@@ -76,13 +75,17 @@ There are four phases to setting up the Pi:
 
 ### Get a shell on the Pi
 If you used a Windows computer to flash the OS onto the MicroSD card, follow these [Instructions](GetShellWithoutMonitorOnWindows.md).
+
 If you used a Mac or a Linux computer, follow these [Instructions](GetShellWithoutMonitorOnLinux.md).
 
 ### Set up the USB storage functionality
 
 Now that you have Wifi up and running, it's time to set up the USB storage and scripts that will manage the dashcam and (optionally) music storage.
 
-1. SSH to the Pi and run `sudo -i`
+1. SSH to the Pi and run
+    ```
+    sudo -i
+    ```
 1. Try to ping your archive server from the Pi. In this example the server is named `nautilus`.
     ```
     ping -c 3 nautilus
@@ -114,30 +117,32 @@ Now that you have Wifi up and running, it's time to set up the USB storage and s
     ```
 1. Run this command:
     ```
-    reboot
+    halt
     ```
+1. Disconnect the Pi from the computer.
 
-After reboot, the Pi hostname will become `teslausb`, so future `ssh` sessions will be `ssh pi@teslausb.local`. 
+On the next boot, the Pi hostname will become `teslausb`, so future `ssh` sessions will be `ssh pi@teslausb.local`. 
 
-### Get the Pi set up for your Tesla.
+Your Pi is now ready to be plugged into your Tesla. If you want to add music to the Pi, follow the instructions in the next section.
-If you set up the Pi with a keyboard and a monitor disconnect it and connect it to a PC. If you're using a cable be sure to use the port labeled "USB" on the circuitboard. 
+
-1. Wait for the Pi to show up on the PC as a USB drive.
+## (Optional) Add music to the Pi
-1. Create a directory named TeslaCam at the root of the drive labeled CAM.
+Connect the Pi to a computer. If you're using a cable be sure to use the port labeled "USB" on the circuitboard. 
+1. Wait for the Pi to show up on the computer as a USB drive.
 1. Copy any music you'd like to the drive labeled MUSIC.
 1. Eject the drives.
 1. Unplug the Pi from the PC.
 1. Plug the Pi into your Tesla.
 
 ## Making changes to the system after setup
-The setup process configures the Pi with read-only file systems for the operating system but with read-write access through the USB
+The setup process configures the Pi with read-only file systems for the operating system but with read-write
- interface. This means that you'll be able to record dashcam video and add and remove music files but you won't be able to make changes 
+access through the USB interface. This means that you'll be able to record dashcam video and add and remove
- to files on / or on /boot. This is to protect against corruption of the operating system when the Tesla cuts power to the Pi.
+music files but you won't be able to make changes to files on / or on /boot. This is to protect against
+corruption of the operating system when the Tesla cuts power to the Pi.
 
 To make changes to the system partitions:
 ```
 ssh pi@teslausb.
 sudo -i
-mount / -o remount,rw
+/root/bin/remountfs_rw
-mount /boot -o remount,rw
 ```
 Then make whatever changes you need to. The next time the system boots the partitions will once again be read-only.

windows_archive/archive-teslacam-clips

@@ -1,7 +1,6 @@
 #!/bin/bash -eu
 
 LOG_FILE=/tmp/archive-teslacam-clips.log
-CAM_MOUNT=/mnt/cam
 ARCHIVE_MOUNT=/mnt/archive
 
 function log () {
@@ -9,66 +8,6 @@ function log () {
   echo "$1" >> "$LOG_FILE"
 }
 
-function retry () {
-  local attempts=0
-  while [ true ]
-  do
-    if eval "$@"
-    then
-      true
-      return
-    fi
-    if [ "$attempts" -ge 10 ]
-    then
-      log "Attempts exhausted."
-      false
-      return
-    fi
-    log "Sleeping before retry..."
-    /bin/sleep 1
-    attempts=$((attempts + 1))
-    log "Retrying..."
-  done
-  false
-  return
-}
-
-function mount_mountpoint () {
-  local mount_point="$1"
-  log "Mounting $mount_point..."
-
-  local mounted=true
-  mount "$mount_point" >> "$LOG_FILE" 2>&1 || mounted=false
-  if [ "$mounted" = true ]
-  then
-    log "Mounted $mount_point."
-    true
-    return
-  else
-    log "Failed to mount $mount_point."
-    false
-    return
-  fi
-}
-
-function ensure_mountpoint_is_mounted () {
-  local mount_point="$1"
-  local mount_exists=true
-
-  findmnt --mountpoint "$mount_point" > /dev/null || mount_exists=false
-
-  if [ "$mount_exists" = true ]
-  then
-    log "$mount_point is already mounted."
-  else
-    mount_mountpoint "$mount_point"
-  fi
-}
-
-function ensure_mountpoint_is_mounted_with_retry () {
-  retry ensure_mountpoint_is_mounted "$1"
-}
-
 function move_clips_to_archive () {
   log "Moving clips to archive..."
   local move_count=0
@@ -97,30 +36,12 @@ function disconnect_usb_drives_from_host () {
   log "Disconnected usb from host."
 }
 
-function fix_errors_on_cam_drive () {
-  log "Running fsck..."
-  /sbin/fsck "$CAM_MOUNT" -- -a >> "$LOG_FILE" 2>&1 || echo ""
-  log "Finished running fsck."
-}
-
 function ensure_archive_is_mounted () {
   log "Ensuring cam archive is mounted..."
   ensure_mountpoint_is_mounted_with_retry "$ARCHIVE_MOUNT"
   log "Ensured cam archive is mounted."
 }
 
-function ensure_cam_drive_is_mounted () {
-  log "Ensuring cam drive is mounted..."
-  ensure_mountpoint_is_mounted_with_retry "$CAM_MOUNT"
-  log "Ensured cam drive is mounted."
-}
-
-function unmount_cam_drive () {
-  log "Unmounting cam drive..."
-  umount "$CAM_MOUNT"
-  log "Unmounted cam drive."
-}
-
 log "Starting..."
 
 if [ ! -r "/root/.teslaCamRsyncConfig" ]
@@ -130,9 +51,9 @@ fi
 
 disconnect_usb_drives_from_host
 
-fix_errors_on_cam_drive
+ensure_cam_file_is_mounted
 
-ensure_cam_drive_is_mounted
+fix_errors_in_cam_file
 
 if [ -r "/root/.teslaCamRsyncConfig" ]
 then
@@ -142,6 +63,6 @@ else
     move_clips_to_archive
 fi
 
-unmount_cam_drive
+unmount_cam_file
 
-connect_usb_drives_to_host
+connect_usb_drives_to_host

windows_archive/archiveloop

@@ -2,11 +2,27 @@
 
 # Change the value on the right side of the equal sign to the name of the server hosting the archive.
 ARCHIVE_HOST_NAME=archiveserver
-LOGFILE=/tmp/archiveloop.log
+
+LOG_FILE=/tmp/archiveloop.log
+
+export CAM_MOUNT=/mnt/cam
+export MUSIC_MOUNT=/mnt/music
 
 function log () {
-  echo "$( date )" >> "$LOGFILE"
+  echo "$( date )" >> "$LOG_FILE"
-  echo "$1" >> "$LOGFILE"
+  echo "$1" >> "$LOG_FILE"
+}
+
+function fix_errors_in_mount_point () {
+  local mount_point="$1"
+  log "Running fsck on $mount_point..."
+  /sbin/fsck "$mount_point" -- -a >> "$LOG_FILE" 2>&1 || echo ""
+  log "Finished fsck on $mount_point."
+}
+
+function fix_errors_in_mounted_files () {
+  fix_errors_in_mount_point "$CAM_MOUNT"
+  fix_errors_in_mount_point "$MUSIC_MOUNT"
 }
 
 function archive_is_reachable () {
@@ -35,10 +51,111 @@ function wait_for_archive_to_be_reachable () {
       log "Archive is reachable."
       break
     fi
+    if [ -e /tmp/archive_is_reachable ]
+    then
+      log "Simulating archive is reachable"
+      rm /tmp/archive_is_reachable
+      break
+    fi
     sleep 1
   done
 }
 
+function retry () {
+  local attempts=0
+  while [ true ]
+  do
+    if eval "$@"
+    then
+      true
+      return
+    fi
+    if [ "$attempts" -ge 10 ]
+    then
+      log "Attempts exhausted."
+      false
+      return
+    fi
+    log "Sleeping before retry..."
+    /bin/sleep 1
+    attempts=$((attempts + 1))
+    log "Retrying..."
+  done
+  false
+  return
+}
+
+function mount_mountpoint () {
+  local mount_point="$1"
+  log "Mounting $mount_point..."
+
+  local mounted=true
+  mount "$mount_point" >> "$LOG_FILE" 2>&1 || mounted=false
+  if [ "$mounted" = true ]
+  then
+    log "Mounted $mount_point."
+    true
+    return
+  else
+    log "Failed to mount $mount_point."
+    false
+    return
+  fi
+}
+
+function ensure_mountpoint_is_mounted () {
+  local mount_point="$1"
+  local mount_exists=true
+
+  findmnt --mountpoint "$mount_point" > /dev/null || mount_exists=false
+
+  if [ "$mount_exists" = true ]
+  then
+    log "$mount_point is already mounted."
+  else
+    mount_mountpoint "$mount_point"
+  fi
+}
+
+function ensure_mountpoint_is_mounted_with_retry () {
+  retry ensure_mountpoint_is_mounted "$1"
+}
+
+function fix_errors_in_cam_file () {
+  fix_errors_in_mount_point "$CAM_MOUNT"
+}
+
+function ensure_cam_file_is_mounted () {
+  log "Ensuring cam file is mounted..."
+  ensure_mountpoint_is_mounted_with_retry "$CAM_MOUNT"
+  log "Ensured cam file is mounted."
+}
+
+function ensure_music_file_is_mounted () {
+  log "Ensuring music backing file is mounted..."
+  ensure_mountpoint_is_mounted_with_retry "$MUSIC_MOUNT"
+  log "Ensured cam drive is mounted."
+}
+
+function unmount_mount_point () {
+  local mount_point="$1"
+  log "Unmounting $mount_point..."
+  umount "$mount_point" >> "$LOG_FILE" 2>&1
+  log "Unmounted $mount_point."
+}
+
+function unmount_cam_file () {
+  unmount_mount_point "$CAM_MOUNT"
+}
+
+function unmount_music_file () {
+  unmount_mount_point "$MUSIC_MOUNT"
+}
+
+function fix_errors_in_music_file () {
+  fix_errors_in_mount_point "$MUSIC_MOUNT"
+}
+
 function archive_clips () {
   log "Archiving..."
   /root/bin/archive-teslacam-clips
@@ -54,20 +171,61 @@ function wait_for_archive_to_be_unreachable () {
         log "Archive is unreachable."
         break
       fi
+      if [ -e /tmp/archive_is_unreachable ]
+      then
+        log "Simulating archive being unreachable."
+        rm /tmp/archive_is_unreachable
+        break
+      fi
       sleep 1
   done
 }
 
+function mount_and_fix_errors_in_cam_file () {
+  ensure_cam_file_is_mounted
+  fix_errors_in_cam_file
+  unmount_cam_file
+}
+
+function mount_and_fix_errors_in_music_file () {
+  if [ -e "$MUSIC_MOUNT" ]
+  then
+    ensure_music_file_is_mounted
+    fix_errors_in_music_file
+    unmount_music_file
+  fi
+}
+
+function mount_and_fix_errors_in_files () {
+  mount_and_fix_errors_in_cam_file
+  mount_and_fix_errors_in_music_file
+}
+
+export -f fix_errors_in_mount_point
+export -f fix_errors_in_cam_file
+export -f retry
+export -f mount_mountpoint
+export -f ensure_mountpoint_is_mounted
+export -f ensure_mountpoint_is_mounted_with_retry
+export -f ensure_cam_file_is_mounted
+export -f fix_errors_in_cam_file
+export -f unmount_mount_point
+export -f unmount_cam_file
 export -f connect_usb_drives_to_host
 
 log "Starting..."
 
 if archive_is_reachable
 then
+  # archive_clips will fix errors in the cam file
+  mount_and_fix_errors_in_music_file
+
   archive_clips
 
   wait_for_archive_to_be_unreachable
 else
+  mount_and_fix_errors_in_files
+
   connect_usb_drives_to_host
 fi
 

windows_archive/create-backingfiles.sh

@@ -21,6 +21,12 @@ function add_drive () {
   echo "$filename $mountpoint vfat noauto,users,umask=000 0 0" >> /etc/fstab
 }
 
+function create_teslacam_directory () {
+  mount /mnt/cam
+  mkdir /mnt/cam/TeslaCam
+  umount /mnt/cam
+}
+
 FREE_1K_BLOCKS="$(df --output=avail --block-size=1K /backingfiles/ | tail -n 1)"
 
 CAM_DISK_SIZE="$(( $FREE_1K_BLOCKS * $CAM_PERCENT / 100 ))"
@@ -36,3 +42,5 @@ then
 else
   echo "options g_mass_storage file=$CAM_DISK_FILE_NAME removable=1 ro=0 stall=0 iSerialNumber=123456" > "$G_MASS_STORAGE_CONF_FILE_NAME"
 fi
+
+create_teslacam_directory

windows_archive/setup-piForHeadlessConfig.ps1

@@ -44,14 +44,18 @@ if ([System.IO.File]::Exists("$wpaSupplicantConfPath")) {
   del "$wpaSupplicantConfPath"
 }
 
-"ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev`
+$wpaSupplicantConfContent=@"
-update_config=1`
+ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev
-`
+update_config=1
-network={`
+
-  ssid=`"$wifiSSID`"`
+network={
-  psk=`"$wifiPSK`"`
+  ssid="$wifiSSID"
-  key_mgmt=WPA-PSK`
+  psk="$wifiPSK"
-}`
+}
-" | Out-File -FilePath "$wpaSupplicantConfPath" -Encoding utf8
+"@
+
+$utf8 = New-Object System.Text.UTF8Encoding $false
+
+Set-Content -Value $utf8.GetBytes($wpaSupplicantConfContent) -Encoding Byte -Path "$wpaSupplicantConfPath"
 
 Write-Verbose "All done."

windows_archive/setup-teslausb

@@ -1,8 +1,10 @@
 #!/bin/bash -eu
 
-REPO=cimryan
+USER_ENABLED_PUSHOVER=false
-BRANCH=master
+
-user_enabled_pushover=false
+REPO=${REPO:-cimryan}
+
+BRANCH=${BRANCH:-master}
 
 if ! [ $(id -u) = 0 ]
 then
@@ -34,7 +36,7 @@ function check_pushover_enabled () {
       echo "STOP: You're trying to setup Pushover, but didn't replace the default User and App key values."
       exit 1
     else
-      user_enabled_pushover=true
+      USER_ENABLED_PUSHOVER=true
       echo "export pushover_enabled=true" > /root/.teslaCamPushoverCredentials
       echo "export pushover_user_key=$pushover_user_key" >> /root/.teslaCamPushoverCredentials
       echo "export pushover_app_key=$pushover_app_key" >> /root/.teslaCamPushoverCredentials
@@ -56,6 +58,40 @@ function check_archive_server_reachable () {
   echo "The archive server is reachable."
 }
 
+function write_archive_credentials_to () {
+  local file_path="$1"
+  
+  echo "username=$shareuser" > "$file_path"
+  echo "password=$sharepassword" >> "$file_path"
+
+}
+
+function check_archive_mountable () {
+  local archive_server_ip_address="$1"
+  
+  local test_mount_location="/tmp/archivetestmount"
+  
+  if [ ! -e "$test_mount_location" ]
+  then
+    mkdir "$test_mount_location"
+  fi
+  
+  local tmp_credentials_file_path="/tmp/teslaCamArchiveCredentials"
+  
+  write_archive_credentials_to "$tmp_credentials_file_path"
+  
+  local mount_failed=false
+  mount -t cifs "//$archive_server_ip_address/$sharename" "$test_mount_location" -o "vers=${cifs_version},credentials=${tmp_credentials_file_path},iocharset=utf8,file_mode=0777,dir_mode=0777" || mount_failed=true
+
+  if [ "$mount_failed" = true ]
+  then
+    echo "STOP: The archive couldn't be mounted with CIFS version ${cifs_version}. Try specifying a lower number for the CIFS version like this: export cifs_version=2"
+	exit 1
+  fi
+  
+  umount "$test_mount_location"
+}
+
 function check_available_space () {
   echo "Verifying that there is sufficient space available on the MicroSD card..."
 
@@ -104,7 +140,10 @@ function create_usb_drive_backing_files () {
 }
 
 function configure_archive () {
+  local archive_server_ip_address="$1"
+
   echo "Configuring the archive..."
+  
   if [ $RSYNC_ENABLE = true ]
   then
     echo "Configuring for Rsync..."
@@ -112,15 +151,21 @@ function configure_archive () {
     echo "server=$RSYNC_SERVER" >> /root/.teslaCamRsyncConfig
     echo "path=$RSYNC_PATH" >> /root/.teslaCamRsyncConfig
   else
-    echo "Configuring for a shared drive..."
+    local archive_path="/mnt/archive"
-    mkdir /mnt/archive
+  
-    local archive_server_ip_address="$(ping -c 1 -w 1 $archiveserver 2>/dev/null | head -n 1 | grep -o -e "(\([[:digit:]]\{1,3\}\.\)\{3\}[[:digit:]]\{1,3\})" | tr -d '()')"
+    if [ ! -e "$archive_path" ]
-    echo "//$archive_server_ip_address/$sharename /mnt/archive cifs vers=3,credentials=/root/.teslaCamArchiveCredentials,iocharset=utf8,file_mode=0777,dir_mode=0777 0" >> /etc/fstab
+    then
+      mkdir "$archive_path"  
+    fi
+  
+    local credentials_file_path="/root/.teslaCamArchiveCredentials"
 
-    echo "username=$shareuser" > /root/.teslaCamArchiveCredentials
+    write_archive_credentials_to "$credentials_file_path"
-    echo "password=$sharepassword" >> /root/.teslaCamArchiveCredentials
+
+    echo "//$archive_server_ip_address/$sharename $archive_path cifs vers=${cifs_version},credentials=${credentials_file_path},iocharset=utf8,file_mode=0777,dir_mode=0777 0" >> /etc/fstab
+  
+    echo "Configured the archive."
   fi
-  echo "Configured the archive."
 }
 
 function configure_archive_scripts () {
@@ -153,9 +198,8 @@ function configure_archive_scripts () {
   echo "Downloaded script to remount filesystems read/write if needed (/root/remountfs_rw)."
 }
 
-
 function configure_pushover_scripts() {
-if [ ${user_enabled_pushover} = "true" ]
+if [ ${USER_ENABLED_PUSHOVER} = "true" ]
 then
   pushd /root/bin
   wget https://raw.githubusercontent.com/"$REPO"/teslausb/"$BRANCH"/windows_archive/send-pushover
@@ -205,6 +249,11 @@ function make_root_fs_readonly () {
 
 echo "Verifying environment variables..."
 
+if [ ! -n "${cifs_version+x}" ]
+then
+  cifs_version=3
+fi
+
 if [ $RSYNC_ENABLE = true ]
 then
     check_variable "RSYNC_USER"
@@ -223,6 +272,10 @@ check_pushover_enabled
 
 check_archive_server_reachable
 
+ARCHIVE_SERVER_IP_ADDRESS="$(ping -c 1 -w 1 $archiveserver 2>/dev/null | head -n 1 | grep -o -e "(\([[:digit:]]\{1,3\}\.\)\{3\}[[:digit:]]\{1,3\})" | tr -d '()')"
+
+check_archive_mountable "$ARCHIVE_SERVER_IP_ADDRESS"
+
 check_available_space
 
 get_ancillary_setup_scripts
@@ -239,7 +292,7 @@ echo "" >> /etc/fstab
 
 create_usb_drive_backing_files
 
-configure_archive
+configure_archive "$ARCHIVE_SERVER_IP_ADDRESS"
 
 configure_rc_local