From 4deb1c35b8473f3086fa43e39fa175145d75d39b Mon Sep 17 00:00:00 2001 From: Ray Tanaka Date: Thu, 25 Oct 2018 12:46:10 -0700 Subject: [PATCH 1/8] add rclone configuration option in main setup script --- setup/pi/setup-teslausb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/setup/pi/setup-teslausb b/setup/pi/setup-teslausb index 1b208ce..b4bf97e 100644 --- a/setup/pi/setup-teslausb +++ b/setup/pi/setup-teslausb @@ -119,6 +119,11 @@ function configure_archive_scripts () { get_script /root/bin archive-clips.sh run/rsync_archive get_script /root/bin connect-archive.sh run/rsync_archive get_script /root/bin disconnect-archive.sh run/rsync_archive + elif [ $RCLONE_ENABLE = true ] + then + get_script /root/bin archive-clips.sh run/rclone_archive + get_script /root/bin connect-archive.sh run/rclone_archive + get_script /root/bin disconnect-archive.sh run/rclone_archive else get_script /root/bin archive-clips.sh run/cifs_archive get_script /root/bin connect-archive.sh run/cifs_archive @@ -181,6 +186,7 @@ function make_root_fs_readonly () { echo "Verifying environment variables..." RSYNC_ENABLE="${RSYNC_ENABLE:-false}" +RCLONE_ENABLE="${RCLONE_ENABLE:-false}" if [ "$RSYNC_ENABLE" = true ] then @@ -188,6 +194,12 @@ then check_variable "RSYNC_SERVER" export archiveserver="$RSYNC_SERVER" check_variable "RSYNC_PATH" +elif [ "$RCLONE_ENABLE" = true ] +then + check_variable "RCLONE_DRIVE" + check_variable "RCLONE_PATH" + # since it's a cloud hosted drive we'll just set this to google dns + export archiveserver="8.8.8.8" else # Else for now, TODO allow both for more redundancy? check_variable "sharename" check_variable "shareuser" @@ -210,6 +222,10 @@ if [ "$RSYNC_ENABLE" = true ] then get_script /root/bin verify-archive-configuration.sh run/rsync_archive get_script /root/bin configure-archive.sh run/rsync_archive +elif [ "$RCLONE_ENABLE" = true ] +then + get_script /root/bin verify-archive-configuration.sh run/clone_archive + get_script /root/bin configure-archive.sh run/rclone_archive else get_script /root/bin verify-archive-configuration.sh run/cifs_archive get_script /root/bin configure-archive.sh run/cifs_archive From 9bb8f1e0463cf5886e17c921ee6d6644b4051a22 Mon Sep 17 00:00:00 2001 From: Ray Tanaka Date: Thu, 25 Oct 2018 15:18:06 -0700 Subject: [PATCH 2/8] adding support and documentation for using rclone --- README.md | 3 + doc/SetupRClone.md | 85 +++++++++++++++++++ run/rclone_archive/archive-clips.sh | 20 +++++ run/rclone_archive/configure-archive.sh | 10 +++ run/rclone_archive/connect-archive.sh | 2 + run/rclone_archive/disconnect-archive.sh | 2 + .../verify-archive-configuration.sh | 1 + 7 files changed, 123 insertions(+) create mode 100644 doc/SetupRClone.md create mode 100644 run/rclone_archive/archive-clips.sh create mode 100644 run/rclone_archive/configure-archive.sh create mode 100644 run/rclone_archive/connect-archive.sh create mode 100644 run/rclone_archive/disconnect-archive.sh create mode 100644 run/rclone_archive/verify-archive-configuration.sh diff --git a/README.md b/README.md index fa902c4..40f525f 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,9 @@ Get the IP address of the archive machine. You'll need this later, so write it d Since sftp/rsync is accessing a computer through SSH, the only requirement for hosting an SFTP/rsync server is to have a box running Linux. An example can be another Raspberry Pi connected to your local network with a USB storage drive plugged in. The official Raspberry Pi site has a good example on [how to mount an external drive](https://www.raspberrypi.org/documentation/configuration/external-storage.md). You will need the username and host/IP of the storage server, as well as the path for the files to go in, and the storage server will need to allow SSH. +### Hosting via rclone (for Google Drive, S3, DropBox, etc) +**EXPERIMENTAL** - This hasn't been thoroughly tested yet but you can follow these [instructions](/doc/SetupRClone.md) and give it a spin. + ### ***TODO: Other hosting solutions*** ## Set up the Raspberry Pi diff --git a/doc/SetupRClone.md b/doc/SetupRClone.md new file mode 100644 index 0000000..e4aaa55 --- /dev/null +++ b/doc/SetupRClone.md @@ -0,0 +1,85 @@ +# Introduction + +This guide will show you how to install and configure [rclone4pi](https://github.com/pageauc/rclone4pi/wiki) (based off [rclone](https://rclone.org/)) to archive your saved TeslaCam footage on a number of different remote storage services including (Google Drive, S3 and Dropbox). + +This guide assumes you have **NOT** run the `setup-teslacam` script yet + +# Step 1: Install rclone4pi + +The first step is to get the [rclone4pi](https://github.com/pageauc/rclone4pi/wiki) binary installed on the raspberry pi. You can do this by executing the following command: + + +``` +curl -L https://raw.github.com/pageauc/rclone4pi/master/rclone-install.sh | bash +``` + +You can also install the script manually by following these [instructions](https://github.com/pageauc/rclone4pi/wiki#manual-install). + + +Once installed, the script will install rclone-install.sh, rclone-sync.sh and create a subfolder rpi-sync in users home eg. /home/pi + +# Step 2: Configure rclone storage system + +Next, run this command as root to configure a storage system. + +``` +rclone config +``` + +**Important:** Run this as root since archiveloop runs as root and the rclone config is bound to the user running the config. + +This will launch an interactive setup with a series of questions. I highly recommend you look at the documents for your storage system by going to [rclone](https://rclone.org/) and selecting your storage system from the pull down menu at the stop. + +I've only personally tested this with Google Drive using these [instructions](https://rclone.org/drive/). One thing to note is the importance of setting the correct [scope](https://rclone.org/drive/#scopes) you are providing access to. Carefully read the documentation on [scopes on rclone](https://rclone.org/drive/#scopes) as well as [Google Drive](https://developers.google.com/drive/api/v3/about-auth). I recommend going with drive.file scope. + +**Important:** Take note of the name you used for this config. You will need it later. The rest of the document will use `gdrive` as the name since that's what I used. + +# Step 3: Verify and create storage directory + +Run the following command (again, as root) to see the name of the remote drive you just created. + +``` +rclone listremotes +``` + +If you don't see the name there, something went wrong and you'll likely have to go back through the config process. If all went well, use + + +``` +rclone lsd gdrive: +``` + +At this point, you should not see anything if you set your scope correctly. Now we need to create a folder to put all our archives in. You can do this by running this command. I used TeslaCam but you can name it whatever you want as long as you set it in the next step below. + +``` +rclone mkdir gdrive:TeslaCam +``` + +Run this one last command again + +``` +rclone lsd gdrive: +``` + +Once you confirm that the directoy you just created is there, we're all set to move on! + +# Step 4: Exports + +To be able to configure the teslausb pi to use rclone, you'll need to export a few things. On your teslausb pi, run: + +``` +export RCLONE_ENABLE=true +export RCLONE_DRIVE= +export RCLONE_PATH= +``` + +An example of my config is listed below: + +``` +export RCLONE_ENABLE=true +export RCLONE_DRIVE=gdrive +export RCLONE_PATH=TeslaCam +``` +**Note:** `RCLONE_ENABLE=true` is going to disable the default archive server. It also will **not** play nicely with `RSYNC_ENABLE=true` Perhaps future releases will allow both to be defined and function at the same time, for redundancy, but for now just pick one that you'll want the most. + +You should be ready to run the setup script now, so return back to step 8 of the [Main Instructions](/README.md). \ No newline at end of file diff --git a/run/rclone_archive/archive-clips.sh b/run/rclone_archive/archive-clips.sh new file mode 100644 index 0000000..6b761d5 --- /dev/null +++ b/run/rclone_archive/archive-clips.sh @@ -0,0 +1,20 @@ +#!/bin/bash -eu + +log "Moving clips to rclone archive..." + +source /root/.teslaCamRcloneConfig + +NUM_FILES_MOVED=0 + +for file_name in "$CAM_MOUNT"/TeslaCam/saved*; do + [ -e "$file_name" ] || continue + log "Moving $file_name ..." + rclone move "$file_name" "$drive:$path" >> "$LOG_FILE" 2>&1 || echo "" + log "Moved $file_name." + NUM_FILES_MOVED=$((NUM_FILES_MOVED + 1)) +done +log "Moved $NUM_FILES_MOVED file(s)." + +/root/bin/send-pushover "$NUM_FILES_MOVED" + +log "Finished moving clips to rclone archive" \ No newline at end of file diff --git a/run/rclone_archive/configure-archive.sh b/run/rclone_archive/configure-archive.sh new file mode 100644 index 0000000..13e291b --- /dev/null +++ b/run/rclone_archive/configure-archive.sh @@ -0,0 +1,10 @@ +#!/bin/bash -eu + +function configure_archive () { + echo "Configuring the archive for Rclone..." + + echo "drive=$RCLONE_DRIVE" > /root/.teslaCamRcloneConfig + echo "path=$RCLONE_PATH" >> /root/.teslaCamRcloneConfig +} + +configure_archive \ No newline at end of file diff --git a/run/rclone_archive/connect-archive.sh b/run/rclone_archive/connect-archive.sh new file mode 100644 index 0000000..cfcbd8e --- /dev/null +++ b/run/rclone_archive/connect-archive.sh @@ -0,0 +1,2 @@ +#!/bin/bash -eu +# Nothing to do. \ No newline at end of file diff --git a/run/rclone_archive/disconnect-archive.sh b/run/rclone_archive/disconnect-archive.sh new file mode 100644 index 0000000..cfcbd8e --- /dev/null +++ b/run/rclone_archive/disconnect-archive.sh @@ -0,0 +1,2 @@ +#!/bin/bash -eu +# Nothing to do. \ No newline at end of file diff --git a/run/rclone_archive/verify-archive-configuration.sh b/run/rclone_archive/verify-archive-configuration.sh new file mode 100644 index 0000000..95dbbf8 --- /dev/null +++ b/run/rclone_archive/verify-archive-configuration.sh @@ -0,0 +1 @@ +#!/bin/bash -eu \ No newline at end of file From 00d77474b12f0b1135867083c1d1c8b1f1513e4a Mon Sep 17 00:00:00 2001 From: Ray Tanaka Date: Thu, 25 Oct 2018 15:27:17 -0700 Subject: [PATCH 3/8] bug fix: script tries to create a filename 0 when files sent is 0 --- run/send-pushover | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/send-pushover b/run/send-pushover index 9d5ffa5..7fc5402 100644 --- a/run/send-pushover +++ b/run/send-pushover @@ -7,7 +7,7 @@ function log () { echo "$1" >> "$LOG_FILE" } -if [ -r "/root/.teslaCamPushoverCredentials" ] && [ $NUM_FILES_MOVED > 0 ] +if [ -r "/root/.teslaCamPushoverCredentials" ] && [ $NUM_FILES_MOVED -gt 0 ] then log "Sending Pushover message for moved files." From 78194641d44ea5822de8350b3b02d518b85e52c9 Mon Sep 17 00:00:00 2001 From: cimryan Date: Thu, 25 Oct 2018 19:00:41 -0700 Subject: [PATCH 4/8] making suggested change Co-Authored-By: rtanaka --- setup/pi/setup-teslausb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/pi/setup-teslausb b/setup/pi/setup-teslausb index 65e25dd..f608edf 100644 --- a/setup/pi/setup-teslausb +++ b/setup/pi/setup-teslausb @@ -224,7 +224,7 @@ then get_script /root/bin configure-archive.sh run/rsync_archive elif [ "$RCLONE_ENABLE" = true ] then - get_script /root/bin verify-archive-configuration.sh run/clone_archive + get_script /root/bin verify-archive-configuration.sh run/rclone_archive get_script /root/bin configure-archive.sh run/rclone_archive else get_script /root/bin verify-archive-configuration.sh run/cifs_archive From 485cec4e413632d8527032ea529d927463f13a09 Mon Sep 17 00:00:00 2001 From: Ray Tanaka Date: Fri, 26 Oct 2018 01:41:15 -0700 Subject: [PATCH 5/8] explicitly adding config file path since it is throwing errors --- run/rclone_archive/archive-clips.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/rclone_archive/archive-clips.sh b/run/rclone_archive/archive-clips.sh index 6b761d5..9f95c01 100644 --- a/run/rclone_archive/archive-clips.sh +++ b/run/rclone_archive/archive-clips.sh @@ -9,7 +9,7 @@ NUM_FILES_MOVED=0 for file_name in "$CAM_MOUNT"/TeslaCam/saved*; do [ -e "$file_name" ] || continue log "Moving $file_name ..." - rclone move "$file_name" "$drive:$path" >> "$LOG_FILE" 2>&1 || echo "" + rclone --config /root/.config/rclone/rclone.conf move "$file_name" "$drive:$path" >> "$LOG_FILE" 2>&1 || echo "" log "Moved $file_name." NUM_FILES_MOVED=$((NUM_FILES_MOVED + 1)) done From b2111722a74e52919b9d7b89eb7581a76493ec3f Mon Sep 17 00:00:00 2001 From: cimryan Date: Sat, 27 Oct 2018 08:54:39 -0700 Subject: [PATCH 6/8] Create a 100MB "mutable" partition after the backingfiles partition. --- setup/pi/create-backingfiles-partition.sh | 25 +++++++++++++++++------ setup/pi/setup-teslausb | 8 +++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/setup/pi/create-backingfiles-partition.sh b/setup/pi/create-backingfiles-partition.sh index e8b7013..120130e 100644 --- a/setup/pi/create-backingfiles-partition.sh +++ b/setup/pi/create-backingfiles-partition.sh @@ -1,16 +1,27 @@ #!/bin/bash -eu BACKINGFILES_MOUNTPOINT="$1" - -PARTITION_TABLE=$(parted -m /dev/mmcblk0 unit s print) -ROOT_PARTITION_LINE=$(echo "$PARTITION_TABLE" | grep -e "^2:") -LAST_ROOT_PARTITION_SECTOR=$(echo "$ROOT_PARTITION_LINE" | sed 's/s//g' | cut -d ":" -f 3) +MUTABLE_MOUNTPOINT="$2" -FIRST_BACKINGFILES_PARTITION_SECTOR=$(( $LAST_ROOT_PARTITION_SECTOR + 1 )) +PARTITION_TABLE=$(parted -m /dev/mmcblk0 unit B print) +DISK_LINE=$(echo "$PARTITION_TABLE" | grep -e "^/dev/mmcblk0:") +DISK_SIZE=$(echo "$DISK_LINE" | cut -d ":" -f 2 | sed 's/B//' ) + +ROOT_PARTITION_LINE=$(echo "$PARTITION_TABLE" | grep -e "^2:") +LAST_ROOT_PARTITION_BYTE=$(echo "$ROOT_PARTITION_LINE" | sed 's/B//g' | cut -d ":" -f 3) + +FIRST_BACKINGFILES_PARTITION_BYTE="$(( $LAST_ROOT_PARTITION_BYTE + 1 ))" +LAST_BACKINGFILES_PARTITION_DESIRED_BYTE="$(( $DISK_SIZE - (100 * (2 ** 20)) - 1))" ORIGINAL_DISK_IDENTIFIER=$( fdisk -l /dev/mmcblk0 | grep -e "^Disk identifier" | sed "s/Disk identifier: 0x//" ) -parted -m /dev/mmcblk0 u s mkpart primary ext4 "$FIRST_BACKINGFILES_PARTITION_SECTOR" 100% +BACKINGFILES_PARTITION_END_SPEC="$(( $LAST_BACKINGFILES_PARTITION_DESIRED_BYTE / 1000000 ))M" +parted -a optimal -m /dev/mmcblk0 unit B mkpart primary ext4 "$FIRST_BACKINGFILES_PARTITION_BYTE" "$BACKINGFILES_PARTITION_END_SPEC" + +LAST_BACKINGFILES_PARTITION_BYTE=$(parted -m /dev/mmcblk0 unit B print | grep -e "^3:" | cut -d ":" -f 3 | sed 's/B//g' ) + +MUTABLE_PARTITION_START_SPEC="$(( $LAST_BACKINGFILES_PARTITION_BYTE / 1000000 ))M" +parted -a optimal -m /dev/mmcblk0 unit B mkpart primary ext4 "$MUTABLE_PARTITION_START_SPEC" 100% NEW_DISK_IDENTIFIER=$( fdisk -l /dev/mmcblk0 | grep -e "^Disk identifier" | sed "s/Disk identifier: 0x//" ) @@ -18,5 +29,7 @@ sed -i "s/${ORIGINAL_DISK_IDENTIFIER}/${NEW_DISK_IDENTIFIER}/g" /etc/fstab sed -i "s/${ORIGINAL_DISK_IDENTIFIER}/${NEW_DISK_IDENTIFIER}/" /boot/cmdline.txt mkfs.ext4 -F /dev/mmcblk0p3 +mkfs.ext4 -F /dev/mmcblk0p4 echo "/dev/mmcblk0p3 $BACKINGFILES_MOUNTPOINT ext4 auto,rw,noatime 0 2" >> /etc/fstab +echo "/dev/mmcblk0p4 $MUTABLE_MOUNTPOINT ext4 auto,rw 0 2" >> /etc/fstab diff --git a/setup/pi/setup-teslausb b/setup/pi/setup-teslausb index 6e64ee8..cc4e8c3 100644 --- a/setup/pi/setup-teslausb +++ b/setup/pi/setup-teslausb @@ -83,6 +83,7 @@ function fix_cmdline_txt_modules_load () } BACKINGFILES_MOUNTPOINT=/backingfiles +MUTABLE_MOUNTPOINT=/mutable function create_usb_drive_backing_files () { if [ ! -e "$BACKINGFILES_MOUNTPOINT" ] @@ -90,9 +91,14 @@ function create_usb_drive_backing_files () { mkdir "$BACKINGFILES_MOUNTPOINT" fi + if [ ! -e "$MUTABLE_MOUNTPOINT" ] + then + mkdir "$MUTABLE_MOUNTPOINT" + fi + if [ ! -e /dev/mmcblk0p3 ] then - /tmp/create-backingfiles-partition.sh "$BACKINGFILES_MOUNTPOINT" + /tmp/create-backingfiles-partition.sh "$BACKINGFILES_MOUNTPOINT" "$MUTABLE_MOUNTPOINT" fi if ! findmnt --mountpoint $BACKINGFILES_MOUNTPOINT From b43e4d709d46fcd05c79486d453f10a89d135e28 Mon Sep 17 00:00:00 2001 From: Ray Tanaka Date: Sun, 28 Oct 2018 10:15:54 -0700 Subject: [PATCH 7/8] making change to leverage the new /mutable directory updating docs for clarification --- README.md | 4 ++-- doc/SetupRClone.md | 4 ++-- setup/pi/make-root-fs-readonly.sh | 33 ++++++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 40f525f..06914af 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Get the IP address of the archive machine. You'll need this later, so write it d Since sftp/rsync is accessing a computer through SSH, the only requirement for hosting an SFTP/rsync server is to have a box running Linux. An example can be another Raspberry Pi connected to your local network with a USB storage drive plugged in. The official Raspberry Pi site has a good example on [how to mount an external drive](https://www.raspberrypi.org/documentation/configuration/external-storage.md). You will need the username and host/IP of the storage server, as well as the path for the files to go in, and the storage server will need to allow SSH. ### Hosting via rclone (for Google Drive, S3, DropBox, etc) -**EXPERIMENTAL** - This hasn't been thoroughly tested yet but you can follow these [instructions](/doc/SetupRClone.md) and give it a spin. +**EXPERIMENTAL** - This hasn't been thoroughly tested yet but you can follow these [instructions](/doc/SetupRClone.md) to setup up and configure rclone to work with teslausb. ### ***TODO: Other hosting solutions*** @@ -107,7 +107,7 @@ Now that you have Wifi up and running, it's time to set up the USB storage and s export campercent= ``` For example, using `export campercent=100` would allocate 100% of the space to recording footage from your car, and would not create a separate music partition. `export campercent=50` would be only allocate half of the space for a dashcam footage drive, and allocates the other half to be a music storage drive. -1. If you are trying to archive on an SFTP/rsync server, then follow these [instructions](doc/SetupRSync.md) and skip step 7. Otherwise, skip this step. +1. **Only for SFTP/rsync or rclone archives**: If you are trying to archive on an SFTP/rsync server, then follow these [instructions](doc/SetupRSync.md) and skip step 7. If you set up rclone you can also skip step 7 and go to step 8. 1. If you are trying to archive on a shared drive, run these commands, subsituting your values for your shared drive: ``` export archiveserver=Nautilus diff --git a/doc/SetupRClone.md b/doc/SetupRClone.md index e4aaa55..72ce8b4 100644 --- a/doc/SetupRClone.md +++ b/doc/SetupRClone.md @@ -16,7 +16,7 @@ curl -L https://raw.github.com/pageauc/rclone4pi/master/rclone-install.sh | bash You can also install the script manually by following these [instructions](https://github.com/pageauc/rclone4pi/wiki#manual-install). -Once installed, the script will install rclone-install.sh, rclone-sync.sh and create a subfolder rpi-sync in users home eg. /home/pi +Once installed, the script will install rclone-install.sh, rclone-sync.sh and create a subfolder rpi-sync in users home eg. /root # Step 2: Configure rclone storage system @@ -26,7 +26,7 @@ Next, run this command as root to configure a storage system. rclone config ``` -**Important:** Run this as root since archiveloop runs as root and the rclone config is bound to the user running the config. +**Important:** Make sure to run this as root since archiveloop runs as root and the rclone config is bound to the user running the config. The config file is also moved with the assumption that the root user ran config. This will launch an interactive setup with a series of questions. I highly recommend you look at the documents for your storage system by going to [rclone](https://rclone.org/) and selecting your storage system from the pull down menu at the stop. diff --git a/setup/pi/make-root-fs-readonly.sh b/setup/pi/make-root-fs-readonly.sh index 1d7882c..fdf2ba6 100644 --- a/setup/pi/make-root-fs-readonly.sh +++ b/setup/pi/make-root-fs-readonly.sh @@ -10,7 +10,7 @@ function append_cmdline_txt_param() { echo "Updating package index files..." apt-get update echo "Removing unwanted packages..." -apt-get remove -y --force-yes --purge triggerhappy logrotate dphys-swapfile fake-hwclock +apt-get remove -y --force-yes --purge triggerhappy logrotate dphys-swapfile apt-get -y --force-yes autoremove --purge # Replace log management with busybox (use logread if needed) echo "Installing ntp and busybox-syslogd..." @@ -22,6 +22,37 @@ append_cmdline_txt_param fastboot append_cmdline_txt_param noswap append_cmdline_txt_param ro +# Move fake-hwclock.data to /mutable directory so it can be updated +if ! findmnt --mountpoint /mutable +then + echo "Mounting the multable partition..." + mount /mutable + echo "Mounted." +fi +if [ ! -e "/mutable/etc" ] +then + mkdir -p /mutable/etc +fi +if [ -e "/etc/fake-hwclock.data" ] +then + echo "Moving fake-hwclock data" + cp /etc/fake-hwclock.data /mutable/etc/fake-hwclock.data + rm /etc/fake-hwclock.data + ln -s /mutable/etc/fake-hwclock.data /etc/fake-hwclock.data +fi + +# Move rclone configs if it exists so we can write to it +if [ ! -e "/mutable/configs" ] +then + mkdir -p /mutable/configs +fi +if [ -e "/root/.config/rclone/rclone.conf" ] +then + echo "Moving rclone configs" + mv /root/.config/rclone /mutable/configs + ln -s /mutable/configs/rclone /root/.config/rclone +fi + # Move /var/spool to /tmp rm -rf /var/spool ln -s /tmp /var/spool From 2fe2e4c475d81a1da66940d96640266fd282bd5c Mon Sep 17 00:00:00 2001 From: cimryan Date: Sun, 28 Oct 2018 22:28:51 -0700 Subject: [PATCH 8/8] Consolidate and standardize documentation. --- README.md | 87 +++++++-------------------- doc/SetupRClone.md | 142 ++++++++++++++++++++++++--------------------- doc/SetupRSync.md | 38 +++++++----- doc/SetupShare.md | 32 ++++++++++ 4 files changed, 152 insertions(+), 147 deletions(-) create mode 100644 doc/SetupShare.md diff --git a/README.md b/README.md index 06914af..c68bd7e 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,12 @@ This repo contains steps and scripts originally from [this thread on Reddit]( ht Many people in that thread suggested that the scripts be hosted on Github but the author didn't seem interested in making that happen. I've hosted the scripts here with his/her permission. -The original post on Reddit assumed that the archive would be hosted on Windows and that the Pi would be set up using a Windows machine but this Git repo welcomes the contribution of instructions for other platforms. - ## Intro -You can configure a Raspberry Pi Zero W so that your Tesla thinks it's a USB drive and will write dashcam footage to it. Since it's a computer, you can run scripts on the Pi to automatically copy the clips to an archive server when you get home. The Pi is going to continually: -1. Wait until it can connect to the archive server -1. Archive the clips -1. Wait until it can't connect to the archive server -1. GOTO 1. - -The scripts in this repo will also allow you to use the Pi to store music that the Tesla can read through the USB interface. +You can configure a Raspberry Pi Zero W so that your Tesla thinks it's a USB drive and will write dashcam footage to it. Since it's a computer: +* Scripts running on the Pi can automatically copy the clips to an archive server when you get home. +* The Pi can hold both dashcam clips and music files. +* The Pi can automatically repair filesystem corruption produced by the Tesla's current failure to properly dismount the USB drives before cutting power to the USB ports. Archiving the clips can take from seconds to hours depending on how many clips you've saved and how strong the WiFi signal is in your Tesla. If you find that the clips aren't getting completely transferred before the car powers down after you park or before you leave you can use the Tesla app to turn on the Climate control. This will send power to the Raspberry Pi, allowing it to complete the archival operation. @@ -24,14 +19,12 @@ Archiving the clips can take from seconds to hours depending on how many clips y ### Assumptions * You park in range of your wireless network. * Your wireless network is configured with WPA2 PSK access. -* You'll be archiving your dashcam clips to a Windows machine, and the Windows machine has a stable IP address on your home network. -* You'll be setting up the Raspberry Pi using a Windows machine. ### Hardware Required: * [Raspberry Pi Zero W](https://www.raspberrypi.org/products/raspberry-pi-zero-w/): [Adafruit](https://www.adafruit.com/product/3400) or [Amazon](https://www.amazon.com/Raspberry-Pi-Zero-Wireless-model/dp/B06XFZC3BX/) - > Note: Of the many varieties of Raspberry Pi only the Raspberry Pi Zero and Raspberry Pi Zero W can be used as simulated USB drives. It may be possible to use a Pi Zero with a USB Wifi adapter to achieve the same result as the Pi Zero W, but this hasn't been confirmed. + > Note: Of the many varieties of Raspberry Pi avaiable only the Raspberry Pi Zero and Raspberry Pi Zero W can be used as simulated USB drives. It may be possible to use a Pi Zero with a USB Wifi adapter to achieve the same result as the Pi Zero W but this hasn't been confirmed. * A Micro SD card, at least 8 GB in size, and an adapter (if necessary) to connect the card to your computer. * A mechanism to connect the Pi to the Tesla. Either: @@ -44,77 +37,41 @@ Optional: ### Software Download [Raspbian Stretch Lite](https://www.raspberrypi.org/downloads/raspbian/) -* Note: Bittorrent is dramatically faster than direct download. - Download and install: * [Etcher](http://etcher.io) -## Create your archive -### Hosting on Windows File Shares, MacOS Sharing, or Samba on Linux -Set up a share to host the archive. These instructions assume that you created a share named "SailfishCam" on the server "Nautilus". It is recommended that you create a new user. Grant the user you'll be using read/write access to the share. These instructions will assume that the user you've created is named "sailfish" and that the password for this user is "pa$$w0rd". - -Get the IP address of the archive machine. You'll need this later, so write it down, somewhere. -* On Windows you can do this by opening a command prompt on the archive machine and typing ipconfig. Get the IP address from the line labeled "IPv4 Address". These instructions will assume that the IP address of the archive server is 192.168.0.41. -* On MacOS or Linux open a terminal and type ifconfig. - -### Hosting via SFTP/rsync -**EXPERIMENTAL - Hosting the archive on SFTP hasn't been thoroughly tested** - -Since sftp/rsync is accessing a computer through SSH, the only requirement for hosting an SFTP/rsync server is to have a box running Linux. An example can be another Raspberry Pi connected to your local network with a USB storage drive plugged in. The official Raspberry Pi site has a good example on [how to mount an external drive](https://www.raspberrypi.org/documentation/configuration/external-storage.md). You will need the username and host/IP of the storage server, as well as the path for the files to go in, and the storage server will need to allow SSH. - -### Hosting via rclone (for Google Drive, S3, DropBox, etc) -**EXPERIMENTAL** - This hasn't been thoroughly tested yet but you can follow these [instructions](/doc/SetupRClone.md) to setup up and configure rclone to work with teslausb. - -### ***TODO: Other hosting solutions*** - ## Set up the Raspberry Pi -There are three phases to setting up the Pi: +There are four phases to setting up the Pi: 1. Get the OS onto the micro sd card. 1. Get a shell on the Pi. +1. Set up the archive for dashcam clips. 1. Set up the USB storage functionality. -### Get the OS onto the micro SD card - +### Get the OS onto the MicroSD card [These instructions](https://www.raspberrypi.org/documentation/installation/installing-images/README.md) tell you how to get Raspbian onto your MicroSD card. Basically: 1. Connect your SD card to your computer. -2. Use Etcher to write the zip file you downloaded to the SD card. Etcher works well and is multi-platform. +2. Use Etcher to write the zip file you downloaded to the SD card. > Note: you don't need to uncompress the zip file you downloaded. ### Get a shell on the Pi -If you used a Windows computer to flash the OS onto the MicroSD card, follow these [Instructions](doc/GetShellWithoutMonitorOnWindows.md). +Follow the instructions corresponding to the OS you used to flash the OS onto the MicroSD card: +* Windows: [Instructions](doc/GetShellWithoutMonitorOnWindows.md). +* MacOS or Linux: [Instructions](doc/GetShellWithoutMonitorOnLinux.md). -If you used a Mac or a Linux computer, follow these [Instructions](doc/GetShellWithoutMonitorOnLinux.md). +Whichever instructions you followed above will leave you in a command shell on the Pi. Use this shell for the rest of the steps in these instructions. + +### Set up the archive for dashcam clips +Follow the instructions corresponding to the technology you'd like to use to host the archive for your dashcam clips. You must choose just one of these technoloies; don't follow more than one of these sets of instructions: +* Windows file share, MacOS Sharing, or Samba on Linux: [Instructions](doc/SetupShare.md). +* SFTP/rsync: [Instructions](doc/SetupRsync.md) +* **Experimental:** Google Drive, Amazon S3, DropBox, Microsoft OneDrive: [Instructions](doc/SetupRClone.md) ### Set up the USB storage functionality - -Now that you have Wifi up and running, it's time to set up the USB storage and scripts that will manage the dashcam and (optionally) music storage. - -1. SSH to the Pi and run - ``` - sudo -i - ``` -1. Try to ping your archive server from the Pi. In this example the server is named `nautilus`. - ``` - ping -c 3 nautilus - ``` -1. If the server can't be reached, ping its IP address: - ``` - ping 192.168.0.41 - ``` -1. If you can't ping the archive server by IP address from the Pi, you should go do whatever you need to on your network to fix that. If you can't reach the archive server by name, from the Pi but you can by IP address, then use its IP address, below, in place of its name. -1. Determine how much, as a percentage, of the drive you want to allocate to recording dashcam footage by using: +1. Indicate how much, as a percentage, of the drive you want to allocate to recording dashcam footage by running this command: ``` export campercent= ``` - For example, using `export campercent=100` would allocate 100% of the space to recording footage from your car, and would not create a separate music partition. `export campercent=50` would be only allocate half of the space for a dashcam footage drive, and allocates the other half to be a music storage drive. -1. **Only for SFTP/rsync or rclone archives**: If you are trying to archive on an SFTP/rsync server, then follow these [instructions](doc/SetupRSync.md) and skip step 7. If you set up rclone you can also skip step 7 and go to step 8. -1. If you are trying to archive on a shared drive, run these commands, subsituting your values for your shared drive: - ``` - export archiveserver=Nautilus - export sharename=SailfishCam - export shareuser=sailfish - export sharepassword=pa$$w0rd - ``` + For example, using `export campercent=100` would allocate 100% of the space to recording footage from your car, and would not create a separate music partition. `export campercent=50` would allocate half of the space for a dashcam footage drive and allocates the other half to for a music storage drive. 1. If you'd like to receive a text message when your Pi finishes archiving clips follow these [Instructions](doc/ConfigureNotificationsForArchive.md). 1. Run these commands: ``` @@ -137,7 +94,7 @@ Connect the Pi to a computer. If you're using a cable be sure to use the port la 1. Wait for the Pi to show up on the computer as a USB drive. 1. Copy any music you'd like to the drive labeled MUSIC. 1. Eject the drives. -1. Unplug the Pi from the PC. +1. Unplug the Pi from the computer. 1. Plug the Pi into your Tesla. ## Making changes to the system after setup diff --git a/doc/SetupRClone.md b/doc/SetupRClone.md index 72ce8b4..02d14ee 100644 --- a/doc/SetupRClone.md +++ b/doc/SetupRClone.md @@ -1,85 +1,93 @@ # Introduction -This guide will show you how to install and configure [rclone4pi](https://github.com/pageauc/rclone4pi/wiki) (based off [rclone](https://rclone.org/)) to archive your saved TeslaCam footage on a number of different remote storage services including (Google Drive, S3 and Dropbox). +This guide will show you how to install and configure [rclone4pi](https://github.com/pageauc/rclone4pi/wiki) (based off [rclone](https://rclone.org/)) to archive your saved TeslaCam footage on one of a number of different remote storage services including Google Drive, S3 and Dropbox. -This guide assumes you have **NOT** run the `setup-teslacam` script yet +You must perform these steps **after** getting a shell on the Pi and **before** running the `setup-teslacam` script on the Pi. -# Step 1: Install rclone4pi +**Make sure to run all commands in these instructions in a single command shell as root. When you return to the [Main Instructions](/README.md) continue running the commands there in this same shell.** This is necessary because: +* The `archiveloop` script runs as root and the rclone config is bound to the user running the config. +* These commands define environment variables that the main setup scripts need. -The first step is to get the [rclone4pi](https://github.com/pageauc/rclone4pi/wiki) binary installed on the raspberry pi. You can do this by executing the following command: +# Quick guide +These instructions will speed you through the process with good defaults. If you encounter an error, or you want to use a different config name than `gdrive` or a different folder name than `TeslaCam`, follow the detailed instuctions, below. +1. Run these commands. Specify the config name `gdrive` when prompted for the config name. + ``` + sudo -i + curl -L https://raw.github.com/pageauc/rclone4pi/master/rclone-install.sh | bash + rclone config + ``` +1. Run these commands: + ``` + export RCLONE_ENABLE=true + export RCLONE_DRIVE=gdrive + export RCLONE_PATH=TeslaCam + + rclone mkdir "$RCLONE_DRIVE:$RCLONE_PATH" + rclone lsd "$RCLONE_DRIVE": + ``` +1. If you didn't encounter any error messages and you see the `TeslaCam` directory listed return to the [Main Instructions](../README.md). -``` -curl -L https://raw.github.com/pageauc/rclone4pi/master/rclone-install.sh | bash -``` +# Detailed instructions +## Step 1: Install rclone4pi +1. Run this command to enter a command shell for the root user. + ``` + sudo -i + ``` +2. Run the following command to install rclone4pi: + ``` + curl -L https://raw.github.com/pageauc/rclone4pi/master/rclone-install.sh | bash + ``` + Alternatively, you can install rclone4pi manually by following these [instructions] (https://github.com/pageauc/rclone4pi/wiki#manual-install). -You can also install the script manually by following these [instructions](https://github.com/pageauc/rclone4pi/wiki#manual-install). - - -Once installed, the script will install rclone-install.sh, rclone-sync.sh and create a subfolder rpi-sync in users home eg. /root - -# Step 2: Configure rclone storage system - -Next, run this command as root to configure a storage system. - -``` -rclone config -``` - -**Important:** Make sure to run this as root since archiveloop runs as root and the rclone config is bound to the user running the config. The config file is also moved with the assumption that the root user ran config. - -This will launch an interactive setup with a series of questions. I highly recommend you look at the documents for your storage system by going to [rclone](https://rclone.org/) and selecting your storage system from the pull down menu at the stop. - -I've only personally tested this with Google Drive using these [instructions](https://rclone.org/drive/). One thing to note is the importance of setting the correct [scope](https://rclone.org/drive/#scopes) you are providing access to. Carefully read the documentation on [scopes on rclone](https://rclone.org/drive/#scopes) as well as [Google Drive](https://developers.google.com/drive/api/v3/about-auth). I recommend going with drive.file scope. - -**Important:** Take note of the name you used for this config. You will need it later. The rest of the document will use `gdrive` as the name since that's what I used. +# Step 2: Configure the archive +1. Run this command to configure an archive: + ``` + rclone config + ``` + This will launch an interactive setup with a series of questions. It is recommended that you look at the documentation for your storage system by going to [rclone](https://rclone.org/) and selecting your storage system from the pull down menu at the stop. + + It has been confirmed that this process works with Google Drive using these [instructions](https://rclone.org/drive/). If you are using another storage system, please feel encouraged to create an "Issue" describing your challenges and/or your success. + + If you are using Google Drive it is important to set the correct [scope](https://rclone.org/drive/#scopes). Carefully read the documentation on [scopes on rclone](https://rclone.org/drive/#scopes) as well as [Google Drive](https://developers.google.com/drive/api/v3/about-auth). The `drive.file` scope is recommended. + + **Important:** During the `rclone config` process you will sepcify a name for the configuration. The rest of the document will assume the use of the name `gdrive`; replace this with your chosen configuration name. +1. Run this command: + ``` + export RCLONE_DRIVE="gdrive" + ``` # Step 3: Verify and create storage directory -Run the following command (again, as root) to see the name of the remote drive you just created. - -``` -rclone listremotes -``` - -If you don't see the name there, something went wrong and you'll likely have to go back through the config process. If all went well, use - - -``` -rclone lsd gdrive: -``` - -At this point, you should not see anything if you set your scope correctly. Now we need to create a folder to put all our archives in. You can do this by running this command. I used TeslaCam but you can name it whatever you want as long as you set it in the next step below. - -``` -rclone mkdir gdrive:TeslaCam -``` - -Run this one last command again - -``` -rclone lsd gdrive: -``` - -Once you confirm that the directoy you just created is there, we're all set to move on! +1. Run the following command to see the name of the remote drive you just created. + ``` + rclone listremotes + ``` + If you don't see the name there, something went wrong. Go back through the `rclone config` process. +1. Run this command: + ``` + rclone lsd "$RCLONE_DRIVE": + ``` + You should not see any files listed. If you do then you did not set your scope correctly during the `rclone config` process. +1. Choose the name of a folder to hold the archived clips. These instructions will assume you chose the name `TeslaCam`. Substitute the name you chose for this name. Run this command: + ``` + export RCLONE_PATH="TeslaCam" + ``` +1. Run the following command to create a folder which will hold the archived clips. + ``` + rclone mkdir "$RCLONE_DRIVE:TeslaCam" + ``` +1. Run this command again: + ``` + rclone lsd "RCLONE_DRIVE": + ``` +Confirm that the directory `TeslaCam` is present. If not, start over. # Step 4: Exports - -To be able to configure the teslausb pi to use rclone, you'll need to export a few things. On your teslausb pi, run: - +Run this command to cause the setup processes which you'll resume in the main instructions to use rclone4pi: ``` export RCLONE_ENABLE=true -export RCLONE_DRIVE= -export RCLONE_PATH= -``` - -An example of my config is listed below: - -``` -export RCLONE_ENABLE=true -export RCLONE_DRIVE=gdrive -export RCLONE_PATH=TeslaCam ``` **Note:** `RCLONE_ENABLE=true` is going to disable the default archive server. It also will **not** play nicely with `RSYNC_ENABLE=true` Perhaps future releases will allow both to be defined and function at the same time, for redundancy, but for now just pick one that you'll want the most. -You should be ready to run the setup script now, so return back to step 8 of the [Main Instructions](/README.md). \ No newline at end of file +Now return to the section "Set up the USB storage functionality" in the [main instructions](../README.md). diff --git a/doc/SetupRSync.md b/doc/SetupRSync.md index 7106a7a..0d586ac 100644 --- a/doc/SetupRSync.md +++ b/doc/SetupRSync.md @@ -1,21 +1,33 @@ # Introduction -This guide will show you how to utilize [rsync](https://rsync.samba.org/) to archive your saved TeslaCam footage on a remote storage server. In my case, I use this for a networked pi storage server. +This guide will show you how to use [rsync](https://rsync.samba.org/) to archive your saved TeslaCam footage on a remote storage server. + +Since sftp/rsync accesses a computer through SSH the only requirement for hosting an SFTP/rsync server is to have a box running SSH. For example, you could use another Raspberry Pi connected to your local network with a USB storage drive plugged in. The official Raspberry Pi site has a good example on [how to mount an external drive](https://www.raspberrypi.org/documentation/configuration/external-storage.md). + +You will need the username and host/IP of the storage server, as well as the path for the files to go in, and the storage server will need to allow SSH. This guide makes the following assumptions: -1. You are running your own ftp/rsync server that you have admin rights to, or can at least add a public key to its `~/.ssh/authorized_keys` file -1. The ftp/rsync server has rsync installed (raspbian automatically does) -2. You have **NOT** run the `setup-teslacam` script yet +* You are running your own ftp/rsync server that you have admin rights to, or can at least add a public key to its `~/.ssh/authorized_keys` file. +* The ftp/rsync server has rsync installed (raspbian automatically does) + # Step 1: Authentication -Similar to sftp, rsync by default utilizes ssh to connect to a remote server and transfer files. This guide will use a generated ssh keypair, hence the first assumption above. +Similar to sftp, rsync by default uses ssh to connect to a remote server and transfer files. This guide will use a generated ssh keypair, hence the first assumption above. -1. On your teslausb pi, run `ssh-keygen` to generate an ssh key **for the ROOT user!** If you followed the previous steps, you should have already ran `sudo -i` to become the root user on the telsausb pi. If you didn't, run `sudo -i` and re-run `ssh-keygen`. You can be sure that it is generating for root if it asks to store the key in `/root/.ssh/` (versus something like `/home/pi/.ssh`). +1. Run these commands to to generate an ssh key for the `root` user: + ``` + sudo -i + ssh-keygen + ``` -1. Add the contents of the newly generated `/root/.ssh/id_rsa.pub` file from your teslausb pi to the storage server's `~/.ssh/authorized_keys` file. This will allow a nice and easy connection through rsync, no passwords needed! +1. Add the contents of the newly generated `/root/.ssh/id_rsa.pub` file from your teslausb pi to the storage server's `~/.ssh/authorized_keys` file. You can do this by connectin via ssh to the archive server from the computer you're using to set up the Pi, editing the `~/.ssh/authorized_keys` in nano, and pasting in the content of the `/root/.ssh/id_rsa.pub` file from the teslausb Pi. -1. Lastly, you will need to authorize the connection to the FTP/Rsync server and test that the key works, so try connecting to the server (through ssh), and **if it asks if you wish to continue connecting, make sure to type `yes`!** If you do not do this, rsync will fail to connect and thus fail to archive your clips. +1. Lastly, you will need to authorize the connection to the FTP/Rsync server and test that the key works, so try connecting to the server (through ssh), and **when you are asked if you wish to continue connecting type `yes`** + ``` + ssh user@archiveserver + ``` + If you do not do this then rsync will fail to connect and thus fail to archive your clips. # Step 2: Exports -To be able to configure the teslausb pi use rsync, you'll need to export a few things. On your teslausb pi, run: +Run this command to cause the setup processes which you'll resume in the main instructions to use rsync: ``` export RSYNC_ENABLE=true @@ -23,21 +35,17 @@ export RSYNC_USER= export RSYNC_SERVER= export RSYNC_PATH= ``` - Explanations for each: * `RSYNC_ENABLE`: `true` for enabling rsync * `RSYNC_USER`: The user on the FTP server * `RSYNC_SERVER`: The IP address/hostname of the destination machine * `RSYNC_PATH`: The path on the destination machine where the files will be saved -An example (of my) config is listed below: - +An example config is below: ``` export RSYNC_ENABLE=true export RSYNC_USER=pi export RSYNC_SERVER=192.168.1.254 export RSYNC_PATH=/mnt/PIHDD/TeslaCam/ ``` -***Note: RSYNC_ENABLE=true is going to disable the default archive server. Perhaps future releases will allow both to be defined and function at the same time, for redundancy, but for now just pick one that you'll want the most.*** - -You should be ready to run the setup script now, so return back to step 8 of the [Main Instructions](README.md). \ No newline at end of file +Now return to the section "Set up the USB storage functionality" in the [main instructions](../README.md). \ No newline at end of file diff --git a/doc/SetupShare.md b/doc/SetupShare.md new file mode 100644 index 0000000..cee21aa --- /dev/null +++ b/doc/SetupShare.md @@ -0,0 +1,32 @@ +# Hosting the archive on Windows File Shares, MacOS Sharing, or Samba on Linux +Set up a share to host the archive on a computer on your home network. These instructions assume that you created a share named "SailfishCam" on the server "Nautilus". + +It is recommended that you create a new user. Grant the user you'll be using read/write access to the share. These instructions will assume that the user you've created is named "sailfish" and that the password for this user is "pa$$w0rd". + +Now, on the Pi: +1. Try to ping your archive server from the Pi. In this example the server is named `nautilus`. + ``` + ping -c 3 nautilus + ``` +1. If the server can't be reached, ping its IP address (These instructions will assume that the IP address of the archive server is `192.168.0.41`.): + ``` + ping 192.168.0.41 + ``` + To get the IP address of the archive machine: + * On Windows: Open a PowerShell session and type `ipconfig`. Get the IP address from the line labeled "IPv4 Address". + * On MacOS or Linux open a terminal and type ifconfig. + + If you can't ping the archive server by IP address from the Pi, go do whatever you need to on your network to fix that. + + If you can't reach the archive server by name but you can by IP address then use its IP address, below, for the `archiveserver` variable. + +1. Run these commands, subsituting your values: + ``` + sudo -i + export archiveserver="Nautilus" + export sharename="SailfishCam" + export shareuser="sailfish" + export sharepassword="pa$$w0rd" + ``` + +Now return to the section "Set up the USB storage functionality" in the [main instructions](/README.md). \ No newline at end of file