From 9e4f612ce63d59ca8b2ca946aae4e5774c91f20e Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Wed, 1 Apr 2020 02:00:13 +0530 Subject: [PATCH 1/4] Use drive api v3 | Sub folder upload | Improve logging a lot.. upload.sh * Use drive api v3. * Sub folder upload ( heirarchially ). * Improve logging a lot. * Add 2 new commands: -D | --debug - Display script command trace. -V | --verbose-progress - Display detailed message and detailed upload progress. * Improve root folder checking. * Merge url_escape.sed contents. * Passes shellcheck now. * Format script using shmft. shfmt -ci -sr -i 4 -bn -w *sh * Remove some useless checks README.md * Update accoding to changes. * Improve script. google-oauth2.sh * Use drive v3 api --- README.md | 72 ++--- google-oauth2.sh | 107 +++++--- upload.sh | 665 +++++++++++++++++++++++++++++++---------------- 3 files changed, 553 insertions(+), 291 deletions(-) mode change 100644 => 100755 README.md diff --git a/README.md b/README.md old mode 100644 new mode 100755 index a43db4b..e02f006 --- a/README.md +++ b/README.md @@ -1,44 +1,56 @@ -**Google drive upload** -------------------- -Google drive upload is a Bash scripts based on v2 google APIs to upload files/directories into google drive. This is a minimalistic shell script which utilizes google OAuth2.0 device flow to generate access tokens to authorize application to upload files to your google drive. +# Google drive upload + +Google drive upload is a Bash script based on v3 google APIs to upload files/directories into google drive. This is a minimalistic shell script which utilizes google OAuth2.0 device flow to generate access tokens to authorize application to upload files to your google drive. Further usage documentation can be found at my blog page [Labbots.com](https://labbots.com/google-drive-upload-bash-script/ "Labbots.com"). -**Dependencies** ----------------- +## Dependencies + This script does not have very many dependencies. Most of the dependencies are available by default in most linux platforms. This script requires the following packages - - Curl - - sed (Stream editor) - - find command - - awk - - getopt +- Curl +- sed (Stream editor) +- find command +- awk +- getopt + +## Features + +- No dependencies at all. +- Upload files and folders. +- Upload sub-folders and content inside it hierarchically. +- Config file support ( easy to use script on multiple machines ). +- Uses latest gdrive v3 api. +- Pretty Logging. -**Usage** ------ -When the script is executed for the first time. It asks for few configuration variables interactively to connect with google APIs. The script requires Client Id and Client secret to access the APIs which can be generated at [google console]. -Script also asks for root folder to be set to which the script uploads documents by default. The default folder will be the root of your google drive. If you want to upload documents to any specific directory by default then provide the google folder id of that folder which root folder id is requested. The google folder id can be found from the URL of the google drive folder. +## Usage -For example ->https://drive.google.com/drive/folders/8ZzWiO_pMAtBrLpARBZJNV09TVmM +When the script is executed for the first time. It asks for few configuration variables interactively to connect with google APIs. The script requires Client ID and Client secret to access the APIs which can be generated at [google console]. +Script also asks for root folder to be set to which the script uploads documents by default. The default folder will be the root of your google drive. If you want to upload documents to any specific directory by default then provide the google folder ID or URL of that folder when root folder ID/URL is requested. + +For example +`https://drive.google.com/drive/folders/8ZzWiO_pMAtBrLpARBZJNV09TVmM` In the above URL. The folder id is ***8ZzWiO_pMAtBrLpARBZJNV09TVmM*** +Note: You can input either URL or ID. + The default configurations of the script are store under **$HOME/.googledrive.conf** The script can be used in the following way - ./upload.sh + ./upload.sh Above command will create a folder under the pre-configured root directory and upload the specified file under the folder name. If the folder already exists then the file is uploaded under the folder. Other Options available are -C | --create-dir - option to create directory. Will provide folder id. - -r | --root-dir - google folder id to which the file/directory to upload. - -v | --verbose - Display detailed message. - -h | --help - Display usage instructions. - -z | --config - Override default config file with custom config file. - -g | --globoff - Turn off URL globbing parser to allow filenames with literal [] and/or {}. + -r | --root-dir - google folder id to which the file/directory to upload. + -v | --verbose - Display detailed message. + -V | --verbose-progress - Display detailed message and detailed upload progress( curl normal progress info ). + -h | --help - Display usage instructions. + -z | --config - Override default config file with custom config file. + -D | --debug - Display script command trace." To create a folder: @@ -47,20 +59,18 @@ This will give the folder id of the newly created folder which can be used to up To Upload file to specific google folder - ./upload.sh -v -r + ./upload.sh -r -The script also allows to upload directories. If directory path is provided as argument to the script then the script recursively uploads all the files in the folder to google drive. +The script also allows to upload directories. If directory path is provided as argument to the script then the script recursively uploads all the sub-folder and files present in the heirarchial way as it is present on the local machine. +## Inspired By -**Inspired By** ----- -* [github-bashutils] - soulseekah/bash-utils -* [deanet-gist] - Uploading File into Google Drive +- [github-bashutils] - soulseekah/bash-utils +- [deanet-gist] - Uploading File into Google Drive -**License** ----- -MIT +## License +MIT [github-bashutils]: [deanet-gist]: diff --git a/google-oauth2.sh b/google-oauth2.sh index bc07304..eb73cf9 100755 --- a/google-oauth2.sh +++ b/google-oauth2.sh @@ -1,66 +1,101 @@ -#!/bin/bash +#!/usr/bin/env bash -# A simple cURL OAuth2 authenticator +# A simple curl OAuth2 authenticator # # Usage: # ./google-oauth2.sh create - authenticates a user # ./google-oauth2.sh refresh - gets a new token # # Set CLIENT_ID and CLIENT_SECRET and SCOPE +# See SCOPES at https://developers.google.com/identity/protocols/oauth2/scopes#docsv1 -#!/bin/bashset -e +function short_help() { + echo -e "\nNo valid arguments provided." + echo -e "Usage:\n" + echo -e " ./google-oauth2.sh create - authenticates a user." + echo -e " ./google-oauth2.sh refresh - gets a new access token." + exit 0 +} + +if [ "$#" = "0" ]; then + short_help + exit 0 +fi + +# Clear nth no. of line to the beginning of the line. +function clear() { + echo -en "\033[""$1""A" + echo -en "\033[2K" +} + +if ! [ "$1" = create ] || [ "$1" = refresh ]; then + short_help +fi + +echo "Starting script.." CLIENT_ID="" CLIENT_SECRET="" -SCOPE=${SCOPE:-"https://docs.google.com/feeds"} +SCOPE="https://www.googleapis.com/auth/drive" +REDIRECT_URI="urn:ietf:wg:oauth:2.0:oob" -if [ -e $HOME/.googledrive.conf ] -then - . $HOME/.googledrive.conf +# shellcheck source=/dev/null +if [ -e "$HOME"/.googledrive.conf ]; then + source "$HOME"/.googledrive.conf fi -if [ -z "$CLIENT_ID" ] -then - read -p "Client ID: " CLIENT_ID +echo "Checking credentials.." + +if [ -z "$CLIENT_ID" ]; then + read -r -p "Client ID: " CLIENT_ID unset token - echo "CLIENT_ID=$CLIENT_ID" >> $HOME/.googledrive.conf + echo "CLIENT_ID=$CLIENT_ID" >> "$HOME"/.googledrive.conf fi -if [ -z "$CLIENT_SECRET" ] -then - read -p "Client Secret: " CLIENT_SECRET +if [ -z "$CLIENT_SECRET" ]; then + read -r -p "Client Secret: " CLIENT_SECRET unset token - echo "CLIENT_SECRET=$CLIENT_SECRET" >> $HOME/.googledrive.conf + echo "CLIENT_SECRET=$CLIENT_SECRET" >> "$HOME"/.googledrive.conf fi +sleep 1 +clear 1 +clear 1 +echo "Required credentials set." +sleep 1 + # Method to extract data from json response function jsonValue() { -KEY=$1 -num=$2 -awk -F"[:,}][^:\/\/]" '{for(i=1;i<=NF;i++){if($i~/\042'$KEY'\042/){print $(i+1)}}}' | tr -d '"' | sed -n ${num}p | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/[,]*$//' + num=$2 + grep \""$1"\" | sed "s/\:/\n/" | grep -v \""$1"\" | sed "s/\"\,//g" | sed 's/["]*$//' | sed 's/[,]*$//' | sed 's/^[ \t]*//' | sed s/\"// | sed -n "${num}"p } if [ "$1" == "create" ]; then - RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/device/code" --data "client_id=$CLIENT_ID&scope=$SCOPE"` - DEVICE_CODE=`echo "$RESPONSE" | jsonValue device_code` - USER_CODE=`echo "$RESPONSE" | jsonValue user_code` - URL=`echo "$RESPONSE" | jsonValue verification_url` - - echo -n "Go to $URL and enter $USER_CODE to grant access to this application. Hit enter when done..." - read + echo "\nVisit the below URL, tap on allow and then enter the code obtained:" + sleep 1 + URL="https://accounts.google.com/o/oauth2/auth?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&scope=$SCOPE&response_type=code&prompt=consent" + echo -e """$URL""\n" + read -r -p "Enter the authorization code: " CODE - RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&code=$DEVICE_CODE&grant_type=http://oauth.net/grant_type/device/1.0"` + CODE="$(echo "$CODE" | tr -d ' ' | tr -d '[:blank:]' | tr -d '[:space:]')" + if [ -n "$CODE" ]; then + RESPONSE="$(curl -s --request POST --data "code=$CODE&client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&redirect_uri=$REDIRECT_URI&grant_type=authorization_code" https://accounts.google.com/o/oauth2/token)" - ACCESS_TOKEN=`echo "$RESPONSE" | jsonValue access_token` - REFRESH_TOKEN=`echo "$RESPONSE" | jsonValue refresh_token` + ACCESS_TOKEN="$(echo "$RESPONSE" | jsonValue access_token)" + REFRESH_TOKEN="$(echo "$RESPONSE" | jsonValue refresh_token)" - echo "Access Token: $ACCESS_TOKEN" - echo "Refresh Token: $REFRESH_TOKEN" + echo "Access Token: $ACCESS_TOKEN" + echo "Refresh Token: $REFRESH_TOKEN" + else + echo -e "\nNo code provided, run the script and try again" + exit 1 + fi elif [ "$1" == "refresh" ]; then - REFRESH_TOKEN=$2 - RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token"` - - ACCESS_TOKEN=`echo $RESPONSE | jsonValue access_token` - - echo "Access Token: $ACCESS_TOKEN" + if [ -n "$REFRESH_TOKEN" ]; then + RESPONSE="$(curl -s --request POST --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token" https://accounts.google.com/o/oauth2/token)" + ACCESS_TOKEN="$(echo "$RESPONSE" | jsonValue access_token)" + echo "Access Token: $ACCESS_TOKEN" + else + echo "Refresh Token not set, use $0 create to generate one." + fi fi diff --git a/upload.sh b/upload.sh index 8a622b0..8ee1ec0 100755 --- a/upload.sh +++ b/upload.sh @@ -1,23 +1,26 @@ -#!/bin/bash - +#!/usr/bin/env bash # Upload a file to Google Drive # Usage: upload.sh -#!/bin/bashset -e - -function usage(){ - - echo -e "\nThe script can be used to upload file/directory to google drive." - echo -e "\nUsage:\n $0 [options..] \n" - echo -e "Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive. \n" - echo -e "File name argument is optional if create directory option is used. \n" - echo -e "Options:\n" - echo -e "-C | --create-dir - option to create directory. Will provide folder id." - echo -e "-r | --root-dir - google folder id to which the file/directory to upload." - echo -e "-v | --verbose - Display detailed message." - echo -e "-z | --config - Override default config file with custom config file." - echo -e "-h | --help - Display usage instructions.\n" - exit 0; +function usage() { + echo -e "\nThe script can be used to upload file/directory to google drive." + echo -e "\nUsage:\n $0 [options..] \n" + echo -e "Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive. \n" + echo -e "File name argument is optional if create directory option is used. \n" + echo -e "Options:\n" + echo -e "-C | --create-dir - option to create directory. Will provide folder id." + echo -e "-r | --root-dir or - google folder ID/URL to which the file/directory is going to upload." + echo -e "-v | --verbose - Display detailed message." + echo -e "-V | --verbose-progress - Display detailed message and detailed upload progress." + echo -e "-z | --config - Override default config file with custom config file." + echo -e "-D | --debug - Display script command trace." + echo -e "-h | --help - Display usage instructions.\n" + exit 0 +} + +function short_help() { + echo -e "\nNo valid arguments provided, use -h/--help flag to see usage." + exit 0 } #Configuration variables @@ -25,27 +28,30 @@ ROOT_FOLDER="" CLIENT_ID="" CLIENT_SECRET="" REFRESH_TOKEN="" -SCOPE=${SCOPE:-"https://docs.google.com/feeds"} +SCOPE="https://www.googleapis.com/auth/drive" +REDIRECT_URI="urn:ietf:wg:oauth:2.0:oob" #Internal variable ACCESS_TOKEN="" -FILE="" +INPUT="" FOLDERNAME="" -curl_args="" +curl_args="-#" -DIR="$( cd "$( dirname "$( readlink "${BASH_SOURCE[0]}" )" )" && pwd )" +DIR="$(pwd)" +STRING="$RANDOM" -if [ -e $HOME/.googledrive.conf ] -then - . $HOME/.googledrive.conf +# shellcheck source=/dev/null +# Config file is created automatically after first run +if [ -e "$HOME"/.googledrive.conf ]; then + source "$HOME"/.googledrive.conf fi PROGNAME=${0##*/} -SHORTOPTS="vhr:C:z:" -LONGOPTS="verbose,help,create-dir:,root-dir:,config:" +SHORTOPTS="v,V,hr:C:D,z:" +LONGOPTS="verbose,verbose-progress,progress-bar,help,create-dir:,root-dir:,debug,config:" -set -o errexit -o noclobber -o pipefail #-o nounset -OPTS=$(getopt -s bash --options $SHORTOPTS --longoptions $LONGOPTS --name $PROGNAME -- "$@" ) +set -o errexit -o noclobber -o pipefail #-o nounset +OPTS=$(getopt -s bash --options $SHORTOPTS --longoptions $LONGOPTS --name "$PROGNAME" -- "$@") # script to parse the input arguments #if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi @@ -53,236 +59,447 @@ OPTS=$(getopt -s bash --options $SHORTOPTS --longoptions $LONGOPTS --name $PROGN eval set -- "$OPTS" VERBOSE=false -HELP=false +VERBOSE_PROGRESS=false +DEBUG=false CONFIG="" ROOTDIR="" while true; do - case "$1" in - -v | --verbose ) VERBOSE=true;curl_args="--progress"; shift ;; - -h | --help ) usage; shift ;; - -C | --create-dir ) FOLDERNAME="$2"; shift 2 ;; - -r | --root-dir ) ROOTDIR="$2";ROOT_FOLDER="$2"; shift 2 ;; - -z | --config ) CONFIG="$2"; shift 2 ;; - -- ) shift; break ;; - * ) break ;; - esac + case "$1" in + -v | --verbose) + VERBOSE=true + shift + ;; + -V | --verbose-progress) + VERBOSE_PROGRESS=true + curl_args="" + shift + ;; + -h | --help) + usage + shift + ;; + -C | --create-dir) + FOLDERNAME="$2" + shift 2 + ;; + -r | --root-dir) + ROOTDIR="$2" + shift 2 + ;; + -z | --config) + CONFIG="$2" + shift 2 + ;; + -D | --debug) + DEBUG=true + shift + ;; + --) + shift + break + ;; + *) break ;; + esac done -if [ ! -z "$CONFIG" ] - then - if [ -e "$CONFIG" ] - then - . $CONFIG - fi - if [ ! -z "$ROOTDIR" ] - then - ROOT_FOLDER="$ROOTDIR" - fi +if [ "$DEBUG" = true ]; then + set -xe +fi + +# shellcheck source=/dev/null +if [ -n "$CONFIG" ]; then + if [ -e "$CONFIG" ]; then + source "$CONFIG" + fi fi -if [ ! -z "$1" ] -then - FILE=$1 +if [ -n "$1" ]; then + input="$1" fi -if [ ! -z "$2" ] && [ -z "$FOLDERNAME" ] -then - FOLDERNAME=$2 +if [ -n "$2" ] && [ -z "$FOLDERNAME" ]; then + FOLDERNAME="$2" fi -if [ "$#" = "0" ] && [ -z "$FOLDERNAME" ] - then - usage +if [ "$#" = "0" ] && [ -z "$FOLDERNAME" ]; then + short_help + exit 0 fi +echo "Starting script.." + +if [ -n "$VERBOSE_PROGRESS" ]; then + if [ -n "$VERBOSE" ]; then + unset "$VERBOSE" + fi +fi + +# Extract file/folder ID from the given input in case of gdrive URL. +extractID() { + ID="$1" + case "$ID" in + 'http'*'://'*'drive.google.com'*'id='*) ID=$(echo "$ID" | sed -e 's/^.*id=//' -e 's|&|\n|' | head -1) ;; + 'http'*'drive.google.com'*'file/d/'* | 'http'*'docs.google.com/file/d/'*) ID=$(echo "$ID" | sed -e's/^.*\/d\///' -e 's/\/.*//') ;; + 'http'*'drive.google.com'*'drive'*'folders'*) ID=$(echo "$ID" | sed -e 's/^.*\/folders\///' -e "s/&.*//" -e -r 's/(.*)\/.*/\1 /') ;; + esac + echo "$ID" +} + +# Clear nth no. of line to the beginning of the line. +function clear() { + echo -en "\033[""$1""A" + echo -en "\033[2K" +} + # Method to extract data from json response function jsonValue() { -KEY=$1 -num=$2 -awk -F"[,:}][^://]" '{for(i=1;i<=NF;i++){if($i~/\042'$KEY'\042/){print $(i+1)}}}' | tr -d '"' | sed -n ${num}p | sed -e 's/[}]*$//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/[,]*$//' + num=$2 + grep \""$1"\" | sed "s/\:/\n/" | grep -v \""$1"\" | sed "s/\"\,//g" | sed 's/["]*$//' | sed 's/[,]*$//' | sed 's/^[ \t]*//' | sed s/\"// | sed -n "${num}"p } -function log() { - - if [ "$VERBOSE" = true ]; then - echo -e "${1}" - - fi +# sed url escaping +function urlEscape() { + sed 's|%|%25|g' \ + | sed 's| |%20|g' \ + | sed 's|<|%3C|g' \ + | sed 's|>|%3E|g' \ + | sed 's|#|%23|g' \ + | sed 's|{|%7B|g' \ + | sed 's|}|%7D|g' \ + | sed 's|\||%7C|g' \ + | sed 's|\\|%5C|g' \ + | sed 's|\^|%5E|g' \ + | sed 's|~|%7E|g' \ + | sed 's|\[|%5B|g' \ + | sed 's|\]|%5D|g' \ + | sed 's|`|%60|g' \ + | sed 's|;|%3B|g' \ + | sed 's|/|%2F|g' \ + | sed 's|?|%3F|g' \ + | sed 's^|^%3A^g' \ + | sed 's|@|%40|g' \ + | sed 's|=|%3D|g' \ + | sed 's|&|%26|g' \ + | sed 's|\$|%24|g' \ + | sed 's|\!|%21|g' \ + | sed 's|\*|%2A|g' } -# Method to create directory in google drive. Requires 3 arguments foldername,root directory id and access token. -function createDirectory(){ - DIRNAME="$1" - ROOTDIR="$2" - ACCESS_TOKEN="$3" - FOLDER_ID="" - QUERY="mimeType='application/vnd.google-apps.folder' and title='$DIRNAME'" - QUERY=$(echo $QUERY | sed -f ${DIR}/url_escape.sed) - - SEARCH_RESPONSE=`/usr/bin/curl \ - --silent \ - -XGET \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "https://www.googleapis.com/drive/v2/files/${ROOTDIR}/children?orderBy=title&q=${QUERY}&fields=items%2Fid"` - - FOLDER_ID=`echo $SEARCH_RESPONSE | jsonValue id` - - - if [ -z "$FOLDER_ID" ] - then - CREATE_FOLDER_POST_DATA="{\"mimeType\": \"application/vnd.google-apps.folder\",\"title\": \"$DIRNAME\",\"parents\": [{\"id\": \"$ROOTDIR\"}]}" - CREATE_FOLDER_RESPONSE=`/usr/bin/curl \ - --silent \ - -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H "Content-Type: application/json; charset=UTF-8" \ - -d "$CREATE_FOLDER_POST_DATA" \ - "https://www.googleapis.com/drive/v2/files?fields=id"` - FOLDER_ID=`echo $CREATE_FOLDER_RESPONSE | jsonValue id` - - fi - echo "$FOLDER_ID" +# Method to get information for a gdrive folder/file. +# Requirements: Given file/folder ID, query, and access_token. +function drive_Info() { + FOLDER_ID="$1" + FETCH="$2" + ACCESS_TOKEN="$3" + SEARCH_RESPONSE="$(curl \ + --silent \ + -XGET \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "https://www.googleapis.com/drive/v3/files/""$FOLDER_ID""?fields=""$FETCH""")" + FETCHED_DATA="$(echo "$SEARCH_RESPONSE" | jsonValue "$FETCH" | head -1)" + echo "$FETCHED_DATA" } -# Method to upload files to google drive. Requires 3 arguments file path, google folder id and access token. -function uploadFile(){ - - FILE="$1" - FOLDER_ID="$2" - ACCESS_TOKEN="$3" - SLUG=`basename "$FILE"` - FILENAME="${SLUG%.*}" - EXTENSION="${SLUG##*.}" - if [ "$FILENAME" == "$EXTENSION" -o ! "$(command -v mimetype)" ] - then - MIME_TYPE=`file --brief --mime-type "$FILE"` - else - MIME_TYPE=`mimetype --output-format %m "$FILE"` - - fi - - - FILESIZE=$(stat -c%s "$FILE") - - # JSON post data to specify the file name and folder under while the file to be created - postData="{\"mimeType\": \"$MIME_TYPE\",\"title\": \"$SLUG\",\"parents\": [{\"id\": \"$FOLDER_ID\"}]}" - postDataSize=$(echo $postData | wc -c) - - # Curl command to initiate resumable upload session and grab the location URL - log "Generating upload link for file $FILE ..." - uploadlink=`/usr/bin/curl \ - --silent \ - -X POST \ - -H "Host: www.googleapis.com" \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H "Content-Type: application/json; charset=UTF-8" \ - -H "X-Upload-Content-Type: $MIME_TYPE" \ - -H "X-Upload-Content-Length: $FILESIZE" \ - -d "$postData" \ - "https://www.googleapis.com/upload/drive/v2/files?uploadType=resumable&supportsAllDrives=true&supportsTeamDrives=true" \ - --dump-header - | sed -ne s/"Location: "//pi | tr -d '\r\n'` - - # Curl command to push the file to google drive. - # If the file size is large then the content can be split to chunks and uploaded. - # In that case content range needs to be specified. - log "Uploading file $FILE to google drive..." - curl \ - -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H "Content-Type: $MIME_TYPE" \ - -H "Content-Length: $FILESIZE" \ - -H "Slug: $SLUG" \ - --upload-file "$FILE" \ - --output /dev/null \ - "$uploadlink" \ - $curl_args +# Method to create directory in google drive. +# Requirements: Foldername, Root folder ID ( the folder in which the new folder will be created ) and access_token. +# First check if a folder exist in given parent directory, if not the case then make the folder. +# Atlast print folder ID ( existing or new one ). +function createDirectory() { + DIRNAME="$1" + ROOTDIR="$2" + ACCESS_TOKEN="$3" + FOLDER_ID="" + QUERY="mimeType='application/vnd.google-apps.folder' and name='$DIRNAME' and trashed=false and '$ROOTDIR' in parents" + QUERY="$(echo "$QUERY" | urlEscape)" + + SEARCH_RESPONSE="$(curl \ + --silent \ + -XGET \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "https://www.googleapis.com/drive/v3/files?q=${QUERY}&fields=files(id)")" + FOLDER_ID="$(echo "$SEARCH_RESPONSE" | jsonValue id | head -1)" + if [ -z "$FOLDER_ID" ]; then + CREATE_FOLDER_POST_DATA="{\"mimeType\": \"application/vnd.google-apps.folder\",\"name\": \"$DIRNAME\",\"parents\": [\"$ROOTDIR\"]}" + CREATE_FOLDER_RESPONSE="$(curl \ + --silent \ + -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json; charset=UTF-8" \ + -d "$CREATE_FOLDER_POST_DATA" \ + "https://www.googleapis.com/drive/v3/files?fields=id")" + FOLDER_ID="$(echo "$CREATE_FOLDER_RESPONSE" | jsonValue id)" + fi + echo "$FOLDER_ID" } -old_umask=`umask` -umask 0077 - -if [ -z "$ROOT_FOLDER" ] -then - read -p "Root Folder ID (Default: root): " ROOT_FOLDER - if [ -z "$ROOT_FOLDER" ] || [ `echo $ROOT_FOLDER | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ] - then - ROOT_FOLDER="root" - echo "ROOT_FOLDER=$ROOT_FOLDER" >> $HOME/.googledrive.conf - else - if expr "$ROOT_FOLDER" : '^[A-Za-z0-9_-]\{28\}$' > /dev/null - then - echo "ROOT_FOLDER=$ROOT_FOLDER" >> $HOME/.googledrive.conf - else - echo "Invalid root folder id" - exit -1 - fi - fi -fi +# Method to upload files to google drive. +# Requirements: Given file path, Google folder ID and access_token. +function uploadFile() { + INPUT="$1" + FOLDER_ID="$2" + ACCESS_TOKEN="$3" + SLUG="$(basename "$INPUT")" + INPUTNAME="${SLUG%.*}" + EXTENSION="${SLUG##*.}" + INPUTSIZE="$(stat -c%s "$INPUT")" + READABLE_SIZE="$(du -sh "$INPUT" | awk '{print $1;}')" + #info "File: ""${INPUT//$DIR\//}""" + echo "File: ""$(basename "$INPUT")"" ""$READABLE_SIZE""" + if [[ "$INPUTNAME" == "$EXTENSION" ]]; then + if command -v mimetype > /dev/null 2>&1; then + MIME_TYPE="$(mimetype --output-format %m "$INPUT")" + elif command -v file > /dev/null 2>&1; then + MIME_TYPE="$(file --brief --mime-type "$INPUT")" + else + echo -e "\nError: file or mimetype command not found." + exit 1 + fi + + fi + + # JSON post data to specify the file name and folder under while the file to be created + postData="{\"mimeType\": \"$MIME_TYPE\",\"name\": \"$SLUG\",\"parents\": [\"$FOLDER_ID\"]}" + + # Curl command to initiate resumable upload session and grab the location URL + echo "Generating upload link..." + uploadlink="$(curl \ + --silent \ + -X POST \ + -H "Host: www.googleapis.com" \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json; charset=UTF-8" \ + -H "X-Upload-Content-Type: $MIME_TYPE" \ + -H "X-Upload-Content-Length: $INPUTSIZE" \ + -d "$postData" \ + "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true&supportsTeamDrives=true" \ + --dump-header - | sed -ne s/"Location: "//pi | tr -d '\r\n')" + + if [ -n "$uploadlink" ]; then + # Curl command to push the file to google drive. + # If the file size is large then the content can be split to chunks and uploaded. + # In that case content range needs to be specified. + clear 1 + echo "Uploading..." + if [ -n "$curl_args" ]; then + curl \ + -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: $MIME_TYPE" \ + -H "Content-Length: $INPUTSIZE" \ + -H "Slug: $SLUG" \ + -T "$INPUT" \ + --output /dev/null \ + --url "$uploadlink" \ + "$curl_args" + else + curl \ + -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: $MIME_TYPE" \ + -H "Content-Length: $INPUTSIZE" \ + -H "Slug: $SLUG" \ + -T "$INPUT" \ + --output /dev/null \ + --url "$uploadlink" + fi + + if [ "$VERBOSE_PROGRESS" = true ]; then + echo "File: ""$INPUTNAME"" (""$READABLE_SIZE"") Uploaded" + else + clear 1 + clear 1 + clear 1 + echo "File: ""$INPUTNAME"" (""$READABLE_SIZE"") Uploaded" + fi + else + echo "Upload link generation error, file not uploaded." + fi +} -if [ -z "$CLIENT_ID" ] -then - read -p "Client ID: " CLIENT_ID - echo "CLIENT_ID=$CLIENT_ID" >> $HOME/.googledrive.conf +echo "Checking credentials.." +# Credentials +if [ -z "$CLIENT_ID" ]; then + read -r -p "Client ID: " CLIENT_ID + echo "CLIENT_ID=$CLIENT_ID" >> "$HOME"/.googledrive.conf fi - -if [ -z "$CLIENT_SECRET" ] -then - read -p "Client Secret: " CLIENT_SECRET - echo "CLIENT_SECRET=$CLIENT_SECRET" >> $HOME/.googledrive.conf +if [ -z "$CLIENT_SECRET" ]; then + read -r -p "Client Secret: " CLIENT_SECRET + echo "CLIENT_SECRET=$CLIENT_SECRET" >> "$HOME"/.googledrive.conf fi -if [ -z "$REFRESH_TOKEN" ] -then - RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/device/code" --data "client_id=$CLIENT_ID&scope=$SCOPE"` - DEVICE_CODE=`echo "$RESPONSE" | jsonValue "device_code"` - USER_CODE=`echo "$RESPONSE" | jsonValue "user_code"` - URL=`echo "$RESPONSE" | jsonValue "verification_url"` - - echo -n "Go to $URL and enter $USER_CODE to grant access to this application. Hit enter when done..." - read - - RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&code=$DEVICE_CODE&grant_type=http://oauth.net/grant_type/device/1.0"` - - ACCESS_TOKEN=`echo "$RESPONSE" | jsonValue access_token` - REFRESH_TOKEN=`echo "$RESPONSE" | jsonValue refresh_token` +# Method to obtain refresh_token. +# Requirements: client_id, client_secret and authorization code. +if [ -z "$REFRESH_TOKEN" ]; then + read -r -p "If you have a refresh token generated, then type the token, else leave blank and press return key.. + Refresh Token: " REFRESH_TOKEN + REFRESH_TOKEN="$(echo "$REFRESH_TOKEN" | tr -d ' ' | tr -d '[:blank:]' | tr -d '[:space:]')" + if [ -n "$REFRESH_TOKEN" ]; then + echo "REFRESH_TOKEN=$REFRESH_TOKEN" >> "$HOME"/.googledrive.conf + else + echo -e "\nVisit the below URL, tap on allow and then enter the code obtained:" + URL="https://accounts.google.com/o/oauth2/auth?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&scope=$SCOPE&response_type=code&prompt=consent" + echo -e """$URL""\n" + read -r -p "Enter the authorization code: " CODE + CODE="$(echo "$CODE" | tr -d ' ' | tr -d '[:blank:]' | tr -d '[:space:]')" + if [ -n "$CODE" ]; then + RESPONSE="$(curl -s --request POST --data "code=$CODE&client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&redirect_uri=$REDIRECT_URI&grant_type=authorization_code" https://accounts.google.com/o/oauth2/token)" + + ACCESS_TOKEN="$(echo "$RESPONSE" | jsonValue access_token)" + REFRESH_TOKEN="$(echo "$RESPONSE" | jsonValue refresh_token)" + + echo "REFRESH_TOKEN=""$REFRESH_TOKEN""" >> "$HOME"/.googledrive.conf + else + echo -e "\nNo code provided, run the script and try again" + exit 1 + fi + fi +fi - echo "REFRESH_TOKEN=$REFRESH_TOKEN" >> $HOME/.googledrive.conf +# Method to regenerate access_token. +# Make a request on https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=$ACCESS_TOKEN url and check if the given token is valid, if not generate one. +# Requirements: Refresh Token +if [ -z "$ACCESS_TOKEN" ]; then + RESPONSE="$(curl -s --request POST --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token" https://accounts.google.com/o/oauth2/token)" + ACCESS_TOKEN="$(echo "$RESPONSE" | jsonValue access_token)" +elif curl -s "https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=$ACCESS_TOKEN" | jsonValue error > /dev/null 2>&1; then + RESPONSE="$(curl -s --request POST --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token" https://accounts.google.com/o/oauth2/token)" + ACCESS_TOKEN="$(echo "$RESPONSE" | jsonValue access_token)" fi -if [ -z "$ACCESS_TOKEN" ] - then - # Access token generation - RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token"` - ACCESS_TOKEN=`echo $RESPONSE | jsonValue access_token` +clear 1 +clear 1 +echo "Required credentials available." +echo "Checking root dir and workspace folder.." +# Setup root directory where all file/folders will be uploaded. +if [ -n "$ROOTDIR" ]; then + ROOT_FOLDER="$(echo "$ROOTDIR" | tr -d ' ' | tr -d '[:blank:]' | tr -d '[:space:]')" + if [ -n "$ROOT_FOLDER" ]; then + ROOT_FOLDER="$(drive_Info "$(extractID "$ROOT_FOLDER")" "id" "$ACCESS_TOKEN")" + if [ -n "$ROOT_FOLDER" ]; then + ROOT_FOLDER="$ROOT_FOLDER" + echo "ROOT_FOLDER=$ROOT_FOLDER" >> "$HOME"/.googledrive.conf + else + echo "Given root folder ID/URL invalid." + exit 1 + fi + fi +elif [ -z "$ROOT_FOLDER" ]; then + read -r -p "Root Folder ID or URL (Default: root): " ROOT_FOLDER + ROOT_FOLDER="$(echo "$ROOT_FOLDER" | tr -d ' ' | tr -d '[:blank:]' | tr -d '[:space:]')" + if [ -n "$ROOT_FOLDER" ]; then + ROOT_FOLDER="$(folderInfo "$(extractID "$ROOT_FOLDER")" "$ACCESS_TOKEN")" + if [ -n "$ROOT_FOLDER" ]; then + ROOT_FOLDER="$ROOT_FOLDER" + echo "ROOT_FOLDER=$ROOT_FOLDER" >> "$HOME"/.googledrive.conf + else + echo "Given root folder ID/URL invalid." + exit 1 + fi + else + ROOT_FOLDER="root" + echo "ROOT_FOLDER=$ROOT_FOLDER" >> "$HOME"/.googledrive.conf + fi fi -# Check to find whether the folder exists in google drive. If not then the folder is created in google drive under the configured root folder -if [ -z "$FOLDERNAME" ] || [[ `echo "$FOLDERNAME" | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ]] - then - if [[ `echo "$FOLDERNAME" | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ]] - then - ROOT_FOLDER="root" - fi - FOLDER_ID=$ROOT_FOLDER +clear 1 +clear 1 +echo "Root dir properly configured." +# Check to find whether the folder exists in google drive. If not then the folder is created in google drive under the configured root folder. +if [ -z "$FOLDERNAME" ]; then + ROOT_FOLDER_ID=$ROOT_FOLDER else - FOLDER_ID=`createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN"` - + ROOT_FOLDER_ID="$(createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN")" +fi +ROOT_FOLDER_NAME="$(drive_Info """$ROOT_FOLDER_ID""" name """$ACCESS_TOKEN""")" +clear 1 +echo "Workspace Folder: $ROOT_FOLDER_NAME ($ROOT_FOLDER_ID)" + +START=$(date +"%s") + +# Check if the argument is a file or a directory. +# In case of file, just upload it. +# In case of folder, do a recursive upload in the same hierarchical manner present. +if [ -n "$input" ]; then + if [ -f "$input" ]; then + echo -e "Given Input: File\n" + uploadFile "$input" "$ROOT_FOLDER_ID" "$ACCESS_TOKEN" + echo + elif [ -d "$input" ]; then + echo "Given Input: Folder" + NEXTROOTDIRID="$ROOT_FOLDER_ID" + # Do not create empty folders during a recursive upload. + # The use of find in this section is important. + # If below command is used, it lists the folder in stair structure, which we later assume while creating sub folders and uploading files. + find "$input" -type d -not -empty | sed "s|$input|$DIR/$input|" > "$STRING"DIRNAMES + # Create a loop and make folders according to list made above. + if [ -n "$(sed '1d' "$STRING"DIRNAMES)" ]; then + echo "Indexing sub-directories..." + fi + + while IFS= read -r dir; do + newdir="$(basename "$dir")" + ID="$(createDirectory "$newdir" "$NEXTROOTDIRID" "$ACCESS_TOKEN")" + # Store sub-folder directory IDs and it's path for later use. + echo "$ID '$dir'" >> "$STRING"DIRIDS + NEXTROOTDIRID=$ID + done < "$STRING"DIRNAMES + + if [ -n "$(sed '1d' "$STRING"DIRNAMES)" ]; then + clear 1 + echo -e "No. of sub-directories: ""$(sed '1d' "$STRING"DIRNAMES | wc -l)""\n" + fi + + if [ -f "$STRING"DIRNAMES ]; then rm "$STRING"DIRNAMES; fi + echo "Indexing files recursively..." + find "$input" -type f | sed "s|$input|$DIR/$input|" > "$STRING"FILENAMES + clear 1 + clear 1 + echo "No. of Files: ""$(grep -c "" "$STRING"FILENAMES)""" + if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then + echo + else + echo -e "\n" + fi + + # shellcheck disable=SC2001 + # Match the path with sub-folder directory ID and upload accordingly. + while read -r i; do sed "s/\(.*\)\/$(basename "$i")/\1/" <<< "$i"; done < "$STRING"FILENAMES > "$STRING"FILES_ROOTDIR + + while IFS= read -r -u 4 rootdirpath && IFS= read -r -u 5 file; do + dirtoupload="$(grep "'$rootdirpath'" "$STRING"DIRIDS | awk '{print $1;}')" + uploadFile "$file" "$dirtoupload" "$ACCESS_TOKEN" + echo "$file" >> "$STRING"UPLOADED + uploaded="$(grep -c "" "$STRING"UPLOADED)" + if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then + echo -e "Status: $uploaded files uploaded.\n" + else + clear 1 + clear 1 + echo "Status: $uploaded files uploaded." + fi + done 4< "$STRING"FILES_ROOTDIR 5< "$STRING"FILENAMES + + if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then + echo -e "Total Files Uploaded: ""$(grep -c "" "$STRING"FILENAMES)""\n" + else + clear 1 + clear 1 + echo -e "Total Files Uploaded: ""$(grep -c "" "$STRING"FILENAMES)""\n" + fi + + if [ -f "$STRING"DIRIDS ]; then rm "$STRING"DIRIDS; fi + if [ -f "$STRING"FILES_ROOTDIR ]; then rm "$STRING"FILES_ROOTDIR; fi + if [ -f "$STRING"FILENAMES ]; then rm "$STRING"FILENAMES; fi + if [ -f "$STRING"UPLOADED ]; then rm "$STRING"UPLOADED; fi + fi fi - log "Folder ID for folder name $FOLDERNAME : $FOLDER_ID" - -# Check whether the given file argument is valid and check whether the argument is file or directory. -# based on the type, if the argument is directory do a recursive upload. -if [ ! -z "$FILE" ]; then - if [ -f "$FILE" ];then - uploadFile "$FILE" "$FOLDER_ID" "$ACCESS_TOKEN" - elif [ -d "$FILE" ];then - FOLDERNAME=$(basename $FILE) - FOLDER_ID=`createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN"` - for file in $(find "$FILE" -type f); - do - uploadFile "$file" "$FOLDER_ID" "$ACCESS_TOKEN" - done - fi + +END="$(date +"%s")" +DIFF="$((END - START))" +echo "Total time elapsed: "$((DIFF / 60))" minute(s) and "$((DIFF % 60))" seconds" + +if [ "$DEBUG" = true ]; then + set +xe fi From a60fbbf0056fc5e6e68ea90380b5644d87ede364 Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Thu, 2 Apr 2020 23:26:58 +0530 Subject: [PATCH 2/4] Fix https://github.com/labbots/google-drive-upload/issues/17 | More improvents to logging * Some code cleanup. --- README.md | 1 + upload.sh | 209 ++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 166 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index e02f006..d8f9d5c 100755 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ Other Options available are -r | --root-dir - google folder id to which the file/directory to upload. -v | --verbose - Display detailed message. -V | --verbose-progress - Display detailed message and detailed upload progress( curl normal progress info ). + -i | --save-info - Save uploaded files info to the given filename." -h | --help - Display usage instructions. -z | --config - Override default config file with custom config file. -D | --debug - Display script command trace." diff --git a/upload.sh b/upload.sh index 8ee1ec0..cf6cfd0 100755 --- a/upload.sh +++ b/upload.sh @@ -12,6 +12,7 @@ function usage() { echo -e "-r | --root-dir or - google folder ID/URL to which the file/directory is going to upload." echo -e "-v | --verbose - Display detailed message." echo -e "-V | --verbose-progress - Display detailed message and detailed upload progress." + echo -e "-i | --save-info - Save uploaded files info to the given filename." echo -e "-z | --config - Override default config file with custom config file." echo -e "-D | --debug - Display script command trace." echo -e "-h | --help - Display usage instructions.\n" @@ -47,14 +48,14 @@ if [ -e "$HOME"/.googledrive.conf ]; then fi PROGNAME=${0##*/} -SHORTOPTS="v,V,hr:C:D,z:" -LONGOPTS="verbose,verbose-progress,progress-bar,help,create-dir:,root-dir:,debug,config:" +SHORTOPTS="v,V,i:,hr:C:D,z:" +LONGOPTS="verbose,verbose-progress,save-info:,help,create-dir:,root-dir:,debug,config:" set -o errexit -o noclobber -o pipefail #-o nounset OPTS=$(getopt -s bash --options $SHORTOPTS --longoptions $LONGOPTS --name "$PROGNAME" -- "$@") # script to parse the input arguments -#if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi +#if [ $? != 0] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi eval set -- "$OPTS" @@ -75,6 +76,10 @@ while true; do curl_args="" shift ;; + -i | --save-info) + LOG_FILE_ID="$2" + shift 2 + ;; -h | --help) usage shift @@ -112,7 +117,6 @@ if [ -n "$CONFIG" ]; then if [ -e "$CONFIG" ]; then source "$CONFIG" fi - fi if [ -n "$1" ]; then @@ -128,7 +132,77 @@ if [ "$#" = "0" ] && [ -z "$FOLDERNAME" ]; then exit 0 fi -echo "Starting script.." +# This refreshes the interactive shell so we can use the $COLUMNS variable. +cat /dev/null + +# https://gist.github.com/TrinityCoder/911059c83e5f7a351b785921cf7ecdaa +if [ "$DEBUG" = true ]; then + # To avoid spamming in debug mode. + function print_center() { + echo -e "${1}" + } + function print_center_justify() { + echo -e "${1}" + } +else + function print_center() { + [[ $# == 0 ]] && return 1 + declare -i TERM_COLS="$COLUMNS" + + out="$1" + + declare -i str_len=${#out} + [[ $str_len -ge $TERM_COLS ]] && { + echo "$out" + return 0 + } + + declare -i filler_len="$(((TERM_COLS - str_len) / 2))" + [[ $# -ge 2 ]] && ch="${2:0:1}" || ch=" " + filler="" + for ((i = 0; i < filler_len; i++)); do + filler="${filler}${ch}" + done + + printf "%s%s%s" "$filler" "$out" "$filler" + [[ $(((TERM_COLS - str_len) % 2)) -ne 0 ]] && printf "%s" "${ch}" + printf "\n" + + return 0 + } + # To avoid entering a new line, and maintaining the output flow. + function print_center_justify() { + [[ $# == 0 ]] && return 1 + declare -i TERM_COLS="$COLUMNS" + + out="$1" + + TO_PRINT="$((TERM_COLS * 98 / 100))" + if [ "${#1}" -gt "$TO_PRINT" ]; then + out="${1:0:$TO_PRINT}.." + fi + + declare -i str_len=${#out} + [[ $str_len -ge $TERM_COLS ]] && { + echo "$out" + return 0 + } + + declare -i filler_len="$(((TERM_COLS - str_len) / 2))" + [[ $# -ge 2 ]] && ch="${2:0:1}" || ch=" " + filler="" + for ((i = 0; i < filler_len; i++)); do + filler="${filler}${ch}" + done + + printf "%s%s%s" "$filler" "$out" "$filler" + [[ $(((TERM_COLS - str_len) % 2)) -ne 0 ]] && printf "%s" "${ch}" + printf "\n" + + return 0 + } +fi +print_center " Starting script " "=" if [ -n "$VERBOSE_PROGRESS" ]; then if [ -n "$VERBOSE" ]; then @@ -245,8 +319,7 @@ function uploadFile() { EXTENSION="${SLUG##*.}" INPUTSIZE="$(stat -c%s "$INPUT")" READABLE_SIZE="$(du -sh "$INPUT" | awk '{print $1;}')" - #info "File: ""${INPUT//$DIR\//}""" - echo "File: ""$(basename "$INPUT")"" ""$READABLE_SIZE""" + print_center_justify " File: ""$(basename "$INPUT")"" | ""$READABLE_SIZE"" " "=" if [[ "$INPUTNAME" == "$EXTENSION" ]]; then if command -v mimetype > /dev/null 2>&1; then MIME_TYPE="$(mimetype --output-format %m "$INPUT")" @@ -263,7 +336,7 @@ function uploadFile() { postData="{\"mimeType\": \"$MIME_TYPE\",\"name\": \"$SLUG\",\"parents\": [\"$FOLDER_ID\"]}" # Curl command to initiate resumable upload session and grab the location URL - echo "Generating upload link..." + print_center " Generating upload link... " "=" uploadlink="$(curl \ --silent \ -X POST \ @@ -281,7 +354,7 @@ function uploadFile() { # If the file size is large then the content can be split to chunks and uploaded. # In that case content range needs to be specified. clear 1 - echo "Uploading..." + print_center " Uploading... " "=" if [ -n "$curl_args" ]; then curl \ -X PUT \ @@ -290,9 +363,9 @@ function uploadFile() { -H "Content-Length: $INPUTSIZE" \ -H "Slug: $SLUG" \ -T "$INPUT" \ - --output /dev/null \ + -o- \ --url "$uploadlink" \ - "$curl_args" + "$curl_args" >> "$STRING" else curl \ -X PUT \ @@ -301,24 +374,40 @@ function uploadFile() { -H "Content-Length: $INPUTSIZE" \ -H "Slug: $SLUG" \ -T "$INPUT" \ - --output /dev/null \ - --url "$uploadlink" + -o- \ + --url "$uploadlink" >> "$STRING" fi + FILE_LINK="$(jsonValue id < "$STRING" | sed 's|^|https://drive.google.com/open?id=|')" + # Log to the filename provided with -i/--save-id flag. + if [ -n "$LOG_FILE_ID" ]; then + if [ -f "$STRING" ]; then + # shellcheck disable=SC2129 + echo "$FILE_LINK" >> "$LOG_FILE_ID" + jsonValue name < "$STRING" | sed "s/^/Name\: /" >> "$LOG_FILE_ID" + jsonValue id < "$STRING" | sed "s/^/ID\: /" >> "$LOG_FILE_ID" + jsonValue mimeType < "$STRING" | sed "s/^/Type\: /" >> "$LOG_FILE_ID" + printf '\n' >> "$LOG_FILE_ID" + fi + fi + + if [ -f "$STRING" ]; then rm "$STRING"; fi + if [ "$VERBOSE_PROGRESS" = true ]; then - echo "File: ""$INPUTNAME"" (""$READABLE_SIZE"") Uploaded" + print_center_justify " File: $SLUG | $READABLE_SIZE | Uploaded. " "=" else clear 1 clear 1 clear 1 - echo "File: ""$INPUTNAME"" (""$READABLE_SIZE"") Uploaded" + print_center_justify " File: $SLUG | $READABLE_SIZE | Uploaded. " "=" fi else - echo "Upload link generation error, file not uploaded." + print_center " Upload link generation error, file not uploaded. " "=" + echo -e "\n" fi } -echo "Checking credentials.." +print_center " Checking credentials... " "=" # Credentials if [ -z "$CLIENT_ID" ]; then read -r -p "Client ID: " CLIENT_ID @@ -351,7 +440,8 @@ if [ -z "$REFRESH_TOKEN" ]; then echo "REFRESH_TOKEN=""$REFRESH_TOKEN""" >> "$HOME"/.googledrive.conf else - echo -e "\nNo code provided, run the script and try again" + echo + print_center "No code provided, run the script and try again" "=" exit 1 fi fi @@ -370,8 +460,8 @@ fi clear 1 clear 1 -echo "Required credentials available." -echo "Checking root dir and workspace folder.." +print_center " Required credentials available " "=" +print_center " Checking root dir and workspace folder... " "=" # Setup root directory where all file/folders will be uploaded. if [ -n "$ROOTDIR" ]; then ROOT_FOLDER="$(echo "$ROOTDIR" | tr -d ' ' | tr -d '[:blank:]' | tr -d '[:space:]')" @@ -381,7 +471,7 @@ if [ -n "$ROOTDIR" ]; then ROOT_FOLDER="$ROOT_FOLDER" echo "ROOT_FOLDER=$ROOT_FOLDER" >> "$HOME"/.googledrive.conf else - echo "Given root folder ID/URL invalid." + print_center " Given root folder ID/URL invalid. " "=" exit 1 fi fi @@ -394,7 +484,7 @@ elif [ -z "$ROOT_FOLDER" ]; then ROOT_FOLDER="$ROOT_FOLDER" echo "ROOT_FOLDER=$ROOT_FOLDER" >> "$HOME"/.googledrive.conf else - echo "Given root folder ID/URL invalid." + print_center " Given root folder ID/URL invalid. " "=" exit 1 fi else @@ -405,7 +495,7 @@ fi clear 1 clear 1 -echo "Root dir properly configured." +print_center " Root dir properly configured " "=" # Check to find whether the folder exists in google drive. If not then the folder is created in google drive under the configured root folder. if [ -z "$FOLDERNAME" ]; then ROOT_FOLDER_ID=$ROOT_FOLDER @@ -414,7 +504,7 @@ else fi ROOT_FOLDER_NAME="$(drive_Info """$ROOT_FOLDER_ID""" name """$ACCESS_TOKEN""")" clear 1 -echo "Workspace Folder: $ROOT_FOLDER_NAME ($ROOT_FOLDER_ID)" +print_center " Workspace Folder: ""$ROOT_FOLDER_NAME"" | ""$ROOT_FOLDER_ID"" " "=" START=$(date +"%s") @@ -423,40 +513,69 @@ START=$(date +"%s") # In case of folder, do a recursive upload in the same hierarchical manner present. if [ -n "$input" ]; then if [ -f "$input" ]; then - echo -e "Given Input: File\n" + print_center " Given Input: FILE " "=" + echo uploadFile "$input" "$ROOT_FOLDER_ID" "$ACCESS_TOKEN" + print_center " DriveLink " "=" + print_center "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" + print_center "$FILE_LINK" echo elif [ -d "$input" ]; then - echo "Given Input: Folder" + FOLDER_NAME="$(basename "$input")" + print_center " Given Input: FOLDER " "=" + echo + print_center " Folder Name: $FOLDER_NAME " "=" NEXTROOTDIRID="$ROOT_FOLDER_ID" # Do not create empty folders during a recursive upload. # The use of find in this section is important. # If below command is used, it lists the folder in stair structure, which we later assume while creating sub folders and uploading files. find "$input" -type d -not -empty | sed "s|$input|$DIR/$input|" > "$STRING"DIRNAMES + NO_OF_SUB_FOLDERS="$(sed '1d' "$STRING"DIRNAMES | wc -l)" # Create a loop and make folders according to list made above. - if [ -n "$(sed '1d' "$STRING"DIRNAMES)" ]; then - echo "Indexing sub-directories..." + if [ -n "$NO_OF_SUB_FOLDERS" ]; then + print_center " ""$NO_OF_SUB_FOLDERS"" Sub-folders found. " "=" + print_center " Creating sub-folders... " "=" + echo fi while IFS= read -r dir; do newdir="$(basename "$dir")" + if [ -n "$NO_OF_SUB_FOLDERS" ]; then + print_center_justify " Name: ""$newdir"" " "=" + fi ID="$(createDirectory "$newdir" "$NEXTROOTDIRID" "$ACCESS_TOKEN")" # Store sub-folder directory IDs and it's path for later use. echo "$ID '$dir'" >> "$STRING"DIRIDS NEXTROOTDIRID=$ID + if [ -n "$NO_OF_SUB_FOLDERS" ]; then + clear 1 + clear 1 + print_center " Status: $(wc -l < "$STRING"DIRIDS) / ""$NO_OF_SUB_FOLDERS"" " "=" + fi done < "$STRING"DIRNAMES - if [ -n "$(sed '1d' "$STRING"DIRNAMES)" ]; then + if [ -n "$NO_OF_SUB_FOLDERS" ]; then clear 1 - echo -e "No. of sub-directories: ""$(sed '1d' "$STRING"DIRNAMES | wc -l)""\n" + clear 1 + clear 1 + print_center " ""$NO_OF_SUB_FOLDERS"" Sub-folders created " "=" fi if [ -f "$STRING"DIRNAMES ]; then rm "$STRING"DIRNAMES; fi - echo "Indexing files recursively..." + + print_center " Indexing files recursively... " "=" find "$input" -type f | sed "s|$input|$DIR/$input|" > "$STRING"FILENAMES - clear 1 - clear 1 - echo "No. of Files: ""$(grep -c "" "$STRING"FILENAMES)""" + NO_OF_FILES="$(wc -l < "$STRING"FILENAMES)" + if [ -n "$NO_OF_SUB_FOLDERS" ]; then + clear 1 + clear 1 + clear 1 + print_center_justify " Folder Name: $FOLDER_NAME | ""$NO_OF_FILES"" File(s) | ""$NO_OF_SUB_FOLDERS"" Sub-folders " "=" + else + clear 1 + clear 1 + print_center_justify " Folder Name: $FOLDER_NAME | ""$NO_OF_FILES"" File(s) | 0 Sub-folders " "=" + fi if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then echo else @@ -473,21 +592,23 @@ if [ -n "$input" ]; then echo "$file" >> "$STRING"UPLOADED uploaded="$(grep -c "" "$STRING"UPLOADED)" if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then - echo -e "Status: $uploaded files uploaded.\n" + print_center " Status: ""$uploaded"" / ""$NO_OF_FILES"" " "=" + echo else clear 1 clear 1 - echo "Status: $uploaded files uploaded." + print_center " Status: ""$uploaded"" / ""$NO_OF_FILES"" " "=" fi done 4< "$STRING"FILES_ROOTDIR 5< "$STRING"FILENAMES - if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then - echo -e "Total Files Uploaded: ""$(grep -c "" "$STRING"FILENAMES)""\n" - else - clear 1 - clear 1 - echo -e "Total Files Uploaded: ""$(grep -c "" "$STRING"FILENAMES)""\n" - fi + clear 1 + clear 1 + print_center " FolderLink " "=" + print_center "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" + print_center "$(head -n1 "$STRING"DIRIDS | awk '{print $1;}' | sed -e 's|^|https://drive.google.com/open?id=|')" + + echo + print_center " Total Files Uploaded: ""$(grep -c "" "$STRING"FILENAMES)"" " "=" if [ -f "$STRING"DIRIDS ]; then rm "$STRING"DIRIDS; fi if [ -f "$STRING"FILES_ROOTDIR ]; then rm "$STRING"FILES_ROOTDIR; fi @@ -498,7 +619,7 @@ fi END="$(date +"%s")" DIFF="$((END - START))" -echo "Total time elapsed: "$((DIFF / 60))" minute(s) and "$((DIFF % 60))" seconds" +print_center " Total time elapsed: ""$((DIFF / 60))"" minute(s) and ""$((DIFF % 60))"" seconds " "=" if [ "$DEBUG" = true ]; then set +xe From 02e8de3188c1d98bf76a01df4b51a79e9bb28a71 Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Sun, 5 Apr 2020 11:04:23 +0530 Subject: [PATCH 3/4] Implement parallel downloading | Add a flag to skip sub folder generation * Use xargs for parallel downloading. * Use a more efficient function for url encoding. * camelCase fucntions. * Use trap to remove temp files in case of abnormal script exit. * Report if any upload fails. * Use local variables in functions. * Add some checks for flags. * Improve jsonValue function. * Check if given argument to -i/--save-info is dir, and don't log --- README.md | 4 + google-oauth2.sh | 27 +- upload.sh | 647 ++++++++++++++++++++++++++++------------------- url_escape.sed | 25 -- 4 files changed, 399 insertions(+), 304 deletions(-) delete mode 100644 url_escape.sed diff --git a/README.md b/README.md index d8f9d5c..89b1976 100755 --- a/README.md +++ b/README.md @@ -13,11 +13,13 @@ This script does not have very many dependencies. Most of the dependencies are a - find command - awk - getopt +- xargs ## Features - No dependencies at all. - Upload files and folders. +- Upload files in parallel. - Upload sub-folders and content inside it hierarchically. - Config file support ( easy to use script on multiple machines ). - Uses latest gdrive v3 api. @@ -46,6 +48,8 @@ Other Options available are -C | --create-dir - option to create directory. Will provide folder id. -r | --root-dir - google folder id to which the file/directory to upload. + -s | --skip-subdirs - Skip creation of sub folders and upload all files inside the INPUT folder/sub-folders in the INPUT folder, use this along with -p/--parallel option to speed up the uploads. + -p | --parallel - Upload multiple files in parallel, only works along with --skip-subdirs/-s option, Max value = 10, low value are recommended. -v | --verbose - Display detailed message. -V | --verbose-progress - Display detailed message and detailed upload progress( curl normal progress info ). -i | --save-info - Save uploaded files info to the given filename." diff --git a/google-oauth2.sh b/google-oauth2.sh index eb73cf9..bb44f98 100755 --- a/google-oauth2.sh +++ b/google-oauth2.sh @@ -9,7 +9,7 @@ # Set CLIENT_ID and CLIENT_SECRET and SCOPE # See SCOPES at https://developers.google.com/identity/protocols/oauth2/scopes#docsv1 -function short_help() { +short_help() { echo -e "\nNo valid arguments provided." echo -e "Usage:\n" echo -e " ./google-oauth2.sh create - authenticates a user." @@ -17,20 +17,15 @@ function short_help() { exit 0 } -if [ "$#" = "0" ]; then - short_help - exit 0 -fi +[ "$#" = "0" ] && short_help # Clear nth no. of line to the beginning of the line. -function clear() { +clear_line() { echo -en "\033[""$1""A" echo -en "\033[2K" } -if ! [ "$1" = create ] || [ "$1" = refresh ]; then - short_help -fi +[ "$1" = create ] || [ "$1" = refresh ] || short_help echo "Starting script.." @@ -40,9 +35,7 @@ SCOPE="https://www.googleapis.com/auth/drive" REDIRECT_URI="urn:ietf:wg:oauth:2.0:oob" # shellcheck source=/dev/null -if [ -e "$HOME"/.googledrive.conf ]; then - source "$HOME"/.googledrive.conf -fi +[ -e "$HOME"/.googledrive.conf ] && source "$HOME"/.googledrive.conf echo "Checking credentials.." @@ -59,15 +52,15 @@ if [ -z "$CLIENT_SECRET" ]; then fi sleep 1 -clear 1 -clear 1 +clear_line 1 +clear_line 1 echo "Required credentials set." sleep 1 # Method to extract data from json response -function jsonValue() { - num=$2 - grep \""$1"\" | sed "s/\:/\n/" | grep -v \""$1"\" | sed "s/\"\,//g" | sed 's/["]*$//' | sed 's/[,]*$//' | sed 's/^[ \t]*//' | sed s/\"// | sed -n "${num}"p +jsonValue() { + num="$2" + grep \""$1"\" | sed "s/\:/\n/" | grep -v \""$1"\" | sed -e "s/\"\,//g" -e 's/["]*$//' -e 's/[,]*$//' -e 's/^[ \t]*//' -e s/\"// | sed -n "${num}"p } if [ "$1" == "create" ]; then diff --git a/upload.sh b/upload.sh index cf6cfd0..3c3add6 100755 --- a/upload.sh +++ b/upload.sh @@ -2,28 +2,54 @@ # Upload a file to Google Drive # Usage: upload.sh -function usage() { +usage() { echo -e "\nThe script can be used to upload file/directory to google drive." echo -e "\nUsage:\n $0 [options..] \n" echo -e "Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive. \n" echo -e "File name argument is optional if create directory option is used. \n" echo -e "Options:\n" - echo -e "-C | --create-dir - option to create directory. Will provide folder id." - echo -e "-r | --root-dir or - google folder ID/URL to which the file/directory is going to upload." - echo -e "-v | --verbose - Display detailed message." - echo -e "-V | --verbose-progress - Display detailed message and detailed upload progress." - echo -e "-i | --save-info - Save uploaded files info to the given filename." - echo -e "-z | --config - Override default config file with custom config file." - echo -e "-D | --debug - Display script command trace." - echo -e "-h | --help - Display usage instructions.\n" + echo -e " -C | --create-dir - option to create directory. Will provide folder id.\n" + echo -e " -r | --root-dir or - google folder ID/URL to which the file/directory is going to upload.\n" + echo -e " -s | --skip-subdirs - Skip creation of sub folders and upload all files inside the INPUT folder/sub-folders in the INPUT folder, use this along with -p/--parallel option to speed up the uploads.\n" + echo -e " -p | --parallel - Upload multiple files in parallel, only works along with --skip-subdirs/-s option, Max value = 10\n" + echo -e " -i | --save-info - Save uploaded files info to the given filename.\n" + echo -e " -z | --config - Override default config file with custom config file.\n" + echo -e " -v | --verbose - Display detailed message.\n" + echo -e " -V | --verbose-progress - Display detailed message and detailed upload progress.\n" + echo -e " -D | --debug - Display script command trace.\n" + echo -e " -h | --help - Display usage instructions.\n" exit 0 } -function short_help() { +shortHelp() { echo -e "\nNo valid arguments provided, use -h/--help flag to see usage." exit 0 } +# Print short help +[ "$#" = "0" ] && shortHelp + +# allow a command to fail with !’s side effect on errexit +# use return value from ${PIPESTATUS[0]}, because ! hosed $? +getopt --test > /dev/null && exit 1 +if [ "${PIPESTATUS[0]}" -ne 4 ]; then + echo 'getopt --test failed in this environment, cannot run the script.' + exit 1 +fi + +PROGNAME=${0##*/} +SHORTOPTS="v,V,i:,s,p:,hr:C:D,z:" +LONGOPTS="verbose,verbose-progress,save-info:,help,create-dir:,root-dir:,debug,config:" + +if ! OPTS=$(getopt -q --options "$SHORTOPTS" --longoptions "$LONGOPTS" --name "$PROGNAME" -- "$@"); then + shortHelp + exit 1 +fi + +set -o errexit -o noclobber -o pipefail # -o nounset + +eval set -- "$OPTS" + #Configuration variables ROOT_FOLDER="" CLIENT_ID="" @@ -32,54 +58,29 @@ REFRESH_TOKEN="" SCOPE="https://www.googleapis.com/auth/drive" REDIRECT_URI="urn:ietf:wg:oauth:2.0:oob" +# shellcheck source=/dev/null +# Config file is created automatically after first run +[ -e "$HOME"/.googledrive.conf ] && source "$HOME"/.googledrive.conf + #Internal variable ACCESS_TOKEN="" INPUT="" FOLDERNAME="" -curl_args="-#" - +CURL_ARGS="-#" DIR="$(pwd)" STRING="$RANDOM" - -# shellcheck source=/dev/null -# Config file is created automatically after first run -if [ -e "$HOME"/.googledrive.conf ]; then - source "$HOME"/.googledrive.conf -fi - -PROGNAME=${0##*/} -SHORTOPTS="v,V,i:,hr:C:D,z:" -LONGOPTS="verbose,verbose-progress,save-info:,help,create-dir:,root-dir:,debug,config:" - -set -o errexit -o noclobber -o pipefail #-o nounset -OPTS=$(getopt -s bash --options $SHORTOPTS --longoptions $LONGOPTS --name "$PROGNAME" -- "$@") - -# script to parse the input arguments -#if [ $? != 0] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi - -eval set -- "$OPTS" - -VERBOSE=false -VERBOSE_PROGRESS=false -DEBUG=false +VERBOSE="" +VERBOSE_PROGRESS="" +DEBUG="" CONFIG="" ROOTDIR="" +LOG_FILE_ID="" +SKIP_SUBDIRS="" +PARALLEL="" +NO_OF_PARALLEL_JOBS="" while true; do - case "$1" in - -v | --verbose) - VERBOSE=true - shift - ;; - -V | --verbose-progress) - VERBOSE_PROGRESS=true - curl_args="" - shift - ;; - -i | --save-info) - LOG_FILE_ID="$2" - shift 2 - ;; + case "${1}" in -h | --help) usage shift @@ -96,6 +97,28 @@ while true; do CONFIG="$2" shift 2 ;; + -i | --save-info) + LOG_FILE_ID="$2" + shift 2 + ;; + -s | --skip-subdirs) + SKIP_SUBDIRS=true + shift + ;; + -p | --parallel) + PARALLEL=true + NO_OF_PARALLEL_JOBS="$2" + shift 2 + ;; + -v | --verbose) + VERBOSE=true + shift + ;; + -V | --verbose-progress) + VERBOSE_PROGRESS=true + CURL_ARGS="" + shift + ;; -D | --debug) DEBUG=true shift @@ -108,44 +131,51 @@ while true; do esac done -if [ "$DEBUG" = true ]; then - set -xe -fi - -# shellcheck source=/dev/null -if [ -n "$CONFIG" ]; then - if [ -e "$CONFIG" ]; then - source "$CONFIG" - fi -fi - if [ -n "$1" ]; then - input="$1" -fi - -if [ -n "$2" ] && [ -z "$FOLDERNAME" ]; then - FOLDERNAME="$2" + INPUT="$1" + if [[ ! -f $INPUT ]] && [[ ! -d $INPUT ]]; then + echo -e "\nError: Invalid Input, no such or directory.\n" + exit 1 + fi +elif [ -z "$FOLDERNAME" ]; then + shortHelp fi -if [ "$#" = "0" ] && [ -z "$FOLDERNAME" ]; then - short_help - exit 0 +if [ -n "$PARALLEL" ]; then + if [ -d "$INPUT" ]; then + if [ "$SKIP_SUBDIRS" != true ]; then + echo -e "\nError: -p/--parallel option can be only used if -s/--skip-dirs option is used." + exit 0 + fi + case "$NO_OF_PARALLEL_JOBS" in + '' | *[!0-9]*) + echo -e "\nError: -p/--parallel values range between 1 to 10." + exit 0 + ;; + *) + [ "$NO_OF_PARALLEL_JOBS" -gt 10 ] && NO_OF_PARALLEL_JOBS=10 + ;; + esac + elif [ -f "$INPUT" ]; then + unset PARALLEL + fi fi -# This refreshes the interactive shell so we can use the $COLUMNS variable. -cat /dev/null - -# https://gist.github.com/TrinityCoder/911059c83e5f7a351b785921cf7ecdaa -if [ "$DEBUG" = true ]; then +if [ -n "$DEBUG" ]; then + set -xe # To avoid spamming in debug mode. - function print_center() { + printCenter() { echo -e "${1}" } - function print_center_justify() { + printCenterJustify() { echo -e "${1}" } else - function print_center() { + # https://gist.github.com/TrinityCoder/911059c83e5f7a351b785921cf7ecdaa + printCenter() { + # This refreshes the interactive shell so we can use the $COLUMNS variable. + cat /dev/null + [[ $# == 0 ]] && return 1 declare -i TERM_COLS="$COLUMNS" @@ -171,7 +201,10 @@ else return 0 } # To avoid entering a new line, and maintaining the output flow. - function print_center_justify() { + printCenterJustify() { + # This refreshes the interactive shell so we can use the $COLUMNS variable. + cat /dev/null + [[ $# == 0 ]] && return 1 declare -i TERM_COLS="$COLUMNS" @@ -179,7 +212,7 @@ else TO_PRINT="$((TERM_COLS * 98 / 100))" if [ "${#1}" -gt "$TO_PRINT" ]; then - out="${1:0:$TO_PRINT}.." + out="${1:0:TO_PRINT}.." fi declare -i str_len=${#out} @@ -202,15 +235,20 @@ else return 0 } fi -print_center " Starting script " "=" -if [ -n "$VERBOSE_PROGRESS" ]; then - if [ -n "$VERBOSE" ]; then - unset "$VERBOSE" - fi -fi +# Check if skip subdirs creation option was enabled or not. +# Then, check for the max value of parallel downloads. + +# shellcheck source=/dev/null +[ -n "$CONFIG" ] && [ -e "$CONFIG" ] && source "$CONFIG" + +[ -n "${2}" ] && [ -z "$FOLDERNAME" ] && FOLDERNAME="${2}" + +printCenter "[ Starting script ]" "=" + +[ -n "$VERBOSE_PROGRESS" ] && [ -n "$VERBOSE" ] && unset "$VERBOSE" -# Extract file/folder ID from the given input in case of gdrive URL. +# Extract file/folder ID from the given INPUT in case of gdrive URL. extractID() { ID="$1" case "$ID" in @@ -222,57 +260,50 @@ extractID() { } # Clear nth no. of line to the beginning of the line. -function clear() { +clearLine() { echo -en "\033[""$1""A" echo -en "\033[2K" } # Method to extract data from json response -function jsonValue() { - num=$2 - grep \""$1"\" | sed "s/\:/\n/" | grep -v \""$1"\" | sed "s/\"\,//g" | sed 's/["]*$//' | sed 's/[,]*$//' | sed 's/^[ \t]*//' | sed s/\"// | sed -n "${num}"p +jsonValue() { + num="$2" + grep \""$1"\" | sed "s/\:/\n/" | grep -v \""$1"\" | sed -e "s/\"\,//g" -e 's/["]*$//' -e 's/[,]*$//' -e 's/^[ \t]*//' -e s/\"// | sed -n "${num}"p } -# sed url escaping -function urlEscape() { - sed 's|%|%25|g' \ - | sed 's| |%20|g' \ - | sed 's|<|%3C|g' \ - | sed 's|>|%3E|g' \ - | sed 's|#|%23|g' \ - | sed 's|{|%7B|g' \ - | sed 's|}|%7D|g' \ - | sed 's|\||%7C|g' \ - | sed 's|\\|%5C|g' \ - | sed 's|\^|%5E|g' \ - | sed 's|~|%7E|g' \ - | sed 's|\[|%5B|g' \ - | sed 's|\]|%5D|g' \ - | sed 's|`|%60|g' \ - | sed 's|;|%3B|g' \ - | sed 's|/|%2F|g' \ - | sed 's|?|%3F|g' \ - | sed 's^|^%3A^g' \ - | sed 's|@|%40|g' \ - | sed 's|=|%3D|g' \ - | sed 's|&|%26|g' \ - | sed 's|\$|%24|g' \ - | sed 's|\!|%21|g' \ - | sed 's|\*|%2A|g' +# Usage: urlEncode "string" +urlEncode() { + local LC_ALL=C + for ((i = 0; i < ${#1}; i++)); do + : "${1:i:1}" + case "$_" in + [a-zA-Z0-9.~_-]) + printf '%s' "$_" + ;; + *) + printf '%%%02X' "'$_" + ;; + esac + done + printf '\n' } # Method to get information for a gdrive folder/file. # Requirements: Given file/folder ID, query, and access_token. -function drive_Info() { +driveInfo() { + local FOLDER_ID FOLDER_ID="$1" + local FETCH FETCH="$2" + local ACCESS_TOKEN ACCESS_TOKEN="$3" + local SEARCH_RESPONSE SEARCH_RESPONSE="$(curl \ --silent \ -XGET \ -H "Authorization: Bearer ${ACCESS_TOKEN}" \ "https://www.googleapis.com/drive/v3/files/""$FOLDER_ID""?fields=""$FETCH""")" - FETCHED_DATA="$(echo "$SEARCH_RESPONSE" | jsonValue "$FETCH" | head -1)" + FETCHED_DATA="$(echo "$SEARCH_RESPONSE" | jsonValue "$FETCH" 1)" echo "$FETCHED_DATA" } @@ -280,22 +311,30 @@ function drive_Info() { # Requirements: Foldername, Root folder ID ( the folder in which the new folder will be created ) and access_token. # First check if a folder exist in given parent directory, if not the case then make the folder. # Atlast print folder ID ( existing or new one ). -function createDirectory() { +createDirectory() { + local DIRNAME DIRNAME="$1" + local ROOTDIR ROOTDIR="$2" + local ACCESS_TOKEN ACCESS_TOKEN="$3" + local FOLDER_ID FOLDER_ID="" - QUERY="mimeType='application/vnd.google-apps.folder' and name='$DIRNAME' and trashed=false and '$ROOTDIR' in parents" - QUERY="$(echo "$QUERY" | urlEscape)" + local QUERY + QUERY="$(urlEncode "mimeType='application/vnd.google-apps.folder' and name='$DIRNAME' and trashed=false and '$ROOTDIR' in parents")" + local SEARCH_RESPONSE SEARCH_RESPONSE="$(curl \ --silent \ -XGET \ -H "Authorization: Bearer ${ACCESS_TOKEN}" \ "https://www.googleapis.com/drive/v3/files?q=${QUERY}&fields=files(id)")" - FOLDER_ID="$(echo "$SEARCH_RESPONSE" | jsonValue id | head -1)" + local FOLDER_ID + FOLDER_ID="$(echo "$SEARCH_RESPONSE" | jsonValue id 1)" if [ -z "$FOLDER_ID" ]; then + local CREATE_FOLDER_POST_DATA CREATE_FOLDER_POST_DATA="{\"mimeType\": \"application/vnd.google-apps.folder\",\"name\": \"$DIRNAME\",\"parents\": [\"$ROOTDIR\"]}" + local CREATE_FOLDER_RESPONSE CREATE_FOLDER_RESPONSE="$(curl \ --silent \ -X POST \ @@ -310,34 +349,46 @@ function createDirectory() { # Method to upload files to google drive. # Requirements: Given file path, Google folder ID and access_token. -function uploadFile() { +uploadFile() { + local INPUT INPUT="$1" + local FOLDER_ID FOLDER_ID="$2" + local ACCESS_TOKEN ACCESS_TOKEN="$3" + local SLUG SLUG="$(basename "$INPUT")" + local INPUTNAME INPUTNAME="${SLUG%.*}" + local EXTENSION EXTENSION="${SLUG##*.}" + local INPUTSIZE INPUTSIZE="$(stat -c%s "$INPUT")" + local READABLE_SIZE READABLE_SIZE="$(du -sh "$INPUT" | awk '{print $1;}')" - print_center_justify " File: ""$(basename "$INPUT")"" | ""$READABLE_SIZE"" " "=" - if [[ "$INPUTNAME" == "$EXTENSION" ]]; then + [ -z "$PARALLEL" ] && printCenterJustify "[ ""$(basename "$INPUT")"" | ""$READABLE_SIZE"" ]" "=" + + if [[ $INPUTNAME == "$EXTENSION" ]]; then if command -v mimetype > /dev/null 2>&1; then + local MIME_TYPE MIME_TYPE="$(mimetype --output-format %m "$INPUT")" elif command -v file > /dev/null 2>&1; then + local MIME_TYPE MIME_TYPE="$(file --brief --mime-type "$INPUT")" else echo -e "\nError: file or mimetype command not found." exit 1 fi - fi # JSON post data to specify the file name and folder under while the file to be created - postData="{\"mimeType\": \"$MIME_TYPE\",\"name\": \"$SLUG\",\"parents\": [\"$FOLDER_ID\"]}" + local POSTDATA + POSTDATA="{\"mimeType\": \"$MIME_TYPE\",\"name\": \"$SLUG\",\"parents\": [\"$FOLDER_ID\"]}" # Curl command to initiate resumable upload session and grab the location URL - print_center " Generating upload link... " "=" - uploadlink="$(curl \ + [ -z "$PARALLEL" ] && printCenter "[ Generating upload link... ]" "=" + local UPLOADLINK + UPLOADLINK="$(curl \ --silent \ -X POST \ -H "Host: www.googleapis.com" \ @@ -345,18 +396,18 @@ function uploadFile() { -H "Content-Type: application/json; charset=UTF-8" \ -H "X-Upload-Content-Type: $MIME_TYPE" \ -H "X-Upload-Content-Length: $INPUTSIZE" \ - -d "$postData" \ + -d "$POSTDATA" \ "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true&supportsTeamDrives=true" \ --dump-header - | sed -ne s/"Location: "//pi | tr -d '\r\n')" - if [ -n "$uploadlink" ]; then + if [ -n "$UPLOADLINK" ]; then # Curl command to push the file to google drive. # If the file size is large then the content can be split to chunks and uploaded. # In that case content range needs to be specified. - clear 1 - print_center " Uploading... " "=" - if [ -n "$curl_args" ]; then - curl \ + [ -z "$PARALLEL" ] && clearLine 1 && printCenter "[ Uploading... ]" "=" + if [ -n "$CURL_ARGS" ]; then + local UPLOAD_BODY + UPLOAD_BODY="$(curl \ -X PUT \ -H "Authorization: Bearer ${ACCESS_TOKEN}" \ -H "Content-Type: $MIME_TYPE" \ @@ -364,10 +415,11 @@ function uploadFile() { -H "Slug: $SLUG" \ -T "$INPUT" \ -o- \ - --url "$uploadlink" \ - "$curl_args" >> "$STRING" + --url "$UPLOADLINK" \ + "$CURL_ARGS")" else - curl \ + local UPLOAD_BODY + UPLOAD_BODY="$(curl \ -X PUT \ -H "Authorization: Bearer ${ACCESS_TOKEN}" \ -H "Content-Type: $MIME_TYPE" \ @@ -375,39 +427,44 @@ function uploadFile() { -H "Slug: $SLUG" \ -T "$INPUT" \ -o- \ - --url "$uploadlink" >> "$STRING" + --url "$UPLOADLINK")" fi - FILE_LINK="$(jsonValue id < "$STRING" | sed 's|^|https://drive.google.com/open?id=|')" + FILE_LINK="$(echo "$UPLOAD_BODY" | jsonValue id | sed 's|^|https://drive.google.com/open?id=|')" # Log to the filename provided with -i/--save-id flag. if [ -n "$LOG_FILE_ID" ]; then - if [ -f "$STRING" ]; then - # shellcheck disable=SC2129 - echo "$FILE_LINK" >> "$LOG_FILE_ID" - jsonValue name < "$STRING" | sed "s/^/Name\: /" >> "$LOG_FILE_ID" - jsonValue id < "$STRING" | sed "s/^/ID\: /" >> "$LOG_FILE_ID" - jsonValue mimeType < "$STRING" | sed "s/^/Type\: /" >> "$LOG_FILE_ID" - printf '\n' >> "$LOG_FILE_ID" + if ! [ -d "$LOG_FILE_ID" ]; then + if [ -n "$UPLOAD_BODY" ]; then + # shellcheck disable=SC2129 + # https://github.com/koalaman/shellcheck/issues/1202#issuecomment-608239163 + echo "$FILE_LINK" >> "$LOG_FILE_ID" + echo "$UPLOAD_BODY" | jsonValue name | sed "s/^/Name\: /" >> "$LOG_FILE_ID" + echo "$UPLOAD_BODY" | jsonValue id | sed "s/^/ID\: /" >> "$LOG_FILE_ID" + echo "$UPLOAD_BODY" | jsonValue mimeType | sed "s/^/Type\: /" >> "$LOG_FILE_ID" + printf '\n' >> "$LOG_FILE_ID" + fi fi fi - if [ -f "$STRING" ]; then rm "$STRING"; fi - - if [ "$VERBOSE_PROGRESS" = true ]; then - print_center_justify " File: $SLUG | $READABLE_SIZE | Uploaded. " "=" + if [ -n "$VERBOSE_PROGRESS" ]; then + printCenterJustify "[ $SLUG | $READABLE_SIZE | Uploaded ]" "=" else - clear 1 - clear 1 - clear 1 - print_center_justify " File: $SLUG | $READABLE_SIZE | Uploaded. " "=" + if [ -z "$PARALLEL" ]; then + clearLine 1 + clearLine 1 + clearLine 1 + fi + printCenterJustify "[ $SLUG | $READABLE_SIZE | Uploaded ]" "=" fi else - print_center " Upload link generation error, file not uploaded. " "=" - echo -e "\n" + printCenter "[ Upload link generation ERROR, $SLUG not uploaded. ]" "=" + echo -e "\n\n" + UPLOAD_STATUS=ERROR + export UPLOAD_STATUS fi } -print_center " Checking credentials... " "=" +printCenter "[ Checking credentials... ]" "=" # Credentials if [ -z "$CLIENT_ID" ]; then read -r -p "Client ID: " CLIENT_ID @@ -441,8 +498,8 @@ if [ -z "$REFRESH_TOKEN" ]; then echo "REFRESH_TOKEN=""$REFRESH_TOKEN""" >> "$HOME"/.googledrive.conf else echo - print_center "No code provided, run the script and try again" "=" - exit 1 + printCenter "No code provided, run the script and try again" "=" + exit 0 fi fi fi @@ -453,25 +510,25 @@ fi if [ -z "$ACCESS_TOKEN" ]; then RESPONSE="$(curl -s --request POST --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token" https://accounts.google.com/o/oauth2/token)" ACCESS_TOKEN="$(echo "$RESPONSE" | jsonValue access_token)" -elif curl -s "https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=$ACCESS_TOKEN" | jsonValue error > /dev/null 2>&1; then +elif curl -s "https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=$ACCESS_TOKEN" | jsonValue ERROR > /dev/null 2>&1; then RESPONSE="$(curl -s --request POST --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token" https://accounts.google.com/o/oauth2/token)" ACCESS_TOKEN="$(echo "$RESPONSE" | jsonValue access_token)" fi -clear 1 -clear 1 -print_center " Required credentials available " "=" -print_center " Checking root dir and workspace folder... " "=" +clearLine 1 +clearLine 1 +printCenter "[ Required credentials available ]" "=" +printCenter "[ Checking root dir and workspace folder.. ]" "=" # Setup root directory where all file/folders will be uploaded. if [ -n "$ROOTDIR" ]; then ROOT_FOLDER="$(echo "$ROOTDIR" | tr -d ' ' | tr -d '[:blank:]' | tr -d '[:space:]')" if [ -n "$ROOT_FOLDER" ]; then - ROOT_FOLDER="$(drive_Info "$(extractID "$ROOT_FOLDER")" "id" "$ACCESS_TOKEN")" + ROOT_FOLDER="$(driveInfo "$(extractID "$ROOT_FOLDER")" "id" "$ACCESS_TOKEN")" if [ -n "$ROOT_FOLDER" ]; then ROOT_FOLDER="$ROOT_FOLDER" echo "ROOT_FOLDER=$ROOT_FOLDER" >> "$HOME"/.googledrive.conf else - print_center " Given root folder ID/URL invalid. " "=" + printCenter "[ Given root folder ID/URL invalid. ]" "=" exit 1 fi fi @@ -484,7 +541,7 @@ elif [ -z "$ROOT_FOLDER" ]; then ROOT_FOLDER="$ROOT_FOLDER" echo "ROOT_FOLDER=$ROOT_FOLDER" >> "$HOME"/.googledrive.conf else - print_center " Given root folder ID/URL invalid. " "=" + printCenter "[ Given root folder ID/URL invalid. ]" "=" exit 1 fi else @@ -493,134 +550,200 @@ elif [ -z "$ROOT_FOLDER" ]; then fi fi -clear 1 -clear 1 -print_center " Root dir properly configured " "=" +clearLine 1 +clearLine 1 +printCenter "[ Root dir properly configured ]" "=" # Check to find whether the folder exists in google drive. If not then the folder is created in google drive under the configured root folder. if [ -z "$FOLDERNAME" ]; then ROOT_FOLDER_ID=$ROOT_FOLDER else ROOT_FOLDER_ID="$(createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN")" fi -ROOT_FOLDER_NAME="$(drive_Info """$ROOT_FOLDER_ID""" name """$ACCESS_TOKEN""")" -clear 1 -print_center " Workspace Folder: ""$ROOT_FOLDER_NAME"" | ""$ROOT_FOLDER_ID"" " "=" - +ROOT_FOLDER_NAME="$(driveInfo """$ROOT_FOLDER_ID""" name """$ACCESS_TOKEN""")" +clearLine 1 +printCenter "[ Workspace Folder: ""$ROOT_FOLDER_NAME"" | ""$ROOT_FOLDER_ID"" ]" "=" START=$(date +"%s") +# To cleanup the TEMP files. +trap '[ -f "$STRING"DIRIDS ] && rm "$STRING"DIRIDS + [ -f "$STRING"DIRNAMES ] && rm "$STRING"DIRNAMES + [ -f "$STRING"SUCCESS ] && rm "$STRING"SUCCESS +[ -f "$STRING"ERROR ] && rm "$STRING"ERROR' EXIT + # Check if the argument is a file or a directory. # In case of file, just upload it. # In case of folder, do a recursive upload in the same hierarchical manner present. -if [ -n "$input" ]; then - if [ -f "$input" ]; then - print_center " Given Input: FILE " "=" +if [ -n "$INPUT" ]; then + if [ -f "$INPUT" ]; then + printCenter "[ Given Input: FILE ]" "=" echo - uploadFile "$input" "$ROOT_FOLDER_ID" "$ACCESS_TOKEN" - print_center " DriveLink " "=" - print_center "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" - print_center "$FILE_LINK" + uploadFile "$INPUT" "$ROOT_FOLDER_ID" "$ACCESS_TOKEN" + printCenter "[ DriveLink ]" "=" + printCenter "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" + printCenter "$FILE_LINK" echo - elif [ -d "$input" ]; then - FOLDER_NAME="$(basename "$input")" - print_center " Given Input: FOLDER " "=" + elif [ -d "$INPUT" ]; then + FOLDER_NAME="$(basename "$INPUT")" + printCenter "[ Given Input: FOLDER ]" "=" echo - print_center " Folder Name: $FOLDER_NAME " "=" + printCenter "[ Folder: $FOLDER_NAME ]" "=" NEXTROOTDIRID="$ROOT_FOLDER_ID" - # Do not create empty folders during a recursive upload. - # The use of find in this section is important. - # If below command is used, it lists the folder in stair structure, which we later assume while creating sub folders and uploading files. - find "$input" -type d -not -empty | sed "s|$input|$DIR/$input|" > "$STRING"DIRNAMES - NO_OF_SUB_FOLDERS="$(sed '1d' "$STRING"DIRNAMES | wc -l)" - # Create a loop and make folders according to list made above. - if [ -n "$NO_OF_SUB_FOLDERS" ]; then - print_center " ""$NO_OF_SUB_FOLDERS"" Sub-folders found. " "=" - print_center " Creating sub-folders... " "=" + + if [ -n "$SKIP_SUBDIRS" ]; then + printCenter "[ Indexing files recursively... ]" "=" + FILENAMES="$(find "$INPUT" -type f)" + NO_OF_FILES="$(wc -l <<< "$FILENAMES")" + clearLine 1 + clearLine 1 + printCenterJustify "[ Folder: $FOLDER_NAME | ""$NO_OF_FILES"" File(s) ]" "=" echo - fi - while IFS= read -r dir; do - newdir="$(basename "$dir")" - if [ -n "$NO_OF_SUB_FOLDERS" ]; then - print_center_justify " Name: ""$newdir"" " "=" + ID="$(createDirectory "$INPUT" "$NEXTROOTDIRID" "$ACCESS_TOKEN")" + echo "$ID" >> "$STRING"DIRIDS + if [ -n "$PARALLEL" ]; then + + export ID + export CURL_ARGS="-s" + export PARALLEL + export ACCESS_TOKEN + export STRING + export -f uploadFile + export -f printCenter + export -f printCenterJustify + export -f clearLine + export -f jsonValue + + # shellcheck disable=SC2016 + echo "$FILENAMES" | xargs -n1 -P"$NO_OF_PARALLEL_JOBS" -i bash -c ' + uploadFile "{}" "$ID" "$ACCESS_TOKEN" + if [ "$UPLOAD_STATUS" = ERROR ]; then + echo 1 >> "$STRING"ERROR + else + echo 1 >> "$STRING"SUCCESS + fi + ' + [ -f "$STRING"SUCCESS ] && SUCESS_STATUS="$(wc -l < "$STRING"SUCCESS)" + [ -f "$STRING"ERROR ] && ERROR_STATUS="$(wc -l < "$STRING"ERROR)" + if [ -z "$VERBOSE" ] && [ -z "$VERBOSE_PROGRESS" ]; then + echo -e "\n\n" + else + echo + fi + else + if [ -z "$VERBOSE" ] && [ -z "$VERBOSE_PROGRESS" ]; then + echo + fi + + while IFS= read -r -u 4 file; do + DIRTOUPLOAD="$ID" + uploadFile "$file" "$DIRTOUPLOAD" "$ACCESS_TOKEN" + [ "$UPLOAD_STATUS" = ERROR ] && ERROR+="\n1" || SUCESS+="\n1" + SUCESS_STATUS="$(echo -e "$SUCESS" | sed 1d | wc -l)" + ERROR_STATUS="$(echo -e "$ERROR" | sed 1d | wc -l)" + if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then + printCenter "[ Status: ""$SUCESS_STATUS"" UPLOADED | ""$ERROR_STATUS"" FAILED ]" "=" + echo + else + clearLine 1 + clearLine 1 + printCenter "[ Status: ""$SUCESS_STATUS"" UPLOADED | ""$ERROR_STATUS"" FAILED ]" "=" + fi + done 4<<< "$FILENAMES" fi - ID="$(createDirectory "$newdir" "$NEXTROOTDIRID" "$ACCESS_TOKEN")" - # Store sub-folder directory IDs and it's path for later use. - echo "$ID '$dir'" >> "$STRING"DIRIDS - NEXTROOTDIRID=$ID + else + # Do not create empty folders during a recursive upload. + # The use of find in this section is important. + # If below command is used, it lists the folder in stair structure, + # which we later assume while creating sub folders( if applicable ) and uploading files. + find "$INPUT" -type d -not -empty | sed "s|$INPUT|$DIR/$INPUT|" > "$STRING"DIRNAMES + NO_OF_SUB_FOLDERS="$(sed '1d' "$STRING"DIRNAMES | wc -l)" + # Create a loop and make folders according to list made above. if [ -n "$NO_OF_SUB_FOLDERS" ]; then - clear 1 - clear 1 - print_center " Status: $(wc -l < "$STRING"DIRIDS) / ""$NO_OF_SUB_FOLDERS"" " "=" + printCenter "[ ""$NO_OF_SUB_FOLDERS"" Sub-folders found ]" "=" + printCenter "[ Creating sub-folders...]" "=" + echo fi - done < "$STRING"DIRNAMES - if [ -n "$NO_OF_SUB_FOLDERS" ]; then - clear 1 - clear 1 - clear 1 - print_center " ""$NO_OF_SUB_FOLDERS"" Sub-folders created " "=" - fi + while IFS= read -r -u 4 dir; do + NEWDIR="$(basename "$dir")" + [ -n "$NO_OF_SUB_FOLDERS" ] && printCenterJustify " Name: ""$NEWDIR"" " "=" 1>&2 + ID="$(createDirectory "$NEWDIR" "$NEXTROOTDIRID" "$ACCESS_TOKEN")" + # Store sub-folder directory IDs and it's path for later use. + echo "$ID '$dir'" + NEXTROOTDIRID=$ID + TEMP+="\n""$NEXTROOTDIRID""" + status="$(echo -e "$TEMP" | sed '/^$/d' | wc -l)" + if [ -n "$NO_OF_SUB_FOLDERS" ]; then + clearLine 1 1>&2 + clearLine 1 1>&2 + printCenter " Status: ""$status"" / ""$NO_OF_SUB_FOLDERS"" " "=" 1>&2 + fi + done 4< "$STRING"DIRNAMES >> "$STRING"DIRIDS - if [ -f "$STRING"DIRNAMES ]; then rm "$STRING"DIRNAMES; fi - - print_center " Indexing files recursively... " "=" - find "$input" -type f | sed "s|$input|$DIR/$input|" > "$STRING"FILENAMES - NO_OF_FILES="$(wc -l < "$STRING"FILENAMES)" - if [ -n "$NO_OF_SUB_FOLDERS" ]; then - clear 1 - clear 1 - clear 1 - print_center_justify " Folder Name: $FOLDER_NAME | ""$NO_OF_FILES"" File(s) | ""$NO_OF_SUB_FOLDERS"" Sub-folders " "=" - else - clear 1 - clear 1 - print_center_justify " Folder Name: $FOLDER_NAME | ""$NO_OF_FILES"" File(s) | 0 Sub-folders " "=" - fi - if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then - echo - else - echo -e "\n" - fi + if [ -n "$NO_OF_SUB_FOLDERS" ]; then + clearLine 1 + clearLine 1 + clearLine 1 + printCenter "[ ""$NO_OF_SUB_FOLDERS"" Sub-folders created ]" "=" + fi - # shellcheck disable=SC2001 - # Match the path with sub-folder directory ID and upload accordingly. - while read -r i; do sed "s/\(.*\)\/$(basename "$i")/\1/" <<< "$i"; done < "$STRING"FILENAMES > "$STRING"FILES_ROOTDIR - - while IFS= read -r -u 4 rootdirpath && IFS= read -r -u 5 file; do - dirtoupload="$(grep "'$rootdirpath'" "$STRING"DIRIDS | awk '{print $1;}')" - uploadFile "$file" "$dirtoupload" "$ACCESS_TOKEN" - echo "$file" >> "$STRING"UPLOADED - uploaded="$(grep -c "" "$STRING"UPLOADED)" - if [ "$VERBOSE" = true ] || [ "$VERBOSE_PROGRESS" = true ]; then - print_center " Status: ""$uploaded"" / ""$NO_OF_FILES"" " "=" + printCenter "[ Indexing files recursively... ]" "=" + FILENAMES="$(find "$INPUT" -type f | sed "s|$INPUT|$DIR/$INPUT|")" + NO_OF_FILES="$(wc -l <<< "$FILENAMES")" + if [ -n "$NO_OF_SUB_FOLDERS" ]; then + clearLine 1 + clearLine 1 + clearLine 1 + printCenterJustify "[ Folder: $FOLDER_NAME | ""$NO_OF_FILES"" File(s) | ""$NO_OF_SUB_FOLDERS"" Sub-folders ]" "=" + else + clearLine 1 + clearLine 1 + printCenterJustify "[ Folder: $FOLDER_NAME | ""$NO_OF_FILES"" File(s) | 0 Sub-folders ]" "=" + fi + if [ -n "$VERBOSE" ] || [ -n "$VERBOSE_PROGRESS" ]; then echo else - clear 1 - clear 1 - print_center " Status: ""$uploaded"" / ""$NO_OF_FILES"" " "=" + echo -e "\n" fi - done 4< "$STRING"FILES_ROOTDIR 5< "$STRING"FILENAMES - clear 1 - clear 1 - print_center " FolderLink " "=" - print_center "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" - print_center "$(head -n1 "$STRING"DIRIDS | awk '{print $1;}' | sed -e 's|^|https://drive.google.com/open?id=|')" + # shellcheck disable=SC2001 + # Match the path with sub-folder directory ID and upload accordingly. + FILES_ROOTDIR="$(while read -r i; do sed "s/\(.*\)\/$(basename "$i")/\1/" <<< "$i"; done <<< "$FILENAMES")" + + while IFS= read -r -u 4 ROOTDIRPATH && IFS= read -r -u 5 file; do + DIRTOUPLOAD="$(grep "'$ROOTDIRPATH'" "$STRING"DIRIDS | awk '{print $1;}')" + uploadFile "$file" "$DIRTOUPLOAD" "$ACCESS_TOKEN" + [ "$UPLOAD_STATUS" = ERROR ] && ERROR+="\n1" || SUCESS+="\n1" + SUCESS_STATUS="$(echo -e "$SUCESS" | sed 1d | wc -l)" + ERROR_STATUS="$(echo -e "$ERROR" | sed 1d | wc -l)" + if [ -n "$VERBOSE" ] || [ -n "$VERBOSE_PROGRESS" ]; then + printCenter "[ Status: ""$SUCESS_STATUS"" UPLOADED | ""$ERROR_STATUS"" FAILED ]" "=" + echo + else + clearLine 1 + clearLine 1 + printCenter "[ Status: ""$SUCESS_STATUS"" UPLOADED | ""$ERROR_STATUS"" FAILED ]" "=" + fi + done 4<<< "$FILES_ROOTDIR" 5<<< "$FILENAMES" + fi + if [ -z "$VERBOSE" ] && [ -z "$VERBOSE_PROGRESS" ]; then + clearLine 1 + clearLine 1 + fi + if ! [ "$SUCESS_STATUS" = 0 ]; then + printCenter "[ FolderLink ]" "=" + printCenter "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" + printCenter "$(head -n1 "$STRING"DIRIDS | awk '{print $1;}' | sed -e 's|^|https://drive.google.com/open?id=|')" + fi echo - print_center " Total Files Uploaded: ""$(grep -c "" "$STRING"FILENAMES)"" " "=" - - if [ -f "$STRING"DIRIDS ]; then rm "$STRING"DIRIDS; fi - if [ -f "$STRING"FILES_ROOTDIR ]; then rm "$STRING"FILES_ROOTDIR; fi - if [ -f "$STRING"FILENAMES ]; then rm "$STRING"FILENAMES; fi - if [ -f "$STRING"UPLOADED ]; then rm "$STRING"UPLOADED; fi + printCenter "[ Total Files Uploaded: ""$SUCESS_STATUS"" ]" "=" + [ -n "$ERROR_STATUS" ] && [ "$ERROR_STATUS" -gt 0 ] && printCenter "[ Total Files Failed: ""$ERROR_STATUS"" ]" "=" fi fi END="$(date +"%s")" DIFF="$((END - START))" -print_center " Total time elapsed: ""$((DIFF / 60))"" minute(s) and ""$((DIFF % 60))"" seconds " "=" +printCenter "[ Time Elapsed: ""$((DIFF / 60))"" minute(s) and ""$((DIFF % 60))"" seconds ]" "=" -if [ "$DEBUG" = true ]; then - set +xe -fi +[ -n "$DEBUG" ] && set +xe diff --git a/url_escape.sed b/url_escape.sed deleted file mode 100644 index eac4002..0000000 --- a/url_escape.sed +++ /dev/null @@ -1,25 +0,0 @@ -# sed url escaping -s:%:%25:g -s: :%20:g -s:<:%3C:g -s:>:%3E:g -s:#:%23:g -s:{:%7B:g -s:}:%7D:g -s:|:%7C:g -s:\\:%5C:g -s:\^:%5E:g -s:~:%7E:g -s:\[:%5B:g -s:\]:%5D:g -s:`:%60:g -s:;:%3B:g -s:/:%2F:g -s:?:%3F:g -s^:^%3A^g -s:@:%40:g -s:=:%3D:g -s:&:%26:g -s:\$:%24:g -s:\!:%21:g -s:\*:%2A:g From 335c21ed9e3ceda791cfd60c61661962bfd7f9ee Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Mon, 6 Apr 2020 19:55:48 +0530 Subject: [PATCH 4/4] Add flag to share file | Use bash getopts | Fix https://github.com/labbots/google-drive-upload/issues/12 * -S/--share to automaticall share the file/folder publicly after uploading * Use bash getops * Use wc to count file size instead of stat * Add a check for internet connection * Fix #12 ( by using bash getops and using wc ) * Improve extractID and jsonValue method --- README.md | 4 +- upload.sh | 173 ++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 113 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 89b1976..bdbb611 100755 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This script does not have very many dependencies. Most of the dependencies are a - sed (Stream editor) - find command - awk -- getopt +- getopts ( bash builtin ) - xargs ## Features @@ -23,6 +23,7 @@ This script does not have very many dependencies. Most of the dependencies are a - Upload sub-folders and content inside it hierarchically. - Config file support ( easy to use script on multiple machines ). - Uses latest gdrive v3 api. +- Share files after uploading. - Pretty Logging. ## Usage @@ -50,6 +51,7 @@ Other Options available are -r | --root-dir - google folder id to which the file/directory to upload. -s | --skip-subdirs - Skip creation of sub folders and upload all files inside the INPUT folder/sub-folders in the INPUT folder, use this along with -p/--parallel option to speed up the uploads. -p | --parallel - Upload multiple files in parallel, only works along with --skip-subdirs/-s option, Max value = 10, low value are recommended. + -S | --share - Share the uploaded input file/folder, grant reader permission to the everyone with the link. -v | --verbose - Display detailed message. -V | --verbose-progress - Display detailed message and detailed upload progress( curl normal progress info ). -i | --save-info - Save uploaded files info to the given filename." diff --git a/upload.sh b/upload.sh index 3c3add6..d4adf40 100755 --- a/upload.sh +++ b/upload.sh @@ -12,6 +12,7 @@ usage() { echo -e " -r | --root-dir or - google folder ID/URL to which the file/directory is going to upload.\n" echo -e " -s | --skip-subdirs - Skip creation of sub folders and upload all files inside the INPUT folder/sub-folders in the INPUT folder, use this along with -p/--parallel option to speed up the uploads.\n" echo -e " -p | --parallel - Upload multiple files in parallel, only works along with --skip-subdirs/-s option, Max value = 10\n" + echo -e " -S | --share - Share the uploaded input file/folder, grant reader permission to the everyone with the link.\n" echo -e " -i | --save-info - Save uploaded files info to the given filename.\n" echo -e " -z | --config - Override default config file with custom config file.\n" echo -e " -v | --verbose - Display detailed message.\n" @@ -29,27 +30,37 @@ shortHelp() { # Print short help [ "$#" = "0" ] && shortHelp -# allow a command to fail with !’s side effect on errexit -# use return value from ${PIPESTATUS[0]}, because ! hosed $? -getopt --test > /dev/null && exit 1 -if [ "${PIPESTATUS[0]}" -ne 4 ]; then - echo 'getopt --test failed in this environment, cannot run the script.' - exit 1 +if ! [[ "$1" == -* ]]; then + FIRST_INPUT="$1" + shift + if [ -n "$FIRST_INPUT" ]; then + if ! [[ "$1" == -* ]]; then + FOLDER_INPUT="$1" + shift + fi + fi fi -PROGNAME=${0##*/} -SHORTOPTS="v,V,i:,s,p:,hr:C:D,z:" -LONGOPTS="verbose,verbose-progress,save-info:,help,create-dir:,root-dir:,debug,config:" - -if ! OPTS=$(getopt -q --options "$SHORTOPTS" --longoptions "$LONGOPTS" --name "$PROGNAME" -- "$@"); then - shortHelp - exit 1 -fi +for arg in "$@"; do + shift + case "$arg" in + "--verbose") eval set -- "$@" "-v" ;; + "--verbose-progress" | "-V") eval set -- "$@" "-V" ;; + "--save-info") eval set -- "$@" "-i" ;; + "--share") eval set -- "$@" "-S" ;; + "--create-dir") eval set -- "$@" "-C" ;; + "--root-dir") eval set -- "$@" "-r" ;; + "--skip-subdirs") eval set -- "$@" "-s" ;; + "--parallel") eval set -- "$@" "-p" ;; + "--debug") eval set -- "$@" "-D" ;; + "--config") eval set -- "$@" "-z" ;; + "--help") eval set -- "$@" "-h" ;; + *) eval set -- "$@" "$arg" ;; + esac +done set -o errexit -o noclobber -o pipefail # -o nounset -eval set -- "$OPTS" - #Configuration variables ROOT_FOLDER="" CLIENT_ID="" @@ -78,67 +89,70 @@ LOG_FILE_ID="" SKIP_SUBDIRS="" PARALLEL="" NO_OF_PARALLEL_JOBS="" +SHARE="" + +SHORTOPTS=":vVi:sp:Shr:C:Dz:" -while true; do - case "${1}" in - -h | --help) +while getopts "${SHORTOPTS}" OPTION; do + case "$OPTION" in + h) usage - shift ;; - -C | --create-dir) - FOLDERNAME="$2" - shift 2 + C) + FOLDERNAME="$OPTARG" ;; - -r | --root-dir) - ROOTDIR="$2" - shift 2 + r) + ROOTDIR="$OPTARG" ;; - -z | --config) - CONFIG="$2" - shift 2 + z) + CONFIG="$OPTARG" ;; - -i | --save-info) - LOG_FILE_ID="$2" - shift 2 + i) + LOG_FILE_ID="$OPTARG" ;; - -s | --skip-subdirs) + s) SKIP_SUBDIRS=true - shift ;; - -p | --parallel) + p) PARALLEL=true - NO_OF_PARALLEL_JOBS="$2" - shift 2 + NO_OF_PARALLEL_JOBS="$OPTARG" + ;; + S) + SHARE=true ;; - -v | --verbose) + v) VERBOSE=true - shift ;; - -V | --verbose-progress) + V) VERBOSE_PROGRESS=true CURL_ARGS="" - shift ;; - -D | --debug) + D) DEBUG=true - shift ;; - --) - shift - break + :) + echo -e "${0}: -$OPTARG: option requires an argument\nTry '"$0 -h/--help"' for more information." && exit 1 + ;; + ?) + echo -e "${0}: -$OPTARG: unknown option\nTry '"$0 -h/--help"' for more information." >&2 && exit 1 ;; - *) break ;; esac done - -if [ -n "$1" ]; then - INPUT="$1" - if [[ ! -f $INPUT ]] && [[ ! -d $INPUT ]]; then - echo -e "\nError: Invalid Input, no such or directory.\n" - exit 1 +shift $((OPTIND - 1)) + +if [ -z "$FIRST_INPUT" ]; then + [ -n "$1" ] && INPUT="$1" + [ -n "$INPUT" ] && [ -n "$2" ] && FOLDER_INPUT="$2" +elif [ -z "$INPUT" ]; then + if [ -n "$FIRST_INPUT" ]; then + INPUT="$FIRST_INPUT" + if [[ ! -f $INPUT ]] && [[ ! -d $INPUT ]]; then + echo -e "\nError: Invalid Input, no such or directory.\n" + exit 1 + fi + elif [ -z "$FOLDERNAME" ]; then + shortHelp fi -elif [ -z "$FOLDERNAME" ]; then - shortHelp fi if [ -n "$PARALLEL" ]; then @@ -161,6 +175,12 @@ if [ -n "$PARALLEL" ]; then fi fi +# If the internet connection is not available, curl gives "000" output, so add a check for it. +if ! curl -Is --write-out "%{http_code}" --output /dev/null "google.com" > /dev/null 2>&1; then + printf '\nError: Internet connection not available.\n\n' + exit 1 +fi + if [ -n "$DEBUG" ]; then set -xe # To avoid spamming in debug mode. @@ -242,7 +262,7 @@ fi # shellcheck source=/dev/null [ -n "$CONFIG" ] && [ -e "$CONFIG" ] && source "$CONFIG" -[ -n "${2}" ] && [ -z "$FOLDERNAME" ] && FOLDERNAME="${2}" +[ -n "$FOLDER_INPUT" ] && [ -z "$FOLDERNAME" ] && FOLDERNAME="${FOLDER_INPUT}" printCenter "[ Starting script ]" "=" @@ -254,7 +274,7 @@ extractID() { case "$ID" in 'http'*'://'*'drive.google.com'*'id='*) ID=$(echo "$ID" | sed -e 's/^.*id=//' -e 's|&|\n|' | head -1) ;; 'http'*'drive.google.com'*'file/d/'* | 'http'*'docs.google.com/file/d/'*) ID=$(echo "$ID" | sed -e's/^.*\/d\///' -e 's/\/.*//') ;; - 'http'*'drive.google.com'*'drive'*'folders'*) ID=$(echo "$ID" | sed -e 's/^.*\/folders\///' -e "s/&.*//" -e -r 's/(.*)\/.*/\1 /') ;; + 'http'*'drive.google.com'*'drive'*'folders'*) ID=$(echo "$ID" | sed -e 's/^.*\/folders\///' -e "s/&.*//" -e "s/?.*//") ;; esac echo "$ID" } @@ -268,7 +288,7 @@ clearLine() { # Method to extract data from json response jsonValue() { num="$2" - grep \""$1"\" | sed "s/\:/\n/" | grep -v \""$1"\" | sed -e "s/\"\,//g" -e 's/["]*$//' -e 's/[,]*$//' -e 's/^[ \t]*//' -e s/\"// | sed -n "${num}"p + grep -o \""$1"\"\:.* | sed -e "s/^.*: //" -e "s/\"\,//g" -e 's/["]*$//' -e 's/[,]*$//' -e 's/^[ \t]*//' -e "s/\"//" -n -e "${num}"p } # Usage: urlEncode "string" @@ -363,9 +383,9 @@ uploadFile() { local EXTENSION EXTENSION="${SLUG##*.}" local INPUTSIZE - INPUTSIZE="$(stat -c%s "$INPUT")" + INPUTSIZE="$(wc -c < "$INPUT")" local READABLE_SIZE - READABLE_SIZE="$(du -sh "$INPUT" | awk '{print $1;}')" + READABLE_SIZE="$(du -h "$INPUT" | awk '{print $1;}')" [ -z "$PARALLEL" ] && printCenterJustify "[ ""$(basename "$INPUT")"" | ""$READABLE_SIZE"" ]" "=" if [[ $INPUTNAME == "$EXTENSION" ]]; then @@ -386,7 +406,7 @@ uploadFile() { POSTDATA="{\"mimeType\": \"$MIME_TYPE\",\"name\": \"$SLUG\",\"parents\": [\"$FOLDER_ID\"]}" # Curl command to initiate resumable upload session and grab the location URL - [ -z "$PARALLEL" ] && printCenter "[ Generating upload link... ]" "=" + [ -z "$PARALLEL" ] && printCenter "[ Generating upload link...]" "=" local UPLOADLINK UPLOADLINK="$(curl \ --silent \ @@ -404,7 +424,7 @@ uploadFile() { # Curl command to push the file to google drive. # If the file size is large then the content can be split to chunks and uploaded. # In that case content range needs to be specified. - [ -z "$PARALLEL" ] && clearLine 1 && printCenter "[ Uploading... ]" "=" + [ -z "$PARALLEL" ] && clearLine 1 && printCenter "[ Uploading...]" "=" if [ -n "$CURL_ARGS" ]; then local UPLOAD_BODY UPLOAD_BODY="$(curl \ @@ -431,13 +451,14 @@ uploadFile() { fi FILE_LINK="$(echo "$UPLOAD_BODY" | jsonValue id | sed 's|^|https://drive.google.com/open?id=|')" + FILE_ID="$(echo "$UPLOAD_BODY" | jsonValue id)" # Log to the filename provided with -i/--save-id flag. if [ -n "$LOG_FILE_ID" ]; then if ! [ -d "$LOG_FILE_ID" ]; then if [ -n "$UPLOAD_BODY" ]; then # shellcheck disable=SC2129 # https://github.com/koalaman/shellcheck/issues/1202#issuecomment-608239163 - echo "$FILE_LINK" >> "$LOG_FILE_ID" + echo "Link: $FILE_LINK" >> "$LOG_FILE_ID" echo "$UPLOAD_BODY" | jsonValue name | sed "s/^/Name\: /" >> "$LOG_FILE_ID" echo "$UPLOAD_BODY" | jsonValue id | sed "s/^/ID\: /" >> "$LOG_FILE_ID" echo "$UPLOAD_BODY" | jsonValue mimeType | sed "s/^/Type\: /" >> "$LOG_FILE_ID" @@ -464,6 +485,30 @@ uploadFile() { fi } +# Method to share a gdrive file/folder +# Requirements: Given file/folder ID, type, role and access_token. +shareID() { + local ID + ID="$1" + local ROLE + ROLE="reader" + local TYPE + TYPE="anyone" + local ACCESS_TOKEN + ACCESS_TOKEN="$2" + local SHARE_POST_DATA + SHARE_POST_DATA="{\"role\":\"$ROLE\",\"type\":\"$TYPE\"}" + + curl \ + --silent \ + --output /dev/null \ + -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json; charset=UTF-8" \ + -d "$SHARE_POST_DATA" \ + "https://www.googleapis.com/drive/v3/files/""$ID""/permissions" +} + printCenter "[ Checking credentials... ]" "=" # Credentials if [ -z "$CLIENT_ID" ]; then @@ -578,6 +623,7 @@ if [ -n "$INPUT" ]; then printCenter "[ Given Input: FILE ]" "=" echo uploadFile "$INPUT" "$ROOT_FOLDER_ID" "$ACCESS_TOKEN" + [ -n "$SHARE" ] && printCenter "[ Making File Public..]" "=" && shareID "$FILE_ID" "$ACCESS_TOKEN" && clearLine 1 printCenter "[ DriveLink ]" "=" printCenter "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" printCenter "$FILE_LINK" @@ -732,6 +778,7 @@ if [ -n "$INPUT" ]; then clearLine 1 fi if ! [ "$SUCESS_STATUS" = 0 ]; then + [ -n "$SHARE" ] && printCenter "[ Making Folder Public..]" "=" && shareID "$(head -n1 "$STRING"DIRIDS | awk '{print $1;}')" "$ACCESS_TOKEN" && clearLine 1 printCenter "[ FolderLink ]" "=" printCenter "$(echo -e "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93")" printCenter "$(head -n1 "$STRING"DIRIDS | awk '{print $1;}' | sed -e 's|^|https://drive.google.com/open?id=|')"