lots of changes - primarily with the dispatcher and handling killing old bots and emptying existing redis queues

Dockerfile

@@ -9,6 +9,7 @@ LABEL maintainer="alex-phillips"
 RUN \
  echo "**** install build packages ****" && \
  apk add --no-cache \
+	grep \
 	logrotate \
 	ncurses \
 	python3 && \

READMETEMPLATE.md

@@ -33,6 +33,7 @@ docker create \
   --name=diskover \
   -v <path to config>:/config \
   -v <path to data>:/data \
+  -v <path to crontab file>:/etc/crontabs/abc \
   -e PGID=<gid> -e PUID=<uid>  \
   -e REDIS_HOST=<REDIS_HOST> \
   -e REDIS_PORT=<REDIS_PORT> \
@@ -41,9 +42,8 @@ docker create \
   -e ES_USER=<ES_USER> \
   -e ES_PASS=<ES_PASS> \
   -e INDEX_NAME=<INDEX_NAME> \
-  -e DISPATCH_OPTS=<DISPATCH_OPTS> \
+  -e DISKOVER_OPTS=<DISKOVER_OPTS> \
   -e WORKER_OPTS=<WORKER_OPTS> \
-  -e USE_CRON=<USE_CRONS> \
   alexphillips/diskover
 ```
 
@@ -61,6 +61,7 @@ http://192.168.x.x:8080 would show you what's running INSIDE the container on po
 | `-p 1234` | the port(s) |
 | `-v /config` | diskover config |
 | `-v /data` | data directory to scan |
+| `-v /etc/crontabs/abc` | crontab file to run dispatcher in cron |
 | `-e PGID` | for GroupID, see below for explanation |
 | `-e PUID` | for UserID, see below for explanation |
 | `-e REDIS_HOST` | Redis host [default: redis] |
@@ -71,9 +72,8 @@ http://192.168.x.x:8080 would show you what's running INSIDE the container on po
 | `-e ES_PASS` | ElasticSearch password [default: changeme] |
 | `-e INDEX_PREFIX` | ElasticSearch index prefix [default: 'diskover-'] |
 | `-e INDEX_NAME` | ElasticSearch index name [default: $INDEX_PREFIX-TIMESTAMP] |
-| `-e DISPATCH_OPTS` | Arguments to pass to the dispatcher [default: '-a -i $INDEX_NAME'] |
+| `-e DISKOVER_OPTS` | Arguments to pass to the dispatcher [default: '-a -i $INDEX_NAME'] |
 | `-e WORKER_OPTS` | Arguments to pass to the bot launcher [default: ''] |
-| `-e USE_CRON` | Whether the dispatcher should use cron to run on a schedule [default: false] |
 
 &nbsp;
 

root/app/dispatcher.sh

@@ -1,19 +1,34 @@
 #!/usr/bin/with-contenv bash
 
+. /config/.env
+
 TODAY=$(date +%Y-%m-%d)
 INDEX_PREFIX=${INDEX_PREFIX:-diskover-}
 INDEX_NAME=${INDEX_NAME:-$INDEX_PREFIX$TODAY}
-DISPATCH_OPTS=${DISPATCH_OPTS:-"-a -i $INDEX_NAME"}
-WORKER_OPTS=${WORKER_OPTS:-""}
-
-sleep 10
+DISKOVER_OPTS=${DISKOVER_OPTS:-"-d /data -a -i $INDEX_NAME"}
 
 cd /app/diskover || exit
 
-# Run the workers
-/bin/bash ./diskover-bot-launcher.sh $WORKER_OPTS > /dev/null 2>&1
+# killing existing workers before starting new ones
+echo "killing existing workers..."
+if [ -f "/tmp/diskover_bot_pids" ]; then
+    /bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1
+    sleep 3
+fi
 
-/usr/bin/python3 ./diskover.py -d /data $DISPATCH_OPTS && \
-    sleep 10 && \
-    echo "dispatcher is done, killing off workers..." && \
-    /bin/bash ./diskover-bot-launcher.sh -k
+# empty existing redis queue
+echo "emptying current redis queues..."
+rq empty -u redis://$REDIS_HOST:$REDIS_PORT diskover_crawl
+sleep 3
+rq empty -u redis://$REDIS_HOST:$REDIS_PORT failed
+sleep 3
+
+echo "killing dangling workers..."
+/bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1
+sleep 3
+
+echo "starting workers with following options: $WORKER_OPTS"
+/bin/bash /app/diskover/diskover-bot-launcher.sh $WORKER_OPTS
+
+echo "starting crawler with following options: $DISKOVER_OPTS"
+/usr/bin/python3 ./diskover.py $DISKOVER_OPTS

root/defaults/crontab

@@ -1 +1 @@
-0 3 * * * /app/diskover/diskover-bot-launcher.sh -r && /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1
+0 3 * * * /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1

root/defaults/diskover.cfg

@@ -2,8 +2,8 @@
 ; if you make any changes, restart worker bots so they get the new config
 
 [excludes]
-; directory names and absolute paths you want to exclude from crawl, case-sensitive, can include wildcard (.* or tmp* or /dir/dirname* etc)
-dirs = .*,.snapshot,.Snapshot
+; directory names and absolute paths you want to exclude from crawl, case-sensitive, can include wildcards (.* or backup* or /dir/dirname* or *tmp or *tmp* etc)
+dirs = .*,.snapshot,.Snapshot,.zfs
 ; files you want to exclude from crawl, case-sensitive, can include wildcards (.*, *.doc or NULLEXT for files with no extension)
 files = .*,Thumbs.db,.DS_Store,._.DS_Store,.localized,desktop.ini
 
@@ -18,6 +18,10 @@ files = .*,Thumbs.db,.DS_Store,._.DS_Store,.localized,desktop.ini
 ;files = [{"name": [], "name_exclude": [], "ext": ["tmp*", "TMP*", "temp*", "TEMP*", "cache*", "CACHE*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}]
 ;dirs = [{"name": ["*tmp*", "*TMP*", "*temp*", "*TEMP*", "*Temp*", "*cache*", "*CACHE*", "*Cache*"], "name_exclude": ["*templates*", "*Templates*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}]
 
+[treewalk]
+; number of processes to run when doing parallel tree walking of rootdir
+procs = 8
+
 [elasticsearch]
 ; uncomment the below three lines if you are using AWS ES
 ;aws = False
@@ -50,9 +54,11 @@ replicas = 1
 host = {{REDIS_HOST}}
 port = {{REDIS_PORT}}
 ;password =
-; how long directory times lives in Redis (seconds) (default 1 week)
+; how long directory keys lives in Redis (seconds)
 ; used for -I index2 when comparing directory times to get metadata from index2 instead of off disk
 dirtimesttl = 604800
+; used for calculating directory sizes at end of crawl
+dirsizesttl = 172800
 
 [workerbot]
 ; enable bot logs (True or False)
@@ -76,6 +82,10 @@ port = 9999
 [dupescheck]
 ; read size (bytes) for md5 sum check
 readsize = 65536
+; max size (bytes) of files to check
+maxsize = 1073741824
+; bytes to check at start and end of file before doing md5 sum check
+checkbytes = 8
 
 [crawlbot]
 ; continuous scanner

root/etc/cont-init.d/30-config

@@ -8,6 +8,20 @@ ES_PORT=${ES_PORT:-9200}
 ES_USER=${ES_USER:-elastic}
 ES_PASS=${ES_PASS:-changeme}
 
+# persist env variables
+if [ ! -f "/config/.env" ]; then
+	echo "export REDIS_HOST="$REDIS_HOST"
+export REDIS_PORT="$REDIS_PORT"
+export ES_HOST="$ES_HOST"
+export ES_PORT="$ES_PORT"
+export ES_USER="$ES_USER"
+export ES_PASS="$ES_PASS"" > /config/.env
+fi
+
+if [ ! -f "/etc/profile.d/diskover.sh" ]; then
+	ln -s /config/.env /etc/profile.d/diskover.sh
+fi
+
 # create folders
 mkdir -p \
 	/config/log/diskover
@@ -30,12 +44,6 @@ if [ ! -f "/config/crontab" ]; then
 	cp /defaults/crontab /config/crontab
 fi
 
-if [ ! -f "/etc/crontabs/abc" ]; then
-	if [ $USE_CRON ]; then
-		ln -s /config/crontab /etc/crontabs/abc
-	fi
-fi
-
 # permissions
 chmod +x \
 	/app/dispatcher.sh
@@ -44,10 +52,12 @@ chown -R abc:abc \
 	/config
 
 # sleep to allow time for elasticsearch to start
-sleep 10
 cd /app/diskover || exit
 
 # run initial retrieval
-echo "Initial run of dispatcher in progress"
-exec \
-	s6-setuidgid abc /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 &
+if [ $RUN_ON_START ]; then
+	sleep 10
+	echo "Initial run of dispatcher in progress"
+	exec \
+		s6-setuidgid abc /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 &
+fi

root/etc/services.d/cron/run

@@ -1,3 +1,3 @@
 #!/usr/bin/with-contenv bash
 
-/usr/sbin/crond -f -S -l 0 -c /etc/crontabs
+# /usr/sbin/crond -f -S -l 0 -c /etc/crontabs