diff --git a/Dockerfile b/Dockerfile index 8b95947..81e25ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ LABEL maintainer="alex-phillips" RUN \ echo "**** install build packages ****" && \ apk add --no-cache \ + grep \ logrotate \ ncurses \ python3 && \ diff --git a/READMETEMPLATE.md b/READMETEMPLATE.md index f313505..291c504 100644 --- a/READMETEMPLATE.md +++ b/READMETEMPLATE.md @@ -33,6 +33,7 @@ docker create \ --name=diskover \ -v :/config \ -v :/data \ + -v :/etc/crontabs/abc \ -e PGID= -e PUID= \ -e REDIS_HOST= \ -e REDIS_PORT= \ @@ -41,9 +42,8 @@ docker create \ -e ES_USER= \ -e ES_PASS= \ -e INDEX_NAME= \ - -e DISPATCH_OPTS= \ + -e DISKOVER_OPTS= \ -e WORKER_OPTS= \ - -e USE_CRON= \ alexphillips/diskover ``` @@ -61,6 +61,7 @@ http://192.168.x.x:8080 would show you what's running INSIDE the container on po | `-p 1234` | the port(s) | | `-v /config` | diskover config | | `-v /data` | data directory to scan | +| `-v /etc/crontabs/abc` | crontab file to run dispatcher in cron | | `-e PGID` | for GroupID, see below for explanation | | `-e PUID` | for UserID, see below for explanation | | `-e REDIS_HOST` | Redis host [default: redis] | @@ -71,9 +72,8 @@ http://192.168.x.x:8080 would show you what's running INSIDE the container on po | `-e ES_PASS` | ElasticSearch password [default: changeme] | | `-e INDEX_PREFIX` | ElasticSearch index prefix [default: 'diskover-'] | | `-e INDEX_NAME` | ElasticSearch index name [default: $INDEX_PREFIX-TIMESTAMP] | -| `-e DISPATCH_OPTS` | Arguments to pass to the dispatcher [default: '-a -i $INDEX_NAME'] | +| `-e DISKOVER_OPTS` | Arguments to pass to the dispatcher [default: '-a -i $INDEX_NAME'] | | `-e WORKER_OPTS` | Arguments to pass to the bot launcher [default: ''] | -| `-e USE_CRON` | Whether the dispatcher should use cron to run on a schedule [default: false] |   diff --git a/root/app/dispatcher.sh b/root/app/dispatcher.sh index 6ba7022..def3788 100644 --- a/root/app/dispatcher.sh +++ b/root/app/dispatcher.sh @@ -1,19 +1,34 @@ #!/usr/bin/with-contenv bash +. /config/.env + TODAY=$(date +%Y-%m-%d) INDEX_PREFIX=${INDEX_PREFIX:-diskover-} INDEX_NAME=${INDEX_NAME:-$INDEX_PREFIX$TODAY} -DISPATCH_OPTS=${DISPATCH_OPTS:-"-a -i $INDEX_NAME"} -WORKER_OPTS=${WORKER_OPTS:-""} - -sleep 10 +DISKOVER_OPTS=${DISKOVER_OPTS:-"-d /data -a -i $INDEX_NAME"} cd /app/diskover || exit -# Run the workers -/bin/bash ./diskover-bot-launcher.sh $WORKER_OPTS > /dev/null 2>&1 +# killing existing workers before starting new ones +echo "killing existing workers..." +if [ -f "/tmp/diskover_bot_pids" ]; then + /bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1 + sleep 3 +fi -/usr/bin/python3 ./diskover.py -d /data $DISPATCH_OPTS && \ - sleep 10 && \ - echo "dispatcher is done, killing off workers..." && \ - /bin/bash ./diskover-bot-launcher.sh -k +# empty existing redis queue +echo "emptying current redis queues..." +rq empty -u redis://$REDIS_HOST:$REDIS_PORT diskover_crawl +sleep 3 +rq empty -u redis://$REDIS_HOST:$REDIS_PORT failed +sleep 3 + +echo "killing dangling workers..." +/bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1 +sleep 3 + +echo "starting workers with following options: $WORKER_OPTS" +/bin/bash /app/diskover/diskover-bot-launcher.sh $WORKER_OPTS + +echo "starting crawler with following options: $DISKOVER_OPTS" +/usr/bin/python3 ./diskover.py $DISKOVER_OPTS diff --git a/root/defaults/crontab b/root/defaults/crontab index 1bf95a0..0467791 100644 --- a/root/defaults/crontab +++ b/root/defaults/crontab @@ -1 +1 @@ -0 3 * * * /app/diskover/diskover-bot-launcher.sh -r && /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 +0 3 * * * /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 diff --git a/root/defaults/diskover.cfg b/root/defaults/diskover.cfg index 3ca16fd..8f28944 100644 --- a/root/defaults/diskover.cfg +++ b/root/defaults/diskover.cfg @@ -2,8 +2,8 @@ ; if you make any changes, restart worker bots so they get the new config [excludes] -; directory names and absolute paths you want to exclude from crawl, case-sensitive, can include wildcard (.* or tmp* or /dir/dirname* etc) -dirs = .*,.snapshot,.Snapshot +; directory names and absolute paths you want to exclude from crawl, case-sensitive, can include wildcards (.* or backup* or /dir/dirname* or *tmp or *tmp* etc) +dirs = .*,.snapshot,.Snapshot,.zfs ; files you want to exclude from crawl, case-sensitive, can include wildcards (.*, *.doc or NULLEXT for files with no extension) files = .*,Thumbs.db,.DS_Store,._.DS_Store,.localized,desktop.ini @@ -18,6 +18,10 @@ files = .*,Thumbs.db,.DS_Store,._.DS_Store,.localized,desktop.ini ;files = [{"name": [], "name_exclude": [], "ext": ["tmp*", "TMP*", "temp*", "TEMP*", "cache*", "CACHE*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}] ;dirs = [{"name": ["*tmp*", "*TMP*", "*temp*", "*TEMP*", "*Temp*", "*cache*", "*CACHE*", "*Cache*"], "name_exclude": ["*templates*", "*Templates*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}] +[treewalk] +; number of processes to run when doing parallel tree walking of rootdir +procs = 8 + [elasticsearch] ; uncomment the below three lines if you are using AWS ES ;aws = False @@ -50,9 +54,11 @@ replicas = 1 host = {{REDIS_HOST}} port = {{REDIS_PORT}} ;password = -; how long directory times lives in Redis (seconds) (default 1 week) +; how long directory keys lives in Redis (seconds) ; used for -I index2 when comparing directory times to get metadata from index2 instead of off disk dirtimesttl = 604800 +; used for calculating directory sizes at end of crawl +dirsizesttl = 172800 [workerbot] ; enable bot logs (True or False) @@ -76,6 +82,10 @@ port = 9999 [dupescheck] ; read size (bytes) for md5 sum check readsize = 65536 +; max size (bytes) of files to check +maxsize = 1073741824 +; bytes to check at start and end of file before doing md5 sum check +checkbytes = 8 [crawlbot] ; continuous scanner diff --git a/root/etc/cont-init.d/30-config b/root/etc/cont-init.d/30-config index e505439..71488ea 100644 --- a/root/etc/cont-init.d/30-config +++ b/root/etc/cont-init.d/30-config @@ -8,6 +8,20 @@ ES_PORT=${ES_PORT:-9200} ES_USER=${ES_USER:-elastic} ES_PASS=${ES_PASS:-changeme} +# persist env variables +if [ ! -f "/config/.env" ]; then + echo "export REDIS_HOST="$REDIS_HOST" +export REDIS_PORT="$REDIS_PORT" +export ES_HOST="$ES_HOST" +export ES_PORT="$ES_PORT" +export ES_USER="$ES_USER" +export ES_PASS="$ES_PASS"" > /config/.env +fi + +if [ ! -f "/etc/profile.d/diskover.sh" ]; then + ln -s /config/.env /etc/profile.d/diskover.sh +fi + # create folders mkdir -p \ /config/log/diskover @@ -30,12 +44,6 @@ if [ ! -f "/config/crontab" ]; then cp /defaults/crontab /config/crontab fi -if [ ! -f "/etc/crontabs/abc" ]; then - if [ $USE_CRON ]; then - ln -s /config/crontab /etc/crontabs/abc - fi -fi - # permissions chmod +x \ /app/dispatcher.sh @@ -44,10 +52,12 @@ chown -R abc:abc \ /config # sleep to allow time for elasticsearch to start -sleep 10 cd /app/diskover || exit # run initial retrieval -echo "Initial run of dispatcher in progress" -exec \ - s6-setuidgid abc /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 & +if [ $RUN_ON_START ]; then + sleep 10 + echo "Initial run of dispatcher in progress" + exec \ + s6-setuidgid abc /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 & +fi diff --git a/root/etc/services.d/cron/run b/root/etc/services.d/cron/run index a1b27f4..766473c 100644 --- a/root/etc/services.d/cron/run +++ b/root/etc/services.d/cron/run @@ -1,3 +1,3 @@ #!/usr/bin/with-contenv bash -/usr/sbin/crond -f -S -l 0 -c /etc/crontabs +# /usr/sbin/crond -f -S -l 0 -c /etc/crontabs