From 0e20d746230e151aecdabd373dda77b3a2f048f4 Mon Sep 17 00:00:00 2001 From: alex-phillips Date: Tue, 31 Jul 2018 12:45:52 -0400 Subject: [PATCH 1/2] fixed issue with INDEX_NAME variable not being properly evaluated within the conditional of the DISKOVER_OPTS conditional. separated it into a separate string concat to get around this (i'm sure there's probably a better way though) --- root/app/dispatcher.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/root/app/dispatcher.sh b/root/app/dispatcher.sh index def3788..060490d 100644 --- a/root/app/dispatcher.sh +++ b/root/app/dispatcher.sh @@ -5,7 +5,9 @@ TODAY=$(date +%Y-%m-%d) INDEX_PREFIX=${INDEX_PREFIX:-diskover-} INDEX_NAME=${INDEX_NAME:-$INDEX_PREFIX$TODAY} -DISKOVER_OPTS=${DISKOVER_OPTS:-"-d /data -a -i $INDEX_NAME"} +DISKOVER_OPTS=${DISKOVER_OPTS:-"-d /data -a"} + +DISKOVER_OPTS="$DISKOVER_OPTS -i $INDEX_NAME" cd /app/diskover || exit From ff4ba8a75e2d096d3ee1c3b4a6f9c73cd7bd9788 Mon Sep 17 00:00:00 2001 From: alex-phillips Date: Thu, 2 Aug 2018 14:56:50 -0400 Subject: [PATCH 2/2] config files renamed to higher numbers to prevent race conditions, config env variables now stored in arrays --- root/app/dispatcher.sh | 18 ++++--- ...{30-diskover-config => 50-diskover-config} | 50 ++++++++++--------- ...over-web-config => 60-diskover-web-config} | 17 ++++--- root/etc/services.d/rq-dashboard/run | 7 +-- 4 files changed, 49 insertions(+), 43 deletions(-) rename root/etc/cont-init.d/{30-diskover-config => 50-diskover-config} (50%) rename root/etc/cont-init.d/{31-diskover-web-config => 60-diskover-web-config} (75%) diff --git a/root/app/dispatcher.sh b/root/app/dispatcher.sh index 060490d..9c71733 100644 --- a/root/app/dispatcher.sh +++ b/root/app/dispatcher.sh @@ -2,10 +2,12 @@ . /config/.env -TODAY=$(date +%Y-%m-%d) -INDEX_PREFIX=${INDEX_PREFIX:-diskover-} -INDEX_NAME=${INDEX_NAME:-$INDEX_PREFIX$TODAY} -DISKOVER_OPTS=${DISKOVER_OPTS:-"-d /data -a"} +# define array for input values +declare -A DISKOVER_ARRAY +DISKOVER_ARRAY[TODAY]=$(date +%Y-%m-%d) +DISKOVER_ARRAY[INDEX_PREFIX]=${INDEX_PREFIX:-diskover-} +DISKOVER_ARRAY[INDEX_NAME]=${INDEX_NAME:-$INDEX_PREFIX$TODAY} +DISKOVER_ARRAY[DISKOVER_OPTS]=${DISKOVER_OPTS:-"-d /data -a"} DISKOVER_OPTS="$DISKOVER_OPTS -i $INDEX_NAME" @@ -29,8 +31,8 @@ echo "killing dangling workers..." /bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1 sleep 3 -echo "starting workers with following options: $WORKER_OPTS" -/bin/bash /app/diskover/diskover-bot-launcher.sh $WORKER_OPTS +echo "starting workers with following options: ${DISKOVER_ARRAY[WORKER_OPTS]}" +/bin/bash /app/diskover/diskover-bot-launcher.sh ${DISKOVER_ARRAY[WORKER_OPTS]} -echo "starting crawler with following options: $DISKOVER_OPTS" -/usr/bin/python3 ./diskover.py $DISKOVER_OPTS +echo "starting crawler with following options: ${DISKOVER_ARRAY[DISKOVER_OPTS]}" +/usr/bin/python3 ./diskover.py ${DISKOVER_ARRAY[DISKOVER_OPTS]} diff --git a/root/etc/cont-init.d/30-diskover-config b/root/etc/cont-init.d/50-diskover-config similarity index 50% rename from root/etc/cont-init.d/30-diskover-config rename to root/etc/cont-init.d/50-diskover-config index 23f8ea0..2c4bb1d 100644 --- a/root/etc/cont-init.d/30-diskover-config +++ b/root/etc/cont-init.d/50-diskover-config @@ -1,21 +1,25 @@ #!/usr/bin/with-contenv bash # set default values for variables -REDIS_HOST=${REDIS_HOST:-redis} -REDIS_PORT=${REDIS_PORT:-6379} -ES_HOST=${ES_HOST:-elasticsearch} -ES_PORT=${ES_PORT:-9200} -ES_USER=${ES_USER:-elastic} -ES_PASS=${ES_PASS:-changeme} +declare -A DISKOVER_CONF +DISKOVER_CONF[REDIS_HOST]=${REDIS_HOST:-redis} +DISKOVER_CONF[REDIS_PORT]=${REDIS_PORT:-6379} +DISKOVER_CONF[ES_HOST]=${ES_HOST:-elasticsearch} +DISKOVER_CONF[ES_PORT]=${ES_PORT:-9200} +DISKOVER_CONF[ES_USER]=${ES_USER:-elastic} +DISKOVER_CONF[ES_PASS]=${ES_PASS:-changeme} +DISKOVER_CONF[RUN_ON_START]=${RUN_ON_START:-false} +DISKOVER_CONF[USE_CRON]=${USE_CRON:-false} # persist env variables if [ ! -f "/config/.env" ]; then - echo "export REDIS_HOST="$REDIS_HOST" -export REDIS_PORT="$REDIS_PORT" -export ES_HOST="$ES_HOST" -export ES_PORT="$ES_PORT" -export ES_USER="$ES_USER" -export ES_PASS="$ES_PASS"" > /config/.env + # sed in values or skip if value not set + for KEY in "${!DISKOVER_CONF[@]}"; do \ + if [[ ${DISKOVER_CONF[$KEY]} == "" ]]; then \ + : + else echo "export $KEY=${DISKOVER_CONF[$KEY]}" >> /config/.env + fi + done fi if [ ! -f "/etc/profile.d/diskover.sh" ]; then @@ -28,12 +32,12 @@ mkdir -p \ if [ ! -f "/config/diskover.cfg" ]; then mv /defaults/diskover.cfg /config/diskover.cfg - sed -i 's|{{REDIS_HOST}}|'$REDIS_HOST'|g' /config/diskover.cfg - sed -i 's|{{REDIS_PORT}}|'$REDIS_PORT'|g' /config/diskover.cfg - sed -i 's|{{ES_HOST}}|'$ES_HOST'|g' /config/diskover.cfg - sed -i 's|{{ES_PORT}}|'$ES_PORT'|g' /config/diskover.cfg - sed -i 's|{{ES_USER}}|'$ES_USER'|g' /config/diskover.cfg - sed -i 's|{{ES_PASS}}|'$ES_PASS'|g' /config/diskover.cfg + for KEY in "${!DISKOVER_CONF[@]}"; do \ + if [[ ${DISKOVER_CONF[$KEY]} == "" ]]; then \ + : + else sed -i 's|{{'$KEY'}}|'${DISKOVER_CONF[$KEY]}'|g' /config/diskover.cfg + fi + done fi if [ ! -f "/app/diskover/diskover.cfg" ]; then @@ -46,7 +50,7 @@ fi # import user crontabs rm /etc/crontabs/* -if [ $USE_CRON ]; then +if [ ${DISKOVER_CONF[USE_CRON]} == true ]; then cp /config/crontabs/* /etc/crontabs/ fi @@ -57,13 +61,11 @@ chown -R abc:abc \ /app/diskover \ /config -# sleep to allow time for elasticsearch to start -cd /app/diskover || exit - # run initial retrieval -if [ $RUN_ON_START ]; then - sleep 10 +if [ ${DISKOVER_CONF[RUN_ON_START]} == true ]; then echo "Initial run of dispatcher in progress" + # sleep to allow time for elasticsearch to start + sleep 10 exec \ s6-setuidgid abc /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 & fi diff --git a/root/etc/cont-init.d/31-diskover-web-config b/root/etc/cont-init.d/60-diskover-web-config similarity index 75% rename from root/etc/cont-init.d/31-diskover-web-config rename to root/etc/cont-init.d/60-diskover-web-config index 49c85e6..9d31663 100644 --- a/root/etc/cont-init.d/31-diskover-web-config +++ b/root/etc/cont-init.d/60-diskover-web-config @@ -6,17 +6,18 @@ if [ $DISABLE_WEB ]; then fi # set default values for variables -ES_HOST=${ES_HOST:-elasticsearch} -ES_PORT=${ES_PORT:-9200} -ES_USER=${ES_USER:-elastic} -ES_PASS=${ES_PASS:-changeme} +declare -A WEB_CONF +WEB_CONF[ES_HOST]=${ES_HOST:-elasticsearch} +WEB_CONF[ES_PORT]=${ES_PORT:-9200} +WEB_CONF[ES_USER]=${ES_USER:-elastic} +WEB_CONF[ES_PASS]=${ES_PASS:-changeme} if [ ! -f "/config/Constants.php" ]; then cp /app/diskover-web/src/diskover/Constants.php.sample /config/Constants.php - sed -i "s|ES_HOST = 'localhost'|ES_HOST = '"$ES_HOST"'|g" /config/Constants.php - sed -i "s|ES_PORT = 9200|ES_PORT = "$ES_PORT"|g" /config/Constants.php - sed -i "s|ES_USER = ''|ES_USER = '"$ES_USER"'|g" /config/Constants.php - sed -i "s|ES_PASS = ''|ES_PASS = '"$ES_PASS"'|g" /config/Constants.php + sed -i "s|ES_HOST = 'localhost'|ES_HOST = '"${WEB_CONF[ES_HOST]}"'|g" /config/Constants.php + sed -i "s|ES_PORT = 9200|ES_PORT = "${WEB_CONF[ES_PORT]}"|g" /config/Constants.php + sed -i "s|ES_USER = ''|ES_USER = '"${WEB_CONF[ES_USER]}"'|g" /config/Constants.php + sed -i "s|ES_PASS = ''|ES_PASS = '"${WEB_CONF[ES_PASS]}"'|g" /config/Constants.php sed -i "s|ENABLE_SOCKET_CLIENT = FALSE|ENABLE_SOCKET_CLIENT = true|g" /config/Constants.php sed -i "s|SOCKET_LISTENER_HOST = '127.0.0.1'|SOCKET_LISTENER_HOST = 'diskover'|g" /config/Constants.php fi diff --git a/root/etc/services.d/rq-dashboard/run b/root/etc/services.d/rq-dashboard/run index f611f47..a30104a 100644 --- a/root/etc/services.d/rq-dashboard/run +++ b/root/etc/services.d/rq-dashboard/run @@ -1,7 +1,8 @@ #!/usr/bin/with-contenv bash -REDIS_HOST=${REDIS_HOST:-redis} -REDIS_PORT=${REDIS_PORT:-6379} +declare -A REDIS_CONF +REDIS_CONF[REDIS_HOST]=${REDIS_HOST:-redis} +REDIS_CONF[REDIS_PORT]=${REDIS_PORT:-6379} exec \ - s6-setuidgid abc /usr/bin/rq-dashboard -H $REDIS_HOST -P $REDIS_PORT + s6-setuidgid abc /usr/bin/rq-dashboard -H ${REDIS_CONF[REDIS_HOST]} -P ${REDIS_CONF[REDIS_PORT]}