LAB-linuxserver/docker-diskover
1
0
Fork 0
mirror of https://github.com/linuxserver/docker-diskover.git synced 2026-02-20 04:57:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

migrated to v2

Browse Source
This commit is contained in:
Alex Phillips 2021-10-11 14:41:13 -04:00
parent 60713305a3
commit 6c3c049ff6
24 changed files with 343 additions and 725 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/external_trigger.yml vendored
View File

@ -18,7 +18,7 @@ jobs:
fi
echo "**** External trigger running off of master branch. To disable this trigger, set a Github secret named \"PAUSE_EXTERNAL_TRIGGER_DISKOVER_MASTER\". ****"
echo "**** Retrieving external version ****"
EXT_RELEASE=$(curl -u "${{ secrets.CR_USER }}:${{ secrets.CR_PAT }}" -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" | jq -r '. | .tag_name')
EXT_RELEASE=$(curl -u "${{ secrets.CR_USER }}:${{ secrets.CR_PAT }}" -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases/latest" | jq -r '. | .tag_name')
if [ -z "${EXT_RELEASE}" ] || [ "${EXT_RELEASE}" == "null" ]; then
echo "**** Can't retrieve external version, exiting ****"
FAILURE_REASON="Can't retrieve external version for diskover branch master"

114
Dockerfile
View File

@ -1,75 +1,57 @@
FROM ghcr.io/linuxserver/baseimage-alpine-nginx:3.12
FROM ghcr.io/linuxserver/baseimage-alpine-nginx:3.14
# set version label
ARG BUILD_DATE
ARG VERSION
ARG DISKOVER_VERSION
ARG DISKOVER_RELEASE
LABEL build_version="Linuxserver.io version:- ${VERSION} Build-date:- ${BUILD_DATE}"
LABEL maintainer="alex-phillips"
RUN \
echo "**** install build packages ****" && \
apk add --no-cache --virtual=build-dependencies \
composer \
curl \
gcc \
musl-dev \
python3-dev && \
echo "**** install runtime packages ****" && \
apk add --no-cache \
grep \
ncurses \
php7-curl \
php7-phar \
py3-pip \
python3 && \
echo "**** install diskover ****" && \
mkdir -p /app/diskover && \
if [ -z ${DISKOVER_VERSION+x} ]; then \
DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
fi && \
curl -o \
/tmp/diskover.tar.gz -L \
"https://github.com/alex-phillips/diskover/archive/${DISKOVER_VERSION}.tar.gz" && \
tar xf \
/tmp/diskover.tar.gz -C \
/app/diskover/ --strip-components=1 && \
echo "**** install diskover-web ****" && \
mkdir -p /app/diskover-web && \
DISKOVER_WEB_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
if [ "${DISKOVER_VERSION}" != "${DISKOVER_WEB_VERSION}" ] || [ -z ${DISKOVER_VERSION+x} ]; then \
DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
fi && \
curl -o \
/tmp/diskover-web.tar.gz -L \
"https://github.com/alex-phillips/diskover-web/archive/${DISKOVER_VERSION}.tar.gz" && \
tar xf \
/tmp/diskover-web.tar.gz -C \
/app/diskover-web/ --strip-components=1 && \
echo "**** install pip packages ****" && \
cd /app/diskover && \
pip3 install --no-cache-dir -r requirements.txt && \
pip3 install rq-dashboard && \
echo "**** install composer packages ****" && \
cd /app/diskover-web && \
composer install && \
echo "**** fix logrotate ****" && \
sed -i "s#/var/log/messages {}.*# #g" /etc/logrotate.conf && \
echo "**** symlink python3 ****" && \
ln -s /usr/bin/python3 /usr/bin/python && \
echo "**** cleanup ****" && \
apk del --purge \
build-dependencies && \
rm -rf \
/root/.cache \
/tmp/*
echo "**** install build packages ****" && \
apk add --no-cache --virtual=build-dependencies \
build-base \
gcc \
py3-pip \
python3-dev \
composer \
curl \
git \
jq \
nodejs \
npm && \
echo "**** install runtime packages ****" && \
apk add --no-cache \
libldap \
ncurses \
php7-curl \
php7-ldap \
py3-requests \
py3-urllib3 \
py3-xxhash \
python3 && \
echo "**** install diskover ****" && \
if [ -z ${DISKOVER_RELEASE+x} ]; then \
DISKOVER_RELEASE=$(curl -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases" \
| jq -r '.[0] | .tag_name'); \
fi && \
curl -o \
/tmp/diskover.tar.gz -L \
"https://github.com/diskoverdata/diskover-community/archive/${DISKOVER_RELEASE}.tar.gz" && \
tar xf \
/tmp/diskover.tar.gz -C \
/app/ --strip-components=1 && \
cd /app/diskover && \
pip3 install --no-cache-dir -r requirements.txt && \
echo "**** overlay-fs workaround ****" && \
mv /app/diskover /app/diskover-tmp && \
mv /app/diskover-web /app/diskover-web-tmp && \
echo "**** cleanup ****" && \
apk del --purge \
build-dependencies && \
rm -rf \
/tmp/* \
/root/.cache
# copy local files
COPY root/ /
# ports and volumes
EXPOSE 8000
VOLUME /config
# add local files
COPY ./root/ /

117
Dockerfile.aarch64
View File

@ -1,78 +1,57 @@
FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm64v8-3.12
FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm64v8-3.14
# set version label
ARG BUILD_DATE
ARG VERSION
ARG DISKOVER_VERSION
ARG DISKOVER_RELEASE
LABEL build_version="Linuxserver.io version:- ${VERSION} Build-date:- ${BUILD_DATE}"
LABEL maintainer="alex-phillips"
RUN \
echo "**** install build packages ****" && \
apk add --no-cache --virtual=build-dependencies \
curl \
gcc \
musl-dev \
python3-dev && \
echo "**** install runtime packages ****" && \
apk add --no-cache \
grep \
ncurses \
php7-curl \
php7-phar \
py3-pip \
python3 && \
echo "**** install composer ****" && \
curl \
-sS https://getcomposer.org/installer \
| php -- --install-dir=/usr/bin --filename=composer && \
echo "**** install diskover ****" && \
mkdir -p /app/diskover && \
if [ -z ${DISKOVER_VERSION+x} ]; then \
DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
fi && \
curl -o \
/tmp/diskover.tar.gz -L \
"https://github.com/alex-phillips/diskover/archive/${DISKOVER_VERSION}.tar.gz" && \
tar xf \
/tmp/diskover.tar.gz -C \
/app/diskover/ --strip-components=1 && \
echo "**** install diskover-web ****" && \
mkdir -p /app/diskover-web && \
DISKOVER_WEB_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
if [ "${DISKOVER_VERSION}" != "${DISKOVER_WEB_VERSION}" ] || [ -z ${DISKOVER_VERSION+x} ]; then \
DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
fi && \
curl -o \
/tmp/diskover-web.tar.gz -L \
"https://github.com/alex-phillips/diskover-web/archive/${DISKOVER_VERSION}.tar.gz" && \
tar xf \
/tmp/diskover-web.tar.gz -C \
/app/diskover-web/ --strip-components=1 && \
echo "**** install pip packages ****" && \
cd /app/diskover && \
pip3 install --no-cache-dir -r requirements.txt && \
pip3 install rq-dashboard && \
echo "**** install composer packages ****" && \
cd /app/diskover-web && \
composer install && \
echo "**** fix logrotate ****" && \
sed -i "s#/var/log/messages {}.*# #g" /etc/logrotate.conf && \
echo "**** symlink python3 ****" && \
ln -s /usr/bin/python3 /usr/bin/python && \
echo "**** cleanup ****" && \
apk del --purge \
build-dependencies && \
rm -rf \
/root/.cache \
/tmp/*
echo "**** install build packages ****" && \
apk add --no-cache --virtual=build-dependencies \
build-base \
gcc \
py3-pip \
python3-dev \
composer \
curl \
git \
jq \
nodejs \
npm && \
echo "**** install runtime packages ****" && \
apk add --no-cache \
libldap \
ncurses \
php7-curl \
php7-ldap \
py3-requests \
py3-urllib3 \
py3-xxhash \
python3 && \
echo "**** install diskover ****" && \
if [ -z ${DISKOVER_RELEASE+x} ]; then \
DISKOVER_RELEASE=$(curl -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases" \
| jq -r '.[0] | .tag_name'); \
fi && \
curl -o \
/tmp/diskover.tar.gz -L \
"https://github.com/diskoverdata/diskover-community/archive/${DISKOVER_RELEASE}.tar.gz" && \
tar xf \
/tmp/diskover.tar.gz -C \
/app/ --strip-components=1 && \
cd /app/diskover && \
pip3 install --no-cache-dir -r requirements.txt && \
echo "**** overlay-fs workaround ****" && \
mv /app/diskover /app/diskover-tmp && \
mv /app/diskover-web /app/diskover-web-tmp && \
echo "**** cleanup ****" && \
apk del --purge \
build-dependencies && \
rm -rf \
/tmp/* \
/root/.cache
# copy local files
COPY root/ /
# ports and volumes
EXPOSE 8000
VOLUME /config
# add local files
COPY ./root/ /

117
Dockerfile.armhf
View File

@ -1,78 +1,57 @@
FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm32v7-3.12
FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm32v7-3.14
# set version label
ARG BUILD_DATE
ARG VERSION
ARG DISKOVER_VERSION
ARG DISKOVER_RELEASE
LABEL build_version="Linuxserver.io version:- ${VERSION} Build-date:- ${BUILD_DATE}"
LABEL maintainer="alex-phillips"
RUN \
echo "**** install build packages ****" && \
apk add --no-cache --virtual=build-dependencies \
curl \
gcc \
musl-dev \
python3-dev && \
echo "**** install runtime packages ****" && \
apk add --no-cache \
grep \
ncurses \
php7-curl \
php7-phar \
py3-pip \
python3 && \
echo "**** install composer ****" && \
curl \
-sS https://getcomposer.org/installer \
| php -- --install-dir=/usr/bin --filename=composer && \
echo "**** install diskover ****" && \
mkdir -p /app/diskover && \
if [ -z ${DISKOVER_VERSION+x} ]; then \
DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
fi && \
curl -o \
/tmp/diskover.tar.gz -L \
"https://github.com/alex-phillips/diskover/archive/${DISKOVER_VERSION}.tar.gz" && \
tar xf \
/tmp/diskover.tar.gz -C \
/app/diskover/ --strip-components=1 && \
echo "**** install diskover-web ****" && \
mkdir -p /app/diskover-web && \
DISKOVER_WEB_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
if [ "${DISKOVER_VERSION}" != "${DISKOVER_WEB_VERSION}" ] || [ -z ${DISKOVER_VERSION+x} ]; then \
DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \
| awk '/tag_name/{print $4;exit}' FS='[""]'); \
fi && \
curl -o \
/tmp/diskover-web.tar.gz -L \
"https://github.com/alex-phillips/diskover-web/archive/${DISKOVER_VERSION}.tar.gz" && \
tar xf \
/tmp/diskover-web.tar.gz -C \
/app/diskover-web/ --strip-components=1 && \
echo "**** install pip packages ****" && \
cd /app/diskover && \
pip3 install --no-cache-dir -r requirements.txt && \
pip3 install rq-dashboard && \
echo "**** install composer packages ****" && \
cd /app/diskover-web && \
composer install && \
echo "**** fix logrotate ****" && \
sed -i "s#/var/log/messages {}.*# #g" /etc/logrotate.conf && \
echo "**** symlink python3 ****" && \
ln -s /usr/bin/python3 /usr/bin/python && \
echo "**** cleanup ****" && \
apk del --purge \
build-dependencies && \
rm -rf \
/root/.cache \
/tmp/*
echo "**** install build packages ****" && \
apk add --no-cache --virtual=build-dependencies \
build-base \
gcc \
py3-pip \
python3-dev \
composer \
curl \
git \
jq \
nodejs \
npm && \
echo "**** install runtime packages ****" && \
apk add --no-cache \
libldap \
ncurses \
php7-curl \
php7-ldap \
py3-requests \
py3-urllib3 \
py3-xxhash \
python3 && \
echo "**** install diskover ****" && \
if [ -z ${DISKOVER_RELEASE+x} ]; then \
DISKOVER_RELEASE=$(curl -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases" \
| jq -r '.[0] | .tag_name'); \
fi && \
curl -o \
/tmp/diskover.tar.gz -L \
"https://github.com/diskoverdata/diskover-community/archive/${DISKOVER_RELEASE}.tar.gz" && \
tar xf \
/tmp/diskover.tar.gz -C \
/app/ --strip-components=1 && \
cd /app/diskover && \
pip3 install --no-cache-dir -r requirements.txt && \
echo "**** overlay-fs workaround ****" && \
mv /app/diskover /app/diskover-tmp && \
mv /app/diskover-web /app/diskover-web-tmp && \
echo "**** cleanup ****" && \
apk del --purge \
build-dependencies && \
rm -rf \
/tmp/* \
/root/.cache
# copy local files
COPY root/ /
# ports and volumes
EXPOSE 8000
VOLUME /config
# add local files
COPY ./root/ /

12
Jenkinsfile vendored
View File

@ -18,8 +18,8 @@ pipeline {
GITLAB_NAMESPACE=credentials('gitlab-namespace-id')
SCARF_TOKEN=credentials('scarf_api_key')
EXT_GIT_BRANCH = 'master'
EXT_USER = 'alex-phillips'
EXT_REPO = 'diskover'
EXT_USER = 'diskoverdata'
EXT_REPO = 'diskover-community'
BUILD_VERSION_ARG = 'DISKOVER_VERSION'
LS_USER = 'linuxserver'
LS_REPO = 'docker-diskover'
@ -467,7 +467,7 @@ pipeline {
--label \"org.opencontainers.image.licenses=GPL-3.0-only\" \
--label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \
--label \"org.opencontainers.image.title=Diskover\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--no-cache --pull -t ${IMAGE}:${META_TAG} \
--build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ."
}
@ -497,7 +497,7 @@ pipeline {
--label \"org.opencontainers.image.licenses=GPL-3.0-only\" \
--label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \
--label \"org.opencontainers.image.title=Diskover\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--no-cache --pull -t ${IMAGE}:amd64-${META_TAG} \
--build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ."
}
@ -524,7 +524,7 @@ pipeline {
--label \"org.opencontainers.image.licenses=GPL-3.0-only\" \
--label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \
--label \"org.opencontainers.image.title=Diskover\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--no-cache --pull -f Dockerfile.armhf -t ${IMAGE}:arm32v7-${META_TAG} \
--build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ."
sh "docker tag ${IMAGE}:arm32v7-${META_TAG} ghcr.io/linuxserver/lsiodev-buildcache:arm32v7-${COMMIT_SHA}-${BUILD_NUMBER}"
@ -558,7 +558,7 @@ pipeline {
--label \"org.opencontainers.image.licenses=GPL-3.0-only\" \
--label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \
--label \"org.opencontainers.image.title=Diskover\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \
--no-cache --pull -f Dockerfile.aarch64 -t ${IMAGE}:arm64v8-${META_TAG} \
--build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ."
sh "docker tag ${IMAGE}:arm64v8-${META_TAG} ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}"

76
README.md
View File

@ -39,9 +39,9 @@ Find us at:
[![Jenkins Build](https://img.shields.io/jenkins/build?labelColor=555555&logoColor=ffffff&style=for-the-badge&jobUrl=https%3A%2F%2Fci.linuxserver.io%2Fjob%2FDocker-Pipeline-Builders%2Fjob%2Fdocker-diskover%2Fjob%2Fmaster%2F&logo=jenkins)](https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-diskover/job/master/)
[![LSIO CI](https://img.shields.io/badge/dynamic/yaml?color=94398d&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=CI&query=CI&url=https%3A%2F%2Fci-tests.linuxserver.io%2Flinuxserver%2Fdiskover%2Flatest%2Fci-status.yml)](https://ci-tests.linuxserver.io/linuxserver/diskover/latest/index.html)
[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.
[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.
[![diskover](https://raw.githubusercontent.com/shirosaidev/diskover/master/docs/diskover.png)](https://github.com/shirosaidev/diskover)
[![diskover](https://raw.githubusercontent.com/shirosaidev/diskover/master/docs/diskover.png)](https://github.com/diskoverdata/diskover-community)
## Supported Architectures
@ -59,19 +59,14 @@ The architectures supported by this image are:
## Application Setup
Once running the URL will be `http://<host-ip>/` initial application spinup will take some time so please reload if you get an empty response. We highly reccomend using Docker compose for this image as it includes multiple database backends to link into.
If you are looking to mount the elasticsearch and redis data to your host machine for access neither of them currently support setting a custom UID or GID they will run by default as:
This application is dependent on an ElasticSearch instance. Please see the example compose file for additional information.
- Redis - UID=999 GID=999
- Elasticsearch - UID=1000 GID=1000
The default username is diskover with the password of **darkdata**, access the container at http://<host-ip>/. The UI may be unusable until a valid index has been created.
ElasticSearch also requires a sysctl setting on the host machine to run properly. Running `sysctl -w vm.max_map_count=262144` will solve this issue. To make this setting persistent through reboots, set this value in `/etc/sysctl.conf`.
The application doesn't start an index by default. A crontab is created inside of the `/config` directory and can be set up to run automated indexes of `/data`. Changes to this crontab file require a restart to apply. You can also manually run an index by executing `/app/diskover/diskover.py` either in interactive or detached mode:
If you simply want the application to work you can mount these to folders with 0777 permissions, otherwise you will need to create these users host level and set the folder ownership properly.
By default this compose example is pointed to a single directory and the UID and GID you pass to the diskover container needs to match that folders ownership. If these are shared folders with many owners the indexing will likely fail.
For specific questions or help setting up diskover in your environment please refer to the project's Github page [Diskover](https://github.com/shirosaidev/diskover).
* `docker exec -u abc -d /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the background
* `docker exec -u abc -it /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the foreground
## Usage
@ -83,49 +78,45 @@ Here are some example snippets to help you get started creating a container.
version: '2'
services:
diskover:
image: linuxserver/diskover
image: ghcr.io/linuxserver/diskover
container_name: diskover
environment:
- PUID=1000
- PGID=1000
- TZ=Europe/London
- REDIS_HOST=redis
- REDIS_PORT=6379
- TZ=America/New_York
- ES_HOST=elasticsearch
- ES_PORT=9200
- ES_USER=elastic
- ES_PASS=changeme
- RUN_ON_START=true
- USE_CRON=true
volumes:
- /path/to/diskover/config:/config
- /path/to/diskover/data:/data
ports:
- 80:80
- 9181:9181
- 9999:9999
mem_limit: 4096m
restart: unless-stopped
depends_on:
- elasticsearch
- redis
elasticsearch:
container_name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:5.6.9
volumes:
- ${DOCKER_HOME}/elasticsearch/data:/usr/share/elasticsearch/data
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
environment:
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms2048m -Xmx2048m"
- "ES_JAVA_OPTS=-Xms1g -Xmx1g"
ulimits:
memlock:
soft: -1
hard: -1
redis:
container_name: redis
image: redis:alpine
volumes:
- ${HOME}/docker/redis:/data
- /path/to/esdata:/usr/share/elasticsearch/data
ports:
- 9200:9200
depends_on:
- elasticsearch-helper
restart: unless-stopped
elasticsearch-helper:
image: alpine
command: sh -c "sysctl -w vm.max_map_count=262144"
privileged: true
```
@ -136,21 +127,12 @@ docker run -d \
--name=diskover \
-e PUID=1000 \
-e PGID=1000 \
-e TZ=Europe/London \
-e REDIS_HOST=redis \
-e REDIS_PORT=6379 \
-e TZ=America/New_York \
-e ES_HOST=elasticsearch \
-e ES_PORT=9200 \
-e ES_USER=elastic \
-e ES_PASS=changeme \
-e INDEX_NAME=diskover- \
-e DISKOVER_OPTS= \
-e WORKER_OPTS= \
-e RUN_ON_START=true \
-e USE_CRON=true \
-p 80:80 \
-p 9181:9181 \
-p 9999:9999 \
-v /path/to/diskover/config:/config \
-v /path/to/diskover/data:/data \
--restart unless-stopped \
@ -164,22 +146,13 @@ Container images are configured using parameters passed at runtime (such as thos
| Parameter | Function |
| :----: | --- |
| `-p 80` | diskover Web UI |
| `-p 9181` | rq-dashboard web UI |
| `-p 9999` | diskover socket server |
| `-e PUID=1000` | for UserID - see below for explanation |
| `-e PGID=1000` | for GroupID - see below for explanation |
| `-e TZ=Europe/London` | Specify a timezone to use EG Europe/London |
| `-e REDIS_HOST=redis` | Redis host (optional) |
| `-e REDIS_PORT=6379` | Redis port (optional) |
| `-e TZ=America/New_York` | Specify a timezone to use EG America/New_York |
| `-e ES_HOST=elasticsearch` | ElasticSearch host (optional) |
| `-e ES_PORT=9200` | ElasticSearch port (optional) |
| `-e ES_USER=elastic` | ElasticSearch username (optional) |
| `-e ES_PASS=changeme` | ElasticSearch password (optional) |
| `-e INDEX_NAME=diskover-` | Index name prefix (optional) |
| `-e DISKOVER_OPTS=` | Optional arguments to pass to the diskover crawler (optional) |
| `-e WORKER_OPTS=` | Optional argumens to pass to the diskover bots launcher (optional) |
| `-e RUN_ON_START=true` | Initiate a crawl every time the container is started (optional) |
| `-e USE_CRON=true` | Run a crawl on as a cron job (optional) |
| `-v /config` | Persistent config files |
| `-v /data` | Default mount point to crawl |
@ -292,6 +265,7 @@ Once registered you can define the dockerfile to use with `-f Dockerfile.aarch64
## Versions
* **11.10.21:** - Updated to diskover-community v2.
* **19.11.20:** - Fix pip packages.
* **19.12.19:** - Rebasing to alpine 3.11.
* **28.06.19:** - Rebasing to alpine 3.10.

4
jenkins-vars.yml
View File

@ -8,8 +8,8 @@ release_tag: latest
ls_branch: master
repo_vars:
- EXT_GIT_BRANCH = 'master'
- EXT_USER = 'alex-phillips'
- EXT_REPO = 'diskover'
- EXT_USER = 'diskoverdata'
- EXT_REPO = 'diskover-community'
- BUILD_VERSION_ARG = 'DISKOVER_VERSION'
- LS_USER = 'linuxserver'
- LS_REPO = 'docker-diskover'

65
readme-vars.yml
View File

@ -2,9 +2,9 @@
# project information
project_name: diskover
project_url: "https://github.com/shirosaidev/diskover"
project_url: "https://github.com/diskoverdata/diskover-community"
project_logo: "https://raw.githubusercontent.com/shirosaidev/diskover/master/docs/diskover.png"
project_blurb: "[{{ project_name }}]({{ project_url }}) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems."
project_blurb: "[{{ project_name }}]({{ project_url }}) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems."
project_lsio_github_repo_url: "https://github.com/linuxserver/docker-{{ project_name }}"
# supported architectures
@ -23,90 +23,73 @@ param_volumes:
param_usage_include_ports: true
param_ports:
- { external_port: "80", internal_port: "80", port_desc: "diskover Web UI" }
- { external_port: "9181", internal_port: "9181", port_desc: "rq-dashboard web UI" }
- { external_port: "9999", internal_port: "9999", port_desc: "diskover socket server" }
param_usage_include_env: true
param_env_vars:
- { env_var: "TZ", env_value: "Europe/London", desc: "Specify a timezone to use EG Europe/London"}
- { env_var: "REDIS_HOST", env_value: "redis", desc: "Redis host (optional)"}
- { env_var: "REDIS_PORT", env_value: "6379", desc: "Redis port (optional)"}
- { env_var: "TZ", env_value: "America/New_York", desc: "Specify a timezone to use EG America/New_York"}
- { env_var: "ES_HOST", env_value: "elasticsearch", desc: "ElasticSearch host (optional)"}
- { env_var: "ES_PORT", env_value: "9200", desc: "ElasticSearch port (optional)"}
- { env_var: "ES_USER", env_value: "elastic", desc: "ElasticSearch username (optional)"}
- { env_var: "ES_PASS", env_value: "changeme", desc: "ElasticSearch password (optional)"}
- { env_var: "INDEX_NAME", env_value: "diskover-", desc: "Index name prefix (optional)"}
- { env_var: "DISKOVER_OPTS", env_value: "", desc: "Optional arguments to pass to the diskover crawler (optional)"}
- { env_var: "WORKER_OPTS", env_value: "", desc: "Optional argumens to pass to the diskover bots launcher (optional)"}
- { env_var: "RUN_ON_START", env_value: "true", desc: "Initiate a crawl every time the container is started (optional)"}
- { env_var: "USE_CRON", env_value: "true", desc: "Run a crawl on as a cron job (optional)"}
custom_compose: |
version: '2'
services:
diskover:
image: linuxserver/diskover
image: ghcr.io/linuxserver/diskover
container_name: diskover
environment:
- PUID=1000
- PGID=1000
- TZ=Europe/London
- REDIS_HOST=redis
- REDIS_PORT=6379
- TZ=America/New_York
- ES_HOST=elasticsearch
- ES_PORT=9200
- ES_USER=elastic
- ES_PASS=changeme
- RUN_ON_START=true
- USE_CRON=true
volumes:
- /path/to/diskover/config:/config
- /path/to/diskover/data:/data
ports:
- 80:80
- 9181:9181
- 9999:9999
mem_limit: 4096m
restart: unless-stopped
depends_on:
- elasticsearch
- redis
elasticsearch:
container_name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:5.6.9
volumes:
- ${DOCKER_HOME}/elasticsearch/data:/usr/share/elasticsearch/data
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
environment:
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms2048m -Xmx2048m"
- "ES_JAVA_OPTS=-Xms1g -Xmx1g"
ulimits:
memlock:
soft: -1
hard: -1
redis:
container_name: redis
image: redis:alpine
volumes:
- ${HOME}/docker/redis:/data
- /path/to/esdata:/usr/share/elasticsearch/data
ports:
- 9200:9200
depends_on:
- elasticsearch-helper
restart: unless-stopped
elasticsearch-helper:
image: alpine
command: sh -c "sysctl -w vm.max_map_count=262144"
privileged: true
# application setup block
app_setup_block_enabled: true
app_setup_block: |
Once running the URL will be `http://<host-ip>/` initial application spinup will take some time so please reload if you get an empty response. We highly reccomend using Docker compose for this image as it includes multiple database backends to link into.
If you are looking to mount the elasticsearch and redis data to your host machine for access neither of them currently support setting a custom UID or GID they will run by default as:
This application is dependent on an ElasticSearch instance. Please see the example compose file for additional information.
- Redis - UID=999 GID=999
- Elasticsearch - UID=1000 GID=1000
The default username is diskover with the password of **darkdata**, access the container at http://<host-ip>/. The UI may be unusable until a valid index has been created.
ElasticSearch also requires a sysctl setting on the host machine to run properly. Running `sysctl -w vm.max_map_count=262144` will solve this issue. To make this setting persistent through reboots, set this value in `/etc/sysctl.conf`.
The application doesn't start an index by default. A crontab is created inside of the `/config` directory and can be set up to run automated indexes of `/data`. Changes to this crontab file require a restart to apply. You can also manually run an index by executing `/app/diskover/diskover.py` either in interactive or detached mode:
If you simply want the application to work you can mount these to folders with 0777 permissions, otherwise you will need to create these users host level and set the folder ownership properly.
By default this compose example is pointed to a single directory and the UID and GID you pass to the diskover container needs to match that folders ownership. If these are shared folders with many owners the indexing will likely fail.
For specific questions or help setting up diskover in your environment please refer to the project's Github page [{{ project_name|capitalize }}]({{ project_url }}).
* `docker exec -u abc -d /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the background
* `docker exec -u abc -it /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the foreground
# changelog
changelogs:
- { date: "11.10.21:", desc: "Updated to diskover-community v2." }
- { date: "19.11.20:", desc: "Fix pip packages." }
- { date: "19.12.19:", desc: "Rebasing to alpine 3.11." }
- { date: "28.06.19:", desc: "Rebasing to alpine 3.10." }

26
root/app/cleanup.sh
View File

@ -1,26 +0,0 @@
#!/usr/bin/with-contenv bash
. /config/.env
# define array for input values
declare -A DISKOVER_ARRAY
DISKOVER_ARRAY[REDIS_HOST]=${REDIS_HOST:-redis}
DISKOVER_ARRAY[REDIS_PORT]=${REDIS_PORT:-6379}
DISKOVER_ARRAY[DISKOVER_OPTS]="${DISKOVER_ARRAY[DISKOVER_OPTS]} -i ${DISKOVER_ARRAY[INDEX_NAME]}"
# killing existing workers before starting new ones
echo "killing existing workers..."
if [ -f "/tmp/diskover_bot_pids" ]; then
/bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1
sleep 3
fi
# empty existing redis queue
echo "emptying current redis queues..."
rq empty -u redis://"${DISKOVER_ARRAY[REDIS_HOST]}":"${DISKOVER_ARRAY[REDIS_PORT]}" diskover_crawl diskover diskover_calcdir failed
sleep 3
echo "killing dangling workers..."
/bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1
sleep 3

25
root/app/dispatcher.sh
View File

@ -1,25 +0,0 @@
#!/usr/bin/with-contenv bash
. /config/.env
# define array for input values
declare -A DISKOVER_ARRAY
DISKOVER_ARRAY[TODAY]=$(date +%Y-%m-%d)
DISKOVER_ARRAY[INDEX_PREFIX]=${INDEX_PREFIX:-diskover-}
DISKOVER_ARRAY[INDEX_NAME]=${INDEX_NAME:-${DISKOVER_ARRAY[INDEX_PREFIX]}${DISKOVER_ARRAY[TODAY]}}
DISKOVER_ARRAY[DISKOVER_OPTS]=${DISKOVER_OPTS:-}" -d /data -a"
DISKOVER_ARRAY[WORKER_OPTS]=${WORKER_OPTS:-""}
DISKOVER_ARRAY[REDIS_HOST]=${REDIS_HOST:-redis}
DISKOVER_ARRAY[REDIS_PORT]=${REDIS_PORT:-6379}
DISKOVER_ARRAY[DISKOVER_OPTS]="${DISKOVER_ARRAY[DISKOVER_OPTS]} -i ${DISKOVER_ARRAY[INDEX_NAME]}"
cd /app/diskover || exit
/bin/bash /app/cleanup.sh
echo "starting workers with following options: ${DISKOVER_ARRAY[WORKER_OPTS]}"
/bin/bash /app/diskover/diskover-bot-launcher.sh ${DISKOVER_ARRAY[WORKER_OPTS]}
echo "starting crawler with following options: ${DISKOVER_ARRAY[DISKOVER_OPTS]}"
/usr/bin/python3 /app/diskover/diskover.py ${DISKOVER_ARRAY[DISKOVER_OPTS]}

1
root/defaults/crontabs/abc Normal file
View File

@ -0,0 +1 @@
#0 3 * * * python3 /app/diskover/diskover.py -i diskover-$(date '+%Y%m%d') /data

63
root/defaults/default
View File

@ -1,45 +1,38 @@
server {
listen 80 default_server;
listen 80 default_server;
listen 443 ssl;
listen 443 ssl;
root /app/diskover-web/public;
index index.php index.html index.htm;
root /app/diskover-web/public;
index index.php index.html index.htm;
server_name _;
server_name _;
ssl_certificate /config/keys/cert.crt;
ssl_certificate_key /config/keys/cert.key;
ssl_certificate /config/keys/cert.crt;
ssl_certificate_key /config/keys/cert.key;
client_max_body_size 0;
client_max_body_size 0;
error_page 599 = @noauth;
location / {
try_files $uri $uri/ /index.php?$query_string;
}
location / {
if (!-f /config/nginx/.htpasswd) {
return 599;
}
auth_basic "Restricted";
auth_basic_user_file /config/nginx/.htpasswd;
try_files $uri $uri/ /index.php?$args;
}
# # unless the request is for a valid file (image, js, css, etc.), send to bootstrap
# if (!-e $request_filename)
# {
# rewrite ^/(.*)$ /index.php?/$1 last;
# break;
# }
location @noauth {
try_files $uri $uri/ /index.php?$args;
}
location ~ \.php(/|$) {
fastcgi_split_path_info ^(.+\.php)(/.+)$;
set $path_info $fastcgi_path_info;
fastcgi_param PATH_INFO $path_info;
try_files $fastcgi_script_name =404;
# With php5-cgi alone:
fastcgi_pass 127.0.0.1:9000;
# With php5-fpm:
#fastcgi_pass unix:/var/run/php5-fpm.sock;
fastcgi_index index.php;
include /etc/nginx/fastcgi_params;
fastcgi_read_timeout 900;
}
location ~ \.php$ {
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_pass 127.0.0.1:9000;
fastcgi_index index.php;
include /etc/nginx/fastcgi_params;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_intercept_errors off;
fastcgi_buffer_size 16k;
fastcgi_buffers 4 16k;
fastcgi_read_timeout 300;
}
}

165
root/defaults/diskover.cfg
View File

@ -1,165 +0,0 @@
; diskover config file
; if you make any changes, restart worker bots so they get the new config
[excludes]
; directory names and absolute paths you want to exclude from crawl, case-sensitive, can include wildcards (.* or backup* or /dir/dirname* or *tmp or *tmp* etc)
dirs = .*,.snapshot,.Snapshot,.zfs
; files you want to exclude from crawl, case-sensitive, can include wildcards (.*, *.doc or NULLEXT for files with no extension)
files = .*,Thumbs.db,.DS_Store,._.DS_Store,.localized,desktop.ini
[includes]
; directory names and absolute paths you want to include (whitelist), case-sensitive, you don't need to whitelist rootdir (-d rootdir)
;dirs = .recycle
; files you want to include (whitelist), case-sensitive
;files =
[ownersgroups]
; control how owner (username) and group fields are stored for file and directory docs
; store uid and gid's instead of trying to get owner and group names (default is False)
;uidgidonly = False
; owner/group names contain domain name set to True (default is False)
;domain = False
; character separator used on cifs/nfs mounts to separte user/group and domain name, usually \\ or @
;domainsep = \\
; if domain name comes first before character separator, set this to True, otherwise False (default is True)
;domainfirst = True
; when indexing owner and group fields, keep the domain name (default is False)
;keepdomain = False
[autotag]
; pattern dictionaries for diskover bots to use when auto-tagging, values are case-sensitive, can include wildcard for ext, name or path (tmp* or TMP* or *tmp or *TMP* etc)
; can also specify file name with json, example files = autotag.files.json
;files = [{"name": [], "name_exclude": [], "ext": ["tmp*", "TMP*", "temp*", "TEMP*", "cache*", "CACHE*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}]
;dirs = [{"name": ["*tmp*", "*TMP*", "*temp*", "*TEMP*", "*Temp*", "*cache*", "*CACHE*", "*Cache*"], "name_exclude": ["*templates*", "*Templates*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}]
[storagecost]
; storage cost per GB (default is 0.03 cents per GB)
costpergb = 0.03
; use decimal base 10 (1000) or binary base 2 (1024) for GB size (default is 2, set to 2 or 10)
base = 2
; pattern dictionaries for diskover bots to use when determing cost per GB (overrides above)
; can also specify file name with json, example paths = storagecost.paths.json
;paths = [{"path": ["*/fastdiskpath1/*", "*/Fastdiskpath2/*"], "path_exclude": [], "costpergb": 0.05}, {"path": ["*/slowdiskpath1/*", "*/Slowdiskpath2/*"], "path_exclude": [], "costpergb": 0.02}]
;times = [{"mtime": 180, "atime": 0, "ctime": 180, "costpergb": 0.02}]
; deciding factor if a match is in both paths and times, can be path or time
;priority = path
[elasticsearch]
; uncomment the below three lines if you are using AWS ES
;aws = False
;host = search-diskover-es-cluster-eg3yztrvzb6qucroyyjk2vokza.ap-northeast-1.es.amazonaws.com
;port = 443
; below two lines are for local ES, comment out if you are using AWS ES
host = {{ES_HOST}}
port = {{ES_PORT}}
; uncomment the below two lines if you installed X-Pack, for http-auth
user = {{ES_USER}}
password = {{ES_PASS}}
; index name for ES, cli arg overwrites this
indexname = diskover-index
; timeout for connection to ES (default is 10)
timeout = 30
; number of connections kept open to ES when crawling (default is 10)
maxsize = 20
; max retries for ES operations (default is 0)
maxretries = 10
; wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait
wait = False
; chunk size for ES bulk operations (default is 500)
chunksize = 1000
; number of shards for index (default is 5)
shards = 1
; number of replicas for index (default is 1)
replicas = 0
; the below settings are to optimize ES for crawling
; index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s
indexrefresh = 30s
; disable replicas during crawl - set to True to turn off replicas or False to keep on (default False), after crawl is set back to replicas value above
disablereplicas = True
; transaction log flush threshold size (default 512mb)
translogsize = 1gb
; search scroll size (default 100 docs)
scrollsize = 1000
[redis]
host = {{REDIS_HOST}}
port = {{REDIS_PORT}}
;password =
; cache directory times in Redis
; used for -I index2 when comparing directory times to get metadata from index2 instead of off disk
; set to True to cache dir times or False to turn off (default False)
cachedirtimes = False
; how long in seconds directory keys lives in Redis (default 1 day)
dirtimesttl = 604800
; database to use (default is 0)
db = 0
; rq job default time out in sec (default 180)
timeout = 3600
; rq default ttl for key/results (default 500)
ttl = 500
; rq queue names to use (default is diskover, diskover_crawl, diskover_calcdir)
queue = diskover
queuecrawl = diskover_crawl
queuecalcdir = diskover_calcdir
[adaptivebatch]
; adaptive batch settings when using -a (intelligent crawling)
; batchsize (numbers of dirs) to start at
startsize = 50
; maximum size of batch
maxsize = 500
; when adjusting batch size use this for +/- (increases when queue is 0, decreases when > 0)
stepsize = 10
; max number of files in batch, above is ignored if file limit reached (default 50000)
maxfiles = 50000
[paths]
; used by diskover socket server
; path to diskover.py (default is ./diskover.py)
diskoverpath = ./diskover.py
; path to python executable (default is python)
pythonpath = python
[socketlistener]
; hostname and port (TCP) for diskover socket server for remote commands
host = 0.0.0.0
port = 9999
; max connections for diskover socket server
maxconnections = 5
; port (TCP) for diskover socket server for messages from diskover treewalk client
twcport = 9998
[dupescheck]
; read size (bytes) for md5 sum check (how many bytes to read in at a time when md5 checking, default 64 KB)
readsize = 65536
; max size (bytes) of files to check (files larger than this will be skipped, default 1 GB)
maxsize = 1073741824
; bytes to check at start and end of file before doing md5 sum check (set large enough to account for file header info, default is 64)
checkbytes = 64
; try to restore times (mtime/atime) for files that get opened by byte check and md5
; set to True or False, default False (useful for cifs which does not work with noatime mount option)
restoretimes = False
; number of threads for calculating md5 checksum of files
threads = 8
[crawlbot]
; continuous scanner when running with --crawlbot
; time to sleep (seconds) between checking for directory changes
sleeptime = 0.1
; number of threads for checking directories, setting this to num of cores x2 is a good starting point
threads = 8
; how often in seconds to get new directory list with updated times from ES (default 3600)
dirlisttime = 3600
[gource]
; should be set to same in diskover-gource.sh
maxfilelag = 0.1
[crawlapi]
; crawl api url endpoint
;url = http://localhost:8080/api
; optional api login
;user = admin
;pass = admin
; number of items per page for each directory list request
;pagesize = 1000

30
root/etc/cont-init.d/49-eol-check Normal file
View File

@ -0,0 +1,30 @@
#!/usr/bin/with-contenv bash
if [ -f /config/diskover.cfg ]; then
echo '
******************************************************
******************************************************
* *
* *
* We have detected that you have an existing *
* *
* diskover v1 config file. Version 1 is now EOL. *
* *
* If you would like to upgrade, please create a *
* *
* /config directory. If you would like to continue *
* *
* using version 1, specify the tag v1.5.0.13-ls33. *
* *
* More information at *
* *
* https://github.com/diskoverdata/diskover-community *
* *
* *
******************************************************
******************************************************'
while true
do
sleep 3600
done
fi

77
root/etc/cont-init.d/50-diskover-config
View File

@ -1,77 +0,0 @@
#!/usr/bin/with-contenv bash
# set default values for variables
declare -A DISKOVER_CONF
DISKOVER_CONF[REDIS_HOST]=${REDIS_HOST:-redis}
DISKOVER_CONF[REDIS_PORT]=${REDIS_PORT:-6379}
DISKOVER_CONF[ES_HOST]=${ES_HOST:-elasticsearch}
DISKOVER_CONF[ES_PORT]=${ES_PORT:-9200}
DISKOVER_CONF[ES_USER]=${ES_USER:-elastic}
DISKOVER_CONF[ES_PASS]=${ES_PASS:-changeme}
DISKOVER_CONF[RUN_ON_START]=${RUN_ON_START:-false}
DISKOVER_CONF[USE_CRON]=${USE_CRON:-false}
# persist env variables
if [ ! -f "/config/.env" ]; then
# sed in values or skip if value not set
for KEY in "${!DISKOVER_CONF[@]}"; do \
if [[ ${DISKOVER_CONF[$KEY]} == "" ]]; then \
:
else echo "export $KEY=${DISKOVER_CONF[$KEY]}" >> /config/.env
fi
done
fi
if [ ! -f "/etc/profile.d/diskover.sh" ]; then
ln -s /config/.env /etc/profile.d/diskover.sh
fi
# create folders
mkdir -p \
/config/{log/diskover,crontabs}
if [ ! -f "/config/diskover.cfg" ]; then
mv /defaults/diskover.cfg /config/diskover.cfg
for KEY in "${!DISKOVER_CONF[@]}"; do \
if [[ ${DISKOVER_CONF[$KEY]} == "" ]]; then \
:
else sed -i 's|{{'$KEY'}}|'${DISKOVER_CONF[$KEY]}'|g' /config/diskover.cfg
fi
done
fi
if [ ! -f "/app/diskover/diskover.cfg" ]; then
ln -s /config/diskover.cfg /app/diskover/diskover.cfg
fi
# copy crontab and proxy defaults if needed
[[ ! -f /config/crontabs/abc ]] && \
cp /etc/crontabs/abc /config/crontabs/
# import user crontabs
rm /etc/crontabs/*
if [ ${DISKOVER_CONF[USE_CRON]} == true ]; then
cp /config/crontabs/* /etc/crontabs/
fi
# permissions
chmod +x \
/app/dispatcher.sh
chown -R abc:abc \
/app/diskover \
/config
# run initial retrieval or just start some worker bots
if [ ${DISKOVER_CONF[RUN_ON_START]} == true ]; then
echo "Initial run of dispatcher in progress"
# sleep to allow time for elasticsearch to start
sleep 10
exec \
s6-setuidgid abc /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 &
else
echo "Starting worker bots"
# sleep to allow time for elasticsearch to start
sleep 10
exec \
s6-setuidgid abc cd /app/diskover && ./diskover-bot-launcher.sh
fi

13
root/etc/cont-init.d/50-permissions Normal file
View File

@ -0,0 +1,13 @@
#!/usr/bin/with-contenv bash
if [ -d /app/diskover-tmp ]; then
echo "New container detected. Setting up app folder and fixing permissions."
mv /app/diskover-tmp /app/diskover
fi
if [ -d /app/diskover-web-tmp ]; then
mv /app/diskover-web-tmp /app/diskover-web
fi
chown -R abc:abc /app

29
root/etc/cont-init.d/51-config-diskover Normal file
View File

@ -0,0 +1,29 @@
#!/usr/bin/with-contenv bash
declare -A DISKOVER_CONF
DISKOVER_CONF[ES_HOST]=${ES_HOST:-elasticsearch}
DISKOVER_CONF[ES_PORT]=${ES_PORT:-9200}
DISKOVER_CONF[ES_USER]=${ES_USER:-}
DISKOVER_CONF[ES_PASS]=${ES_PASS:-}
# copy config
if [[ ! -e /config/diskover.conf.d ]]; then
cp -r /app/diskover/configs_sample /config/diskover.conf.d
sed -i "s|host:|host: ${WEB_CONF[ES_HOST]}|g" /config/diskover.conf.d/diskover/config.yaml
sed -i "s|port:|port: ${WEB_CONF[ES_PORT]}|g" /config/diskover.conf.d/diskover/config.yaml
sed -i "s|user:|user: ${WEB_CONF[ES_USER]}|g" /config/diskover.conf.d/diskover/config.yaml
sed -i "s|password:|password: ${WEB_CONF[ES_PASS]}|g" /config/diskover.conf.d/diskover/config.yaml
fi
ls /config/diskover.conf.d | \
xargs -I {} \
ln -sf /config/diskover.conf.d/{} /etc/{}
# install crontab
[[ ! -e /config/crontab ]] && \
cp /defaults/crontabs/abc /config/crontab
crontab -u abc /config/crontab
chown -R abc:abc /config

37
root/etc/cont-init.d/52-config-diskover-web Normal file
View File

@ -0,0 +1,37 @@
#!/usr/bin/with-contenv bash
# set default values for variables
declare -A WEB_CONF
WEB_CONF[ES_HOST]=${ES_HOST:-elasticsearch}
WEB_CONF[ES_PORT]=${ES_PORT:-9200}
WEB_CONF[ES_USER]=${ES_USER:-}
WEB_CONF[ES_PASS]=${ES_PASS:-}
mkdir -p \
/config/diskover-web.conf.d
# copy over sample config files
cd /app/diskover-web/public
for file in *.sample; do
dest=${file%.sample}
[[ ! -e "/config/diskover-web.conf.d/$dest" ]] && \
cp /app/diskover-web/public/$file /config/diskover-web.conf.d/$dest
ln -sf /config/diskover-web.conf.d/$dest /app/diskover-web/public/$dest
done
# copy and set variables in Constants.php
if [[ ! -e "/config/diskover-web.conf.d/Constants.php" ]]; then
cp /app/diskover-web/src/diskover/Constants.php.sample /config/diskover-web.conf.d/Constants.php
sed -i "s|const TIMEZONE = 'America/Vancouver';|const TIMEZONE = '${TZ}';|g" /config/diskover-web.conf.d/Constants.php
sed -i "s|const ES_HOST = 'localhost';|const ES_HOST = '${WEB_CONF[ES_HOST]}';|g" /config/diskover-web.conf.d/Constants.php
sed -i "s|const ES_PORT = 9200;|const ES_PORT = ${WEB_CONF[ES_PORT]};|g" /config/diskover-web.conf.d/Constants.php
sed -i "s|const ES_USER = '';|const ES_USER = '${WEB_CONF[ES_USER]}';|g" /config/diskover-web.conf.d/Constants.php
sed -i "s|const ES_PASS = '';|const ES_PASS = '${WEB_CONF[ES_PASS]}';|g" /config/diskover-web.conf.d/Constants.php
fi
ln -sf /config/diskover-web.conf.d/Constants.php /app/diskover-web/src/diskover/Constants.php
chown -R abc:abc /config

61
root/etc/cont-init.d/60-diskover-web-config
View File

@ -1,61 +0,0 @@
#!/usr/bin/with-contenv bash
if [ $DISABLE_WEB ]; then
echo "Not running web UI"
exit
fi
# set default values for variables
declare -A WEB_CONF
WEB_CONF[ES_HOST]=${ES_HOST:-elasticsearch}
WEB_CONF[ES_PORT]=${ES_PORT:-9200}
WEB_CONF[ES_USER]=${ES_USER:-elastic}
WEB_CONF[ES_PASS]=${ES_PASS:-changeme}
if [ ! -f "/config/Constants.php" ]; then
cp /app/diskover-web/src/diskover/Constants.php.sample /config/Constants.php
sed -i "s|ES_HOST = 'localhost'|ES_HOST = '"${WEB_CONF[ES_HOST]}"'|g" /config/Constants.php
sed -i "s|ES_PORT = 9200|ES_PORT = "${WEB_CONF[ES_PORT]}"|g" /config/Constants.php
sed -i "s|ES_USER = ''|ES_USER = '"${WEB_CONF[ES_USER]}"'|g" /config/Constants.php
sed -i "s|ES_PASS = ''|ES_PASS = '"${WEB_CONF[ES_PASS]}"'|g" /config/Constants.php
sed -i "s|ENABLE_SOCKET_CLIENT = FALSE|ENABLE_SOCKET_CLIENT = true|g" /config/Constants.php
sed -i "s|SOCKET_LISTENER_HOST = '127.0.0.1'|SOCKET_LISTENER_HOST = 'diskover'|g" /config/Constants.php
fi
if [ ! -f "/app/diskover-web/src/diskover/Constants.php" ]; then
ln -s /config/Constants.php /app/diskover-web/src/diskover/Constants.php
fi
if [ ! -f "/app/diskover-web/public/index.php" ]; then
cd /app/diskover-web/public && \
ln -s dashboard.php index.php
fi
if [ ! -f "/config/smartsearches.txt" ]; then
cp /app/diskover-web/public/smartsearches.txt.sample /config/smartsearches.txt
fi
if [ ! -f "/app/diskover-web/public/smartsearches.txt" ]; then
ln -s /config/smartsearches.txt /app/diskover-web/public/smartsearches.txt
fi
if [ ! -f "/config/customtags.txt" ]; then
cp /app/diskover-web/public/customtags.txt.sample /config/customtags.txt
fi
if [ ! -f "/app/diskover-web/public/customtags.txt" ]; then
ln -s /config/customtags.txt /app/diskover-web/public/customtags.txt
fi
if [ ! -f "/config/extrafields.txt.sample" ]; then
cp /app/diskover-web/public/extrafields.txt.sample /config/extrafields.txt
fi
if [ ! -f "/app/diskover-web/public/extrafields.txt" ]; then
ln -s /config/extrafields.txt /app/diskover-web/public/extrafields.txt
fi
# permissions
chown -R abc:abc \
/app/diskover-web \
/config

1
root/etc/crontabs/abc
View File

@ -1 +0,0 @@
0 3 * * * /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1

9
root/etc/logrotate.d/diskover
View File

@ -1,9 +0,0 @@
/config/log/diskover/*.log {
daily
rotate 14
compress
nodateext
notifempty
missingok
su abc abc
}

3
root/etc/services.d/cron/run Normal file
View File

@ -0,0 +1,3 @@
#!/usr/bin/with-contenv bash
exec /usr/sbin/crond -f -S -l 5 -c /etc/crontabs

17
root/etc/services.d/rq-dashboard/run
View File

@ -1,17 +0,0 @@
#!/usr/bin/with-contenv bash
# need to safely handle exiting the existing worker bots and
# redis queue when container restarts or stops
_term() {
/bin/bash /app/cleanup.sh
}
trap _term SIGTERM
declare -A REDIS_CONF
REDIS_CONF[REDIS_HOST]=${REDIS_HOST:-redis}
REDIS_CONF[REDIS_PORT]=${REDIS_PORT:-6379}
exec \
s6-setuidgid abc /usr/bin/rq-dashboard -H ${REDIS_CONF[REDIS_HOST]} -P ${REDIS_CONF[REDIS_PORT]}

4
root/etc/services.d/socket_server/run
View File

@ -1,4 +0,0 @@
#!/usr/bin/with-contenv bash
exec \
s6-setuidgid abc /usr/bin/python /app/diskover/diskover.py -l