diff --git a/.github/workflows/external_trigger.yml b/.github/workflows/external_trigger.yml index 2a5a687..8e66463 100644 --- a/.github/workflows/external_trigger.yml +++ b/.github/workflows/external_trigger.yml @@ -18,7 +18,7 @@ jobs: fi echo "**** External trigger running off of master branch. To disable this trigger, set a Github secret named \"PAUSE_EXTERNAL_TRIGGER_DISKOVER_MASTER\". ****" echo "**** Retrieving external version ****" - EXT_RELEASE=$(curl -u "${{ secrets.CR_USER }}:${{ secrets.CR_PAT }}" -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" | jq -r '. | .tag_name') + EXT_RELEASE=$(curl -u "${{ secrets.CR_USER }}:${{ secrets.CR_PAT }}" -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases/latest" | jq -r '. | .tag_name') if [ -z "${EXT_RELEASE}" ] || [ "${EXT_RELEASE}" == "null" ]; then echo "**** Can't retrieve external version, exiting ****" FAILURE_REASON="Can't retrieve external version for diskover branch master" diff --git a/Dockerfile b/Dockerfile index 43c9482..e9af665 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,75 +1,58 @@ -FROM ghcr.io/linuxserver/baseimage-alpine-nginx:3.12 +FROM ghcr.io/linuxserver/baseimage-alpine-nginx:3.14 # set version label ARG BUILD_DATE ARG VERSION -ARG DISKOVER_VERSION +ARG DISKOVER_RELEASE LABEL build_version="Linuxserver.io version:- ${VERSION} Build-date:- ${BUILD_DATE}" LABEL maintainer="alex-phillips" RUN \ - echo "**** install build packages ****" && \ - apk add --no-cache --virtual=build-dependencies \ - composer \ - curl \ - gcc \ - musl-dev \ - python3-dev && \ - echo "**** install runtime packages ****" && \ - apk add --no-cache \ - grep \ - ncurses \ - php7-curl \ - php7-phar \ - py3-pip \ - python3 && \ - echo "**** install diskover ****" && \ - mkdir -p /app/diskover && \ - if [ -z ${DISKOVER_VERSION+x} ]; then \ - DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - fi && \ - curl -o \ - /tmp/diskover.tar.gz -L \ - "https://github.com/alex-phillips/diskover/archive/${DISKOVER_VERSION}.tar.gz" && \ - tar xf \ - /tmp/diskover.tar.gz -C \ - /app/diskover/ --strip-components=1 && \ - echo "**** install diskover-web ****" && \ - mkdir -p /app/diskover-web && \ - DISKOVER_WEB_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - if [ "${DISKOVER_VERSION}" != "${DISKOVER_WEB_VERSION}" ] || [ -z ${DISKOVER_VERSION+x} ]; then \ - DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - fi && \ - curl -o \ - /tmp/diskover-web.tar.gz -L \ - "https://github.com/alex-phillips/diskover-web/archive/${DISKOVER_VERSION}.tar.gz" && \ - tar xf \ - /tmp/diskover-web.tar.gz -C \ - /app/diskover-web/ --strip-components=1 && \ - echo "**** install pip packages ****" && \ - cd /app/diskover && \ - pip3 install --no-cache-dir -r requirements.txt && \ - pip3 install rq-dashboard && \ - echo "**** install composer packages ****" && \ - cd /app/diskover-web && \ - composer install && \ - echo "**** fix logrotate ****" && \ - sed -i "s#/var/log/messages {}.*# #g" /etc/logrotate.conf && \ - echo "**** symlink python3 ****" && \ - ln -s /usr/bin/python3 /usr/bin/python && \ - echo "**** cleanup ****" && \ - apk del --purge \ - build-dependencies && \ - rm -rf \ - /root/.cache \ - /tmp/* + echo "**** install build packages ****" && \ + apk add --no-cache --virtual=build-dependencies \ + build-base \ + gcc \ + py3-pip \ + python3-dev \ + composer \ + curl \ + git \ + jq \ + nodejs \ + npm && \ + echo "**** install runtime packages ****" && \ + apk add --no-cache \ + libldap \ + ncurses \ + php7-curl \ + php7-ldap \ + py3-requests \ + py3-urllib3 \ + py3-xxhash \ + python3 && \ + echo "**** install diskover ****" && \ + if [ -z ${DISKOVER_RELEASE+x} ]; then \ + DISKOVER_RELEASE=$(curl -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases" \ + | jq -r '.[0] | .tag_name'); \ + fi && \ + curl -o \ + /tmp/diskover.tar.gz -L \ + "https://github.com/diskoverdata/diskover-community/archive/${DISKOVER_RELEASE}.tar.gz" && \ + tar xf \ + /tmp/diskover.tar.gz -C \ + /app/ --strip-components=1 && \ + cd /app/diskover && \ + pip3 install --no-cache-dir -r requirements.txt && \ + sed -i 's@;clear_env = no@clear_env = no@' "/etc/php7/php-fpm.d/www.conf" && \ + echo "**** overlay-fs workaround ****" && \ + mv /app/diskover /app/diskover-tmp && \ + mv /app/diskover-web /app/diskover-web-tmp && \ + echo "**** cleanup ****" && \ + apk del --purge \ + build-dependencies && \ + rm -rf \ + /tmp/* \ + /root/.cache -# copy local files -COPY root/ / - -# ports and volumes -EXPOSE 8000 -VOLUME /config +# add local files +COPY ./root/ / diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index dd51fd4..d1630d1 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -1,78 +1,58 @@ -FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm64v8-3.12 +FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm64v8-3.14 # set version label ARG BUILD_DATE ARG VERSION -ARG DISKOVER_VERSION +ARG DISKOVER_RELEASE LABEL build_version="Linuxserver.io version:- ${VERSION} Build-date:- ${BUILD_DATE}" LABEL maintainer="alex-phillips" RUN \ - echo "**** install build packages ****" && \ - apk add --no-cache --virtual=build-dependencies \ - curl \ - gcc \ - musl-dev \ - python3-dev && \ - echo "**** install runtime packages ****" && \ - apk add --no-cache \ - grep \ - ncurses \ - php7-curl \ - php7-phar \ - py3-pip \ - python3 && \ - echo "**** install composer ****" && \ - curl \ - -sS https://getcomposer.org/installer \ - | php -- --install-dir=/usr/bin --filename=composer && \ - echo "**** install diskover ****" && \ - mkdir -p /app/diskover && \ - if [ -z ${DISKOVER_VERSION+x} ]; then \ - DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - fi && \ - curl -o \ - /tmp/diskover.tar.gz -L \ - "https://github.com/alex-phillips/diskover/archive/${DISKOVER_VERSION}.tar.gz" && \ - tar xf \ - /tmp/diskover.tar.gz -C \ - /app/diskover/ --strip-components=1 && \ - echo "**** install diskover-web ****" && \ - mkdir -p /app/diskover-web && \ - DISKOVER_WEB_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - if [ "${DISKOVER_VERSION}" != "${DISKOVER_WEB_VERSION}" ] || [ -z ${DISKOVER_VERSION+x} ]; then \ - DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - fi && \ - curl -o \ - /tmp/diskover-web.tar.gz -L \ - "https://github.com/alex-phillips/diskover-web/archive/${DISKOVER_VERSION}.tar.gz" && \ - tar xf \ - /tmp/diskover-web.tar.gz -C \ - /app/diskover-web/ --strip-components=1 && \ - echo "**** install pip packages ****" && \ - cd /app/diskover && \ - pip3 install --no-cache-dir -r requirements.txt && \ - pip3 install rq-dashboard && \ - echo "**** install composer packages ****" && \ - cd /app/diskover-web && \ - composer install && \ - echo "**** fix logrotate ****" && \ - sed -i "s#/var/log/messages {}.*# #g" /etc/logrotate.conf && \ - echo "**** symlink python3 ****" && \ - ln -s /usr/bin/python3 /usr/bin/python && \ - echo "**** cleanup ****" && \ - apk del --purge \ - build-dependencies && \ - rm -rf \ - /root/.cache \ - /tmp/* + echo "**** install build packages ****" && \ + apk add --no-cache --virtual=build-dependencies \ + build-base \ + gcc \ + py3-pip \ + python3-dev \ + composer \ + curl \ + git \ + jq \ + nodejs \ + npm && \ + echo "**** install runtime packages ****" && \ + apk add --no-cache \ + libldap \ + ncurses \ + php7-curl \ + php7-ldap \ + py3-requests \ + py3-urllib3 \ + py3-xxhash \ + python3 && \ + echo "**** install diskover ****" && \ + if [ -z ${DISKOVER_RELEASE+x} ]; then \ + DISKOVER_RELEASE=$(curl -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases" \ + | jq -r '.[0] | .tag_name'); \ + fi && \ + curl -o \ + /tmp/diskover.tar.gz -L \ + "https://github.com/diskoverdata/diskover-community/archive/${DISKOVER_RELEASE}.tar.gz" && \ + tar xf \ + /tmp/diskover.tar.gz -C \ + /app/ --strip-components=1 && \ + cd /app/diskover && \ + pip3 install --no-cache-dir -r requirements.txt && \ + sed -i 's@;clear_env = no@clear_env = no@' "/etc/php7/php-fpm.d/www.conf" && \ + echo "**** overlay-fs workaround ****" && \ + mv /app/diskover /app/diskover-tmp && \ + mv /app/diskover-web /app/diskover-web-tmp && \ + echo "**** cleanup ****" && \ + apk del --purge \ + build-dependencies && \ + rm -rf \ + /tmp/* \ + /root/.cache -# copy local files -COPY root/ / - -# ports and volumes -EXPOSE 8000 -VOLUME /config +# add local files +COPY ./root/ / diff --git a/Dockerfile.armhf b/Dockerfile.armhf index 7bed87f..50d2a1b 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -1,78 +1,58 @@ -FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm32v7-3.12 +FROM ghcr.io/linuxserver/baseimage-alpine-nginx:arm32v7-3.14 # set version label ARG BUILD_DATE ARG VERSION -ARG DISKOVER_VERSION +ARG DISKOVER_RELEASE LABEL build_version="Linuxserver.io version:- ${VERSION} Build-date:- ${BUILD_DATE}" LABEL maintainer="alex-phillips" RUN \ - echo "**** install build packages ****" && \ - apk add --no-cache --virtual=build-dependencies \ - curl \ - gcc \ - musl-dev \ - python3-dev && \ - echo "**** install runtime packages ****" && \ - apk add --no-cache \ - grep \ - ncurses \ - php7-curl \ - php7-phar \ - py3-pip \ - python3 && \ - echo "**** install composer ****" && \ - curl \ - -sS https://getcomposer.org/installer \ - | php -- --install-dir=/usr/bin --filename=composer && \ - echo "**** install diskover ****" && \ - mkdir -p /app/diskover && \ - if [ -z ${DISKOVER_VERSION+x} ]; then \ - DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - fi && \ - curl -o \ - /tmp/diskover.tar.gz -L \ - "https://github.com/alex-phillips/diskover/archive/${DISKOVER_VERSION}.tar.gz" && \ - tar xf \ - /tmp/diskover.tar.gz -C \ - /app/diskover/ --strip-components=1 && \ - echo "**** install diskover-web ****" && \ - mkdir -p /app/diskover-web && \ - DISKOVER_WEB_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - if [ "${DISKOVER_VERSION}" != "${DISKOVER_WEB_VERSION}" ] || [ -z ${DISKOVER_VERSION+x} ]; then \ - DISKOVER_VERSION=$(curl -sX GET "https://api.github.com/repos/alex-phillips/diskover-web/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - fi && \ - curl -o \ - /tmp/diskover-web.tar.gz -L \ - "https://github.com/alex-phillips/diskover-web/archive/${DISKOVER_VERSION}.tar.gz" && \ - tar xf \ - /tmp/diskover-web.tar.gz -C \ - /app/diskover-web/ --strip-components=1 && \ - echo "**** install pip packages ****" && \ - cd /app/diskover && \ - pip3 install --no-cache-dir -r requirements.txt && \ - pip3 install rq-dashboard && \ - echo "**** install composer packages ****" && \ - cd /app/diskover-web && \ - composer install && \ - echo "**** fix logrotate ****" && \ - sed -i "s#/var/log/messages {}.*# #g" /etc/logrotate.conf && \ - echo "**** symlink python3 ****" && \ - ln -s /usr/bin/python3 /usr/bin/python && \ - echo "**** cleanup ****" && \ - apk del --purge \ - build-dependencies && \ - rm -rf \ - /root/.cache \ - /tmp/* + echo "**** install build packages ****" && \ + apk add --no-cache --virtual=build-dependencies \ + build-base \ + gcc \ + py3-pip \ + python3-dev \ + composer \ + curl \ + git \ + jq \ + nodejs \ + npm && \ + echo "**** install runtime packages ****" && \ + apk add --no-cache \ + libldap \ + ncurses \ + php7-curl \ + php7-ldap \ + py3-requests \ + py3-urllib3 \ + py3-xxhash \ + python3 && \ + echo "**** install diskover ****" && \ + if [ -z ${DISKOVER_RELEASE+x} ]; then \ + DISKOVER_RELEASE=$(curl -sX GET "https://api.github.com/repos/diskoverdata/diskover-community/releases" \ + | jq -r '.[0] | .tag_name'); \ + fi && \ + curl -o \ + /tmp/diskover.tar.gz -L \ + "https://github.com/diskoverdata/diskover-community/archive/${DISKOVER_RELEASE}.tar.gz" && \ + tar xf \ + /tmp/diskover.tar.gz -C \ + /app/ --strip-components=1 && \ + cd /app/diskover && \ + pip3 install --no-cache-dir -r requirements.txt && \ + sed -i 's@;clear_env = no@clear_env = no@' "/etc/php7/php-fpm.d/www.conf" && \ + echo "**** overlay-fs workaround ****" && \ + mv /app/diskover /app/diskover-tmp && \ + mv /app/diskover-web /app/diskover-web-tmp && \ + echo "**** cleanup ****" && \ + apk del --purge \ + build-dependencies && \ + rm -rf \ + /tmp/* \ + /root/.cache -# copy local files -COPY root/ / - -# ports and volumes -EXPOSE 8000 -VOLUME /config +# add local files +COPY ./root/ / diff --git a/Jenkinsfile b/Jenkinsfile index c8dacdd..ea04d40 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,8 +18,8 @@ pipeline { GITLAB_NAMESPACE=credentials('gitlab-namespace-id') SCARF_TOKEN=credentials('scarf_api_key') EXT_GIT_BRANCH = 'master' - EXT_USER = 'alex-phillips' - EXT_REPO = 'diskover' + EXT_USER = 'diskoverdata' + EXT_REPO = 'diskover-community' BUILD_VERSION_ARG = 'DISKOVER_VERSION' LS_USER = 'linuxserver' LS_REPO = 'docker-diskover' @@ -467,7 +467,7 @@ pipeline { --label \"org.opencontainers.image.licenses=GPL-3.0-only\" \ --label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \ --label \"org.opencontainers.image.title=Diskover\" \ - --label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ + --label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ --no-cache --pull -t ${IMAGE}:${META_TAG} \ --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." } @@ -497,7 +497,7 @@ pipeline { --label \"org.opencontainers.image.licenses=GPL-3.0-only\" \ --label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \ --label \"org.opencontainers.image.title=Diskover\" \ - --label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ + --label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ --no-cache --pull -t ${IMAGE}:amd64-${META_TAG} \ --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." } @@ -524,7 +524,7 @@ pipeline { --label \"org.opencontainers.image.licenses=GPL-3.0-only\" \ --label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \ --label \"org.opencontainers.image.title=Diskover\" \ - --label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ + --label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ --no-cache --pull -f Dockerfile.armhf -t ${IMAGE}:arm32v7-${META_TAG} \ --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." sh "docker tag ${IMAGE}:arm32v7-${META_TAG} ghcr.io/linuxserver/lsiodev-buildcache:arm32v7-${COMMIT_SHA}-${BUILD_NUMBER}" @@ -558,7 +558,7 @@ pipeline { --label \"org.opencontainers.image.licenses=GPL-3.0-only\" \ --label \"org.opencontainers.image.ref.name=${COMMIT_SHA}\" \ --label \"org.opencontainers.image.title=Diskover\" \ - --label \"org.opencontainers.image.description=[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ + --label \"org.opencontainers.image.description=[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems.\" \ --no-cache --pull -f Dockerfile.aarch64 -t ${IMAGE}:arm64v8-${META_TAG} \ --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." sh "docker tag ${IMAGE}:arm64v8-${META_TAG} ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}" diff --git a/README.md b/README.md index 57f644e..d6dea3a 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,9 @@ Find us at: [![Jenkins Build](https://img.shields.io/jenkins/build?labelColor=555555&logoColor=ffffff&style=for-the-badge&jobUrl=https%3A%2F%2Fci.linuxserver.io%2Fjob%2FDocker-Pipeline-Builders%2Fjob%2Fdocker-diskover%2Fjob%2Fmaster%2F&logo=jenkins)](https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-diskover/job/master/) [![LSIO CI](https://img.shields.io/badge/dynamic/yaml?color=94398d&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=CI&query=CI&url=https%3A%2F%2Fci-tests.linuxserver.io%2Flinuxserver%2Fdiskover%2Flatest%2Fci-status.yml)](https://ci-tests.linuxserver.io/linuxserver/diskover/latest/index.html) -[diskover](https://github.com/shirosaidev/diskover) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems. +[diskover](https://github.com/diskoverdata/diskover-community) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems. -[![diskover](https://raw.githubusercontent.com/shirosaidev/diskover/master/docs/diskover.png)](https://github.com/shirosaidev/diskover) +[![diskover](https://raw.githubusercontent.com/diskoverdata/diskover-community/master/diskover-web/public/images/diskover.png)](https://github.com/diskoverdata/diskover-community) ## Supported Architectures @@ -59,19 +59,14 @@ The architectures supported by this image are: ## Application Setup -Once running the URL will be `http:///` initial application spinup will take some time so please reload if you get an empty response. We highly reccomend using Docker compose for this image as it includes multiple database backends to link into. -If you are looking to mount the elasticsearch and redis data to your host machine for access neither of them currently support setting a custom UID or GID they will run by default as: +This application is dependent on an ElasticSearch instance. Please see the example compose file for additional information. -- Redis - UID=999 GID=999 -- Elasticsearch - UID=1000 GID=1000 +The default username is diskover with the password of **darkdata**, access the container at http:///. The UI may be unusable until a valid index has been created. -ElasticSearch also requires a sysctl setting on the host machine to run properly. Running `sysctl -w vm.max_map_count=262144` will solve this issue. To make this setting persistent through reboots, set this value in `/etc/sysctl.conf`. +The application doesn't start an index by default. A crontab is created inside of the `/config` directory and can be set up to run automated indexes of `/data`. Changes to this crontab file require a restart to apply. You can also manually run an index by executing `/app/diskover/diskover.py` either in interactive or detached mode: -If you simply want the application to work you can mount these to folders with 0777 permissions, otherwise you will need to create these users host level and set the folder ownership properly. - -By default this compose example is pointed to a single directory and the UID and GID you pass to the diskover container needs to match that folders ownership. If these are shared folders with many owners the indexing will likely fail. - -For specific questions or help setting up diskover in your environment please refer to the project's Github page [Diskover](https://github.com/shirosaidev/diskover). +* `docker exec -u abc -d /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the background +* `docker exec -u abc -it /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the foreground ## Usage @@ -83,49 +78,45 @@ Here are some example snippets to help you get started creating a container. version: '2' services: diskover: - image: linuxserver/diskover + image: ghcr.io/linuxserver/diskover container_name: diskover environment: - PUID=1000 - PGID=1000 - - TZ=Europe/London - - REDIS_HOST=redis - - REDIS_PORT=6379 + - TZ=America/New_York - ES_HOST=elasticsearch - ES_PORT=9200 - - ES_USER=elastic - - ES_PASS=changeme - - RUN_ON_START=true - - USE_CRON=true volumes: - /path/to/diskover/config:/config - /path/to/diskover/data:/data ports: - 80:80 - - 9181:9181 - - 9999:9999 mem_limit: 4096m restart: unless-stopped depends_on: - elasticsearch - - redis elasticsearch: container_name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.9 - volumes: - - ${DOCKER_HOME}/elasticsearch/data:/usr/share/elasticsearch/data + image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 environment: + - discovery.type=single-node - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" ulimits: memlock: soft: -1 hard: -1 - redis: - container_name: redis - image: redis:alpine volumes: - - ${HOME}/docker/redis:/data + - /path/to/esdata:/usr/share/elasticsearch/data + ports: + - 9200:9200 + depends_on: + - elasticsearch-helper + restart: unless-stopped + elasticsearch-helper: + image: alpine + command: sh -c "sysctl -w vm.max_map_count=262144" + privileged: true ``` @@ -136,21 +127,12 @@ docker run -d \ --name=diskover \ -e PUID=1000 \ -e PGID=1000 \ - -e TZ=Europe/London \ - -e REDIS_HOST=redis \ - -e REDIS_PORT=6379 \ + -e TZ=America/New_York \ -e ES_HOST=elasticsearch \ -e ES_PORT=9200 \ -e ES_USER=elastic \ -e ES_PASS=changeme \ - -e INDEX_NAME=diskover- \ - -e DISKOVER_OPTS= \ - -e WORKER_OPTS= \ - -e RUN_ON_START=true \ - -e USE_CRON=true \ -p 80:80 \ - -p 9181:9181 \ - -p 9999:9999 \ -v /path/to/diskover/config:/config \ -v /path/to/diskover/data:/data \ --restart unless-stopped \ @@ -164,22 +146,13 @@ Container images are configured using parameters passed at runtime (such as thos | Parameter | Function | | :----: | --- | | `-p 80` | diskover Web UI | -| `-p 9181` | rq-dashboard web UI | -| `-p 9999` | diskover socket server | | `-e PUID=1000` | for UserID - see below for explanation | | `-e PGID=1000` | for GroupID - see below for explanation | -| `-e TZ=Europe/London` | Specify a timezone to use EG Europe/London | -| `-e REDIS_HOST=redis` | Redis host (optional) | -| `-e REDIS_PORT=6379` | Redis port (optional) | +| `-e TZ=America/New_York` | Specify a timezone to use EG America/New_York | | `-e ES_HOST=elasticsearch` | ElasticSearch host (optional) | | `-e ES_PORT=9200` | ElasticSearch port (optional) | | `-e ES_USER=elastic` | ElasticSearch username (optional) | | `-e ES_PASS=changeme` | ElasticSearch password (optional) | -| `-e INDEX_NAME=diskover-` | Index name prefix (optional) | -| `-e DISKOVER_OPTS=` | Optional arguments to pass to the diskover crawler (optional) | -| `-e WORKER_OPTS=` | Optional argumens to pass to the diskover bots launcher (optional) | -| `-e RUN_ON_START=true` | Initiate a crawl every time the container is started (optional) | -| `-e USE_CRON=true` | Run a crawl on as a cron job (optional) | | `-v /config` | Persistent config files | | `-v /data` | Default mount point to crawl | @@ -292,6 +265,7 @@ Once registered you can define the dockerfile to use with `-f Dockerfile.aarch64 ## Versions +* **11.10.21:** - Updated to diskover-community v2. * **19.11.20:** - Fix pip packages. * **19.12.19:** - Rebasing to alpine 3.11. * **28.06.19:** - Rebasing to alpine 3.10. diff --git a/jenkins-vars.yml b/jenkins-vars.yml index 32bb408..262a813 100644 --- a/jenkins-vars.yml +++ b/jenkins-vars.yml @@ -8,8 +8,8 @@ release_tag: latest ls_branch: master repo_vars: - EXT_GIT_BRANCH = 'master' - - EXT_USER = 'alex-phillips' - - EXT_REPO = 'diskover' + - EXT_USER = 'diskoverdata' + - EXT_REPO = 'diskover-community' - BUILD_VERSION_ARG = 'DISKOVER_VERSION' - LS_USER = 'linuxserver' - LS_REPO = 'docker-diskover' diff --git a/readme-vars.yml b/readme-vars.yml index e3d6cd9..5629f28 100644 --- a/readme-vars.yml +++ b/readme-vars.yml @@ -2,9 +2,9 @@ # project information project_name: diskover -project_url: "https://github.com/shirosaidev/diskover" -project_logo: "https://raw.githubusercontent.com/shirosaidev/diskover/master/docs/diskover.png" -project_blurb: "[{{ project_name }}]({{ project_url }}) is a file system crawler and disk space usage software that uses Elasticsearch to index and manage data across heterogeneous storage systems." +project_url: "https://github.com/diskoverdata/diskover-community" +project_logo: "https://raw.githubusercontent.com/diskoverdata/diskover-community/master/diskover-web/public/images/diskover.png" +project_blurb: "[{{ project_name }}]({{ project_url }}) is an open source file system indexer that uses Elasticsearch to index and manage data across heterogeneous storage systems." project_lsio_github_repo_url: "https://github.com/linuxserver/docker-{{ project_name }}" # supported architectures @@ -23,90 +23,73 @@ param_volumes: param_usage_include_ports: true param_ports: - { external_port: "80", internal_port: "80", port_desc: "diskover Web UI" } - - { external_port: "9181", internal_port: "9181", port_desc: "rq-dashboard web UI" } - - { external_port: "9999", internal_port: "9999", port_desc: "diskover socket server" } param_usage_include_env: true param_env_vars: - - { env_var: "TZ", env_value: "Europe/London", desc: "Specify a timezone to use EG Europe/London"} - - { env_var: "REDIS_HOST", env_value: "redis", desc: "Redis host (optional)"} - - { env_var: "REDIS_PORT", env_value: "6379", desc: "Redis port (optional)"} + - { env_var: "TZ", env_value: "America/New_York", desc: "Specify a timezone to use EG America/New_York"} - { env_var: "ES_HOST", env_value: "elasticsearch", desc: "ElasticSearch host (optional)"} - { env_var: "ES_PORT", env_value: "9200", desc: "ElasticSearch port (optional)"} - { env_var: "ES_USER", env_value: "elastic", desc: "ElasticSearch username (optional)"} - { env_var: "ES_PASS", env_value: "changeme", desc: "ElasticSearch password (optional)"} - - { env_var: "INDEX_NAME", env_value: "diskover-", desc: "Index name prefix (optional)"} - - { env_var: "DISKOVER_OPTS", env_value: "", desc: "Optional arguments to pass to the diskover crawler (optional)"} - - { env_var: "WORKER_OPTS", env_value: "", desc: "Optional argumens to pass to the diskover bots launcher (optional)"} - - { env_var: "RUN_ON_START", env_value: "true", desc: "Initiate a crawl every time the container is started (optional)"} - - { env_var: "USE_CRON", env_value: "true", desc: "Run a crawl on as a cron job (optional)"} custom_compose: | version: '2' services: diskover: - image: linuxserver/diskover + image: ghcr.io/linuxserver/diskover container_name: diskover environment: - PUID=1000 - PGID=1000 - - TZ=Europe/London - - REDIS_HOST=redis - - REDIS_PORT=6379 + - TZ=America/New_York - ES_HOST=elasticsearch - ES_PORT=9200 - - ES_USER=elastic - - ES_PASS=changeme - - RUN_ON_START=true - - USE_CRON=true volumes: - /path/to/diskover/config:/config - /path/to/diskover/data:/data ports: - 80:80 - - 9181:9181 - - 9999:9999 mem_limit: 4096m restart: unless-stopped depends_on: - elasticsearch - - redis elasticsearch: container_name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.9 - volumes: - - ${DOCKER_HOME}/elasticsearch/data:/usr/share/elasticsearch/data + image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 environment: + - discovery.type=single-node - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" ulimits: memlock: soft: -1 hard: -1 - redis: - container_name: redis - image: redis:alpine volumes: - - ${HOME}/docker/redis:/data + - /path/to/esdata:/usr/share/elasticsearch/data + ports: + - 9200:9200 + depends_on: + - elasticsearch-helper + restart: unless-stopped + elasticsearch-helper: + image: alpine + command: sh -c "sysctl -w vm.max_map_count=262144" + privileged: true # application setup block app_setup_block_enabled: true app_setup_block: | - Once running the URL will be `http:///` initial application spinup will take some time so please reload if you get an empty response. We highly reccomend using Docker compose for this image as it includes multiple database backends to link into. - If you are looking to mount the elasticsearch and redis data to your host machine for access neither of them currently support setting a custom UID or GID they will run by default as: + This application is dependent on an ElasticSearch instance. Please see the example compose file for additional information. - - Redis - UID=999 GID=999 - - Elasticsearch - UID=1000 GID=1000 + The default username is diskover with the password of **darkdata**, access the container at http:///. The UI may be unusable until a valid index has been created. - ElasticSearch also requires a sysctl setting on the host machine to run properly. Running `sysctl -w vm.max_map_count=262144` will solve this issue. To make this setting persistent through reboots, set this value in `/etc/sysctl.conf`. + The application doesn't start an index by default. A crontab is created inside of the `/config` directory and can be set up to run automated indexes of `/data`. Changes to this crontab file require a restart to apply. You can also manually run an index by executing `/app/diskover/diskover.py` either in interactive or detached mode: - If you simply want the application to work you can mount these to folders with 0777 permissions, otherwise you will need to create these users host level and set the folder ownership properly. - - By default this compose example is pointed to a single directory and the UID and GID you pass to the diskover container needs to match that folders ownership. If these are shared folders with many owners the indexing will likely fail. - - For specific questions or help setting up diskover in your environment please refer to the project's Github page [{{ project_name|capitalize }}]({{ project_url }}). + * `docker exec -u abc -d /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the background + * `docker exec -u abc -it /app/diskover/diskover.py -i diskover-my_index_name /data` Will run an index in the foreground # changelog changelogs: + - { date: "11.10.21:", desc: "Updated to diskover-community v2." } - { date: "19.11.20:", desc: "Fix pip packages." } - { date: "19.12.19:", desc: "Rebasing to alpine 3.11." } - { date: "28.06.19:", desc: "Rebasing to alpine 3.10." } diff --git a/root/app/cleanup.sh b/root/app/cleanup.sh deleted file mode 100644 index ba221ae..0000000 --- a/root/app/cleanup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/with-contenv bash - -. /config/.env - -# define array for input values -declare -A DISKOVER_ARRAY -DISKOVER_ARRAY[REDIS_HOST]=${REDIS_HOST:-redis} -DISKOVER_ARRAY[REDIS_PORT]=${REDIS_PORT:-6379} - -DISKOVER_ARRAY[DISKOVER_OPTS]="${DISKOVER_ARRAY[DISKOVER_OPTS]} -i ${DISKOVER_ARRAY[INDEX_NAME]}" - -# killing existing workers before starting new ones -echo "killing existing workers..." -if [ -f "/tmp/diskover_bot_pids" ]; then - /bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1 - sleep 3 -fi - -# empty existing redis queue -echo "emptying current redis queues..." -rq empty -u redis://"${DISKOVER_ARRAY[REDIS_HOST]}":"${DISKOVER_ARRAY[REDIS_PORT]}" diskover_crawl diskover diskover_calcdir failed -sleep 3 - -echo "killing dangling workers..." -/bin/bash /app/diskover/diskover-bot-launcher.sh -k > /dev/null 2>&1 -sleep 3 \ No newline at end of file diff --git a/root/app/dispatcher.sh b/root/app/dispatcher.sh deleted file mode 100644 index 4cd2380..0000000 --- a/root/app/dispatcher.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/with-contenv bash - -. /config/.env - -# define array for input values -declare -A DISKOVER_ARRAY -DISKOVER_ARRAY[TODAY]=$(date +%Y-%m-%d) -DISKOVER_ARRAY[INDEX_PREFIX]=${INDEX_PREFIX:-diskover-} -DISKOVER_ARRAY[INDEX_NAME]=${INDEX_NAME:-${DISKOVER_ARRAY[INDEX_PREFIX]}${DISKOVER_ARRAY[TODAY]}} -DISKOVER_ARRAY[DISKOVER_OPTS]=${DISKOVER_OPTS:-}" -d /data -a" -DISKOVER_ARRAY[WORKER_OPTS]=${WORKER_OPTS:-""} -DISKOVER_ARRAY[REDIS_HOST]=${REDIS_HOST:-redis} -DISKOVER_ARRAY[REDIS_PORT]=${REDIS_PORT:-6379} - -DISKOVER_ARRAY[DISKOVER_OPTS]="${DISKOVER_ARRAY[DISKOVER_OPTS]} -i ${DISKOVER_ARRAY[INDEX_NAME]}" - -cd /app/diskover || exit - -/bin/bash /app/cleanup.sh - -echo "starting workers with following options: ${DISKOVER_ARRAY[WORKER_OPTS]}" -/bin/bash /app/diskover/diskover-bot-launcher.sh ${DISKOVER_ARRAY[WORKER_OPTS]} - -echo "starting crawler with following options: ${DISKOVER_ARRAY[DISKOVER_OPTS]}" -/usr/bin/python3 /app/diskover/diskover.py ${DISKOVER_ARRAY[DISKOVER_OPTS]} \ No newline at end of file diff --git a/root/defaults/crontabs/abc b/root/defaults/crontabs/abc new file mode 100644 index 0000000..50da62f --- /dev/null +++ b/root/defaults/crontabs/abc @@ -0,0 +1 @@ +#0 3 * * * python3 /app/diskover/diskover.py /data diff --git a/root/defaults/default b/root/defaults/default index a5a6d89..75f24f5 100644 --- a/root/defaults/default +++ b/root/defaults/default @@ -1,45 +1,38 @@ server { - listen 80 default_server; + listen 80 default_server; - listen 443 ssl; + listen 443 ssl; - root /app/diskover-web/public; - index index.php index.html index.htm; + root /app/www/public; + index index.php index.html index.htm; - server_name _; + server_name _; - ssl_certificate /config/keys/cert.crt; - ssl_certificate_key /config/keys/cert.key; + ssl_certificate /config/keys/cert.crt; + ssl_certificate_key /config/keys/cert.key; - client_max_body_size 0; + client_max_body_size 0; - error_page 599 = @noauth; + location / { + try_files $uri $uri/ /index.php?$query_string; + } - location / { - if (!-f /config/nginx/.htpasswd) { - return 599; - } - - auth_basic "Restricted"; - auth_basic_user_file /config/nginx/.htpasswd; - try_files $uri $uri/ /index.php?$args; - } +# # unless the request is for a valid file (image, js, css, etc.), send to bootstrap +# if (!-e $request_filename) +# { +# rewrite ^/(.*)$ /index.php?/$1 last; +# break; +# } - location @noauth { - try_files $uri $uri/ /index.php?$args; - } - - location ~ \.php(/|$) { - fastcgi_split_path_info ^(.+\.php)(/.+)$; - set $path_info $fastcgi_path_info; - fastcgi_param PATH_INFO $path_info; - try_files $fastcgi_script_name =404; - # With php5-cgi alone: - fastcgi_pass 127.0.0.1:9000; - # With php5-fpm: - #fastcgi_pass unix:/var/run/php5-fpm.sock; - fastcgi_index index.php; - include /etc/nginx/fastcgi_params; - fastcgi_read_timeout 900; - } + location ~ \.php$ { + fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_pass 127.0.0.1:9000; + fastcgi_index index.php; + include /etc/nginx/fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_intercept_errors off; + fastcgi_buffer_size 16k; + fastcgi_buffers 4 16k; + fastcgi_read_timeout 300; + } } diff --git a/root/defaults/diskover.cfg b/root/defaults/diskover.cfg deleted file mode 100644 index f08f490..0000000 --- a/root/defaults/diskover.cfg +++ /dev/null @@ -1,165 +0,0 @@ -; diskover config file -; if you make any changes, restart worker bots so they get the new config - -[excludes] -; directory names and absolute paths you want to exclude from crawl, case-sensitive, can include wildcards (.* or backup* or /dir/dirname* or *tmp or *tmp* etc) -dirs = .*,.snapshot,.Snapshot,.zfs -; files you want to exclude from crawl, case-sensitive, can include wildcards (.*, *.doc or NULLEXT for files with no extension) -files = .*,Thumbs.db,.DS_Store,._.DS_Store,.localized,desktop.ini - -[includes] -; directory names and absolute paths you want to include (whitelist), case-sensitive, you don't need to whitelist rootdir (-d rootdir) -;dirs = .recycle -; files you want to include (whitelist), case-sensitive -;files = - -[ownersgroups] -; control how owner (username) and group fields are stored for file and directory docs -; store uid and gid's instead of trying to get owner and group names (default is False) -;uidgidonly = False -; owner/group names contain domain name set to True (default is False) -;domain = False -; character separator used on cifs/nfs mounts to separte user/group and domain name, usually \\ or @ -;domainsep = \\ -; if domain name comes first before character separator, set this to True, otherwise False (default is True) -;domainfirst = True -; when indexing owner and group fields, keep the domain name (default is False) -;keepdomain = False - -[autotag] -; pattern dictionaries for diskover bots to use when auto-tagging, values are case-sensitive, can include wildcard for ext, name or path (tmp* or TMP* or *tmp or *TMP* etc) -; can also specify file name with json, example files = autotag.files.json -;files = [{"name": [], "name_exclude": [], "ext": ["tmp*", "TMP*", "temp*", "TEMP*", "cache*", "CACHE*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}] -;dirs = [{"name": ["*tmp*", "*TMP*", "*temp*", "*TEMP*", "*Temp*", "*cache*", "*CACHE*", "*Cache*"], "name_exclude": ["*templates*", "*Templates*"], "path": ["*/Application Support/*", "*/Containers/*"], "path_exclude": [], "mtime": 90, "atime": 0, "ctime": 90, "tag": "delete", "tag_custom": "autotag"}] - -[storagecost] -; storage cost per GB (default is 0.03 cents per GB) -costpergb = 0.03 -; use decimal base 10 (1000) or binary base 2 (1024) for GB size (default is 2, set to 2 or 10) -base = 2 -; pattern dictionaries for diskover bots to use when determing cost per GB (overrides above) -; can also specify file name with json, example paths = storagecost.paths.json -;paths = [{"path": ["*/fastdiskpath1/*", "*/Fastdiskpath2/*"], "path_exclude": [], "costpergb": 0.05}, {"path": ["*/slowdiskpath1/*", "*/Slowdiskpath2/*"], "path_exclude": [], "costpergb": 0.02}] -;times = [{"mtime": 180, "atime": 0, "ctime": 180, "costpergb": 0.02}] -; deciding factor if a match is in both paths and times, can be path or time -;priority = path - -[elasticsearch] -; uncomment the below three lines if you are using AWS ES -;aws = False -;host = search-diskover-es-cluster-eg3yztrvzb6qucroyyjk2vokza.ap-northeast-1.es.amazonaws.com -;port = 443 -; below two lines are for local ES, comment out if you are using AWS ES -host = {{ES_HOST}} -port = {{ES_PORT}} -; uncomment the below two lines if you installed X-Pack, for http-auth -user = {{ES_USER}} -password = {{ES_PASS}} -; index name for ES, cli arg overwrites this -indexname = diskover-index -; timeout for connection to ES (default is 10) -timeout = 30 -; number of connections kept open to ES when crawling (default is 10) -maxsize = 20 -; max retries for ES operations (default is 0) -maxretries = 10 -; wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait -wait = False -; chunk size for ES bulk operations (default is 500) -chunksize = 1000 -; number of shards for index (default is 5) -shards = 1 -; number of replicas for index (default is 1) -replicas = 0 -; the below settings are to optimize ES for crawling -; index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s -indexrefresh = 30s -; disable replicas during crawl - set to True to turn off replicas or False to keep on (default False), after crawl is set back to replicas value above -disablereplicas = True -; transaction log flush threshold size (default 512mb) -translogsize = 1gb -; search scroll size (default 100 docs) -scrollsize = 1000 - -[redis] -host = {{REDIS_HOST}} -port = {{REDIS_PORT}} -;password = -; cache directory times in Redis -; used for -I index2 when comparing directory times to get metadata from index2 instead of off disk -; set to True to cache dir times or False to turn off (default False) -cachedirtimes = False -; how long in seconds directory keys lives in Redis (default 1 day) -dirtimesttl = 604800 -; database to use (default is 0) -db = 0 -; rq job default time out in sec (default 180) -timeout = 3600 -; rq default ttl for key/results (default 500) -ttl = 500 -; rq queue names to use (default is diskover, diskover_crawl, diskover_calcdir) -queue = diskover -queuecrawl = diskover_crawl -queuecalcdir = diskover_calcdir - -[adaptivebatch] -; adaptive batch settings when using -a (intelligent crawling) -; batchsize (numbers of dirs) to start at -startsize = 50 -; maximum size of batch -maxsize = 500 -; when adjusting batch size use this for +/- (increases when queue is 0, decreases when > 0) -stepsize = 10 -; max number of files in batch, above is ignored if file limit reached (default 50000) -maxfiles = 50000 - -[paths] -; used by diskover socket server -; path to diskover.py (default is ./diskover.py) -diskoverpath = ./diskover.py -; path to python executable (default is python) -pythonpath = python - -[socketlistener] -; hostname and port (TCP) for diskover socket server for remote commands -host = 0.0.0.0 -port = 9999 -; max connections for diskover socket server -maxconnections = 5 -; port (TCP) for diskover socket server for messages from diskover treewalk client -twcport = 9998 - -[dupescheck] -; read size (bytes) for md5 sum check (how many bytes to read in at a time when md5 checking, default 64 KB) -readsize = 65536 -; max size (bytes) of files to check (files larger than this will be skipped, default 1 GB) -maxsize = 1073741824 -; bytes to check at start and end of file before doing md5 sum check (set large enough to account for file header info, default is 64) -checkbytes = 64 -; try to restore times (mtime/atime) for files that get opened by byte check and md5 -; set to True or False, default False (useful for cifs which does not work with noatime mount option) -restoretimes = False -; number of threads for calculating md5 checksum of files -threads = 8 - -[crawlbot] -; continuous scanner when running with --crawlbot -; time to sleep (seconds) between checking for directory changes -sleeptime = 0.1 -; number of threads for checking directories, setting this to num of cores x2 is a good starting point -threads = 8 -; how often in seconds to get new directory list with updated times from ES (default 3600) -dirlisttime = 3600 - -[gource] -; should be set to same in diskover-gource.sh -maxfilelag = 0.1 - -[crawlapi] -; crawl api url endpoint -;url = http://localhost:8080/api -; optional api login -;user = admin -;pass = admin -; number of items per page for each directory list request -;pagesize = 1000 \ No newline at end of file diff --git a/root/etc/cont-init.d/49-eol-check b/root/etc/cont-init.d/49-eol-check new file mode 100644 index 0000000..d74a14c --- /dev/null +++ b/root/etc/cont-init.d/49-eol-check @@ -0,0 +1,30 @@ +#!/usr/bin/with-contenv bash + +if [ -f "/config/diskover.cfg" ]; then + echo ' +****************************************************** +****************************************************** +* * +* * +* We have detected that you have an existing * +* * +* diskover v1 config file. Version 1 is now EOL. * +* * +* If you would like to upgrade, please create a * +* * +* new /config directory. If you want to continue * +* * +* using version 1, specify the tag v1.5.0.13-ls33. * +* * +* More information at * +* * +* https://github.com/diskoverdata/diskover-community * +* * +* * +****************************************************** +******************************************************' + while true + do + sleep 3600 + done +fi diff --git a/root/etc/cont-init.d/50-diskover-config b/root/etc/cont-init.d/50-diskover-config deleted file mode 100644 index cbd83be..0000000 --- a/root/etc/cont-init.d/50-diskover-config +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/with-contenv bash - -# set default values for variables -declare -A DISKOVER_CONF -DISKOVER_CONF[REDIS_HOST]=${REDIS_HOST:-redis} -DISKOVER_CONF[REDIS_PORT]=${REDIS_PORT:-6379} -DISKOVER_CONF[ES_HOST]=${ES_HOST:-elasticsearch} -DISKOVER_CONF[ES_PORT]=${ES_PORT:-9200} -DISKOVER_CONF[ES_USER]=${ES_USER:-elastic} -DISKOVER_CONF[ES_PASS]=${ES_PASS:-changeme} -DISKOVER_CONF[RUN_ON_START]=${RUN_ON_START:-false} -DISKOVER_CONF[USE_CRON]=${USE_CRON:-false} - -# persist env variables -if [ ! -f "/config/.env" ]; then - # sed in values or skip if value not set - for KEY in "${!DISKOVER_CONF[@]}"; do \ - if [[ ${DISKOVER_CONF[$KEY]} == "" ]]; then \ - : - else echo "export $KEY=${DISKOVER_CONF[$KEY]}" >> /config/.env - fi - done -fi - -if [ ! -f "/etc/profile.d/diskover.sh" ]; then - ln -s /config/.env /etc/profile.d/diskover.sh -fi - -# create folders -mkdir -p \ - /config/{log/diskover,crontabs} - -if [ ! -f "/config/diskover.cfg" ]; then - mv /defaults/diskover.cfg /config/diskover.cfg - for KEY in "${!DISKOVER_CONF[@]}"; do \ - if [[ ${DISKOVER_CONF[$KEY]} == "" ]]; then \ - : - else sed -i 's|{{'$KEY'}}|'${DISKOVER_CONF[$KEY]}'|g' /config/diskover.cfg - fi - done -fi - -if [ ! -f "/app/diskover/diskover.cfg" ]; then - ln -s /config/diskover.cfg /app/diskover/diskover.cfg -fi - -# copy crontab and proxy defaults if needed -[[ ! -f /config/crontabs/abc ]] && \ - cp /etc/crontabs/abc /config/crontabs/ - -# import user crontabs -rm /etc/crontabs/* -if [ ${DISKOVER_CONF[USE_CRON]} == true ]; then - cp /config/crontabs/* /etc/crontabs/ -fi - -# permissions -chmod +x \ - /app/dispatcher.sh -chown -R abc:abc \ - /app/diskover \ - /config - -# run initial retrieval or just start some worker bots -if [ ${DISKOVER_CONF[RUN_ON_START]} == true ]; then - echo "Initial run of dispatcher in progress" - # sleep to allow time for elasticsearch to start - sleep 10 - exec \ - s6-setuidgid abc /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 & -else - echo "Starting worker bots" - # sleep to allow time for elasticsearch to start - sleep 10 - exec \ - s6-setuidgid abc cd /app/diskover && ./diskover-bot-launcher.sh -fi \ No newline at end of file diff --git a/root/etc/cont-init.d/50-permissions b/root/etc/cont-init.d/50-permissions new file mode 100644 index 0000000..0b16751 --- /dev/null +++ b/root/etc/cont-init.d/50-permissions @@ -0,0 +1,13 @@ +#!/usr/bin/with-contenv bash + +if [ -d "/app/diskover-tmp" ]; then + echo "New container detected. Setting up app folder and fixing permissions." + mv /app/diskover-tmp /app/diskover +fi + + +if [ -d "/app/diskover-web-tmp" ]; then + mv /app/diskover-web-tmp /app/www +fi + +chown -R abc:abc /app diff --git a/root/etc/cont-init.d/51-config-diskover b/root/etc/cont-init.d/51-config-diskover new file mode 100644 index 0000000..2111fb6 --- /dev/null +++ b/root/etc/cont-init.d/51-config-diskover @@ -0,0 +1,18 @@ +#!/usr/bin/with-contenv bash + +# copy config +if [[ ! -e /config/diskover.conf.d ]]; then + cp -r /app/diskover/configs_sample /config/diskover.conf.d +fi + +ls /config/diskover.conf.d | \ + xargs -I {} \ + ln -sf /config/diskover.conf.d/{} /etc/{} + +# install crontab +[[ ! -e "/config/crontab" ]] && \ + cp /defaults/crontabs/abc /config/crontab + +crontab -u abc /config/crontab + +chown -R abc:abc /config diff --git a/root/etc/cont-init.d/52-config-diskover-web b/root/etc/cont-init.d/52-config-diskover-web new file mode 100644 index 0000000..a01f091 --- /dev/null +++ b/root/etc/cont-init.d/52-config-diskover-web @@ -0,0 +1,24 @@ +#!/usr/bin/with-contenv bash + +mkdir -p \ + /config/diskover-web.conf.d + +# copy over sample config files +cd /app/www/public +for file in *.sample; do + dest=${file%.sample} + + [[ ! -e "/config/diskover-web.conf.d/$dest" ]] && \ + cp /app/www/public/$file /config/diskover-web.conf.d/$dest + + ln -sf /config/diskover-web.conf.d/$dest /app/www/public/$dest +done + +# copy and set variables in Constants.php +if [[ ! -e "/config/diskover-web.conf.d/Constants.php" ]]; then + cp /app/www/src/diskover/Constants.php.sample /config/diskover-web.conf.d/Constants.php +fi + +ln -sf /config/diskover-web.conf.d/Constants.php /app/www/src/diskover/Constants.php + +chown -R abc:abc /config diff --git a/root/etc/cont-init.d/60-diskover-web-config b/root/etc/cont-init.d/60-diskover-web-config deleted file mode 100644 index 8ab1ed2..0000000 --- a/root/etc/cont-init.d/60-diskover-web-config +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/with-contenv bash - -if [ $DISABLE_WEB ]; then - echo "Not running web UI" - exit -fi - -# set default values for variables -declare -A WEB_CONF -WEB_CONF[ES_HOST]=${ES_HOST:-elasticsearch} -WEB_CONF[ES_PORT]=${ES_PORT:-9200} -WEB_CONF[ES_USER]=${ES_USER:-elastic} -WEB_CONF[ES_PASS]=${ES_PASS:-changeme} - -if [ ! -f "/config/Constants.php" ]; then - cp /app/diskover-web/src/diskover/Constants.php.sample /config/Constants.php - sed -i "s|ES_HOST = 'localhost'|ES_HOST = '"${WEB_CONF[ES_HOST]}"'|g" /config/Constants.php - sed -i "s|ES_PORT = 9200|ES_PORT = "${WEB_CONF[ES_PORT]}"|g" /config/Constants.php - sed -i "s|ES_USER = ''|ES_USER = '"${WEB_CONF[ES_USER]}"'|g" /config/Constants.php - sed -i "s|ES_PASS = ''|ES_PASS = '"${WEB_CONF[ES_PASS]}"'|g" /config/Constants.php - sed -i "s|ENABLE_SOCKET_CLIENT = FALSE|ENABLE_SOCKET_CLIENT = true|g" /config/Constants.php - sed -i "s|SOCKET_LISTENER_HOST = '127.0.0.1'|SOCKET_LISTENER_HOST = 'diskover'|g" /config/Constants.php -fi - -if [ ! -f "/app/diskover-web/src/diskover/Constants.php" ]; then - ln -s /config/Constants.php /app/diskover-web/src/diskover/Constants.php -fi - -if [ ! -f "/app/diskover-web/public/index.php" ]; then - cd /app/diskover-web/public && \ - ln -s dashboard.php index.php -fi - -if [ ! -f "/config/smartsearches.txt" ]; then - cp /app/diskover-web/public/smartsearches.txt.sample /config/smartsearches.txt -fi - -if [ ! -f "/app/diskover-web/public/smartsearches.txt" ]; then - ln -s /config/smartsearches.txt /app/diskover-web/public/smartsearches.txt -fi - -if [ ! -f "/config/customtags.txt" ]; then - cp /app/diskover-web/public/customtags.txt.sample /config/customtags.txt -fi - -if [ ! -f "/app/diskover-web/public/customtags.txt" ]; then - ln -s /config/customtags.txt /app/diskover-web/public/customtags.txt -fi - -if [ ! -f "/config/extrafields.txt.sample" ]; then - cp /app/diskover-web/public/extrafields.txt.sample /config/extrafields.txt -fi - -if [ ! -f "/app/diskover-web/public/extrafields.txt" ]; then - ln -s /config/extrafields.txt /app/diskover-web/public/extrafields.txt -fi - -# permissions -chown -R abc:abc \ - /app/diskover-web \ - /config diff --git a/root/etc/crontabs/abc b/root/etc/crontabs/abc deleted file mode 100644 index 0467791..0000000 --- a/root/etc/crontabs/abc +++ /dev/null @@ -1 +0,0 @@ -0 3 * * * /app/dispatcher.sh >> /config/log/diskover/dispatcher.log 2>&1 diff --git a/root/etc/logrotate.d/diskover b/root/etc/logrotate.d/diskover deleted file mode 100644 index ce51163..0000000 --- a/root/etc/logrotate.d/diskover +++ /dev/null @@ -1,9 +0,0 @@ -/config/log/diskover/*.log { - daily - rotate 14 - compress - nodateext - notifempty - missingok - su abc abc -} diff --git a/root/etc/services.d/cron/run b/root/etc/services.d/cron/run new file mode 100644 index 0000000..29cd322 --- /dev/null +++ b/root/etc/services.d/cron/run @@ -0,0 +1,3 @@ +#!/usr/bin/with-contenv bash + +exec /usr/sbin/crond -f -S -l 5 -c /etc/crontabs diff --git a/root/etc/services.d/rq-dashboard/run b/root/etc/services.d/rq-dashboard/run deleted file mode 100644 index caaae99..0000000 --- a/root/etc/services.d/rq-dashboard/run +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/with-contenv bash - -# need to safely handle exiting the existing worker bots and -# redis queue when container restarts or stops - -_term() { - /bin/bash /app/cleanup.sh -} - -trap _term SIGTERM - -declare -A REDIS_CONF -REDIS_CONF[REDIS_HOST]=${REDIS_HOST:-redis} -REDIS_CONF[REDIS_PORT]=${REDIS_PORT:-6379} - -exec \ - s6-setuidgid abc /usr/bin/rq-dashboard -H ${REDIS_CONF[REDIS_HOST]} -P ${REDIS_CONF[REDIS_PORT]} \ No newline at end of file diff --git a/root/etc/services.d/socket_server/run b/root/etc/services.d/socket_server/run deleted file mode 100644 index 3b7c50f..0000000 --- a/root/etc/services.d/socket_server/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/with-contenv bash - -exec \ - s6-setuidgid abc /usr/bin/python /app/diskover/diskover.py -l