From f4c070ada721f35aeb76f9fc662e069d2ecec3bd Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Dec 2024 20:08:36 -0500 Subject: [PATCH 1/7] Reduce image size and keep pycache in a volume Not only are volumes typically faster to access than the image overlay, the cachee can be reused between containers. --- Dockerfile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4941821b..a472f37a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -138,6 +138,9 @@ COPY pip.conf /etc/pip.conf # Add Pipfile COPY Pipfile /app/Pipfile +# Do not include compiled byte-code +ENV PIP_ROOT_USER_ACTION='ignore' PIP_NO_COMPILE=1 + # Switch workdir to the the app WORKDIR /app @@ -163,7 +166,7 @@ RUN set -x && \ useradd -M -d /app -s /bin/false -g app app && \ # Install non-distro packages cp -at /tmp/ "${HOME}" && \ - PIPENV_VERBOSITY=64 HOME="/tmp/${HOME#/}" pipenv install --system --skip-lock && \ + PIPENV_VERBOSITY=64 HOME="/tmp/${HOME#/}" pipenv install --system --skip-lock --no-cache-dir && \ # Clean up rm /app/Pipfile && \ pipenv --clear && \ @@ -184,7 +187,7 @@ RUN set -x && \ apt-get -y autoclean && \ rm -rf /var/lib/apt/lists/* && \ rm -rf /var/cache/apt/* && \ - rm -rf /tmp/* + rm -rfv /tmp/* # Copy app @@ -196,11 +199,12 @@ RUN set -x && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database rm -rf /app/db.sqlite3 && \ # Run any required app commands - /usr/bin/python3 /app/manage.py compilescss && \ - /usr/bin/python3 /app/manage.py collectstatic --no-input --link && \ + /usr/bin/python3 -B /app/manage.py compilescss && \ + /usr/bin/python3 -B /app/manage.py collectstatic --no-input --link && \ # Create config, downloads and run dirs mkdir -v -p /run/app && \ mkdir -v -p /config/media && \ + mkdir -v -p /config/pycache && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video @@ -219,7 +223,7 @@ COPY config/root / HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1:8080/healthcheck # ENVS and ports -ENV PYTHONPATH="/app" +ENV PYTHONPATH="/app" PYTHONPYCACHEPREFIX="/config/pycache" EXPOSE 4848 # Volumes From a2c97d27912e5118797637c2776f9a9317609f9a Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Dec 2024 20:22:08 -0500 Subject: [PATCH 2/7] pipenv did not run with that option --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a472f37a..05fd7686 100644 --- a/Dockerfile +++ b/Dockerfile @@ -139,7 +139,9 @@ COPY pip.conf /etc/pip.conf COPY Pipfile /app/Pipfile # Do not include compiled byte-code -ENV PIP_ROOT_USER_ACTION='ignore' PIP_NO_COMPILE=1 +ENV PIP_NO_COMPILE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_ROOT_USER_ACTION='ignore' # Switch workdir to the the app WORKDIR /app From 2c26936035d798dcfd598ccd2d50fe39b9e41c5b Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 21 Dec 2024 20:23:47 -0500 Subject: [PATCH 3/7] fixup: remove the --no-cache-dir option --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 05fd7686..0434d08b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,7 +168,7 @@ RUN set -x && \ useradd -M -d /app -s /bin/false -g app app && \ # Install non-distro packages cp -at /tmp/ "${HOME}" && \ - PIPENV_VERBOSITY=64 HOME="/tmp/${HOME#/}" pipenv install --system --skip-lock --no-cache-dir && \ + PIPENV_VERBOSITY=64 HOME="/tmp/${HOME#/}" pipenv install --system --skip-lock && \ # Clean up rm /app/Pipfile && \ pipenv --clear && \ From b9eb28fd36eaf2b1383651bab4fd544e3b2072f1 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 22 Dec 2024 01:45:16 -0500 Subject: [PATCH 4/7] There is a /config/cache directory we can safely use --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0434d08b..b89a48b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -206,7 +206,7 @@ RUN set -x && \ # Create config, downloads and run dirs mkdir -v -p /run/app && \ mkdir -v -p /config/media && \ - mkdir -v -p /config/pycache && \ + mkdir -v -p /config/cache/pycache && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video @@ -225,7 +225,7 @@ COPY config/root / HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1:8080/healthcheck # ENVS and ports -ENV PYTHONPATH="/app" PYTHONPYCACHEPREFIX="/config/pycache" +ENV PYTHONPATH="/app" PYTHONPYCACHEPREFIX="/config/cache/pycache" EXPOSE 4848 # Volumes From 2f89c333a49339b9cc17de882ed58a3f0c8aa64e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 22 Dec 2024 01:52:15 -0500 Subject: [PATCH 5/7] yt_dlp controls this directory It's probably safe, and `yt_dlp` plays nicely when cleaning up, but this is more correct. --- tubesync/tubesync/local_settings.py.container | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index a0426a4c..84e29c5f 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -60,7 +60,7 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: MEDIA_ROOT = CONFIG_BASE_DIR / 'media' DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR -YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache' +YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache/youtube' YOUTUBE_DL_TEMPDIR = DOWNLOAD_ROOT / 'cache' COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' From 42a1033e7eeef1dd3e9a17cbda7ff60d5c971fc4 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 22 Dec 2024 02:42:30 -0500 Subject: [PATCH 6/7] Rename old_youtube_cache_dirs It makes sense to do this here because anyone who had the previous setting was, more than likely, using this file. --- tubesync/tubesync/local_settings.py.container | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 84e29c5f..e75778b8 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -88,3 +88,13 @@ SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240")) + + +# ensure that the current directory exists +if not YOUTUBE_DL_CACHEDIR.is_dir(): + YOUTUBE_DL_CACHEDIR.mkdir(parents=True) +# rename any old yt_dlp cache directories to the current directory +old_youtube_cache_dirs = list(YOUTUBE_DL_CACHEDIR.parent.glob('youtube-*')) +for cache_dir in old_youtube_cache_dirs: + cache_dir.rename(YOUTUBE_DL_CACHEDIR / cache_dir.name) + From 7343e39f35bc466ca6efdea387787ee7083ec7f9 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 25 Dec 2024 00:32:03 -0500 Subject: [PATCH 7/7] Verbose output is no longer needed I confirmed that the changes reduced the amount of files being removed. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b89a48b6..b33af383 100644 --- a/Dockerfile +++ b/Dockerfile @@ -189,7 +189,7 @@ RUN set -x && \ apt-get -y autoclean && \ rm -rf /var/lib/apt/lists/* && \ rm -rf /var/cache/apt/* && \ - rm -rfv /tmp/* + rm -rf /tmp/* # Copy app