From 279c10f4ece0387fe8ea7aee807d19b38437ed91 Mon Sep 17 00:00:00 2001 From: quietsy Date: Thu, 13 Feb 2025 17:15:37 +0200 Subject: [PATCH 1/6] Add initial date --- root/app/models.py | 3 ++- root/app/updater.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/root/app/models.py b/root/app/models.py index 76294b0..c23a72a 100644 --- a/root/app/models.py +++ b/root/app/models.py @@ -1,7 +1,7 @@ from pydantic import BaseModel # Increment when updating schema or forcing an update on start -IMAGES_SCHEMA_VERSION = 2 +IMAGES_SCHEMA_VERSION = 3 class Tag(BaseModel): @@ -92,6 +92,7 @@ class Config(BaseModel): class Image(BaseModel): name: str + initial_date: str github_url: str project_url: str | None = None project_logo: str | None = None diff --git a/root/app/updater.py b/root/app/updater.py index b849fee..caf0cb7 100644 --- a/root/app/updater.py +++ b/root/app/updater.py @@ -31,13 +31,15 @@ def get_architectures(readme_vars): archs.append(Architecture(arch=item["arch"], tag=item["tag"])) return archs -def get_changelogs(readme_vars): +def get_changelog(readme_vars): if "changelogs" not in readme_vars: return None - changelogs = [] + changelog = [] for item in readme_vars["changelogs"][0:3]: - changelogs.append(Changelog(date=item["date"][0:-1], desc=item["desc"])) - return changelogs + changelog.append(Changelog(date=item["date"][0:-1], desc=item["desc"])) + first_changelog = readme_vars["changelogs"][-1] + initial_date = first_changelog["date"][0:-1] + return changelog, initial_date def get_description(readme_vars): description = readme_vars.get("project_blurb", "No description") @@ -153,6 +155,7 @@ def get_image(repo): application_setup = None if readme_vars.get("app_setup_block_enabled", False): application_setup = f"{repo.html_url}?tab=readme-ov-file#application-setup" + changelog, initial_date = get_changelog(readme_vars) config = Config( application_setup=application_setup, readonly_supported=readme_vars.get("readonly_supported", None), @@ -171,6 +174,7 @@ def get_image(repo): ) return Image( name=project_name, + initial_date=initial_date, github_url=repo.html_url, stars=repo.stargazers_count, project_url=readme_vars.get("project_url", None), @@ -183,7 +187,7 @@ def get_image(repo): deprecated=deprecated, tags=tags, architectures=get_architectures(readme_vars), - changelog=get_changelogs(readme_vars), + changelog=changelog, config=config, ) From 03c3f94f0679502bc8c41a857fbcc406e23d8190 Mon Sep 17 00:00:00 2001 From: quietsy Date: Thu, 13 Feb 2025 18:21:44 +0200 Subject: [PATCH 2/6] Add monthly pulls --- root/app/models.py | 3 ++- root/app/updater.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/root/app/models.py b/root/app/models.py index c23a72a..8193b14 100644 --- a/root/app/models.py +++ b/root/app/models.py @@ -92,7 +92,7 @@ class Config(BaseModel): class Image(BaseModel): name: str - initial_date: str + initial_date: str | None = None github_url: str project_url: str | None = None project_logo: str | None = None @@ -103,6 +103,7 @@ class Image(BaseModel): stable: bool deprecated: bool stars: int + monthly_pulls: int | None = None tags: list[Tag] architectures: list[Architecture] changelog: list[Changelog] | None = None diff --git a/root/app/updater.py b/root/app/updater.py index caf0cb7..748585c 100644 --- a/root/app/updater.py +++ b/root/app/updater.py @@ -6,10 +6,13 @@ from models import Repository, ImagesData, ImagesResponse, IMAGES_SCHEMA_VERSION import datetime import os +import requests import time +import traceback CI = os.environ.get("CI", None) INVALIDATE_HOURS = int(os.environ.get("INVALIDATE_HOURS", "24")) +SCARF_TOKEN = os.environ.get("SCARF_TOKEN", None) def get_tags(readme_vars): @@ -33,7 +36,7 @@ def get_architectures(readme_vars): def get_changelog(readme_vars): if "changelogs" not in readme_vars: - return None + return None, None changelog = [] for item in readme_vars["changelogs"][0:3]: changelog.append(Changelog(date=item["date"][0:-1], desc=item["desc"])) @@ -138,7 +141,23 @@ def get_mac_address(readme_vars): hostname = readme_vars.get("param_mac_address", False) return MacAddress(mac_address=hostname, desc=readme_vars.get("param_mac_address_desc", ""), optional=optional) -def get_image(repo): +def get_monthly_pulls(): + pulls_map = {} + try: + response = requests.get("https://api.scarf.sh/v2/packages/linuxserver-ci/overview?per_page=1000", headers={"Authorization": f"Bearer {SCARF_TOKEN}"}) + results = response.json()["results"] + for result in results: + name = result["package"]["name"].replace("linuxserver/", "") + if "total_installs" not in result: + continue + monthly_pulls = result["total_installs"] + pulls_map[name] = monthly_pulls + except Exception: + print(traceback.format_exc()) + raise + return pulls_map + +def get_image(repo, pulls_map): print(f"Processing {repo.name}") if not repo.name.startswith("docker-") or repo.name.startswith("docker-baseimage-"): return None @@ -176,7 +195,6 @@ def get_image(repo): name=project_name, initial_date=initial_date, github_url=repo.html_url, - stars=repo.stargazers_count, project_url=readme_vars.get("project_url", None), project_logo=readme_vars.get("project_logo", None), description=get_description(readme_vars), @@ -185,6 +203,8 @@ def get_image(repo): category=categories, stable=stable, deprecated=deprecated, + stars=repo.stargazers_count, + monthly_pulls=pulls_map.get(project_name, None), tags=tags, architectures=get_architectures(readme_vars), changelog=changelog, @@ -198,16 +218,17 @@ def update_images(): print(f"{datetime.datetime.now()} - skipped - already updated") return print(f"{datetime.datetime.now()} - updating images") + pulls_map = get_monthly_pulls() images = [] repos = gh.get_repos() for repo in sorted(repos, key=lambda repo: repo.name): - image = get_image(repo) + image = get_image(repo, pulls_map) if not image: continue images.append(image) data = ImagesData(repositories=Repository(linuxserver=images)) - last_updated = datetime.datetime.now(datetime.timezone.utc).isoformat(' ', 'seconds') + last_updated = datetime.datetime.now(datetime.timezone.utc).isoformat(" ", "seconds") response = ImagesResponse(status="OK", last_updated=last_updated, data=data) new_state = response.model_dump_json(exclude_none=True) kv.set_value("images", new_state, IMAGES_SCHEMA_VERSION) From eb84d04e25fb1f3df2dd7e1da5879ba1ca772435 Mon Sep 17 00:00:00 2001 From: quietsy Date: Thu, 13 Feb 2025 18:46:09 +0200 Subject: [PATCH 3/6] Normalize date formats --- root/app/updater.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/root/app/updater.py b/root/app/updater.py index 748585c..658a73d 100644 --- a/root/app/updater.py +++ b/root/app/updater.py @@ -39,10 +39,13 @@ def get_changelog(readme_vars): return None, None changelog = [] for item in readme_vars["changelogs"][0:3]: - changelog.append(Changelog(date=item["date"][0:-1], desc=item["desc"])) + date = item["date"][0:-1] + normalized_date = str(datetime.datetime.strptime(date, "%d.%m.%y").date()) + changelog.append(Changelog(date=normalized_date, desc=item["desc"])) first_changelog = readme_vars["changelogs"][-1] initial_date = first_changelog["date"][0:-1] - return changelog, initial_date + normalized_initial_date = str(datetime.datetime.strptime(initial_date, "%d.%m.%y").date()) + return changelog, normalized_initial_date def get_description(readme_vars): description = readme_vars.get("project_blurb", "No description") From af0ca76b9aab0ec02103e574520a46a755a00fcf Mon Sep 17 00:00:00 2001 From: quietsy Date: Thu, 13 Feb 2025 19:23:23 +0200 Subject: [PATCH 4/6] Refactor --- root/app/models.py | 1 + root/app/updater.py | 60 ++++++++++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/root/app/models.py b/root/app/models.py index 8193b14..4d6830e 100644 --- a/root/app/models.py +++ b/root/app/models.py @@ -2,6 +2,7 @@ from pydantic import BaseModel # Increment when updating schema or forcing an update on start IMAGES_SCHEMA_VERSION = 3 +SCARF_SCHEMA_VERSION = 1 class Tag(BaseModel): diff --git a/root/app/updater.py b/root/app/updater.py index 658a73d..8cd95d4 100644 --- a/root/app/updater.py +++ b/root/app/updater.py @@ -2,9 +2,10 @@ import lsio_github as gh from keyvaluestore import KeyValueStore, set_db_schema from models import Architecture, Changelog, Tag, EnvVar, Volume, Port, Config from models import Custom, SecurityOpt, Device, Cap, Hostname, MacAddress, Image -from models import Repository, ImagesData, ImagesResponse, IMAGES_SCHEMA_VERSION +from models import Repository, ImagesData, ImagesResponse, IMAGES_SCHEMA_VERSION, SCARF_SCHEMA_VERSION import datetime +import json import os import requests import time @@ -144,23 +145,7 @@ def get_mac_address(readme_vars): hostname = readme_vars.get("param_mac_address", False) return MacAddress(mac_address=hostname, desc=readme_vars.get("param_mac_address_desc", ""), optional=optional) -def get_monthly_pulls(): - pulls_map = {} - try: - response = requests.get("https://api.scarf.sh/v2/packages/linuxserver-ci/overview?per_page=1000", headers={"Authorization": f"Bearer {SCARF_TOKEN}"}) - results = response.json()["results"] - for result in results: - name = result["package"]["name"].replace("linuxserver/", "") - if "total_installs" not in result: - continue - monthly_pulls = result["total_installs"] - pulls_map[name] = monthly_pulls - except Exception: - print(traceback.format_exc()) - raise - return pulls_map - -def get_image(repo, pulls_map): +def get_image(repo, scarf_data): print(f"Processing {repo.name}") if not repo.name.startswith("docker-") or repo.name.startswith("docker-baseimage-"): return None @@ -207,7 +192,7 @@ def get_image(repo, pulls_map): stable=stable, deprecated=deprecated, stars=repo.stargazers_count, - monthly_pulls=pulls_map.get(project_name, None), + monthly_pulls=scarf_data.get(project_name, None), tags=tags, architectures=get_architectures(readme_vars), changelog=changelog, @@ -218,14 +203,14 @@ def update_images(): with KeyValueStore(invalidate_hours=INVALIDATE_HOURS, readonly=False) as kv: is_current_schema = kv.is_current_schema("images", IMAGES_SCHEMA_VERSION) if ("images" in kv and is_current_schema) or CI == "1": - print(f"{datetime.datetime.now()} - skipped - already updated") + print(f"{datetime.datetime.now()} - images skipped - already updated") return print(f"{datetime.datetime.now()} - updating images") - pulls_map = get_monthly_pulls() images = [] + scarf_data = json.loads(kv["scarf"]) repos = gh.get_repos() for repo in sorted(repos, key=lambda repo: repo.name): - image = get_image(repo, pulls_map) + image = get_image(repo, scarf_data) if not image: continue images.append(image) @@ -237,10 +222,41 @@ def update_images(): kv.set_value("images", new_state, IMAGES_SCHEMA_VERSION) print(f"{datetime.datetime.now()} - updated images") +def get_monthly_pulls(): + pulls_map = {} + try: + response = requests.get("https://api.scarf.sh/v2/packages/linuxserver-ci/overview?per_page=1000", headers={"Authorization": f"Bearer {SCARF_TOKEN}"}) + results = response.json()["results"] + for result in results: + name = result["package"]["name"].replace("linuxserver/", "") + if "total_installs" not in result: + continue + monthly_pulls = result["total_installs"] + pulls_map[name] = monthly_pulls + except Exception: + print(traceback.format_exc()) + return pulls_map + +def update_scarf(): + with KeyValueStore(invalidate_hours=INVALIDATE_HOURS, readonly=False) as kv: + is_current_schema = kv.is_current_schema("scarf", SCARF_SCHEMA_VERSION) + if ("scarf" in kv and is_current_schema) or CI == "1": + print(f"{datetime.datetime.now()} - scarf skipped - already updated") + return + print(f"{datetime.datetime.now()} - updating scarf") + pulls_map = get_monthly_pulls() + if not pulls_map: + return + new_state = json.dumps(pulls_map) + kv.set_value("scarf", new_state, SCARF_SCHEMA_VERSION) + print(f"{datetime.datetime.now()} - updated scarf") + + def main(): set_db_schema() while True: gh.print_rate_limit() + update_scarf() update_images() gh.print_rate_limit() time.sleep(INVALIDATE_HOURS*60*60) From af6a726c2f6eb6888e92822fd05446913b31ba2e Mon Sep 17 00:00:00 2001 From: quietsy Date: Fri, 14 Feb 2025 09:15:41 +0200 Subject: [PATCH 5/6] Add token to readme --- README.md | 1 + readme-vars.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 7177e3d..5faebba 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ docker run -d \ -e DB_FILE=/config/api.db `#optional` \ -e INVALIDATE_HOURS=24 `#optional` \ -e PAT=token `#optional` \ +-e SCARF_TOKEN=token `#optional` \ -e URL=http://localhost:8000 `#optional` \ -p 8000:8000 \ -v /path/to/lsio-api/config:/config \ diff --git a/readme-vars.yml b/readme-vars.yml index aade1b4..dccad73 100644 --- a/readme-vars.yml +++ b/readme-vars.yml @@ -40,6 +40,7 @@ full_custom_readme: | -e DB_FILE=/config/api.db `#optional` \ -e INVALIDATE_HOURS=24 `#optional` \ -e PAT=token `#optional` \ + -e SCARF_TOKEN=token `#optional` \ -e URL=http://localhost:8000 `#optional` \ -p 8000:8000 \ -v /path/to/lsio-api/config:/config \ From e151a3a429ff47e9889da064d41b79f3dc9435c7 Mon Sep 17 00:00:00 2001 From: quietsy Date: Fri, 14 Feb 2025 11:17:59 +0200 Subject: [PATCH 6/6] Update health when the updater fails --- root/app/api.py | 11 ++++++++++- root/app/updater.py | 44 ++++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/root/app/api.py b/root/app/api.py index e176b6a..414d3d2 100644 --- a/root/app/api.py +++ b/root/app/api.py @@ -18,9 +18,18 @@ api.mount("/static", StaticFiles(directory="static"), name="static") async def swagger_ui_html(): return get_swagger_ui_html(openapi_url="/openapi.json", title="LinuxServer API", swagger_favicon_url="/static/logo.png") +async def get_status(): + with KeyValueStore() as kv: + return kv["status"] + @api.get("/health", summary="Get the health status") async def health(): - return "Success" + try: + content = await get_status() + return JSONResponse(content=content) + except Exception: + print(traceback.format_exc()) + raise HTTPException(status_code=404, detail="Not found") async def get_images(): with KeyValueStore() as kv: diff --git a/root/app/updater.py b/root/app/updater.py index 8cd95d4..c0b3226 100644 --- a/root/app/updater.py +++ b/root/app/updater.py @@ -224,17 +224,14 @@ def update_images(): def get_monthly_pulls(): pulls_map = {} - try: - response = requests.get("https://api.scarf.sh/v2/packages/linuxserver-ci/overview?per_page=1000", headers={"Authorization": f"Bearer {SCARF_TOKEN}"}) - results = response.json()["results"] - for result in results: - name = result["package"]["name"].replace("linuxserver/", "") - if "total_installs" not in result: - continue - monthly_pulls = result["total_installs"] - pulls_map[name] = monthly_pulls - except Exception: - print(traceback.format_exc()) + response = requests.get("https://api.scarf.sh/v2/packages/linuxserver-ci/overview?per_page=1000", headers={"Authorization": f"Bearer {SCARF_TOKEN}"}) + results = response.json()["results"] + for result in results: + name = result["package"]["name"].replace("linuxserver/", "") + if "total_installs" not in result: + continue + monthly_pulls = result["total_installs"] + pulls_map[name] = monthly_pulls return pulls_map def update_scarf(): @@ -250,15 +247,26 @@ def update_scarf(): new_state = json.dumps(pulls_map) kv.set_value("scarf", new_state, SCARF_SCHEMA_VERSION) print(f"{datetime.datetime.now()} - updated scarf") - + +def update_status(status): + with KeyValueStore(invalidate_hours=0, readonly=False) as kv: + print(f"{datetime.datetime.now()} - updating status") + kv.set_value("status", status, 0) + print(f"{datetime.datetime.now()} - updated status") def main(): - set_db_schema() - while True: - gh.print_rate_limit() - update_scarf() - update_images() - gh.print_rate_limit() + try: + set_db_schema() + while True: + gh.print_rate_limit() + update_scarf() + update_images() + gh.print_rate_limit() + update_status("Success") + time.sleep(INVALIDATE_HOURS*60*60) + except: + print(traceback.format_exc()) + update_status("Failed") time.sleep(INVALIDATE_HOURS*60*60) if __name__ == "__main__":