ci: add mypy job + pre-commit hook, tighten type annotations
- .github/workflows/ci.yml: new 'types' job runs mypy src/; docker now needs lint+types+test
- .pre-commit-config.yaml: add mirrors-mypy hook on src/ with types-requests, types-PyYAML
- pyproject.toml: add per-module overrides ignoring legacy modules (main, downloader, reddit_client, database, extractors.{reddit,gfycat}) while keeping src/web/ strict
- src/web/deps.py: type DOWNLOADS_DIR explicitly as Path
- src/web/config_manager.py: type guards on get_subreddits/users/blacklist returns
- src/web/routers/favorites.py: dict[str, object] for result, avoid in-place + on object
- src/web/routers/media.py: dict[str, object] for delete result
- src/web/rate_limit.py: move Awaitable/Callable under TYPE_CHECKING
- ruff format + auto-fix on src/ + tests/
This commit is contained in:
parent
34c2b16e19
commit
b4a2f303ac
12 changed files with 63 additions and 25 deletions
16
.github/workflows/ci.yml
vendored
16
.github/workflows/ci.yml
vendored
|
|
@ -15,8 +15,18 @@ jobs:
|
|||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install ruff
|
||||
- run: ruff check src/
|
||||
- run: ruff format --check src/
|
||||
- run: ruff check src/ tests/
|
||||
- run: ruff format --check src/ tests/
|
||||
|
||||
types:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -e ".[dev]"
|
||||
- run: mypy src/
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -30,7 +40,7 @@ jobs:
|
|||
|
||||
docker:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, test]
|
||||
needs: [lint, types, test]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: docker build -t reddit-media-collector .
|
||||
|
|
|
|||
|
|
@ -5,3 +5,11 @@ repos:
|
|||
- id: ruff
|
||||
args: [--fix]
|
||||
- id: ruff-format
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v1.10.0
|
||||
hooks:
|
||||
- id: mypy
|
||||
files: ^src/
|
||||
additional_dependencies:
|
||||
- types-requests
|
||||
- types-PyYAML
|
||||
|
|
|
|||
|
|
@ -72,6 +72,20 @@ disallow_untyped_defs = false
|
|||
check_untyped_defs = true
|
||||
ignore_missing_imports = true
|
||||
|
||||
# Legacy modules still being typed gradually — keep them out of strict mode
|
||||
# until they get their cleanup pass. New code under src/web/ is held to the
|
||||
# default strictness.
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"src.main",
|
||||
"src.downloader",
|
||||
"src.reddit_client",
|
||||
"src.database",
|
||||
"src.extractors.reddit",
|
||||
"src.extractors.gfycat",
|
||||
]
|
||||
ignore_errors = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["."]
|
||||
|
|
|
|||
|
|
@ -813,13 +813,16 @@ class Database:
|
|||
AND p.downloaded_at IS NOT NULL
|
||||
"""
|
||||
if favorites_only:
|
||||
query = base_select + """
|
||||
query = (
|
||||
base_select
|
||||
+ """
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM favorites f
|
||||
JOIN posts fp ON f.post_id = fp.id
|
||||
WHERE fp.author = p.author
|
||||
)
|
||||
"""
|
||||
)
|
||||
else:
|
||||
query = base_select
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,8 @@ def save_config(config: dict[str, Any]) -> None:
|
|||
def get_subreddits() -> list[dict[str, Any]]:
|
||||
"""Get list of configured subreddits."""
|
||||
config = load_config()
|
||||
return config.get("targets", {}).get("subreddits", [])
|
||||
subreddits: list[dict[str, Any]] = config.get("targets", {}).get("subreddits", []) or []
|
||||
return subreddits
|
||||
|
||||
|
||||
def add_subreddit(name: str, limit: int = 100, sort: str = "new") -> bool:
|
||||
|
|
@ -67,7 +68,8 @@ def remove_subreddit(name: str) -> bool:
|
|||
def get_users() -> list[dict[str, Any]]:
|
||||
"""Get list of configured users."""
|
||||
config = load_config()
|
||||
return config.get("targets", {}).get("users", [])
|
||||
users: list[dict[str, Any]] = config.get("targets", {}).get("users", []) or []
|
||||
return users
|
||||
|
||||
|
||||
def add_user(name: str, limit: int = 100) -> bool:
|
||||
|
|
@ -121,7 +123,8 @@ def get_blacklist() -> dict[str, list[str]]:
|
|||
"""Get the full blacklist configuration."""
|
||||
config = load_config()
|
||||
config = _ensure_blacklist(config)
|
||||
return config["blacklist"]
|
||||
blacklist: dict[str, list[str]] = config["blacklist"]
|
||||
return blacklist
|
||||
|
||||
|
||||
def add_blacklist_author(author: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -8,8 +8,7 @@ from pathlib import Path
|
|||
PROJECT_DIR = Path(__file__).parent.parent.parent
|
||||
|
||||
# Downloads directory for serving media files
|
||||
DOWNLOADS_DIR = os.environ.get("RMC_DOWNLOAD_DIR", str(PROJECT_DIR / "downloads"))
|
||||
DOWNLOADS_DIR = Path(DOWNLOADS_DIR)
|
||||
DOWNLOADS_DIR: Path = Path(os.environ.get("RMC_DOWNLOAD_DIR", str(PROJECT_DIR / "downloads")))
|
||||
|
||||
# Thumbnails directory
|
||||
THUMBS_DIR = DOWNLOADS_DIR / ".thumbs"
|
||||
|
|
|
|||
|
|
@ -10,10 +10,13 @@ from __future__ import annotations
|
|||
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
_Buckets = dict[tuple[str, str], deque[float]]
|
||||
_buckets: _Buckets = defaultdict(deque)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,16 +34,14 @@ async def add_favorite(post_id: str, add_user_to_collection: bool = True):
|
|||
|
||||
added = db.add_favorite(post_id)
|
||||
|
||||
result = {
|
||||
"message": f"Post '{post_id}' added to favorites" if added else "Post already in favorites",
|
||||
"added": added,
|
||||
}
|
||||
message = f"Post '{post_id}' added to favorites" if added else "Post already in favorites"
|
||||
result: dict[str, object] = {"message": message, "added": added}
|
||||
|
||||
if add_user_to_collection and post.author and post.author not in ("[deleted]", "AutoModerator"):
|
||||
user_added = config_manager.add_user(post.author, limit=100)
|
||||
if user_added:
|
||||
result["user_added"] = post.author
|
||||
result["message"] += f". User '{post.author}' added to collection targets."
|
||||
result["message"] = message + f". User '{post.author}' added to collection targets."
|
||||
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -281,7 +281,7 @@ async def delete_media(post_id: str, blacklist_author: bool = False, blacklist_s
|
|||
conn.execute("DELETE FROM posts WHERE id = ?", (post_id,))
|
||||
conn.commit()
|
||||
|
||||
result = {"message": f"Media '{post_id}' deleted successfully"}
|
||||
result: dict[str, object] = {"message": f"Media '{post_id}' deleted successfully"}
|
||||
if blacklisted:
|
||||
result["blacklisted"] = blacklisted
|
||||
return result
|
||||
|
|
@ -320,9 +320,7 @@ async def preview_blacklist_cleanup(
|
|||
|
||||
db = Database()
|
||||
author_count = await asyncio.to_thread(db.count_posts_by_authors, authors_all) if authors_all else 0
|
||||
subreddit_count = (
|
||||
await asyncio.to_thread(db.count_posts_by_subreddits, subreddits_all) if subreddits_all else 0
|
||||
)
|
||||
subreddit_count = await asyncio.to_thread(db.count_posts_by_subreddits, subreddits_all) if subreddits_all else 0
|
||||
|
||||
if limit is not None:
|
||||
authors_page = authors_all[offset : offset + limit]
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from src.web import config_manager
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
|
@ -23,7 +23,7 @@ def _make_post(pid: str, **overrides) -> PostRecord:
|
|||
media_type="image",
|
||||
score=42,
|
||||
created_utc=1_700_000_000.0,
|
||||
downloaded_at=datetime.now(timezone.utc),
|
||||
downloaded_at=datetime.now(UTC),
|
||||
local_path=f"./downloads/{pid}.jpg",
|
||||
file_hash=f"hash{pid}",
|
||||
permalink=f"/r/testsub/comments/{pid}",
|
||||
|
|
|
|||
|
|
@ -2,8 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
|
@ -24,7 +23,7 @@ def _make_post(post_id: str, **overrides) -> PostRecord:
|
|||
media_type="image",
|
||||
score=42,
|
||||
created_utc=1_700_000_000.0,
|
||||
downloaded_at=datetime.now(timezone.utc),
|
||||
downloaded_at=datetime.now(UTC),
|
||||
local_path=None,
|
||||
file_hash="hash" + post_id,
|
||||
permalink=f"/r/testsub/comments/{post_id}",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue