From 657082e940660a3343cfc0d36019c5a56c78ca67 Mon Sep 17 00:00:00 2001 From: mauvehed Date: Sat, 5 Sep 2026 10:25:02 -0500 Subject: [PATCH] ci: validate memorial json against schema and site conventions Add schema/person.schema.json and scripts/validate_people.py, run via a pre-commit hook and a GitHub Actions job on pull requests. Checks: - people/*.json match the schema (keys, types, item shapes) - every peoplelist.json entry has a file and every file is listed once - filenames are lowercase alphanumeric (two legacy names grandfathered) - local image paths exist under images/ - link fields are http(s) with no surrounding whitespace - text-only fields contain no HTML; maintext stays raw HTML by design Warns (non-fatal) on gallery items with no title, which the page uses as alt text. Adds a pip dependabot entry for scripts/requirements.txt. --- .github/dependabot.yml | 8 + .github/workflows/validate-people.yml | 32 ++++ .pre-commit-config.yaml | 9 ++ CONTRIBUTING.md | 2 + schema/person.schema.json | 93 ++++++++++++ scripts/requirements.txt | 1 + scripts/validate_people.py | 204 ++++++++++++++++++++++++++ 7 files changed, 349 insertions(+) create mode 100644 .github/workflows/validate-people.yml create mode 100644 schema/person.schema.json create mode 100644 scripts/requirements.txt create mode 100755 scripts/validate_people.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6383546..b4f5f75 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,3 +13,11 @@ updates: target-branch: "main" assignees: - "mauvehed" + # Python dependencies for scripts/validate_people.py + - package-ecosystem: "pip" + directory: "/scripts" + schedule: + interval: "weekly" + target-branch: "main" + assignees: + - "mauvehed" diff --git a/.github/workflows/validate-people.yml b/.github/workflows/validate-people.yml new file mode 100644 index 0000000..3053dae --- /dev/null +++ b/.github/workflows/validate-people.yml @@ -0,0 +1,32 @@ +# Validates memorial data (people/*.json, peoplelist.json) against +# schema/person.schema.json and the site's cross-file conventions. +name: Validate memorial data + +on: + pull_request: + paths: + - "people/**" + - "peoplelist.json" + - "images/**" + - "schema/**" + - "scripts/**" + - ".github/workflows/validate-people.yml" + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/requirements.txt + - run: pip install -r scripts/requirements.txt + - run: python scripts/validate_people.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 084a626..323f244 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,5 +13,14 @@ repos: hooks: - id: prettier types_or: [json, html, css, javascript, yaml, markdown] +- repo: local + hooks: + - id: validate-people + name: validate memorial data + entry: python scripts/validate_people.py + language: python + additional_dependencies: [jsonschema==4.26.0] + pass_filenames: false + files: ^(people/.*\.json|peoplelist\.json|images/.*|schema/.*|scripts/validate_people\.py)$ ci: autofix_prs: true # set false to stop pull-request commits being added by pre-commit.ci diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 244684f..d76ac69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,6 +16,8 @@ Verify that an Issue [does not already exist](https://github.com/restincode/rest Alternatively you can open a regular issue or you can also choose to submit the json file (located in [people/](https://github.com/restincode/restincode/tree/master/people)) directly (via a pull request). If you wish to submit your own json, please utilize this [template](https://github.com/restincode/restincode/blob/master/people/_template.json) to get started. +Before opening a pull request, run `pre-commit run --all-files`. This formats the files and runs `scripts/validate_people.py`, which checks every `people/*.json` against [schema/person.schema.json](schema/person.schema.json), confirms `peoplelist.json` and the image files line up, and rejects HTML in text-only fields. + ### Adding data to existing person Every person on RiC should have an open [Issue](https://github.com/restincode/restincode/issues) within the project. The first step is to search and locate the existing Issue for the person who's data you want to add to. Once you've located it, just leave a comment on the Issue with the new information to be added. diff --git a/schema/person.schema.json b/schema/person.schema.json new file mode 100644 index 0000000..d98af6a --- /dev/null +++ b/schema/person.schema.json @@ -0,0 +1,93 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://restincode.com/schema/person.schema.json", + "title": "RestInCode memorial entry", + "description": "Shape of a people/.json file. See people/_template.json for a starting point.", + "type": "object", + "additionalProperties": false, + "required": [ + "firstname", + "lastname", + "handle", + "birth", + "death", + "obituary", + "issue", + "affiliations", + "mainimage", + "maintext", + "socialmedialinks", + "references", + "contributions", + "gallery" + ], + "properties": { + "firstname": { "type": "string" }, + "lastname": { "type": "string" }, + "handle": { "type": "string" }, + "birth": { "type": "string" }, + "death": { "type": "string" }, + "obituary": { "type": "string" }, + "issue": { + "type": "string", + "pattern": "^[0-9]*$", + "description": "GitHub issue number as a string, or empty" + }, + "affiliations": { "type": "string" }, + "mainimage": { "type": "string", "minLength": 1 }, + "maintext": { + "type": "string", + "description": "Raw HTML. This is the only field rendered as HTML." + }, + "socialmedialinks": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["sitename", "siteurl"], + "properties": { + "sitename": { "type": "string", "minLength": 1 }, + "siteurl": { "type": "string", "minLength": 1 } + } + } + }, + "references": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["title", "url"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "url": { "type": "string", "minLength": 1 } + } + } + }, + "contributions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["title", "url", "description"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "url": { "type": "string" }, + "description": { "type": "string" } + } + } + }, + "gallery": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["url", "caption"], + "properties": { + "url": { "type": "string", "minLength": 1 }, + "title": { "type": "string" }, + "caption": { "type": "string" } + } + } + } + } +} diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..3ee1ea1 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1 @@ +jsonschema==4.26.0 diff --git a/scripts/validate_people.py b/scripts/validate_people.py new file mode 100755 index 0000000..41bc8cb --- /dev/null +++ b/scripts/validate_people.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Validate memorial data files. + +Checks people/*.json against schema/person.schema.json and enforces the +cross-file conventions the site relies on: + + - every peoplelist.json entry points at an existing people/.json + - every people/*.json (except _template.json) is listed exactly once + - filenames are lowercase alphanumeric + - local image paths exist under images/ + - link fields are http(s) URLs with no surrounding whitespace + - text-only fields contain no HTML (maintext is the one HTML field) + +people/_template.json is scaffolding with intentional blanks and is skipped. + +Errors exit non-zero. Warnings are printed but do not fail the run. + +Usage: scripts/validate_people.py [--root DIR] +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +import jsonschema + +FILENAME_RE = re.compile(r"^[a-z0-9]+$") +HTTP_RE = re.compile(r"^https?://\S+$") +HTML_RE = re.compile(r"<[a-zA-Z/!]|&[a-zA-Z#][a-zA-Z0-9]*;") +# Filenames that predate the lowercase-alphanumeric rule. Renaming them would +# change public memorial URLs, so they are allowed as-is. Do not add to this +# list for new entries. +LEGACY_FILENAMES = {"karenKrystalia", "obsèquesdepaolo"} + +TEXT_FIELDS = ("firstname", "lastname", "handle", "affiliations", "birth", "death") + +PEOPLELIST_SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["people"], + "properties": { + "people": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": ["displayname", "filename"], + "properties": { + "displayname": {"type": "string", "minLength": 1}, + "filename": {"type": "string"}, + }, + }, + } + }, +} + + +class Report: + def __init__(self) -> None: + self.errors: list[str] = [] + self.warnings: list[str] = [] + + def error(self, where: str, msg: str) -> None: + self.errors.append(f"{where}: {msg}") + + def warn(self, where: str, msg: str) -> None: + self.warnings.append(f"{where}: {msg}") + + +def load_json(path: Path, report: Report): + try: + with path.open(encoding="utf-8") as fh: + return json.load(fh) + except (OSError, ValueError) as exc: + report.error(f"{path.parent.name}/{path.name}", f"cannot parse JSON: {exc}") + return None + + +def check_schema(path: Path, data, validator: jsonschema.Validator, report: Report) -> None: + for err in sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path)): + loc = "/".join(str(p) for p in err.absolute_path) or "(root)" + report.error(str(path), f"schema: {loc}: {err.message}") + + +def check_text(where: str, field: str, value, report: Report) -> None: + if isinstance(value, str) and HTML_RE.search(value): + report.error(where, f"{field}: HTML is not allowed in this field: {value!r}") + + +def check_link(where: str, field: str, value, report: Report) -> None: + if not isinstance(value, str) or value == "": + return + if value != value.strip(): + report.error(where, f"{field}: URL has surrounding whitespace: {value!r}") + elif not HTTP_RE.match(value): + report.error(where, f"{field}: not an http(s) URL: {value!r}") + + +def check_image(where: str, field: str, value, root: Path, report: Report) -> None: + if not isinstance(value, str) or value == "": + return + if value.startswith("/images/"): + if not (root / value.lstrip("/")).is_file(): + report.error(where, f"{field}: image not found: {value}") + else: + check_link(where, field, value, report) + + +def check_person(path: Path, data: dict, root: Path, report: Report) -> None: + where = str(path.relative_to(root)) + + for field in TEXT_FIELDS: + check_text(where, field, data.get(field), report) + + check_link(where, "obituary", data.get("obituary"), report) + check_image(where, "mainimage", data.get("mainimage"), root, report) + + for i, item in enumerate(data.get("socialmedialinks") or []): + check_text(where, f"socialmedialinks[{i}].sitename", item.get("sitename"), report) + check_link(where, f"socialmedialinks[{i}].siteurl", item.get("siteurl"), report) + + for i, item in enumerate(data.get("references") or []): + check_text(where, f"references[{i}].title", item.get("title"), report) + check_link(where, f"references[{i}].url", item.get("url"), report) + + for i, item in enumerate(data.get("contributions") or []): + check_text(where, f"contributions[{i}].title", item.get("title"), report) + check_text(where, f"contributions[{i}].description", item.get("description"), report) + check_link(where, f"contributions[{i}].url", item.get("url"), report) + + for i, item in enumerate(data.get("gallery") or []): + check_text(where, f"gallery[{i}].title", item.get("title"), report) + check_text(where, f"gallery[{i}].caption", item.get("caption"), report) + check_image(where, f"gallery[{i}].url", item.get("url"), root, report) + if not item.get("title"): + report.warn(where, f"gallery[{i}]: no title (used as the image alt text)") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parent.parent) + args = parser.parse_args() + root: Path = args.root + report = Report() + + schema = load_json(root / "schema" / "person.schema.json", report) + if schema is None: + return finish(report) + validator_cls = jsonschema.validators.validator_for(schema) + validator_cls.check_schema(schema) + person_validator = validator_cls(schema) + list_validator = jsonschema.Draft202012Validator(PEOPLELIST_SCHEMA) + + people_dir = root / "people" + files = {p.stem: p for p in sorted(people_dir.glob("*.json")) if p.stem != "_template"} + + peoplelist_path = root / "peoplelist.json" + peoplelist = load_json(peoplelist_path, report) + listed: dict[str, int] = {} + if peoplelist is not None: + check_schema(peoplelist_path.relative_to(root), peoplelist, list_validator, report) + for entry in peoplelist.get("people", []): + name = entry.get("filename", "") + if name == "": + continue # intentional no-link card + listed[name] = listed.get(name, 0) + 1 + if name not in files: + report.error("peoplelist.json", f"'{name}' has no people/{name}.json") + for name, count in listed.items(): + if count > 1: + report.error("peoplelist.json", f"'{name}' is listed {count} times") + + for stem, path in files.items(): + where = str(path.relative_to(root)) + if stem not in listed: + report.error(where, "not listed in peoplelist.json") + if not FILENAME_RE.match(stem) and stem not in LEGACY_FILENAMES: + report.error(where, "filename must be lowercase alphanumeric (a-z, 0-9)") + data = load_json(path, report) + if data is None: + continue + check_schema(path.relative_to(root), data, person_validator, report) + if isinstance(data, dict): + check_person(path, data, root, report) + + print(f"checked {len(files)} memorial files, {len(listed)} peoplelist entries") + return finish(report) + + +def finish(report: Report) -> int: + for msg in report.warnings: + print(f"WARNING: {msg}") + for msg in report.errors: + print(f"ERROR: {msg}") + print(f"{len(report.errors)} error(s), {len(report.warnings)} warning(s)") + return 1 if report.errors else 0 + + +if __name__ == "__main__": + sys.exit(main())