diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6383546..b4f5f75 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,3 +13,11 @@ updates: target-branch: "main" assignees: - "mauvehed" + # Python dependencies for scripts/validate_people.py + - package-ecosystem: "pip" + directory: "/scripts" + schedule: + interval: "weekly" + target-branch: "main" + assignees: + - "mauvehed" diff --git a/.github/workflows/validate-people.yml b/.github/workflows/validate-people.yml new file mode 100644 index 0000000..3053dae --- /dev/null +++ b/.github/workflows/validate-people.yml @@ -0,0 +1,32 @@ +# Validates memorial data (people/*.json, peoplelist.json) against +# schema/person.schema.json and the site's cross-file conventions. +name: Validate memorial data + +on: + pull_request: + paths: + - "people/**" + - "peoplelist.json" + - "images/**" + - "schema/**" + - "scripts/**" + - ".github/workflows/validate-people.yml" + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/requirements.txt + - run: pip install -r scripts/requirements.txt + - run: python scripts/validate_people.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 084a626..323f244 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,5 +13,14 @@ repos: hooks: - id: prettier types_or: [json, html, css, javascript, yaml, markdown] +- repo: local + hooks: + - id: validate-people + name: validate memorial data + entry: python scripts/validate_people.py + language: python + additional_dependencies: [jsonschema==4.26.0] + pass_filenames: false + files: ^(people/.*\.json|peoplelist\.json|images/.*|schema/.*|scripts/validate_people\.py)$ ci: autofix_prs: true # set false to stop pull-request commits being added by pre-commit.ci diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 244684f..d76ac69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,6 +16,8 @@ Verify that an Issue [does not already exist](https://github.com/restincode/rest Alternatively you can open a regular issue or you can also choose to submit the json file (located in [people/](https://github.com/restincode/restincode/tree/master/people)) directly (via a pull request). If you wish to submit your own json, please utilize this [template](https://github.com/restincode/restincode/blob/master/people/_template.json) to get started. +Before opening a pull request, run `pre-commit run --all-files`. This formats the files and runs `scripts/validate_people.py`, which checks every `people/*.json` against [schema/person.schema.json](schema/person.schema.json), confirms `peoplelist.json` and the image files line up, and rejects HTML in text-only fields. + ### Adding data to existing person Every person on RiC should have an open [Issue](https://github.com/restincode/restincode/issues) within the project. The first step is to search and locate the existing Issue for the person who's data you want to add to. Once you've located it, just leave a comment on the Issue with the new information to be added. diff --git a/people/_template.json b/people/_template.json index 3d9024f..9a751b3 100644 --- a/people/_template.json +++ b/people/_template.json @@ -40,7 +40,7 @@ "contributions": [ { "title": "[project name]", - "url": "htps://www.cyberz.tech/", + "url": "https://www.cyberz.tech/", "description": "Research paper on [project name]. As part of a team studying lorem ipsum dolor sit amet, consectetur adipisicing elit. Eveniet quibusdam, officia eos." }, { diff --git a/people/barnabyjack.json b/people/barnabyjack.json index 5cc4a4d..dd1057c 100644 --- a/people/barnabyjack.json +++ b/people/barnabyjack.json @@ -49,7 +49,7 @@ "caption": "The showman and the cash-spewing ATMs: Barnaby Jack at the 2010 Black Hat convention." }, { - "url": "/images/Barnaby_Jack3.png", + "url": "/images/Barnaby_Jack3.jpg", "title": "Barnaby Jack shows one of his two ATM machines", "caption": "Barnaby Jack shows one of his two ATM machines he bought on line and has hacked into, at his San Jose home on Tuesday, Aug. 31, 2010." } diff --git a/people/bill.json b/people/bill.json index 8cbe427..af0e153 100644 --- a/people/bill.json +++ b/people/bill.json @@ -7,7 +7,7 @@ "obituary": "", "issue": "59", "affiliations": "", - "mainimage": "/images/bill.jpg", + "mainimage": "/images/Bill.jpg", "maintext": "25 years later, still Wolfgame. IT Consultant, Cider Enthusiast, Vaper, Gamer, and cranky fucker with no time for your shit.", "socialmedialinks": [ { @@ -19,7 +19,7 @@ "contributions": [], "gallery": [ { - "url": "/images/bill.jpg", + "url": "/images/Bill.jpg", "title": "Bill (Wolfgame) at a 2600 NYC dinner in 1996", "caption": "Bill (@wolfgame) in the middle wearing a birthday crown at a 2600 NYC dinner for Meme's birthday in 1996. Photo by Pamela Howell (@serendipitousP)." } diff --git a/people/bradsmith.json b/people/bradsmith.json index 50245d7..de0e1fe 100644 --- a/people/bradsmith.json +++ b/people/bradsmith.json @@ -43,7 +43,7 @@ "contributions": [ { "title": "2014 - Presentation - Hacker Halted - Cracking BlueTooth for Phun and Profit", - "url": " https://www.yumpu.com/en/document/view/22139939/cracking-bluetooth-for-phun-and-profit-brad-smith-hacker-halted", + "url": "https://www.yumpu.com/en/document/view/22139939/cracking-bluetooth-for-phun-and-profit-brad-smith-hacker-halted", "description": "" }, { diff --git a/people/bruceesquibel.json b/people/bruceesquibel.json index ac61ace..08bb3cd 100644 --- a/people/bruceesquibel.json +++ b/people/bruceesquibel.json @@ -23,17 +23,17 @@ "contributions": [ { "title": "Dr. Ripco's Last Stand", - "url": " https://archive.org/details/ripcoslaststand", + "url": "https://archive.org/details/ripcoslaststand", "description": "A collection of gathered media of interest to Bruce \"Dr. Ripco\" Esquibel. Archived by Jason Scott." }, { "title": "Closing the Net", - "url": " http://www.costik.com/hackers.html", + "url": "http://www.costik.com/hackers.html", "description": "January 1991 issue of Reason magazine. Detailing events that took place during Operation Sun Devil." }, { "title": "BBS Documentary Video", - "url": " https://archive.org/details/bbs-20040123-ripco/Doctor+Ripco+-+Tape+1.avi", + "url": "https://archive.org/details/bbs-20040123-ripco/Doctor+Ripco+-+Tape+1.avi", "description": "Full BBS Documentary Interview: Dr. Ripco" }, { diff --git a/people/larrytesler.json b/people/larrytesler.json index 7950ca1..7b6b766 100644 --- a/people/larrytesler.json +++ b/people/larrytesler.json @@ -23,7 +23,7 @@ "contributions": [ { "title": "Computer scientist", - "url": "htps://www.cyberz.tech/", + "url": "", "description": "" } ], diff --git a/schema/person.schema.json b/schema/person.schema.json new file mode 100644 index 0000000..d98af6a --- /dev/null +++ b/schema/person.schema.json @@ -0,0 +1,93 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://restincode.com/schema/person.schema.json", + "title": "RestInCode memorial entry", + "description": "Shape of a people/.json file. See people/_template.json for a starting point.", + "type": "object", + "additionalProperties": false, + "required": [ + "firstname", + "lastname", + "handle", + "birth", + "death", + "obituary", + "issue", + "affiliations", + "mainimage", + "maintext", + "socialmedialinks", + "references", + "contributions", + "gallery" + ], + "properties": { + "firstname": { "type": "string" }, + "lastname": { "type": "string" }, + "handle": { "type": "string" }, + "birth": { "type": "string" }, + "death": { "type": "string" }, + "obituary": { "type": "string" }, + "issue": { + "type": "string", + "pattern": "^[0-9]*$", + "description": "GitHub issue number as a string, or empty" + }, + "affiliations": { "type": "string" }, + "mainimage": { "type": "string", "minLength": 1 }, + "maintext": { + "type": "string", + "description": "Raw HTML. This is the only field rendered as HTML." + }, + "socialmedialinks": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["sitename", "siteurl"], + "properties": { + "sitename": { "type": "string", "minLength": 1 }, + "siteurl": { "type": "string", "minLength": 1 } + } + } + }, + "references": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["title", "url"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "url": { "type": "string", "minLength": 1 } + } + } + }, + "contributions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["title", "url", "description"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "url": { "type": "string" }, + "description": { "type": "string" } + } + } + }, + "gallery": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["url", "caption"], + "properties": { + "url": { "type": "string", "minLength": 1 }, + "title": { "type": "string" }, + "caption": { "type": "string" } + } + } + } + } +} diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..3ee1ea1 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1 @@ +jsonschema==4.26.0 diff --git a/scripts/validate_people.py b/scripts/validate_people.py new file mode 100755 index 0000000..53840ef --- /dev/null +++ b/scripts/validate_people.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Validate memorial data files. + +Checks people/*.json against schema/person.schema.json and enforces the +cross-file conventions the site relies on: + + - every peoplelist.json entry points at an existing people/.json + - every people/*.json (except _template.json) is listed exactly once + - filenames are lowercase alphanumeric + - local image paths exist under images/ + - link fields are http(s) URLs with no surrounding whitespace + - text-only fields contain no HTML (maintext is the one HTML field) + +people/_template.json is scaffolding with intentional blanks and is skipped. + +Errors exit non-zero. Warnings are printed but do not fail the run. + +Usage: scripts/validate_people.py [--root DIR] +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +import jsonschema + +FILENAME_RE = re.compile(r"^[a-z0-9]+$") +HTTP_RE = re.compile(r"^https?://\S+$") +HTML_RE = re.compile(r"<[a-zA-Z/!]|&[a-zA-Z#][a-zA-Z0-9]*;") +# Filenames that predate the lowercase-alphanumeric rule. Renaming them would +# change public memorial URLs, so they are allowed as-is. Do not add to this +# list for new entries. +LEGACY_FILENAMES = {"karenKrystalia", "obsèquesdepaolo"} + +TEXT_FIELDS = ("firstname", "lastname", "handle", "affiliations", "birth", "death") + +PEOPLELIST_SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["people"], + "properties": { + "people": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": ["displayname", "filename"], + "properties": { + "displayname": {"type": "string", "minLength": 1}, + "filename": {"type": "string"}, + }, + }, + } + }, +} + + +class Report: + def __init__(self) -> None: + self.errors: list[str] = [] + self.warnings: list[str] = [] + + def error(self, where: str, msg: str) -> None: + self.errors.append(f"{where}: {msg}") + + def warn(self, where: str, msg: str) -> None: + self.warnings.append(f"{where}: {msg}") + + +def load_json(path: Path, report: Report): + try: + with path.open(encoding="utf-8") as fh: + return json.load(fh) + except (OSError, ValueError) as exc: + report.error(f"{path.parent.name}/{path.name}", f"cannot parse JSON: {exc}") + return None + + +def check_schema(path: Path, data, validator: jsonschema.Validator, report: Report) -> bool: + """Report schema errors. Returns True when the data is valid. + + The convention checks below assume schema-valid structure (arrays of + objects with the expected keys), so callers skip them on False. + """ + errors = sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path)) + for err in errors: + loc = "/".join(str(p) for p in err.absolute_path) or "(root)" + report.error(str(path), f"schema: {loc}: {err.message}") + return not errors + + +def check_text(where: str, field: str, value, report: Report) -> None: + if isinstance(value, str) and HTML_RE.search(value): + report.error(where, f"{field}: HTML is not allowed in this field: {value!r}") + + +def check_link(where: str, field: str, value, report: Report) -> None: + if not isinstance(value, str) or value == "": + return + if value != value.strip(): + report.error(where, f"{field}: URL has surrounding whitespace: {value!r}") + elif not HTTP_RE.match(value): + report.error(where, f"{field}: not an http(s) URL: {value!r}") + + +def list_images(root: Path) -> set[str]: + """Site-relative paths of every file under images/, e.g. 'images/Foo.jpg'. + + Compared as exact strings so a case mismatch fails here the same way it + fails on GitHub Pages, even on a case-insensitive local filesystem. + """ + images_dir = root / "images" + return {p.relative_to(root).as_posix() for p in images_dir.rglob("*") if p.is_file()} + + +def check_image( + where: str, field: str, value, images: set[str], report: Report +) -> None: + if not isinstance(value, str) or value == "": + return + if value.startswith("/images/"): + if value.lstrip("/") not in images: + report.error(where, f"{field}: image not found (case-sensitive): {value}") + else: + check_link(where, field, value, report) + + +def check_person( + path: Path, data: dict, root: Path, images: set[str], report: Report +) -> None: + where = str(path.relative_to(root)) + + for field in TEXT_FIELDS: + check_text(where, field, data.get(field), report) + + check_link(where, "obituary", data.get("obituary"), report) + check_image(where, "mainimage", data.get("mainimage"), images, report) + + for i, item in enumerate(data.get("socialmedialinks") or []): + check_text(where, f"socialmedialinks[{i}].sitename", item.get("sitename"), report) + check_link(where, f"socialmedialinks[{i}].siteurl", item.get("siteurl"), report) + + for i, item in enumerate(data.get("references") or []): + check_text(where, f"references[{i}].title", item.get("title"), report) + check_link(where, f"references[{i}].url", item.get("url"), report) + + for i, item in enumerate(data.get("contributions") or []): + check_text(where, f"contributions[{i}].title", item.get("title"), report) + check_text(where, f"contributions[{i}].description", item.get("description"), report) + check_link(where, f"contributions[{i}].url", item.get("url"), report) + + for i, item in enumerate(data.get("gallery") or []): + check_text(where, f"gallery[{i}].title", item.get("title"), report) + check_text(where, f"gallery[{i}].caption", item.get("caption"), report) + check_image(where, f"gallery[{i}].url", item.get("url"), images, report) + if not item.get("title"): + report.warn(where, f"gallery[{i}]: no title (used as the image alt text)") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parent.parent) + args = parser.parse_args() + root: Path = args.root + report = Report() + + schema = load_json(root / "schema" / "person.schema.json", report) + if schema is None: + return finish(report) + validator_cls = jsonschema.validators.validator_for(schema) + validator_cls.check_schema(schema) + person_validator = validator_cls(schema) + list_validator = jsonschema.Draft202012Validator(PEOPLELIST_SCHEMA) + + people_dir = root / "people" + images = list_images(root) + files = {p.stem: p for p in sorted(people_dir.glob("*.json")) if p.stem != "_template"} + + peoplelist_path = root / "peoplelist.json" + peoplelist = load_json(peoplelist_path, report) + listed: dict[str, int] = {} + peoplelist_ok = peoplelist is not None and check_schema( + peoplelist_path.relative_to(root), peoplelist, list_validator, report + ) + if peoplelist_ok: + for entry in peoplelist["people"]: + name = entry["filename"] + if name == "": + continue # intentional no-link card + listed[name] = listed.get(name, 0) + 1 + if name not in files: + report.error("peoplelist.json", f"'{name}' has no people/{name}.json") + for name, count in listed.items(): + if count > 1: + report.error("peoplelist.json", f"'{name}' is listed {count} times") + + for stem, path in files.items(): + where = str(path.relative_to(root)) + if peoplelist_ok and stem not in listed: + report.error(where, "not listed in peoplelist.json") + if not FILENAME_RE.match(stem) and stem not in LEGACY_FILENAMES: + report.error(where, "filename must be lowercase alphanumeric (a-z, 0-9)") + data = load_json(path, report) + if data is None: + continue + if check_schema(path.relative_to(root), data, person_validator, report): + check_person(path, data, root, images, report) + + print(f"checked {len(files)} memorial files, {len(listed)} peoplelist entries") + return finish(report) + + +def finish(report: Report) -> int: + for msg in report.warnings: + print(f"WARNING: {msg}") + for msg in report.errors: + print(f"ERROR: {msg}") + print(f"{len(report.errors)} error(s), {len(report.warnings)} warning(s)") + return 1 if report.errors else 0 + + +if __name__ == "__main__": + sys.exit(main())