Merge pull request #351 from restincode/ci/validate-people-json

ci: validate memorial json against schema and site conventions

.github/dependabot.yml

@@ -13,3 +13,11 @@ updates:
     target-branch: "main"
     assignees:
       - "mauvehed"
+  # Python dependencies for scripts/validate_people.py
+  - package-ecosystem: "pip"
+    directory: "/scripts"
+    schedule:
+      interval: "weekly"
+    target-branch: "main"
+    assignees:
+      - "mauvehed"

.github/workflows/validate-people.yml

@@ -0,0 +1,32 @@
+# Validates memorial data (people/*.json, peoplelist.json) against
+# schema/person.schema.json and the site's cross-file conventions.
+name: Validate memorial data
+
+on:
+  pull_request:
+    paths:
+      - "people/**"
+      - "peoplelist.json"
+      - "images/**"
+      - "schema/**"
+      - "scripts/**"
+      - ".github/workflows/validate-people.yml"
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: scripts/requirements.txt
+      - run: pip install -r scripts/requirements.txt
+      - run: python scripts/validate_people.py

.pre-commit-config.yaml

@@ -13,5 +13,14 @@ repos:
     hooks:
     -   id: prettier
         types_or: [json, html, css, javascript, yaml, markdown]
+-   repo: local
+    hooks:
+    -   id: validate-people
+        name: validate memorial data
+        entry: python scripts/validate_people.py
+        language: python
+        additional_dependencies: [jsonschema==4.26.0]
+        pass_filenames: false
+        files: ^(people/.*\.json|peoplelist\.json|images/.*|schema/.*|scripts/validate_people\.py)$
 ci:
     autofix_prs: true # set false to stop pull-request commits being added by pre-commit.ci

CONTRIBUTING.md

@@ -16,6 +16,8 @@ Verify that an Issue [does not already exist](https://github.com/restincode/rest
 
 Alternatively you can open a <a href="https://github.com/restincode/restincode/issues/new">regular issue</a> or you can also choose to submit the json file (located in [people/](https://github.com/restincode/restincode/tree/master/people)) directly (via a pull request). If you wish to submit your own json, please utilize this [template](https://github.com/restincode/restincode/blob/master/people/_template.json) to get started.
 
+Before opening a pull request, run `pre-commit run --all-files`. This formats the files and runs `scripts/validate_people.py`, which checks every `people/*.json` against [schema/person.schema.json](schema/person.schema.json), confirms `peoplelist.json` and the image files line up, and rejects HTML in text-only fields.
+
 ### Adding data to existing person
 
 Every person on RiC should have an open [Issue](https://github.com/restincode/restincode/issues) within the project. The first step is to search and locate the existing Issue for the person who's data you want to add to. Once you've located it, just leave a comment on the Issue with the new information to be added.

people/_template.json

@@ -40,7 +40,7 @@
   "contributions": [
     {
       "title": "[project name]",
-      "url": "htps://www.cyberz.tech/",
+      "url": "https://www.cyberz.tech/",
       "description": "Research paper on [project name]. As part of a team studying lorem ipsum dolor sit amet, consectetur adipisicing elit. Eveniet quibusdam, officia eos."
     },
     {

people/barnabyjack.json

@@ -49,7 +49,7 @@
       "caption": "The showman and the cash-spewing ATMs: Barnaby Jack at the 2010 Black Hat convention."
     },
     {
-      "url": "/images/Barnaby_Jack3.png",
+      "url": "/images/Barnaby_Jack3.jpg",
       "title": "Barnaby Jack shows one of his two ATM machines",
       "caption": "Barnaby Jack shows one of his two ATM machines he bought on line and has hacked into, at his San Jose home on Tuesday, Aug. 31, 2010."
     }

people/bill.json

@@ -7,7 +7,7 @@
   "obituary": "",
   "issue": "59",
   "affiliations": "",
-  "mainimage": "/images/bill.jpg",
+  "mainimage": "/images/Bill.jpg",
   "maintext": "25 years later, still Wolfgame.  IT Consultant, Cider Enthusiast, Vaper, Gamer, and cranky fucker with no time for your shit.",
   "socialmedialinks": [
     {
@@ -19,7 +19,7 @@
   "contributions": [],
   "gallery": [
     {
-      "url": "/images/bill.jpg",
+      "url": "/images/Bill.jpg",
       "title": "Bill (Wolfgame) at a 2600 NYC dinner in 1996",
       "caption": "Bill (@wolfgame) in the middle wearing a birthday crown at a 2600 NYC dinner for Meme's birthday in 1996. Photo by Pamela Howell (@serendipitousP)."
     }

people/bradsmith.json

@@ -43,7 +43,7 @@
   "contributions": [
     {
       "title": "2014 - Presentation - Hacker Halted - Cracking BlueTooth for Phun and Profit",
-      "url": "  https://www.yumpu.com/en/document/view/22139939/cracking-bluetooth-for-phun-and-profit-brad-smith-hacker-halted",
+      "url": "https://www.yumpu.com/en/document/view/22139939/cracking-bluetooth-for-phun-and-profit-brad-smith-hacker-halted",
       "description": ""
     },
     {

people/bruceesquibel.json

@@ -23,17 +23,17 @@
   "contributions": [
     {
       "title": "Dr. Ripco's Last Stand",
-      "url": " https://archive.org/details/ripcoslaststand",
+      "url": "https://archive.org/details/ripcoslaststand",
       "description": "A collection of gathered media of interest to Bruce \"Dr. Ripco\" Esquibel. Archived by Jason Scott."
     },
     {
       "title": "Closing the Net",
-      "url": " http://www.costik.com/hackers.html",
+      "url": "http://www.costik.com/hackers.html",
       "description": "January 1991 issue of Reason magazine. Detailing events that took place during Operation Sun Devil."
     },
     {
       "title": "BBS Documentary Video",
-      "url": " https://archive.org/details/bbs-20040123-ripco/Doctor+Ripco+-+Tape+1.avi",
+      "url": "https://archive.org/details/bbs-20040123-ripco/Doctor+Ripco+-+Tape+1.avi",
       "description": "Full BBS Documentary Interview: Dr. Ripco"
     },
     {

people/larrytesler.json

@@ -23,7 +23,7 @@
   "contributions": [
     {
       "title": "Computer scientist",
-      "url": "htps://www.cyberz.tech/",
+      "url": "",
       "description": ""
     }
   ],

schema/person.schema.json

@@ -0,0 +1,93 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://restincode.com/schema/person.schema.json",
+  "title": "RestInCode memorial entry",
+  "description": "Shape of a people/<name>.json file. See people/_template.json for a starting point.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "firstname",
+    "lastname",
+    "handle",
+    "birth",
+    "death",
+    "obituary",
+    "issue",
+    "affiliations",
+    "mainimage",
+    "maintext",
+    "socialmedialinks",
+    "references",
+    "contributions",
+    "gallery"
+  ],
+  "properties": {
+    "firstname": { "type": "string" },
+    "lastname": { "type": "string" },
+    "handle": { "type": "string" },
+    "birth": { "type": "string" },
+    "death": { "type": "string" },
+    "obituary": { "type": "string" },
+    "issue": {
+      "type": "string",
+      "pattern": "^[0-9]*$",
+      "description": "GitHub issue number as a string, or empty"
+    },
+    "affiliations": { "type": "string" },
+    "mainimage": { "type": "string", "minLength": 1 },
+    "maintext": {
+      "type": "string",
+      "description": "Raw HTML. This is the only field rendered as HTML."
+    },
+    "socialmedialinks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["sitename", "siteurl"],
+        "properties": {
+          "sitename": { "type": "string", "minLength": 1 },
+          "siteurl": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "references": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["title", "url"],
+        "properties": {
+          "title": { "type": "string", "minLength": 1 },
+          "url": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "contributions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["title", "url", "description"],
+        "properties": {
+          "title": { "type": "string", "minLength": 1 },
+          "url": { "type": "string" },
+          "description": { "type": "string" }
+        }
+      }
+    },
+    "gallery": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["url", "caption"],
+        "properties": {
+          "url": { "type": "string", "minLength": 1 },
+          "title": { "type": "string" },
+          "caption": { "type": "string" }
+        }
+      }
+    }
+  }
+}

scripts/requirements.txt

@@ -0,0 +1 @@
+jsonschema==4.26.0

scripts/validate_people.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Validate memorial data files.
+
+Checks people/*.json against schema/person.schema.json and enforces the
+cross-file conventions the site relies on:
+
+  - every peoplelist.json entry points at an existing people/<filename>.json
+  - every people/*.json (except _template.json) is listed exactly once
+  - filenames are lowercase alphanumeric
+  - local image paths exist under images/
+  - link fields are http(s) URLs with no surrounding whitespace
+  - text-only fields contain no HTML (maintext is the one HTML field)
+
+people/_template.json is scaffolding with intentional blanks and is skipped.
+
+Errors exit non-zero. Warnings are printed but do not fail the run.
+
+Usage: scripts/validate_people.py [--root DIR]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+import jsonschema
+
+FILENAME_RE = re.compile(r"^[a-z0-9]+$")
+HTTP_RE = re.compile(r"^https?://\S+$")
+HTML_RE = re.compile(r"<[a-zA-Z/!]|&[a-zA-Z#][a-zA-Z0-9]*;")
+# Filenames that predate the lowercase-alphanumeric rule. Renaming them would
+# change public memorial URLs, so they are allowed as-is. Do not add to this
+# list for new entries.
+LEGACY_FILENAMES = {"karenKrystalia", "obsèquesdepaolo"}
+
+TEXT_FIELDS = ("firstname", "lastname", "handle", "affiliations", "birth", "death")
+
+PEOPLELIST_SCHEMA = {
+    "type": "object",
+    "additionalProperties": False,
+    "required": ["people"],
+    "properties": {
+        "people": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "additionalProperties": False,
+                "required": ["displayname", "filename"],
+                "properties": {
+                    "displayname": {"type": "string", "minLength": 1},
+                    "filename": {"type": "string"},
+                },
+            },
+        }
+    },
+}
+
+
+class Report:
+    def __init__(self) -> None:
+        self.errors: list[str] = []
+        self.warnings: list[str] = []
+
+    def error(self, where: str, msg: str) -> None:
+        self.errors.append(f"{where}: {msg}")
+
+    def warn(self, where: str, msg: str) -> None:
+        self.warnings.append(f"{where}: {msg}")
+
+
+def load_json(path: Path, report: Report):
+    try:
+        with path.open(encoding="utf-8") as fh:
+            return json.load(fh)
+    except (OSError, ValueError) as exc:
+        report.error(f"{path.parent.name}/{path.name}", f"cannot parse JSON: {exc}")
+        return None
+
+
+def check_schema(path: Path, data, validator: jsonschema.Validator, report: Report) -> bool:
+    """Report schema errors. Returns True when the data is valid.
+
+    The convention checks below assume schema-valid structure (arrays of
+    objects with the expected keys), so callers skip them on False.
+    """
+    errors = sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path))
+    for err in errors:
+        loc = "/".join(str(p) for p in err.absolute_path) or "(root)"
+        report.error(str(path), f"schema: {loc}: {err.message}")
+    return not errors
+
+
+def check_text(where: str, field: str, value, report: Report) -> None:
+    if isinstance(value, str) and HTML_RE.search(value):
+        report.error(where, f"{field}: HTML is not allowed in this field: {value!r}")
+
+
+def check_link(where: str, field: str, value, report: Report) -> None:
+    if not isinstance(value, str) or value == "":
+        return
+    if value != value.strip():
+        report.error(where, f"{field}: URL has surrounding whitespace: {value!r}")
+    elif not HTTP_RE.match(value):
+        report.error(where, f"{field}: not an http(s) URL: {value!r}")
+
+
+def list_images(root: Path) -> set[str]:
+    """Site-relative paths of every file under images/, e.g. 'images/Foo.jpg'.
+
+    Compared as exact strings so a case mismatch fails here the same way it
+    fails on GitHub Pages, even on a case-insensitive local filesystem.
+    """
+    images_dir = root / "images"
+    return {p.relative_to(root).as_posix() for p in images_dir.rglob("*") if p.is_file()}
+
+
+def check_image(
+    where: str, field: str, value, images: set[str], report: Report
+) -> None:
+    if not isinstance(value, str) or value == "":
+        return
+    if value.startswith("/images/"):
+        if value.lstrip("/") not in images:
+            report.error(where, f"{field}: image not found (case-sensitive): {value}")
+    else:
+        check_link(where, field, value, report)
+
+
+def check_person(
+    path: Path, data: dict, root: Path, images: set[str], report: Report
+) -> None:
+    where = str(path.relative_to(root))
+
+    for field in TEXT_FIELDS:
+        check_text(where, field, data.get(field), report)
+
+    check_link(where, "obituary", data.get("obituary"), report)
+    check_image(where, "mainimage", data.get("mainimage"), images, report)
+
+    for i, item in enumerate(data.get("socialmedialinks") or []):
+        check_text(where, f"socialmedialinks[{i}].sitename", item.get("sitename"), report)
+        check_link(where, f"socialmedialinks[{i}].siteurl", item.get("siteurl"), report)
+
+    for i, item in enumerate(data.get("references") or []):
+        check_text(where, f"references[{i}].title", item.get("title"), report)
+        check_link(where, f"references[{i}].url", item.get("url"), report)
+
+    for i, item in enumerate(data.get("contributions") or []):
+        check_text(where, f"contributions[{i}].title", item.get("title"), report)
+        check_text(where, f"contributions[{i}].description", item.get("description"), report)
+        check_link(where, f"contributions[{i}].url", item.get("url"), report)
+
+    for i, item in enumerate(data.get("gallery") or []):
+        check_text(where, f"gallery[{i}].title", item.get("title"), report)
+        check_text(where, f"gallery[{i}].caption", item.get("caption"), report)
+        check_image(where, f"gallery[{i}].url", item.get("url"), images, report)
+        if not item.get("title"):
+            report.warn(where, f"gallery[{i}]: no title (used as the image alt text)")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parent.parent)
+    args = parser.parse_args()
+    root: Path = args.root
+    report = Report()
+
+    schema = load_json(root / "schema" / "person.schema.json", report)
+    if schema is None:
+        return finish(report)
+    validator_cls = jsonschema.validators.validator_for(schema)
+    validator_cls.check_schema(schema)
+    person_validator = validator_cls(schema)
+    list_validator = jsonschema.Draft202012Validator(PEOPLELIST_SCHEMA)
+
+    people_dir = root / "people"
+    images = list_images(root)
+    files = {p.stem: p for p in sorted(people_dir.glob("*.json")) if p.stem != "_template"}
+
+    peoplelist_path = root / "peoplelist.json"
+    peoplelist = load_json(peoplelist_path, report)
+    listed: dict[str, int] = {}
+    peoplelist_ok = peoplelist is not None and check_schema(
+        peoplelist_path.relative_to(root), peoplelist, list_validator, report
+    )
+    if peoplelist_ok:
+        for entry in peoplelist["people"]:
+            name = entry["filename"]
+            if name == "":
+                continue  # intentional no-link card
+            listed[name] = listed.get(name, 0) + 1
+            if name not in files:
+                report.error("peoplelist.json", f"'{name}' has no people/{name}.json")
+        for name, count in listed.items():
+            if count > 1:
+                report.error("peoplelist.json", f"'{name}' is listed {count} times")
+
+    for stem, path in files.items():
+        where = str(path.relative_to(root))
+        if peoplelist_ok and stem not in listed:
+            report.error(where, "not listed in peoplelist.json")
+        if not FILENAME_RE.match(stem) and stem not in LEGACY_FILENAMES:
+            report.error(where, "filename must be lowercase alphanumeric (a-z, 0-9)")
+        data = load_json(path, report)
+        if data is None:
+            continue
+        if check_schema(path.relative_to(root), data, person_validator, report):
+            check_person(path, data, root, images, report)
+
+    print(f"checked {len(files)} memorial files, {len(listed)} peoplelist entries")
+    return finish(report)
+
+
+def finish(report: Report) -> int:
+    for msg in report.warnings:
+        print(f"WARNING: {msg}")
+    for msg in report.errors:
+        print(f"ERROR: {msg}")
+    print(f"{len(report.errors)} error(s), {len(report.warnings)} warning(s)")
+    return 1 if report.errors else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())