fix CRO module: use correct CKAN resource, add SQL search fallback, integrate CWS API
This commit is contained in:
parent
ee84906f6d
commit
0168261e00
1 changed files with 185 additions and 61 deletions
|
|
@ -1,18 +1,24 @@
|
|||
"""Irish Companies Registration Office (CRO) Module — Company & director lookup via Open Data API."""
|
||||
"""Irish Companies Registration Office (CRO) Module — Company & director lookup via Open Data + CWS API."""
|
||||
import logging
|
||||
from typing import List, Dict, Optional
|
||||
import base64
|
||||
import os
|
||||
from typing import List, Dict
|
||||
from eirescope.core.entity import Entity, EntityType, Investigation
|
||||
from eirescope.modules.base import BaseOSINTModule
|
||||
from eirescope.utils.http_client import OSINTHTTPClient
|
||||
|
||||
logger = logging.getLogger("eirescope.modules.irish_cro")
|
||||
|
||||
# CRO Open Data Portal — CKAN API
|
||||
CRO_API_BASE = "https://opendata.cro.ie/api/3/action"
|
||||
CRO_COMPANY_RESOURCE = "e64eb540-fb97-44c2-b461-766f2babbdf6"
|
||||
# CRO Open Data Portal — CKAN API (individual company records)
|
||||
CRO_CKAN_BASE = "https://opendata.cro.ie/api/3/action"
|
||||
CRO_COMPANY_RESOURCE = "3fef41bc-b8f4-4b10-8434-ce51c29b1bba"
|
||||
|
||||
# Also available: CORE search (public web)
|
||||
CRO_CORE_SEARCH = "https://core.cro.ie/api/company/search"
|
||||
# CRO Company Web Services (CWS) — RESTful API
|
||||
CRO_CWS_BASE = "https://services.cro.ie/cws"
|
||||
|
||||
# Test credentials (limited to 'ryanair', 'google', company_num 83740)
|
||||
CRO_TEST_EMAIL = "[email protected]"
|
||||
CRO_TEST_KEY = "da093a04-c9d7-46d7-9c83-9c9f8630d5e0"
|
||||
|
||||
|
||||
class IrishCROModule(BaseOSINTModule):
|
||||
|
|
@ -27,6 +33,16 @@ class IrishCROModule(BaseOSINTModule):
|
|||
def __init__(self, config=None):
|
||||
super().__init__(config)
|
||||
self.http = OSINTHTTPClient(timeout=15, max_retries=2, rate_limit=0.5)
|
||||
# CWS API credentials (optional — falls back to test creds)
|
||||
self.cws_email = os.environ.get("CRO_EMAIL", "")
|
||||
self.cws_key = os.environ.get("CRO_API_KEY", "")
|
||||
|
||||
def _get_cws_auth_header(self) -> Dict[str, str]:
|
||||
"""Build Basic Auth header for CWS API."""
|
||||
email = self.cws_email or CRO_TEST_EMAIL
|
||||
key = self.cws_key or CRO_TEST_KEY
|
||||
token = base64.b64encode(f"{email}:{key}".encode()).decode()
|
||||
return {"Authorization": f"Basic {token}", "Accept": "application/json"}
|
||||
|
||||
def execute(self, entity: Entity, investigation: Investigation) -> List[Entity]:
|
||||
"""Search CRO for company/person data."""
|
||||
|
|
@ -38,82 +54,145 @@ class IrishCROModule(BaseOSINTModule):
|
|||
if entity.entity_type == EntityType.DOMAIN:
|
||||
query = query.split(".")[0]
|
||||
|
||||
# 1. Search CRO Open Data (CKAN API)
|
||||
results = self._search_ckan(query)
|
||||
if results:
|
||||
entity.metadata["cro_results_count"] = len(results)
|
||||
for company in results[:10]: # Top 10 matches
|
||||
# 1. Search CRO Open Data (CKAN API — full company register)
|
||||
ckan_results = self._search_ckan(query)
|
||||
if ckan_results:
|
||||
entity.metadata["cro_ckan_results"] = len(ckan_results)
|
||||
for company in ckan_results[:10]:
|
||||
comp_name = (
|
||||
company.get("company_name")
|
||||
or company.get("Company Name")
|
||||
or company.get("company_name_english")
|
||||
or "Unknown"
|
||||
)
|
||||
comp_num = str(
|
||||
company.get("company_num")
|
||||
or company.get("Company Number")
|
||||
or company.get("company_number")
|
||||
or ""
|
||||
)
|
||||
company_entity = Entity(
|
||||
entity_type=EntityType.COMPANY,
|
||||
value=company.get("company_name", "Unknown"),
|
||||
value=comp_name,
|
||||
source_module=self.name,
|
||||
confidence=0.85,
|
||||
metadata={
|
||||
"company_number": company.get("company_num", ""),
|
||||
"company_name": company.get("company_name", ""),
|
||||
"company_status": company.get("company_status_desc", ""),
|
||||
"company_type": company.get("company_type_desc", ""),
|
||||
"registered_address": company.get("company_addr_1", ""),
|
||||
"address_2": company.get("company_addr_2", ""),
|
||||
"address_3": company.get("company_addr_3", ""),
|
||||
"address_4": company.get("company_addr_4", ""),
|
||||
"eircode": company.get("company_addr_eircode", ""),
|
||||
"company_number": comp_num,
|
||||
"company_name": comp_name,
|
||||
"company_status": company.get("company_status", company.get("company_status_desc", "")),
|
||||
"company_type": company.get("company_type_desc", company.get("company_type", "")),
|
||||
"registered_address": company.get("company_addr_1", company.get("company_address_1", "")),
|
||||
"address_2": company.get("company_addr_2", company.get("company_address_2", "")),
|
||||
"address_3": company.get("company_addr_3", company.get("company_address_3", "")),
|
||||
"address_4": company.get("company_addr_4", company.get("company_address_4", "")),
|
||||
"county": company.get("County", company.get("county", "")),
|
||||
"registration_date": company.get("company_reg_date", ""),
|
||||
"last_annual_return": company.get("last_arr_date", ""),
|
||||
"last_accounts_date": company.get("last_accounts_date", ""),
|
||||
"place_of_business": company.get("place_of_business", ""),
|
||||
"source": "CRO Open Data",
|
||||
"cro_url": f"https://core.cro.ie/company/{company.get('company_num', '')}",
|
||||
"dissolved_date": company.get("comp_dissolved_date", ""),
|
||||
"source": "CRO Open Data (CKAN)",
|
||||
"cro_url": f"https://core.cro.ie/company/{comp_num}" if comp_num else "",
|
||||
},
|
||||
)
|
||||
added = investigation.add_entity(company_entity)
|
||||
investigation.add_relationship(
|
||||
source_id=entity.id,
|
||||
target_id=added.id,
|
||||
rel_type="search_found_company",
|
||||
rel_type="cro_company_record",
|
||||
confidence=0.85,
|
||||
evidence={"query": query, "company_number": company.get("company_num", "")},
|
||||
evidence={"query": query, "company_number": comp_num},
|
||||
)
|
||||
found_entities.append(added)
|
||||
|
||||
# 2. Try CORE search as well (web-based)
|
||||
core_results = self._search_core(query)
|
||||
if core_results:
|
||||
for company in core_results[:5]:
|
||||
name = company.get("companyName", "")
|
||||
if not any(e.value == name for e in found_entities):
|
||||
# 2. Also try CKAN SQL search for more flexible matching
|
||||
if not ckan_results:
|
||||
sql_results = self._search_ckan_sql(query)
|
||||
if sql_results:
|
||||
entity.metadata["cro_sql_results"] = len(sql_results)
|
||||
for company in sql_results[:10]:
|
||||
comp_name = (
|
||||
company.get("company_name")
|
||||
or company.get("Company Name")
|
||||
or "Unknown"
|
||||
)
|
||||
# Skip if already found
|
||||
if any(e.value == comp_name for e in found_entities):
|
||||
continue
|
||||
comp_num = str(company.get("company_num", company.get("Company Number", "")))
|
||||
company_entity = Entity(
|
||||
entity_type=EntityType.COMPANY,
|
||||
value=name,
|
||||
value=comp_name,
|
||||
source_module=self.name,
|
||||
confidence=0.8,
|
||||
confidence=0.80,
|
||||
metadata={
|
||||
"company_number": company.get("companyNumber", ""),
|
||||
"company_name": name,
|
||||
"company_status": company.get("companyStatusDesc", ""),
|
||||
"company_type": company.get("companyTypeDesc", ""),
|
||||
"registered_address": company.get("companyAddress", ""),
|
||||
"source": "CRO CORE",
|
||||
"cro_url": f"https://core.cro.ie/company/{company.get('companyNumber', '')}",
|
||||
"company_number": comp_num,
|
||||
"company_name": comp_name,
|
||||
"company_status": company.get("company_status", ""),
|
||||
"county": company.get("County", ""),
|
||||
"source": "CRO Open Data (SQL)",
|
||||
"cro_url": f"https://core.cro.ie/company/{comp_num}" if comp_num else "",
|
||||
},
|
||||
)
|
||||
added = investigation.add_entity(company_entity)
|
||||
investigation.add_relationship(
|
||||
source_id=entity.id,
|
||||
target_id=added.id,
|
||||
rel_type="search_found_company",
|
||||
confidence=0.8,
|
||||
rel_type="cro_company_record",
|
||||
confidence=0.80,
|
||||
)
|
||||
found_entities.append(added)
|
||||
|
||||
# 3. Try CWS API (services.cro.ie) — requires API key or test mode
|
||||
cws_results = self._search_cws(query)
|
||||
if cws_results:
|
||||
entity.metadata["cro_cws_results"] = len(cws_results)
|
||||
for company in cws_results[:5]:
|
||||
name = company.get("company_name", "")
|
||||
if not name or any(e.value == name for e in found_entities):
|
||||
continue
|
||||
comp_num = str(company.get("company_num", ""))
|
||||
company_entity = Entity(
|
||||
entity_type=EntityType.COMPANY,
|
||||
value=name,
|
||||
source_module=self.name,
|
||||
confidence=0.90,
|
||||
metadata={
|
||||
"company_number": comp_num,
|
||||
"company_name": name,
|
||||
"company_status": company.get("company_status_desc", company.get("status", "")),
|
||||
"company_type": company.get("company_type_desc", company.get("type", "")),
|
||||
"registered_address": company.get("company_addr_1", ""),
|
||||
"source": "CRO CWS API",
|
||||
"cro_url": f"https://core.cro.ie/company/{comp_num}" if comp_num else "",
|
||||
},
|
||||
)
|
||||
added = investigation.add_entity(company_entity)
|
||||
investigation.add_relationship(
|
||||
source_id=entity.id,
|
||||
target_id=added.id,
|
||||
rel_type="cro_company_record",
|
||||
confidence=0.90,
|
||||
)
|
||||
found_entities.append(added)
|
||||
|
||||
if not found_entities:
|
||||
entity.metadata["cro_note"] = (
|
||||
"No results found. The CKAN open data may use a different resource format. "
|
||||
"Try searching directly at https://core.cro.ie or set CRO_EMAIL and CRO_API_KEY "
|
||||
"environment variables for full CWS API access."
|
||||
)
|
||||
logger.warning(f"CRO: no results for '{query}' across all sources")
|
||||
|
||||
logger.info(f"CRO lookup complete: {len(found_entities)} companies found")
|
||||
return found_entities
|
||||
|
||||
# ── CKAN full-text search ──────────────────────────────────────────
|
||||
|
||||
def _search_ckan(self, query: str) -> List[Dict]:
|
||||
"""Search CRO Open Data Portal via CKAN datastore API."""
|
||||
"""Search CRO Open Data Portal via CKAN datastore full-text search."""
|
||||
try:
|
||||
resp = self.http.get(
|
||||
f"{CRO_API_BASE}/datastore_search",
|
||||
f"{CRO_CKAN_BASE}/datastore_search",
|
||||
params={
|
||||
"resource_id": CRO_COMPANY_RESOURCE,
|
||||
"q": query,
|
||||
|
|
@ -123,28 +202,73 @@ class IrishCROModule(BaseOSINTModule):
|
|||
if resp and resp.status_code == 200:
|
||||
data = resp.json()
|
||||
if data.get("success"):
|
||||
return data.get("result", {}).get("records", [])
|
||||
records = data.get("result", {}).get("records", [])
|
||||
if records:
|
||||
logger.info(f"CKAN search returned {len(records)} records")
|
||||
return records
|
||||
else:
|
||||
logger.info("CKAN search returned 0 records")
|
||||
elif resp:
|
||||
logger.warning(f"CKAN returned status {resp.status_code}")
|
||||
except Exception as e:
|
||||
logger.debug(f"CKAN search failed: {e}")
|
||||
logger.warning(f"CKAN search failed: {e}")
|
||||
return []
|
||||
|
||||
def _search_core(self, query: str) -> List[Dict]:
|
||||
"""Search CRO CORE website API."""
|
||||
def _search_ckan_sql(self, query: str) -> List[Dict]:
|
||||
"""Search CKAN using SQL API for flexible LIKE matching."""
|
||||
try:
|
||||
# Escape single quotes in query
|
||||
safe_q = query.replace("'", "''")
|
||||
sql = (
|
||||
f'SELECT * FROM "{CRO_COMPANY_RESOURCE}" '
|
||||
f"WHERE \"company_name\" ILIKE '%{safe_q}%' "
|
||||
f"LIMIT 15"
|
||||
)
|
||||
resp = self.http.get(
|
||||
"https://services.cro.ie/cw/company",
|
||||
params={
|
||||
"company_name": query,
|
||||
"skip": 0,
|
||||
"take": 10,
|
||||
},
|
||||
headers={"Accept": "application/json"},
|
||||
f"{CRO_CKAN_BASE}/datastore_search_sql",
|
||||
params={"sql": sql},
|
||||
)
|
||||
if resp and resp.status_code == 200:
|
||||
data = resp.json()
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
return data.get("companies", data.get("results", []))
|
||||
if data.get("success"):
|
||||
records = data.get("result", {}).get("records", [])
|
||||
if records:
|
||||
logger.info(f"CKAN SQL search returned {len(records)} records")
|
||||
return records
|
||||
elif resp:
|
||||
logger.warning(f"CKAN SQL returned status {resp.status_code}")
|
||||
except Exception as e:
|
||||
logger.debug(f"CORE search failed: {e}")
|
||||
logger.warning(f"CKAN SQL search failed: {e}")
|
||||
return []
|
||||
|
||||
# ── CWS REST API ───────────────────────────────────────────────────
|
||||
|
||||
def _search_cws(self, query: str) -> List[Dict]:
|
||||
"""Search CRO Company Web Services (CWS) REST API."""
|
||||
try:
|
||||
resp = self.http.get(
|
||||
f"{CRO_CWS_BASE}/companies",
|
||||
params={
|
||||
"company_name": query,
|
||||
"company_bus_ind": "C",
|
||||
"skip": 0,
|
||||
"max": 10,
|
||||
"htmlEnc": "false",
|
||||
},
|
||||
headers=self._get_cws_auth_header(),
|
||||
)
|
||||
if resp and resp.status_code == 200:
|
||||
data = resp.json()
|
||||
# CWS returns a list or an object with companies
|
||||
if isinstance(data, list):
|
||||
logger.info(f"CWS returned {len(data)} companies")
|
||||
return data
|
||||
companies = data.get("companies", data.get("results", []))
|
||||
if companies:
|
||||
logger.info(f"CWS returned {len(companies)} companies")
|
||||
return companies
|
||||
elif resp:
|
||||
logger.warning(f"CWS returned status {resp.status_code}: {resp.text[:200]}")
|
||||
except Exception as e:
|
||||
logger.warning(f"CWS search failed: {e}")
|
||||
return []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue