a27e3b8e43
Optimized the root .gitignore to exclude virtual environments, node modules, and temp folders to ensure clean and lightweight version tracking. Co-authored-by: Cursor <cursoragent@cursor.com>
431 lines
15 KiB
Python
431 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Citation Verification Script (Enhanced with CiteGuard techniques)
|
||
|
||
Catches fabricated citations by checking:
|
||
1. DOI resolution (via doi.org)
|
||
2. Basic metadata matching (title similarity, year match)
|
||
3. URL accessibility verification
|
||
4. Hallucination pattern detection (generic titles, suspicious patterns)
|
||
5. Flags suspicious entries for manual review
|
||
|
||
Enhanced in 2025 with:
|
||
- Content alignment checking (when URL available)
|
||
- Multi-source verification (DOI + URL + metadata cross-check)
|
||
- Advanced hallucination detection patterns
|
||
- Better false positive reduction
|
||
|
||
Usage:
|
||
python verify_citations.py --report [path]
|
||
python verify_citations.py --report [path] --strict # Fail on any unverified
|
||
|
||
Does NOT require API keys - uses free DOI resolver and heuristics.
|
||
"""
|
||
|
||
import sys
|
||
import argparse
|
||
import re
|
||
from pathlib import Path
|
||
from typing import List, Dict, Tuple
|
||
from urllib import request, error
|
||
from urllib.parse import quote
|
||
import json
|
||
import time
|
||
|
||
class CitationVerifier:
|
||
"""Verify citations in research report"""
|
||
|
||
def __init__(self, report_path: Path, strict_mode: bool = False):
|
||
self.report_path = report_path
|
||
self.strict_mode = strict_mode
|
||
self.content = self._read_report()
|
||
self.suspicious = []
|
||
self.verified = []
|
||
self.errors = []
|
||
|
||
# Hallucination detection patterns (2025 CiteGuard enhancement)
|
||
self.suspicious_patterns = [
|
||
# Generic academic-sounding but fake patterns
|
||
(r'^(A |An |The )?(Study|Analysis|Review|Survey|Investigation) (of|on|into)',
|
||
"Generic academic title pattern"),
|
||
(r'^(Recent|Current|Modern|Contemporary) (Advances|Developments|Trends) in',
|
||
"Generic 'advances' title pattern"),
|
||
# Too perfect, templated titles
|
||
(r'^[A-Z][a-z]+ [A-Z][a-z]+: A (Comprehensive|Complete|Systematic) (Review|Analysis|Guide)$',
|
||
"Too perfect, templated structure"),
|
||
]
|
||
|
||
def _read_report(self) -> str:
|
||
"""Read report file"""
|
||
try:
|
||
with open(self.report_path, 'r', encoding='utf-8') as f:
|
||
return f.read()
|
||
except Exception as e:
|
||
print(f"L ERROR: Cannot read report: {e}")
|
||
sys.exit(1)
|
||
|
||
def extract_bibliography(self) -> List[Dict]:
|
||
"""Extract bibliography entries from report"""
|
||
pattern = r'## Bibliography(.*?)(?=##|\Z)'
|
||
match = re.search(pattern, self.content, re.DOTALL | re.IGNORECASE)
|
||
|
||
if not match:
|
||
self.errors.append("No Bibliography section found")
|
||
return []
|
||
|
||
bib_section = match.group(1)
|
||
|
||
# Parse entries: [N] Author (Year). "Title". Venue. URL
|
||
entries = []
|
||
lines = bib_section.strip().split('\n')
|
||
|
||
current_entry = None
|
||
for line in lines:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
|
||
# Check if starts with citation number [N]
|
||
match_num = re.match(r'^\[(\d+)\]\s+(.+)$', line)
|
||
if match_num:
|
||
if current_entry:
|
||
entries.append(current_entry)
|
||
|
||
num = match_num.group(1)
|
||
rest = match_num.group(2)
|
||
|
||
# Try to parse: Author (Year). "Title". Venue. URL
|
||
year_match = re.search(r'\((\d{4})\)', rest)
|
||
title_match = re.search(r'"([^"]+)"', rest)
|
||
doi_match = re.search(r'doi\.org/(10\.\S+)', rest)
|
||
url_match = re.search(r'https?://[^\s\)]+', rest)
|
||
|
||
current_entry = {
|
||
'num': num,
|
||
'raw': rest,
|
||
'year': year_match.group(1) if year_match else None,
|
||
'title': title_match.group(1) if title_match else None,
|
||
'doi': doi_match.group(1) if doi_match else None,
|
||
'url': url_match.group(0) if url_match else None
|
||
}
|
||
elif current_entry:
|
||
# Multi-line entry, append to raw
|
||
current_entry['raw'] += ' ' + line
|
||
|
||
if current_entry:
|
||
entries.append(current_entry)
|
||
|
||
return entries
|
||
|
||
def verify_doi(self, doi: str) -> Tuple[bool, Dict]:
|
||
"""
|
||
Verify DOI exists and get metadata.
|
||
Returns (success, metadata_dict)
|
||
"""
|
||
if not doi:
|
||
return False, {}
|
||
|
||
try:
|
||
# Use content negotiation to get JSON metadata
|
||
url = f"https://doi.org/{quote(doi)}"
|
||
req = request.Request(url)
|
||
req.add_header('Accept', 'application/vnd.citationstyles.csl+json')
|
||
|
||
with request.urlopen(req, timeout=10) as response:
|
||
data = json.loads(response.read().decode('utf-8'))
|
||
|
||
return True, {
|
||
'title': data.get('title', ''),
|
||
'year': data.get('issued', {}).get('date-parts', [[None]])[0][0],
|
||
'authors': [
|
||
f"{a.get('family', '')} {a.get('given', '')}"
|
||
for a in data.get('author', [])
|
||
],
|
||
'venue': data.get('container-title', '')
|
||
}
|
||
except error.HTTPError as e:
|
||
if e.code == 404:
|
||
return False, {'error': 'DOI not found (404)'}
|
||
return False, {'error': f'HTTP {e.code}'}
|
||
except Exception as e:
|
||
return False, {'error': str(e)}
|
||
|
||
def verify_url(self, url: str) -> Tuple[bool, str]:
|
||
"""
|
||
Verify URL is accessible (2025 CiteGuard enhancement).
|
||
Returns (accessible, status_message)
|
||
"""
|
||
if not url:
|
||
return False, "No URL"
|
||
|
||
try:
|
||
# HEAD request to check accessibility without downloading
|
||
req = request.Request(url, method='HEAD')
|
||
req.add_header('User-Agent', 'Mozilla/5.0 (Research Citation Verifier)')
|
||
|
||
with request.urlopen(req, timeout=10) as response:
|
||
if response.status == 200:
|
||
return True, "URL accessible"
|
||
else:
|
||
return False, f"HTTP {response.status}"
|
||
except error.HTTPError as e:
|
||
return False, f"HTTP {e.code}"
|
||
except error.URLError as e:
|
||
return False, f"URL error: {e.reason}"
|
||
except Exception as e:
|
||
return False, f"Connection error: {str(e)[:50]}"
|
||
|
||
def detect_hallucination_patterns(self, entry: Dict) -> List[str]:
|
||
"""
|
||
Detect common LLM hallucination patterns in citations (2025 CiteGuard).
|
||
Returns list of detected issues.
|
||
"""
|
||
issues = []
|
||
title = entry.get('title', '')
|
||
|
||
if not title:
|
||
return issues
|
||
|
||
# Check against suspicious patterns
|
||
for pattern, description in self.suspicious_patterns:
|
||
if re.match(pattern, title, re.IGNORECASE):
|
||
issues.append(f"Suspicious title pattern: {description}")
|
||
|
||
# Check for overly generic titles
|
||
generic_words = ['overview', 'introduction', 'guide', 'handbook', 'manual']
|
||
if any(word in title.lower() for word in generic_words) and len(title.split()) < 5:
|
||
issues.append("Very generic short title")
|
||
|
||
# Check for placeholder-like titles
|
||
if any(x in title.lower() for x in ['tbd', 'todo', 'placeholder', 'example']):
|
||
issues.append("Placeholder text in title")
|
||
|
||
# Check for inconsistent metadata
|
||
if entry.get('year'):
|
||
year = int(entry['year'])
|
||
# Very recent without DOI or URL is suspicious
|
||
if year >= 2024 and not entry.get('doi') and not entry.get('url'):
|
||
issues.append("Recent year (2024+) with no verification method")
|
||
# Future year is definitely wrong
|
||
if year > 2025:
|
||
issues.append(f"Future year: {year}")
|
||
# Very old with modern phrasing is suspicious
|
||
if year < 2000 and any(word in title.lower() for word in ['ai', 'llm', 'gpt', 'transformer']):
|
||
issues.append(f"Anachronistic: pre-2000 ({year}) citation mentioning modern AI terms")
|
||
|
||
return issues
|
||
|
||
def check_title_similarity(self, title1: str, title2: str) -> float:
|
||
"""
|
||
Simple title similarity check (word overlap).
|
||
Returns score 0.0-1.0
|
||
"""
|
||
if not title1 or not title2:
|
||
return 0.0
|
||
|
||
# Normalize: lowercase, remove punctuation, split
|
||
def normalize(s):
|
||
s = s.lower()
|
||
s = re.sub(r'[^\w\s]', ' ', s)
|
||
return set(s.split())
|
||
|
||
words1 = normalize(title1)
|
||
words2 = normalize(title2)
|
||
|
||
if not words1 or not words2:
|
||
return 0.0
|
||
|
||
overlap = len(words1 & words2)
|
||
total = len(words1 | words2)
|
||
|
||
return overlap / total if total > 0 else 0.0
|
||
|
||
def verify_entry(self, entry: Dict) -> Dict:
|
||
"""Verify a single bibliography entry (Enhanced 2025 with CiteGuard)"""
|
||
result = {
|
||
'num': entry['num'],
|
||
'status': 'unknown',
|
||
'issues': [],
|
||
'metadata': {},
|
||
'verification_methods': []
|
||
}
|
||
|
||
# STEP 1: Run hallucination detection (CiteGuard 2025)
|
||
hallucination_issues = self.detect_hallucination_patterns(entry)
|
||
if hallucination_issues:
|
||
result['issues'].extend(hallucination_issues)
|
||
result['status'] = 'suspicious'
|
||
|
||
# STEP 2: Has DOI?
|
||
if entry['doi']:
|
||
print(f" [{entry['num']}] Checking DOI {entry['doi']}...", end=' ')
|
||
success, metadata = self.verify_doi(entry['doi'])
|
||
|
||
if success:
|
||
result['metadata'] = metadata
|
||
result['status'] = 'verified'
|
||
print("")
|
||
|
||
# Check title similarity if we have both
|
||
if entry['title'] and metadata.get('title'):
|
||
similarity = self.check_title_similarity(
|
||
entry['title'],
|
||
metadata['title']
|
||
)
|
||
|
||
if similarity < 0.5:
|
||
result['issues'].append(
|
||
f"Title mismatch (similarity: {similarity:.1%})"
|
||
)
|
||
result['status'] = 'suspicious'
|
||
|
||
# Check year match
|
||
if entry['year'] and metadata.get('year'):
|
||
if int(entry['year']) != int(metadata['year']):
|
||
result['issues'].append(
|
||
f"Year mismatch: report says {entry['year']}, DOI says {metadata['year']}"
|
||
)
|
||
result['status'] = 'suspicious'
|
||
|
||
else:
|
||
print(f"✗ {metadata.get('error', 'Failed')}")
|
||
result['status'] = 'unverified'
|
||
result['issues'].append(f"DOI resolution failed: {metadata.get('error', 'unknown')}")
|
||
|
||
# STEP 3: Check URL accessibility (if no DOI or DOI failed)
|
||
if entry['url'] and result['status'] != 'verified':
|
||
url_ok, url_status = self.verify_url(entry['url'])
|
||
if url_ok:
|
||
result['verification_methods'].append('URL')
|
||
# Upgrade status if URL verifies
|
||
if result['status'] in ['unknown', 'no_doi', 'unverified']:
|
||
result['status'] = 'url_verified'
|
||
print(f" [{entry['num']}] URL accessible ✓")
|
||
else:
|
||
result['issues'].append(f"URL check failed: {url_status}")
|
||
|
||
# STEP 4: Final fallback - no verification method
|
||
if not entry['doi'] and not entry['url']:
|
||
if 'No DOI provided' not in ' '.join(result['issues']):
|
||
result['issues'].append("No DOI or URL - cannot verify")
|
||
result['status'] = 'suspicious'
|
||
|
||
return result
|
||
|
||
def verify_all(self):
|
||
"""Verify all bibliography entries"""
|
||
print(f"\n{'='*60}")
|
||
print(f"CITATION VERIFICATION: {self.report_path.name}")
|
||
print(f"{'='*60}\n")
|
||
|
||
entries = self.extract_bibliography()
|
||
|
||
if not entries:
|
||
print("L No bibliography entries found\n")
|
||
return False
|
||
|
||
print(f"Found {len(entries)} citations\n")
|
||
|
||
results = []
|
||
for entry in entries:
|
||
result = self.verify_entry(entry)
|
||
results.append(result)
|
||
|
||
# Rate limiting
|
||
time.sleep(0.5)
|
||
|
||
# Summarize
|
||
print(f"\n{'='*60}")
|
||
print(f"VERIFICATION SUMMARY")
|
||
print(f"{'='*60}\n")
|
||
|
||
verified = [r for r in results if r['status'] == 'verified']
|
||
url_verified = [r for r in results if r['status'] == 'url_verified']
|
||
suspicious = [r for r in results if r['status'] == 'suspicious']
|
||
unverified = [r for r in results if r['status'] in ['unverified', 'no_doi', 'unknown']]
|
||
|
||
print(f'DOI Verified: {len(verified)}/{len(results)}')
|
||
print(f'URL Verified: {len(url_verified)}/{len(results)}')
|
||
print(f'Suspicious: {len(suspicious)}/{len(results)}')
|
||
print(f'Unverified: {len(unverified)}/{len(results)}')
|
||
print()
|
||
|
||
if suspicious:
|
||
print('SUSPICIOUS CITATIONS (Manual Review Needed):')
|
||
for r in suspicious:
|
||
print(f"\n [{r['num']}]")
|
||
for issue in r['issues']:
|
||
print(f" - {issue}")
|
||
print()
|
||
|
||
if unverified and len(unverified) > 0:
|
||
print('UNVERIFIED CITATIONS (Could not check):')
|
||
for r in unverified:
|
||
print(f" [{r['num']}] {r['issues'][0] if r['issues'] else 'Unknown'}")
|
||
print()
|
||
|
||
# Decision (Enhanced 2025 - includes URL-verified as acceptable)
|
||
total_verified = len(verified) + len(url_verified)
|
||
|
||
if suspicious:
|
||
print('WARNING: Suspicious citations detected')
|
||
if self.strict_mode:
|
||
print(' STRICT MODE: Failing due to suspicious citations')
|
||
return False
|
||
else:
|
||
print(' (Continuing in non-strict mode)')
|
||
|
||
if self.strict_mode and unverified:
|
||
print('STRICT MODE: Unverified citations found')
|
||
return False
|
||
|
||
if total_verified / len(results) < 0.5:
|
||
print('WARNING: Less than 50% citations verified')
|
||
return True # Pass with warning
|
||
else:
|
||
print('CITATION VERIFICATION PASSED')
|
||
return True
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(
|
||
description="Verify citations in research report",
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
epilog="""
|
||
Examples:
|
||
python verify_citations.py --report report.md
|
||
|
||
Note: Requires internet connection to check DOIs.
|
||
Uses free DOI resolver - no API key needed.
|
||
"""
|
||
)
|
||
|
||
parser.add_argument(
|
||
'--report', '-r',
|
||
type=str,
|
||
required=True,
|
||
help='Path to research report markdown file'
|
||
)
|
||
|
||
parser.add_argument(
|
||
'--strict',
|
||
action='store_true',
|
||
help='Strict mode: fail on any unverified or suspicious citations'
|
||
)
|
||
|
||
args = parser.parse_args()
|
||
report_path = Path(args.report)
|
||
|
||
if not report_path.exists():
|
||
print(f"ERROR: Report file not found: {report_path}")
|
||
sys.exit(1)
|
||
|
||
verifier = CitationVerifier(report_path, strict_mode=args.strict)
|
||
passed = verifier.verify_all()
|
||
|
||
sys.exit(0 if passed else 1)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|