User:PharyngealImplosive7/IPAlinker.py
Description
[edit]This contains old code for PharyngealBOT task 2. It is NOT the current code and is retained for historical reasons. To see that, go to the github repository. – PharyngealImplosive7 (talk) 00:00, 1 June 2025 (UTC)
Actual code
[edit]import pywikibot
from mwparserfromhell import parse, nodes
import urllib.parse
import re
import unicodedata
from dataclasses import dataclass
from typing import Set, List, Tuple, Optional, Dict, Any
from functools import lru_cache
import yaml
import sys
from pathlib import Path
# Pre-intern common strings
_STRINGS = {s: sys.intern(s) for s in ['ipa', 'separator', 'IPA link']}
@dataclass
class ProcessingStats:
__slots__ = ['changes', 'processed_count', 'modified_count', 'skipped_count']
def __init__(self):
self.changes = 0
self.processed_count = 0
self.modified_count = 0
self.skipped_count = 0
class IPAProcessor:
"""Optimized IPA processor with combined functionality."""
def __init__(self, config_path: str = "ipa_config.yaml"):
self.config_path = config_path
self._load_config()
self._compile_patterns()
self._build_char_maps()
self.stats = ProcessingStats()
print(f"IPA Processor initialized with config: {config_path}")
def _load_config(self):
"""Load and cache configuration."""
config_file = Path(self.config_path)
if not config_file.exists():
raise FileNotFoundError(f"Configuration file not found: {self.config_path}")
with open(config_file, 'r', encoding='utf-8') as f:
c = yaml.safe_load(f)
# Store config values directly as attributes for faster access
self.max_ipa_length = c.get('max_ipa_length', 1)
self.brackets = c.get('brackets', {})
self.brackets_to_remove = frozenset(c.get('brackets_to_remove', []))
self.multi_char_exceptions = frozenset(c.get('multi_char_exceptions', []))
self.vowel_symbols = frozenset(c.get('vowel_symbols', []))
self.tone_symbols = frozenset(c.get('tone_symbols', []))
self.click_symbols = frozenset(c.get('click_symbols', []))
self.diacritics = frozenset(c.get('diacritics', []))
self.non_ipa_diacritics = frozenset(c.get('non_ipa_diacritics', []))
self.allophone_separators = frozenset(c.get('allophone_separators', []))
self.separator_symbols = frozenset(c.get('separator_symbols', []))
self.phonetic_terms = frozenset(term.lower() for term in c.get('phonetic_terms', []))
self.tiebar = sys.intern(c.get('tiebar', '͡'))
self.ejective_marker = sys.intern(c.get('ejective_marker', 'ʼ'))
# Process IPA brackets
ipa_brackets_raw = c.get('ipa_brackets', {})
self.ipa_brackets = {}
for open_b, bracket_data in ipa_brackets_raw.items():
self.ipa_brackets[sys.intern(open_b)] = (
sys.intern(bracket_data[0]),
sys.intern(bracket_data[1])
)
self._superscript_pattern = c.get('superscript_regex_pattern',
r"[\u02B0-\u02FF\u1D00-\u1DBF\u2070-\u209F\u1D2C-\u1D6B]")
def _compile_patterns(self):
"""Compile all regex patterns once."""
self.superscript_regex = re.compile(self._superscript_pattern)
separator_pattern = '|'.join(re.escape(sep) for sep in self.separator_symbols)
self.separator_split_pattern = re.compile(rf'(\s*)({separator_pattern})(\s*)')
self.table_pattern = re.compile(r'(?:^|\n)\s*:{0,4}\s*\{\|.*?\n\|\}', re.MULTILINE | re.DOTALL)
self.word_pattern = re.compile(r'\b\w+\b')
self.space_pattern = re.compile(r'(\s+)')
def _build_char_maps(self):
"""Build efficient character classification maps."""
self.char_types = {}
for char in self.diacritics:
self.char_types[char] = 'diacritic'
for char in self.tone_symbols:
self.char_types[char] = 'tone'
for char in self.vowel_symbols:
self.char_types[char] = 'vowel'
for char in self.click_symbols:
self.char_types[char] = 'click'
for char in self.non_ipa_diacritics:
self.char_types[char] = 'non_ipa'
@lru_cache(maxsize=1000)
def analyze_segment(self, segment: str) -> Dict[str, Any]:
"""Single-pass segment analysis with caching."""
seg_clean = ''.join(c for c in segment if c not in self.brackets_to_remove and not c.isspace())
normalized = unicodedata.normalize('NFD', seg_clean)
has_tone = any(c in self.tone_symbols for c in segment) or segment in self.tone_symbols
has_ejective = self.ejective_marker in segment
has_diacritic = any(d in normalized for d in self.diacritics) or bool(self.superscript_regex.search(seg_clean))
has_non_ipa = any(c in self.non_ipa_diacritics for c in normalized)
has_tiebar = self.tiebar in normalized
has_click = any(click in normalized for click in self.click_symbols)
# Count vowels for diphthong detection
clean_for_vowels = ''.join(c for c in seg_clean
if c not in self.diacritics and not self.superscript_regex.match(c))
vowel_count = sum(1 for c in clean_for_vowels.lower() if c in self.vowel_symbols)
is_diphthong = vowel_count >= 2
# Check if ejective consonant
is_ejective_consonant = False
if has_ejective:
base = segment.replace(self.ejective_marker, '')
is_ejective_consonant = (len(base.strip()) == 1 and
base.lower() not in self.vowel_symbols and base.strip())
# Determine if special segment
is_special = (seg_clean.lower() in self.multi_char_exceptions or
has_diacritic or has_tiebar or has_click or
is_ejective_consonant or len(seg_clean.strip()) == 1)
# Should link determination
should_link = (not has_tone and not is_diphthong and not has_non_ipa and
seg_clean.strip() and (is_ejective_consonant or is_special))
return {
'has_tone': has_tone,
'is_diphthong': is_diphthong,
'has_non_ipa': has_non_ipa,
'is_ejective_consonant': is_ejective_consonant,
'should_link': should_link,
'clean': seg_clean
}
def detect_ipa_brackets(self, segment: str) -> Tuple[Optional[str], Optional[str], str, Optional[str]]:
"""Fast bracket detection."""
segment = segment.strip()
# Check IPA-specific brackets first
for open_b, (close_b, template_name) in self.ipa_brackets.items():
if segment.startswith(open_b) and segment.endswith(close_b):
content = segment[len(open_b):-len(close_b)].strip()
return open_b, close_b, content, template_name
# Fallback to regular brackets
for open_b, close_b in self.brackets.items():
if segment.startswith(open_b) and segment.endswith(close_b):
content = segment[len(open_b):-len(close_b)].strip()
return open_b, close_b, content, None
return None, None, segment, None
def tokenize_content(self, content: str) -> List:
"""Fast tokenization with single regex split."""
result = []
parts = self.separator_split_pattern.split(content)
for part in parts:
if not part:
continue
if part.strip() in self.separator_symbols:
result.append((_STRINGS['separator'], part.strip(), ''))
elif part.isspace():
continue
else:
if part.strip():
space_parts = self.space_pattern.split(part)
for space_part in space_parts:
if space_part.strip():
result.append(space_part.strip())
elif space_part.isspace():
result.append(space_part)
return result
def contains_phonetic_terms(self, text: str, min_terms: int = 3) -> Tuple[bool, List[str]]:
"""Fast phonetic term detection."""
words = set(self.word_pattern.findall(text.lower()))
matched = [w for w in words if w in self.phonetic_terms]
return len(matched) >= min_terms, matched[:min_terms] if matched else []
def find_tables(self, text: str) -> List[Tuple[int, int, str]]:
"""Find table boundaries."""
return [(m.start(), m.end(), m.group()) for m in self.table_pattern.finditer(text)]
def is_in_table(self, pos: int, tables: List[Tuple[int, int, str]]) -> Tuple[bool, Optional[str]]:
"""Check if position is in any table."""
for start, end, content in tables:
if start <= pos <= end:
return True, content
return False, None
def process_ipa_template(self, node: nodes.Template, parent_list: List, index: int) -> None:
"""Process IPA template with optimized logic."""
if node.name.strip().lower() != _STRINGS['ipa']:
return
raw_content = str(node.params[0].value).strip()
open_b, close_b, inner_content, template_name = self.detect_ipa_brackets(raw_content)
# Handle IPA brackets with separators
if (open_b and close_b and template_name and
any(sep in inner_content for sep in self.separator_symbols)):
self._process_bracketed_allophones(node, parent_list, index, raw_content,
open_b, close_b, inner_content, template_name)
return
# Handle simple IPA brackets
if open_b and close_b and template_name:
self._process_simple_brackets(node, parent_list, index, raw_content,
open_b, close_b, inner_content, template_name)
return
# Fallback processing
segments = self.tokenize_content(raw_content)
if not any(isinstance(s, str) and not s.isspace() and
self.analyze_segment(s if isinstance(s, str) else s[1])['should_link']
for s in segments):
return
new_nodes = self._create_nodes(segments)
if new_nodes:
parent_list[index:index+1] = new_nodes
self.stats.changes += 1
print(f"Converted IPA template: {raw_content}")
def _process_simple_brackets(self, node, parent_list, index, raw_content,
open_b, close_b, inner_content, template_name):
"""Handle simple IPA brackets."""
if not inner_content.strip():
return
analysis = self.analyze_segment(inner_content)
if analysis['should_link']:
ipa_link = nodes.Template(name=template_name)
ipa_link.add("1", inner_content.strip())
parent_list[index:index+1] = [ipa_link]
self.stats.changes += 1
print(f"Converted: {raw_content} -> {{{{{template_name}|{inner_content.strip()}}}}}")
def _process_bracketed_allophones(self, node, parent_list, index, raw_content,
open_b, close_b, inner_content, template_name):
"""Handle bracketed allophone sets."""
segments = self.tokenize_content(inner_content)
new_nodes = []
for segment in segments:
if isinstance(segment, tuple) and len(segment) == 3:
new_nodes.append(nodes.Text(segment[1]))
if segment[2]:
new_nodes.append(nodes.Text(segment[2]))
continue
if isinstance(segment, str):
if segment.isspace():
new_nodes.append(nodes.Text(segment))
continue
if not segment.strip():
continue
analysis = self.analyze_segment(segment)
if analysis['should_link']:
ipa_link = nodes.Template(name=template_name)
ipa_link.add("1", segment.strip())
new_nodes.append(ipa_link)
else:
new_nodes.append(nodes.Text(segment))
if new_nodes:
parent_list[index:index+1] = new_nodes
self.stats.changes += 1
print(f"Converted allophone template: {raw_content}")
def _create_nodes(self, segments: List) -> List[nodes.Node]:
"""Create processed nodes efficiently."""
new_nodes = []
for segment in segments:
if isinstance(segment, tuple) and len(segment) == 3:
new_nodes.append(nodes.Text(segment[1]))
if segment[2]:
new_nodes.append(nodes.Text(segment[2]))
continue
if isinstance(segment, str) and segment.isspace():
new_nodes.append(nodes.Text(segment))
continue
open_b, close_b, core = self.detect_ipa_brackets(segment)
if not core.strip():
continue
analysis = self.analyze_segment(core)
if not analysis['should_link']:
return [] # Preserve original if any segment shouldn't be linked
if open_b:
new_nodes.append(nodes.Text(open_b))
ipa_link = nodes.Template(name=_STRINGS['IPA link'])
ipa_link.add("1", core)
new_nodes.append(ipa_link)
if close_b:
new_nodes.append(nodes.Text(close_b))
return new_nodes
def process_page(self, page: pywikibot.Page) -> bool:
"""Process single page efficiently."""
print(f"\nProcessing: {page.title()}")
try:
text = page.get()
except pywikibot.exceptions.NoPageError:
print("Page doesn't exist!")
return False
wikicode = parse(text)
self.stats.changes = 0
tables = self.find_tables(text)
self._process_nodes_with_context(wikicode.nodes, 0, tables)
if self.stats.changes:
new_text = str(wikicode)
print(f"\nFound {self.stats.changes} IPA conversion(s)")
pywikibot.showDiff(text, new_text)
if input("Save changes? (y/n): ").lower() == 'y':
page.text = new_text
page.save(summary=f"IPA conversion in phonetic tables ({self.stats.changes} templates)", bot=True)
return True
else:
print("No IPA templates needed conversion")
return False
def _process_nodes_with_context(self, node_list: List[nodes.Node], text_offset: int, tables: List):
"""Process nodes with table context."""
i = 0
current_offset = text_offset
while i < len(node_list):
node = node_list[i]
node_str = str(node)
in_table, table_content = self.is_in_table(current_offset, tables)
if isinstance(node, nodes.Template) and node.name.strip().lower() == _STRINGS['ipa']:
if in_table:
is_relevant, _ = self.contains_phonetic_terms(table_content, 3)
if is_relevant:
self.process_ipa_template(node, node_list, i)
elif isinstance(node, nodes.Tag) and hasattr(node, 'contents') and hasattr(node.contents, 'nodes'):
tag_start_len = len(f"<{node.tag}>")
self._process_nodes_with_context(node.contents.nodes, current_offset + tag_start_len, tables)
current_offset += len(node_str)
i += 1
def process_category(self, category_name: str, depth: int = 0,
max_pages: Optional[int] = None, skip_pages: int = 0) -> Tuple[int, int]:
"""Process category efficiently."""
site = pywikibot.Site('en', 'wikipedia')
cat = pywikibot.Category(site, f"Category:{category_name}")
print(f"\n=== Processing Category: {cat.title()} ===")
all_pages = list(cat.articles(recurse=depth))
pages = [page for page in all_pages if page.namespace() == 0]
print(f"Found {len(all_pages)} total pages, {len(pages)} article space pages")
if max_pages:
print(f"Will process up to {max_pages} pages")
if skip_pages:
print(f"Skipping first {skip_pages} pages")
self.stats.processed_count = 0
self.stats.modified_count = 0
self.stats.skipped_count = 0
for i, page in enumerate(pages):
if max_pages and self.stats.processed_count >= max_pages:
break
if self.stats.skipped_count < skip_pages:
self.stats.skipped_count += 1
continue
print(f"\n[{i+1}/{len(pages)}] Processing article: {page.title()}")
try:
if self.process_page(page):
self.stats.modified_count += 1
self.stats.processed_count += 1
except Exception as e:
print(f"Error processing page {page.title()}: {e}")
print(f"\n=== Category Processing Complete ===")
print(f"Processed {self.stats.processed_count} pages")
print(f"Made changes to {self.stats.modified_count} pages")
return self.stats.processed_count, self.stats.modified_count
def reload_config(self):
"""Reload configuration."""
self._load_config()
self._compile_patterns()
self._build_char_maps()
# Clear cache
self.analyze_segment.cache_clear()
print(f"Configuration reloaded from {self.config_path}")
def main():
"""Main entry point."""
# URL decoding workaround
if not hasattr(pywikibot, 'tools'):
pywikibot.tools = type('', (), {})()
pywikibot.tools.chars = type('', (), {})()
pywikibot.tools.chars.url2string = lambda text, encodings=None: urllib.parse.unquote(
text, encoding=(encodings or ['utf-8'])[0]
)
try:
site = pywikibot.Site('en', 'wikipedia')
if site.logged_in():
print(f"Successfully logged in as: {site.username()}")
else:
print("Not logged in. Please check your authentication.")
return
config_path = input("Enter config file path (or press Enter for default 'ipa_config.yaml'): ").strip()
processor = IPAProcessor(config_path or "ipa_config.yaml")
while True:
print("\nOptions:")
print("1. Process a specific page")
print("2. Process a category")
print("3. Reload configuration")
print("4. Exit")
choice = input("Enter your choice (1-4): ").strip()
if choice == '1':
page_title = input("Enter page title: ").strip()
page = pywikibot.Page(site, page_title)
processor.process_page(page)
elif choice == '2':
category_name = input("Enter category name (without 'Category:' prefix): ").strip()
depth = int(input("Enter recursion depth (0 for just this category): ").strip() or "0")
max_pages_str = input("Enter maximum pages (or enter for no limit): ").strip()
max_pages = int(max_pages_str) if max_pages_str else None
skip_pages = int(input("Enter pages to skip (or enter for none): ").strip() or "0")
processor.process_category(category_name, depth, max_pages, skip_pages)
elif choice == '3':
processor.reload_config()
elif choice == '4':
print("Exiting program.")
break
else:
print("Invalid choice. Please enter 1, 2, 3, or 4.")
except KeyboardInterrupt:
print("\nOperation interrupted by user.")
except Exception as e:
print(f"An error occurred: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
main()