User:IngenuityBot/fixpages.py

import requests, re
from getpagemetadata import getpagemetadata

urls = {
    "cbc.ca": "[[CBC News]]",
    "ctvnews.ca": "[[CTV News]]",
    "globalnews.ca": "[[Global News]]",
    "thestar.com": "[[Toronto Star]]",
    "washingtonpost.com": "[[The Washington Post]]",
    "nytimes.com": "[[The New York Times]]",
    "theglobeandmail.com": "[[The Globe and Mail]]",
    "nationalpost.com": "[[National Post]]",
    "apnews.com": "[[Associated Press]]",
    "reuters.com": "[[Reuters]]",
    "bbc.com": "[[BBC News]]",
    "theguardian.com": "[[The Guardian]]",
    "aljazeera.com": "[[Al Jazeera]]",
    "npr.org": "[[NPR]]",
    "nbcnews.com": "[[NBC News]]",
    "usatoday.com": "[[USA Today]]",
    "latimes.com": "[[Los Angeles Times]]",
    "wsj.com": "[[The Wall Street Journal]]",
    "politico.com": "[[Politico]]",
    "bloomberg.com": "[[Bloomberg News]]",
    "axios.com": "[[Axios (website)|Axios]]",
    "businessinsider.com": "[[Business Insider]]",
    "thehill.com": "[[The Hill (newspaper)|The Hill]]",
    "nypost.com": "[[New York Post]]",
    "chicagotribune.com": "[[Chicago Tribune]]",
    "vox.com": "[[Vox (website)|Vox]]",
    "slate.com": "[[Slate (magazine)|Slate]]",
    "theatlantic.com": "[[The Atlantic]]",
    "newyorker.com": "[[The New Yorker]]",
    "time.com": "[[Time (magazine)|Time]]",
    "smh.com.au": "[[The Sydney Morning Herald]]",
    "space.com": "[[Space.com]]",
    "rollingstone.com": "[[Rolling Stone]]",
    "nzherald.co.nz": "[[The New Zealand Herald]]",
    "news.com.au": "[[News.com.au]]",
    "nasa.gov": "[[NASA]]",
    "msnbc.com": "[[MSNBC]]",
    "thejc.com": "[[The Jewish Chronicle]]",
    "theconversation.com": "[[The Conversation (website)|The Conversation]]",
    "hollywoodreporter.com": "[[The Hollywood Reporter]]",
    "gizmodo.com": "[[Gizmodo]]",
    "thediplomat.com": "[[The Diplomat]]",
    "deadline.com": "[[Deadline Hollywood]]",
    "abcnews.go.com": "[[ABC News]]",
    "cnn.com": "[[CNN]]",
    "theverge.com": "[[The Verge]]",
    "theage.com.au": "[[The Age]]",
    "afp.com": "[[Agence France-Presse]]",
    "arstechica.com": "[[Ars Technica]]",
    "theaustralian.com.au": "[[The Australian]]",
    "avclub.com": "[[The A.V. Club]]",
    "buzzfeednews.com": "[[BuzzFeed News]]",
    "csmonitor.com": "[[The Christian Science Monitor]]",
    "cnet.com": "[[CNET]]",
    "telegraph.co.uk": "[[The Daily Telegraph]]",
    "ew.com": "[[Entertainment Weekly]]",
    "forbes.com": "[[Forbes]]",
    "foxnews.com": "[[Fox News]]",
    "ign.com": "[[IGN]]",
    "qz.com": "[[Quartz (publication)|Quartz]]",
    "scientificamerican.com": "[[Scientific American]]",
    "scmp.com": "[[South China Morning Post]]",
    "variety.com": "[[Variety (magazine)|Variety]]",
    "vogue.com": "[[Vogue (magazine)|Vogue]]",
    "vox.com": "[[Vox (website)|Vox]]",
    "wired.com": "[[Wired (magazine)|Wired]]"
}


def get_wikipedia_content(titles):
    endpoint = "https://en.wikipedia.org/w/api.php"
    params = {
        "action": "query",
        "prop": "revisions",
        "rvprop": "content",
        "format": "json",
        "titles": "|".join(titles[:50])
    }
    response = requests.get(endpoint, params=params)
    data = response.json()
    pages = data["query"]["pages"]
    content_dict = {}
    for _, page_data in pages.items():
        try:
            content_dict[page_data["title"]] = page_data["revisions"][0]["*"]
        except:
            pass
    if len(titles) > 50:
        content_dict.update(get_wikipedia_content(titles[50:]))
    return content_dict


def get_wikipedia_pages():
    with open("pages.txt", "r") as f:
        return f.read().split("\n")


def parse_time(timestamp):
    return timestamp.split("T")[0]


def metadata_to_wikitext(metadata):
    if not metadata["title"] or not metadata["url"]:
        return None

    metadata["title"] = metadata["title"].replace('|', '{{!}}')

    args = []

    for key, value in metadata.items():
        if value:
            args.append(f"|{key}={value}")

    argtext = " ".join(args)
    return "{{cite web " + argtext + "}}"


def main():
    pages = get_wikipedia_pages()
    print(f"Fetching content of {len(pages)} pages...")
    content = get_wikipedia_content(pages)
    print("Done.")

    regex = re.compile(r"<ref(?:\s+name=\"?[^>]+\"?)?>\[?(http[^ <\]]+)]?(?: ?{{bare[^}]+?}})?<\/ref>")

    for item in pages:
        if item not in content:
            continue

        matches = re.findall(regex, content[item])

        if not matches:
            continue

        to_replace = []

        for item in matches:
            for url in urls:
                if re.findall("[\.\/]" + url, item):
                    to_replace.append(item)
                    break
        
        if not to_replace:
            continue

        for item in to_replace:
            metadata = getpagemetadata(item)
            metadata["date"] = parse_time(metadata["date"]) if "date" in metadata else None
            wikitext = metadata_to_wikitext(metadata)
            if not wikitext:
                continue
            print(f"Original URL: {item}")
            print(f"\tReplaced with: {wikitext}\n")

            with open("results.txt", "a") as f:
                f.write(f"{item}\n{wikitext}\n\n")


if __name__ == "__main__":
    main()