from bs4 import BeautifulSoup
import requests
import threading
import queue


def get_all_mods(letter, q: queue.Queue):
    # Step through the alphabet, and query each page
    url = f"https://warframe.fandom.com/wiki/Category:Mods?from={letter}"
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")

    # Find the div element with the specified class
    div_element = soup.find("div", {"class": "category-page__members"}).find_all("li")

    # add them all to the main list
    q.put([li.text.strip() for li in div_element if "Category" not in li.text])


threads = []
q = queue.Queue(maxsize=0)
for letter in [chr(i) for i in range(ord("a"), ord("z") + 1)]:
    t = threading.Thread(target=get_all_mods, args=(letter, q))
    t.start()
    threads.append(t)

# Block until all threads / queues are done working
q.join()
for x in threads:
    x.join()

# Flatten the list of lists, list(q.queue) is how you unwrap the data from the queue
print(sorted([item for sublist in list(q.queue) for item in sublist]))