34 lines
1.0 KiB
Python
Executable File
34 lines
1.0 KiB
Python
Executable File
from bs4 import BeautifulSoup
|
|
import requests
|
|
import threading
|
|
import queue
|
|
|
|
|
|
def get_all_mods(letter, q: queue.Queue):
|
|
# Step through the alphabet, and query each page
|
|
url = f"https://warframe.fandom.com/wiki/Category:Mods?from={letter}"
|
|
response = requests.get(url)
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
# Find the div element with the specified class
|
|
div_element = soup.find("div", {"class": "category-page__members"}).find_all("li")
|
|
|
|
# add them all to the main list
|
|
q.put([li.text.strip() for li in div_element if "Category" not in li.text])
|
|
|
|
|
|
threads = []
|
|
q = queue.Queue(maxsize=0)
|
|
for letter in [chr(i) for i in range(ord("a"), ord("z") + 1)]:
|
|
t = threading.Thread(target=get_all_mods, args=(letter, q))
|
|
t.start()
|
|
threads.append(t)
|
|
|
|
# Block until all threads / queues are done working
|
|
q.join()
|
|
for x in threads:
|
|
x.join()
|
|
|
|
# Flatten the list of lists, list(q.queue) is how you unwrap the data from the queue
|
|
print(sorted([item for sublist in list(q.queue) for item in sublist]))
|