46 lines
1.3 KiB
Python
Executable File
46 lines
1.3 KiB
Python
Executable File
from bs4 import BeautifulSoup
|
|
import requests
|
|
import threading
|
|
import queue
|
|
|
|
|
|
def get_all_mods(letter, q: queue.Queue):
|
|
all_mods = []
|
|
# Step through the alphabet, and query each page
|
|
url = "https://warframe.fandom.com/wiki/Category:Mods?from=%s" % letter
|
|
response = requests.get(url)
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
# Find the div element with the specified class
|
|
div_element = soup.find("div", {"class": "category-page__members"})
|
|
|
|
# Find all the li elements within the div
|
|
li_elements = div_element.find_all("li")
|
|
|
|
# add them all to the main list
|
|
q.put(
|
|
[
|
|
li.text.strip()
|
|
for li in soup.find("div", {"class": "category-page__members"}).find_all(
|
|
"li"
|
|
)
|
|
if "Category" not in li.text
|
|
]
|
|
)
|
|
|
|
|
|
threads = []
|
|
q = queue.Queue(maxsize=0)
|
|
for letter in [chr(i) for i in range(ord("a"), ord("z") + 1)]:
|
|
t = threading.Thread(target=get_all_mods, args=(letter, q))
|
|
t.start()
|
|
threads.append(t)
|
|
|
|
# Block until all threads / queues are done working
|
|
q.join()
|
|
for x in threads:
|
|
x.join()
|
|
|
|
# Flatten the list of lists, list(q.queue) is how you unwrap the data from the queue
|
|
print(sorted([item for sublist in list(q.queue) for item in sublist]))
|