#!/usr/bin/env python from flask import Flask, render_template, request, redirect import requests import html import re from bs4 import BeautifulSoup from urllib.parse import quote, unquote def scrape(url): data = requests.get(url) our_path = re.sub(r".*://.*/", "/", request.url) path = re.sub(r".*://.*/", "/", data.url) if our_path != path and \ quote(unquote(re.sub("[?&=]", "", our_path))) != re.sub("[?&=]", "", path): # this is bad ^ return f"REDIRECT {path}" ret = [] soup = BeautifulSoup(data.text, "html.parser") defs = [(div, div.get('data-defid')) for div in soup.find_all("div") if div.get('data-defid')] try: thumbs_data = { str(entry['defid']): entry for entry in requests.get( 'https://api.urbandictionary.com/v0/uncacheable?ids=' + ','.join(defid for (_, defid) in defs) ).json()['thumbs'] } except: thumbs_data = {} for (definition, defid) in defs: word = definition.select("div div h1 a, div div h2 a")[0].text meaning = definition.find(attrs={"class" : ["break-words meaning mb-4"]}).decode_contents() example = definition.find(attrs={"class" : ["break-words example italic mb-4"]}).decode_contents() contributor = definition.find(attrs={"class" : ["contributor font-bold"]}) thumbs_up = thumbs_data.get(defid, {}).get('up') thumbs_down = thumbs_data.get(defid, {}).get('down') ret.append([defid, word, meaning, example, contributor, thumbs_up, thumbs_down]) pages = soup.find(attrs={"class" : ["pagination text-xl text-center"]}) if pages == None: pages = "" return (ret, pages) app = Flask(__name__, template_folder="templates", static_folder="static") @app.route('/', defaults={'path': ''}) @app.route('/') def catch_all(path): scraped = scrape(f"https://urbandictionary.com/{re.sub(r'.*://.*/', '/', request.url)}") if type(scraped) == str and scraped.startswith("REDIRECT"): return redirect(scraped.replace("REDIRECT ", ""), 302) return render_template('index.html', data=scraped) if __name__ == '__main__': from waitress import serve serve(app, host="0.0.0.0", port=8080)