from bs4 import BeautifulSoup baseurl = "https://ncatlab.org" about = { "website": baseurl + "/nlab/show/HomePage", "use_official_api": False, "require_api_key": False, "results": "html" } # basic search def request(query, params): params["url"] = baseurl + f"/nlab/search?query={query}" return params def response(resp): res = [] soup = BeautifulSoup(resp.content) for entry in soup.select("#Content ul li a"): url = baseurl + entry["href"] title = entry.string res_dict = { "url": url, "title": title, } res.append(res_dict) return res