Here’s a complete working example that puts it all together:
import requestsfrom main import URLFetcherdef main(): # Base URL of the API base_url = "https://meawfy.com/internal/api/results.json" # User-agent string user_agent = "MyCustomUserAgent/1.0" # Initialize the URLFetcher with the base URL and user-agent url_fetcher = URLFetcher(base_url, user_agent) # Example search query search_query = "example_query" # Fetch URLs based on the search query urls = url_fetcher.fetch_urls(search_query) # Display the retrieved URLs if urls: print("Retrieved URLs:") for url in urls: print(url) else: print("No URLs found or an error occurred.")if __name__ == "__main__": main()
The library includes built-in error handling for common issues:
try: urls = url_fetcher.fetch_urls(search_query) if urls: print(f"Found {len(urls)} URLs") else: print("No results found")except Exception as e: print(f"Unexpected error: {e}")
The fetch_urls() method catches requests.exceptions.RequestException internally and returns an empty list on error. Check the console output for error details.
urls = url_fetcher.fetch_urls("data science")# Filter for specific domainsmega_urls = [url for url in urls if "mega.nz" in url]print(f"Found {len(mega_urls)} Mega.nz links")