1  """googlesearch is a Python library for searching Google, easily."""
   2  from time import sleep
   3  from bs4 import BeautifulSoup
   4  from requests import get
   5  from urllib.parse import unquote # to decode the url
   6  from .user_agents import get_useragent
   7  
   8  
   9  def _req(term, results, lang, start, proxies, timeout, safe, ssl_verify, region):
  10      resp = get(
  11          url="https://www.google.com/search",
  12          headers={
  13              "User-Agent": get_useragent(),
  14              "Accept": "*/*"
  15          },
  16          params={
  17              "q": term,
  18              "num": results + 2,  # Prevents multiple requests
  19              "hl": lang,
  20              "start": start,
  21              "safe": safe,
  22              "gl": region,
  23          },
  24          proxies=proxies,
  25          timeout=timeout,
  26          verify=ssl_verify,
  27          cookies = {
  28              'CONSENT': 'PENDING+987', # Bypasses the consent page
  29              'SOCS': 'CAESHAgBEhIaAB',
  30          }
  31      )
  32      resp.raise_for_status()
  33      return resp
  34  
  35  
  36  class SearchResult:
  37      def __init__(self, url, title, description):
  38          self.url = url
  39          self.title = title
  40          self.description = description
  41  
  42      def __repr__(self):
  43          return f"SearchResult(url={self.url}, title={self.title}, description={self.description})"
  44  
  45  
  46  def search(term, num_results=10, lang="en", proxy=None, advanced=False, sleep_interval=0, timeout=5, safe="active", ssl_verify=None, region=None, start_num=0, unique=False):
  47      """Search the Google search engine"""
  48  
  49      # Proxy setup
  50      proxies = {"https": proxy, "http": proxy} if proxy and (proxy.startswith("https") or proxy.startswith("http")) else None
  51  
  52      start = start_num
  53      fetched_results = 0  # Keep track of the total fetched results
  54      fetched_links = set() # to keep track of links that are already seen previously
  55  
  56      while fetched_results < num_results:
  57          # Send request
  58          resp = _req(term, num_results - start,
  59                      lang, start, proxies, timeout, safe, ssl_verify, region)
  60          
  61          # put in file - comment for debugging purpose
  62          # with open('google.html', 'w') as f:
  63          #     f.write(resp.text)
  64          
  65          # Parse
  66          soup = BeautifulSoup(resp.text, "html.parser")
  67          result_block = soup.find_all("div", class_="ezO2md")
  68          new_results = 0  # Keep track of new results in this iteration
  69  
  70          for result in result_block:
  71              # Find the link tag within the result block
  72              link_tag = result.find("a", href=True)
  73              # Find the title tag within the link tag
  74              title_tag = link_tag.find("span", class_="CVA68e") if link_tag else None
  75              # Find the description tag within the result block
  76              description_tag = result.find("span", class_="FrIlee")
  77  
  78              # Check if all necessary tags are found
  79              if link_tag and title_tag and description_tag:
  80                  # Extract and decode the link URL
  81                  link = unquote(link_tag["href"].split("&")[0].replace("/url?q=", "")) if link_tag else ""
  82              # Extract and decode the link URL
  83              link = unquote(link_tag["href"].split("&")[0].replace("/url?q=", "")) if link_tag else ""
  84              # Check if the link has already been fetched and if unique results are required
  85              if link in fetched_links and unique:
  86                  continue  # Skip this result if the link is not unique
  87              # Add the link to the set of fetched links
  88              fetched_links.add(link)
  89              # Extract the title text
  90              title = title_tag.text if title_tag else ""
  91              # Extract the description text
  92              description = description_tag.text if description_tag else ""
  93              # Increment the count of fetched results
  94              fetched_results += 1
  95              # Increment the count of new results in this iteration
  96              new_results += 1
  97              # Yield the result based on the advanced flag
  98              if advanced:
  99                  yield SearchResult(link, title, description)  # Yield a SearchResult object
 100              else:
 101                  yield link  # Yield only the link
 102  
 103              if fetched_results >= num_results:
 104                  break  # Stop if we have fetched the desired number of results
 105  
 106          if new_results == 0:
 107              #If you want to have printed to your screen that the desired amount of queries can not been fulfilled, uncomment the line below:
 108              #print(f"Only {fetched_results} results found for query requiring {num_results} results. Moving on to the next query.")
 109              break  # Break the loop if no new results were found in this iteration
 110  
 111          start += 10  # Prepare for the next set of results
 112          sleep(sleep_interval)
 113