cid/Google_Search_Result_Scraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added user-agent

Browse Source
This commit is contained in:
cid
2023-12-30 03:42:36 +08:00
parent 6544a50965
commit 135bfdc697
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/python3.12/site-packages/main.py
View File

@ -4,7 +4,7 @@ from bs4 import BeautifulSoup
search_keyword = input("Enter a search keyword: ") search_keyword = input("Enter a search keyword: ")
def scrape_body(url): def scrape_body(url):
response = requests.get(url) response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser') soup = BeautifulSoup(response.text, 'html.parser')
body = soup.find('body') body = soup.find('body')
body_list.append(body.prettify()) body_list.append(body.prettify())
@ -12,6 +12,7 @@ def scrape_body(url):
if __name__ == "__main__": if __name__ == "__main__":
body_list = [] body_list = []
headers = {'User-Agent': 'myprogram/1.0'}
for i in range(1, 11): for i in range(1, 11):
search_url = 'https://www.google.com/search?q={}&start=' search_url = 'https://www.google.com/search?q={}&start='
scrape_body('https://www.google.com/search?q={}&start=' + str(i)) scrape_body('https://www.google.com/search?q={}&start=' + str(i))