21 lines
707 B
Python
21 lines
707 B
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
search_keyword = input("Enter a search keyword: ")
|
|
|
|
def scrape_body(url):
|
|
response = requests.get(url, headers=headers)
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
body = soup.find('body')
|
|
body_list.append(body.prettify())
|
|
return body_list
|
|
|
|
if __name__ == "__main__":
|
|
body_list = []
|
|
headers = {'User-Agent': 'myprogram/1.0'}
|
|
for i in range(1, 11):
|
|
search_url = 'https://www.google.com/search?q={}&start='
|
|
scrape_body('https://www.google.com/search?q={}&start=' + str(i))
|
|
search_url = search_url.format(search_keyword)
|
|
with open('output.txt', 'w+') as f:
|
|
f.writelines(body_list) |