cid/Web_Scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: cid:thierry-dev
Branches Tags
cid:master cid:thierry-dev
...
compare: cid:0c9cea522599bcd4783d7dc3666afe4a69c3e0e9
Branches Tags
cid:master cid:thierry-dev

1 Commits
thierry-de ... 0c9cea5225

Author SHA1 Message Date
cid
0c9cea5225 Udated main.py "added user-agent headers" 2023-12-29 15:02:15 -05:00
1 changed files with 2 additions and 1 deletions
Expand all files Collapse all files

3
main.py
View File

@ -3,6 +3,7 @@ from bs4 import BeautifulSoup
from urllib.parse import urlparse, urlunparse from urllib.parse import urlparse, urlunparse
import pandas as pd import pandas as pd
headers = {'User-Agent': 'myprogram/1.0'}
def scrape_headings(url): def scrape_headings(url):
try: try:
# Check if the URL has a scheme (http/https), and add one if missing # Check if the URL has a scheme (http/https), and add one if missing
@ -11,7 +12,7 @@ def scrape_headings(url):
url = urlunparse(('http',) + parsed_url[1:]) url = urlunparse(('http',) + parsed_url[1:])
# Send an HTTP GET request to the specified URL # Send an HTTP GET request to the specified URL
response = requests.get(url) response = requests.get(url, headers=headers)
# Check if the request was successful # Check if the request was successful
if response.status_code == 200: if response.status_code == 200: