Приложение 2
Код для парсинга на Python
import csv
import time
import re
from datetime import datetime
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
driver = webdriver.Firefox(executable_path=r'c:\work\project\parser\geckodriver.exe')
from selenium.webdriver.common.keys import Keys
from urllib.request import Request, urlopen
def get_user_city(user_link):
time.sleep(2)
html = requests.get(user_link)
soup = BeautifulSoup(html.text, 'lxml')
user_city = soup.find(class_='_2iel _50f7').text
print ('user_city=', user_city)
return user_city
def get_html(url):
#response = requests.get(url)
#return response.text
# req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
# response = urlopen(req).read()
profile = webdriver.FirefoxProfile()
profile.set_preference("general.useragent.override", "Mozilla/5.0")
driver = webdriver.Firefox(profile)
driver.get(url)
#Login in FB
#username= 'alexeyivanovfb@yandex.ru'
#password = 'fbpython'
#UN = driver.find_element_by_id('email')
#UN.send_keys(username)
#PS = driver.find_element_by_id('pass')
#PS.send_keys(password)
#LI = driver.find_element_by_id('loginbutton')
#LI.click()
# Scrolling page
# driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
SCROLL_PAUSE_TIME = 2
# Get scroll height
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
# Scroll down to bottom
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
# Wait to load page
time.sleep(SCROLL_PAUSE_TIME)
# Calculate new scroll height and compare with last scroll height
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
html = driver.page_source
#driver.quit()
return html
def get_clear_tag(tags_list):
try:
cl_tag = []
for i in range(len(tags_list)):
cl_tag.insert(i,tags_list[i].text)
except:
cl_tag = ''
return cl_tag
def get_page_data(html):
try:
user_name = html.find(class_='profileLink').text
except:
user_name = ''
try:
comment_body = html.find("p").text
except:
comment_body = ''
try:
comment_tags = html.find_all(class_='_2ien _50f7')
except:
comment_tags = ''
try:
user_link = html.find(class_="profileLink").get('href')
except:
user_link = ''
try:
user_city = get_user_city(html.find(class_="profileLink").get('href'))
except:
user_city = ''
data = {'User name:': user_name,
'Comment body:': comment_body,
'Comment tags:': get_clear_tag(comment_tags),
'User city:': user_city,
'User Link:': user_link
}
return data
def write_csv_heder(url):
header=re.findall(r'/pg/\w+',url)
with open('fbcomments.csv', 'a', encoding='utf8') as f:
writer = csv.writer(f, delimiter=' ', quotechar='|')
writer.writerow(header)
print(header, 'parsed')
def write_csv(data):
with open('fbcomments.csv', 'a', encoding='utf8') as f:
writer = csv.writer(f, delimiter=' ', quotechar='|')
writer.writerow((data['User name:'],
data['Comment body:'],
data['Comment tags:'],
data['User city:'],
data['User Link:']))
print(data['User name:'], 'parsed')
def read_file(filename):
try:
url_list = []
with open(filename,'r') as hotels:
#for line in hotels:
url_list = [line.rstrip() for line in hotels]
except:
print("An IOError has occurred in input file!")
url_list=''
return url_list
def main():
start = datetime.now()
all_hotels = (read_file('hotels-full.txt'))
for h in range(len(all_hotels)):
print('len(all_hotels)', len(all_hotels))
print('h = ',h)
write_csv_heder(all_hotels[h])
soup = BeautifulSoup(get_html(all_hotels[h]), 'lxml')
all_p = soup.find_all(class_="_1dwg _1w_m _q7o")
for i in range(len(all_p)):
data = get_page_data(all_p[i])
write_csv(data)
#driver.close(all_hotels[h])
end = datetime.now()
total = end - start
print(str(total))
if __name__ == '__main__':
main()
#driver.close()
#driver.quit()