Парсинг сайтов на Python: подробный видеокурс и программный код

importcsv

fromdatetimeimportdatetime

importrequests

frombs4 importBeautifulSoup

defget_html(url):

response=requests.get(url)

returnresponse.text

defget_all_links(html):

soup=BeautifulSoup(html,'lxml')

tds=soup.find('table',id='currencies-all').find_all('td',class_='currency-name')

links=[]

fortd intds:

a=td.find('a',class_='currency-name-container').get('href')

link='https://coinmarketcap.com'+a

links.append(link)

returnlinks

deftext_before_word(text,word):

line=text.split(word)[0].strip()

returnline

defget_page_data(html):

soup=BeautifulSoup(html,'lxml')

try:

name=text_before_word(soup.find('title').text,'price')

except:

name=''

try:

price=text_before_word(soup.find('div',

class_='col-xs-6 col-sm-8 col-md-4 text-left').text,'USD')

except:

price=''

data={'name':name,

'price':price}

returndata

defwrite_csv(i,data):

withopen('coinmarketcap.csv','a')asf:

writer=csv.writer(f)

writer.writerow((data['name'],

data['price']))

print(i,data['name'],'parsed')

defmain():

start=datetime.now()

url='https://coinmarketcap.com/all/views/all'

all_links=get_all_links(get_html(url))

fori,link inenumerate(all_links):

html=get_html(link)

data=get_page_data(html)

write_csv(i,data)

end=datetime.now()

total=end-start

print(str(total))

a=input()

if__name__=='__main__':

main()

frommultiprocessingimportPool

importcsv

fromdatetimeimportdatetime

importrequests

frombs4 importBeautifulSoup

defget_html(url):

response=requests.get(url)

returnresponse.text

defget_all_links(html):

soup=BeautifulSoup(html,'lxml')

tds=soup.find('table',id='currencies-all').find_all('td',class_='currency-name')

links=[]

fortd intds:

a=td.find('a',class_='currency-name-container').get('href')

link='https://coinmarketcap.com'+a

links.append(link)

returnlinks

deftext_before_word(text,word):

line=text.split(word)[0].strip()

returnline

defget_page_data(html):

soup=BeautifulSoup(html,'lxml')

try:

name=text_before_word(soup.find('title').text,'price')

except:

name=''

try:

price=text_before_word(soup.find('div',

class_='col-xs-6 col-sm-8 col-md-4 text-left').text,'USD')

except:

price=''

data={'name':name,

'price':price}

returndata

defwrite_csv(data):

withopen('coinmarketcap.csv','a')asf:

writer=csv.writer(f)

writer.writerow((data['name'],

data['price']))

print(data['name'],'parsed')

defmake_all(link):

html=get_html(link)

data=get_page_data(html)

write_csv(data)

defmain():

start=datetime.now()

url='https://coinmarketcap.com/all/views/all'

all_links=get_all_links(get_html(url))

withPool(40)asp:

p.map(make_all,all_links)

end=datetime.now()

total=end-start

print(str(total))

a=input()

if__name__=='__main__':

main()

importrequests

frombs4 importBeautifulSoup

importcsv

defget_html(url):

r=requests.get(url)

returnr.text

defget_total_pages(html):

soup=BeautifulSoup(html,'lxml')

divs=soup.find('div',class_='pagination-pages clearfix')

pages=divs.find_all('a',class_='pagination-page')[-1].get('href')

total_pages=pages.split('=')[1].split('&')[0]

returnint(total_pages)

defwrite_csv(data):

withopen('avito.csv','a')asf:

writer=csv.writer(f)

writer.writerow((data['title'],

data['price'],

data['metro'],

data['url']))

defget_page_data(html):

soup=BeautifulSoup(html,'lxml')

divs=soup.find('div',class_='catalog-list')

ads=divs.find_all('div',class_='item_table')

forad inads:

try:

div=ad.find('div',class_='description').find('h3')

if'htc'notindiv.text.lower():

continue

else:

title=div.text.strip()

except:

title=''

try:

div=ad.find('div',class_='description').find('h3')

url="https://avito.ru"+div.find('a').get('href')

except:

url=''

try:

price=ad.find('div',class_='about').text.strip()

except:

price=''

try:

div=ad.find('div',class_='data')

metro=div.find_all('p')[-1].text.strip()

except:

metro=''

data={'title':title,

'price':price,

'metro':metro,

'url':url}

write_csv(data)

defmain():

url="https://avito.ru/moskva/telefony?p=1&q=htc"

base_url="https://avito.ru/moskva/telefony?"

page_part="p="

query_par="&q=htc"

# total_pages = get_total_pages(get_html(url))

foriinrange(1,3):

url_gen=base_url+page_part+str(i)+query_par

html=get_html(url_gen)

get_page_data(html)

if__name__=='__main__':

main()

fromselenium importwebdriver

fromtimeimportsleep

fromPIL importImage

frompytesseract importimage_to_string

classBot:

def__init__(self):

self.driver=webdriver.Chrome()# or Firefox() or smth else

self.navigate()

deftake_screenshot(self):

self.driver.save_screenshot('avito_screenshot.png')

deftel_recon(self):

image=Image.open('tel.gif')

print(image_to_string(image))

defcrop(self,location,size):

image=Image.open('avito_screenshot.png')

x=location['x']

y=location['y']

width=size['width']

height=size['height']

image.crop((x,y,x+width,y+height)).save('tel.gif')

self.tel_recon()

defnavigate(self):

self.driver.get('https://www.avito.ru/velikie_luki/telefony/lenovo_a319_1161088336')

button=self.driver.find_element_by_xpath(

'//button[@class="button item-phone-button js-item-phone-button

button-origin button-origin-blue button-origin_full-width

button-origin_large-extra item-phone-button_hide-phone

item-phone-button_card js-item-phone-button_card"]')

button.click()

sleep(3)

self.take_screenshot()

image=self.driver.find_element_by_xpath(

'//div[@class="item-phone-big-number js-item-phone-big-number"]//*')

location=image.location

size=image.size

self.crop(location,size)

defmain():

b=Bot()

if__name__=='__main__':

main()

importrequests

frombs4 importBeautifulSoup

fromrandomimportchoice,uniform

fromtimeimportsleep

defget_html(url,useragent=None,proxy=None):

print('get_html')

r=requests.get(url,headers=useragent,proxies=proxy)

returnr.text

defget_ip(html):

print('get_ip')

print('New Proxy & User-Agent:')

soup=BeautifulSoup(html,'lxml')

ip=soup.find('span',class_='ip').text.strip()

ua=soup.find('span',class_='ip').find_next_sibling('span').text.strip()

print(ip)

print(ua)

print('---------------------------')

defmain():

url='https://sitespy.ru/my-ip'

useragents=open('useragents.txt').read().split(' ')

proxies=open('proxies.txt').read().split(' ')

foriinrange(10):

# sleep(uniform(3, 6))

proxy={'http':'http://'+choice(proxies)}

useragent={'User-Agent':choice(useragents)}

try:

html=get_html(url,useragent,proxy)

except:

continue

get_ip(html)

if__name__=='__main__':

main()

importos

importrequests

# url = 'https://www.hdwallpapers.in/thumbs/2018/spiral_silver_metal-t1.jpg'

# r = requests.get(url, stream=True)    # stream for partial download

# with open('1.jpg', 'bw') as f:

#     for chunk in r.iter_content(8192):

#         f.write(chunk)

urls=[

'https://www.hdwallpapers.in/walls/messier_106_spiral_galaxy_5k-wide.jpg',

'https://www.hdwallpapers.in/walls/helix_nebula_5k-wide.jpg',

'https://www.hdwallpapers.in/walls/orion_nebula_hubble_space_telescope_5k-wide.jpg',

'https://www.hdwallpapers.in/walls/solar_space-wide.jpg'

]

defget_file(url):

r=requests.get(url,stream=True)

returnr

defget_name(url):

name=url.split('/')[-1]

folder=name.split('.')[0]

ifnotos.path.exists(folder):

os.makedirs(folder)

path=os.path.abspath(folder)

returnpath+'/'+name

defsave_image(name,file_object):

withopen(name,'bw')asf:

forchunkinfile_object.iter_content(8192):

f.write(chunk)

defmain():

forurl inurls:

save_image(get_name(url),get_file(url))

if__name__=='__main__':

main()