Tout d'abord, je pense que cela vaut la peine de le dire, je sais qu'il y a un tas de questions similaires, mais AUCUNE d'entre elles ne fonctionne pour moi ...
Je suis novice sur Python, html et web scraper. J'essaie de supprimer les informations utilisateur d'un site Web qui doit d'abord se connecter. Dans mes tests, j'utilise grattoir mes paramètres de messagerie à partir de github comme exemples. La page principale est ' https://github.com/login ' et la page cible est ' https://github.com/settings/emails '
Voici une liste de méthodes que j'ai essayées
##################################### Method 1
import mechanize
import cookielib
from BeautifulSoup import BeautifulSoup
import html2text
br = mechanize.Browser()
cj = cookielib.LWPCookieJar()
br.set_cookiejar(cj)
# Browser options
br.set_handle_equiv(True)
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
br.addheaders = [('User-agent', 'Chrome')]
# The site we will navigate into, handling it's session
br.open('https://github.com/login')
for f in br.forms():
print f
br.select_form(nr=0)
# User credentials
br.form['login'] = 'myusername'
br.form['password'] = 'mypwd'
# Login
br.submit()
br.open('github.com/settings/emails').read()
################ Method 2
import urllib, urllib2, cookielib
username = 'myusername'
password = 'mypwd'
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
login_data = urllib.urlencode({'username' : username, 'j_password' : password})
opener.open('https://github.com/login', login_data)
resp = opener.open('https://github.com/settings/emails')
print resp.read()
############# Method 3
import urllib
opener = urllib.FancyURLopener()
print opener.open('http://myusername:[email protected]/settings/emails').read()
########## Method 4
import mechanize
import cookielib
br = mechanize.Browser()
cj = cookielib.LWPCookieJar()
br.set_cookiejar(cj)
br.set_handle_equiv(True)
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
#br.set_debug_http(True)
#br.set_debug_redirects(True)
#br.set_debug_responses(True)
br.addheaders = [('User-agent', 'Chrome')]
br.add_password('https://github.com/settings/emails', 'myusername', 'mypwd')
br.open('https://github.com/settings/emails')
print br.response().read()
############ Methods 5
from requests import session
payload = {
'action': 'login',
'username': 'myusername',
'password': 'mypwd'
}
with session() as c:
c.post('https://github.com/login', data=payload)
request = c.get('https://github.com/settings/emails')
print request.headers
print request.text
########### Method 6
import requests
from requests.packages.urllib3 import add_stderr_logger
import sys
from bs4 import BeautifulSoup as bs
add_stderr_logger()
s = requests.Session()
s.headers['User-Agent'] = 'Chrome'
username = 'myusername'
password = 'mypwd'
url = 'https://github.com/login'
# after examining the HTML of the website you're trying to log into
# set name_form to the name of the form element that contains the name and
# set password_form to the name of the form element that will contain the password
login = {'login': username, 'password': password}
login_response = s.post(url, data=login)
for r in login_response.history:
if r.status_code == 401: # 401 means authentication failed
print 'error!'
sys.exit(1) # abort
pdf_response = s.get('https://github.com/settings/emails') # Your cookies and headers are automatically included
soup = bs(pdf_response.content)
J'ai également lu quelques discussions sur les différences entre l'authentification HTTP et les cookies. Aucun d'entre eux ne fonctionnait toujours.
Veuillez aider et toute aide serait appréciée. Merci beaucoup.
Cela fonctionne pour moi:
##################################### Method 1
import mechanize
import cookielib
from BeautifulSoup import BeautifulSoup
import html2text
# Browser
br = mechanize.Browser()
# Cookie Jar
cj = cookielib.LWPCookieJar()
br.set_cookiejar(cj)
# Browser options
br.set_handle_equiv(True)
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
br.addheaders = [('User-agent', 'Chrome')]
# The site we will navigate into, handling it's session
br.open('https://github.com/login')
# View available forms
for f in br.forms():
print f
# Select the second (index one) form (the first form is a search query box)
br.select_form(nr=1)
# User credentials
br.form['login'] = 'mylogin'
br.form['password'] = 'mypass'
# Login
br.submit()
print(br.open('https://github.com/settings/emails').read())
Vous n'étiez pas loin du tout!
aimerait ajouter ma solution à côté. cette réponse suit principalement l'approche hacky/paresseuse que je suis toujours dans tout ce que je fais. a continué principalement parce que j'étais trop paresseux pour gérer les cookies, les données de session, etc.
cette solution est très utile si vous souhaitez gratter plusieurs pages d'un site Web après vous être connecté avec des informations d'identification de compte unique (par exemple, tous vos tableaux de Pinterest). pas si vous voulez automatiser l'authentification en utilisant plusieurs comptes
donc ma solution est Selenium avec des profils firefox.
j'ai imaginé ce mécanisme quand je suis tombé sur le besoin de gratter quelques pages pinterest, j'ai ajouté quelques lignes de code de l'exemple montrant comment utiliser le profil. adapter le code selon vos besoins.
from Selenium import webdriver
from Selenium.webdriver.common.by import By
from Selenium.webdriver.common.keys import Keys
from Selenium.webdriver.support.ui import Select
from Selenium.webdriver.support.ui import WebDriverWait
from Selenium.common.exceptions import TimeoutException
from Selenium.webdriver.support import expected_conditions as EC
from Selenium.common.exceptions import NoSuchElementException
from Selenium.common.exceptions import NoAlertPresentException
#replace with your firefox profile
fp=webdriver.FirefoxProfile('C:/Users/SJ/AppData/Roaming/Mozilla/Firefox/Profiles/hlsfrs2o.scrape')
#enter your url here
url=""
driver = webdriver.Firefox(fp)
driver.get(url)
html_source = driver.page_source
La manière classique d'aborder ce problème est:
En tant que fans d'automatisation, nous avons pensé que nous pourrions écrire du code pour automatiser le point 2 (qui est en fait le plus long) et le résultat est un formulaire de connexion, une bibliothèque pour remplir automatiquement les formulaires de connexion compte tenu de la page de connexion, du nom d'utilisateur et du mot de passe. Voici le code d'une simple araignée qui utiliserait loginform pour se connecter automatiquement aux sites.
githubloginspider.py
from scrapy.spider import BaseSpider
from scrapy.http import FormRequest
from scrapy.http.request import Request
from loginform import fill_login_form
from scrapy import log
from scraping.articles import ArticleItem
class GitHubLogin(BaseSpider):
name = 'GitHubLogin'
allowed_domains = ['github.com']
start_urls = ['http://github.com/login']
login_user = 'ranvijay5686'
login_pass = ''
def parse(self, response):
(args, url, method) = fill_login_form(response.url,
response.body, self.login_user, self.login_pass)
return FormRequest(url, method=method, formdata=args,
callback=self.after_login)
def after_login(self, response):
# for link in response.xpath("//*[@id='site-container']/div[2]/div[4]/p/a/@href").extract():
item = ArticleItem()
item['title'] = 'ranvijay'
log.msg('*************** : '
+ str(response.xpath("//form[@class='subnav-search left']/input/@value"
).extract()))
item['url'] = \
response.xpath("//*[@id='site-container']/div[1]/div/div/span/span/text()"
).extract()
yield item
items.py
from scrapy.item import Item, Field
class ArticleItem(Item):
title = Field()
url = Field()
loginform.py
import sys
from argparse import ArgumentParser
from collections import defaultdict
from lxml import html
__version__ = '1.0' # also update setup.py
def _form_score(form):
score = 0
# In case of user/pass or user/pass/remember-me
if len(form.inputs.keys()) in (2, 3):
score += 10
typecount = defaultdict(int)
for x in form.inputs:
type_ = (x.type if isinstance(x, html.InputElement) else 'other'
)
typecount[type_] += 1
if typecount['text'] > 1:
score += 10
if not typecount['text']:
score -= 10
if typecount['password'] == 1:
score += 10
if not typecount['password']:
score -= 10
if typecount['checkbox'] > 1:
score -= 10
if typecount['radio']:
score -= 10
return score
def _pick_form(forms):
"""Return the form most likely to be a login form"""
return sorted(forms, key=_form_score, reverse=True)[0]
def _pick_fields(form):
"""Return the most likely field names for username and password"""
userfield = passfield = emailfield = None
for x in form.inputs:
if not isinstance(x, html.InputElement):
continue
type_ = x.type
if type_ == 'password' and passfield is None:
passfield = x.name
Elif type_ == 'text' and userfield is None:
userfield = x.name
Elif type_ == 'email' and emailfield is None:
emailfield = x.name
return (userfield or emailfield, passfield)
def submit_value(form):
"""Returns the value for the submit input, if any"""
for x in form.inputs:
if x.type == 'submit' and x.name:
return [(x.name, x.value)]
else:
return []
def fill_login_form(
url,
body,
username,
password,
):
doc = html.document_fromstring(body, base_url=url)
form = _pick_form(doc.xpath('//form'))
(userfield, passfield) = _pick_fields(form)
form.fields[userfield] = username
form.fields[passfield] = password
form_values = form.form_values() + submit_value(form)
return (form_values, form.action or form.base_url, form.method)
def main():
ap = ArgumentParser()
ap.add_argument('-u', '--username', default='username')
ap.add_argument('-p', '--password', default='secret')
ap.add_argument('url')
args = ap.parse_args()
try:
import requests
except ImportError:
print 'requests library is required to use loginform as a tool'
r = requests.get(args.url)
(values, action, method) = fill_login_form(args.url, r.text,
args.username, args.password)
print '''url: {0}
method: {1}
payload:'''.format(action, method)
for (k, v) in values:
print '- {0}: {1}'.format(k, v)
if __name__ == '__main__':
sys.exit(main())