From 739538f62998b7ea25e810ff157bca8e37ed0b2c Mon Sep 17 00:00:00 2001 From: Mincka Date: Sat, 15 Feb 2020 10:16:06 +0100 Subject: [PATCH] Fixes #71, thanks to @cajuncooks --- dmarchiver/__init__.py | 2 +- dmarchiver/core.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dmarchiver/__init__.py b/dmarchiver/__init__.py index c6f931f..72a183e 100644 --- a/dmarchiver/__init__.py +++ b/dmarchiver/__init__.py @@ -5,4 +5,4 @@ without having to deal with the API limitations. """ -__version__ = "0.2.5" +__version__ = "0.2.6" diff --git a/dmarchiver/core.py b/dmarchiver/core.py index 4b6b08f..bad4f6a 100644 --- a/dmarchiver/core.py +++ b/dmarchiver/core.py @@ -261,6 +261,9 @@ class Crawler(object): _http_headers = { 'User-Agent': _user_agent} + _login_headers = { + 'User-Agent': _user_agent, + 'Referer': 'https://mobile.twitter.com/login'} _ajax_headers = { 'User-Agent': _user_agent, 'Accept': 'application/json, text/javascript, */*; q=0.01', @@ -273,6 +276,7 @@ class Crawler(object): _session = None def authenticate(self, username, password, save_session, raw_output): + force_nojs = 'https://mobile.twitter.com/i/nojs_router?path=%2Flogin' login_url = self._twitter_base_url + '/login' sessions_url = self._twitter_base_url + '/sessions' messages_url = self._twitter_base_url + '/messages' @@ -299,15 +303,15 @@ def authenticate(self, username, password, save_session, raw_output): raw_output_file = open( 'authentication-{0}.txt'.format(username), 'wb') - response = self._session.get( - login_url, - headers=self._http_headers) + response = self._session.post( + force_nojs, + headers=self._login_headers) if raw_output: raw_output_file.write(response.content) raw_output_file.close() - document = lxml.html.document_fromstring(response.text) + document = lxml.html.document_fromstring(response.content) authenticity_token = document.xpath( '//input[@name="authenticity_token"]/@value')[0]