Merge pull request #1919 from justin2004/oauth-gmail-imap

OAuth 2.0 flow for Gmail IMAP

docs/formats.md

@@ -106,6 +106,8 @@ eleventyNavigation:
 ## imap {#imap}
 - `vd "imap://[email protected]:[email protected]"` opens a connection to the IMAP server
  - e.g. `vd "imap://[email protected]:[email protected]:993"`
+ - e.g. `vd "imap://[email protected]@imap.gmail.com"`
+ - note that you don't specifiy a password for gmail here -- instead, you will be prompted to follow some instructions
 
 ### using VisiData as a pager within psql
 

docs/gmail.md

@@ -0,0 +1,165 @@
+# Using Gmail with OAuth 2.0
+
+## Why
+
+As of May 30 2022, Google [doesn't allow](https://support.google.com/accounts/answer/6010255?hl=en) you to log into your IMAP gmail account using only a username/password combination.
+So to open your Gmail in Visidata you need to create a Google API App, attach some scopes to it, generate a client ID and secret, then let Visidata use that client ID and secret (in a json file).
+
+## How
+
+First, go to the [Google Console](https://console.cloud.google.com/apis/dashboard)
+
+<br/>
+
+![](assets/gmail_oauth/1.png)
+
+<br/>
+
+And click `CREATE PROJECT`
+
+---
+
+
+![](assets/gmail_oauth/2.png)
+
+Give the project a name then click `CREATE`
+
+
+---
+
+Open a new tab and go to the [API Library](https://console.cloud.google.com/apis/library)
+
+<br/>
+
+![](assets/gmail_oauth/enable/1.png)
+
+Search for `gmail`
+
+---
+
+![](assets/gmail_oauth/enable/2.png)
+
+Click the search result `Gmail API`
+
+---
+
+![](assets/gmail_oauth/enable/3.png)
+
+Click `ENABLE`
+
+---
+
+Go back to your first tab
+
+On the left, select `OAuth consent screen` 
+
+![](assets/gmail_oauth/3.png)
+
+Then select `External` and click `CREATE` to create an App 
+
+---
+
+![](assets/gmail_oauth/4.png)
+
+Give the App a name and input your gmail address.
+
+---
+
+![](assets/gmail_oauth/5.png)
+
+Click `ADD OR REMOVE SCOPES`
+
+---
+
+![](assets/gmail_oauth/6.png)
+
+Search for `gmail`
+
+---
+
+![](assets/gmail_oauth/7.png)
+
+Click the checkbox by the row with the scope value `https://mail.google.com/`
+
+Then scroll to the bottom
+
+---
+
+![](assets/gmail_oauth/8.png)
+
+And click `UPDATE`
+
+---
+
+![](assets/gmail_oauth/9.png)
+
+You should see your selected scopes.
+
+Click `SAVE AND CONTINUE`
+
+---
+
+![](assets/gmail_oauth/10.png)
+
+Click `ADD USERS`
+
+---
+
+![](assets/gmail_oauth/11.png)
+
+Type in your gmail email address then click `ADD`
+
+---
+
+![](assets/gmail_oauth/12.png)
+
+On the left, click `Credentials`
+
+---
+
+![](assets/gmail_oauth/13.png)
+
+Near the top click `CREATE CREDENTIALS`
+
+Then click `OAuth client ID`
+
+---
+
+![](assets/gmail_oauth/14.png)
+
+Select the application type `Desktop App` and give your OAuth 2.0 client a name then click `CREATE`
+
+---
+
+![](assets/gmail_oauth/15.png)
+
+Click `DOWNLOAD JSON` and move the downloaded file into the visidata project directory at the path `vdplus/api/google/` and call the file `google-creds.json`
+
+---
+
+Now, on the command line run the equivalent for you:
+
+`vd "imap://[email protected]@imap.gmail.com"`
+
+Then you should get a web browser popup:
+
+![](assets/gmail_oauth/16.png)
+
+Select the account whose email address you have been using in these instructions.
+
+---
+
+![](assets/gmail_oauth/17.png)
+
+Click `Select all`
+
+---
+
+![](assets/gmail_oauth/18.png)
+
+See your gmail in Visidata.
+
+<Chef's Kiss>
+
+---
+

vdplus/api/google/__init__.py

@@ -1,35 +1,2 @@
-
-from visidata import vd, VisiData, Sheet, IndexSheet, SequenceSheet, ColumnItem, Path, AttrDict, ColumnAttr, asyncthread
 from .gsheets import *
 from .gdrive import *
-
-
-def _google_creds_fn():
- from pkg_resources import resource_filename
- return resource_filename('vdplus.api.google', 'google-creds.json')
-
-
-@VisiData.api
-def google_auth(vd, scopes='spreadsheets.readonly'):
- import pickle
- import os.path
- GSHEETS_TOKEN_FILE = Path(vd.options.visidata_dir)/f'google-{scopes}.pickle'
- creds = None
- if os.path.exists(GSHEETS_TOKEN_FILE):
- with open(GSHEETS_TOKEN_FILE, 'rb') as fp:
- creds = pickle.load(fp)
-
- if not creds or not creds.valid:
- if creds and creds.expired and creds.refresh_token:
- from google.auth.transport.requests import Request
- creds.refresh(Request())
- else:
- from google_auth_oauthlib.flow import InstalledAppFlow
- SCOPES = [f'https://www.googleapis.com/auth/{x}' for x in scopes.split()]
- flow = InstalledAppFlow.from_client_secrets_file(_google_creds_fn(), SCOPES)
- creds = flow.run_local_server(port=0)
-
- with open(GSHEETS_TOKEN_FILE, 'wb') as fp:
- pickle.dump(creds, fp)
-
- return creds

visidata/loaders/google.py

@@ -0,0 +1,44 @@
+
+from visidata import vd, VisiData, Sheet, IndexSheet, SequenceSheet, ColumnItem, Path, AttrDict, ColumnAttr, asyncthread
+
+
+def _google_creds_fn():
+ from pkg_resources import resource_filename
+ import os
+ if not os.path.exists(Path(resource_filename('vdplus.api.google', 'google-creds.json'))):
+ vd.error('google-creds.json file does not exist. create it by following this guide: https://github.com/saulpw/visidata/blob/develop/docs/gmail.md')
+ else:
+ return resource_filename('vdplus.api.google', 'google-creds.json')
+
+
+@VisiData.api
+def google_auth(vd, scopes=None):
+ import pickle
+ import os.path
+ import urllib.parse
+
+ SCOPES = []
+ for scope in scopes.split():
+ if not scope.startswith('https://'):
+ scope = 'https://www.googleapis.com/auth/' + scope
+ SCOPES.append(scope)
+
+ GOOGLE_TOKEN_FILE = Path(vd.options.visidata_dir)/f'google-{urllib.parse.quote_plus(str(scopes))}.pickle'
+ creds = None
+ if os.path.exists(GOOGLE_TOKEN_FILE):
+ with open(GOOGLE_TOKEN_FILE, 'rb') as fp:
+ creds = pickle.load(fp)
+
+ if not creds or not creds.valid:
+ if creds and creds.expired and creds.refresh_token:
+ from google.auth.transport.requests import Request
+ creds.refresh(Request())
+ else:
+ from google_auth_oauthlib.flow import InstalledAppFlow
+ flow = InstalledAppFlow.from_client_secrets_file(_google_creds_fn(), SCOPES)
+ creds = flow.run_local_server(port=0)
+
+ with open(GOOGLE_TOKEN_FILE, 'wb') as fp:
+ pickle.dump(creds, fp)
+
+ return creds

visidata/loaders/imap.py

@@ -1,12 +1,12 @@
 from visidata import VisiData, vd, TableSheet, asyncthread, ColumnItem, Column, ColumnAttr, Progress
+import visidata.loaders.google
 from urllib.parse import urlparse
 
 
 @VisiData.api
 def openurl_imap(vd, url, **kwargs):
  url_parsed = urlparse(str(url))
- password = url_parsed.password or vd.error('no password given in url') # vd.input("imap password for %s" % user, display=False))
- return ImapSheet(url_parsed.hostname, source=url_parsed, password=password)
+ return ImapSheet(url_parsed.hostname, source=url_parsed, password=url_parsed.password)
 
 
 class ImapSheet(TableSheet):
@@ -29,8 +29,18 @@ def reload(self):
  import email.parser
 
  m = imaplib.IMAP4_SSL(host=self.source.hostname)
+ # m.debug=4
  user = self.source.username
- m.login(user, self.password)
+
+ if self.source.hostname == 'imap.gmail.com':
+ credentials=vd.google_auth(scopes='https://mail.google.com/')
+ header_template = 'user=%s\1auth=Bearer %s\1\1'
+ m.authenticate('XOAUTH2', lambda x: header_template % (user, credentials.token))
+ else:
+ if self.password is None:
+ vd.error('no password given in url') # vd.input("imap password for %s" % user, display=False))
+ m.login(user, self.source.password)
+
  typ, folders = m.list()
  for r in Progress(folders, gerund="downloading"):
  fname = r.decode('utf-8').split()[-1]