Add a web-scraping script to determine, sort and filter dependent packages

skimage_filter_dependents.py

@@ -0,0 +1,346 @@
+"""
+At the time this script was created (July 2020), GitHub did not offer an
+official way to query dependent packages through their API. So, we went
+for a web-scraping approach using BeautifulSoup, patterned after a response
+in this stack-overflow thread:
+https://stackoverflow.com/questions/58734176/how-to-use-github-api-to-get-a-repositorys-dependents-information-in-github
+
+To retrieve topic lists via the GitHub API, the user must have defined a
+GITHUB_TOKEN environment variable.
+
+This script generates three lists of packages:
+
+1.) One that has ALL dependents that are active repositories (i.e. no "Ghost"
+icon in the web page).
+2.) Another one that only retains packages with >= min_stars stars, but also
+includes a list of the GitHub "topics" associated with each package.
+3.) A third list that is based on filtering the second list. During filtering,
+a package is retained if either:
+ a.) Any string from repo_name_terms is in the repository organization/name
+ b.) A topic in the repo's topic list matches a topic in topic_search_terms
+
+The three variables containing the lists described above are:
+
+Outputs
+-------
+all_packages : list of tuple
+ Each element is a (name, forks, stars) tuple.
+popular_packages : list of tuple
+ Each element is a (name, forks, stars, topics) tuple.
+popular_filtered_packages : list, tuple
+ Each element is a (name, forks, stars, topics) tuple.
+"""
+
+import os
+import pickle
+
+from bs4 import BeautifulSoup
+from github import Github
+import pandas
+import requests
+
+# we use PyGitHub to retrieve topic lists
+token = os.environ['GITHUB_TOKEN']
+g = Github(token)
+
+# ----------------------------------
+# START OF USER-CONFIGURABLE OPTIONS
+# ----------------------------------
+
+# The repository we will query (whose dependents we want to find)
+repo_to_query = "scikit-image/scikit-image"
+
+# Retrieve detailed topic lists only for packages with >= min_stars stars.
+min_stars = 5
+
+# If True, will write the three lists to .pickle files in the current directory
+save_to_pickle = False
+# If True, will write the three lists to .csv files in the current directory
+save_to_csv = True
+
+# Search terms of interest in the repository organization/name.
+# (see description at top)
+# All terms should be in lower case.
+repo_name_terms = [
+ 'brain',
+ 'cell',
+ 'ecg',
+ 'eeg',
+ 'medi',
+ 'mri',
+ 'neuro',
+ 'pathol',
+ 'retin',
+ 'slide',
+ 'spectro',
+ 'tissue',
+ 'tomo',
+]
+
+# Search terms of interest in the repository's topics (see description at top).
+# This list was created to match bio-image applications by manually curating
+# topic names from the full list of dependent packages.
+topic_search_terms = [
+ 'airways',
+ 'anatomy',
+ 'arteries',
+ 'astrocytes',
+ 'atomic-force-microscopy',
+ 'afm',
+ 'axon',
+ 'bioimage-informatics',
+ 'bioinformatics',
+ 'biologists',
+ 'biomedical-image-processing',
+ 'bionic-vision',
+ 'biophysics',
+ 'brain-connectivity',
+ 'brain-imaging',
+ 'brain-mri',
+ 'brain-tumor-segmentation',
+ 'brats',
+ 'calcium',
+ 'cancer-research',
+ 'cell-biology',
+ 'cell-detection',
+ 'cell-segmentation',
+ 'computational-pathology',
+ 'connectome',
+ 'connectomics',
+ 'cryo-em',
+ 'ct-data',
+ 'deconvolution-microscopy',
+ 'dicom',
+ 'dicom-rt',
+ 'digital-pathology-data',
+ 'digital-pathology',
+ 'digital-slide-archive',
+ 'dmri',
+ 'electron-microscopy',
+ 'electrophysiology',
+ 'fluorescence',
+ 'fluorescence-microscopy-imaging',
+ 'fmri',
+ 'fmri-preprocessing',
+ 'functional-connectomes',
+ 'healthcare-imaging',
+ 'histology',
+ 'voxel',
+ 'microorganism-colonies',
+ 'microscopy',
+ 'microscopy-images',
+ 'neuroimaging',
+ 'medical',
+ 'medical-image-computing',
+ 'medical-image-processing',
+ 'medical-images',
+ 'medical-imaging',
+ 'mri',
+ 'myelin',
+ 'neural-engineering',
+ 'neuroanatomy',
+ 'neuroimaging',
+ 'neuroimaging-analysis',
+ 'neuropoly',
+ 'neuroscience',
+ 'nih-brain-initiative',
+ 'openslide',
+ 'pathology',
+ 'pathology-image',
+ 'radiation-oncology',
+ 'radiation-physics',
+ 'raman',
+ 'retinal-implants',
+ 'scanning-probe-microscopy',
+ 'scanning-tunnelling-microscopy',
+ 'single-cell-imaging',
+ 'slide-images',
+ 'spectroscopy',
+ 'spinalcord',
+ 'stm',
+ 'stem',
+ 'stitching',
+ 'structural-connectomes',
+ 'tissue-localization',
+ 'tomography',
+ 'volumetric-images',
+ 'whole-slide-image',
+ 'whole-slide-imaging',
+]
+
+# Omit the following repositories from the filtered list.
+# These match at least one of the search terms above, but do not appear to be
+# biology-focused. (e.g. the term "cell" appears in "Marcello").
+omit_list = [
+ 'Marcello-Sega/pytim',
+ 'PMEAL/porespy'
+]
+
+# --------------------------------
+# END OF USER-CONFIGURABLE OPTIONS
+# --------------------------------
+
+# Parse at most this many web pages.
+# Parsing should automatically stop when reaching the last page.
+max_page_num = 100
+
+packages = True
+url = ('https://github.com/{}/network/dependents'
+ '?dependent_type=PACKAGE').format(repo_to_query)
+
+package_list = []
+ghost_list = []
+prev_len = 0
+for i in range(max_page_num):
+ # retrieve HTML for the current URL
+ print("GET " + url)
+ r = requests.get(url)
+ soup = BeautifulSoup(r.content, "html.parser")
+
+ page_package_list = []
+ page_ghost_list = []
+ for t in soup.findAll("div", {"class": "Box-row"}):
+ try:
+ # find repository org/name
+ name = "{}/{}".format(
+ t.find('a', {"data-repository-hovercards-enabled": ""}).text,
+ t.find('a', {"data-hovercard-type": "repository"}).text
+ )
+ except AttributeError:
+ # Ghost repositories will give None for the find() calls above.
+ # This results in an AttributeError when trying to access .text
+ page_ghost_list.append(t.text)
+ continue
+
+ # extract the number of stars
+ stars = 'unknown'
+ for span in t.find_all('span', attrs={'class': 'text-gray-light'}):
+ svg_star = span.find_all('svg', attrs={'class': 'octicon-star'})
+ if svg_star:
+ # replace ","" in e.g. "1,000" before casting to int
+ stars = int(span.text.strip().replace(",", ""))
+ break
+
+ # extract the number of forks
+ forks = 'unknown'
+ for span in t.find_all('span', attrs={'class': 'text-gray-light'}):
+ svg_fork = span.find_all('svg',
+ attrs={'class': 'octicon-repo-forked'})
+ if svg_fork:
+ # replace ","" in e.g. "1,000" before casting to int
+ forks = int(span.text.strip().replace(",", ""))
+ break
+
+ page_package_list.append((name, forks, stars))
+
+ # append packages from the current page to the overall lists
+ package_list = package_list + page_package_list
+ ghost_list = ghost_list + page_ghost_list
+
+ # remove any duplicates
+ package_list = list(set(package_list))
+ ghost_list = list(set(ghost_list))
+
+ # terminate if no change from the prior URL
+ new_len = len(package_list) + len(ghost_list)
+ if new_len == prev_len:
+ print("no change in package lists... stopping scraping")
+ break
+ prev_len = new_len
+
+ # find the URL for the "Next" page of packages
+ paginationContainers = soup.find(
+ "div", {"class": "paginate-container"}).find_all('a')
+ url = None
+ for paginationContainer in paginationContainers:
+ # Make sure we are retrieving the "Next" page and not the "Previous"
+ if paginationContainer.text == "Next":
+ url = paginationContainer["href"]
+ if url is None:
+ print("No additional next page found, ... stopping scraping")
+ break
+
+# sort by descending number of stars
+# This is the first list mentioned at the top.
+all_packages = sorted(package_list, key=lambda x: x[2], reverse=True)
+
+# Create the second list by retaining only those with >= min_stars
+# Note that in the package list, the tuple is:
+# (name, # of forks, # of stars)
+_popular_packages = [p for p in all_packages if p[2] >= min_stars]
+n_popular = len(_popular_packages)
+
+# add a 4th term to each tuple, containing the GitHub topic list
+popular_packages = []
+
+for n, p in enumerate(_popular_packages):
+ print("Retrieving topics for package {} of {}".format(n + 1, n_popular))
+ repo_name = p[0]
+ repo = g.get_repo(repo_name)
+ topics = repo.get_topics()
+ popular_packages.append(p + (topics,))
+
+print("Applying filtering")
+popular_filtered_packages = []
+for p in popular_packages:
+ name = p[0]
+ name_lower = name.lower()
+ if name in omit_list:
+ continue
+ topics = p[3]
+ keep = False # unless we match a term below, we will exclude the package
+
+ # check match based on repository organization/name
+ for m in repo_name_terms:
+ if m in name_lower:
+ keep = True
+ break
+
+ # If not already a match, search based on topic search terms
+ if not keep:
+ for topic in topics:
+ if topic in topic_search_terms:
+ keep = True
+ break
+ if keep:
+ popular_filtered_packages.append(p)
+
+# dump output lists to pickle
+fname_base = repo_to_query.replace('/', '_')
+if save_to_pickle:
+ print("Writing pickle files")
+
+ os.chdir('/media/lee8rx/data/Dropbox/Dropbox/Grants/CZI')
+ with open(fname_base + '_all_packages.pickle', 'wb') as f:
+ pickle.dump(all_packages, f)
+
+ with open(fname_base + '_popular_packages.pickle', 'wb') as f:
+ pickle.dump(popular_packages, f)
+
+ with open(fname_base + '_popular_filtered_packages.pickle', 'wb') as f:
+ pickle.dump(popular_filtered_packages, f)
+
+if save_to_csv:
+ print("Writing CSV files")
+ df_all = pandas.DataFrame(
+ all_packages,
+ columns=('name', '# of forks', '# of stars')
+ )
+ df_all = df_all.set_index('name')
+ df_all.to_csv(fname_base + '_all_dependents.csv')
+
+ df_popular = pandas.DataFrame(
+ popular_packages,
+ columns=('name', '# of forks', '# of stars', 'topics')
+ )
+ df_popular = df_popular.set_index('name')
+ df_popular.to_csv(fname_base + '_popular_dependents.csv')
+
+ df_filtered_popular = pandas.DataFrame(
+ popular_filtered_packages,
+ columns=('name', '# of forks', '# of stars', 'topics')
+ )
+ df_filtered_popular = df_filtered_popular.set_index('name')
+ df_filtered_popular.to_csv(fname_base + '_filtered_dependents.csv')
+
+ # print(df_filtered_popular.to_markdown())