Added git services, db caching and front end API changes

.eslintrc.json

@@ -16,6 +16,7 @@
  "no-console": "off",
  "func-names": "off",
  "no-alert": "off",
- "prefer-const": "off"
+ "prefer-const": ["error", {"destructuring": "all"}],
+ "no-underscore-dangle": ["error", { "allow": ["_json"] }]
  }
 }

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .vscode
-config/.dev.env
+config/.dev.env
+filelog-debug.log

config/winston.js

@@ -0,0 +1,24 @@
+const winston = require('winston');
+
+const tsFormat = () => (new Date()).toLocaleTimeString();
+
+const logger = new (winston.Logger)({
+ transports: [
+ // colorize the output to the console
+ new (winston.transports.Console)({
+ timestamp: tsFormat,
+ colorize: true,
+ }),
+ new (winston.transports.File)({
+ name: 'debug-file',
+ filename: 'filelog-debug.log',
+ level: 'debug',
+ }),
+ ],
+});
+
+logger.level = 'info';
+
+// logger.debug('Test Log Message', { anything: 'This is metadata' });
+
+module.exports.logger = logger;

package.json

@@ -28,7 +28,10 @@
  "express-session": "^1.15.3",
  "mongodb": "^2.2.27",
  "mongoose": "^4.10.4",
+ "mongoose-paginate": "^5.0.3",
  "passport": "^0.3.2",
- "passport-github2": "^0.1.10"
+ "passport-github2": "^0.1.10",
+ "request": "^2.81.0",
+ "winston": "^2.3.1"
  }
 }

public/js/languageSelection.js

@@ -42,7 +42,7 @@ const selectLanguage = function selectLanguage(event) {
  languageSelected = event.target.value;
  content.innerHTML = '';
  reqNo = Math.floor(Math.random() * 3) + 1;
- projectsPerPage = (languageSelected == ANY_LANGUAGE) ? 2 : 100;
+ projectsPerPage = (languageSelected == ANY_LANGUAGE) ? 2 : 10;
  getData();
 };
 

public/js/main.js

@@ -17,8 +17,12 @@ const moreDataNeeded = function moreDataNeeded() {
  return ((allUsersChecked()) && (projectsCurrentCall < MIN_PROJECTS_PER_CALL));
 };
 
-const userFormatter = function userFormatter(username) {
- return `<a href='https://github.com/${username}?tab=stars'>${username}</a>`;
+const userFormatter = function userFormatter(usernames) {
+ let usernameStr = '';
+ for (username of usernames) {
+ usernameStr += `<a href='https://github.com/${username}?tab=stars'>${username}</a>, `;
+ }
+ return usernameStr.slice(0, -2);
 };
 
 const nFormatter = function nFormatter(num) {
@@ -40,9 +44,10 @@ const dataCollector = function dataCollector(response, username) {
  let innerContent = `<li><span class='link'><a href='${entry.html_url}' target='_blank'>${entry.name}<span> - ${String(entry.description)}</span><br/></a></span>`;
  innerContent += "<div class='additional'>";
  innerContent += `${nFormatter(entry.stargazers_count)} <i class='fa fa-star'></i>`;
- innerContent += `&emsp;${nFormatter(entry.forks)} <i class='fa fa-code-fork'></i>`;
+ innerContent += `&emsp;${nFormatter(entry.forks_count)} <i class='fa fa-code-fork'></i>`;
  innerContent += (entry.language != null) ? `&emsp;${entry.language}` : '';
- innerContent += `&emsp;(from ${userFormatter(username)})`;
+ // innerContent += `&emsp;(from ${userFormatter(username)})`;
+ innerContent += `&emsp;(from ${userFormatter(entry.stargazersLogin)})`;
  innerContent += '</div></li>';
  innerContent = EMOJI.replace_unified(innerContent);
  CONTENT.innerHTML += EMOJI.replace_colons(innerContent);
@@ -63,8 +68,12 @@ const getData = function getData() {
  usersCurrentCall = 0;
  callInProgress = true;
  reqNo += 1;
+ if (reqNo > 10) {
+ return console.log('rejecting request', reqNo);
+ }
  USERNAMES.forEach((username) => {
- const url = `https://api.github.com/users/${username}/starred?per_page=${projectsPerPage}&access_token=${accessToken}&page=${reqNo}`;
+ // const url = `https://api.github.com/users/${username}/starred?per_page=${projectsPerPage}&access_token=${accessToken}&page=${reqNo}`;
+ const url = `http://localhost:3000/api/repos/v1/search?stargazer=${username}&language=${languageSelected}&per_page=${projectsPerPage}&page=${reqNo}`;
  axios({
  url,
  method: 'get',

server/app.js

@@ -3,7 +3,8 @@ require('./../config/config');
 const express = require('express');
 const session = require('express-session');
 const { mongoose } = require('./db/mongoose');
-const { usernameRoutes } = require('./routes/api/username');
+const { stargazersRoutes } = require('./routes/api/stargazers');
+const { reposRoutes } = require('./routes/api/repos');
 const { userRoutes } = require('./routes/user');
 const { passport } = require('./auth');
 
@@ -19,7 +20,8 @@ app.use(passport.session());
 app.use(express.static(`${__dirname}/../public`));
 
 app.use('/user',userRoutes);
-app.use('/api/username', usernameRoutes);
+app.use('/api/stargazers', stargazersRoutes);
+app.use('/api/repos', reposRoutes);
 
 app.listen(port, () => {
  console.log(`Starting server on port ${port}.`);

server/db/mongoose.js

@@ -1,4 +1,4 @@
-let mongoose = require('mongoose');
+const mongoose = require('mongoose');
 
 mongoose.Promise = global.Promise;
 

server/db/repositories.js

@@ -1,10 +1,11 @@
 // Repositories
 const mongoose = require('./mongoose');
+const mongoosePaginate = require('mongoose-paginate');
 
 const Schema = mongoose.Schema;
 
 const repositorySchema = new Schema({
- _id: mongoose.Schema.Types.ObjectId,
+ githubId: Number,
  name: String,
  html_url: String,
  description: String,
@@ -13,8 +14,39 @@ const repositorySchema = new Schema({
  created_at: Date,
  updated_at: Date,
  language: String,
-});
+ stargazersLogin: [String],
+}, { timestamps: { updatedAt: 'recordUpdated_at', createdAt: 'recordCreated_at' } });
 
+repositorySchema.statics.updateRepoWithStargazer = function (repo, stargazer) {
+ const Repository = this;
+
+ return new Promise((resolve, reject) => {
+ Repository.findOne({ githubId: repo.githubId })
+ .then((repository) => {
+ if (repository) {
+ if (repository.stargazersLogin.indexOf(stargazer) === -1) {
+ repository.stargazersLogin.push(stargazer);
+ }
+ Object.assign(repository, repo);
+ repository.save((error, doc) => {
+ if (error) throw error;
+ //console.log(`Updated: ${repo.name} from: ${stargazer}`);
+ });
+ } else {
+ repository = new Repository(repo);
+ repository.stargazersLogin.push(stargazer);
+ repository.save((error, doc) => {
+ if (error) throw error;
+ //console.log(`Added: ${repo.name} from: ${stargazer}`);
+ });
+ }
+ resolve(repository);
+ })
+ .catch(e => reject(e));
+ });
+};
+
+repositorySchema.plugin(mongoosePaginate);
 
 const Repository = mongoose.model('Repository', repositorySchema);
 

server/db/stargazers.js

@@ -0,0 +1,21 @@
+// Schema of stargazers in Github
+const mongoose = require('./mongoose');
+
+const Schema = mongoose.Schema;
+
+const stargazersSchema = new Schema({
+ githubId: Number,
+ login: String,
+ name: String,
+ html_url: String,
+ location: String,
+ bio: String,
+ public_repos: Number,
+ public_gists: Number,
+ followers: Number,
+}, { timestamps: { updatedAt: 'recordUpdated_at', createdAt: 'recordCreated_at' } });
+
+
+const Stargazers = mongoose.model('Stargazers', stargazersSchema);
+
+module.exports.Stargazers = Stargazers;

server/db/user.js

@@ -19,6 +19,10 @@ userSchema.statics.findOrCreate = function (profile, accessToken) {
  if (user) {
  // TODO:Need to update accessToken for this user
  user.accessToken = accessToken;
+ user.save((err) => {
+ if (err) return handleError(err);
+ // saved!
+ });
  return Promise.resolve(user);
  }
  // Create new user

server/gitUtility/gitService.js

@@ -0,0 +1,89 @@
+const request = require('request');
+const { logger } = require('./../../config/winston');
+
+const gitService = {
+
+ getUserDetails(login, token) {
+ if (token) {
+ headers = {
+ 'User-Agent': 'request',
+ Authorization: `token ${token}`,
+ };
+ } else {
+ headers = {
+ 'User-Agent': 'request',
+ };
+ }
+ const options = {
+ url: `https://api.github.com/users/${login}`,
+ headers,
+ };
+ return new Promise((resolve, reject) => {
+ logger.info('Calling api.github.com/users/user', { login });
+ request(options, (error, response, body) => {
+ if (!error && response.statusCode == 200) {
+ const user = JSON.parse(body);
+ const stargazer = {
+ githubId: user.id,
+ login: user.login,
+ name: user.name,
+ html_url: user.html_url,
+ location: user.location,
+ bio: user.bio,
+ public_repos: user.public_repos,
+ public_gists: user.public_gists,
+ followers: user.followers,
+ };
+ resolve(stargazer);
+ } else {
+ logger.warn('Error api.github.com/users/user', { Error: response.body });
+ reject(response.body);
+ }
+ });
+ });
+ },
+
+ getStarredRepository(login, token, page = 1) {
+ if (token) {
+ headers = {
+ 'User-Agent': 'request',
+ Authorization: `token ${token}`,
+ };
+ } else {
+ headers = {
+ 'User-Agent': 'request',
+ };
+ }
+ const options = {
+ url: `https://api.github.com/users/${login}/starred?per_page=100&page=${page}`,
+ headers,
+ };
+ return new Promise((resolve, reject) => {
+ logger.info('Calling api.github.com/users/login/starred', { login, page });
+ request(options, (error, response, body) => {
+ if (!error && response.statusCode == 200) {
+ const repos = JSON.parse(body);
+ // Only select required info from data received.
+ const filterRepos = repos.map(repo => ({
+ githubId: repo.id,
+ name: repo.name,
+ html_url: repo.html_url,
+ description: repo.description,
+ stargazers_count: repo.stargazers_count,
+ forks_count: repo.forks_count,
+ created_at: repo.created_at,
+ updated_at: repo.updated_at,
+ language: repo.language,
+ }));
+ logger.debug('Data Received:', { login, length: filterRepos.length, currentPage: page });
+ resolve(filterRepos);
+ } else {
+ logger.warn('Error api.github.com/users/login/starred', { err: response.body });
+ reject(response.body);
+ }
+ });
+ });
+ },
+};
+
+module.exports.gitService = gitService;

server/gitUtility/initialSeed.js

@@ -0,0 +1,54 @@
+require('./../../config/config');
+const { gitService } = require('./gitService');
+const { Stargazers } = require('./../db/stargazers');
+const { Repository } = require('./../db/repositories');
+const { stargazerList } = require('./stargazerList');
+const { logger } = require('./../../config/winston');
+
+// stargazerList = ['Kureev', 'yyx990803'];
+
+function seedStargazerRecords(token) {
+ // Get List from stargazerList.js and update records for stargazer.
+ for (const stargazer of stargazerList) {
+ logger.info(`*****Getting Record for stargazer:${stargazer}*****`);
+ gitService.getUserDetails(stargazer, token).then((user) => {
+ Stargazers.findOneAndUpdate({ login: user.login }, user,
+ { new: true, upsert: true },
+ (error, doc) => {
+ if (error) throw error;
+ //logger.debug(`Updated: ${stargazer}`);
+ });
+ }).catch(e => logger.error(e));
+ }
+ logger.info('*_*_*_*_*Stargazer collection is updated.*_*_*_*_*');
+}
+
+function seedRepositoryRecords(token) {
+ // Get List from stargazerList.js and update records for Starred Repos.
+ for (const stargazer of stargazerList) {
+ logger.info(`*****Getting Starred Repos of:${stargazer}*****`);
+ getRepositoryRecord(stargazer, token);
+ }
+}
+
+function getRepositoryRecord(stargazer, token, page = 1) {
+ gitService.getStarredRepository(stargazer, token, page).then((repos) => {
+ if (repos.length === 100) {
+ // If repos.length is 100 there might be another page of data to calling for next page.
+ page += 1;
+ getRepositoryRecord(stargazer, token, page);
+ }
+ for (const repo of repos) {
+ // Calling static method of Repository modal to store repos and update stargazersLogin array.
+ Repository.updateRepoWithStargazer(repo, stargazer)
+ .catch(e => console.log(e));
+ }
+ });
+}
+
+updateRecords = function updateRecords(token) {
+ seedStargazerRecords(token);
+ seedRepositoryRecords(token);
+};
+
+module.exports.updateRecords = updateRecords;