#86 Decoupling the scraper from the backend and editing the scraper to make it more versatile with different quarters

backend-py/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get -y install locales && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+
+ENV LANG en_US.UTF-8 
+ENV LANGUAGE en_US:en 
+ENV LC_ALL en_US.UTF-8 
+
+WORKDIR /app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+ENV PYTHONPATH /app
+
+EXPOSE 5000
+
+CMD ["bash", "./docker-run.sh"] 

backend/application.py → backend-py/application.py

@@ -6,6 +6,12 @@
 from flask_caching import Cache
 from backend import generate_class_json, get_all_classes_in, get_departments
 
+import configparser, os
+
+config = configparser.ConfigParser()
+config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), "config", "config.example.ini"))
+DEFAULT_QUARTER = config["DB"]["DEFAULT_QUARTER"]
+
 application = Flask(__name__)
 CORS(application)
 Compress(application)
@@ -25,10 +31,12 @@ def return_db_data():
  classes = request_json['classes']
  ret_classes = {}
 
+ quarter = request_json['quarter'] if 'quarter' in request_json else DEFAULT_QUARTER
+
  for Class in classes:
  department, course_num = Class['department'], Class['courseNum']
  full_name = "{} {}".format(department, course_num)
- ret_classes[full_name] = generate_class_json(department, course_num)
+ ret_classes[full_name] = generate_class_json(department, course_num, quarter)
 
  return jsonify(ret_classes)
 
@@ -44,7 +52,10 @@ def return_department_list():
 @cache.cached(timeout=3600, key_prefix="class_summaries", query_string=True)
 def return_classes():
  department = request.args.get('department')
- classes = get_all_classes_in(department)
+
+ # quarter = request.args.get('quarter') if 'quarter' in request.args else DEFAULT_QUARTER
+ quarter = request.args.get('quarter', DEFAULT_QUARTER, type=str)
+ classes = get_all_classes_in(department, quarter)
  return jsonify(classes)
 
 

backend/backend.py → backend-py/backend.py

@@ -21,17 +21,18 @@
  pool_recycle=3600)
 
 # Caching departments and class types
+# TODO: might have to cache different data for different quarters
 departments = []
 class_types = []
 
 
-def get_all_classes_in(department):
+def get_all_classes_in(department, quarter):
  # Must order this one separately because doing it lexically won't work
  ret_dict = {}
  sql = text(
  "SELECT DISTINCT COURSE_NUM, DEPARTMENT, "
- "INSTRUCTOR, TYPE, DESCRIPTION FROM CLASS_DATA WHERE DEPARTMENT = :department")
- result = cursor.execute(sql, department=department).fetchall()
+ "INSTRUCTOR, TYPE, DESCRIPTION FROM :quarter WHERE DEPARTMENT = :department")
+ result = cursor.execute(sql, quarter=quarter,department=department).fetchall()
  # use dict here for fast lookup
  ret_dict["CLASS_SUMMARY"] = {}
  for row in result:
@@ -52,16 +53,16 @@ def get_departments():
  return departments
 
 
-def generate_class_json(department, course_num):
+def generate_class_json(department, course_num, quarter):
  """
  Generates a set of classes of the same version and ID.
  For example returns all the CSE 20 classes given that ID.
  :param course_num: the course number
  :param department: the department
  :return: returns all the classes with the same ID in a list
  """
- sql = text("SELECT * FROM CLASS_DATA WHERE DEPARTMENT = :department AND COURSE_NUM = :course_num")
- result = cursor.execute(sql, department=department, course_num=course_num).fetchall()
+ sql = text("SELECT * FROM :quarter WHERE DEPARTMENT = :department AND COURSE_NUM = :course_num")
+ result = cursor.execute(sql, quarter=quarter, department=department, course_num=course_num).fetchall()
  class_versions = [dict(row) for row in result]
  # The different sections of the given class
  return class_versions

backend-py/config/config.example.ini

@@ -0,0 +1,12 @@
+[DB]
+USERNAME=dev
+PASSWORD=password
+DB_NAME=classes
+ENDPOINT=
+DEFAULT_QUARTER=SP19
+
+
+[VARS]
+QUARTERS=["SP19"] ; Example: QUARTERS=["SP19", "WI19"]
+
+#sdschedule-database

backend/docker-run.sh → backend-py/docker-run.sh

@@ -1,10 +1,5 @@
 #!/bin/bash
 
-# download data or not
-if [[ $SDSCHEDULE_SCRAPE -eq 1 ]]; then
- python3 -u datautil/webreg_scrape_upload.py
-fi
-
 # uwsgi or flask
 if [[ $ENV == "PROD" ]]; then
  useradd app

backend-py/requirements.txt

@@ -0,0 +1,7 @@
+uwsgi>=2.0.17.1
+SQLAlchemy>=1.2.15
+mysqlclient>=1.3.14
+Flask>=1.0.2
+Flask-Compress>=1.4.0
+Flask-Cors>=3.0.7
+Flask-Caching>=1.4.0

backend/backend.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ "path/filepath"
+ "strconv"
+
+ "gopkg.in/ini.v1"
+
+ "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/environ"
+ "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/route"
+ "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/store"
+)
+
+func readConfig(configFile string) (config *ini.File, err error) {
+ config, err = ini.Load(configFile)
+
+ if err != nil {
+ return nil, fmt.Errorf("Error reading config file: %v", err)
+ }
+
+ if config == nil {
+ return nil, fmt.Errorf("Could not find config file: %v", configFile)
+ }
+
+ return
+}
+
+func main() {
+
+ // config, err := readConfig(filepath.Join(".", "config", "config.example.ini"))
+ config, err := readConfig(filepath.Join(".", "config", "config.dev.ini"))
+
+ if err != nil {
+ panic("Failed to load config: " + err.Error())
+ }
+
+ db, err := store.NewDBConfig(config)
+ if err != nil {
+ panic("Failed to init db: " + err.Error())
+ }
+ defer db.Close()
+
+ env, err := environ.NewEnvConfig(db, config)
+ if err != nil {
+ panic("Failed to init env: " + err.Error())
+ }
+
+ log.Printf("Start server on port: %v\n", env.Port)
+
+ // TODO: use env in handlers
+ // TODO: make a handler factory
+ http.HandleFunc("/api_course_nums", route.MakeHandler(route.GetCourseNums, db, route.LogPrefixCourseNums))
+ http.HandleFunc("/api_departments", route.MakeHandler(route.GetDepartments, db, route.LogPrefixDepartment))
+ http.HandleFunc("/api_instructors", route.MakeHandler(route.GetInstructors, db, route.LogPrefixInstructors))
+ http.HandleFunc("/api_types", route.MakeHandler(route.GetTypes, db, route.LogPrefixTypes))
+ http.HandleFunc("/api_class_data", route.MakeHandler(route.GetClassData, db, route.LogPrefixClassData))
+
+ err = http.ListenAndServe(":"+strconv.Itoa(env.Port), nil)
+
+ if err != nil {
+ log.Panic(err) // non-fatal to run deferred calls
+ }
+}

backend/config/config.dev.ini

@@ -0,0 +1,14 @@
+[BACKEND]
+PORT=8080
+
+[DB]
+USERNAME=dev
+PASSWORD=password
+DB_NAME=classes
+ENDPOINT=localhost:22180
+DEFAULT_QUARTER=SP19
+
+[VARS]
+QUARTERS=["SP19"] ; Example: QUARTERS=["SP19", "WI19"]
+
+#sdschedule-database

backend/config/config.example.ini

@@ -1,4 +1,14 @@
+[BACKEND]
+PORT=8080
+
 [DB]
-USERNAME=root
+USERNAME=splanner
 PASSWORD=password
-ENDPOINT=sdschedule-database
+DB_NAME=classes
+ENDPOINT=
+DEFAULT_QUARTER=SP19
+
+[VARS]
+QUARTERS=["SP19"] ; Example: QUARTERS=["SP19", "WI19"]
+
+#sdschedule-database