#86 Decoupling the scraper from the backend and editing the scraper to make it more versatile with different quarters

backend/backend.go

@@ -12,15 +12,20 @@ import (
  "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/routes"
 )
 
+// TODO: make this config-able
 const port = 8080
 
+// create closure for http handler func
 func makeHandler(f func(http.ResponseWriter, *http.Request, *db.DatabaseStruct), ds *db.DatabaseStruct) http.HandlerFunc {
  return func(w http.ResponseWriter, r *http.Request) { f(w, r, ds) }
 }
 
 func main() {
+
  var config, err = ini.Load(filepath.Join(".", "config", "config.example.ini"))
 
+ // var config, err = ini.Load(filepath.Join(".", "config", "config.dev.ini"))
+
  if err != nil {
  panic("Error reading config file: " + err.Error())
  }
@@ -35,6 +40,7 @@ func main() {
  panic("Failed to init db: " + err.Error())
  }
 
+ // only for the completeness of the code
  defer ds.Close()
 
  log.Printf("Starting server on port: %v\n", port)
@@ -45,6 +51,9 @@ func main() {
  http.HandleFunc("/api_instructors", makeHandler(routes.GetInstructors, ds))
  http.HandleFunc("/api_types", makeHandler(routes.GetTypes, ds))
 
- // close db no matter what
- log.Panic(http.ListenAndServe(":"+strconv.Itoa(port), nil))
+ // not fatal to close db (defer above)
+ err = http.ListenAndServe(":"+strconv.Itoa(port), nil)
+ if err != nil {
+ log.Panic(err)
+ }
 }

backend/db/db.go

@@ -1,6 +1,10 @@
 // Package db provides a simple way to communicate with database
 package db
 
+// TODO: if necessary, covert the db struct to a Env struct
+// that stores environment of the whole server, e.g. *DB, port, hostname, config, etc.
+// Env will be constructed in main() and passed to all other handlers
+
 import (
  "database/sql"
  "encoding/json"
@@ -36,7 +40,17 @@ func New(config *ini.File) (*DatabaseStruct, error) {
 
  tableNames = append(tableNames, "DEPARTMENT")
 
- db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s%s", username, password, endpoint, databaseName))
+ // TODEL: must delete, for back compatibility temporarily
+ tableNames = append(tableNames, "CLASS_DATA")
+
+ db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s", username, password, endpoint, databaseName))
+ // db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s%s", username, password, endpoint, databaseName))
+
+ if err != nil {
+ return nil, err
+ }
+
+ err = db.Ping() // validate the connection
 
  if err != nil {
  return nil, err
@@ -61,14 +75,18 @@ func (ds *DatabaseStruct) Close() {
 
 // Query using the input SQL query
 func (ds *DatabaseStruct) Query(tableName string, sqlQuery string, params ...interface{}) (*sql.Rows, error) {
+
  if !ds.isValidTable(tableName) {
  return nil, fmt.Errorf("Table name '%s' is not valid, cannot continue query", tableName)
  }
+
  // Query will creates a connection and automatically release it
  // ref: https://golang.org/src/database/sql/sql.go?s=40984:41081#L1522
  results, err := ds.db.Query(sqlQuery, params...)
+
  if err != nil {
  return nil, errors.New(err.Error())
  }
+
  return results, nil
 }

backend/routes/ClassData.go

@@ -5,16 +5,14 @@ import (
  "encoding/json"
  "fmt"
  "io/ioutil"
- "log"
  "net/http"
 
  "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/db"
 )
 
-// TODO create some error checking functions to handle duplicated error checking codes
 const logTagClassData = "[ClassData]"
 
-// TODO make this graceful
+// TODO: make this graceful, maybe an Env var or config
 // set default quarter to SP19
 const defaultQuarter = "SP19"
 
@@ -58,32 +56,31 @@ type Subclass struct {
  Description string `json:"description"`
 }
 
-// GetClassData is a pre-http.HandlerFunc for class data route ready to be a closure with *DatabaseStruct
+// GetClassData is a pre-http.HandlerFunc for class data route and will become a closure with *DatabaseStruct
 func GetClassData(writer http.ResponseWriter, request *http.Request, ds *db.DatabaseStruct) {
 
  if request.Method != "POST" {
- http.Error(writer, "Unsupported request type", http.StatusMethodNotAllowed)
+ errInvalidMethod(logTagClassData, writer, request)
  return
  }
 
  body, err := ioutil.ReadAll(request.Body)
 
  if err != nil {
- http.Error(writer, "Error handling POST request", http.StatusBadRequest)
- log.Printf("%s Failed to read request to %q from %q: %s", logTagClassData, request.RequestURI, request.RemoteAddr, err.Error())
+ errRequestBodyPost(logTagClassData, err, writer, request)
  return
  }
 
  var classesToQuery []ClassDataRequest
  err = json.Unmarshal(body, &classesToQuery)
 
  if err != nil {
- http.Error(writer, "Failed to parse request", http.StatusInternalServerError)
- log.Printf("%s Failed to parse request to %q from %q: %s", logTagClassData, request.RequestURI, request.RemoteAddr, err.Error())
+ errRequestBodyParse(logTagClassData, err, writer, request)
  return
  }
 
  // TODO: reduce to one query?
+
  ret := make(map[string][]Subclass)
  for i := 0; i < len(classesToQuery); i++ {
  currentClass := classesToQuery[i]
@@ -92,16 +89,12 @@ func GetClassData(writer http.ResponseWriter, request *http.Request, ds *db.Data
  rows, err := ds.Query(currentClass.quarter, query, currentClass.department, currentClass.courseNumber)
 
  if err != nil {
- http.Error(writer, "Error querying for class data", http.StatusInternalServerError)
- // TODO think about better ways to log the query, and fix all the other such logs in other files
- log.Printf("%s Failed to query data with %q from %q: %s", logTagClassData, query+" "+currentClass.department+" "+currentClass.courseNumber, request.RemoteAddr, err.Error())
+ errQuery(logTagClassData, err, writer, request, query, currentClass.department, currentClass.courseNumber)
  return
  }
 
- // TODO this might not be an error, better error message?
  if rows == nil {
- http.Error(writer, "Error querying for class data", http.StatusInternalServerError)
- log.Printf("%s Empty query data with %q from %q: %s", logTagClassData, query+" "+currentClass.department+" "+currentClass.courseNumber, request.RemoteAddr, err.Error())
+ errEmptyQuery(logTagClassData, writer, request, query, currentClass.department, currentClass.courseNumber)
  return
  }
 
@@ -122,8 +115,7 @@ func GetClassData(writer http.ResponseWriter, request *http.Request, ds *db.Data
  &subClass.Description)
 
  if err != nil {
- http.Error(writer, "Error retrieving data", http.StatusInternalServerError)
- log.Printf("%s Failed to query data with %q from %q: %s", logTagClassData, query+" "+currentClass.department+" "+currentClass.courseNumber, request.RemoteAddr, err.Error())
+ errParseQueryResult(logTagClassData, err, writer, request)
  return
  }
 
@@ -133,18 +125,16 @@ func GetClassData(writer http.ResponseWriter, request *http.Request, ds *db.Data
 
  retJSON, err := json.Marshal(ret)
 
- // TODO use a function to deal with same error checks for the routes
  if err != nil {
- http.Error(writer, "Error retrieving data", http.StatusInternalServerError)
- log.Printf("%s Failed to read data from ret in response to %q: %s", logTagClassData, request.RemoteAddr, err.Error())
+ errCreateResponse(logTagClassData, err, writer, request)
  return
  }
 
- _, err = writer.Write(retJSON)
+ status, err := writer.Write(retJSON)
 
  if err != nil {
- http.Error(writer, "Failed to send response", http.StatusInternalServerError)
- log.Printf("%s Failed to write data in response to %q: %s", logTagClassData, request.RemoteAddr, err.Error()) // log JSON?
+ errWriteResponse(logTagClassData, err, writer, request, status)
  return
  }
+
 }

backend/routes/Department.go

@@ -1,65 +1,28 @@
 package routes
 
 import (
- "encoding/json"
- "log"
+ "database/sql"
  "net/http"
 
  "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/db"
 )
 
 const logTagDepartment = "[Department]"
 
-// GetDepartments is a pre-http.HandlerFunc for department route ready to be a closure with *DatabaseStruct
+// GetDepartments is a pre-http.HandlerFunc for department route and will become a closure with *DatabaseStruct
 func GetDepartments(writer http.ResponseWriter, request *http.Request, ds *db.DatabaseStruct) {
 
  if request.Method != "GET" {
- http.Error(writer, "Method not allowed", http.StatusMethodNotAllowed)
+ errInvalidMethod(logTagDepartment, writer, request)
  return
  }
 
- rows, err := ds.Query("DEPARTMENT", "SELECT DISTINCT DEPT_CODE FROM DEPARTMENT")
+ queryAndResponse(ds, logTagDepartment, writer, request,
+ func(rows *sql.Rows) (interface{}, error) {
+ var val string
+ err := rows.Scan(&val)
+ return val, err
+ },
+ "SELECT DISTINCT DEPT_CODE FROM DEPARTMENT", "DEPARTMENT")
 
- if err != nil {
- http.Error(writer, "Error querying departments", http.StatusInternalServerError)
- log.Printf("%s Failed to query data with %q from %q: %s", logTagDepartment, "SELECT DISTINCT DEPT_CODE FROM DEPARTMENT", request.RemoteAddr, err.Error())
- return
- }
-
- if rows == nil {
- http.Error(writer, "Error querying departments", http.StatusInternalServerError)
- log.Printf("%s Empty query data with %q from %q: %s", logTagDepartment, "SELECT DISTINCT DEPT_CODE FROM DEPARTMENT", request.RemoteAddr, err.Error())
- return
- }
-
- var departments []string
- for rows.Next() {
- var (
- department string
- )
- err := rows.Scan(&department)
- if err != nil {
- http.Error(writer, "Error parsing departments", http.StatusInternalServerError)
- log.Printf("%s Failed to parse departments: %s", logTagDepartment, err.Error())
- return
- }
-
- departments = append(departments, department)
- }
-
- departmentJSON, err := json.Marshal(departments)
-
- if err != nil {
- http.Error(writer, "Error fetching departments", http.StatusInternalServerError)
- log.Printf("%s Failed to read data from departments in response to %q: %s", logTagDepartment, request.RemoteAddr, err.Error())
- return
- }
-
- _, err = writer.Write(departmentJSON)
-
- if err != nil {
- http.Error(writer, "Failed to send response", http.StatusInternalServerError)
- log.Printf("%s Failed to write data in response to %q: %s", logTagDepartment, request.RemoteAddr, err.Error()) // log JSON?
- return
- }
 }

backend/routes/DepartmentSummary.go

@@ -1,8 +1,7 @@
 package routes
 
 import (
- "encoding/json"
- "log"
+ "database/sql"
  "net/http"
 
  "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/db"
@@ -17,74 +16,30 @@ type DepartmentSummary struct {
  Description string `json:"description"`
 }
 
-// GetCourseNums is a pre-http.HandlerFunc for course num route ready to be a closure with *DatabaseStruct
+// GetCourseNums is a pre-http.HandlerFunc for course num route ready and will become a closure with *DatabaseStruct
 func GetCourseNums(writer http.ResponseWriter, request *http.Request, ds *db.DatabaseStruct) {
- if request.Method != "GET" {
- http.Error(writer, "Invalid method type", http.StatusMethodNotAllowed)
- return
- }
 
- // TODO create a function for query value check
- keys, ok := request.URL.Query()["department"]
- department := keys[0]
-
- if !ok {
- http.Error(writer, "Request does not contain department!", http.StatusBadRequest)
- // not logging invalid input yet
+ if request.Method != "GET" {
+ errInvalidMethod(logTagDepartmentSummary, writer, request)
  return
  }
 
- keys, ok = request.URL.Query()["quarter"]
- quarter := keys[0]
+ // NOTE: hack way to re-use function
+ department, _, quarter, missing := readDeptCourseNumQuarter(request)
 
- if !ok {
- http.Error(writer, "Request does not contain department!", http.StatusBadRequest)
- // not logging invalid input yet
+ if missing != "courseNum" {
+ errMissingInput(logTagInstructors, writer, request, missing)
  return
  }
 
  query := "SELECT DISTINCT DEPARTMENT, COURSE_NUM, DESCRIPTION FROM " + quarter + " WHERE DEPARTMENT=?"
- rows, err := ds.Query(quarter, query, department)
-
- if err != nil {
- http.Error(writer, "Error querying department summary", http.StatusInternalServerError)
- log.Printf("%s Failed to query data with %q from %q: %s", logTagDepartmentSummary, query, request.RemoteAddr, err.Error())
- return
- }
-
- if rows == nil {
- http.Error(writer, "Could not query correctly.", http.StatusInternalServerError)
- log.Printf("%s Failed to query data with %q from %q: %s", logTagDepartmentSummary, query, request.RemoteAddr, err.Error())
- return
- }
 
- var ret []DepartmentSummary
+ queryAndResponse(ds, logTagDepartmentSummary, writer, request,
+ func(rows *sql.Rows) (interface{}, error) {
+ val := DepartmentSummary{}
+ err := rows.Scan(&val.Department, &val.CourseNum, &val.Description)
+ return val, err
+ },
+ query, quarter, department)
 
- for rows.Next() {
- departmentSummary := DepartmentSummary{}
- err := rows.Scan(&departmentSummary.Department, &departmentSummary.CourseNum, &departmentSummary.Description)
-
- if err != nil {
- http.Error(writer, "Could not scan data into struct", http.StatusInternalServerError)
- log.Printf("%s Failed to parse department summary: %s", logTagDepartmentSummary, err.Error())
- return
- }
-
- ret = append(ret, departmentSummary)
- }
-
- retJSON, err := json.Marshal(ret)
-
- if err != nil {
- http.Error(writer, "Error converting data", http.StatusInternalServerError)
- log.Printf("%s Failed to read data from departments in response to %q: %s", logTagDepartmentSummary, request.RemoteAddr, err.Error())
- return
- }
-
- _, err = writer.Write(retJSON)
- if err != nil {
- http.Error(writer, "Failed to send response", http.StatusInternalServerError)
- log.Printf("%s Failed to write data in response to %q: %s", logTagDepartmentSummary, request.RemoteAddr, err.Error()) // log JSON?
- return
- }
 }