#86 Decoupling the scraper from the backend and editing the scraper to make it more versatile with different quarters

backend/Dockerfile

@@ -1,24 +1,33 @@
-FROM joyzoursky/python-chromedriver:3.7
+# build
+# note the consitency of building and running environments
+FROM golang:buster as builder
 
-ENV DEBIAN_FRONTEND noninteractive
+WORKDIR /app
+
+# cache mod packages
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
 
-RUN apt-get update
-RUN apt-get -y install locales 
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG en_US.UTF-8 
-ENV LANGUAGE en_US:en 
-ENV LC_ALL en_US.UTF-8 
+COPY . .
 
-RUN apt-get -y install python3-lxml python3-pip
-RUN apt-get -y install default-libmysqlclient-dev 
+# build stripped binary by default
+ARG BUILD_ARGS="-ldflags '-s -w'"
+RUN go build -v $BUILD_ARGS -o backend
+
+# ship
+FROM debian:buster
 
 WORKDIR /app
-COPY ./requirements.txt /app/requirements.txt
-RUN pip3 install -r requirements.txt
-COPY . /app
 
-ENV PYTHONPATH /app
+# to detect if mysql service is up
+RUN apt-get -q update && apt-get -qy install netcat
+COPY ./wait-for .
+
+# only need config and compiled binary file
+COPY ./config ./config
+COPY --from=builder /app/backend .
 
-EXPOSE 5000
+# handle port exposure in docker compose
 
-CMD ["bash", "./docker-run.sh"] 
+ENTRYPOINT ["/app/backend"]

backend/backend.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ "path/filepath"
+ "strconv"
+
+ "gopkg.in/ini.v1"
+
+ "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/environ"
+ "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/route"
+ "github.com/ucsdscheduleplanner/UCSD-Schedule-Planner/backend/store"
+)
+
+// list of config files to search in order
+var configFiles = []string{
+ filepath.Join(".", "config", "config.ini"),
+ filepath.Join(".", "config", "config.example.ini"),
+}
+
+// Config for the backend
+type Config struct {
+ File *ini.File
+ Port int
+}
+
+// return the first successful read
+// return error of last failed
+func readConfig() (c *Config, e error) {
+ for _, f := range configFiles {
+ if c, e = doReadConfig(f); e == nil {
+ break
+ }
+ }
+ return
+}
+
+func doReadConfig(configFile string) (*Config, error) {
+
+ config, err := ini.Load(configFile)
+
+ if err != nil {
+ return nil, fmt.Errorf("Error reading config file: %v", err)
+ }
+
+ if config == nil {
+ return nil, fmt.Errorf("Could not find config file: %v", configFile)
+ }
+
+ port, err := config.Section("BACKEND").Key("PORT").Int()
+ if err != nil {
+ return nil, fmt.Errorf("Error reading port number: %v", err.Error())
+ }
+
+ return &Config{File: config, Port: port}, nil
+}
+
+func main() {
+
+ config, err := readConfig()
+
+ if err != nil {
+ panic("Failed to load config: " + err.Error())
+ }
+
+ db, err := store.NewDBConfig(config.File)
+ if err != nil {
+ panic("Failed to init db: " + err.Error())
+ }
+ defer db.Close()
+
+ env, err := environ.NewEnvConfig(db, config.File)
+ if err != nil {
+ panic("Failed to init env: " + err.Error())
+ }
+
+ log.Printf("Start server on port: %v\n", config.Port)
+
+ routeFactory := route.HandlerFactory{Env: env}
+
+ http.HandleFunc("/api_course_nums", routeFactory.MakeHandler(route.HandlerCourseNum))
+ http.HandleFunc("/api_departments", routeFactory.MakeHandler(route.HandlerDepartment))
+ http.HandleFunc("/api_instructors", routeFactory.MakeHandler(route.HandlerInstructors))
+ http.HandleFunc("/api_types", routeFactory.MakeHandler(route.HandlerTypes))
+ http.HandleFunc("/api_class_data", routeFactory.MakeHandler(route.HandlerClassData))
+
+ err = http.ListenAndServe(":"+strconv.Itoa(config.Port), nil)
+
+ if err != nil {
+ log.Panic(err) // non-fatal to run deferred calls
+ }
+}

backend/config/config.dev.ini

@@ -0,0 +1,14 @@
+[BACKEND]
+PORT=8080
+
+[DB]
+USERNAME=dev
+PASSWORD=password
+DB_NAME=classes
+ENDPOINT=localhost:22180
+DEFAULT_QUARTER=SP19
+
+[VARS]
+QUARTERS=["SP19"] ; Example: QUARTERS=["SP19", "WI19"]
+
+#sdschedule-database

backend/config/config.example.ini

@@ -1,4 +1,12 @@
+[BACKEND]
+PORT=8080
+
 [DB]
-USERNAME=root
+USERNAME=splanner
 PASSWORD=password
-ENDPOINT=sdschedule-database
+DB_NAME=classes
+ENDPOINT=
+DEFAULT_QUARTER=SP19
+
+[VARS]
+QUARTERS=["SP19"] ; Example: QUARTERS=["SP19", "WI19"]