[#12920] Create script to migrate noSQL test data to SQL schema format (

#12922)

* Add classes to migrate test json data

* Add toposort  script

* Add function to remove foreign key data

* Cleanup

* WIP

* Simplify keys for students and instructors

* Fix lint issues

* Output SQL JSON in same folder as JSON

* Change output file name

* Fix bug: wrong jsonkey used

* Fix lint error

* Make section and team name unique

* Set read notification key to be unique

* Delete python file

src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java

@@ -0,0 +1,228 @@
+package teammates.client.scripts.testdataconversion;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FilenameUtils;
+
+import com.google.gson.JsonObject;
+
+import teammates.common.datatransfer.DataBundle;
+import teammates.common.datatransfer.SqlDataBundle;
+import teammates.common.exception.InvalidParametersException;
+import teammates.common.util.JsonUtils;
+import teammates.storage.sqlentity.Account;
+import teammates.storage.sqlentity.AccountRequest;
+import teammates.storage.sqlentity.Course;
+import teammates.storage.sqlentity.DeadlineExtension;
+import teammates.storage.sqlentity.FeedbackQuestion;
+import teammates.storage.sqlentity.FeedbackResponse;
+import teammates.storage.sqlentity.FeedbackResponseComment;
+import teammates.storage.sqlentity.FeedbackSession;
+import teammates.storage.sqlentity.Instructor;
+import teammates.storage.sqlentity.Notification;
+import teammates.storage.sqlentity.ReadNotification;
+import teammates.storage.sqlentity.Section;
+import teammates.storage.sqlentity.Student;
+import teammates.storage.sqlentity.Team;
+import teammates.test.FileHelper;
+
+/**
+ * Class to create JSON test data in SQL format from a noSQL JSON file.
+ * File can be run using the gradle execScript task and accepts a single argument which is the JSON path
+ * ./gradlew execScript -PuserScript="testdataconversion/ConvertDatastoreJsonToSqlJson" --args="JSON_FILE_PATH_HERE"
+ */
+public class ConvertDatastoreJsonToSqlJson {
+ private DataStoreToSqlConverter entityConverter;
+ private DataBundle dataStoreBundle;
+ private SqlDataBundle sqlDataBundle;
+
+ private String[] entitiesReferencedForeignKeys = new String[] {
+ "course",
+ "feedbackSession",
+ "section",
+ "account",
+ "giverSection",
+ "recipientSection",
+ "notification"};
+
+ protected ConvertDatastoreJsonToSqlJson(File inputFile) throws IOException {
+ this.entityConverter = new DataStoreToSqlConverter();
+
+ this.dataStoreBundle = loadDataBundle(inputFile.getCanonicalPath());
+ }
+
+ private String removeWhitespace(String string) {
+ return string.replaceAll("\\s", "");
+ }
+
+ private DataBundle loadDataBundle(String pathToJsonFile) throws IOException {
+ String jsonString = FileHelper.readFile(pathToJsonFile);
+ return JsonUtils.fromJson(jsonString, DataBundle.class);
+ }
+
+ private void saveFile(String filePath, String content) throws IOException {
+ FileHelper.saveFile(filePath, content);
+ System.out.println(filePath + " created!");
+ }
+
+ /**
+ * Amends foreign key references to only have ID field.
+ */
+ private void removeForeignKeyData(JsonObject obj) {
+ for (String entityName : entitiesReferencedForeignKeys) {
+ if (obj.get(entityName) != null) {
+ JsonObject entity = obj.get(entityName).getAsJsonObject();
+ for (String field : entity.deepCopy().keySet()) {
+ if (!"id".equals(field)) {
+ entity.remove(field);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Read datstore json file and creates a SQL equivalent.
+ */
+ private void createSqlJson(File outputFile) throws IOException, InvalidParametersException {
+ sqlDataBundle = new SqlDataBundle();
+
+ migrateIndepedentEntities();
+ migrateDependentEntities();
+
+ // Iterates through all entities in JSON file and removes foreign entitity data except its ID
+ JsonObject sqlJsonString = JsonUtils.toJsonObject(sqlDataBundle);
+ for (String entityCollectionName : sqlJsonString.keySet()) {
+ JsonObject entityCollection = sqlJsonString.get(entityCollectionName).getAsJsonObject();
+ for (String entityName : entityCollection.getAsJsonObject().keySet()) {
+ JsonObject entity = entityCollection.get(entityName).getAsJsonObject();
+ removeForeignKeyData(entity);
+ }
+ }
+
+ String jsonString = JsonUtils.toJson(sqlJsonString);
+ saveFile(outputFile.getCanonicalPath(), jsonString + System.lineSeparator());
+ }
+
+ /**
+ * Migrate entities with no foreign key reference.
+ * Entities are account requests, usage statistics, courses, accouns, notifications
+ */
+ private void migrateIndepedentEntities() {
+ assert sqlDataBundle != null;
+
+ dataStoreBundle.accounts.forEach((k, datastoreAccount) -> {
+ Account sqlAccount = entityConverter.convert(datastoreAccount);
+ sqlDataBundle.accounts.put(k, sqlAccount);
+ });
+
+ dataStoreBundle.courses.forEach((k, datastoreCourse) -> {
+ Course sqlCourse = entityConverter.convert(datastoreCourse);
+ sqlDataBundle.courses.put(k, sqlCourse);
+ });
+
+ dataStoreBundle.accountRequests.forEach((k, accountRequest) -> {
+ AccountRequest sqlAccountRequest = entityConverter.convert(accountRequest);
+ sqlDataBundle.accountRequests.put(k, sqlAccountRequest);
+ });
+
+ dataStoreBundle.notifications.forEach((k, notification) -> {
+ Notification sqlNotification = entityConverter.convert(notification);
+ sqlDataBundle.notifications.put(k, sqlNotification);
+ });
+ }
+
+ /**
+ * Migrate entities which have dependence on each other or on the independent entities.
+ * The order which the entities were migrated was generated using a topological sort
+ * of its foreign key dependencies.
+ * Dependent entities: feedback sessions, sections, teams, users, students, instructors,
+ * deadline extensions, feedback questions, read notifications,
+ * feedback responses and feedback response comments.
+ */
+ private void migrateDependentEntities() {
+
+ dataStoreBundle.feedbackSessions.forEach((k, feedbackSession) -> {
+ FeedbackSession sqlFeedbackSession = entityConverter.convert(feedbackSession);
+ sqlDataBundle.feedbackSessions.put(k, sqlFeedbackSession);
+ });
+
+ dataStoreBundle.students.forEach((k, student) -> {
+ String jsonKey = removeWhitespace(String.format("%s-%s",
+ student.getCourse(), student.getSection()));
+
+ if (!sqlDataBundle.sections.containsKey(jsonKey)) {
+ Section sqlSection = entityConverter.createSection(student);
+ sqlDataBundle.sections.put(jsonKey, sqlSection);
+ }
+ });
+
+ dataStoreBundle.students.forEach((k, student) -> {
+ String jsonKey = removeWhitespace(String.format("%s-%s-%s",
+ student.getCourse(), student.getSection(), student.getTeam()));
+
+ if (!sqlDataBundle.teams.containsKey(jsonKey)) {
+ Team sqlTeam = entityConverter.createTeam(student);
+ sqlDataBundle.teams.put(jsonKey, sqlTeam);
+ }
+ });
+
+ dataStoreBundle.instructors.forEach((k, instructor) -> {
+ Instructor sqlInstructor = entityConverter.convert(instructor);
+ sqlDataBundle.instructors.put(k, sqlInstructor);
+ });
+
+ dataStoreBundle.students.forEach((k, student) -> {
+ Student sqlStudent = entityConverter.convert(student);
+ sqlDataBundle.students.put(k, sqlStudent);
+ });
+
+ dataStoreBundle.deadlineExtensions.forEach((k, deadlineExtension) -> {
+ DeadlineExtension sqlDeadline = entityConverter.convert(deadlineExtension);
+ sqlDataBundle.deadlineExtensions.put(k, sqlDeadline);
+ });
+
+ dataStoreBundle.feedbackQuestions.forEach((k, feedbackQuestion) -> {
+ FeedbackQuestion sqlFeedbackQuestion = entityConverter.convert(feedbackQuestion);
+ sqlDataBundle.feedbackQuestions.put(k, sqlFeedbackQuestion);
+ });
+
+ dataStoreBundle.accounts.forEach((k, account) -> {
+ List<ReadNotification> sqlReadNotifications = entityConverter.createReadNotifications(account);
+ sqlReadNotifications.forEach(notif -> {
+ String jsonKey = removeWhitespace(String.format("%s-%s",
+ notif.getNotification().getTitle(), account.getEmail()));
+ sqlDataBundle.readNotifications.put(jsonKey, notif);
+ });
+ });
+
+ dataStoreBundle.feedbackResponses.forEach((k, feedbackResponse) -> {
+ FeedbackResponse sqlFeedbackResponse = entityConverter.convert(feedbackResponse);
+ sqlDataBundle.feedbackResponses.put(k, sqlFeedbackResponse);
+ });
+
+ dataStoreBundle.feedbackResponseComments.forEach((k, feedbackReponseComment) -> {
+ FeedbackResponseComment sqlFeedbackResponseComment = entityConverter.convert(feedbackReponseComment);
+ sqlDataBundle.feedbackResponseComments.put(k, sqlFeedbackResponseComment);
+ });
+ }
+
+ public static void main(String[] args) throws IOException, InvalidParametersException {
+ if (args.length > 0) {
+ File inputFile = new File(args[0]);
+ String fileExtension = FilenameUtils.getExtension(inputFile.getName());
+ if (!"json".equals(fileExtension)) {
+ throw new InvalidParametersException("The file provided is not a JSON file");
+ }
+
+ ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(inputFile);
+ String outputFileName = FilenameUtils.getBaseName(inputFile.getName()) + "Sql.json";
+ File outputFile = new File(inputFile.getParent(), outputFileName);
+ script.createSqlJson(outputFile);
+ } else {
+ throw new InvalidParametersException("Required the path of the script to convert");
+ }
+ }
+}