refactor: move assessment event fields to parent class

pom-dependency-tree.txt

@@ -1,7 +1,7 @@
-ai.elimu:webapp:war:2.6.44-SNAPSHOT
-+- ai.elimu:model:jar:model-2.0.97:compile
-|  \- com.google.code.gson:gson:jar:2.13.0:compile
-|     \- com.google.errorprone:error_prone_annotations:jar:2.37.0:compile
+ai.elimu:webapp:war:2.6.47-SNAPSHOT
++- ai.elimu:model:jar:model-2.0.111:compile
+|  \- com.google.code.gson:gson:jar:2.13.1:compile
+|     \- com.google.errorprone:error_prone_annotations:jar:2.38.0:compile
 +- org.springframework:spring-context:jar:6.0.9:compile
 |  +- org.springframework:spring-aop:jar:6.0.9:compile
 |  +- org.springframework:spring-beans:jar:6.0.9:compile

pom.xml

@@ -9,7 +9,7 @@
   <properties>
     <java.version>17</java.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <model.version>2.0.97</model.version>
+    <model.version>2.0.111</model.version>
     <hibernate.version>6.1.7.Final</hibernate.version>
     <jetty.version>11.0.24</jetty.version>
     <spring.version>6.0.9</spring.version>

src/main/java/ai/elimu/dao/LetterSoundAssessmentEventDao.java

@@ -1,12 +1,15 @@
 package ai.elimu.dao;
 
+import java.util.Calendar;
 import java.util.List;
 
 import org.springframework.dao.DataAccessException;
 
 import ai.elimu.entity.analytics.LetterSoundAssessmentEvent;
 
 public interface LetterSoundAssessmentEventDao extends GenericDao<LetterSoundAssessmentEvent> {
+
+    LetterSoundAssessmentEvent read(Calendar timestamp, String androidId, String packageName) throws DataAccessException;
 
     List<LetterSoundAssessmentEvent> readAll(String androidId) throws DataAccessException;
 }

src/main/java/ai/elimu/dao/jpa/LetterSoundAssessmentEventDaoJpa.java

@@ -1,14 +1,35 @@
 package ai.elimu.dao.jpa;
 
+import java.util.Calendar;
 import java.util.List;
 
 import org.springframework.dao.DataAccessException;
 
 import ai.elimu.dao.LetterSoundAssessmentEventDao;
 import ai.elimu.entity.analytics.LetterSoundAssessmentEvent;
+import jakarta.persistence.NoResultException;
 
 public class LetterSoundAssessmentEventDaoJpa extends GenericDaoJpa<LetterSoundAssessmentEvent> implements LetterSoundAssessmentEventDao {
 
+    @Override
+    public LetterSoundAssessmentEvent read(Calendar timestamp, String androidId, String packageName) throws DataAccessException {
+        try {
+            return (LetterSoundAssessmentEvent) em.createQuery(
+                "SELECT event " +
+                "FROM LetterSoundAssessmentEvent event " +
+                "WHERE event.timestamp = :timestamp " +
+                "AND event.androidId = :androidId " + 
+                "AND event.packageName = :packageName")
+                .setParameter("timestamp", timestamp)
+                .setParameter("androidId", androidId)
+                .setParameter("packageName", packageName)
+                .getSingleResult();
+        } catch (NoResultException e) {
+            logger.info("LetterSoundAssessmentEvent (" + timestamp.getTimeInMillis() + ", " + androidId + ", \"" + packageName + "\") was not found");
+            return null;
+        }
+    }
+
     @Override
     public List<LetterSoundAssessmentEvent> readAll(String androidId) throws DataAccessException {
         return em.createQuery(

src/main/java/ai/elimu/entity/analytics/AssessmentEvent.java

@@ -44,12 +44,23 @@ public abstract class AssessmentEvent extends BaseEntity {
   @ManyToOne
   private Application application;
 
+  /**
+   * A value in the range [0.0, 1.0].
+   */
+  private Float masteryScore;
+
+  /**
+   * The number of milliseconds passed between the student opening the assessment task 
+   * and submitting a response. E.g. <code>15000</code>.
+   */
+  private Long timeSpentMs;
+
   /**
    * Any additional data should be stored in the format of a JSON object.
    * 
    * Example:
    * <pre>
-   * {'word_ids_presented': [1,2,3], 'word_id_selected': 2}
+   * {'word_ids_presented': [1,2,3], 'word_id_selected': [2]}
    * </pre>
    */
   @Column(length = 1024)

src/main/java/ai/elimu/entity/analytics/LetterSoundAssessmentEvent.java

@@ -1,6 +1,7 @@
 package ai.elimu.entity.analytics;
 
 import jakarta.persistence.Entity;
+import jakarta.validation.constraints.NotNull;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -12,11 +13,13 @@ public class LetterSoundAssessmentEvent extends AssessmentEvent {
     /**
      * The sequence of letters. E.g. <code>"sh"</code>.
      */
+    @NotNull
     private String letterSoundLetters;
 
     /**
      * The sequence of sounds (IPA values). E.g. <code>"ʃ"</code>.
      */
+    @NotNull
     private String letterSoundSounds;
 
     /**
@@ -25,15 +28,4 @@ public class LetterSoundAssessmentEvent extends AssessmentEvent {
      * In this case, the {@link #letterSoundId} will be {@code null}.
      */
     private Long letterSoundId;
-
-    /**
-     * A value in the range [0.0, 1.0].
-     */
-    private Float masteryScore;
-
-    /**
-     * The number of milliseconds passed between the student opening the assessment task 
-     * and submitting a response. E.g. <code>15000</code>.
-     */
-    private Long timeSpentMs;
 }

src/main/java/ai/elimu/entity/analytics/WordAssessmentEvent.java

@@ -20,15 +20,4 @@ public class WordAssessmentEvent extends AssessmentEvent {
      * In this case, the {@link #wordId} will be {@code null}.
      */
     private Long wordId;
-
-    /**
-     * A value in the range [0.0, 1.0].
-     */
-    private Float masteryScore;
-
-    /**
-     * The number of milliseconds passed between the student opening the assessment task 
-     * and submitting a response. E.g. <code>15000</code>.
-     */
-    private Long timeSpentMs;
 }

src/main/java/ai/elimu/tasks/analytics/LetterSoundAssessmentEventImportScheduler.java

@@ -0,0 +1,110 @@
+package ai.elimu.tasks.analytics;
+
+import ai.elimu.dao.StudentDao;
+import ai.elimu.dao.LetterSoundAssessmentEventDao;
+import ai.elimu.entity.analytics.LetterSoundAssessmentEvent;
+import ai.elimu.entity.analytics.students.Student;
+import ai.elimu.model.v2.enums.Language;
+import ai.elimu.rest.v2.analytics.LetterSoundAssessmentEventsRestController;
+import ai.elimu.util.ConfigHelper;
+import ai.elimu.util.csv.CsvAnalyticsExtractionHelper;
+import java.io.File;
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+/**
+ * Extracts assessment events from CSV files previously received by the {@link LetterSoundAssessmentEventsRestController}, and imports them into the database.
+ * <p/>
+ * <p>
+ * Expected folder structure:
+ * <pre>
+ * ├── lang-ENG
+ * │   ├── analytics
+ * │   │   ├── android-id-e387e38700000001
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── letter-sound-assessment-events
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-11.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-14.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-18.csv
+ * │   │   │           └── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-20.csv
+ * │   │   ├── android-id-e387e38700000002
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── letter-sound-assessment-events
+ * │   │   │           ├── e387e38700000002_3001018_letter-sound-assessment-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000002_3001018_letter-sound-assessment-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000002_3001018_letter-sound-assessment-events_2024-10-11.csv
+ * </pre>
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class LetterSoundAssessmentEventImportScheduler {
+
+  private final LetterSoundAssessmentEventDao letterSoundAssessmentEventDao;
+
+  private final StudentDao studentDao;
+
+  @Scheduled(cron = "00 15 * * * *") // 15 minutes past every hour
+  public synchronized void execute() {
+    log.info("execute");
+
+    // Lookup CSV files stored on the filesystem
+    File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
+    File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
+    File analyticsDir = new File(languageDir, "analytics");
+    log.info("analyticsDir: " + analyticsDir);
+    analyticsDir.mkdirs();
+    for (File analyticsDirFile : analyticsDir.listFiles()) {
+      if (analyticsDirFile.getName().startsWith("android-id-")) {
+        File androidIdDir = new File(analyticsDir, analyticsDirFile.getName());
+        for (File androidIdDirFile : androidIdDir.listFiles()) {
+          if (androidIdDirFile.getName().startsWith("version-code-")) {
+            File versionCodeDir = new File(androidIdDir, androidIdDirFile.getName());
+            for (File versionCodeDirFile : versionCodeDir.listFiles()) {
+              if (versionCodeDirFile.getName().equals("letter-sound-assessment-events")) {
+                File letterSoundAssessmentEventsDir = new File(versionCodeDir, versionCodeDirFile.getName());
+                for (File csvFile : letterSoundAssessmentEventsDir.listFiles()) {
+                  log.info("csvFile: " + csvFile);
+
+                  // Convert from CSV to Java
+                  List<LetterSoundAssessmentEvent> events = CsvAnalyticsExtractionHelper.extractLetterSoundAssessmentEvents(csvFile);
+                  log.info("events.size(): " + events.size());
+
+                  // Store in database
+                  for (LetterSoundAssessmentEvent event : events) {
+                    // Check if the event has already been stored in the database
+                    LetterSoundAssessmentEvent existingLetterSoundAssessmentEvent = letterSoundAssessmentEventDao.read(event.getTimestamp(), event.getAndroidId(), event.getPackageName());
+                    if (existingLetterSoundAssessmentEvent != null) {
+                      log.warn("The event has already been stored in the database. Skipping data import.");
+                      continue;
+                    }
+
+                    // Generate Student ID
+                    Student existingStudent = studentDao.read(event.getAndroidId());
+                    if (existingStudent == null) {
+                      Student student = new Student();
+                      student.setAndroidId(event.getAndroidId());
+                      studentDao.create(student);
+                      log.info("Stored Student in database with ID " + student.getId());
+                    }
+
+                    // Store the event in the database
+                    letterSoundAssessmentEventDao.create(event);
+                    log.info("Stored event in database with ID " + event.getId());
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    log.info("execute complete");
+  }
+}

src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java

@@ -1,5 +1,6 @@
 package ai.elimu.util.csv;
 
+import ai.elimu.entity.analytics.LetterSoundAssessmentEvent;
 import ai.elimu.entity.analytics.LetterSoundLearningEvent;
 import ai.elimu.entity.analytics.StoryBookLearningEvent;
 import ai.elimu.entity.analytics.VideoLearningEvent;
@@ -30,6 +31,68 @@
 @Slf4j
 public class CsvAnalyticsExtractionHelper {
 
+    public static List<LetterSoundAssessmentEvent> extractLetterSoundAssessmentEvents(File csvFile) {
+        log.info("extractLetterSoundAssessmentEvents");
+
+        Integer versionCode = AnalyticsHelper.extractVersionCodeFromCsvFilename(csvFile.getName());
+        log.info("versionCode: " + versionCode);
+
+        List<LetterSoundAssessmentEvent> letterSoundAssessmentEvents = new ArrayList<>();
+
+        // Iterate each row in the CSV file
+        Path csvFilePath = Paths.get(csvFile.toURI());
+        log.info("csvFilePath: " + csvFilePath);
+        try {
+            Reader reader = Files.newBufferedReader(csvFilePath);
+            CSVFormat csvFormat = CSVFormat.DEFAULT.withFirstRecordAsHeader();
+            log.info("header: " + Arrays.toString(csvFormat.getHeader()));
+            CSVParser csvParser = new CSVParser(reader, csvFormat);
+            for (CSVRecord csvRecord : csvParser) {
+                log.info("csvRecord: " + csvRecord);
+
+                // Convert from CSV to Java
+
+                LetterSoundAssessmentEvent letterSoundAssessmentEvent = new LetterSoundAssessmentEvent();
+
+                long timestampInMillis = Long.valueOf(csvRecord.get("timestamp"));
+                Calendar timestamp = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+                timestamp.setTimeInMillis(timestampInMillis);
+                letterSoundAssessmentEvent.setTimestamp(timestamp);
+
+                String androidId = AnalyticsHelper.extractAndroidIdFromCsvFilename(csvFile.getName());
+                letterSoundAssessmentEvent.setAndroidId(androidId);
+
+                String packageName = csvRecord.get("package_name");
+                letterSoundAssessmentEvent.setPackageName(packageName);
+
+                Float masteryScore = Float.valueOf(csvRecord.get("mastery_score"));
+                letterSoundAssessmentEvent.setMasteryScore(masteryScore);
+
+                Long timeSpentMs = Long.valueOf(csvRecord.get("time_spent_ms"));
+                letterSoundAssessmentEvent.setTimeSpentMs(timeSpentMs);
+
+                String additionalData = csvRecord.get("additional_data");
+                letterSoundAssessmentEvent.setAdditionalData(additionalData);
+
+                String letterSoundLetters = csvRecord.get("letter_sound_letters");
+                letterSoundAssessmentEvent.setLetterSoundLetters(letterSoundLetters);
+
+                String letterSoundSounds = csvRecord.get("letter_sound_sounds");
+                letterSoundAssessmentEvent.setLetterSoundLetters(letterSoundSounds);
+
+                Long letterSoundId = Long.valueOf(csvRecord.get("letter_sound_id"));
+                letterSoundAssessmentEvent.setLetterSoundId(letterSoundId);
+
+                letterSoundAssessmentEvents.add(letterSoundAssessmentEvent);
+            }
+            csvParser.close();
+        } catch (IOException ex) {
+            log.error(ex.getMessage());
+        }
+
+        return letterSoundAssessmentEvents;
+    }
+
     public static List<LetterSoundLearningEvent> extractLetterSoundLearningEvents(File csvFile) {
         log.info("extractLetterSoundLearningEvents");
 
@@ -125,12 +188,6 @@ public static List<WordAssessmentEvent> extractWordAssessmentEvents(File csvFile
                 String packageName = csvRecord.get("package_name");
                 wordAssessmentEvent.setPackageName(packageName);
 
-                String wordText = csvRecord.get("word_text");
-                wordAssessmentEvent.setWordText(wordText);
-
-                Long wordId = Long.valueOf(csvRecord.get("word_id"));
-                wordAssessmentEvent.setWordId(wordId);
-
                 Float masteryScore = Float.valueOf(csvRecord.get("mastery_score"));
                 wordAssessmentEvent.setMasteryScore(masteryScore);
 
@@ -140,6 +197,12 @@ public static List<WordAssessmentEvent> extractWordAssessmentEvents(File csvFile
                 // String additionalData = csvRecord.get("additional_data");
                 // wordAssessmentEvent.setAdditionalData(additionalData);
 
+                String wordText = csvRecord.get("word_text");
+                wordAssessmentEvent.setWordText(wordText);
+
+                Long wordId = Long.valueOf(csvRecord.get("word_id"));
+                wordAssessmentEvent.setWordId(wordId);
+
                 wordAssessmentEvents.add(wordAssessmentEvent);
             }
             csvParser.close();

src/main/java/ai/elimu/web/analytics/students/LetterSoundAssessmentEventsCsvExportController.java

@@ -47,12 +47,12 @@ public void handleRequest(
             "id",
             "timestamp",
             "package_name",
-            "letter_sound_letters",
-            "letter_sound_sounds",
-            "letter_sound_id",
             "mastery_score",
             "time_spent_ms",
-            "additional_data"
+            "additional_data",
+            "letter_sound_letters",
+            "letter_sound_sounds",
+            "letter_sound_id"
         )
         .build();
 
@@ -66,12 +66,12 @@ public void handleRequest(
           letterSoundAssessmentEvent.getId(),
           letterSoundAssessmentEvent.getTimestamp().getTimeInMillis(),
           letterSoundAssessmentEvent.getPackageName(),
-          letterSoundAssessmentEvent.getLetterSoundLetters(),
-          letterSoundAssessmentEvent.getLetterSoundSounds(),
-          letterSoundAssessmentEvent.getLetterSoundId(),
           letterSoundAssessmentEvent.getMasteryScore(),
           letterSoundAssessmentEvent.getTimeSpentMs(),
-          letterSoundAssessmentEvent.getAdditionalData()
+          letterSoundAssessmentEvent.getAdditionalData(),
+          letterSoundAssessmentEvent.getLetterSoundLetters(),
+          letterSoundAssessmentEvent.getLetterSoundSounds(),
+          letterSoundAssessmentEvent.getLetterSoundId()
       );
     }
     csvPrinter.flush();