diff --git a/pom-dependency-tree.txt b/pom-dependency-tree.txt index db32b148b..88b7535eb 100644 --- a/pom-dependency-tree.txt +++ b/pom-dependency-tree.txt @@ -1,7 +1,7 @@ -ai.elimu:webapp:war:2.6.44-SNAPSHOT -+- ai.elimu:model:jar:model-2.0.97:compile -| \- com.google.code.gson:gson:jar:2.13.0:compile -| \- com.google.errorprone:error_prone_annotations:jar:2.37.0:compile +ai.elimu:webapp:war:2.6.47-SNAPSHOT ++- ai.elimu:model:jar:model-2.0.111:compile +| \- com.google.code.gson:gson:jar:2.13.1:compile +| \- com.google.errorprone:error_prone_annotations:jar:2.38.0:compile +- org.springframework:spring-context:jar:6.0.9:compile | +- org.springframework:spring-aop:jar:6.0.9:compile | +- org.springframework:spring-beans:jar:6.0.9:compile diff --git a/pom.xml b/pom.xml index 9c861b92e..94465880e 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ 17 UTF-8 - 2.0.97 + 2.0.111 6.1.7.Final 11.0.24 6.0.9 diff --git a/src/main/java/ai/elimu/dao/LetterSoundAssessmentEventDao.java b/src/main/java/ai/elimu/dao/LetterSoundAssessmentEventDao.java index 3e8349baf..6051f6d51 100644 --- a/src/main/java/ai/elimu/dao/LetterSoundAssessmentEventDao.java +++ b/src/main/java/ai/elimu/dao/LetterSoundAssessmentEventDao.java @@ -1,5 +1,6 @@ package ai.elimu.dao; +import java.util.Calendar; import java.util.List; import org.springframework.dao.DataAccessException; @@ -7,6 +8,8 @@ import ai.elimu.entity.analytics.LetterSoundAssessmentEvent; public interface LetterSoundAssessmentEventDao extends GenericDao { + + LetterSoundAssessmentEvent read(Calendar timestamp, String androidId, String packageName) throws DataAccessException; List readAll(String androidId) throws DataAccessException; } diff --git a/src/main/java/ai/elimu/dao/jpa/LetterSoundAssessmentEventDaoJpa.java b/src/main/java/ai/elimu/dao/jpa/LetterSoundAssessmentEventDaoJpa.java index 067e22e56..3ca2748cb 100644 --- a/src/main/java/ai/elimu/dao/jpa/LetterSoundAssessmentEventDaoJpa.java +++ b/src/main/java/ai/elimu/dao/jpa/LetterSoundAssessmentEventDaoJpa.java @@ -1,14 +1,35 @@ package ai.elimu.dao.jpa; +import java.util.Calendar; import java.util.List; import org.springframework.dao.DataAccessException; import ai.elimu.dao.LetterSoundAssessmentEventDao; import ai.elimu.entity.analytics.LetterSoundAssessmentEvent; +import jakarta.persistence.NoResultException; public class LetterSoundAssessmentEventDaoJpa extends GenericDaoJpa implements LetterSoundAssessmentEventDao { + @Override + public LetterSoundAssessmentEvent read(Calendar timestamp, String androidId, String packageName) throws DataAccessException { + try { + return (LetterSoundAssessmentEvent) em.createQuery( + "SELECT event " + + "FROM LetterSoundAssessmentEvent event " + + "WHERE event.timestamp = :timestamp " + + "AND event.androidId = :androidId " + + "AND event.packageName = :packageName") + .setParameter("timestamp", timestamp) + .setParameter("androidId", androidId) + .setParameter("packageName", packageName) + .getSingleResult(); + } catch (NoResultException e) { + logger.info("LetterSoundAssessmentEvent (" + timestamp.getTimeInMillis() + ", " + androidId + ", \"" + packageName + "\") was not found"); + return null; + } + } + @Override public List readAll(String androidId) throws DataAccessException { return em.createQuery( diff --git a/src/main/java/ai/elimu/entity/analytics/AssessmentEvent.java b/src/main/java/ai/elimu/entity/analytics/AssessmentEvent.java index 5489899d4..929a3b0ef 100644 --- a/src/main/java/ai/elimu/entity/analytics/AssessmentEvent.java +++ b/src/main/java/ai/elimu/entity/analytics/AssessmentEvent.java @@ -44,12 +44,23 @@ public abstract class AssessmentEvent extends BaseEntity { @ManyToOne private Application application; + /** + * A value in the range [0.0, 1.0]. + */ + private Float masteryScore; + + /** + * The number of milliseconds passed between the student opening the assessment task + * and submitting a response. E.g. 15000. + */ + private Long timeSpentMs; + /** * Any additional data should be stored in the format of a JSON object. * * Example: * 
-   * {'word_ids_presented': [1,2,3], 'word_id_selected': 2}
+   * {'word_ids_presented': [1,2,3], 'word_id_selected': [2]}
    *
*/ @Column(length = 1024) diff --git a/src/main/java/ai/elimu/entity/analytics/LetterSoundAssessmentEvent.java b/src/main/java/ai/elimu/entity/analytics/LetterSoundAssessmentEvent.java index 1e59fcafa..8234f0329 100644 --- a/src/main/java/ai/elimu/entity/analytics/LetterSoundAssessmentEvent.java +++ b/src/main/java/ai/elimu/entity/analytics/LetterSoundAssessmentEvent.java @@ -1,6 +1,7 @@ package ai.elimu.entity.analytics; import jakarta.persistence.Entity; +import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; @@ -12,11 +13,13 @@ public class LetterSoundAssessmentEvent extends AssessmentEvent { /** * The sequence of letters. E.g. "sh". */ + @NotNull private String letterSoundLetters; /** * The sequence of sounds (IPA values). E.g. "ʃ". */ + @NotNull private String letterSoundSounds; /** @@ -25,15 +28,4 @@ public class LetterSoundAssessmentEvent extends AssessmentEvent { * In this case, the {@link #letterSoundId} will be {@code null}. */ private Long letterSoundId; - - /** - * A value in the range [0.0, 1.0]. - */ - private Float masteryScore; - - /** - * The number of milliseconds passed between the student opening the assessment task - * and submitting a response. E.g. 15000. - */ - private Long timeSpentMs; } diff --git a/src/main/java/ai/elimu/entity/analytics/WordAssessmentEvent.java b/src/main/java/ai/elimu/entity/analytics/WordAssessmentEvent.java index b900c0569..ba1a921bf 100644 --- a/src/main/java/ai/elimu/entity/analytics/WordAssessmentEvent.java +++ b/src/main/java/ai/elimu/entity/analytics/WordAssessmentEvent.java @@ -20,15 +20,4 @@ public class WordAssessmentEvent extends AssessmentEvent { * In this case, the {@link #wordId} will be {@code null}. */ private Long wordId; - - /** - * A value in the range [0.0, 1.0]. - */ - private Float masteryScore; - - /** - * The number of milliseconds passed between the student opening the assessment task - * and submitting a response. E.g. 15000. - */ - private Long timeSpentMs; } diff --git a/src/main/java/ai/elimu/tasks/analytics/LetterSoundAssessmentEventImportScheduler.java b/src/main/java/ai/elimu/tasks/analytics/LetterSoundAssessmentEventImportScheduler.java new file mode 100644 index 000000000..f2ca682fd --- /dev/null +++ b/src/main/java/ai/elimu/tasks/analytics/LetterSoundAssessmentEventImportScheduler.java @@ -0,0 +1,110 @@ +package ai.elimu.tasks.analytics; + +import ai.elimu.dao.StudentDao; +import ai.elimu.dao.LetterSoundAssessmentEventDao; +import ai.elimu.entity.analytics.LetterSoundAssessmentEvent; +import ai.elimu.entity.analytics.students.Student; +import ai.elimu.model.v2.enums.Language; +import ai.elimu.rest.v2.analytics.LetterSoundAssessmentEventsRestController; +import ai.elimu.util.ConfigHelper; +import ai.elimu.util.csv.CsvAnalyticsExtractionHelper; +import java.io.File; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * Extracts assessment events from CSV files previously received by the {@link LetterSoundAssessmentEventsRestController}, and imports them into the database. + *

+ *

+ * Expected folder structure: + * 

+ * ├── lang-ENG
+ * │   ├── analytics
+ * │   │   ├── android-id-e387e38700000001
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── letter-sound-assessment-events
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-11.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-14.csv
+ * │   │   │           ├── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-18.csv
+ * │   │   │           └── e387e38700000001_3001018_letter-sound-assessment-events_2024-10-20.csv
+ * │   │   ├── android-id-e387e38700000002
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── letter-sound-assessment-events
+ * │   │   │           ├── e387e38700000002_3001018_letter-sound-assessment-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000002_3001018_letter-sound-assessment-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000002_3001018_letter-sound-assessment-events_2024-10-11.csv
+ *
+ */ +@Service +@RequiredArgsConstructor +@Slf4j +public class LetterSoundAssessmentEventImportScheduler { + + private final LetterSoundAssessmentEventDao letterSoundAssessmentEventDao; + + private final StudentDao studentDao; + + @Scheduled(cron = "00 15 * * * *") // 15 minutes past every hour + public synchronized void execute() { + log.info("execute"); + + // Lookup CSV files stored on the filesystem + File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai"); + File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language"))); + File analyticsDir = new File(languageDir, "analytics"); + log.info("analyticsDir: " + analyticsDir); + analyticsDir.mkdirs(); + for (File analyticsDirFile : analyticsDir.listFiles()) { + if (analyticsDirFile.getName().startsWith("android-id-")) { + File androidIdDir = new File(analyticsDir, analyticsDirFile.getName()); + for (File androidIdDirFile : androidIdDir.listFiles()) { + if (androidIdDirFile.getName().startsWith("version-code-")) { + File versionCodeDir = new File(androidIdDir, androidIdDirFile.getName()); + for (File versionCodeDirFile : versionCodeDir.listFiles()) { + if (versionCodeDirFile.getName().equals("letter-sound-assessment-events")) { + File letterSoundAssessmentEventsDir = new File(versionCodeDir, versionCodeDirFile.getName()); + for (File csvFile : letterSoundAssessmentEventsDir.listFiles()) { + log.info("csvFile: " + csvFile); + + // Convert from CSV to Java + List events = CsvAnalyticsExtractionHelper.extractLetterSoundAssessmentEvents(csvFile); + log.info("events.size(): " + events.size()); + + // Store in database + for (LetterSoundAssessmentEvent event : events) { + // Check if the event has already been stored in the database + LetterSoundAssessmentEvent existingLetterSoundAssessmentEvent = letterSoundAssessmentEventDao.read(event.getTimestamp(), event.getAndroidId(), event.getPackageName()); + if (existingLetterSoundAssessmentEvent != null) { + log.warn("The event has already been stored in the database. Skipping data import."); + continue; + } + + // Generate Student ID + Student existingStudent = studentDao.read(event.getAndroidId()); + if (existingStudent == null) { + Student student = new Student(); + student.setAndroidId(event.getAndroidId()); + studentDao.create(student); + log.info("Stored Student in database with ID " + student.getId()); + } + + // Store the event in the database + letterSoundAssessmentEventDao.create(event); + log.info("Stored event in database with ID " + event.getId()); + } + } + } + } + } + } + } + } + + log.info("execute complete"); + } +} diff --git a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java index c47f4f1c3..19a079434 100644 --- a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java +++ b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java @@ -1,5 +1,6 @@ package ai.elimu.util.csv; +import ai.elimu.entity.analytics.LetterSoundAssessmentEvent; import ai.elimu.entity.analytics.LetterSoundLearningEvent; import ai.elimu.entity.analytics.StoryBookLearningEvent; import ai.elimu.entity.analytics.VideoLearningEvent; @@ -30,6 +31,68 @@ @Slf4j public class CsvAnalyticsExtractionHelper { + public static List extractLetterSoundAssessmentEvents(File csvFile) { + log.info("extractLetterSoundAssessmentEvents"); + + Integer versionCode = AnalyticsHelper.extractVersionCodeFromCsvFilename(csvFile.getName()); + log.info("versionCode: " + versionCode); + + List letterSoundAssessmentEvents = new ArrayList<>(); + + // Iterate each row in the CSV file + Path csvFilePath = Paths.get(csvFile.toURI()); + log.info("csvFilePath: " + csvFilePath); + try { + Reader reader = Files.newBufferedReader(csvFilePath); + CSVFormat csvFormat = CSVFormat.DEFAULT.withFirstRecordAsHeader(); + log.info("header: " + Arrays.toString(csvFormat.getHeader())); + CSVParser csvParser = new CSVParser(reader, csvFormat); + for (CSVRecord csvRecord : csvParser) { + log.info("csvRecord: " + csvRecord); + + // Convert from CSV to Java + + LetterSoundAssessmentEvent letterSoundAssessmentEvent = new LetterSoundAssessmentEvent(); + + long timestampInMillis = Long.valueOf(csvRecord.get("timestamp")); + Calendar timestamp = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + timestamp.setTimeInMillis(timestampInMillis); + letterSoundAssessmentEvent.setTimestamp(timestamp); + + String androidId = AnalyticsHelper.extractAndroidIdFromCsvFilename(csvFile.getName()); + letterSoundAssessmentEvent.setAndroidId(androidId); + + String packageName = csvRecord.get("package_name"); + letterSoundAssessmentEvent.setPackageName(packageName); + + Float masteryScore = Float.valueOf(csvRecord.get("mastery_score")); + letterSoundAssessmentEvent.setMasteryScore(masteryScore); + + Long timeSpentMs = Long.valueOf(csvRecord.get("time_spent_ms")); + letterSoundAssessmentEvent.setTimeSpentMs(timeSpentMs); + + String additionalData = csvRecord.get("additional_data"); + letterSoundAssessmentEvent.setAdditionalData(additionalData); + + String letterSoundLetters = csvRecord.get("letter_sound_letters"); + letterSoundAssessmentEvent.setLetterSoundLetters(letterSoundLetters); + + String letterSoundSounds = csvRecord.get("letter_sound_sounds"); + letterSoundAssessmentEvent.setLetterSoundLetters(letterSoundSounds); + + Long letterSoundId = Long.valueOf(csvRecord.get("letter_sound_id")); + letterSoundAssessmentEvent.setLetterSoundId(letterSoundId); + + letterSoundAssessmentEvents.add(letterSoundAssessmentEvent); + } + csvParser.close(); + } catch (IOException ex) { + log.error(ex.getMessage()); + } + + return letterSoundAssessmentEvents; + } + public static List extractLetterSoundLearningEvents(File csvFile) { log.info("extractLetterSoundLearningEvents"); @@ -125,12 +188,6 @@ public static List extractWordAssessmentEvents(File csvFile String packageName = csvRecord.get("package_name"); wordAssessmentEvent.setPackageName(packageName); - String wordText = csvRecord.get("word_text"); - wordAssessmentEvent.setWordText(wordText); - - Long wordId = Long.valueOf(csvRecord.get("word_id")); - wordAssessmentEvent.setWordId(wordId); - Float masteryScore = Float.valueOf(csvRecord.get("mastery_score")); wordAssessmentEvent.setMasteryScore(masteryScore); @@ -140,6 +197,12 @@ public static List extractWordAssessmentEvents(File csvFile // String additionalData = csvRecord.get("additional_data"); // wordAssessmentEvent.setAdditionalData(additionalData); + String wordText = csvRecord.get("word_text"); + wordAssessmentEvent.setWordText(wordText); + + Long wordId = Long.valueOf(csvRecord.get("word_id")); + wordAssessmentEvent.setWordId(wordId); + wordAssessmentEvents.add(wordAssessmentEvent); } csvParser.close(); diff --git a/src/main/java/ai/elimu/web/analytics/students/LetterSoundAssessmentEventsCsvExportController.java b/src/main/java/ai/elimu/web/analytics/students/LetterSoundAssessmentEventsCsvExportController.java index 0da480bc2..a09237ce5 100644 --- a/src/main/java/ai/elimu/web/analytics/students/LetterSoundAssessmentEventsCsvExportController.java +++ b/src/main/java/ai/elimu/web/analytics/students/LetterSoundAssessmentEventsCsvExportController.java @@ -47,12 +47,12 @@ public void handleRequest( "id", "timestamp", "package_name", - "letter_sound_letters", - "letter_sound_sounds", - "letter_sound_id", "mastery_score", "time_spent_ms", - "additional_data" + "additional_data", + "letter_sound_letters", + "letter_sound_sounds", + "letter_sound_id" ) .build(); @@ -66,12 +66,12 @@ public void handleRequest( letterSoundAssessmentEvent.getId(), letterSoundAssessmentEvent.getTimestamp().getTimeInMillis(), letterSoundAssessmentEvent.getPackageName(), - letterSoundAssessmentEvent.getLetterSoundLetters(), - letterSoundAssessmentEvent.getLetterSoundSounds(), - letterSoundAssessmentEvent.getLetterSoundId(), letterSoundAssessmentEvent.getMasteryScore(), letterSoundAssessmentEvent.getTimeSpentMs(), - letterSoundAssessmentEvent.getAdditionalData() + letterSoundAssessmentEvent.getAdditionalData(), + letterSoundAssessmentEvent.getLetterSoundLetters(), + letterSoundAssessmentEvent.getLetterSoundSounds(), + letterSoundAssessmentEvent.getLetterSoundId() ); } csvPrinter.flush(); diff --git a/src/main/java/ai/elimu/web/analytics/students/WordAssessmentEventsCsvExportController.java b/src/main/java/ai/elimu/web/analytics/students/WordAssessmentEventsCsvExportController.java index 662a801a6..d68f34d80 100644 --- a/src/main/java/ai/elimu/web/analytics/students/WordAssessmentEventsCsvExportController.java +++ b/src/main/java/ai/elimu/web/analytics/students/WordAssessmentEventsCsvExportController.java @@ -47,11 +47,11 @@ public void handleRequest( "id", "timestamp", "package_name", - "word_text", - "word_id", "mastery_score", "time_spent_ms", - "additional_data" + "additional_data", + "word_text", + "word_id" ) .build(); @@ -65,11 +65,11 @@ public void handleRequest( wordAssessmentEvent.getId(), wordAssessmentEvent.getTimestamp().getTimeInMillis(), wordAssessmentEvent.getPackageName(), - wordAssessmentEvent.getWordText(), - wordAssessmentEvent.getWordId(), wordAssessmentEvent.getMasteryScore(), wordAssessmentEvent.getTimeSpentMs(), - wordAssessmentEvent.getAdditionalData() + wordAssessmentEvent.getAdditionalData(), + wordAssessmentEvent.getWordText(), + wordAssessmentEvent.getWordId() ); } csvPrinter.flush(); diff --git a/src/main/resources/META-INF/jpa-schema-export.sql b/src/main/resources/META-INF/jpa-schema-export.sql index 4b7d58180..c7bfe4367 100644 --- a/src/main/resources/META-INF/jpa-schema-export.sql +++ b/src/main/resources/META-INF/jpa-schema-export.sql @@ -312,13 +312,13 @@ id bigint not null auto_increment, additionalData text, androidId varchar(255), + masteryScore float(23), packageName varchar(255), + timeSpentMs bigint, timestamp datetime, letterSoundId bigint, letterSoundLetters varchar(255), letterSoundSounds varchar(255), - masteryScore float(23), - timeSpentMs bigint, application_id bigint, primary key (id) ) type=MyISAM; @@ -610,10 +610,10 @@ id bigint not null auto_increment, additionalData text, androidId varchar(255), - packageName varchar(255), - timestamp datetime, masteryScore float(23), + packageName varchar(255), timeSpentMs bigint, + timestamp datetime, wordId bigint, wordText varchar(255), application_id bigint, diff --git a/src/main/resources/db/migration/2006047.sql b/src/main/resources/db/migration/2006047.sql new file mode 100644 index 000000000..3bb7c543d --- /dev/null +++ b/src/main/resources/db/migration/2006047.sql @@ -0,0 +1,4 @@ +# 2.6.47 + +ALTER TABLE `LetterSoundAssessmentEvent` MODIFY `letterSoundLetters` VARCHAR(255) NOT NULL; +ALTER TABLE `LetterSoundAssessmentEvent` MODIFY `letterSoundSounds` VARCHAR(255) NOT NULL;