feat(vocab): implement SRS pair fingerprinting and deduplicate course-wide SRS items

backend/migrations-active/20260828000000-dedupe-course-wide-srs-items.cjs

@@ -0,0 +1,50 @@
+'use strict';
+
+/**
+ * Legacy SRS keys included lesson_id while the course pool used NULL. The same
+ * word pair could therefore receive parallel schedules. Keep the most advanced
+ * record, preserve its earliest due time, and remove only exact course-wide
+ * duplicates.
+ */
+module.exports = {
+  async up(queryInterface) {
+    await queryInterface.sequelize.query(`
+      WITH grouped AS (
+        SELECT
+          id,
+          FIRST_VALUE(id) OVER bucket AS keeper_id,
+          MIN(next_due_at) OVER bucket AS earliest_due_at,
+          MAX(correct_count) OVER bucket AS max_correct_count,
+          MAX(wrong_count) OVER bucket AS max_wrong_count,
+          MAX(lapse_count) OVER bucket AS max_lapse_count,
+          BOOL_OR(is_hard) OVER bucket AS any_hard,
+          ROW_NUMBER() OVER bucket AS row_number
+        FROM community.vocab_srs_item
+        WINDOW bucket AS (
+          PARTITION BY user_id, course_id, UPPER(direction),
+            LOWER(REGEXP_REPLACE(learning, '[[:punct:][:space:]]+', ' ', 'g')),
+            LOWER(REGEXP_REPLACE(reference, '[[:punct:][:space:]]+', ' ', 'g'))
+          ORDER BY correct_count DESC, last_reviewed_at DESC NULLS LAST, id ASC
+          ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+        )
+      ), updated AS (
+        UPDATE community.vocab_srs_item AS item
+        SET next_due_at = grouped.earliest_due_at,
+            correct_count = grouped.max_correct_count,
+            wrong_count = grouped.max_wrong_count,
+            lapse_count = grouped.max_lapse_count,
+            is_hard = grouped.any_hard
+        FROM grouped
+        WHERE item.id = grouped.keeper_id
+        RETURNING item.id
+      )
+      DELETE FROM community.vocab_srs_item AS item
+      USING grouped
+      WHERE item.id = grouped.id
+        AND grouped.row_number > 1;
+    `);
+  },
+  async down() {
+    // Duplicate rows are intentionally not recreated.
+  }
+};

backend/services/vocabService.js

@@ -123,6 +123,15 @@ export default class VocabService {
     return crypto.createHash('sha1').update(raw).digest('hex');
   }
 
+  _buildSrsPairFingerprint({ courseId, learning, reference, direction = 'BOTH' }) {
+    return [
+      Number(courseId) || 0,
+      String(direction || 'BOTH').toUpperCase(),
+      this._normalizeSrsText(learning),
+      this._normalizeSrsText(reference)
+    ].join('|');
+  }
+
   _decorateSrsVocabs(vocabs = [], { courseId, lessonId = null } = {}) {
     return (Array.isArray(vocabs) ? vocabs : [])
       .map((entry) => {
@@ -255,20 +264,24 @@ export default class VocabService {
       return [];
     }
 
+    // SRS is course-wide. A pair may originate from a lesson pool or a course
+    // pool, but must still remain one scheduling item. Matching only itemKey
+    // used to create duplicates because that key includes lessonId.
     const existing = await VocabSrsItem.findAll({
-      where: {
-        userId,
-        itemKey: {
-          [Op.in]: decorated.map((entry) => entry.itemKey)
-        }
-      }
+      where: { userId, courseId: Number(courseId) },
+      order: [['correctCount', 'DESC'], ['lastReviewedAt', 'DESC'], ['id', 'ASC']]
+    });
+    const existingByPair = new Map();
+    existing.forEach((entry) => {
+      const fingerprint = this._buildSrsPairFingerprint(entry);
+      if (!existingByPair.has(fingerprint)) existingByPair.set(fingerprint, entry);
     });
-    const existingByKey = new Map(existing.map((entry) => [entry.itemKey, entry]));
     const now = new Date();
 
     const createdItems = [];
     for (const entry of decorated) {
-      if (existingByKey.has(entry.itemKey)) {
+      const fingerprint = this._buildSrsPairFingerprint(entry);
+      if (existingByPair.has(fingerprint)) {
         continue;
       }
 
@@ -295,11 +308,11 @@ export default class VocabService {
         nextDueAt: safeNextDue
       });
       createdItems.push(created);
-      existingByKey.set(entry.itemKey, created);
+      existingByPair.set(fingerprint, created);
     }
 
     return decorated.map((entry) => {
-      const item = existingByKey.get(entry.itemKey);
+      const item = existingByPair.get(this._buildSrsPairFingerprint(entry));
       return {
         ...entry,
         srs: item ? {
@@ -2045,21 +2058,21 @@ export default class VocabService {
       ]
     });
     const validPool = await this.getCompletedLessonVocabPool(hashedUserId, course.id);
-    const validKeys = new Set(
+    const validPairs = new Set(
       (Array.isArray(validPool?.vocabs) ? validPool.vocabs : [])
-        .map((entry) => String(entry?.itemKey || '').trim())
+        .map((entry) => this._buildSrsPairFingerprint(entry))
         .filter(Boolean)
     );
     const validDueRows = scheduledRows.filter((item) =>
       this._isTrainableSrsPair(item)
-      && (!validKeys.size || validKeys.has(String(item.itemKey || '').trim()))
+      && (!validPairs.size || validPairs.has(this._buildSrsPairFingerprint(item)))
       && this._getSrsScheduleDateKey(item.nextDueAt) <= todayKey
     );
     const upcomingByDate = new Map();
     scheduledRows
       .filter((item) => (
         this._isTrainableSrsPair(item)
-        && (!validKeys.size || validKeys.has(String(item.itemKey || '').trim()))
+        && (!validPairs.size || validPairs.has(this._buildSrsPairFingerprint(item)))
         && this._getSrsScheduleDateKey(item.nextDueAt) > todayKey
       ))
       .forEach((item) => {