diff --git a/backend/migrations-active/20260828000000-dedupe-course-wide-srs-items.cjs b/backend/migrations-active/20260828000000-dedupe-course-wide-srs-items.cjs new file mode 100644 index 0000000..edde00f --- /dev/null +++ b/backend/migrations-active/20260828000000-dedupe-course-wide-srs-items.cjs @@ -0,0 +1,50 @@ +'use strict'; + +/** + * Legacy SRS keys included lesson_id while the course pool used NULL. The same + * word pair could therefore receive parallel schedules. Keep the most advanced + * record, preserve its earliest due time, and remove only exact course-wide + * duplicates. + */ +module.exports = { + async up(queryInterface) { + await queryInterface.sequelize.query(` + WITH grouped AS ( + SELECT + id, + FIRST_VALUE(id) OVER bucket AS keeper_id, + MIN(next_due_at) OVER bucket AS earliest_due_at, + MAX(correct_count) OVER bucket AS max_correct_count, + MAX(wrong_count) OVER bucket AS max_wrong_count, + MAX(lapse_count) OVER bucket AS max_lapse_count, + BOOL_OR(is_hard) OVER bucket AS any_hard, + ROW_NUMBER() OVER bucket AS row_number + FROM community.vocab_srs_item + WINDOW bucket AS ( + PARTITION BY user_id, course_id, UPPER(direction), + LOWER(REGEXP_REPLACE(learning, '[[:punct:][:space:]]+', ' ', 'g')), + LOWER(REGEXP_REPLACE(reference, '[[:punct:][:space:]]+', ' ', 'g')) + ORDER BY correct_count DESC, last_reviewed_at DESC NULLS LAST, id ASC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) + ), updated AS ( + UPDATE community.vocab_srs_item AS item + SET next_due_at = grouped.earliest_due_at, + correct_count = grouped.max_correct_count, + wrong_count = grouped.max_wrong_count, + lapse_count = grouped.max_lapse_count, + is_hard = grouped.any_hard + FROM grouped + WHERE item.id = grouped.keeper_id + RETURNING item.id + ) + DELETE FROM community.vocab_srs_item AS item + USING grouped + WHERE item.id = grouped.id + AND grouped.row_number > 1; + `); + }, + async down() { + // Duplicate rows are intentionally not recreated. + } +}; diff --git a/backend/services/vocabService.js b/backend/services/vocabService.js index 01124bc..673f737 100755 --- a/backend/services/vocabService.js +++ b/backend/services/vocabService.js @@ -123,6 +123,15 @@ export default class VocabService { return crypto.createHash('sha1').update(raw).digest('hex'); } + _buildSrsPairFingerprint({ courseId, learning, reference, direction = 'BOTH' }) { + return [ + Number(courseId) || 0, + String(direction || 'BOTH').toUpperCase(), + this._normalizeSrsText(learning), + this._normalizeSrsText(reference) + ].join('|'); + } + _decorateSrsVocabs(vocabs = [], { courseId, lessonId = null } = {}) { return (Array.isArray(vocabs) ? vocabs : []) .map((entry) => { @@ -255,20 +264,24 @@ export default class VocabService { return []; } + // SRS is course-wide. A pair may originate from a lesson pool or a course + // pool, but must still remain one scheduling item. Matching only itemKey + // used to create duplicates because that key includes lessonId. const existing = await VocabSrsItem.findAll({ - where: { - userId, - itemKey: { - [Op.in]: decorated.map((entry) => entry.itemKey) - } - } + where: { userId, courseId: Number(courseId) }, + order: [['correctCount', 'DESC'], ['lastReviewedAt', 'DESC'], ['id', 'ASC']] + }); + const existingByPair = new Map(); + existing.forEach((entry) => { + const fingerprint = this._buildSrsPairFingerprint(entry); + if (!existingByPair.has(fingerprint)) existingByPair.set(fingerprint, entry); }); - const existingByKey = new Map(existing.map((entry) => [entry.itemKey, entry])); const now = new Date(); const createdItems = []; for (const entry of decorated) { - if (existingByKey.has(entry.itemKey)) { + const fingerprint = this._buildSrsPairFingerprint(entry); + if (existingByPair.has(fingerprint)) { continue; } @@ -295,11 +308,11 @@ export default class VocabService { nextDueAt: safeNextDue }); createdItems.push(created); - existingByKey.set(entry.itemKey, created); + existingByPair.set(fingerprint, created); } return decorated.map((entry) => { - const item = existingByKey.get(entry.itemKey); + const item = existingByPair.get(this._buildSrsPairFingerprint(entry)); return { ...entry, srs: item ? { @@ -2045,21 +2058,21 @@ export default class VocabService { ] }); const validPool = await this.getCompletedLessonVocabPool(hashedUserId, course.id); - const validKeys = new Set( + const validPairs = new Set( (Array.isArray(validPool?.vocabs) ? validPool.vocabs : []) - .map((entry) => String(entry?.itemKey || '').trim()) + .map((entry) => this._buildSrsPairFingerprint(entry)) .filter(Boolean) ); const validDueRows = scheduledRows.filter((item) => this._isTrainableSrsPair(item) - && (!validKeys.size || validKeys.has(String(item.itemKey || '').trim())) + && (!validPairs.size || validPairs.has(this._buildSrsPairFingerprint(item))) && this._getSrsScheduleDateKey(item.nextDueAt) <= todayKey ); const upcomingByDate = new Map(); scheduledRows .filter((item) => ( this._isTrainableSrsPair(item) - && (!validKeys.size || validKeys.has(String(item.itemKey || '').trim())) + && (!validPairs.size || validPairs.has(this._buildSrsPairFingerprint(item))) && this._getSrsScheduleDateKey(item.nextDueAt) > todayKey )) .forEach((item) => {