Fixed bisaya course
All checks were successful
Deploy to production / deploy (push) Successful in 2m8s
All checks were successful
Deploy to production / deploy (push) Successful in 2m8s
This commit is contained in:
131
backend/scripts/repair-invalid-srs-fragment-sentence-pairs.js
Normal file
131
backend/scripts/repair-invalid-srs-fragment-sentence-pairs.js
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Removes stale SRS cards that should not appear in daily typing review:
|
||||||
|
* - compact German question cues against full Bisaya questions, e.g.
|
||||||
|
* "wo/wohin" -> "Asa ka padulong?"
|
||||||
|
* - German speaking/task prompts against model-answer sentences, e.g.
|
||||||
|
* "Kumusta ka? ..." -> "Begrüße jemanden ..."
|
||||||
|
*
|
||||||
|
* Default: dry-run. Use --apply to delete matching rows.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Op } from 'sequelize';
|
||||||
|
import { sequelize } from '../utils/sequelize.js';
|
||||||
|
import VocabSrsItem from '../models/community/vocab_srs_item.js';
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
return {
|
||||||
|
apply: argv.includes('--apply'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const QUESTION_CUES = new Set([
|
||||||
|
'wo',
|
||||||
|
'wo wohin',
|
||||||
|
'wohin',
|
||||||
|
'was',
|
||||||
|
'wer',
|
||||||
|
'wann',
|
||||||
|
'warum',
|
||||||
|
'wieso',
|
||||||
|
'wie',
|
||||||
|
]);
|
||||||
|
|
||||||
|
function normalizeText(value) {
|
||||||
|
return String(value || '')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.normalize('NFKC')
|
||||||
|
.replace(/[\p{P}\p{S}]+/gu, ' ')
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function wordCount(value) {
|
||||||
|
return normalizeText(value).split(/\s+/).filter(Boolean).length;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isQuestionCue(value) {
|
||||||
|
return QUESTION_CUES.has(normalizeText(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isFullQuestion(value) {
|
||||||
|
const text = String(value || '').trim();
|
||||||
|
return wordCount(text) >= 2 && /\?\s*$/.test(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isStaleQuestionCueCard(left, right) {
|
||||||
|
return (isQuestionCue(left) && isFullQuestion(right)) || (isQuestionCue(right) && isFullQuestion(left));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isInstructionLikeText(value) {
|
||||||
|
const text = String(value || '').trim();
|
||||||
|
if (!text) return false;
|
||||||
|
const words = normalizeText(text).split(/\s+/).filter(Boolean);
|
||||||
|
if (words.length < 3) return false;
|
||||||
|
|
||||||
|
const normalized = text.toLowerCase().normalize('NFKC');
|
||||||
|
const startsWithTaskVerb = /^(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|beginne|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige|begrüße|begruesse|grüße|gruesse|drücke|druecke)\b/i.test(normalized);
|
||||||
|
const containsTaskChain = /\b(und|,)\s*(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige|begrüße|begruesse|grüße|gruesse|drücke|druecke)\b/i.test(normalized);
|
||||||
|
const containsPracticeMarker = /\b(laut|jeweils|zu jedem|zu jeder|umgebung|alltagsszene|rollenspiel|mini-dialog|szene)\b/i.test(normalized);
|
||||||
|
|
||||||
|
return startsWithTaskVerb || (containsTaskChain && containsPracticeMarker);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isStaleInstructionCard(left, right) {
|
||||||
|
return isInstructionLikeText(left) || isInstructionLikeText(right);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const { apply } = parseArgs(process.argv.slice(2));
|
||||||
|
const items = await VocabSrsItem.findAll({
|
||||||
|
where: {
|
||||||
|
[Op.or]: [
|
||||||
|
{ learning: { [Op.like]: '%/%' } },
|
||||||
|
{ reference: { [Op.like]: '%/%' } },
|
||||||
|
{ learning: { [Op.like]: '%?%' } },
|
||||||
|
{ reference: { [Op.like]: '%?%' } },
|
||||||
|
{ learning: { [Op.iLike]: 'Begrüße %' } },
|
||||||
|
{ reference: { [Op.iLike]: 'Begrüße %' } },
|
||||||
|
{ learning: { [Op.iLike]: 'Begruesse %' } },
|
||||||
|
{ reference: { [Op.iLike]: 'Begruesse %' } },
|
||||||
|
{ learning: { [Op.iLike]: 'Drücke %' } },
|
||||||
|
{ reference: { [Op.iLike]: 'Drücke %' } },
|
||||||
|
{ learning: { [Op.iLike]: 'Druecke %' } },
|
||||||
|
{ reference: { [Op.iLike]: 'Druecke %' } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
order: [['id', 'ASC']],
|
||||||
|
});
|
||||||
|
|
||||||
|
const matches = items.filter((item) =>
|
||||||
|
isStaleQuestionCueCard(item.learning, item.reference) || isStaleInstructionCard(item.learning, item.reference)
|
||||||
|
);
|
||||||
|
|
||||||
|
matches.forEach((item) => {
|
||||||
|
console.log(
|
||||||
|
`SRS ${item.id} course:${item.courseId} lesson:${item.lessonId || '-'} "${item.learning}" | "${item.reference}"`
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (apply && matches.length) {
|
||||||
|
await VocabSrsItem.destroy({
|
||||||
|
where: {
|
||||||
|
id: {
|
||||||
|
[Op.in]: matches.map((item) => item.id),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`${apply ? 'Deleted' : 'Would delete'} ${matches.length} invalid SRS item(s).`);
|
||||||
|
}
|
||||||
|
|
||||||
|
main()
|
||||||
|
.catch((error) => {
|
||||||
|
console.error(error);
|
||||||
|
process.exitCode = 1;
|
||||||
|
})
|
||||||
|
.finally(async () => {
|
||||||
|
await sequelize.close();
|
||||||
|
});
|
||||||
@@ -73,13 +73,13 @@ export default class VocabService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const normalized = text.toLowerCase().normalize('NFKC');
|
const normalized = text.toLowerCase().normalize('NFKC');
|
||||||
const startsWithTaskVerb = /^(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|beginne|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige)\b/i.test(normalized);
|
const startsWithTaskVerb = /^(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|beginne|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige|begrüße|begruesse|grüße|gruesse|drücke|druecke)\b/i.test(normalized);
|
||||||
const startsWithTakeTask = /^nimm\b/i.test(normalized)
|
const startsWithTakeTask = /^nimm\b/i.test(normalized)
|
||||||
&& (
|
&& (
|
||||||
/\b(ein|eine|einen|zwei|drei|vier|fünf|fuenf|sechs|sieben|acht|neun|zehn|\d+)\b/i.test(normalized)
|
/\b(ein|eine|einen|zwei|drei|vier|fünf|fuenf|sechs|sieben|acht|neun|zehn|\d+)\b/i.test(normalized)
|
||||||
|| /\b(w[oö]rter|verben|gegenstände|gegenstaende|sätze|saetze|muster|beispiele)\b/i.test(normalized)
|
|| /\b(w[oö]rter|verben|gegenstände|gegenstaende|sätze|saetze|muster|beispiele)\b/i.test(normalized)
|
||||||
);
|
);
|
||||||
const containsTaskChain = /\b(und|,)\s*(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige)\b/i.test(normalized);
|
const containsTaskChain = /\b(und|,)\s*(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige|begrüße|begruesse|grüße|gruesse|drücke|druecke)\b/i.test(normalized);
|
||||||
const containsPracticeMarker = /\b(laut|jeweils|zu jedem|zu jeder|umgebung|alltagsszene|rollenspiel|mini-dialog|szene)\b/i.test(normalized);
|
const containsPracticeMarker = /\b(laut|jeweils|zu jedem|zu jeder|umgebung|alltagsszene|rollenspiel|mini-dialog|szene)\b/i.test(normalized);
|
||||||
|
|
||||||
return startsWithTaskVerb || startsWithTakeTask || (containsTaskChain && containsPracticeMarker);
|
return startsWithTaskVerb || startsWithTakeTask || (containsTaskChain && containsPracticeMarker);
|
||||||
@@ -1974,6 +1974,11 @@ export default class VocabService {
|
|||||||
err.status = 400;
|
err.status = 400;
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
|
if (!this._isTrainableSrsPair({ learning, reference })) {
|
||||||
|
const err = new Error('Invalid SRS item text');
|
||||||
|
err.status = 400;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
const lessonId = payload?.lessonId == null
|
const lessonId = payload?.lessonId == null
|
||||||
? null
|
? null
|
||||||
|
|||||||
@@ -225,9 +225,25 @@ export default {
|
|||||||
return Math.round((this.wrongCount / this.totalCount) * 100);
|
return Math.round((this.wrongCount / this.totalCount) * 100);
|
||||||
},
|
},
|
||||||
currentPrompt() {
|
currentPrompt() {
|
||||||
|
if (!this.current) return '';
|
||||||
|
return this.currentDisplayPrompt || this.currentAnswerPrompt;
|
||||||
|
},
|
||||||
|
currentAnswerPrompt() {
|
||||||
if (!this.current) return '';
|
if (!this.current) return '';
|
||||||
return this.direction === 'L2R' ? this.current.learning : this.current.reference;
|
return this.direction === 'L2R' ? this.current.learning : this.current.reference;
|
||||||
},
|
},
|
||||||
|
currentDisplayPrompt() {
|
||||||
|
if (!this.current || this.direction !== 'L2R') return '';
|
||||||
|
const learning = this.normalizeSpacedText(this.current.learning);
|
||||||
|
const reference = this.normalizeSpacedText(this.current.reference);
|
||||||
|
if ((learning === 'wo' || learning === 'wo wohin' || learning === 'wohin') && reference === 'asa') {
|
||||||
|
return '_______(wo/wohin) ka padulong?';
|
||||||
|
}
|
||||||
|
if (learning === 'was' && reference === 'unsa') {
|
||||||
|
return '_______(was) imong plano?';
|
||||||
|
}
|
||||||
|
return '';
|
||||||
|
},
|
||||||
directionLabel() {
|
directionLabel() {
|
||||||
return this.direction === 'L2R'
|
return this.direction === 'L2R'
|
||||||
? this.$t('socialnetwork.vocab.practice.dirLearningToRef')
|
? this.$t('socialnetwork.vocab.practice.dirLearningToRef')
|
||||||
@@ -650,13 +666,13 @@ export default {
|
|||||||
if (wordCount < 3) return false;
|
if (wordCount < 3) return false;
|
||||||
|
|
||||||
const normalized = text.toLowerCase().normalize('NFKC');
|
const normalized = text.toLowerCase().normalize('NFKC');
|
||||||
const startsWithTaskVerb = /^(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|beginne|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige)\b/i.test(normalized);
|
const startsWithTaskVerb = /^(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|beginne|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige|begrüße|begruesse|grüße|gruesse|drücke|druecke)\b/i.test(normalized);
|
||||||
const startsWithTakeTask = /^nimm\b/i.test(normalized)
|
const startsWithTakeTask = /^nimm\b/i.test(normalized)
|
||||||
&& (
|
&& (
|
||||||
/\b(ein|eine|einen|zwei|drei|vier|fünf|fuenf|sechs|sieben|acht|neun|zehn|\d+)\b/i.test(normalized)
|
/\b(ein|eine|einen|zwei|drei|vier|fünf|fuenf|sechs|sieben|acht|neun|zehn|\d+)\b/i.test(normalized)
|
||||||
|| /\b(w[oö]rter|verben|gegenstände|gegenstaende|sätze|saetze|muster|beispiele)\b/i.test(normalized)
|
|| /\b(w[oö]rter|verben|gegenstände|gegenstaende|sätze|saetze|muster|beispiele)\b/i.test(normalized)
|
||||||
);
|
);
|
||||||
const containsTaskChain = /\b(und|,)\s*(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige)\b/i.test(normalized);
|
const containsTaskChain = /\b(und|,)\s*(sage|sag|frage|frag|bitte|stelle|sprich|erzähle|erzaehle|beschreibe|bilde|wähle|waehle|ordne|übersetze|uebersetze|nenne|nenn|verwende|nutze|reagiere|kombiniere|spiele|löse|loese|beantworte|ergänze|ergaenze|formuliere|lies|entscheide|zeige|begrüße|begruesse|grüße|gruesse|drücke|druecke)\b/i.test(normalized);
|
||||||
const containsPracticeMarker = /\b(laut|jeweils|zu jedem|zu jeder|umgebung|alltagsszene|rollenspiel|mini-dialog|szene)\b/i.test(normalized);
|
const containsPracticeMarker = /\b(laut|jeweils|zu jedem|zu jeder|umgebung|alltagsszene|rollenspiel|mini-dialog|szene)\b/i.test(normalized);
|
||||||
|
|
||||||
return startsWithTaskVerb || startsWithTakeTask || (containsTaskChain && containsPracticeMarker);
|
return startsWithTaskVerb || startsWithTakeTask || (containsTaskChain && containsPracticeMarker);
|
||||||
@@ -678,6 +694,37 @@ export default {
|
|||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.length;
|
.length;
|
||||||
},
|
},
|
||||||
|
normalizeSpacedText(value) {
|
||||||
|
return String(value || '')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.normalize('NFKC')
|
||||||
|
.replace(/[\p{P}\p{S}]+/gu, ' ')
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.trim();
|
||||||
|
},
|
||||||
|
isGermanQuestionCue(value) {
|
||||||
|
return new Set(['wo', 'wo wohin', 'wohin', 'was', 'wer', 'wann', 'warum', 'wieso', 'wie'])
|
||||||
|
.has(this.normalizeSpacedText(value));
|
||||||
|
},
|
||||||
|
firstWordOfQuestion(value) {
|
||||||
|
const text = String(value || '').trim();
|
||||||
|
if (!/\?\s*$/.test(text) || this.wordCount(text) < 2) return '';
|
||||||
|
return text.split(/\s+/).find(Boolean) || '';
|
||||||
|
},
|
||||||
|
normalizeQuestionCuePair(item) {
|
||||||
|
const learning = String(item?.learning || '').trim();
|
||||||
|
const reference = String(item?.reference || '').trim();
|
||||||
|
if (this.isGermanQuestionCue(learning)) {
|
||||||
|
const firstWord = this.firstWordOfQuestion(reference);
|
||||||
|
if (firstWord) return { ...item, learning, reference: firstWord };
|
||||||
|
}
|
||||||
|
if (this.isGermanQuestionCue(reference)) {
|
||||||
|
const firstWord = this.firstWordOfQuestion(learning);
|
||||||
|
if (firstWord) return { ...item, learning: firstWord, reference };
|
||||||
|
}
|
||||||
|
return { ...item, learning, reference };
|
||||||
|
},
|
||||||
looksLikeFragmentMismatch(left, right) {
|
looksLikeFragmentMismatch(left, right) {
|
||||||
const leftWords = this.wordCount(left);
|
const leftWords = this.wordCount(left);
|
||||||
const rightWords = this.wordCount(right);
|
const rightWords = this.wordCount(right);
|
||||||
@@ -730,14 +777,15 @@ export default {
|
|||||||
|
|
||||||
const mapped = expanded
|
const mapped = expanded
|
||||||
.map(({ candidate, index, altIndex }) => {
|
.map(({ candidate, index, altIndex }) => {
|
||||||
const learning = String(candidate?.learning || '').trim();
|
const normalizedCandidate = this.normalizeQuestionCuePair(candidate);
|
||||||
const reference = String(candidate?.reference || '').trim();
|
const learning = String(normalizedCandidate?.learning || '').trim();
|
||||||
|
const reference = String(normalizedCandidate?.reference || '').trim();
|
||||||
if (!this.isTrainablePair(learning, reference)) return null;
|
if (!this.isTrainablePair(learning, reference)) return null;
|
||||||
const key = `${this.normalize(learning)}|${this.normalize(reference)}`;
|
const key = `${this.normalize(learning)}|${this.normalize(reference)}`;
|
||||||
if (seen.has(key)) return null;
|
if (seen.has(key)) return null;
|
||||||
seen.add(key);
|
seen.add(key);
|
||||||
return {
|
return {
|
||||||
...candidate,
|
...normalizedCandidate,
|
||||||
id: candidate?.id || candidate?.itemKey || candidate?.key || `${key}|${index}|${altIndex}`,
|
id: candidate?.id || candidate?.itemKey || candidate?.key || `${key}|${index}|${altIndex}`,
|
||||||
learning,
|
learning,
|
||||||
reference
|
reference
|
||||||
@@ -782,7 +830,7 @@ export default {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Aktuelle Frage behalten, nur UI/Antwortmodus neu aufbauen
|
// Aktuelle Frage behalten, nur UI/Antwortmodus neu aufbauen
|
||||||
const prompt = this.currentPrompt;
|
const prompt = this.currentAnswerPrompt;
|
||||||
this.acceptableAnswers = this.getAnswersForPrompt(prompt, this.direction);
|
this.acceptableAnswers = this.getAnswersForPrompt(prompt, this.direction);
|
||||||
if (this.simpleMode) {
|
if (this.simpleMode) {
|
||||||
this.buildChoices();
|
this.buildChoices();
|
||||||
@@ -1067,7 +1115,7 @@ export default {
|
|||||||
return items[items.length - 1];
|
return items[items.length - 1];
|
||||||
},
|
},
|
||||||
buildChoices() {
|
buildChoices() {
|
||||||
const prompt = this.currentPrompt;
|
const prompt = this.currentAnswerPrompt;
|
||||||
const acceptable = this.getAnswersForPrompt(prompt, this.direction);
|
const acceptable = this.getAnswersForPrompt(prompt, this.direction);
|
||||||
this.acceptableAnswers = acceptable;
|
this.acceptableAnswers = acceptable;
|
||||||
|
|
||||||
@@ -1290,7 +1338,7 @@ export default {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!this.current) return;
|
if (!this.current) return;
|
||||||
const prompt = this.currentPrompt;
|
const prompt = this.currentAnswerPrompt;
|
||||||
this.acceptableAnswers = this.getAnswersForPrompt(prompt, this.direction);
|
this.acceptableAnswers = this.getAnswersForPrompt(prompt, this.direction);
|
||||||
if (this.simpleMode) this.buildChoices();
|
if (this.simpleMode) this.buildChoices();
|
||||||
this.$nextTick(() => {
|
this.$nextTick(() => {
|
||||||
|
|||||||
Reference in New Issue
Block a user