From f841a35501b15d34e792f4092010754d4e5cab7e Mon Sep 17 00:00:00 2001
From: "Torsten Schulz (local)" <tsschulz@gmx.net>
Date: Tue, 14 Apr 2026 15:33:00 +0200
Subject: [PATCH] feat(bisaya-course): enhance gap-fill exercise processing and
 validation

- Added `buildCorePatternGlossLookup` to create a mapping of core patterns for improved hint sanitization.
- Implemented `sanitizeGapFillHintText` to validate and replace hints based on answer word count and gloss availability, generating fixes and warnings as needed.
- Updated `sanitizeExerciseForConsistency` to incorporate new sanitization logic for gap-fill exercises, ensuring consistent exercise data handling.
- Enhanced logging in `createBisayaCourseContent` to provide detailed feedback on exercise processing and validation outcomes.
---
 .../scripts/create-bisaya-course-content.js   | 110 ++++++++++++++++--
 1 file changed, 100 insertions(+), 10 deletions(-)

diff --git a/backend/scripts/create-bisaya-course-content.js b/backend/scripts/create-bisaya-course-content.js
index 165275b..4dd4bc5 100644
--- a/backend/scripts/create-bisaya-course-content.js
+++ b/backend/scripts/create-bisaya-course-content.js
@@ -74,6 +74,84 @@ function collectExerciseAuditWarnings(lessonTitle, exerciseData, exerciseNumber)
   return warnings;
 }
 
+function buildCorePatternGlossLookup(didactics) {
+  const map = new Map();
+  const patterns = Array.isArray(didactics?.corePatterns) ? didactics.corePatterns : [];
+  patterns.forEach((entry) => {
+    const normalized = normalizeCorePatternEntry(entry);
+    if (!normalized?.target || !normalized?.gloss) return;
+    map.set(normalized.target.toLowerCase(), normalized.gloss);
+  });
+  return map;
+}
+
+function sanitizeGapFillHintText(lessonTitle, text, answers, glossLookup) {
+  const source = String(text || '');
+  const normalizedAnswers = Array.isArray(answers)
+    ? answers.map((answer) => normalizeText(answer))
+    : [];
+  if (!source || normalizedAnswers.length === 0) {
+    return { text: source, fixes: [], warnings: [] };
+  }
+
+  const hintRegex = /\(([^)]+)\)/g;
+  let hintIndex = 0;
+  const fixes = [];
+  const warnings = [];
+  const rebuilt = source.replace(hintRegex, (full, inner) => {
+    const answer = normalizedAnswers[hintIndex] || '';
+    const hint = normalizeText(inner);
+    hintIndex += 1;
+    if (!answer || !hint) return full;
+
+    const answerWords = countWords(answer);
+    const hintWords = countWords(hint);
+    if (!(answerWords <= 2 && hintWords >= 4)) {
+      return full;
+    }
+
+    const mappedGloss = normalizeText(glossLookup.get(answer.toLowerCase()) || '');
+    if (mappedGloss && countWords(mappedGloss) <= 3) {
+      fixes.push(`[${lessonTitle}] "${answer}": "${hint}" -> "${mappedGloss}"`);
+      return `(${mappedGloss})`;
+    }
+
+    warnings.push(
+      `[${lessonTitle}] Keine sichere Gloss für "${answer}" gefunden; langer Hinweis bleibt: "${hint}"`
+    );
+    return full;
+  });
+
+  return { text: rebuilt, fixes, warnings };
+}
+
+function sanitizeExerciseForConsistency(lesson, exerciseData, didactics) {
+  const exercise = { ...exerciseData };
+  const questionData = { ...(exerciseData?.questionData || {}) };
+  const answerData = { ...(exerciseData?.answerData || {}) };
+  const fixes = [];
+  const warnings = [];
+
+  if (questionData.type === 'gap_fill') {
+    const glossLookup = buildCorePatternGlossLookup(didactics);
+    const sanitized = sanitizeGapFillHintText(
+      lesson.title,
+      questionData.text,
+      Array.isArray(answerData.answers) ? answerData.answers : [],
+      glossLookup
+    );
+    if (sanitized.text !== questionData.text) {
+      questionData.text = sanitized.text;
+      fixes.push(...sanitized.fixes);
+    }
+    warnings.push(...sanitized.warnings);
+  }
+
+  exercise.questionData = questionData;
+  exercise.answerData = answerData;
+  return { exercise, fixes, warnings };
+}
+
 function normalizeCorePatternEntry(entry) {
   if (entry === null || entry === undefined || entry === '') {
     return null;
@@ -4292,6 +4370,7 @@ function getExercisesForLesson(lesson) {
 async function createBisayaCourseContent() {
   await sequelize.authenticate();
   console.log('Datenbankverbindung erfolgreich hergestellt.\n');
+  const forceRebuildAll = process.env.VOCAB_FORCE_REBUILD_ALL === '1';
 
   const systemUser = await findOrCreateSystemUser();
   console.log(`Verwende System-Benutzer: ${systemUser.username} (ID: ${systemUser.id})\n`);
@@ -4362,33 +4441,44 @@ async function createBisayaCourseContent() {
         where: { lessonId: lesson.id }
       });
 
-      if (existingCount > 0 && !replacePlaceholders) {
+      if (existingCount > 0 && !replacePlaceholders && !forceRebuildAll) {
         console.log(`   ⏭️  Lektion ${lesson.lessonNumber}: "${lesson.title}" - bereits ${existingCount} Übung(en) vorhanden`);
         continue;
       }
 
-      if (replacePlaceholders && existingCount > 0) {
+      if ((replacePlaceholders || forceRebuildAll) && existingCount > 0) {
         const deleted = await VocabGrammarExercise.destroy({ where: { lessonId: lesson.id } });
-        console.log(`   🗑️  Lektion ${lesson.lessonNumber}: "${lesson.title}" - ${deleted} Platzhalter entfernt`);
+        const reason = forceRebuildAll ? 'vollständig neu aufgebaut' : 'Platzhalter entfernt';
+        console.log(`   🗑️  Lektion ${lesson.lessonNumber}: "${lesson.title}" - ${deleted} Übungen entfernt (${reason})`);
       }
       
       // Erstelle Übungen
+      const lessonDidactics = getLessonDidactics(lesson);
       let exerciseNumber = 1;
       for (const exerciseData of exercises) {
+        const { exercise, fixes, warnings } = sanitizeExerciseForConsistency(
+          lesson,
+          exerciseData,
+          lessonDidactics
+        );
         if (process.env.VOCAB_STRICT_AUDIT === '1') {
-          const warnings = collectExerciseAuditWarnings(lesson.title, exerciseData, exerciseNumber);
+          fixes.forEach((fix) => console.log(`   🛠️  ${fix}`));
           warnings.forEach((warning) => console.warn(`   ⚠️  ${warning}`));
         }
-        const exerciseTypeId = await resolveExerciseTypeId(exerciseData);
+        if (process.env.VOCAB_STRICT_AUDIT === '1') {
+          const warnings = collectExerciseAuditWarnings(lesson.title, exercise, exerciseNumber);
+          warnings.forEach((warning) => console.warn(`   ⚠️  ${warning}`));
+        }
+        const exerciseTypeId = await resolveExerciseTypeId(exercise);
         await VocabGrammarExercise.create({
           lessonId: lesson.id,
           exerciseTypeId,
           exerciseNumber: exerciseNumber++,
-          title: exerciseData.title,
-          instruction: exerciseData.instruction,
-          questionData: JSON.stringify(exerciseData.questionData),
-          answerData: JSON.stringify(exerciseData.answerData),
-          explanation: exerciseData.explanation,
+          title: exercise.title,
+          instruction: exercise.instruction,
+          questionData: JSON.stringify(exercise.questionData),
+          answerData: JSON.stringify(exercise.answerData),
+          explanation: exercise.explanation,
           createdByUserId: course.ownerUserId || systemUser.id
         });
         totalExercisesAdded++;