harheimertc/server/utils/qttr-import.js

import { promises as fs } from 'fs'
import { getServerDataPath } from './paths.js'

const QTTR_URL = 'https://www.mytischtennis.de/rankings/andro-rangliste?continent=all&country=Deutschland&all-players=on&as=DE.WE.R4.07&di=DE.WE.R4.07.04&area=DE.WE.R4.07.04.43&clubnr-search=Harheimer+TC&clubnr=43030&fednickname=HeTTV&gender=all&current-ranking=no&ttr-range=100%3B3000&birth-range=1926%3B2021'
const OUTPUT_FILE = getServerDataPath('qttr-values.json')

function decodeHtmlEntities(value) {
  const namedEntities = {
    amp: '&',
    apos: "'",
    gt: '>',
    lt: '<',
    nbsp: ' ',
    quot: '"'
  }

  return String(value || '')
    .replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, entity) => {
      if (entity.startsWith('#x')) {
        const codePoint = Number.parseInt(entity.slice(2), 16)
        return Number.isNaN(codePoint) ? match : String.fromCodePoint(codePoint)
      }

      if (entity.startsWith('#')) {
        const codePoint = Number.parseInt(entity.slice(1), 10)
        return Number.isNaN(codePoint) ? match : String.fromCodePoint(codePoint)
      }

      return Object.prototype.hasOwnProperty.call(namedEntities, entity) ? namedEntities[entity] : match
    })
}

function stripTags(value) {
  return decodeHtmlEntities(String(value || '')
    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
    .replace(/<br\s*\/?/gi, ' ')
    .replace(/<[^>]+>/g, ' ')
    .replace(/\s+/g, ' ')
    .trim())
}

function normalizeHeaderKey(value) {
  return String(value || '')
    .normalize('NFKD')
    .replace(/[\u0300-\u036f]/g, '')
    .toLowerCase()
    .replace(/ß/g, 'ss')
    .replace(/[^a-z0-9]+/g, '_')
    .replace(/^_+|_+$/g, '')
}

function toNumberOrNull(value) {
  const raw = String(value || '').replace(',', '.').match(/-?\d+(?:\.\d+)?/)
  if (!raw) return null
  const numberValue = Number(raw[0])
  return Number.isNaN(numberValue) ? null : numberValue
}

function normalizeName(value) {
  return String(value || '')
    .trim()
    .toLowerCase()
    .normalize('NFKD')
    .replace(/[\u0300-\u036f]/g, '')
    .replace(/\s+/g, ' ')
    .replace(/[’'`]/g, '')
}

function normalizeGender(value) {
  const normalized = String(value || '').trim().toLowerCase()
  if (normalized === 'm' || normalized === 'männlich') return 'männlich'
  if (normalized === 'w' || normalized === 'weiblich') return 'weiblich'
  return normalized || null
}

function extractTableBlocks(html) {
  return [...String(html || '').matchAll(/<table\b[^>]*>[\s\S]*?<\/table>/gi)].map((match) => match[0])
}

function extractCellTexts(rowHtml) {
  return [...String(rowHtml || '').matchAll(/<(?:t[hd])\b[^>]*>([\s\S]*?)<\/t[hd]>/gi)].map((match) => stripTags(match[1]))
}

function findBestTable(html) {
  const tables = extractTableBlocks(html)
  if (tables.length === 0) return null

  return (
    tables.find((table) => /Harheimer TC/i.test(table) && /Q-?TTR/i.test(table)) ||
    tables.find((table) => /Harheimer TC/i.test(table)) ||
    tables.find((table) => /Q-?TTR/i.test(table)) ||
    tables[0]
  )
}

function extractTableTitle(html, tableHtml) {
  const tableIndex = String(html || '').indexOf(tableHtml)
  if (tableIndex === -1) return null

  const prefix = String(html || '').slice(0, tableIndex)
  const headingMatches = [...prefix.matchAll(/<(h[1-6])\b[^>]*>([\s\S]*?)<\/\1>/gi)]
  if (headingMatches.length === 0) return null

  return stripTags(headingMatches[headingMatches.length - 1][2]) || null
}

function extractRowsFromTable(tableHtml) {
  const rows = [...String(tableHtml || '').matchAll(/<tr\b[^>]*>([\s\S]*?)<\/tr>/gi)]
  if (rows.length === 0) return { headers: [], rows: [] }

  const headerRow = rows.find((row) => /<th\b/i.test(row[0]))
  const headerCells = headerRow ? extractCellTexts(headerRow[1]) : extractCellTexts(rows[0][1])
  const headers = headerCells.map((cell) => ({
    label: cell,
    key: normalizeHeaderKey(cell)
  }))

  const dataRows = rows
    .filter((row) => row !== headerRow)
    .map((row) => extractCellTexts(row[1]))
    .filter((cells) => cells.length > 0 && cells.some((cell) => cell !== ''))

  return { headers, rows: dataRows }
}

function deriveQttrFields(headers, cells) {
  const valuesByHeader = {}
  headers.forEach((header, index) => {
    valuesByHeader[header.key] = cells[index] ?? null
  })

  const lookup = (pattern) => {
    const entry = headers.find((header) => pattern.test(header.key))
    return entry ? valuesByHeader[entry.key] : null
  }

  const rank = lookup(/platz|rank|rang/) ?? cells[0] ?? null
  const playerNumber = toNumberOrNull(cells[1] ?? lookup(/spieler.*nr|spielernr|id/))
  let gender = lookup(/geschlecht/) ?? null
  let playerName = lookup(/^(name|spielername)$/) ?? lookup(/\bspieler\b/) ?? null
  const combinedPlayerCell = cells[2] ?? lookup(/\bspieler\b/) ?? null
  const clubName = lookup(/verein|club/) ?? cells[3] ?? null
  const currentQttr = toNumberOrNull(cells[4] ?? lookup(/aktuell.*q.*ttr|current.*q.*ttr|q.*ttr/))
  const previousQttr = toNumberOrNull(lookup(/vorher|previous/))

  if (combinedPlayerCell) {
    const genderAndName = String(combinedPlayerCell).match(/^(m|w|männlich|weiblich)\s+(.*)$/i)
    if (genderAndName) {
      gender = gender ?? normalizeGender(genderAndName[1])
      playerName = genderAndName[2].trim()
    } else {
      playerName = playerName ?? String(combinedPlayerCell).trim()
    }
  }

  gender = normalizeGender(gender)

  return {
    rank: toNumberOrNull(rank),
    playerNumber,
    gender,
    playerName,
    clubName,
    currentQttr,
    previousQttr,
    valuesByHeader,
    rawCells: cells
  }
}

export async function importQttrValues(options = {}) {
  const url = options.url || QTTR_URL
  const response = await fetch(url, {
    headers: {
      accept: 'text/html,application/xhtml+xml',
      'accept-language': 'de-DE,de;q=0.9'
    }
  })

  if (!response.ok) {
    throw new Error(`QTTR-Download fehlgeschlagen: HTTP ${response.status}`)
  }

  const html = await response.text()
  const tableHtml = findBestTable(html)

  if (!tableHtml) {
    throw new Error('Keine QTTR-Tabelle im HTML gefunden')
  }

  const { headers, rows } = extractRowsFromTable(tableHtml)
  if (headers.length === 0 || rows.length === 0) {
    throw new Error('QTTR-Tabelle ist leer oder unvollständig')
  }

  const parsedRows = rows.map((cells) => deriveQttrFields(headers, cells))
  const payload = {
    format: 'harheimertc.qttr.v1',
    importedAt: new Date().toISOString(),
    source: {
      url
    },
    title: extractTableTitle(html, tableHtml),
    headerCount: headers.length,
    rowCount: parsedRows.length,
    headers,
    rows: parsedRows
  }

  await fs.mkdir(getServerDataPath(), { recursive: true })
  await fs.writeFile(OUTPUT_FILE, `${JSON.stringify(payload, null, 2)}\n`, 'utf8')

  return {
    outputFile: OUTPUT_FILE,
    tableCount: 1,
    rowCount: parsedRows.length,
    ...payload
  }
}