diff --git a/backend/migrations/create_http_page_fetch_log.sql b/backend/migrations/create_http_page_fetch_log.sql new file mode 100644 index 00000000..126e39d7 --- /dev/null +++ b/backend/migrations/create_http_page_fetch_log.sql @@ -0,0 +1,25 @@ +-- Migration: Create http_page_fetch_log table for logging HTTP page fetches (HTTV/click-TT etc.) +-- Dient zum Verständnis der externen Seiten-Struktur und URL-Varianten je nach Verband/Saison + +CREATE TABLE IF NOT EXISTS http_page_fetch_log ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NULL COMMENT 'Optional: User der den Aufruf ausgelöst hat', + fetch_type VARCHAR(64) NOT NULL COMMENT 'z.B. leaguePage, clubInfoDisplay, regionMeetingFilter', + base_domain VARCHAR(255) NOT NULL COMMENT 'z.B. httv.click-tt.de', + full_url TEXT NOT NULL COMMENT 'Vollständige aufgerufene URL', + association VARCHAR(64) NULL COMMENT 'Verband (z.B. HeTTV, RTTV)', + championship VARCHAR(128) NULL COMMENT 'Championship-Parameter (z.B. HTTV 25/26)', + club_id_param VARCHAR(64) NULL COMMENT 'Club-ID falls clubInfoDisplay', + http_status INT NULL COMMENT 'HTTP-Status der Response', + success BOOLEAN NOT NULL DEFAULT FALSE, + response_snippet TEXT NULL COMMENT 'Gekürzter Response-Anfang (max 2000 Zeichen) zur Strukturanalyse', + content_type VARCHAR(128) NULL COMMENT 'Content-Type der Response', + error_message TEXT NULL, + execution_time_ms INT NULL COMMENT 'Laufzeit in Millisekunden', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + INDEX idx_created_at (created_at), + INDEX idx_base_domain_fetch_type (base_domain, fetch_type), + INDEX idx_association_championship (association, championship), + FOREIGN KEY (user_id) REFERENCES user(id) ON DELETE SET NULL ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/models/HttpPageFetchLog.js b/backend/models/HttpPageFetchLog.js new file mode 100644 index 00000000..394e4f0d --- /dev/null +++ b/backend/models/HttpPageFetchLog.js @@ -0,0 +1,93 @@ +import { DataTypes } from 'sequelize'; +import sequelize from '../database.js'; +import User from './User.js'; + +const HttpPageFetchLog = sequelize.define('HttpPageFetchLog', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + }, + userId: { + type: DataTypes.INTEGER, + allowNull: true, + references: { + model: User, + key: 'id', + }, + onDelete: 'SET NULL', + onUpdate: 'CASCADE', + }, + fetchType: { + type: DataTypes.STRING(64), + allowNull: false, + comment: 'z.B. leaguePage, clubInfoDisplay, regionMeetingFilter', + }, + baseDomain: { + type: DataTypes.STRING(255), + allowNull: false, + comment: 'z.B. httv.click-tt.de', + }, + fullUrl: { + type: DataTypes.TEXT, + allowNull: false, + }, + association: { + type: DataTypes.STRING(64), + allowNull: true, + comment: 'Verband (z.B. HeTTV, RTTV)', + }, + championship: { + type: DataTypes.STRING(128), + allowNull: true, + comment: 'Championship-Parameter (z.B. HTTV 25/26)', + }, + clubIdParam: { + type: DataTypes.STRING(64), + allowNull: true, + comment: 'Club-ID falls clubInfoDisplay', + }, + httpStatus: { + type: DataTypes.INTEGER, + allowNull: true, + }, + success: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }, + responseSnippet: { + type: DataTypes.TEXT, + allowNull: true, + comment: 'Gekürzter Response-Anfang zur Strukturanalyse', + }, + contentType: { + type: DataTypes.STRING(128), + allowNull: true, + }, + errorMessage: { + type: DataTypes.TEXT, + allowNull: true, + }, + executionTimeMs: { + type: DataTypes.INTEGER, + allowNull: true, + comment: 'Laufzeit in Millisekunden', + }, +}, { + underscored: true, + tableName: 'http_page_fetch_log', + timestamps: true, + updatedAt: false, + indexes: [ + { fields: ['created_at'] }, + { fields: ['base_domain', 'fetch_type'] }, + { fields: ['association', 'championship'] }, + ], +}); + +HttpPageFetchLog.belongsTo(User, { foreignKey: 'userId', as: 'user' }); +User.hasMany(HttpPageFetchLog, { foreignKey: 'userId', as: 'httpPageFetchLogs' }); + +export default HttpPageFetchLog; diff --git a/backend/routes/clickTtHttpPageRoutes.js b/backend/routes/clickTtHttpPageRoutes.js new file mode 100644 index 00000000..7d3bb575 --- /dev/null +++ b/backend/routes/clickTtHttpPageRoutes.js @@ -0,0 +1,234 @@ +/** + * API-Routes für HTTV/click-TT HTTP-Seiten mit Logging. + * Ermöglicht das Testen verschiedener URLs und das Auslesen der Logs. + */ + +import express from 'express'; +import clickTtHttpPageService from '../services/clickTtHttpPageService.js'; +import HttpPageFetchLog from '../models/HttpPageFetchLog.js'; +import { authenticate } from '../middleware/authenticate.js'; +import { Op } from 'sequelize'; + +const router = express.Router(); + +/** + * GET /api/clicktt/league-page + * Ruft die Ligenübersicht ab (leaguePage) + * Query: association, championship + */ +router.get('/league-page', authenticate, async (req, res, next) => { + try { + const association = req.query.association || 'HeTTV'; + const championship = req.query.championship || 'HTTV 25/26'; + const userId = req.user?.id ?? null; + + const result = await clickTtHttpPageService.fetchLeaguePage({ + association, + championship, + userId, + }); + + res.json({ + success: result.success, + status: result.status, + contentType: result.contentType, + executionTimeMs: result.executionTimeMs, + bodyLength: result.body?.length ?? 0, + body: result.body, + }); + } catch (error) { + next(error); + } +}); + +/** + * GET /api/clicktt/club-info + * Ruft die Vereinsinfo ab (clubInfoDisplay) + * Query: association, clubId + */ +router.get('/club-info', authenticate, async (req, res, next) => { + try { + const association = req.query.association || 'HeTTV'; + const clubId = req.query.clubId; + const userId = req.user?.id ?? null; + + if (!clubId) { + return res.status(400).json({ error: 'clubId ist erforderlich' }); + } + + const result = await clickTtHttpPageService.fetchClubInfoDisplay({ + association, + clubId, + userId, + }); + + res.json({ + success: result.success, + status: result.status, + contentType: result.contentType, + executionTimeMs: result.executionTimeMs, + bodyLength: result.body?.length ?? 0, + body: result.body, + }); + } catch (error) { + next(error); + } +}); + +/** + * GET /api/clicktt/region-meetings + * Ruft den Regionsspielplan ab (regionMeetingFilter) + */ +router.get('/region-meetings', authenticate, async (req, res, next) => { + try { + const association = req.query.association || 'HeTTV'; + const championship = req.query.championship || 'HTTV 25/26'; + const userId = req.user?.id ?? null; + + const result = await clickTtHttpPageService.fetchRegionMeetingFilter({ + association, + championship, + userId, + }); + + res.json({ + success: result.success, + status: result.status, + contentType: result.contentType, + executionTimeMs: result.executionTimeMs, + bodyLength: result.body?.length ?? 0, + body: result.body, + }); + } catch (error) { + next(error); + } +}); + +/** + * GET /api/clicktt/fetch + * Ruft eine beliebige URL ab (für manuelle Tests) + * Query: url (erforderlich) + */ +router.get('/fetch', authenticate, async (req, res, next) => { + try { + const url = req.query.url; + const userId = req.user?.id ?? null; + + if (!url) { + return res.status(400).json({ error: 'url ist erforderlich' }); + } + + // Nur click-tt.de und httv.de erlauben + if (!url.includes('click-tt.de') && !url.includes('httv.de')) { + return res.status(400).json({ + error: 'Nur URLs von click-tt.de oder httv.de sind erlaubt', + }); + } + + const result = await clickTtHttpPageService.fetchArbitraryUrl(url, { + fetchType: 'arbitrary', + userId, + }); + + res.json({ + success: result.success, + status: result.status, + contentType: result.contentType, + executionTimeMs: result.executionTimeMs, + bodyLength: result.body?.length ?? 0, + body: result.body, + }); + } catch (error) { + next(error); + } +}); + +/** + * GET /api/clicktt/logs + * Liefert die letzten HTTP-Page-Fetch-Logs + * Query: limit (default 50), fetchType, association, success + */ +router.get('/logs', authenticate, async (req, res, next) => { + try { + const limit = Math.min(parseInt(req.query.limit, 10) || 50, 200); + const { fetchType, association, success } = req.query; + + const where = {}; + if (fetchType) where.fetchType = fetchType; + if (association) where.association = association; + if (success !== undefined) where.success = success === 'true'; + + const logs = await HttpPageFetchLog.findAll({ + where: Object.keys(where).length > 0 ? where : undefined, + order: [['createdAt', 'DESC']], + limit, + attributes: [ + 'id', + 'fetchType', + 'baseDomain', + 'fullUrl', + 'association', + 'championship', + 'clubIdParam', + 'httpStatus', + 'success', + 'responseSnippet', + 'contentType', + 'errorMessage', + 'executionTimeMs', + 'createdAt', + ], + }); + + res.json({ + success: true, + count: logs.length, + logs, + }); + } catch (error) { + next(error); + } +}); + +/** + * GET /api/clicktt/url-info + * Liefert Informationen zur URL-Struktur (Verband→Domain, Beispiel-URLs) + */ +router.get('/url-info', authenticate, async (req, res, next) => { + try { + const info = { + associationToDomain: clickTtHttpPageService.ASSOCIATION_TO_DOMAIN, + exampleUrls: { + leaguePage: clickTtHttpPageService.buildLeaguePageUrl({ + association: 'HeTTV', + championship: 'HTTV 25/26', + }), + leaguePageBezirk: clickTtHttpPageService.buildLeaguePageUrl({ + association: 'HeTTV', + championship: 'K43 25/26', + }), + clubInfoDisplay: clickTtHttpPageService.buildClubInfoDisplayUrl({ + association: 'HeTTV', + clubId: '1060', + }), + regionMeetingFilter: clickTtHttpPageService.buildRegionMeetingFilterUrl({ + association: 'HeTTV', + championship: 'HTTV 25/26', + }), + }, + championshipExamples: [ + 'HTTV 25/26', + 'HTTV 24/25', + 'K43 25/26', + 'K16 25/26', + 'RL-OL West 25/26', + ], + }; + + res.json({ success: true, ...info }); + } catch (error) { + next(error); + } +}); + +export default router; diff --git a/backend/server.js b/backend/server.js index 60b5d0f9..628b1a91 100644 --- a/backend/server.js +++ b/backend/server.js @@ -48,6 +48,7 @@ import nuscoreApiRoutes from './routes/nuscoreApiRoutes.js'; import memberActivityRoutes from './routes/memberActivityRoutes.js'; import permissionRoutes from './routes/permissionRoutes.js'; import apiLogRoutes from './routes/apiLogRoutes.js'; +import clickTtHttpPageRoutes from './routes/clickTtHttpPageRoutes.js'; import memberTransferConfigRoutes from './routes/memberTransferConfigRoutes.js'; import trainingGroupRoutes from './routes/trainingGroupRoutes.js'; import trainingTimeRoutes from './routes/trainingTimeRoutes.js'; @@ -115,6 +116,7 @@ app.use('/api/nuscore', nuscoreApiRoutes); app.use('/api/member-activities', memberActivityRoutes); app.use('/api/permissions', permissionRoutes); app.use('/api/logs', apiLogRoutes); +app.use('/api/clicktt', clickTtHttpPageRoutes); app.use('/api/member-transfer-config', memberTransferConfigRoutes); app.use('/api/training-groups', trainingGroupRoutes); app.use('/api/training-times', trainingTimeRoutes); diff --git a/backend/services/clickTtHttpPageService.js b/backend/services/clickTtHttpPageService.js new file mode 100644 index 00000000..fbb8061c --- /dev/null +++ b/backend/services/clickTtHttpPageService.js @@ -0,0 +1,242 @@ +/** + * Service für HTTP-Aufrufe an click-TT/HTTV-Seiten mit Logging. + * Dient zum Verständnis der Seiten-Struktur und URL-Varianten je nach Verband/Saison. + * + * URL-Struktur httv.click-tt.de (und andere Verbände): + * - leaguePage: /cgi-bin/WebObjects/nuLigaTTDE.woa/wa/leaguePage?championship=HTTV+25%2F26 + * - regionMeetingFilter: /cgi-bin/WebObjects/nuLigaTTDE.woa/wa/regionMeetingFilter?championship=HTTV+25%2F26 + * - clubInfoDisplay: /cgi-bin/WebObjects/nuLigaTTDE.woa/wa/clubInfoDisplay?club=1060 + */ + +import fetch from 'node-fetch'; +import HttpPageFetchLog from '../models/HttpPageFetchLog.js'; +import { devLog } from '../utils/logger.js'; + +/** Verband → click-TT Subdomain */ +const ASSOCIATION_TO_DOMAIN = { + HeTTV: 'httv.click-tt.de', + HTTV: 'httv.click-tt.de', + RTTV: 'rttv.click-tt.de', + WTTV: 'wttv.click-tt.de', + TTVNw: 'ttvnw.click-tt.de', + BTTV: 'battv.click-tt.de', +}; + +const SNIPPET_MAX_LEN = 2000; + +/** + * Ermittelt die click-TT-Domain für einen Verband + */ +function getDomainForAssociation(association) { + if (!association) return null; + const key = association.replace(/\s/g, ''); + return ASSOCIATION_TO_DOMAIN[association] ?? ASSOCIATION_TO_DOMAIN[key] ?? null; +} + +/** + * Baut die leaguePage-URL für einen Verband + * @param {Object} opts + * @param {string} opts.association - z.B. HeTTV, HTTV + * @param {string} opts.championship - z.B. "HTTV 25/26" oder "K43 25/26" (Frankfurt) + */ +function buildLeaguePageUrl(opts) { + const { association, championship } = opts; + const domain = getDomainForAssociation(association) || 'httv.click-tt.de'; + const champEncoded = encodeURIComponent(championship || 'HTTV 25/26'); + return `https://${domain}/cgi-bin/WebObjects/nuLigaTTDE.woa/wa/leaguePage?championship=${champEncoded}`; +} + +/** + * Baut die clubInfoDisplay-URL + */ +function buildClubInfoDisplayUrl(opts) { + const { association, clubId } = opts; + const domain = getDomainForAssociation(association) || 'httv.click-tt.de'; + return `https://${domain}/cgi-bin/WebObjects/nuLigaTTDE.woa/wa/clubInfoDisplay?club=${encodeURIComponent(clubId)}`; +} + +/** + * Baut die regionMeetingFilter-URL + */ +function buildRegionMeetingFilterUrl(opts) { + const { association, championship } = opts; + const domain = getDomainForAssociation(association) || 'httv.click-tt.de'; + const champEncoded = encodeURIComponent(championship || 'HTTV 25/26'); + return `https://${domain}/cgi-bin/WebObjects/nuLigaTTDE.woa/wa/regionMeetingFilter?championship=${champEncoded}`; +} + +/** + * Extrahiert die Basis-Domain aus einer URL + */ +function extractBaseDomain(url) { + try { + const u = new URL(url); + return u.hostname; + } catch { + return 'unknown'; + } +} + +/** + * Erstellt einen Response-Snippet (gekürzt) + */ +function createResponseSnippet(body, maxLen = SNIPPET_MAX_LEN) { + if (!body || typeof body !== 'string') return null; + const trimmed = body.trim(); + if (trimmed.length <= maxLen) return trimmed; + return trimmed.substring(0, maxLen) + '\n...[gekürzt]'; +} + +/** + * Führt einen HTTP-Aufruf durch und loggt das Ergebnis + */ +async function fetchWithLogging(options) { + const { + url, + fetchType = 'unknown', + association = null, + championship = null, + clubIdParam = null, + userId = null, + method = 'GET', + } = options; + + const baseDomain = extractBaseDomain(url); + const startTime = Date.now(); + + try { + const response = await fetch(url, { + method, + headers: { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'de,en-US;q=0.7,en;q=0.3', + }, + timeout: 30000, + }); + + const executionTimeMs = Date.now() - startTime; + const contentType = response.headers.get('content-type') || null; + const body = await response.text(); + const success = response.ok; + const snippet = createResponseSnippet(body); + + await HttpPageFetchLog.create({ + userId, + fetchType, + baseDomain, + fullUrl: url, + association, + championship, + clubIdParam, + httpStatus: response.status, + success, + responseSnippet: snippet, + contentType, + errorMessage: success ? null : `HTTP ${response.status}: ${response.statusText}`, + executionTimeMs, + }); + + devLog(`[ClickTT] ${fetchType} ${url} → ${response.status} (${executionTimeMs}ms)`); + + return { + success, + status: response.status, + contentType, + body, + executionTimeMs, + }; + } catch (error) { + const executionTimeMs = Date.now() - startTime; + + await HttpPageFetchLog.create({ + userId, + fetchType, + baseDomain, + fullUrl: url, + association, + championship, + clubIdParam, + httpStatus: null, + success: false, + responseSnippet: null, + contentType: null, + errorMessage: error.message, + executionTimeMs, + }); + + devLog(`[ClickTT] ${fetchType} ${url} → ERROR: ${error.message}`); + + throw error; + } +} + +/** + * Ruft die leaguePage-Seite ab (Ligenübersicht) + */ +async function fetchLeaguePage(opts) { + const { association = 'HeTTV', championship = 'HTTV 25/26', userId } = opts; + const url = buildLeaguePageUrl({ association, championship }); + return fetchWithLogging({ + url, + fetchType: 'leaguePage', + association, + championship, + userId, + }); +} + +/** + * Ruft die clubInfoDisplay-Seite ab (Vereinsinfo) + */ +async function fetchClubInfoDisplay(opts) { + const { association = 'HeTTV', clubId, userId } = opts; + const url = buildClubInfoDisplayUrl({ association, clubId }); + return fetchWithLogging({ + url, + fetchType: 'clubInfoDisplay', + association, + clubIdParam: String(clubId), + userId, + }); +} + +/** + * Ruft die regionMeetingFilter-Seite ab (Regionsspielplan) + */ +async function fetchRegionMeetingFilter(opts) { + const { association = 'HeTTV', championship = 'HTTV 25/26', userId } = opts; + const url = buildRegionMeetingFilterUrl({ association, championship }); + return fetchWithLogging({ + url, + fetchType: 'regionMeetingFilter', + association, + championship, + userId, + }); +} + +/** + * Ruft eine beliebige URL ab (für manuelle Tests) + */ +async function fetchArbitraryUrl(url, opts = {}) { + const { fetchType = 'arbitrary', userId } = opts; + return fetchWithLogging({ + url, + fetchType, + userId, + }); +} + +export default { + fetchLeaguePage, + fetchClubInfoDisplay, + fetchRegionMeetingFilter, + fetchArbitraryUrl, + fetchWithLogging, + buildLeaguePageUrl, + buildClubInfoDisplayUrl, + buildRegionMeetingFilterUrl, + getDomainForAssociation, + ASSOCIATION_TO_DOMAIN, +};