diff --git a/backend/migrations/20260310_add_clicktt_payload_logging.sql b/backend/migrations/20260310_add_clicktt_payload_logging.sql new file mode 100644 index 00000000..65fa9321 --- /dev/null +++ b/backend/migrations/20260310_add_clicktt_payload_logging.sql @@ -0,0 +1,10 @@ +ALTER TABLE http_page_fetch_log + ADD COLUMN IF NOT EXISTS request_method VARCHAR(16) NULL COMMENT 'HTTP-Methode des ausgehenden Requests' AFTER club_id_param, + ADD COLUMN IF NOT EXISTS request_headers LONGTEXT NULL COMMENT 'Gesendete Request-Header als JSON' AFTER request_method, + ADD COLUMN IF NOT EXISTS request_body LONGTEXT NULL COMMENT 'Gesendeter Request-Body im Originalformat' AFTER request_headers, + ADD COLUMN IF NOT EXISTS response_headers LONGTEXT NULL COMMENT 'Empfangene Response-Header als JSON' AFTER content_type, + ADD COLUMN IF NOT EXISTS response_body LONGTEXT NULL COMMENT 'Vollstaendiger Response-Body' AFTER response_headers, + ADD COLUMN IF NOT EXISTS response_url TEXT NULL COMMENT 'Finale Response-URL nach Redirects' AFTER response_body; + +ALTER TABLE http_page_fetch_log + MODIFY COLUMN response_snippet LONGTEXT NULL COMMENT 'Gekuerzter oder kompletter Response-Anfang zur Strukturanalyse'; diff --git a/backend/migrations/create_http_page_fetch_log.sql b/backend/migrations/create_http_page_fetch_log.sql index 126e39d7..88aa6e7a 100644 --- a/backend/migrations/create_http_page_fetch_log.sql +++ b/backend/migrations/create_http_page_fetch_log.sql @@ -10,10 +10,16 @@ CREATE TABLE IF NOT EXISTS http_page_fetch_log ( association VARCHAR(64) NULL COMMENT 'Verband (z.B. HeTTV, RTTV)', championship VARCHAR(128) NULL COMMENT 'Championship-Parameter (z.B. HTTV 25/26)', club_id_param VARCHAR(64) NULL COMMENT 'Club-ID falls clubInfoDisplay', + request_method VARCHAR(16) NULL COMMENT 'HTTP-Methode des ausgehenden Requests', + request_headers LONGTEXT NULL COMMENT 'Gesendete Request-Header als JSON', + request_body LONGTEXT NULL COMMENT 'Gesendeter Request-Body im Originalformat', http_status INT NULL COMMENT 'HTTP-Status der Response', success BOOLEAN NOT NULL DEFAULT FALSE, response_snippet TEXT NULL COMMENT 'Gekürzter Response-Anfang (max 2000 Zeichen) zur Strukturanalyse', content_type VARCHAR(128) NULL COMMENT 'Content-Type der Response', + response_headers LONGTEXT NULL COMMENT 'Empfangene Response-Header als JSON', + response_body LONGTEXT NULL COMMENT 'Vollständiger Response-Body', + response_url TEXT NULL COMMENT 'Finale Response-URL nach Redirects', error_message TEXT NULL, execution_time_ms INT NULL COMMENT 'Laufzeit in Millisekunden', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, diff --git a/backend/models/HttpPageFetchLog.js b/backend/models/HttpPageFetchLog.js index 394e4f0d..0c4deecf 100644 --- a/backend/models/HttpPageFetchLog.js +++ b/backend/models/HttpPageFetchLog.js @@ -48,6 +48,21 @@ const HttpPageFetchLog = sequelize.define('HttpPageFetchLog', { allowNull: true, comment: 'Club-ID falls clubInfoDisplay', }, + requestMethod: { + type: DataTypes.STRING(16), + allowNull: true, + comment: 'HTTP-Methode des ausgehenden Requests', + }, + requestHeaders: { + type: DataTypes.TEXT('long'), + allowNull: true, + comment: 'Gesendete Request-Header als JSON', + }, + requestBody: { + type: DataTypes.TEXT('long'), + allowNull: true, + comment: 'Gesendeter Request-Body im Originalformat', + }, httpStatus: { type: DataTypes.INTEGER, allowNull: true, @@ -58,7 +73,7 @@ const HttpPageFetchLog = sequelize.define('HttpPageFetchLog', { defaultValue: false, }, responseSnippet: { - type: DataTypes.TEXT, + type: DataTypes.TEXT('long'), allowNull: true, comment: 'Gekürzter Response-Anfang zur Strukturanalyse', }, @@ -66,6 +81,21 @@ const HttpPageFetchLog = sequelize.define('HttpPageFetchLog', { type: DataTypes.STRING(128), allowNull: true, }, + responseHeaders: { + type: DataTypes.TEXT('long'), + allowNull: true, + comment: 'Empfangene Response-Header als JSON', + }, + responseBody: { + type: DataTypes.TEXT('long'), + allowNull: true, + comment: 'Vollstaendiger Response-Body', + }, + responseUrl: { + type: DataTypes.TEXT, + allowNull: true, + comment: 'Finale Response-URL nach Redirects', + }, errorMessage: { type: DataTypes.TEXT, allowNull: true, diff --git a/backend/routes/clickTtHttpPageRoutes.js b/backend/routes/clickTtHttpPageRoutes.js index 48cfa031..0c4c9f48 100644 --- a/backend/routes/clickTtHttpPageRoutes.js +++ b/backend/routes/clickTtHttpPageRoutes.js @@ -40,6 +40,22 @@ function getOrCreateSid(req) { return req.query.sid || crypto.randomBytes(16).toString('hex'); } +function serializeFormBody(req) { + if (typeof req.rawBody === 'string' && req.rawBody.length > 0) { + return req.rawBody; + } + + if (typeof req.body === 'string') { + return req.body; + } + + if (req.body && typeof req.body === 'object' && Object.keys(req.body).length > 0) { + return new URLSearchParams(req.body).toString(); + } + + return null; +} + /** Domains, deren Links durch den Proxy umgeleitet werden (für Folge-Logs) */ const PROXY_DOMAINS = ['click-tt.de', 'httv.de']; @@ -264,9 +280,7 @@ router.post('/proxy', async (req, res, next) => { } const contentType = req.get('content-type') || 'application/x-www-form-urlencoded'; - const body = typeof req.body === 'string' ? req.body : (req.body && Object.keys(req.body).length - ? new URLSearchParams(req.body).toString() - : null); + const body = serializeFormBody(req); const result = await clickTtHttpPageService.fetchWithLogging({ url: targetUrl, @@ -487,37 +501,52 @@ router.get('/logs', authenticate, async (req, res, next) => { try { const limit = Math.min(parseInt(req.query.limit, 10) || 50, 200); const { fetchType, association, success } = req.query; + const includePayloads = req.query.includePayloads === 'true'; const where = {}; if (fetchType) where.fetchType = fetchType; if (association) where.association = association; if (success !== undefined) where.success = success === 'true'; + const attributes = [ + 'id', + 'fetchType', + 'baseDomain', + 'fullUrl', + 'association', + 'championship', + 'clubIdParam', + 'requestMethod', + 'httpStatus', + 'success', + 'responseSnippet', + 'contentType', + 'responseUrl', + 'errorMessage', + 'executionTimeMs', + 'createdAt', + ]; + + if (includePayloads) { + attributes.push( + 'requestHeaders', + 'requestBody', + 'responseHeaders', + 'responseBody', + ); + } + const logs = await HttpPageFetchLog.findAll({ where: Object.keys(where).length > 0 ? where : undefined, order: [['createdAt', 'DESC']], limit, - attributes: [ - 'id', - 'fetchType', - 'baseDomain', - 'fullUrl', - 'association', - 'championship', - 'clubIdParam', - 'httpStatus', - 'success', - 'responseSnippet', - 'contentType', - 'errorMessage', - 'executionTimeMs', - 'createdAt', - ], + attributes, }); res.json({ success: true, count: logs.length, + includePayloads, logs, }); } catch (error) { diff --git a/backend/server.js b/backend/server.js index 628b1a91..823ad307 100644 --- a/backend/server.js +++ b/backend/server.js @@ -59,6 +59,11 @@ import HttpError from './exceptions/HttpError.js'; const app = express(); const port = process.env.PORT || 3005; +function captureRawBody(req, res, buf, encoding) { + if (!buf || buf.length === 0) return; + req.rawBody = buf.toString(encoding || 'utf8'); +} + const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -70,8 +75,8 @@ app.use(cors({ allowedHeaders: ['Content-Type', 'Authorization', 'authcode', 'userid'] })); -app.use(express.json()); -app.use(express.urlencoded({ extended: true })); +app.use(express.json({ verify: captureRawBody })); +app.use(express.urlencoded({ extended: true, verify: captureRawBody })); // Request Logging Middleware - loggt alle API-Requests // Wichtig: userId wird später in authMiddleware gesetzt, aber Middleware funktioniert auch ohne @@ -421,4 +426,4 @@ app.use((err, req, res, next) => { } catch (err) { console.error('Unable to synchronize the database:', err); } -})(); \ No newline at end of file +})(); diff --git a/backend/services/clickTtHttpPageService.js b/backend/services/clickTtHttpPageService.js index 9985f084..497b3e4e 100644 --- a/backend/services/clickTtHttpPageService.js +++ b/backend/services/clickTtHttpPageService.js @@ -24,6 +24,32 @@ const ASSOCIATION_TO_DOMAIN = { const SNIPPET_MAX_LEN = 2000; +function serializeHeaders(headers) { + if (!headers) return null; + + if (typeof headers.raw === 'function') { + try { + return JSON.stringify(headers.raw()); + } catch { + // Fallback below + } + } + + try { + const entries = {}; + if (typeof headers.forEach === 'function') { + headers.forEach((value, key) => { + entries[key] = value; + }); + } else { + Object.assign(entries, headers); + } + return JSON.stringify(entries); + } catch { + return null; + } +} + /** * Ermittelt die click-TT-Domain für einen Verband */ @@ -134,6 +160,8 @@ async function fetchWithLogging(options) { const responseBody = await response.text(); const success = response.ok; const snippet = createResponseSnippet(responseBody); + const requestHeaders = serializeHeaders(headers); + const responseHeaders = serializeHeaders(response.headers); await HttpPageFetchLog.create({ userId, @@ -143,10 +171,16 @@ async function fetchWithLogging(options) { association, championship, clubIdParam, + requestMethod: method, + requestHeaders, + requestBody: body, httpStatus: response.status, success, responseSnippet: snippet, contentType, + responseHeaders, + responseBody, + responseUrl: response.url || url, errorMessage: success ? null : `HTTP ${response.status}: ${response.statusText}`, executionTimeMs, }); @@ -172,10 +206,16 @@ async function fetchWithLogging(options) { association, championship, clubIdParam, + requestMethod: method, + requestHeaders: serializeHeaders(headers), + requestBody: body, httpStatus: null, success: false, responseSnippet: null, contentType: null, + responseHeaders: null, + responseBody: null, + responseUrl: null, errorMessage: error.message, executionTimeMs, });