torsten/trainingstagebuch
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(logging): enhance HTTP request logging with additional payload details

Browse Source 
- Introduced new fields in the HttpPageFetchLog model to capture request and response headers, bodies, and method for improved logging granularity.
- Updated the logging service to serialize and store these new details during HTTP fetch operations, enhancing traceability and debugging capabilities.
- Modified the clickTtHttpPageRoutes to include the new logging features, allowing for optional payload inclusion in log queries.
This commit is contained in:
Torsten Schulz (local)
2026-03-10 22:26:37 +01:00
parent 4f3a1829ca
commit 0e4d1707fd
6 changed files with 143 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/migrations/20260310_add_clicktt_payload_logging.sql Normal file
View File

@@ -0,0 +1,10 @@
ALTER TABLE http_page_fetch_log
ADD COLUMN IF NOT EXISTS request_method VARCHAR(16) NULL COMMENT 'HTTP-Methode des ausgehenden Requests' AFTER club_id_param,
ADD COLUMN IF NOT EXISTS request_headers LONGTEXT NULL COMMENT 'Gesendete Request-Header als JSON' AFTER request_method,
ADD COLUMN IF NOT EXISTS request_body LONGTEXT NULL COMMENT 'Gesendeter Request-Body im Originalformat' AFTER request_headers,
ADD COLUMN IF NOT EXISTS response_headers LONGTEXT NULL COMMENT 'Empfangene Response-Header als JSON' AFTER content_type,
ADD COLUMN IF NOT EXISTS response_body LONGTEXT NULL COMMENT 'Vollstaendiger Response-Body' AFTER response_headers,
ADD COLUMN IF NOT EXISTS response_url TEXT NULL COMMENT 'Finale Response-URL nach Redirects' AFTER response_body;
ALTER TABLE http_page_fetch_log
MODIFY COLUMN response_snippet LONGTEXT NULL COMMENT 'Gekuerzter oder kompletter Response-Anfang zur Strukturanalyse';

6
backend/migrations/create_http_page_fetch_log.sql
View File

@@ -10,10 +10,16 @@ CREATE TABLE IF NOT EXISTS http_page_fetch_log (
association VARCHAR(64) NULL COMMENT 'Verband (z.B. HeTTV, RTTV)',
championship VARCHAR(128) NULL COMMENT 'Championship-Parameter (z.B. HTTV 25/26)',
club_id_param VARCHAR(64) NULL COMMENT 'Club-ID falls clubInfoDisplay',
request_method VARCHAR(16) NULL COMMENT 'HTTP-Methode des ausgehenden Requests',
request_headers LONGTEXT NULL COMMENT 'Gesendete Request-Header als JSON',
request_body LONGTEXT NULL COMMENT 'Gesendeter Request-Body im Originalformat',
http_status INT NULL COMMENT 'HTTP-Status der Response',
success BOOLEAN NOT NULL DEFAULT FALSE,
response_snippet TEXT NULL COMMENT 'Gekürzter Response-Anfang (max 2000 Zeichen) zur Strukturanalyse',
content_type VARCHAR(128) NULL COMMENT 'Content-Type der Response',
response_headers LONGTEXT NULL COMMENT 'Empfangene Response-Header als JSON',
response_body LONGTEXT NULL COMMENT 'Vollständiger Response-Body',
response_url TEXT NULL COMMENT 'Finale Response-URL nach Redirects',
error_message TEXT NULL,
execution_time_ms INT NULL COMMENT 'Laufzeit in Millisekunden',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,

32
backend/models/HttpPageFetchLog.js
View File

@@ -48,6 +48,21 @@ const HttpPageFetchLog = sequelize.define('HttpPageFetchLog', {
allowNull: true,
comment: 'Club-ID falls clubInfoDisplay',
},
requestMethod: {
type: DataTypes.STRING(16),
allowNull: true,
comment: 'HTTP-Methode des ausgehenden Requests',
},
requestHeaders: {
type: DataTypes.TEXT('long'),
allowNull: true,
comment: 'Gesendete Request-Header als JSON',
},
requestBody: {
type: DataTypes.TEXT('long'),
allowNull: true,
comment: 'Gesendeter Request-Body im Originalformat',
},
httpStatus: {
type: DataTypes.INTEGER,
allowNull: true,
@@ -58,7 +73,7 @@ const HttpPageFetchLog = sequelize.define('HttpPageFetchLog', {
defaultValue: false,
},
responseSnippet: {
type: DataTypes.TEXT,
type: DataTypes.TEXT('long'),
allowNull: true,
comment: 'Gekürzter Response-Anfang zur Strukturanalyse',
},
@@ -66,6 +81,21 @@ const HttpPageFetchLog = sequelize.define('HttpPageFetchLog', {
type: DataTypes.STRING(128),
allowNull: true,
},
responseHeaders: {
type: DataTypes.TEXT('long'),
allowNull: true,
comment: 'Empfangene Response-Header als JSON',
},
responseBody: {
type: DataTypes.TEXT('long'),
allowNull: true,
comment: 'Vollstaendiger Response-Body',
},
responseUrl: {
type: DataTypes.TEXT,
allowNull: true,
comment: 'Finale Response-URL nach Redirects',
},
errorMessage: {
type: DataTypes.TEXT,
allowNull: true,

67
backend/routes/clickTtHttpPageRoutes.js
View File

@@ -40,6 +40,22 @@ function getOrCreateSid(req) {
return req.query.sid || crypto.randomBytes(16).toString('hex');
}
function serializeFormBody(req) {
if (typeof req.rawBody === 'string' && req.rawBody.length > 0) {
return req.rawBody;
}
if (typeof req.body === 'string') {
return req.body;
}
if (req.body && typeof req.body === 'object' && Object.keys(req.body).length > 0) {
return new URLSearchParams(req.body).toString();
}
return null;
}
/** Domains, deren Links durch den Proxy umgeleitet werden (für Folge-Logs) */
const PROXY_DOMAINS = ['click-tt.de', 'httv.de'];
@@ -264,9 +280,7 @@ router.post('/proxy', async (req, res, next) => {
}
const contentType = req.get('content-type') || 'application/x-www-form-urlencoded';
const body = typeof req.body === 'string' ? req.body : (req.body && Object.keys(req.body).length
? new URLSearchParams(req.body).toString()
: null);
const body = serializeFormBody(req);
const result = await clickTtHttpPageService.fetchWithLogging({
url: targetUrl,
@@ -487,37 +501,52 @@ router.get('/logs', authenticate, async (req, res, next) => {
try {
const limit = Math.min(parseInt(req.query.limit, 10) || 50, 200);
const { fetchType, association, success } = req.query;
const includePayloads = req.query.includePayloads === 'true';
const where = {};
if (fetchType) where.fetchType = fetchType;
if (association) where.association = association;
if (success !== undefined) where.success = success === 'true';
const attributes = [
'id',
'fetchType',
'baseDomain',
'fullUrl',
'association',
'championship',
'clubIdParam',
'requestMethod',
'httpStatus',
'success',
'responseSnippet',
'contentType',
'responseUrl',
'errorMessage',
'executionTimeMs',
'createdAt',
];
if (includePayloads) {
attributes.push(
'requestHeaders',
'requestBody',
'responseHeaders',
'responseBody',
);
}
const logs = await HttpPageFetchLog.findAll({
where: Object.keys(where).length > 0 ? where : undefined,
order: [['createdAt', 'DESC']],
limit,
attributes: [
'id',
'fetchType',
'baseDomain',
'fullUrl',
'association',
'championship',
'clubIdParam',
'httpStatus',
'success',
'responseSnippet',
'contentType',
'errorMessage',
'executionTimeMs',
'createdAt',
],
attributes,
});
res.json({
success: true,
count: logs.length,
includePayloads,
logs,
});
} catch (error) {

11
backend/server.js
View File

@@ -59,6 +59,11 @@ import HttpError from './exceptions/HttpError.js';
const app = express();
const port = process.env.PORT || 3005;
function captureRawBody(req, res, buf, encoding) {
if (!buf || buf.length === 0) return;
req.rawBody = buf.toString(encoding || 'utf8');
}
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@@ -70,8 +75,8 @@ app.use(cors({
allowedHeaders: ['Content-Type', 'Authorization', 'authcode', 'userid']
}));
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(express.json({ verify: captureRawBody }));
app.use(express.urlencoded({ extended: true, verify: captureRawBody }));
// Request Logging Middleware - loggt alle API-Requests
// Wichtig: userId wird später in authMiddleware gesetzt, aber Middleware funktioniert auch ohne
@@ -421,4 +426,4 @@ app.use((err, req, res, next) => {
} catch (err) {
console.error('Unable to synchronize the database:', err);
}
})();
})();

40
backend/services/clickTtHttpPageService.js
View File

@@ -24,6 +24,32 @@ const ASSOCIATION_TO_DOMAIN = {
const SNIPPET_MAX_LEN = 2000;
function serializeHeaders(headers) {
if (!headers) return null;
if (typeof headers.raw === 'function') {
try {
return JSON.stringify(headers.raw());
} catch {
// Fallback below
}
}
try {
const entries = {};
if (typeof headers.forEach === 'function') {
headers.forEach((value, key) => {
entries[key] = value;
});
} else {
Object.assign(entries, headers);
}
return JSON.stringify(entries);
} catch {
return null;
}
}
/**
* Ermittelt die click-TT-Domain für einen Verband
*/
@@ -134,6 +160,8 @@ async function fetchWithLogging(options) {
const responseBody = await response.text();
const success = response.ok;
const snippet = createResponseSnippet(responseBody);
const requestHeaders = serializeHeaders(headers);
const responseHeaders = serializeHeaders(response.headers);
await HttpPageFetchLog.create({
userId,
@@ -143,10 +171,16 @@ async function fetchWithLogging(options) {
association,
championship,
clubIdParam,
requestMethod: method,
requestHeaders,
requestBody: body,
httpStatus: response.status,
success,
responseSnippet: snippet,
contentType,
responseHeaders,
responseBody,
responseUrl: response.url || url,
errorMessage: success ? null : `HTTP ${response.status}: ${response.statusText}`,
executionTimeMs,
});
@@ -172,10 +206,16 @@ async function fetchWithLogging(options) {
association,
championship,
clubIdParam,
requestMethod: method,
requestHeaders: serializeHeaders(headers),
requestBody: body,
httpStatus: null,
success: false,
responseSnippet: null,
contentType: null,
responseHeaders: null,
responseBody: null,
responseUrl: null,
errorMessage: error.message,
executionTimeMs,
});