Declare context fields on the catalogue and add always_keep

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## Unreleased
+
+### Changed (breaking)
+
+- Context fields an application declares are now declared on the catalogue as
+  `context_fields` instead of passed to `configure()` as
+  `extra_context_fields`, which is removed. The event overview reads them from
+  there, so it now lists them with the context fields.
+- `ALWAYS_KEEP_FIELDS` is removed, since which fields every stream keeps now
+  depends on the catalogue. `always_kept_field_names()` returns them.
+
+### Added
+
+- `always_keep` on `ContextField`. A declared context field with
+  `always_keep=True` reaches every stream an event routes to, like the standard
+  ones, without an entry in each allow-list. Without it, a declared field still
+  reaches only the streams whose allow-list names it.
+
 ## 0.6.0 - 2026-10-29
 
 ### Changed (breaking)

docs/STARTING_GUIDE.md

@@ -72,10 +72,26 @@ Subclass `EventCatalogue` instead to start from nothing, declaring routing as
 well as ids for all of `REQUIRED_EVENTS`.
 
 `fields` is a **per-stream allow-list**. A field not listed for a stream never
-reaches it, which is what keeps application detail out of SIEM. Correlation
-metadata (`request_id`, `ip`, `user_agent`, `client_trace_id`, `correlation_id`)
-is always retained. An event with no `fields` sends everything to every stream it
-declares.
+reaches it, which is what keeps application detail out of SIEM. The standard
+context fields (`request_id`, `ip`, `user_agent`, `client_trace_id`,
+`correlation_id`, `endpoint`, `method`, `scope`) are always retained. An event
+with no `fields` sends everything to every stream it declares.
+
+Declare a context field of the application's own on the catalogue. It is read
+from its header on every request, and like any other field it reaches only the
+streams whose allow-list names it. With `always_keep=True` it reaches every
+stream an event routes to, like the standard ones, so set that only for what
+every stream, SIEM included, may carry:
+
+```python
+from gfmodules.logging import ContextField
+
+class Log(Base):
+    context_fields = (
+        ContextField(name="tenant_id", header="X-Tenant-Id"),
+        ContextField(name="caller_id", header="X-Caller-Id", always_keep=True),
+    )
+```
 
 Field names in `RESERVED_FIELDS` cannot be used: the standard library refuses to
 overwrite its own `LogRecord` attributes, so `name`, `module`, `lineno` and the
@@ -104,7 +120,6 @@ gflog.configure(
     config=config.logging,          # a gflog.ConfigLogging
     loglevel=config.app.loglevel,
     catalogue=Log,
-    extra_context_fields=(gflog.ContextField(name="tenant_id", header="X-Tenant-Id"),),
     strict_fields=False,            # True in the test suite, see Testing
 )
 ```
@@ -371,7 +386,7 @@ with update_context({"tenant_id": subject.tenant}):
 ```
 
 A field only survives to the record if it was declared, either as a standard
-field or through `extra_context_fields`.
+field or in the catalogue's `context_fields`.
 
 `correlation_headers()` returns the correlation id and client trace id as
 headers, to pass to a downstream call so the trace does not end here. To read a

examples/fastapi_app/app.py

@@ -18,9 +18,6 @@
 
 logger = logging.getLogger("app.api")
 
-# Declaring the field is what makes the header survive to the record.
-TENANT_ID = gflog.ContextField(name="tenant_id", header="X-Tenant-Id")
-
 
 class CreateResource(BaseModel):
     resource_id: str
@@ -36,7 +33,6 @@ def setup_logging(settings: Settings, *, strict_fields: bool = False) -> None:
         config=settings.logging,
         loglevel=settings.app.loglevel,
         catalogue=Log,
-        extra_context_fields=(TENANT_ID,),
         strict_fields=strict_fields,
     )
     gflog.install_excepthook(logger)

examples/fastapi_app/events.md

@@ -22,6 +22,10 @@ Every event carries these on every stream, whenever the request context holds a
 - `method`
 - `scope`
 
+The request context also holds these, but an event carries them only on the streams whose fields name them.
+
+- `tenant_id`
+
 ## Stream `app`
 
 | Event | Id | Level | Fields |

examples/fastapi_app/events.py

@@ -7,7 +7,7 @@
 from collections.abc import Mapping
 from typing import ClassVar
 
-from gfmodules.logging import DefaultEventCatalogue, LogEvent, LoggingStreams
+from gfmodules.logging import ContextField, DefaultEventCatalogue, LogEvent, LoggingStreams
 
 APP = LoggingStreams.APP
 SIEM = LoggingStreams.SIEM
@@ -16,6 +16,9 @@
 
 
 class Log(Base):
+    # Declaring the field is what makes the header survive to the record.
+    context_fields = (ContextField(name="tenant_id", header="X-Tenant-Id"),)
+
     # The system events: this system's ids, the library's routing. Leaving one
     # out fails in configure(), naming the slot.
     SYS_APP_STARTED = Base.SYS_APP_STARTED.with_id("100601")

gfmodules/logging/__init__.py

@@ -5,7 +5,6 @@
 """
 
 import logging
-from collections.abc import Sequence
 from logging.config import dictConfig
 
 from gfmodules.logging.config import ConfigLogging
@@ -120,7 +119,6 @@ def configure(
     config: ConfigLogging,
     loglevel: str,
     catalogue: type[EventCatalogue],
-    extra_context_fields: Sequence[ContextField] = (),
     strict_fields: bool = False,
     logger_root: str = DEFAULT_LOGGER_ROOT,
 ) -> None:
@@ -133,7 +131,7 @@ def configure(
 
     validate_catalogue(catalogue, access_logs=config.access_logs)
     set_strict_fields(strict_fields)
-    register_context_fields(extra_context_fields)
+    register_context_fields(catalogue.context_fields)
     register_access_logs(config.access_logs)
     register_catalogue(catalogue, access_logs=config.access_logs)
     register_logger_root(logger_root)

gfmodules/logging/context.py

@@ -48,16 +48,17 @@ class ContextField:
     header: str | None
     sanitize: bool = True
     sanitizer: Callable[[str], str] = sanitize_header_value
+    always_keep: bool = False
 
 
-REQUEST_ID = ContextField(name="request_id", header=None)
-IP = ContextField(name="ip", header=None)
-USER_AGENT = ContextField(name="user_agent", header=USER_AGENT_HEADER, sanitizer=sanitize_free_text)
-CLIENT_TRACE_ID = ContextField(name="client_trace_id", header=CLIENT_TRACE_ID_HEADER)
-CORRELATION_ID = ContextField(name="correlation_id", header=CORRELATION_ID_HEADER)
-ENDPOINT = ContextField(name="endpoint", header=None)
-METHOD = ContextField(name="method", header=None)
-SCOPE = ContextField(name="scope", header=SCOPE_HEADER, sanitizer=sanitize_free_text)
+REQUEST_ID = ContextField(name="request_id", header=None, always_keep=True)
+IP = ContextField(name="ip", header=None, always_keep=True)
+USER_AGENT = ContextField(name="user_agent", header=USER_AGENT_HEADER, sanitizer=sanitize_free_text, always_keep=True)
+CLIENT_TRACE_ID = ContextField(name="client_trace_id", header=CLIENT_TRACE_ID_HEADER, always_keep=True)
+CORRELATION_ID = ContextField(name="correlation_id", header=CORRELATION_ID_HEADER, always_keep=True)
+ENDPOINT = ContextField(name="endpoint", header=None, always_keep=True)
+METHOD = ContextField(name="method", header=None, always_keep=True)
+SCOPE = ContextField(name="scope", header=SCOPE_HEADER, sanitizer=sanitize_free_text, always_keep=True)
 
 # Order matters: it is the order these keys appear in every emitted record.
 STANDARD_FIELDS: tuple[ContextField, ...] = (
@@ -71,8 +72,6 @@ class ContextField:
     SCOPE,
 )
 
-ALWAYS_KEEP_FIELDS: frozenset[str] = frozenset(field.name for field in STANDARD_FIELDS)
-
 _fields: tuple[ContextField, ...] = STANDARD_FIELDS
 _context_var: ContextVar[Mapping[str, str] | None] = ContextVar("gfmodules_logging_context", default=None)
 
@@ -92,6 +91,14 @@ def registered_fields() -> tuple[ContextField, ...]:
     return _fields
 
 
+def registered_field_names() -> frozenset[str]:
+    return frozenset(field.name for field in _fields)
+
+
+def always_kept_field_names() -> frozenset[str]:
+    return frozenset(field.name for field in _fields if field.always_keep)
+
+
 def extract_context(headers: Mapping[str, str]) -> dict[str, str]:
     lowered = {key.lower(): value for key, value in headers.items()}
     extracted: dict[str, str] = {}
@@ -123,7 +130,7 @@ def bind_context(values: Mapping[str, Any]) -> Generator[None]:
 def update_context(values: Mapping[str, Any]) -> Generator[None]:
     """Adds to the bound context instead of replacing it.
 
-    The field still has to be declared through :func:`register_context_fields`,
+    The field still has to be declared in the catalogue's ``context_fields``,
     or nothing collects it.
     """
     context_mapping = _context_var.get() or {}

gfmodules/logging/event_overview.py

@@ -4,7 +4,7 @@
 import sys
 from collections.abc import Iterable, Sequence
 
-from gfmodules.logging.context import STANDARD_FIELDS
+from gfmodules.logging.context import STANDARD_FIELDS, ContextField
 from gfmodules.logging.events import EventCatalogue, LogEvent, _excused_events, declared_events
 from gfmodules.logging.streams import LoggingStreams
 
@@ -15,6 +15,10 @@
 
 _CONTEXT_INTRO = "Every event carries these on every stream, whenever the request context holds a value for them."
 
+_ALLOW_LISTED_CONTEXT_INTRO = (
+    "The request context also holds these, but an event carries them only on the streams whose fields name them."
+)
+
 _ACCESS_INTRO = "`ACCESS_REQUEST` carries these ids instead of its own on these routes."
 
 
@@ -50,6 +54,20 @@ def _stream_section(stream: LoggingStreams, events: Sequence[tuple[str, LogEvent
     return _section(f"Stream `{stream.name.lower()}`", _table(("Event", "Id", "Level", "Fields"), rows))
 
 
+def _field_list(fields: Iterable[ContextField]) -> str:
+    return "\n".join(f"- `{field.name}`" for field in fields)
+
+
+def _context_section(catalogue: type[EventCatalogue]) -> str:
+    fields = (*STANDARD_FIELDS, *catalogue.context_fields)
+    kept = [field for field in fields if field.always_keep]
+    allow_listed = [field for field in fields if not field.always_keep]
+    paragraphs = [_CONTEXT_INTRO, _field_list(kept)]
+    if allow_listed:
+        paragraphs += [_ALLOW_LISTED_CONTEXT_INTRO, _field_list(allow_listed)]
+    return _section("Context fields", *paragraphs)
+
+
 def _access_section(catalogue: type[EventCatalogue]) -> str:
     rows = [(method, f"`{route}`", event_id) for (method, route), event_id in catalogue.access_event_id.items()]
     return _section("Access event ids per route", _ACCESS_INTRO, _table(("Method", "Route", "Id"), rows))
@@ -66,7 +84,7 @@ def render_event_overview(catalogue: type[EventCatalogue], *, access_logs: bool
             _LEGEND,
         )
     )
-    context = _section("Context fields", _CONTEXT_INTRO, "\n".join(f"- `{field.name}`" for field in STANDARD_FIELDS))
+    context = _context_section(catalogue)
     reaching = {
         stream: [(name, event) for name, event in events if stream in event.streams] for stream in LoggingStreams
     }

gfmodules/logging/events.py

@@ -11,7 +11,7 @@
 from dataclasses import dataclass, field
 from typing import Any, ClassVar, Self
 
-from gfmodules.logging.context import ALWAYS_KEEP_FIELDS, registered_fields
+from gfmodules.logging.context import ContextField, always_kept_field_names, registered_field_names
 from gfmodules.logging.loggers import report_outside_root, within_root
 from gfmodules.logging.streams import LoggingStreams
 
@@ -156,7 +156,7 @@ def set_strict_fields(enabled: bool) -> None:
 def unrouted_fields(event: LogEvent, names: Iterable[str]) -> tuple[str, ...]:
     if not event.fields:
         return ()
-    allowed = {name for allow_list in event.fields.values() for name in allow_list} | ALWAYS_KEEP_FIELDS
+    allowed = {name for allow_list in event.fields.values() for name in allow_list} | always_kept_field_names()
     return tuple(sorted(set(names) - allowed))
 
 
@@ -179,7 +179,7 @@ def emit(
     if reserved:
         raise ValueError(f"event {event.event_id} names fields a log record reserves: {', '.join(reserved)}")
 
-    context_fields = sorted({context_field.name for context_field in registered_fields()} & values.keys())
+    context_fields = sorted(registered_field_names() & values.keys())
     if context_fields:
         raise ValueError(
             f"event {event.event_id} names fields the request context already binds: {', '.join(context_fields)}. "
@@ -215,6 +215,8 @@ class EventCatalogue:
     #: (method, route path) -> event id, for routes with an access id of their own.
     access_event_id: ClassVar[Mapping[tuple[str, str], str]] = {}
 
+    context_fields: ClassVar[tuple[ContextField, ...]] = ()
+
     @classmethod
     def event(
         cls,

gfmodules/logging/formatter.py

@@ -4,7 +4,7 @@
 from datetime import UTC, datetime
 from typing import Any
 
-from gfmodules.logging.context import ALWAYS_KEEP_FIELDS, collect_context
+from gfmodules.logging.context import always_kept_field_names, collect_context
 from gfmodules.logging.events import RESERVED_FIELDS
 from gfmodules.logging.streams import LoggingStreams
 
@@ -19,7 +19,7 @@ def _allowed_on(
     field_streams: dict[LoggingStreams, tuple[str, ...]] | None = getattr(record, "field_streams", None)
     if stream is None or not field_streams:
         return data
-    allowed = set(field_streams.get(stream, ())) | ALWAYS_KEEP_FIELDS
+    allowed = set(field_streams.get(stream, ())) | always_kept_field_names()
     return {key: value for key, value in data.items() if key in allowed}
 
 

tests/test_context.py

@@ -1,18 +1,19 @@
 import pytest
 
 from gfmodules.logging.context import (
-    ALWAYS_KEEP_FIELDS,
     CLIENT_TRACE_ID_HEADER,
     CORRELATION_ID_HEADER,
     STANDARD_FIELDS,
     UNSET,
     USER_AGENT_HEADER,
     ContextField,
+    always_kept_field_names,
     bind_context,
     collect_context,
     correlation_headers,
     extract_context,
     register_context_fields,
+    registered_field_names,
     registered_fields,
     sanitize_free_text,
     sanitize_header_value,
@@ -36,6 +37,17 @@ def test_extras_are_appended_after_the_standard_fields(self) -> None:
 
         assert registered_fields() == (*STANDARD_FIELDS, TENANT_ID)
 
+    def test_the_registered_names_cover_the_standard_fields_and_the_extras(self) -> None:
+        register_context_fields((TENANT_ID,))
+
+        assert registered_field_names() == {field.name for field in STANDARD_FIELDS} | {"tenant_id"}
+
+    def test_an_extra_is_always_kept_only_when_it_says_so(self) -> None:
+        caller = ContextField(name="caller_id", header="X-Caller-Id", always_keep=True)
+        register_context_fields((TENANT_ID, caller))
+
+        assert always_kept_field_names() == {field.name for field in STANDARD_FIELDS} | {"caller_id"}
+
     def test_registering_replaces_the_previous_extras(self) -> None:
         register_context_fields((TENANT_ID,))
         register_context_fields(())
@@ -223,7 +235,7 @@ def test_the_ip_is_not_read_from_a_header(self) -> None:
         assert "ip" not in extract_context({"X-Forwarded-For": "203.0.113.7"})
 
     def test_the_always_kept_fields_are_request_and_correlation_metadata(self) -> None:
-        assert ALWAYS_KEEP_FIELDS == {
+        assert always_kept_field_names() == {
             "request_id",
             "ip",
             "user_agent",