From b40121ee5aa1677cb43ddc22e9055cd5e3813828 Mon Sep 17 00:00:00 2001 From: grs Date: Tue, 27 Oct 2026 08:23:50 +0000 Subject: [PATCH 1/5] [BUGFIX] Detect failed feed fetches (#18) Use the SimplePie initialization result so failed requests render the configured error output even when feed getters are selected. Related: #5 --- Classes/Service/FeedRuntimeInitializer.php | 4 +- .../Functional/Frontend/FeedFrontendTest.php | 6 +- Tests/Unit/Service/FeedDataServiceTest.php | 60 +++++++++++++++---- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/Classes/Service/FeedRuntimeInitializer.php b/Classes/Service/FeedRuntimeInitializer.php index 15fa2308..1e61268f 100644 --- a/Classes/Service/FeedRuntimeInitializer.php +++ b/Classes/Service/FeedRuntimeInitializer.php @@ -53,9 +53,7 @@ public function initializeFeed(SimplePie $feed, array $settings): bool } $feed->enable_cache(false); // Feed autodiscovery can hit SimplePie's IRI handling before getters run. - SimplePieDeprecationHandler::run($feed->init(...)); - - return true; + return SimplePieDeprecationHandler::run($feed->init(...)) === true; } private function shouldUseTypo3HttpClient(string $feedUrl): bool diff --git a/Tests/Functional/Frontend/FeedFrontendTest.php b/Tests/Functional/Frontend/FeedFrontendTest.php index 0652875b..a7aaa4d3 100644 --- a/Tests/Functional/Frontend/FeedFrontendTest.php +++ b/Tests/Functional/Frontend/FeedFrontendTest.php @@ -146,13 +146,13 @@ public function feedImageGetsPublishedToTemporaryAssets(): void #[Test] public function invalidFeedDisplaysTheDefaultErrorMessage(): void { - $this->initializeFrontendRootPage($this->instancePath . '/fileadmin/missing-feed.xml', [ - 'getFields.feed' => '', - ]); + $this->initializeFrontendRootPage($this->instancePath . '/fileadmin/missing-feed.xml'); $body = $this->requestPage(); self::assertStringContainsString('Sorry, the feed could not be fetched.', $body); + self::assertStringNotContainsString('class="feed-title"', $body); + self::assertStringNotContainsString('class="feed-items"', $body); } #[Test] diff --git a/Tests/Unit/Service/FeedDataServiceTest.php b/Tests/Unit/Service/FeedDataServiceTest.php index 2808b26a..c7a8f8b5 100644 --- a/Tests/Unit/Service/FeedDataServiceTest.php +++ b/Tests/Unit/Service/FeedDataServiceTest.php @@ -21,6 +21,11 @@ use ErHaWeb\FeedDisplay\Service\FeedDataService; use ErHaWeb\FeedDisplay\Service\FeedRuntimeInitializer; use GuzzleHttp\Client; +use GuzzleHttp\Handler\MockHandler; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Psr7\HttpFactory; +use GuzzleHttp\Psr7\Response; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; use Psr\EventDispatcher\EventDispatcherInterface; use Psr\Http\Message\RequestFactoryInterface; @@ -73,7 +78,7 @@ public function buildDataKeepsSimplePieDefaultTransportForLocalFeedPaths(): void $feed->expects(self::once())->method('set_feed_url')->with('/var/www/public/fileadmin/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -108,7 +113,7 @@ public function buildDataMapsFeedAndItemFieldsAndDispatchesEvent(): void $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_title')->willReturn('Feed title'); $feed->expects(self::once())->method('subscribe_url')->willReturn('https://example.com/subscribe'); $feed->expects(self::once())->method('get_image_url')->willReturn('file://' . $sourceImagePath); @@ -158,7 +163,7 @@ public function buildDataIgnoresLegacyFaviconFeedFieldWithoutCallingSimplePieFav $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::never())->method('get_favicon'); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); @@ -193,7 +198,7 @@ public function buildDataKeepsFeedGetterValueWhenKnownSimplePieIriDeprecationIsT $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_description')->willReturnCallback( static fn(): string => require $deprecationFile ); @@ -275,7 +280,7 @@ public function buildDataSupportsGetterCallsWithThreeAndFourFieldParts(): void $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 1)->willReturn([$item]); $eventDispatcher->expects(self::once())->method('dispatch')->willReturnArgument(0); @@ -311,7 +316,7 @@ public function buildDataDoesNotOverwriteAnAlreadyGeneratedTemporaryImage(): voi $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_image_url')->willReturn('file://' . $sourceImagePath); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -344,7 +349,7 @@ public function buildDataStripsSlashesFromConfiguredFeedUrlBeforeInitializingSim $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -375,7 +380,7 @@ public function buildDataReturnsNullForUnknownGettersAndUnsupportedArgumentCount $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 1)->willReturn([$item]); $eventDispatcher->expects(self::once())->method('dispatch')->willReturnArgument(0); @@ -409,7 +414,7 @@ public function buildDataReturnsNullIfFeedImageUrlCannotBeResolvedToATemporaryFi $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_image_url')->willReturn('file:///path/to/non-existing.svg'); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -442,7 +447,7 @@ public function buildDataReturnsNullIfFeedImageUrlHasNoFilenameOrExtension(): vo $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_image_url')->willReturn('https://example.com/images/'); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -476,7 +481,7 @@ public function buildDataReturnsSerializableDataForObjectRichFieldsAndEventInjec $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::exactly(2))->method('get_items')->willReturnCallback( static function (int $start, int $end) use ($item): array { self::assertSame(0, $start); @@ -512,6 +517,39 @@ static function (SingleFeedDataEvent $event): SingleFeedDataEvent { self::assertNotSame('', serialize($data)); } + #[Test] + #[DataProvider('httpFeedResponses')] + public function initializerReportsWhetherTheHttpFeedWasParsed(Response $response, bool $expected): void + { + $client = new Client(['handler' => HandlerStack::create(new MockHandler([$response]))]); + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->expects(self::once())->method('getClient')->willReturn($client); + $httpFactory = new HttpFactory(); + $subject = new FeedRuntimeInitializer($factory, $httpFactory, $httpFactory); + + self::assertSame( + $expected, + $subject->initializeFeed(new SimplePie(), ['feedUrl' => 'https://example.test/feed.xml']) + ); + } + + /** + * @return iterable + */ + public static function httpFeedResponses(): iterable + { + yield 'HTTP failure' => [new Response(503), false]; + yield 'invalid XML' => [new Response(200, ['Content-Type' => 'application/rss+xml'], ''), false]; + yield 'valid empty feed' => [ + new Response( + 200, + ['Content-Type' => 'application/rss+xml'], + 'Feedhttps://example.test/Empty' + ), + true, + ]; + } + private function createSourceImage(string $fileName): string { $sourceImagePath = Environment::getPublicPath() . '/typo3temp/var/tests/' . $fileName; From 875c33885271e32da3629aeb414180065b925fe4 Mon Sep 17 00:00:00 2001 From: Eric Harrer Date: Sun, 8 Nov 2026 01:52:08 +0100 Subject: [PATCH 2/5] [FEATURE] Persist the last successful feed response (#18) Store responses as binary documents with their status, selected headers, MIME type and final URIs. Feeds need not be UTF-8, and character sets and relative links depend on this context. Keep serving the last successful response after failed updates, and keep rendering when the table is missing or a write fails. Share source freshness between plugin configurations, repair unusable saved documents and read payloads only on display-cache misses. Page-cache lifetimes of live plugins stay unchanged. Co-authored-by: grs --- Build/Scripts/classicFixture.php | 10 + Build/Scripts/testClassic.sh | 11 + Build/Scripts/verifyTerArtefact.php | 3 + Classes/Controller/FeedController.php | 23 +- Classes/Service/FeedDataService.php | 41 +-- Classes/Service/FeedFetcher.php | 88 ++++++ Classes/Service/FeedHttpRecorder.php | 92 +++++++ Classes/Service/FeedParser.php | 76 ++++++ Classes/Service/FeedProvider.php | 237 ++++++++++++++++ Classes/Service/FeedRefresher.php | 113 ++++++++ Classes/Service/FeedRuntimeInitializer.php | 5 +- Classes/Service/FetchedFeed.php | 27 ++ Classes/Service/RefreshResult.php | 30 ++ Classes/Storage/FeedDocument.php | 97 +++++++ Classes/Storage/FeedStorage.php | 163 +++++++++++ Classes/Storage/StoredFeed.php | 37 +++ Configuration/Services.yaml | 4 - Tests/Fixtures/StoredDocument.php | 31 +++ Tests/Functional/Fixtures/FeedHttp.php | 54 ++++ .../Frontend/AbstractHttpFeedTestCase.php | 37 +++ .../Functional/Frontend/FeedFrontendTest.php | 5 +- .../Frontend/PersistentFeedTest.php | 123 +++++++++ .../Frontend/PersistentHttpFeedTest.php | 258 ++++++++++++++++++ Tests/Functional/Storage/FeedFailureTest.php | 200 ++++++++++++++ Tests/Functional/Storage/FeedStorageTest.php | 137 ++++++++++ Tests/Unit/Controller/FeedControllerTest.php | 239 ---------------- Tests/Unit/Service/FeedHttpTest.php | 212 ++++++++++++++ Tests/Unit/Service/FeedSnapshotTest.php | 160 +++++++++++ Tests/Unit/Storage/FeedDocumentTest.php | 57 ++++ Tests/Unit/Storage/FeedStorageTest.php | 75 +++++ ext_tables.sql | 13 + 31 files changed, 2375 insertions(+), 283 deletions(-) create mode 100644 Classes/Service/FeedFetcher.php create mode 100644 Classes/Service/FeedHttpRecorder.php create mode 100644 Classes/Service/FeedParser.php create mode 100644 Classes/Service/FeedProvider.php create mode 100644 Classes/Service/FeedRefresher.php create mode 100644 Classes/Service/FetchedFeed.php create mode 100644 Classes/Service/RefreshResult.php create mode 100644 Classes/Storage/FeedDocument.php create mode 100644 Classes/Storage/FeedStorage.php create mode 100644 Classes/Storage/StoredFeed.php create mode 100644 Tests/Fixtures/StoredDocument.php create mode 100644 Tests/Functional/Fixtures/FeedHttp.php create mode 100644 Tests/Functional/Frontend/AbstractHttpFeedTestCase.php create mode 100644 Tests/Functional/Frontend/PersistentFeedTest.php create mode 100644 Tests/Functional/Frontend/PersistentHttpFeedTest.php create mode 100644 Tests/Functional/Storage/FeedFailureTest.php create mode 100644 Tests/Functional/Storage/FeedStorageTest.php delete mode 100644 Tests/Unit/Controller/FeedControllerTest.php create mode 100644 Tests/Unit/Service/FeedHttpTest.php create mode 100644 Tests/Unit/Service/FeedSnapshotTest.php create mode 100644 Tests/Unit/Storage/FeedDocumentTest.php create mode 100644 Tests/Unit/Storage/FeedStorageTest.php create mode 100644 ext_tables.sql diff --git a/Build/Scripts/classicFixture.php b/Build/Scripts/classicFixture.php index 42e30cf3..011bb7cb 100644 --- a/Build/Scripts/classicFixture.php +++ b/Build/Scripts/classicFixture.php @@ -27,6 +27,16 @@ echo 'Classic autoloading resolved both SimplePie APIs from ' . $expectedPath . PHP_EOL; exit(0); } +if ($mode === 'verifyStored') { + $pool = $container->get(\TYPO3\CMS\Core\Database\ConnectionPool::class); + $row = $pool->getConnectionForTable('tx_feeddisplay_feed')->select(['response_data', 'revision'], 'tx_feeddisplay_feed')->fetchAssociative(); + if (!is_array($row) || !is_string($row['response_data']) || hash('sha256', $row['response_data']) !== $row['revision']) { + throw new RuntimeException('Classic rendering did not persist a complete feed document.'); + } + \ErHaWeb\FeedDisplay\Storage\FeedDocument::decode($row['response_data']); + echo 'Verified the persistent Classic source document.' . PHP_EOL; + exit(0); +} if ($mode !== 'seed') { throw new InvalidArgumentException('Expected seed or probe.'); } diff --git a/Build/Scripts/testClassic.sh b/Build/Scripts/testClassic.sh index 6c958636..c9f5cc7d 100644 --- a/Build/Scripts/testClassic.sh +++ b/Build/Scripts/testClassic.sh @@ -76,6 +76,17 @@ for mode in linked packaged; do exit 1 fi done + TYPO3_CONTEXT=Development php Build/Scripts/classicFixture.php "${instance}" verifyStored + rm "${instance}/fileadmin/feed.xml" + ( + cd "${instance}" + TYPO3_CONTEXT=Development php typo3/sysext/core/bin/typo3 cache:flush + ) + curl --fail --silent --show-error "${base}" -o "${instance}/persistent.html" + if ! grep -q 'Classic bundle integration item' "${instance}/persistent.html"; then + cat "${instance}/server.log" >&2 + exit 1 + fi kill "${serverPid}" wait "${serverPid}" 2>/dev/null || true serverPid='' diff --git a/Build/Scripts/verifyTerArtefact.php b/Build/Scripts/verifyTerArtefact.php index b89d2d87..c6c479d2 100644 --- a/Build/Scripts/verifyTerArtefact.php +++ b/Build/Scripts/verifyTerArtefact.php @@ -10,6 +10,9 @@ $manifest = json_decode($zip->getFromName('composer.json'), true, 512, JSON_THROW_ON_ERROR); $bundlePath = $manifest['extra']['typo3/cms']['Package']['providesPackages']['simplepie/simplepie'] ?? null; $isBundle = $bundlePath === 'Resources/Private/Php/ComposerVendor'; +if (is_file($sourceRoot . '/ext_tables.sql') && $zip->locateName('ext_tables.sql') === false) { + throw new RuntimeException('The TER package is missing the extension database schema.'); +} $temporary = sys_get_temp_dir() . '/feed-display-artefact-' . bin2hex(random_bytes(8)); mkdir($temporary); try { diff --git a/Classes/Controller/FeedController.php b/Classes/Controller/FeedController.php index 35503918..2c3fbe45 100644 --- a/Classes/Controller/FeedController.php +++ b/Classes/Controller/FeedController.php @@ -17,17 +17,13 @@ namespace ErHaWeb\FeedDisplay\Controller; -use ErHaWeb\FeedDisplay\Service\FeedDataService; +use ErHaWeb\FeedDisplay\Service\FeedProvider; use Psr\Http\Message\ResponseInterface; -use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface; use TYPO3\CMS\Extbase\Mvc\Controller\ActionController; class FeedController extends ActionController { - public function __construct( - private readonly FrontendInterface $cache, - private readonly FeedDataService $feedDataService - ) {} + public function __construct(private readonly FeedProvider $feedProvider) {} public function displayAction(): ResponseInterface { @@ -35,20 +31,7 @@ public function displayAction(): ResponseInterface return $this->htmlResponse(); } - $cacheIdentifier = 'feeddisplay_' . md5(serialize($this->settings)); - $data = $this->cache->get($cacheIdentifier); - $cacheDuration = (int)$this->settings['cacheDuration']; - - if ($cacheDuration === 0) { - $data = $this->feedDataService->buildData($this->settings); - $this->cache->remove($cacheIdentifier); - } elseif ($data === false || $data['settings'] !== $this->settings) { - $data = $this->feedDataService->buildData($this->settings); - $this->cache->set($cacheIdentifier, $data, [], $cacheDuration); - } - - $this->view->assign('data', $data); - + $this->view->assign('data', $this->feedProvider->getData($this->settings)); return $this->htmlResponse(); } } diff --git a/Classes/Service/FeedDataService.php b/Classes/Service/FeedDataService.php index 5b19a140..b570e6dc 100644 --- a/Classes/Service/FeedDataService.php +++ b/Classes/Service/FeedDataService.php @@ -49,18 +49,25 @@ public function __construct( */ public function buildData(array $settings): array { - $data = [ - 'settings' => $settings, - ]; - if (!SimplePieDeprecationHandler::run( fn(): bool => $this->initializer->initializeFeed($this->feed, $settings) )) { - return $data; + return ['settings' => $settings]; } - $this->appendFeedProperties($data, $settings); - $this->appendItemProperties($data, $settings); + return $this->buildInitializedData($this->feed, $settings); + } + + /** + * @param FeedSettings $settings + * @return FeedData + */ + public function buildInitializedData(SimplePie $feed, array $settings): array + { + $data = ['settings' => $settings]; + + $this->appendFeedProperties($feed, $data, $settings); + $this->appendItemProperties($feed, $data, $settings); return $data; } @@ -69,7 +76,7 @@ public function buildData(array $settings): array * @param FeedSettings $settings * @param FeedData $data */ - private function appendFeedProperties(array &$data, array $settings): void + private function appendFeedProperties(SimplePie $feed, array &$data, array $settings): void { $getFeedFields = GeneralUtility::trimExplode(',', (string)($settings['getFields']['feed'] ?? '')); foreach ($getFeedFields as $getFeedField) { @@ -79,7 +86,7 @@ private function appendFeedProperties(array &$data, array $settings): void $fieldParts = GeneralUtility::trimExplode('|', $getFeedField); $field = GeneralUtility::underscoredToLowerCamelCase($fieldParts[0]); - $value = $this->getFeedFieldValue($getFeedField, $fieldParts); + $value = $this->getFeedFieldValue($feed, $getFeedField, $fieldParts); if ($getFeedField === 'image_url') { $data['feed']['image'] = $this->getImage($value); @@ -95,19 +102,19 @@ private function appendFeedProperties(array &$data, array $settings): void * @param FeedSettings $settings * @param FeedData $data */ - private function appendItemProperties(array &$data, array $settings): void + private function appendItemProperties(SimplePie $feed, array &$data, array $settings): void { $maxFeedCount = (int)($settings['maxFeedCount'] ?? 0); $getItemsFields = GeneralUtility::trimExplode(',', (string)($settings['getFields']['items'] ?? '')); /** @var list $items */ $items = SimplePieDeprecationHandler::run( - fn(): array => $this->feed->get_items(0, $maxFeedCount) + fn(): array => $feed->get_items(0, $maxFeedCount) ); foreach ($items as $item) { $itemProperties = $this->buildNormalizedItemProperties($item, $getItemsFields); - $itemProperties = $this->dispatchSingleFeedDataEvent($itemProperties, $item, $settings); + $itemProperties = $this->dispatchSingleFeedDataEvent($feed, $itemProperties, $item, $settings); if ($itemProperties !== []) { $data['items'][] = $itemProperties; @@ -118,12 +125,12 @@ private function appendItemProperties(array &$data, array $settings): void /** * @param list $fieldParts */ - private function getFeedFieldValue(string $getFeedField, array $fieldParts): mixed + private function getFeedFieldValue(SimplePie $feed, string $getFeedField, array $fieldParts): mixed { $fieldName = $fieldParts[0] ?? $getFeedField; if ($fieldName === 'subscribe_url') { - return $this->callSimplePieGetter($this->feed->subscribe_url(...)); + return $this->callSimplePieGetter($feed->subscribe_url(...)); } // Avoid calling SimplePie's deprecated favicon handling for legacy configurations. @@ -131,7 +138,7 @@ private function getFeedFieldValue(string $getFeedField, array $fieldParts): mix return null; } - return $this->getValue($this->feed, $fieldParts); + return $this->getValue($feed, $fieldParts); } /** @@ -164,10 +171,10 @@ private function buildNormalizedItemProperties(Item $item, array $getItemsFields * @param array $itemProperties * @return array */ - private function dispatchSingleFeedDataEvent(array $itemProperties, Item $item, array $settings): array + private function dispatchSingleFeedDataEvent(SimplePie $feed, array $itemProperties, Item $item, array $settings): array { $event = $this->eventDispatcher->dispatch( - new SingleFeedDataEvent($itemProperties, $item, $settings, $this->feed) + new SingleFeedDataEvent($itemProperties, $item, $settings, $feed) ); /** @var SingleFeedDataEvent $event */ $normalizedProps = SimplePieDeprecationHandler::run( diff --git a/Classes/Service/FeedFetcher.php b/Classes/Service/FeedFetcher.php new file mode 100644 index 00000000..921af9be --- /dev/null +++ b/Classes/Service/FeedFetcher.php @@ -0,0 +1,88 @@ + Misc::fix_protocol($url, 1)); + if (Misc::is_remote_uri($target)) { + $url = $target; + $recorder = new FeedHttpRecorder( + $this->clientFactory->getClient(), + $bounded ? $this->timeout('timeout') : null, + $bounded ? $this->timeout('connect_timeout') : null, + $bounded ? FeedHttpRecorder::MAX_BODY_SIZE : null, + ); + } + if ($bounded) { + // Scheduled fetches read one direct source, which keeps their limits predictable. + $feed->set_autodiscovery_level(SimplePie::LOCATOR_NONE); + if ($recorder === null && (!is_file($url) || filesize($url) > FeedHttpRecorder::MAX_BODY_SIZE)) { + throw new \RuntimeException('The local feed source must be a file of at most 2 MiB.'); + } + } + $initialized = SimplePieDeprecationHandler::run( + fn(): bool => $this->initializer->initializeFeed($feed, ['feedUrl' => $url], $recorder) + ); + if (!$initialized) { + $error = $feed->error() ?? ''; + if (is_array($error)) { + $error = implode('; ', $error); + } + throw new \RuntimeException($error !== '' ? $error : 'The source did not return a usable feed.'); + } + $snapshot = $feed->getSnapshot(); + // Every stored byte must have passed the recorder's size accounting. + if ($recorder !== null && !$recorder->received($snapshot['response']['body'])) { + throw new \RuntimeException('The selected feed is not part of the recorded responses.'); + } + $document = new FeedDocument(['version' => 2, 'kind' => 'snapshot', 'snapshot' => $snapshot]); + return new FetchedFeed($feed, $document); + } + + public function restore(FeedDocument $document): SimplePie + { + $feed = new FeedParser($document->payload['snapshot']); + if (!SimplePieDeprecationHandler::run($feed->init(...))) { + throw new \UnexpectedValueException('The stored feed could not be parsed.'); + } + return $feed; + } + + private function timeout(string $key): float + { + $configured = (float)($GLOBALS['TYPO3_CONF_VARS']['HTTP'][$key] ?? 0); + return $configured > 0 ? min($configured, 10.0) : 10.0; + } +} diff --git a/Classes/Service/FeedHttpRecorder.php b/Classes/Service/FeedHttpRecorder.php new file mode 100644 index 00000000..2faa5235 --- /dev/null +++ b/Classes/Service/FeedHttpRecorder.php @@ -0,0 +1,92 @@ +>, body: string} + */ +final class FeedHttpRecorder implements ClientInterface +{ + public const MAX_BODY_SIZE = 2 * 1024 * 1024; + private const HEADERS = ['content-type', 'content-encoding', 'location', 'link', 'content-language']; + + /** @var list */ + private array $steps = []; + private int $bodySize = 0; + + public function __construct( + private readonly GuzzleClientInterface $client, + private readonly ?float $timeout = null, + private readonly ?float $connectTimeout = null, + private readonly ?int $maxBodySize = self::MAX_BODY_SIZE, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $uri = (string)$request->getUri(); + if ($request->getMethod() !== 'GET' || !in_array(strtolower($request->getUri()->getScheme()), ['http', 'https'], true)) { + throw new RequestException('Expected an HTTP(S) GET feed request.', $request); + } + + $options = ['allow_redirects' => false, 'http_errors' => false]; + if ($this->timeout !== null) { + $options['timeout'] = $this->timeout; + $options['connect_timeout'] = $this->connectTimeout ?? $this->timeout; + } + $response = $this->client->send($request, $options); + $stream = $response->getBody(); + try { + if ($stream->isSeekable()) { + $stream->rewind(); + } + $body = Utils::copyToString($stream, $this->maxBodySize === null ? -1 : $this->maxBodySize - $this->bodySize + 1); + } finally { + $stream->close(); + } + $this->bodySize += strlen($body); + if ($this->maxBodySize !== null && $this->bodySize > $this->maxBodySize) { + throw new RequestException('The feed responses exceed the 2 MiB response limit.', $request); + } + + $headers = []; + foreach ($response->getHeaders() as $name => $values) { + if (in_array(strtolower($name), self::HEADERS, true)) { + $headers[$name] = array_values($values); + } + } + + $status = $response->getStatusCode(); + $this->steps[] = ['uri' => $uri, 'status' => $status, 'headers' => $headers, 'body' => $body]; + + return new Response($status, $headers, $body); + } + + public function received(string $body): bool + { + return in_array($body, array_column($this->steps, 'body'), true); + } +} diff --git a/Classes/Service/FeedParser.php b/Classes/Service/FeedParser.php new file mode 100644 index 00000000..c2225763 --- /dev/null +++ b/Classes/Service/FeedParser.php @@ -0,0 +1,76 @@ +>, body: string} + * @phpstan-type SourceSnapshot array{response: SourceResponse, mime: string} + */ +final class FeedParser extends SimplePie +{ + /** @param SourceSnapshot|null $snapshot */ + public function __construct(private ?array $snapshot = null) + { + parent::__construct(); + if ($snapshot !== null) { + $this->feed_url = $snapshot['response']['uri']; + $this->permanent_url = $snapshot['response']['base']; + $this->enable_cache(false); + } + } + + /** @return SourceSnapshot */ + public function getSnapshot(): array + { + return $this->snapshot ?? throw new \LogicException('No successfully fetched parser input.'); + } + + /** + * @param \SimplePie\Cache\Base|\SimplePie\Cache\DataCache|false $cache + * @return array{array, string}|bool + */ + protected function fetch_data(&$cache) + { + if ($this->snapshot === null) { + $result = parent::fetch_data($cache); + if (is_array($result)) { + $this->snapshot = [ + 'response' => [ + 'uri' => $this->feed_url, + 'base' => $this->permanent_url ?? $this->feed_url, + 'status' => $this->status_code, + 'headers' => array_map(static fn(string $value): array => [$value], $result[0]), + 'body' => (string)$this->get_raw_data(), + ], + 'mime' => $result[1], + ]; + } + return $result; + } + + $response = $this->snapshot['response']; + $this->raw_data = $response['body']; + $this->feed_url = $response['uri']; + $this->permanent_url = $response['base']; + $this->status_code = $response['status']; + return [array_map(static fn(array $values): string => implode(', ', $values), $response['headers']), $this->snapshot['mime']]; + } +} diff --git a/Classes/Service/FeedProvider.php b/Classes/Service/FeedProvider.php new file mode 100644 index 00000000..59dcdbb8 --- /dev/null +++ b/Classes/Service/FeedProvider.php @@ -0,0 +1,237 @@ + $settings + * @return array + */ + public function getData(array $settings, bool $scheduler = false): array + { + $url = trim(stripslashes((string)($settings['feedUrl'] ?? ''))); + $duration = (int)($settings['cacheDuration'] ?? 0); + $identifier = 'feeddisplay_' . md5(serialize($settings)); + $cached = $this->readCache($identifier, $settings); + if ($url === '') { + return ['settings' => $settings]; + } + if (!$scheduler && $duration > 0 && $cached !== false + && is_string($cached[self::REVISION] ?? null) && str_starts_with($cached[self::REVISION], 'unstored:') + ) { + // Persistence failed: retain the successful display result for its original TTL. + return $this->displayData($cached); + } + try { + $stored = $this->storage->findState($url); + } catch (\Throwable $exception) { + $this->logger->warning('Could not read the persistent feed. Apply the extension database schema.', ['url' => $url, 'exception' => $exception]); + if (!$scheduler && $duration > 0 && $cached !== false) { + return $this->displayData($cached); + } + $fetched = $scheduler ? null : $this->fetchWithoutStorage($url); + if ($fetched === null) { + return $this->cacheData($identifier, ['settings' => $settings], '', $duration); + } + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler); + } + // A pre-upgrade cache hit remains usable until its original TTL expires. + if (!$scheduler && $duration > 0 && $cached !== false + && !array_key_exists(self::REVISION, $cached) && !($stored->hasDocument ?? false) + ) { + return $cached; + } + if ($stored !== null && $stored->invalidationPending) { + try { + $this->refresher->invalidate($stored); + } catch (\Throwable $exception) { + $this->logger->warning('Could not invalidate feed pages; the operation remains pending.', ['url' => $url, 'exception' => $exception]); + } + } + $now = (int)$this->context->getPropertyFromAspect('date', 'timestamp'); + $lastAttempt = $stored->lastAttemptAt ?? 0; + $needsFetch = $scheduler + ? !($stored->hasDocument ?? false) && ($lastAttempt === 0 || $lastAttempt + 60 <= $now) + : $duration <= 0 || $lastAttempt === 0 || $lastAttempt + $duration <= $now; + $fetched = null; + $attempted = false; + // The second pass repairs an unusable stored document, at most once per request. + while (true) { + if ($needsFetch) { + $attempted = true; + $result = $this->refresher->refresh($url, $scheduler); + $fetched = $result->fetched; + if ($result->status === 'failed' && $fetched !== null && !$result->persisted) { + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler); + } + try { + // Re-read after a busy lock, too: another worker may have finished. + $stored = $this->storage->findState($url); + } catch (\Throwable $exception) { + $this->logger->warning('Could not read the refreshed feed.', ['url' => $url, 'exception' => $exception]); + if ($fetched !== null) { + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler, $result->persisted); + } + $stored = null; + } + if ($fetched !== null && !($stored->hasDocument ?? false)) { + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler, $result->persisted); + } + if (!$scheduler && $result->status === 'locked' && !($stored->hasDocument ?? false)) { + $fresh = $this->fetchWithoutStorage($url); + if ($fresh !== null) { + return $this->dataService->buildInitializedData($fresh->feed, $settings); + } + } + } + if ($duration > 0 && $cached !== false && $stored !== null + && $stored->hasDocument && ($cached[self::REVISION] ?? null) === $stored->revision + ) { + return $this->displayData($cached); + } + $feed = $this->restoreFeed($url, $stored, $fetched); + if ($feed !== null && $stored !== null) { + // Public events and getters remain outside transport/storage catches. + $data = $this->dataService->buildInitializedData($feed, $settings); + return $this->cacheData($identifier, $data, $stored->revision, $duration); + } + $lastAttempt = $stored->lastAttemptAt ?? 0; + if ($attempted || !($stored->hasDocument ?? false) || ($lastAttempt !== 0 && $lastAttempt + 60 > $now)) { + break; + } + // A successful revision whose payload cannot be used follows bootstrap retry rules. + $needsFetch = true; + } + if (!$scheduler && $duration > 0 && $cached !== false && (isset($cached['feed']) || isset($cached['items']))) { + return $this->displayData($cached); + } + return $this->cacheData($identifier, ['settings' => $settings], '', $duration); + } + + private function restoreFeed(string $url, ?StoredFeed &$stored, ?FetchedFeed $fetched): ?SimplePie + { + try { + // Retry a concurrently replaced revision once, never spin in the frontend. + for ($attempt = 0; $attempt < 2 && $stored !== null && $stored->hasDocument; ++$attempt) { + if ($fetched !== null && $fetched->document->revision() === $stored->revision) { + return $fetched->feed; + } + $document = $this->storage->loadDocument($url, $stored->revision); + if ($document !== null) { + return $this->fetcher->restore($document); + } + $current = $this->storage->findState($url); + if ($current?->revision === $stored->revision) { + return null; + } + $stored = $current; + } + } catch (\Throwable $exception) { + $this->logger->warning('Could not replay the persistent feed.', ['url' => $url, 'exception' => $exception]); + } + return null; + } + + /** + * @param array $settings + * @return array + */ + private function fetchedData(string $identifier, FetchedFeed $fetched, array $settings, int $duration, bool $scheduler, bool $persisted = false): array + { + $data = $this->dataService->buildInitializedData($fetched->feed, $settings); + $revision = $fetched->document->revision(); + $cacheRevision = $persisted ? $revision : 'unstored:' . $revision; + return $scheduler ? $data : $this->cacheData($identifier, $data, $cacheRevision, $duration); + } + + private function fetchWithoutStorage(string $url): ?FetchedFeed + { + try { + return $this->fetcher->fetch($url); + } catch (\Throwable $exception) { + $this->logger->warning('Could not fetch the live feed.', ['url' => $url, 'exception' => $exception]); + return null; + } + } + + /** + * @param array $settings + * @return array|false + */ + private function readCache(string $identifier, array $settings): array|false + { + try { + $cached = $this->cache->get($identifier); + return is_array($cached) && ($cached['settings'] ?? null) === $settings ? $cached : false; + } catch (\Throwable $exception) { + $this->logger->warning('Could not read the feed display cache.', ['exception' => $exception]); + return false; + } + } + + /** + * @param array $data + * @return array + */ + private function cacheData(string $identifier, array $data, string $revision, int $duration): array + { + try { + if ($duration > 0) { + $this->cache->set($identifier, $data + [self::REVISION => $revision], [], $duration); + } else { + $this->cache->remove($identifier); + } + } catch (\Throwable $exception) { + $this->logger->warning('Could not update the feed display cache.', ['exception' => $exception]); + } + return $data; + } + + /** + * @param array $cached + * @return array + */ + private function displayData(array $cached): array + { + unset($cached[self::REVISION]); + return $cached; + } +} diff --git a/Classes/Service/FeedRefresher.php b/Classes/Service/FeedRefresher.php new file mode 100644 index 00000000..68c5c68c --- /dev/null +++ b/Classes/Service/FeedRefresher.php @@ -0,0 +1,113 @@ +withLock($url, 'refresh', function () use ($url, $bounded): RefreshResult { + $now = (int)$this->context->getPropertyFromAspect('date', 'timestamp'); + try { + $fetched = $this->fetcher->fetch($url, $bounded); + } catch (\Throwable $exception) { + try { + $this->storage->storeFailure($url, $exception->getMessage(), $now); + } catch (\Throwable $storageException) { + $this->logger->warning('Could not record feed fetch failure.', ['url' => $url, 'exception' => $storageException]); + } + throw $exception; + } + $persisted = false; + try { + $changed = $this->storage->storeSuccess($url, $fetched->document, $now); + $persisted = true; + $stored = $this->storage->findState($url); + if ($stored !== null) { + $this->invalidate($stored); + } + } catch (\Throwable $exception) { + // Keep the fresh parser usable for this request even when persistence fails. + return new RefreshResult('failed', $fetched, $exception, $persisted); + } + return new RefreshResult($changed ? 'updated' : 'unchanged', $fetched, persisted: true); + }); + } + + public function invalidate(StoredFeed $stored): void + { + if (!$stored->invalidationPending) { + return; + } + $this->cacheManager->flushCachesInGroupByTag('pages', FeedStorage::cacheTag($stored->url)); + $this->storage->markInvalidated($stored->url, $stored->revision); + } + + /** @param callable(): RefreshResult $operation */ + private function withLock(string $url, string $name, callable $operation): RefreshResult + { + $lock = null; + $acquired = false; + $result = new RefreshResult('locked'); + $mode = LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + try { + $lock = $this->lockFactory->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + try { + $acquired = $lock->acquire($mode); + } catch (LockAcquireWouldBlockException) { + $acquired = false; + } + if ($acquired) { + $result = $operation(); + } + } catch (\Throwable $exception) { + $result = new RefreshResult('failed', null, $exception); + } finally { + if ($lock !== null && $acquired) { + try { + $lock->release(); + } catch (\Throwable $exception) { + $result = new RefreshResult('failed', $result->fetched, $exception, $result->persisted); + } + } + } + if ($result->error !== null) { + $this->logger->warning('Feed source operation failed.', ['url' => $url, 'operation' => $name, 'exception' => $result->error]); + } + return $result; + } +} diff --git a/Classes/Service/FeedRuntimeInitializer.php b/Classes/Service/FeedRuntimeInitializer.php index 1e61268f..383c28ad 100644 --- a/Classes/Service/FeedRuntimeInitializer.php +++ b/Classes/Service/FeedRuntimeInitializer.php @@ -17,6 +17,7 @@ namespace ErHaWeb\FeedDisplay\Service; +use Psr\Http\Client\ClientInterface; use Psr\Http\Message\RequestFactoryInterface; use Psr\Http\Message\UriFactoryInterface; use SimplePie\SimplePie; @@ -37,7 +38,7 @@ public function __construct( /** * @param FeedSettings $settings */ - public function initializeFeed(SimplePie $feed, array $settings): bool + public function initializeFeed(SimplePie $feed, array $settings, ?ClientInterface $httpClient = null): bool { $feedUrl = $settings['feedUrl'] ?? ''; if ($feedUrl === '') { @@ -47,7 +48,7 @@ public function initializeFeed(SimplePie $feed, array $settings): bool $feedUrl = stripslashes((string)$feedUrl); $feed->set_feed_url($feedUrl); if ($this->shouldUseTypo3HttpClient($feedUrl)) { - $httpClient = $this->guzzleClientFactory->getClient(); + $httpClient ??= $this->guzzleClientFactory->getClient(); assert($httpClient instanceof \Psr\Http\Client\ClientInterface); $feed->set_http_client($httpClient, $this->requestFactory, $this->uriFactory); } diff --git a/Classes/Service/FetchedFeed.php b/Classes/Service/FetchedFeed.php new file mode 100644 index 00000000..cef4de44 --- /dev/null +++ b/Classes/Service/FetchedFeed.php @@ -0,0 +1,27 @@ +payload); + } + + public function revision(): string + { + return hash('sha256', $this->encode()); + } + + public static function decode(string $bytes): self + { + // Corrupt internal data is rejected below; never instantiate serialized objects. + $value = @unserialize($bytes, ['allowed_classes' => false]); + if (!is_array($value) || ($value['version'] ?? null) !== 2 || ($value['kind'] ?? null) !== 'snapshot') { + throw new \UnexpectedValueException('Unsupported stored feed format.'); + } + $snapshot = $value['snapshot'] ?? null; + if (!is_array($snapshot) || !is_string($snapshot['mime'] ?? null)) { + throw new \UnexpectedValueException('Invalid stored parser input.'); + } + return new self(['version' => 2, 'kind' => 'snapshot', 'snapshot' => [ + 'response' => self::decodeResponse($snapshot['response'] ?? null), + 'mime' => $snapshot['mime'], + ]]); + } + + /** @return SourceResponse */ + private static function decodeResponse(mixed $value): array + { + if (!is_array($value) || !is_string($value['uri'] ?? null) || !is_string($value['base'] ?? null) + || !is_int($value['status'] ?? null) || !is_string($value['body'] ?? null) || !self::validHeaders($value['headers'] ?? null) + ) { + throw new \UnexpectedValueException('Invalid stored source response.'); + } + // Validate PSR-7 header and status semantics, not just PHP types. + new Response($value['status'], $value['headers'], $value['body']); + return [ + 'uri' => $value['uri'], + 'base' => $value['base'], + 'status' => $value['status'], + 'headers' => $value['headers'], + 'body' => $value['body'], + ]; + } + + /** @phpstan-assert-if-true array> $headers */ + private static function validHeaders(mixed $headers): bool + { + if (!is_array($headers)) { + return false; + } + foreach ($headers as $name => $values) { + if (!is_string($name) || !is_array($values) || !array_is_list($values)) { + return false; + } + foreach ($values as $value) { + if (!is_string($value)) { + return false; + } + } + } + return true; + } +} diff --git a/Classes/Storage/FeedStorage.php b/Classes/Storage/FeedStorage.php new file mode 100644 index 00000000..309cf1e7 --- /dev/null +++ b/Classes/Storage/FeedStorage.php @@ -0,0 +1,163 @@ +connection()->select(['*'], self::TABLE, ['url_hash' => self::hashUrl($url)])->fetchAssociative(); + if (!is_array($row)) { + return null; + } + $document = (string)$row['revision'] === '' ? null : $this->decodeDocument($url, (string)$row['revision'], $row['response_data']); + return $this->stateFromRow($row, $document); + } + + public function findState(string $url): ?StoredFeed + { + $row = $this->connection()->select( + ['url', 'revision', 'fetched_at', 'last_attempt_at', 'last_error', 'invalidation_pending'], + self::TABLE, + ['url_hash' => self::hashUrl($url)], + )->fetchAssociative(); + return is_array($row) ? $this->stateFromRow($row) : null; + } + + public function loadDocument(string $url, string $revision): ?FeedDocument + { + $row = $this->connection()->select(['response_data'], self::TABLE, [ + 'url_hash' => self::hashUrl($url), 'revision' => $revision, + ])->fetchAssociative(); + // A different revision is a concurrent replacement, not corrupt data. + return is_array($row) ? $this->decodeDocument($url, $revision, $row['response_data']) : null; + } + + private function decodeDocument(string $url, string $revision, mixed $bytes): ?FeedDocument + { + try { + if (is_resource($bytes)) { + $bytes = stream_get_contents($bytes); + } + if (!is_string($bytes) || !hash_equals($revision, hash('sha256', $bytes))) { + throw new \UnexpectedValueException('Stored feed checksum mismatch.'); + } + return FeedDocument::decode($bytes); + } catch (\Throwable $exception) { + $this->logger->warning('Could not decode the stored feed.', ['url' => $url, 'exception' => $exception]); + return null; + } + } + + /** @param array $row */ + private function stateFromRow(array $row, ?FeedDocument $document = null): StoredFeed + { + return new StoredFeed( + (string)$row['url'], + $document, + (string)$row['revision'], + (int)$row['fetched_at'], + (int)$row['last_attempt_at'], + (string)$row['last_error'], + (bool)$row['invalidation_pending'], + ); + } + + public function storeSuccess(string $url, FeedDocument $document, int $now): bool + { + $previous = $this->find($url); + $revision = $document->revision(); + $changed = $previous === null || $previous->document === null || $previous->revision !== $revision; + $fields = ['fetched_at' => $now, 'last_attempt_at' => $now, 'last_error' => '']; + if ($changed) { + $fields += ['response_data' => $document->encode(), 'revision' => $revision, 'invalidation_pending' => 1]; + } + $this->write($url, $fields, $previous !== null); + return $changed; + } + + public function storeFailure(string $url, string $error, int $now): void + { + $this->write($url, [ + 'last_attempt_at' => $now, + 'last_error' => mb_convert_encoding($error, 'UTF-8', 'UTF-8'), + ], $this->findState($url) !== null); + } + + public function markInvalidated(string $url, string $revision): void + { + // Never clear a pending invalidation belonging to a concurrently newer revision. + $this->connection()->update(self::TABLE, ['invalidation_pending' => 0], [ + 'url_hash' => self::hashUrl($url), 'revision' => $revision, + ]); + } + + /** @return list */ + public function findUrls(): array + { + return array_map(strval(...), $this->connection()->select(['url'], self::TABLE, [], [], ['uid' => 'ASC'])->fetchFirstColumn()); + } + + public function remove(string $url): void + { + $this->connection()->delete(self::TABLE, ['url_hash' => self::hashUrl($url)]); + } + + /** @param array $fields */ + private function write(string $url, array $fields, bool $exists): void + { + $connection = $this->connection(); + $identifier = ['url_hash' => self::hashUrl($url)]; + $types = isset($fields['response_data']) ? ['response_data' => Connection::PARAM_LOB] : []; + if ($exists) { + // All successful-response fields change in one statement; no delete/insert gap. + $connection->update(self::TABLE, $fields, $identifier, $types); + return; + } + try { + $connection->insert(self::TABLE, $identifier + ['url' => $url] + $fields, $types); + } catch (UniqueConstraintViolationException) { + $connection->update(self::TABLE, $fields, $identifier, $types); + } + } + + private function connection(): Connection + { + return $this->connectionPool->getConnectionForTable(self::TABLE); + } +} diff --git a/Classes/Storage/StoredFeed.php b/Classes/Storage/StoredFeed.php new file mode 100644 index 00000000..0eccfe42 --- /dev/null +++ b/Classes/Storage/StoredFeed.php @@ -0,0 +1,37 @@ +hasDocument = $revision !== ''; + } +} diff --git a/Configuration/Services.yaml b/Configuration/Services.yaml index 392b4ca6..266b1a85 100644 --- a/Configuration/Services.yaml +++ b/Configuration/Services.yaml @@ -8,10 +8,6 @@ services: resource: '../Classes/*' exclude: '../Classes/Domain/Model/*' - ErHaWeb\FeedDisplay\Controller\FeedController: - arguments: - $cache: '@cache.feeddisplay' - # Register the external dependency explicitly so FeedDataService can autowire it. SimplePie\SimplePie: ~ diff --git a/Tests/Fixtures/StoredDocument.php b/Tests/Fixtures/StoredDocument.php new file mode 100644 index 00000000..901932ed --- /dev/null +++ b/Tests/Fixtures/StoredDocument.php @@ -0,0 +1,31 @@ + 2, 'kind' => 'snapshot', 'snapshot' => [ + 'response' => ['uri' => $uri, 'base' => $uri, 'status' => 200, 'headers' => ['content-type' => ['application/rss+xml']], 'body' => $body], + 'mime' => 'application/rss+xml', + ]]); + } +} diff --git a/Tests/Functional/Fixtures/FeedHttp.php b/Tests/Functional/Fixtures/FeedHttp.php new file mode 100644 index 00000000..0b12fc12 --- /dev/null +++ b/Tests/Functional/Fixtures/FeedHttp.php @@ -0,0 +1,54 @@ + */ + public static array $requests = []; + /** @var array */ + public static array $responses = []; + + public static function middleware(callable $next): callable + { + return static function (RequestInterface $request, array $options) { + $uri = (string)$request->getUri(); + self::$requests[] = $uri; + $response = self::$responses[$uri] ?? new Response(503, [], 'Unavailable'); + // The recorder closes consumed streams. Every request needs a fresh body. + return Create::promiseFor(new Response($response->getStatusCode(), $response->getHeaders(), (string)$response->getBody())); + }; + } + + public static function rss(string $title, string $extra = ''): Response + { + return new Response(200, ['Content-Type' => 'application/rss+xml'], + '' . $title . 'https://example.test/Test' + . $extra . 'Itemhttps://example.test/item'); + } + + public static function countRequests(string $url): int + { + return count(array_filter(self::$requests, static fn(string $request): bool => $request === $url)); + } +} diff --git a/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php b/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php new file mode 100644 index 00000000..4013e65d --- /dev/null +++ b/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php @@ -0,0 +1,37 @@ +configurationToUseInTestInstance['SYS']['caching']['cacheConfigurations']['pages']['backend'] = Typo3DatabaseBackend::class; + parent::setUp(); + FeedHttp::$requests = []; + FeedHttp::$responses = []; + // Frontend Bootstrap::init() reloads HTTP configuration from the test instance. + file_put_contents($this->instancePath . '/typo3conf/system/additional.php', 'requestPage(); self::assertSame(2, substr_count($body, 'class="item-title"')); - self::assertStringContainsString('Updated feed title', $body); + self::assertStringContainsString('Original feed title', $body); self::assertStringContainsString('First item', $body); self::assertStringContainsString('Second item', $body); @@ -68,6 +68,9 @@ public function flexFormOverridesTypoScriptAndSelectsSeparateCacheEntries(): voi self::assertSame(1, substr_count($body, 'class="item-title"')); self::assertStringContainsString('Original feed title', $body); self::assertStringNotContainsString('Updated feed title', $body); + // Both settings use one source; a refresh invalidates both display revisions. + $this->get(\ErHaWeb\FeedDisplay\Service\FeedRefresher::class)->refresh($feedUrl); + self::assertStringContainsString('Updated feed title', $this->requestPage()); } #[Test] diff --git a/Tests/Functional/Frontend/PersistentFeedTest.php b/Tests/Functional/Frontend/PersistentFeedTest.php new file mode 100644 index 00000000..5ecc20e2 --- /dev/null +++ b/Tests/Functional/Frontend/PersistentFeedTest.php @@ -0,0 +1,123 @@ +writeFeedFixture('Persisted title', ['Original item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + self::assertStringContainsString('Persisted title', $this->requestPage()); + $storage = $this->get(FeedStorage::class); + $stored = $storage->find($url); + self::assertNotNull($stored); + self::assertNotNull($stored->document); + unlink($url); + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['last_attempt_at' => 0], ['url_hash' => FeedStorage::hashUrl($url)]); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('Original item', $this->requestPage()); + self::assertSame($stored->revision, $storage->find($url)?->revision); + self::assertSame($stored->fetchedAt, $storage->find($url)->fetchedAt); + self::assertNotSame('', $storage->find($url)->lastError); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('Original item', $this->requestPage()); + } + + #[Test] + #[DataProvider('uncachedDurations')] + public function uncachedLiveFetchesStillPersistAndFallBack(int $duration): void + { + $url = $this->writeFeedFixture('Initial', ['Original item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => $duration]); + self::assertStringContainsString('Initial', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + $this->writeFeedFixture('Updated', []); + self::assertStringContainsString('Updated', $this->requestPage()); + unlink($url); + $body = $this->requestPage(); + self::assertStringContainsString('Updated', $body); + self::assertStringNotContainsString('Original item', $body); + } + + /** @return iterable */ + public static function uncachedDurations(): iterable + { + yield 'zero' => [0]; + yield 'negative' => [-1]; + } + + #[Test] + public function legacyDisplayCacheRemainsUsableBeforeTheFirstNewFetch(): void + { + $url = $this->writeFeedFixture('Old cache', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + self::assertStringContainsString('Old cache', $this->requestPage()); + $cached = $this->getCachedFeedData(); + self::assertIsArray($cached); + unset($cached['_feeddisplay_revision']); + $this->setCachedFeedData($cached); + $this->get(FeedStorage::class)->remove($url); + $this->writeFeedFixture('New source', ['Item']); + self::assertStringContainsString('Old cache', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->find($url)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + self::assertStringContainsString('New source', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } + + #[Test] + public function liveRenderingKeepsWorkingWhenTheSchemaUpdateWasMissed(): void + { + $url = $this->writeFeedFixture('Without schema', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + try { + self::assertStringContainsString('Without schema', $this->requestPage()); + unlink($url); + self::assertStringContainsString('Without schema', $this->requestPage()); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function liveFirstRequestCanReadWithoutWritingWhileAnotherWorkerHoldsTheLock(): void + { + $url = $this->writeFeedFixture('Parallel live', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + self::assertTrue($lock->acquire($mode)); + try { + self::assertStringContainsString('Parallel live', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->find($url)); + } finally { + $lock->release(); + } + self::assertStringContainsString('Parallel live', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } +} diff --git a/Tests/Functional/Frontend/PersistentHttpFeedTest.php b/Tests/Functional/Frontend/PersistentHttpFeedTest.php new file mode 100644 index 00000000..c7969601 --- /dev/null +++ b/Tests/Functional/Frontend/PersistentHttpFeedTest.php @@ -0,0 +1,258 @@ + 'text/html'], ''); + FeedHttp::$responses[$feedUrl] = FeedHttp::rss('DISCOVERED_ORIGINAL'); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 3600]); + self::assertStringContainsString('DISCOVERED_ORIGINAL', $this->requestPage()); + $before = $this->get(FeedStorage::class)->find(self::URL); + self::assertNotNull($before?->document); + FeedHttp::$responses = []; + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('DISCOVERED_ORIGINAL', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertSame(1, FeedHttp::countRequests($feedUrl)); + self::assertSame($before->revision, $this->get(FeedStorage::class)->find(self::URL)?->revision); + } + + #[Test] + public function changedLiveSourcesLeaveExistingLivePagesCached(): void + { + FeedHttp::$responses[self::URL] = FeedHttp::rss('Original'); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 3600]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nconfig.cache_period = 7200"); + self::assertStringContainsString('Original', $this->requestPage()); + $pages = $this->getConnectionPool()->getConnectionForTable('cache_pages'); + $identifiers = $pages->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative(); + self::assertCount(1, $identifiers); + self::assertGreaterThan(time() + 3600, (int)$identifiers[0]['expires']); + foreach ([3600, 0, -1] as $duration) { + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['last_attempt_at' => 1], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Changed ' . $duration); + $data = $this->get(FeedProvider::class)->getData(['feedUrl' => self::URL, 'cacheDuration' => $duration, 'getFields' => ['feed' => 'title']]); + self::assertSame('Changed ' . $duration, $data['feed']['title']); + self::assertSame($identifiers, $pages->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative()); + self::assertStringContainsString('Original', $this->requestPage()); + } + } + + #[Test] + public function validDisplayCacheDoesNotReadOrDecodeACorruptBlob(): void + { + FeedHttp::$responses[self::URL] = FeedHttp::rss('Cached success'); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 600]); + self::assertStringContainsString('Cached success', $this->requestPage()); + $connection = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $connection->update(FeedStorage::TABLE, ['response_data' => 'corrupt'], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + self::assertStringContainsString('Cached success', $this->requestPage()); + self::assertStringContainsString('Cached success', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + // A miss diagnoses and repairs the unusable document despite a positive live TTL. + $now = (int)$this->get(Context::class)->getPropertyFromAspect('date', 'timestamp'); + $connection->update(FeedStorage::TABLE, ['last_attempt_at' => $now - 61], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Repaired'); + self::assertStringContainsString('Repaired', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } + + #[Test] + #[DataProvider('writeFailureCases')] + public function failedWritesKeepTheSuccessfulDisplayResult(int $duration, bool $existing): void + { + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + if ($existing) { + $this->get(FeedStorage::class)->storeSuccess(self::URL, StoredDocument::create('OldTest'), 1); + } + $before = $this->get(FeedStorage::class)->find(self::URL); + $connection = $this->createMock(Connection::class); + $connection->method('select')->willReturnCallback($real->select(...)); + $connection->method('update')->willThrowException(new \RuntimeException('Simulated packet limit')); + $connection->method('insert')->willThrowException(new \RuntimeException('Simulated packet limit')); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $storage = new FeedStorage($pool, new NullLogger()); + $fetcher = $this->get(FeedFetcher::class); + $context = $this->get(Context::class); + $refresher = new FeedRefresher($storage, $fetcher, $this->get(LockFactory::class), $this->get(CacheManager::class), $context, new NullLogger()); + $provider = new FeedProvider($storage, $refresher, $fetcher, $this->get(FeedDataService::class), $this->get(CacheManager::class)->getCache('feeddisplay'), $context, new NullLogger()); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Fresh but not stored'); + for ($request = 0; $request < 3; ++$request) { + $data = $provider->getData(['feedUrl' => self::URL, 'cacheDuration' => $duration, 'getFields' => ['feed' => 'title']]); + self::assertSame('Fresh but not stored', $data['feed']['title']); + self::assertArrayNotHasKey('_feeddisplay_revision', $data); + } + self::assertSame($duration > 0 ? 1 : 3, FeedHttp::countRequests(self::URL)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + self::assertSame('Fresh but not stored', $provider->getData(['feedUrl' => self::URL, 'cacheDuration' => $duration, 'getFields' => ['feed' => 'title']])['feed']['title']); + self::assertSame($duration > 0 ? 2 : 4, FeedHttp::countRequests(self::URL)); + self::assertSame($before?->revision, $this->get(FeedStorage::class)->find(self::URL)?->revision); + self::assertSame($before?->lastAttemptAt, $this->get(FeedStorage::class)->find(self::URL)?->lastAttemptAt); + } + + /** @return iterable */ + public static function writeFailureCases(): iterable + { + foreach ([600, 0, -1] as $duration) { + foreach ([false, true] as $existing) { + yield $duration . ($existing ? ' old row' : ' no row') => [$duration, $existing]; + } + } + } + + #[Test] + public function missingSchemaStillCachesLiveErrorsForTheConfiguredDuration(): void + { + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 600]); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + try { + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function liveOutageKeepsTheStoredResponse(): void + { + $document = StoredDocument::create((string)FeedHttp::rss('Stored live')->getBody(), self::URL . '/'); + $storage = $this->get(FeedStorage::class); + $storage->storeSuccess(self::URL, $document, 1); + $storage->markInvalidated(self::URL, $document->revision()); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 3600]); + self::assertStringContainsString('Stored live', $this->requestPage()); + self::assertStringContainsString('Stored live', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + $after = $storage->findState(self::URL); + self::assertNotNull($after); + self::assertSame($document->revision(), $after->revision); + self::assertSame(1, $after->fetchedAt); + self::assertNotSame('', $after->lastError); + } + + #[Test] + #[DataProvider('readFailuresAfterPersistence')] + public function persistedResultsKeepTheirRevisionAndRetryInvalidation(string $failure): void + { + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $reads = 0; + $connection = self::createStub(Connection::class); + $connection->method('select')->willReturnCallback(static function (array $columns, string $table, array $where = []) use ($real, &$reads, $failure): \Doctrine\DBAL\Result { + if (in_array('last_attempt_at', $columns, true) && ++$reads === 3) { + if ($failure === 'exception') { + throw new \RuntimeException('Read failed after successful write'); + } + $empty = self::createStub(\Doctrine\DBAL\Result::class); + $empty->method('fetchAssociative')->willReturn(false); + return $empty; + } + return $real->select($columns, $table, $where); + }); + $connection->method('insert')->willReturnCallback($real->insert(...)); + $connection->method('update')->willReturnCallback($real->update(...)); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $storage = new FeedStorage($pool, new NullLogger()); + $flushes = 0; + $manager = $this->createMock(CacheManager::class); + $manager->expects(self::exactly(2))->method('flushCachesInGroupByTag')->willReturnCallback(static function () use (&$flushes): void { + if (++$flushes === 1) { + throw new \RuntimeException('Page invalidation temporarily unavailable'); + } + }); + $context = $this->get(Context::class); + $fetcher = $this->get(FeedFetcher::class); + $refresher = new FeedRefresher($storage, $fetcher, $this->get(LockFactory::class), $manager, $context, new NullLogger()); + $cache = $this->get(CacheManager::class)->getCache('feeddisplay'); + $provider = new FeedProvider($storage, $refresher, $fetcher, $this->get(FeedDataService::class), $cache, $context, new NullLogger()); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Persisted'); + $settings = ['feedUrl' => self::URL, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + self::assertSame('Persisted', $provider->getData($settings)['feed']['title']); + $state = $this->get(FeedStorage::class)->findState(self::URL); + self::assertNotNull($state); + self::assertTrue($state->invalidationPending); + $cached = $cache->get('feeddisplay_' . md5(serialize($settings))); + self::assertIsArray($cached); + self::assertSame($state->revision, $cached['_feeddisplay_revision']); + self::assertSame('Persisted', $provider->getData($settings)['feed']['title']); + self::assertFalse($this->get(FeedStorage::class)->findState(self::URL)?->invalidationPending); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + } + + /** @return iterable */ + public static function readFailuresAfterPersistence(): iterable + { + yield 'exception' => ['exception']; + yield 'missing state' => ['missing']; + } + + #[Test] + public function cacheHitsOnlyReadMetadataAndDoNotLogCorruptPayloads(): void + { + FeedHttp::$responses[self::URL] = FeedHttp::rss('Large'); + $settings = ['feedUrl' => self::URL, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + $this->get(FeedProvider::class)->getData($settings); + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $real->update(FeedStorage::TABLE, ['response_data' => 'corrupt'], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + $connection = $this->createMock(Connection::class); + $connection->expects(self::exactly(3))->method('select')->with( + ['url', 'revision', 'fetched_at', 'last_attempt_at', 'last_error', 'invalidation_pending'], + FeedStorage::TABLE, + ['url_hash' => FeedStorage::hashUrl(self::URL)], + )->willReturnCallback($real->select(...)); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $logger = $this->createMock(LoggerInterface::class); + $logger->expects(self::never())->method('warning'); + $storage = new FeedStorage($pool, $logger); + $provider = new FeedProvider($storage, $this->get(FeedRefresher::class), $this->get(FeedFetcher::class), $this->get(FeedDataService::class), $this->get(CacheManager::class)->getCache('feeddisplay'), $this->get(Context::class), $logger); + for ($i = 0; $i < 3; ++$i) { + self::assertSame('Large', $provider->getData($settings)['feed']['title']); + } + } +} diff --git a/Tests/Functional/Storage/FeedFailureTest.php b/Tests/Functional/Storage/FeedFailureTest.php new file mode 100644 index 00000000..6498cf55 --- /dev/null +++ b/Tests/Functional/Storage/FeedFailureTest.php @@ -0,0 +1,200 @@ +instancePath . '/new.xml'; + file_put_contents($path, 'FreshTestItem'); + return $path; + } + + #[Test] + public function failedDatabaseWritePreservesTheOldDocumentAndReturnsTheFreshParser(): void + { + $url = $this->source(); + $storage = $this->get(FeedStorage::class); + $original = StoredDocument::create('Old response'); + $storage->storeSuccess($url, $original, 1000); + $row = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->select(['*'], FeedStorage::TABLE)->fetchAssociative(); + $result = self::createStub(Result::class); + $result->method('fetchAssociative')->willReturn($row); + $connection = $this->createMock(Connection::class); + $connection->method('select')->willReturn($result); + $connection->expects(self::once())->method('update')->willThrowException(new \RuntimeException('Simulated write failure')); + $connection->expects(self::never())->method('delete'); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $failing = new FeedStorage($pool, new NullLogger()); + $refresher = new FeedRefresher($failing, $this->get(FeedFetcher::class), $this->get(LockFactory::class), $this->get(CacheManager::class), $this->get(Context::class), new NullLogger()); + $refreshed = $refresher->refresh($url); + self::assertSame('failed', $refreshed->status); + self::assertSame('Fresh', \ErHaWeb\FeedDisplay\Service\SimplePieDeprecationHandler::run(static fn(): ?string => $refreshed->fetched?->feed->get_title())); + self::assertSame($original->revision(), $storage->find($url)?->revision); + self::assertSame(1000, $storage->find($url)->fetchedAt); + } + + #[Test] + public function corruptDocumentsProduceADiagnosis(): void + { + $logger = $this->createMock(LoggerInterface::class); + $logger->expects(self::once())->method('warning')->with('Could not decode the stored feed.', self::callback(static fn(array $context): bool => $context['url'] === 'broken')); + $storage = new FeedStorage($this->getConnectionPool(), $logger); + $storage->storeSuccess('broken', StoredDocument::create('Original'), 1000); + $bytes = serialize(['version' => 2, 'kind' => 'snapshot', 'snapshot' => ['response' => ['uri' => 'https://example.test', 'base' => 'https://example.test', 'status' => 200, 'body' => 'Body', 'headers' => ['Bad Header' => ['x']]], 'mime' => 'text/xml']]); + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['response_data' => $bytes, 'revision' => hash('sha256', $bytes)], ['url_hash' => FeedStorage::hashUrl('broken')]); + self::assertNull($storage->find('broken')?->document); + } + + #[Test] + #[DataProvider('lockFailures')] + public function lockFailuresAreDiagnosedAndBusySourcesAreSkipped(string $stage, string $expected): void + { + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($expected === 'failed' ? self::once() : self::never())->method('warning'); + $url = $this->source(); + $locks = $this->createMock(LockFactory::class); + $lock = $this->createMock(LockingStrategyInterface::class); + if ($stage === 'create') { + $locks->method('createLocker')->willThrowException(new \RuntimeException('Create failed')); + } else { + $locks->method('createLocker')->willReturn($lock); + } + if ($stage === 'acquire' || $stage === 'would-block') { + $lock->method('acquire')->willThrowException($stage === 'acquire' ? new \RuntimeException('Acquire failed') : new LockAcquireWouldBlockException('Busy')); + } else { + $lock->method('acquire')->willReturn($stage !== 'busy'); + } + if ($stage === 'release') { + $lock->expects(self::once())->method('release')->willThrowException(new \RuntimeException('Release failed')); + } else { + $lock->expects(self::never())->method('release'); + } + $refresher = new FeedRefresher($this->get(FeedStorage::class), $this->get(FeedFetcher::class), $locks, $this->get(CacheManager::class), $this->get(Context::class), $logger); + self::assertSame($expected, $refresher->refresh($url)->status); + } + + /** @return iterable */ + public static function lockFailures(): iterable + { + yield 'creation' => ['create', 'failed']; + yield 'acquisition' => ['acquire', 'failed']; + yield 'release' => ['release', 'failed']; + yield 'busy return' => ['busy', 'locked']; + yield 'busy exception' => ['would-block', 'locked']; + } + + #[Test] + public function publicEventFailuresAreNotSwallowedByTheProvider(): void + { + $dispatcher = self::createStub(EventDispatcherInterface::class); + $dispatcher->method('dispatch')->willThrowException(new \DomainException('Listener failed')); + $data = new FeedDataService(new SimplePie(), $dispatcher, $this->get(FeedRuntimeInitializer::class)); + $provider = new FeedProvider($this->get(FeedStorage::class), $this->get(FeedRefresher::class), $this->get(FeedFetcher::class), $data, $this->get(CacheManager::class)->getCache('feeddisplay'), $this->get(Context::class), new NullLogger()); + $this->expectException(\DomainException::class); + $this->expectExceptionMessage('Listener failed'); + $provider->getData(['feedUrl' => $this->source(), 'cacheDuration' => 0, 'getFields' => ['items' => 'title']]); + } + #[Test] + public function concurrentRevisionIsNotLabelledWithAnEarlierParsersData(): void + { + $url = $this->source(); + $storage = $this->get(FeedStorage::class); + $next = StoredDocument::create('Concurrent resultTest'); + $lock = $this->createMock(LockingStrategyInterface::class); + $lock->method('acquire')->willReturn(true); + $lock->method('release')->willReturnCallback(static function () use ($storage, $url, $next): bool { + // Model a second writer completing after our lock release, before re-reading. + $storage->storeSuccess($url, $next, time()); + return true; + }); + $locks = self::createStub(LockFactory::class); + $locks->method('createLocker')->willReturn($lock); + $cache = $this->get(CacheManager::class)->getCache('feeddisplay'); + $fetcher = $this->get(FeedFetcher::class); + $context = $this->get(Context::class); + $refresher = new FeedRefresher($storage, $fetcher, $locks, $this->get(CacheManager::class), $context, new NullLogger()); + $provider = new FeedProvider($storage, $refresher, $fetcher, $this->get(FeedDataService::class), $cache, $context, new NullLogger()); + $settings = ['feedUrl' => $url, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + $data = $provider->getData($settings); + self::assertSame('Concurrent result', $data['feed']['title']); + self::assertArrayNotHasKey('_feeddisplay_revision', $data); + $cached = $cache->get('feeddisplay_' . md5(serialize($settings))); + self::assertIsArray($cached); + self::assertSame('Concurrent result', $cached['feed']['title']); + self::assertSame($next->revision(), $cached['_feeddisplay_revision']); + } + + #[Test] + public function concurrentReplacementDuringLazyLoadingIsRetriedWithItsOwnRevision(): void + { + $url = $this->source(); + $storage = $this->get(FeedStorage::class); + $fetcher = $this->get(FeedFetcher::class); + $storage->storeSuccess($url, $fetcher->fetch($url)->document, time()); + $next = StoredDocument::create('Concurrent lazy resultTest'); + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $swapped = false; + $connection = self::createStub(Connection::class); + $connection->method('select')->willReturnCallback(static function (array $columns, string $table, array $where = []) use ($storage, $url, $next, $real, &$swapped): Result { + if ($columns === ['response_data'] && !$swapped) { + $swapped = true; + $storage->storeSuccess($url, $next, time()); + } + return $real->select($columns, $table, $where); + }); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $lazy = new FeedStorage($pool, new NullLogger()); + $cache = $this->get(CacheManager::class)->getCache('feeddisplay'); + $provider = new FeedProvider($lazy, $this->get(FeedRefresher::class), $fetcher, $this->get(FeedDataService::class), $cache, $this->get(Context::class), new NullLogger()); + $settings = ['feedUrl' => $url, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + self::assertSame('Concurrent lazy result', $provider->getData($settings)['feed']['title']); + $cached = $cache->get('feeddisplay_' . md5(serialize($settings))); + self::assertIsArray($cached); + self::assertSame($next->revision(), $cached['_feeddisplay_revision']); + } + +} diff --git a/Tests/Functional/Storage/FeedStorageTest.php b/Tests/Functional/Storage/FeedStorageTest.php new file mode 100644 index 00000000..2ce07265 --- /dev/null +++ b/Tests/Functional/Storage/FeedStorageTest.php @@ -0,0 +1,137 @@ +Stra\xDFeTest'; + + #[Test] + public function binaryDocumentSurvivesEveryCacheFlushAndFailedRefresh(): void + { + $storage = $this->get(FeedStorage::class); + $document = StoredDocument::create("\x00Stra\xDFe\xff"); + self::assertTrue($storage->storeSuccess(self::URL, $document, 1000)); + $this->get(CacheManager::class)->flushCaches(); + $storage->storeFailure(self::URL, 'HTTP 503', 2000); + $stored = $storage->find(self::URL); + self::assertNotNull($stored); + self::assertSame($document->payload, $stored->document?->payload); + self::assertSame(1000, $stored->fetchedAt); + self::assertSame(2000, $stored->lastAttemptAt); + self::assertSame('HTTP 503', $stored->lastError); + } + + #[Test] + public function unchangedSuccessRenewsFreshnessAndPreservesPendingInvalidation(): void + { + $storage = $this->get(FeedStorage::class); + $one = StoredDocument::create('one'); + $two = StoredDocument::create('two'); + $storage->storeSuccess(self::URL, $one, 1000); + $storage->markInvalidated(self::URL, $one->revision()); + self::assertFalse($storage->storeSuccess(self::URL, $one, 2000)); + self::assertSame(2000, $storage->find(self::URL)?->fetchedAt); + self::assertFalse($storage->find(self::URL)->invalidationPending); + $storage->storeSuccess(self::URL, $two, 3000); + $storage->markInvalidated(self::URL, $one->revision()); + self::assertTrue($storage->find(self::URL)?->invalidationPending); + self::assertFalse($storage->storeSuccess(self::URL, $two, 4000)); + self::assertTrue($storage->find(self::URL)?->invalidationPending); + } + + #[Test] + public function unchangedRefreshRetriesAFailedPageInvalidation(): void + { + $calls = 0; + $manager = $this->createMock(CacheManager::class); + $manager->expects(self::exactly(2))->method('flushCachesInGroupByTag')->willReturnCallback( + static function () use (&$calls): void { + if (++$calls === 1) { + throw new \RuntimeException('Page cache temporarily unavailable'); + } + } + ); + $body = str_replace('Stra\\xDFe', "Stra\xDFe", self::BODY); + $client = new Client(['handler' => HandlerStack::create(new MockHandler([ + new Response(200, ['Content-Type' => 'application/rss+xml'], $body), + new Response(200, ['Content-Type' => 'application/rss+xml'], $body), + ]))]); + $factory = self::createStub(GuzzleClientFactory::class); + $factory->method('getClient')->willReturn($client); + $http = new HttpFactory(); + $initializer = new FeedRuntimeInitializer($factory, $http, $http); + $lock = self::createStub(LockingStrategyInterface::class); + $lock->method('acquire')->willReturn(true); + $locks = self::createStub(LockFactory::class); + $locks->method('createLocker')->willReturn($lock); + $context = new Context(); + $context->setAspect('date', new DateTimeAspect(new \DateTimeImmutable('@1000'))); + $storage = $this->get(FeedStorage::class); + $subject = new FeedRefresher($storage, new FeedFetcher($initializer, $factory), $locks, $manager, $context, self::createStub(LoggerInterface::class)); + $first = $subject->refresh(self::URL); + self::assertSame('failed', $first->status); + self::assertTrue($first->persisted); + self::assertTrue($storage->find(self::URL)?->invalidationPending); + $context->setAspect('date', new DateTimeAspect(new \DateTimeImmutable('@2000'))); + self::assertSame('unchanged', $subject->refresh(self::URL)->status); + self::assertFalse($storage->find(self::URL)?->invalidationPending); + self::assertSame(2000, $storage->find(self::URL)->fetchedAt); + } + #[Test] + public function lazyReadsDoNotReturnAPayloadFromADifferentRevision(): void + { + $storage = $this->get(FeedStorage::class); + $one = StoredDocument::create("one\xFF\x00"); + $two = StoredDocument::create("two\xDF\x00"); + $storage->storeSuccess(self::URL, $one, 1000); + $state = $storage->findState(self::URL); + self::assertNotNull($state); + self::assertTrue($state->hasDocument); + self::assertNull($state->document); + self::assertSame($one->payload, $storage->loadDocument(self::URL, $state->revision)?->payload); + $storage->storeSuccess(self::URL, $two, 2000); + self::assertNull($storage->loadDocument(self::URL, $state->revision)); + $current = $storage->findState(self::URL); + self::assertNotNull($current); + self::assertSame($two->payload, $storage->loadDocument(self::URL, $current->revision)?->payload); + } + +} diff --git a/Tests/Unit/Controller/FeedControllerTest.php b/Tests/Unit/Controller/FeedControllerTest.php deleted file mode 100644 index 9aef6e56..00000000 --- a/Tests/Unit/Controller/FeedControllerTest.php +++ /dev/null @@ -1,239 +0,0 @@ -createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - - $cache->expects(self::never())->method('get'); - $feedDataService->expects(self::never())->method('buildData'); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings([]); - $subject->setView(new RecordingView()); - - $subject->displayAction(); - } - - #[Test] - public function displayActionBypassesCacheIfCacheDurationIsZero(): void - { - $settings = [ - 'cacheDuration' => 0, - ]; - $data = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'First item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn(false); - $cache->expects(self::once())->method('remove')->with($cacheIdentifier); - $cache->expects(self::never())->method('set'); - $feedDataService->expects(self::once())->method('buildData')->with($settings)->willReturn($data); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($data, $view->assigned['data']); - } - - #[Test] - public function displayActionCachesBuiltDataOnCacheMiss(): void - { - $settings = [ - 'cacheDuration' => 600, - ]; - $data = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'Cached item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn(false); - $cache->expects(self::once())->method('set')->with($cacheIdentifier, $data, [], 600); - $cache->expects(self::never())->method('remove'); - $feedDataService->expects(self::once())->method('buildData')->with($settings)->willReturn($data); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($data, $view->assigned['data']); - } - - #[Test] - public function displayActionReusesCachedDataWhenSettingsMatch(): void - { - $settings = [ - 'cacheDuration' => 600, - ]; - $cachedData = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'Cached item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn($cachedData); - $cache->expects(self::never())->method('set'); - $cache->expects(self::never())->method('remove'); - $feedDataService->expects(self::never())->method('buildData'); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($cachedData, $view->assigned['data']); - } - - #[Test] - public function displayActionRebuildsCacheWhenCachedSettingsDoNotMatch(): void - { - $settings = [ - 'cacheDuration' => 600, - ]; - $cachedData = [ - 'settings' => [ - 'cacheDuration' => 60, - ], - 'items' => [ - ['title' => 'Outdated item'], - ], - ]; - $rebuiltData = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'Fresh item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn($cachedData); - $cache->expects(self::once())->method('set')->with($cacheIdentifier, $rebuiltData, [], 600); - $cache->expects(self::never())->method('remove'); - $feedDataService->expects(self::once())->method('buildData')->with($settings)->willReturn($rebuiltData); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($rebuiltData, $view->assigned['data']); - } -} - -final class TestableFeedController extends FeedController -{ - /** - * @param array $settings - */ - public function setSettings(array $settings): void - { - $this->settings = $settings; - } - - public function setView(ViewInterface $view): void - { - $this->view = $view; - } - - protected function htmlResponse(?string $html = null): ResponseInterface - { - return new HtmlResponse($html ?? ''); - } -} - -final class RecordingView implements ViewInterface -{ - /** @var array */ - public array $assigned = []; - - public function assign(string $key, mixed $value): self - { - $this->assigned[$key] = $value; - return $this; - } - - /** - * @param array $values - */ - public function assignMultiple(array $values): self - { - foreach ($values as $key => $value) { - $this->assigned[$key] = $value; - } - return $this; - } - - public function render(string $templateFileName = ''): string - { - return ''; - } - - /** - * @param array $variables - */ - public function renderSection(string $sectionName, array $variables = [], bool $ignoreUnknown = false): string - { - return ''; - } - - /** - * @param array $variables - */ - public function renderPartial(string $partialName, string $sectionName, array $variables, bool $ignoreUnknown = false): string - { - return ''; - } -} diff --git a/Tests/Unit/Service/FeedHttpTest.php b/Tests/Unit/Service/FeedHttpTest.php new file mode 100644 index 00000000..234667fb --- /dev/null +++ b/Tests/Unit/Service/FeedHttpTest.php @@ -0,0 +1,212 @@ + $responses + */ + #[Test] + #[DataProvider('feedResponses')] + public function recordingAndReplayMatchLiveParsing(array $responses): void + { + $liveClient = new Client(['handler' => HandlerStack::create(new MockHandler($responses))]); + $guzzleFactory = $this->createMock(GuzzleClientFactory::class); + $guzzleFactory->expects(self::once())->method('getClient')->willReturn($liveClient); + $httpFactory = new HttpFactory(); + $initializer = new FeedRuntimeInitializer($guzzleFactory, $httpFactory, $httpFactory); + $live = new SimplePie(); + self::assertTrue($initializer->initializeFeed($live, ['feedUrl' => self::URL])); + + $recordingClient = new Client(['handler' => HandlerStack::create(new MockHandler($responses))]); + $recordingFactory = self::createStub(GuzzleClientFactory::class); + $recordingFactory->method('getClient')->willReturn($recordingClient); + $fetcher = new \ErHaWeb\FeedDisplay\Service\FeedFetcher(new FeedRuntimeInitializer($recordingFactory, $httpFactory, $httpFactory), $recordingFactory); + $fetched = $fetcher->fetch(self::URL); + $recorded = $fetched->feed; + $replayed = $fetcher->restore(\ErHaWeb\FeedDisplay\Storage\FeedDocument::decode($fetched->document->encode())); + + $liveData = $this->getFeedValues($live); + self::assertSame($liveData, $this->getFeedValues($recorded)); + self::assertSame($liveData, $this->getFeedValues($replayed)); + + $dispatcher = $this->createMock(EventDispatcherInterface::class); + $dispatcher->expects(self::once())->method('dispatch')->willReturnCallback( + static function (SingleFeedDataEvent $event) use ($replayed): SingleFeedDataEvent { + self::assertSame($replayed, $event->getFeed()); + self::assertSame($replayed->get_item(0), $event->getItem()); + return $event; + } + ); + (new FeedDataService($live, $dispatcher, $initializer))->buildInitializedData($replayed, [ + 'getFields' => ['items' => 'title'], + 'maxFeedCount' => 1, + ]); + } + + /** + * @return iterable}> + */ + public static function feedResponses(): iterable + { + $rss = '%sTest' + . 'Itemarticle.html'; + $atom = 'News' + . 'Itemurn:news:1'; + + foreach ([301, 302] as $status) { + yield 'RSS redirect ' . $status => [[ + new Response($status, ['Location' => 'https://new.example.test/rss/feed.xml']), + new Response(200, ['Content-Type' => 'application/rss+xml'], sprintf($rss, 'News')), + ]]; + yield 'Atom redirect ' . $status => [[ + new Response($status, ['Location' => 'https://new.example.test/atom/feed.xml']), + new Response(200, ['Content-Type' => 'application/atom+xml'], $atom), + ]]; + } + + yield 'HTML feed autodiscovery' => [[ + new Response(200, ['Content-Type' => 'text/html'], 'News'), + new Response(200, ['Content-Type' => 'application/rss+xml'], sprintf($rss, 'Discovered feed')), + ]]; + + yield 'non-UTF-8 body' => [[new Response( + 200, + ['Content-Type' => 'application/rss+xml'], + '' . sprintf($rss, "Stra\xDFe") + )]]; + yield 'encoding declared in lowercase header' => [[new Response( + 200, + ['content-type' => 'application/rss+xml; charset=windows-1252'], + sprintf($rss, "Price 10 \x80") + )]]; + yield 'encoding declared in mixed case header' => [[new Response( + 200, + ['Content-Type' => 'application/rss+xml; charset=windows-1252'], + sprintf($rss, "Price 10 \x80") + )]]; + yield 'encoding contradicts body' => [[new Response( + 200, + ['Content-Type' => 'application/rss+xml; charset=ISO-8859-1'], + sprintf($rss, 'Straße') + )]]; + foreach ([['Link', 'Content-Language'], ['link', 'content-language']] as [$linkHeader, $languageHeader]) { + yield $linkHeader . ' and ' . $languageHeader => [[new Response( + 200, + [ + 'Content-Type' => 'application/rss+xml', + $linkHeader => '; rel="self"', + $languageHeader => 'de', + 'Set-Cookie' => 'not-stored=1', + ], + sprintf($rss, 'News') + )]]; + } + } + + #[Test] + public function recorderReadsOnlyTheRemainingBodyBudgetAndClosesTheStream(): void + { + $readBytes = 0; + $stream = new PumpStream(static function (int $length) use (&$readBytes): string { + $readBytes += $length; + return str_repeat('x', $length); + }); + $client = self::createStub(GuzzleClientInterface::class); + $client->method('send')->willReturn(new Response(200, [], $stream)); + $recorder = new FeedHttpRecorder($client, 10, 10); + $this->expectException(RequestException::class); + $this->expectExceptionMessage('2 MiB'); + + try { + $recorder->sendRequest(new Request('GET', self::URL)); + } finally { + self::assertSame(2 * 1024 * 1024 + 1, $readBytes); + self::assertTrue($stream->eof()); + } + } + + #[Test] + public function recorderAppliesTheBudgetAcrossRedirectResponses(): void + { + $client = new Client(['handler' => HandlerStack::create(new MockHandler([ + new Response(302, ['Location' => self::URL], str_repeat('x', 1024 * 1024)), + new Response(200, [], str_repeat('x', 1024 * 1024 + 1)), + ]))]); + $recorder = new FeedHttpRecorder($client, 10, 10); + $recorder->sendRequest(new Request('GET', self::URL)); + $this->expectException(RequestException::class); + $recorder->sendRequest(new Request('GET', self::URL)); + } + + #[Test] + public function localReplayPreservesSourceContextWithoutReadingTheFileAgain(): void + { + $factory = self::createStub(GuzzleClientFactory::class); + $http = new HttpFactory(); + $fetcher = new \ErHaWeb\FeedDisplay\Service\FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + $path = tempnam(__DIR__ . '/../../../.Build/Web/typo3temp', 'feed-replay-'); + self::assertIsString($path); + $this->testFilesToDelete[] = $path . '.moved'; + file_put_contents($path, 'Stra' . "\xDF" . 'eTestItemarticle.html'); + $live = $fetcher->fetch($path); + rename($path, $path . '.moved'); + $restored = $fetcher->restore(\ErHaWeb\FeedDisplay\Storage\FeedDocument::decode($live->document->encode())); + self::assertSame($this->getFeedValues($live->feed), $this->getFeedValues($restored)); + } + + /** + * @return array + */ + private function getFeedValues(SimplePie $feed): array + { + return SimplePieDeprecationHandler::run(static fn(): array => [ + 'title' => $feed->get_title(), + 'subscription' => $feed->subscribe_url(), + 'permanentSubscription' => $feed->subscribe_url(true), + 'base' => $feed->get_base(), + 'language' => $feed->get_language(), + 'self' => $feed->get_link(0, 'self'), + 'itemLink' => $feed->get_item(0)?->get_permalink(), + ]); + } +} diff --git a/Tests/Unit/Service/FeedSnapshotTest.php b/Tests/Unit/Service/FeedSnapshotTest.php new file mode 100644 index 00000000..733ff3c7 --- /dev/null +++ b/Tests/Unit/Service/FeedSnapshotTest.php @@ -0,0 +1,160 @@ + $responses */ + private function fetcher(array $responses): FeedFetcher + { + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->expects(self::once())->method('getClient')->willReturn(new Client(['handler' => HandlerStack::create(new MockHandler($responses))])); + $http = new HttpFactory(); + return new FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + } + + private static function rss(string $title): string + { + return '' . $title . 'TestItemarticle.html'; + } + + /** @param list $responses */ + #[Test] + #[DataProvider('responses')] + public function snapshotRetainsSelectedFeedAndItsMetadata(array $responses): void + { + $fetcher = $this->fetcher($responses); + $fetched = $fetcher->fetch(self::URL); + $document = FeedDocument::decode($fetched->document->encode()); + $replay = $fetcher->restore($document); + $values = static fn(SimplePie $feed): array => SimplePieDeprecationHandler::run(static fn(): array => [ + $feed->get_title(), $feed->subscribe_url(), $feed->subscribe_url(true), $feed->get_base(), + $feed->get_language(), $feed->get_link(0, 'self'), $feed->get_item(0)?->get_permalink(), + $feed->get_item(0)?->get_title(), (new FeedValueNormalizer())->normalizeValue($feed->get_all_discovered_feeds()), + $feed->data['headers'] ?? [], + ]); + self::assertSame($values($fetched->feed), $values($replay)); + self::assertSame($fetched->document->revision(), $document->revision()); + } + + /** @return iterable}> */ + public static function responses(): iterable + { + yield from FeedHttpTest::feedResponses(); + $html = ''; + yield 'two discovered feeds retain the first selected feed' => [[ + new Response(200, ['Content-Type' => 'text/html'], $html), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('SELECTED_A')), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('ALTERNATIVE_B')), + ]]; + yield 'trailing non-feed success does not replace the selected feed' => [[ + new Response(200, ['Content-Type' => 'text/html'], $html), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('SELECTED_A')), + new Response(200, ['Content-Type' => 'text/html'], 'No feed here'), + ]]; + } + + #[Test] + public function boundedFetchesUseOnlyTheDirectSource(): void + { + $html = ''; + $mock = new MockHandler([ + new Response(200, ['Content-Type' => 'text/html'], $html), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('NOT_REQUESTED')), + ]); + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->method('getClient')->willReturn(new Client(['handler' => HandlerStack::create($mock)])); + $http = new HttpFactory(); + $fetcher = new FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + try { + $fetcher->fetch(self::URL, true); + self::fail('A bounded fetch must not continue with another source.'); + } catch (\RuntimeException $exception) { + self::assertStringContainsString('could not be found', $exception->getMessage()); + } + self::assertCount(1, $mock); + } + + #[Test] + public function boundedLocalSourcesMustBeFilesWithinTheLimit(): void + { + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->expects(self::never())->method('getClient'); + $http = new HttpFactory(); + $fetcher = new FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + $path = tempnam(sys_get_temp_dir(), 'feed-snapshot-'); + self::assertIsString($path); + try { + file_put_contents($path, str_repeat(' ', FeedHttpRecorder::MAX_BODY_SIZE + 1)); + foreach ([$path, sys_get_temp_dir()] as $source) { + try { + $fetcher->fetch($source, true); + self::fail('The source must be rejected.'); + } catch (\RuntimeException $exception) { + self::assertStringContainsString('at most 2 MiB', $exception->getMessage()); + } + } + } finally { + unlink($path); + } + } + + /** @return iterable */ + public static function httpSourcesWithoutHttpScheme(): iterable + { + yield 'feed scheme' => ['feed://example.test/feed']; + yield 'host only' => ['example.test']; + } + + #[Test] + #[DataProvider('httpSourcesWithoutHttpScheme')] + public function httpSourcesWithoutHttpSchemeAreRecorded(string $source): void + { + $fetched = $this->fetcher([new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('RECORDED'))])->fetch($source, true); + self::assertSame('RECORDED', SimplePieDeprecationHandler::run($fetched->feed->get_title(...))); + } + + #[Test] + public function recorderConfirmsOnlyReceivedBodies(): void + { + $http = new HttpFactory(); + $recorder = new FeedHttpRecorder(new Client(['handler' => HandlerStack::create(new MockHandler([new Response(200, [], 'RECEIVED')]))])); + $recorder->sendRequest($http->createRequest('GET', self::URL)); + self::assertTrue($recorder->received('RECEIVED')); + self::assertFalse($recorder->received('OTHER')); + } +} diff --git a/Tests/Unit/Storage/FeedDocumentTest.php b/Tests/Unit/Storage/FeedDocumentTest.php new file mode 100644 index 00000000..24de8b79 --- /dev/null +++ b/Tests/Unit/Storage/FeedDocumentTest.php @@ -0,0 +1,57 @@ +payload, FeedDocument::decode($document->encode())->payload); + } + + #[Test] + #[DataProvider('invalidDocuments')] + public function malformedDocumentsAreRejected(mixed $payload): void + { + $this->expectException(\Exception::class); + FeedDocument::decode(serialize($payload)); + } + + /** @return iterable */ + public static function invalidDocuments(): iterable + { + yield 'incomplete snapshot' => [['version' => 2, 'kind' => 'snapshot', 'snapshot' => []]]; + $response = ['uri' => 'https://example.test', 'base' => 'https://example.test', 'status' => 200, 'headers' => [], 'body' => 'XML']; + foreach ([['base' => null], ['status' => 700], ['status' => '200'], ['headers' => ['Bad Header' => ['x']]], ['body' => new \stdClass()]] as $invalid) { + yield 'snapshot ' . json_encode($invalid) => [['version' => 2, 'kind' => 'snapshot', 'snapshot' => [ + 'response' => array_replace($response, $invalid), 'mime' => 'text/xml', + ]]]; + } + yield 'serialized object' => [(object)['version' => 1]]; + yield 'missing format' => [[]]; + yield 'unsupported format' => [['version' => 1, 'kind' => 'http', 'steps' => []]]; + } +} diff --git a/Tests/Unit/Storage/FeedStorageTest.php b/Tests/Unit/Storage/FeedStorageTest.php new file mode 100644 index 00000000..2bf63995 --- /dev/null +++ b/Tests/Unit/Storage/FeedStorageTest.php @@ -0,0 +1,75 @@ +encode()); + rewind($stream); + try { + $row = ['url' => 'source', 'response_data' => $stream, 'revision' => $document->revision(), 'fetched_at' => 1000, 'last_attempt_at' => 1000, 'last_error' => '', 'invalidation_pending' => 0]; + $result = self::createStub(Result::class); + $result->method('fetchAssociative')->willReturn($row); + $connection = self::createStub(Connection::class); + $connection->method('select')->willReturn($result); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + self::assertSame($document->payload, (new FeedStorage($pool, new NullLogger()))->find('source')?->document?->payload); + } finally { + fclose($stream); + } + } + #[Test] + public function lazyDocumentReadsSupportLobStreamsAndMatchTheRequestedRevision(): void + { + $document = StoredDocument::create("Binary \x00\xFF\xDF"); + $stream = fopen('php://memory', 'w+b'); + self::assertIsResource($stream); + fwrite($stream, $document->encode()); + rewind($stream); + try { + $result = self::createStub(Result::class); + $result->method('fetchAssociative')->willReturn(['response_data' => $stream]); + $connection = $this->createMock(Connection::class); + $connection->expects(self::once())->method('select')->with(['response_data'], FeedStorage::TABLE, [ + 'url_hash' => FeedStorage::hashUrl('source'), 'revision' => $document->revision(), + ])->willReturn($result); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + self::assertSame($document->payload, (new FeedStorage($pool, new NullLogger()))->loadDocument('source', $document->revision())?->payload); + } finally { + fclose($stream); + } + } + +} diff --git a/ext_tables.sql b/ext_tables.sql new file mode 100644 index 00000000..716598ad --- /dev/null +++ b/ext_tables.sql @@ -0,0 +1,13 @@ +CREATE TABLE tx_feeddisplay_feed ( + uid int(11) unsigned NOT NULL auto_increment, + url_hash char(64) DEFAULT '' NOT NULL, + url text, + response_data longblob, + revision char(64) DEFAULT '' NOT NULL, + fetched_at int(11) unsigned DEFAULT '0' NOT NULL, + last_attempt_at int(11) unsigned DEFAULT '0' NOT NULL, + last_error text, + invalidation_pending smallint(5) unsigned DEFAULT '0' NOT NULL, + PRIMARY KEY (uid), + UNIQUE KEY url_hash (url_hash) +); From d3977740ad960b77ed68da535d8fb8aa364e1ef8 Mon Sep 17 00:00:00 2001 From: Eric Harrer Date: Sun, 8 Nov 2026 01:55:38 +0100 Subject: [PATCH 3/5] [FEATURE] Add scheduled feed refresh (#18) Use the shared persistent source for scheduled rendering. Bootstrap a missing response once in the frontend and refresh configured or stored sources through the schedulable command afterwards. Limit these fetches in size and time and read only the configured URL, so a slow or large source cannot hold up the job. Retry failed page invalidation and discard newly cached pages whose rendered source revisions have already changed, so a refresh during rendering cannot leave outdated pages behind. Resolves: #5 Co-authored-by: grs --- Classes/Command/RefreshFeedsCommand.php | 69 ++++++ Classes/Controller/FeedController.php | 23 +- .../EventListener/VerifyPageFeedRevisions.php | 68 +++++ Classes/Service/ConfiguredFeedUrls.php | 76 ++++++ Classes/Service/FeedDataService.php | 12 +- Classes/Service/FeedProvider.php | 38 +-- Classes/Service/ProvidedFeedData.php | 25 ++ Classes/Storage/FeedStorage.php | 11 + Configuration/FlexForms/FeedDisplay.xml | 22 ++ .../Sets/FeedDisplay/constants.typoscript | 4 +- Configuration/Sets/FeedDisplay/labels.xlf | 6 + .../FeedDisplay/settings.definitions.yaml | 9 +- .../Sets/FeedDisplay/setup.typoscript | 1 + Resources/Private/Language/locallang_be.xlf | 19 +- Tests/Functional/Fixtures/FeedHttp.php | 5 + .../Frontend/AbstractHttpFeedTestCase.php | 1 + .../Functional/Frontend/ScheduledFeedTest.php | 234 ++++++++++++++++++ .../Frontend/ScheduledHttpFeedTest.php | 121 +++++++++ .../Frontend/ScheduledPageCacheTest.php | 191 ++++++++++++++ Tests/Unit/Storage/FeedStorageTest.php | 23 ++ 20 files changed, 931 insertions(+), 27 deletions(-) create mode 100644 Classes/Command/RefreshFeedsCommand.php create mode 100644 Classes/EventListener/VerifyPageFeedRevisions.php create mode 100644 Classes/Service/ConfiguredFeedUrls.php create mode 100644 Classes/Service/ProvidedFeedData.php create mode 100644 Tests/Functional/Frontend/ScheduledFeedTest.php create mode 100644 Tests/Functional/Frontend/ScheduledHttpFeedTest.php create mode 100644 Tests/Functional/Frontend/ScheduledPageCacheTest.php diff --git a/Classes/Command/RefreshFeedsCommand.php b/Classes/Command/RefreshFeedsCommand.php new file mode 100644 index 00000000..ef37a6f0 --- /dev/null +++ b/Classes/Command/RefreshFeedsCommand.php @@ -0,0 +1,69 @@ +addOption('url', null, InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Additional feed source (repeatable; commas remain part of the URL)'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + /** @var list $additional */ + $additional = $input->getOption('url'); + try { + $urls = $this->urls->collect($additional); + } catch (\Throwable $exception) { + $io->error(OutputFormatter::escape('Could not collect feed sources: ' . $exception->getMessage())); + return self::FAILURE; + } + if ($urls === []) { + $io->success('No feed sources found. Explicit FlexForm URLs, stored sources and --url are considered.'); + return self::SUCCESS; + } + $status = self::SUCCESS; + foreach ($urls as $url) { + $result = $this->refresher->refresh($url, true); + $io->writeln(OutputFormatter::escape($result->status . ': ' . $url)); + if ($result->error !== null) { + $io->error(OutputFormatter::escape($result->error->getMessage())); + $status = self::FAILURE; + } + } + return $status; + } +} diff --git a/Classes/Controller/FeedController.php b/Classes/Controller/FeedController.php index 2c3fbe45..253da1a2 100644 --- a/Classes/Controller/FeedController.php +++ b/Classes/Controller/FeedController.php @@ -18,7 +18,10 @@ namespace ErHaWeb\FeedDisplay\Controller; use ErHaWeb\FeedDisplay\Service\FeedProvider; +use ErHaWeb\FeedDisplay\Storage\FeedStorage; use Psr\Http\Message\ResponseInterface; +use TYPO3\CMS\Core\Cache\CacheDataCollector; +use TYPO3\CMS\Core\Cache\CacheTag; use TYPO3\CMS\Extbase\Mvc\Controller\ActionController; class FeedController extends ActionController @@ -27,11 +30,29 @@ public function __construct(private readonly FeedProvider $feedProvider) {} public function displayAction(): ResponseInterface { + $scheduler = trim((string)($this->settings['fetchMode'] ?? 'live')) === 'scheduler'; + if (array_key_exists('fetchMode', $this->settings)) { + unset($this->settings['fetchMode']); + $this->view->assign('settings', $this->settings); + } if ($this->settings === []) { return $this->htmlResponse(); } - $this->view->assign('data', $this->feedProvider->getData($this->settings)); + $url = trim(stripslashes((string)($this->settings['feedUrl'] ?? ''))); + $collector = $this->request->getAttribute('frontend.cache.collector'); + if ($scheduler && $url !== '' && $collector instanceof CacheDataCollector) { + $collector->addCacheTags(new CacheTag(FeedStorage::cacheTag($url))); + } + $provided = $this->feedProvider->provide($this->settings, $scheduler); + $data = $provided->data; + if ($scheduler && $url !== '' && $provided->revision !== null && $collector instanceof CacheDataCollector) { + $collector->addCacheTags(new CacheTag(FeedStorage::revisionTag($url, $provided->revision))); + } + if ($scheduler && !isset($data['feed']) && !isset($data['items']) && $collector instanceof CacheDataCollector) { + $collector->restrictMaximumLifetime(60); + } + $this->view->assign('data', $data); return $this->htmlResponse(); } } diff --git a/Classes/EventListener/VerifyPageFeedRevisions.php b/Classes/EventListener/VerifyPageFeedRevisions.php new file mode 100644 index 00000000..3daf78e5 --- /dev/null +++ b/Classes/EventListener/VerifyPageFeedRevisions.php @@ -0,0 +1,68 @@ +getRequest()->getAttribute('frontend.cache.collector'); + if (!$collector instanceof CacheDataCollector) { + return; + } + $revisions = []; + foreach ($collector->getCacheTags() as $tag) { + if (preg_match('/^feeddisplay_rev_([0-9a-f]{64})_([0-9a-f]{64})$/D', $tag->name, $match) === 1) { + $revisions[$match[1]][$match[2]] = true; + } + } + foreach ($revisions as $urlHash => $usedRevisions) { + $context = ['urlHash' => $urlHash, 'cacheIdentifier' => $event->getCacheIdentifier()]; + try { + $current = $this->storage->findRevisionByHash($urlHash); + } catch (\Throwable $exception) { + $this->logger->warning('Could not verify the cached feed revision.', $context + ['exception' => $exception]); + continue; + } + if ($current !== null && count($usedRevisions) === 1 && isset($usedRevisions[$current])) { + continue; + } + try { + $this->cacheManager->getCache('pages')->remove($event->getCacheIdentifier()); + } catch (\Throwable $exception) { + $this->logger->warning('Could not remove an obsolete feed page.', $context + ['exception' => $exception]); + } + return; + } + } +} diff --git a/Classes/Service/ConfiguredFeedUrls.php b/Classes/Service/ConfiguredFeedUrls.php new file mode 100644 index 00000000..effdac8a --- /dev/null +++ b/Classes/Service/ConfiguredFeedUrls.php @@ -0,0 +1,76 @@ + $additional + * @return list + */ + public function collect(array $additional): array + { + $urls = $this->storage->findUrls(); + $query = $this->connections->getQueryBuilderForTable('tt_content'); + $query->getRestrictions()->removeAll(); + $records = $query->select('*')->from('tt_content')->where( + $query->expr()->eq('deleted', 0), + $query->expr()->eq('t3ver_wsid', 0), + $query->expr()->or( + $query->expr()->eq('CType', $query->createNamedParameter('feeddisplay_pi1')), + $query->expr()->eq('CType', $query->createNamedParameter('list')), + ), + )->executeQuery()->fetchAllAssociative(); + foreach ($records as $record) { + if ($record['CType'] !== 'feeddisplay_pi1' && !($record['CType'] === 'list' && ($record['list_type'] ?? '') === 'feeddisplay_pi1')) { + continue; + } + if (trim((string)($record['pi_flexform'] ?? '')) === '') { + continue; + } + $form = GeneralUtility::xml2array((string)$record['pi_flexform']); + if (!is_array($form)) { + $this->logger->warning('Could not read feed FlexForm.', ['contentElementUid' => (int)$record['uid'], 'error' => $form]); + continue; + } + $sheets = $form['data'] ?? []; + if (!is_array($sheets)) { + $this->logger->warning('Invalid feed FlexForm structure.', ['contentElementUid' => (int)$record['uid']]); + continue; + } + foreach ($sheets as $sheet) { + if (!is_array($sheet)) { + continue; + } + $url = $sheet['lDEF']['settings.feedUrl']['vDEF'] ?? null; + if (is_string($url)) { + $urls[] = $url; + } + } + } + $urls = array_map(static fn(string $url): string => trim(stripslashes($url)), [...$urls, ...$additional]); + return array_values(array_unique(array_filter($urls, static fn(string $url): bool => $url !== ''))); + } +} diff --git a/Classes/Service/FeedDataService.php b/Classes/Service/FeedDataService.php index b570e6dc..22133bec 100644 --- a/Classes/Service/FeedDataService.php +++ b/Classes/Service/FeedDataService.php @@ -232,16 +232,16 @@ private function getImage(mixed $fileUrl): ?string } $temporaryFileName = md5($filename) . '.' . $extension; + $tempFilePath = 'typo3temp/assets/images/' . $temporaryFileName; + $absoluteTempFilePath = Environment::getPublicPath() . '/' . $tempFilePath; + if (@is_file($absoluteTempFilePath)) { + return $tempFilePath; + } $content = GeneralUtility::getUrl($fileUrl); if ($content === false) { return null; } - - $tempFilePath = 'typo3temp/assets/images/' . $temporaryFileName; - $absoluteTempFilePath = Environment::getPublicPath() . '/' . $tempFilePath; - if (!@is_file($absoluteTempFilePath)) { - GeneralUtility::writeFileToTypo3tempDir($absoluteTempFilePath, $content); - } + GeneralUtility::writeFileToTypo3tempDir($absoluteTempFilePath, $content); return $tempFilePath; } diff --git a/Classes/Service/FeedProvider.php b/Classes/Service/FeedProvider.php index 59dcdbb8..e6cd5122 100644 --- a/Classes/Service/FeedProvider.php +++ b/Classes/Service/FeedProvider.php @@ -46,30 +46,36 @@ public function __construct( * @return array */ public function getData(array $settings, bool $scheduler = false): array + { + return $this->provide($settings, $scheduler)->data; + } + + /** @param array $settings */ + public function provide(array $settings, bool $scheduler = false): ProvidedFeedData { $url = trim(stripslashes((string)($settings['feedUrl'] ?? ''))); $duration = (int)($settings['cacheDuration'] ?? 0); $identifier = 'feeddisplay_' . md5(serialize($settings)); $cached = $this->readCache($identifier, $settings); if ($url === '') { - return ['settings' => $settings]; + return new ProvidedFeedData(['settings' => $settings]); } if (!$scheduler && $duration > 0 && $cached !== false && is_string($cached[self::REVISION] ?? null) && str_starts_with($cached[self::REVISION], 'unstored:') ) { // Persistence failed: retain the successful display result for its original TTL. - return $this->displayData($cached); + return new ProvidedFeedData($this->displayData($cached)); } try { $stored = $this->storage->findState($url); } catch (\Throwable $exception) { $this->logger->warning('Could not read the persistent feed. Apply the extension database schema.', ['url' => $url, 'exception' => $exception]); if (!$scheduler && $duration > 0 && $cached !== false) { - return $this->displayData($cached); + return new ProvidedFeedData($this->displayData($cached)); } $fetched = $scheduler ? null : $this->fetchWithoutStorage($url); if ($fetched === null) { - return $this->cacheData($identifier, ['settings' => $settings], '', $duration); + return new ProvidedFeedData($this->cacheData($identifier, ['settings' => $settings], '', $duration)); } return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler); } @@ -77,7 +83,7 @@ public function getData(array $settings, bool $scheduler = false): array if (!$scheduler && $duration > 0 && $cached !== false && !array_key_exists(self::REVISION, $cached) && !($stored->hasDocument ?? false) ) { - return $cached; + return new ProvidedFeedData($cached); } if ($stored !== null && $stored->invalidationPending) { try { @@ -118,20 +124,20 @@ public function getData(array $settings, bool $scheduler = false): array if (!$scheduler && $result->status === 'locked' && !($stored->hasDocument ?? false)) { $fresh = $this->fetchWithoutStorage($url); if ($fresh !== null) { - return $this->dataService->buildInitializedData($fresh->feed, $settings); + return new ProvidedFeedData($this->dataService->buildInitializedData($fresh->feed, $settings)); } } } if ($duration > 0 && $cached !== false && $stored !== null && $stored->hasDocument && ($cached[self::REVISION] ?? null) === $stored->revision ) { - return $this->displayData($cached); + return new ProvidedFeedData($this->displayData($cached), $stored->revision); } $feed = $this->restoreFeed($url, $stored, $fetched); if ($feed !== null && $stored !== null) { // Public events and getters remain outside transport/storage catches. $data = $this->dataService->buildInitializedData($feed, $settings); - return $this->cacheData($identifier, $data, $stored->revision, $duration); + return new ProvidedFeedData($this->cacheData($identifier, $data, $stored->revision, $duration), $stored->revision); } $lastAttempt = $stored->lastAttemptAt ?? 0; if ($attempted || !($stored->hasDocument ?? false) || ($lastAttempt !== 0 && $lastAttempt + 60 > $now)) { @@ -141,9 +147,9 @@ public function getData(array $settings, bool $scheduler = false): array $needsFetch = true; } if (!$scheduler && $duration > 0 && $cached !== false && (isset($cached['feed']) || isset($cached['items']))) { - return $this->displayData($cached); + return new ProvidedFeedData($this->displayData($cached)); } - return $this->cacheData($identifier, ['settings' => $settings], '', $duration); + return new ProvidedFeedData($this->cacheData($identifier, ['settings' => $settings], '', $duration)); } private function restoreFeed(string $url, ?StoredFeed &$stored, ?FetchedFeed $fetched): ?SimplePie @@ -170,16 +176,16 @@ private function restoreFeed(string $url, ?StoredFeed &$stored, ?FetchedFeed $fe return null; } - /** - * @param array $settings - * @return array - */ - private function fetchedData(string $identifier, FetchedFeed $fetched, array $settings, int $duration, bool $scheduler, bool $persisted = false): array + /** @param array $settings */ + private function fetchedData(string $identifier, FetchedFeed $fetched, array $settings, int $duration, bool $scheduler, bool $persisted = false): ProvidedFeedData { $data = $this->dataService->buildInitializedData($fetched->feed, $settings); $revision = $fetched->document->revision(); $cacheRevision = $persisted ? $revision : 'unstored:' . $revision; - return $scheduler ? $data : $this->cacheData($identifier, $data, $cacheRevision, $duration); + if (!$scheduler) { + $data = $this->cacheData($identifier, $data, $cacheRevision, $duration); + } + return new ProvidedFeedData($data, $persisted ? $revision : null); } private function fetchWithoutStorage(string $url): ?FetchedFeed diff --git a/Classes/Service/ProvidedFeedData.php b/Classes/Service/ProvidedFeedData.php new file mode 100644 index 00000000..14dd4401 --- /dev/null +++ b/Classes/Service/ProvidedFeedData.php @@ -0,0 +1,25 @@ + $data */ + public function __construct(public array $data, public ?string $revision = null) {} +} diff --git a/Classes/Storage/FeedStorage.php b/Classes/Storage/FeedStorage.php index 309cf1e7..f8e85f8b 100644 --- a/Classes/Storage/FeedStorage.php +++ b/Classes/Storage/FeedStorage.php @@ -39,6 +39,17 @@ public static function cacheTag(string $url): string return 'feeddisplay_url_' . self::hashUrl($url); } + public static function revisionTag(string $url, string $revision): string + { + return 'feeddisplay_rev_' . self::hashUrl($url) . '_' . $revision; + } + + public function findRevisionByHash(string $urlHash): ?string + { + $revision = $this->connection()->select(['revision'], self::TABLE, ['url_hash' => $urlHash])->fetchOne(); + return $revision === false ? null : (string)$revision; + } + public function find(string $url): ?StoredFeed { $row = $this->connection()->select(['*'], self::TABLE, ['url_hash' => self::hashUrl($url)])->fetchAssociative(); diff --git a/Configuration/FlexForms/FeedDisplay.xml b/Configuration/FlexForms/FeedDisplay.xml index 0c40cbeb..7d8d2041 100644 --- a/Configuration/FlexForms/FeedDisplay.xml +++ b/Configuration/FlexForms/FeedDisplay.xml @@ -226,6 +226,28 @@ number + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.fetchMode.description + + select + selectSingle + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.placeholder.typoscript + + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.fetchMode.live + live + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.fetchMode.scheduler + scheduler + + + + diff --git a/Configuration/Sets/FeedDisplay/constants.typoscript b/Configuration/Sets/FeedDisplay/constants.typoscript index 5c2831e7..86e89726 100644 --- a/Configuration/Sets/FeedDisplay/constants.typoscript +++ b/Configuration/Sets/FeedDisplay/constants.typoscript @@ -40,6 +40,8 @@ plugin.tx_feeddisplay_pi1 { errorMessage = # cat=plugin.tx_feeddisplay_pi1/200_advanced/340; type=int+; label=LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:constants.settings.cacheDuration cacheDuration = 3600 + # cat=plugin.tx_feeddisplay_pi1/200_advanced/345; type=options[live,scheduler]; label=LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:constants.settings.fetchMode + fetchMode = live getFields { # Each field is internally converted to a SimplePie getter method. @@ -95,5 +97,5 @@ plugin.tx_feeddisplay_pi1 { } # cat=plugin.tx_feeddisplay_pi1/200_advanced/350; type=string; label=LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:constants.settings.ignoreFlexFormSettingsIfEmpty - ignoreFlexFormSettingsIfEmpty = feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,getFields.feed,getFields.items + ignoreFlexFormSettingsIfEmpty = feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,fetchMode,getFields.feed,getFields.items } diff --git a/Configuration/Sets/FeedDisplay/labels.xlf b/Configuration/Sets/FeedDisplay/labels.xlf index 5f21770f..dd44d0cb 100644 --- a/Configuration/Sets/FeedDisplay/labels.xlf +++ b/Configuration/Sets/FeedDisplay/labels.xlf @@ -123,6 +123,12 @@ Time in seconds in which the data is to be read from the cache until the next renewal of the cache. (0 = no cache) + + Fetch mode + + + Fetch the feed during page rendering or through the refresh command. + Ignore FlexForm Settings if empty diff --git a/Configuration/Sets/FeedDisplay/settings.definitions.yaml b/Configuration/Sets/FeedDisplay/settings.definitions.yaml index c6be1954..3eeaf647 100644 --- a/Configuration/Sets/FeedDisplay/settings.definitions.yaml +++ b/Configuration/Sets/FeedDisplay/settings.definitions.yaml @@ -24,7 +24,7 @@ settings: category: feeddisplay.templates plugin.tx_feeddisplay_pi1.settings.ignoreFlexFormSettingsIfEmpty: type: string - default: 'feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,getFields.feed,getFields.items' + default: 'feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,fetchMode,getFields.feed,getFields.items' category: feeddisplay.general plugin.tx_feeddisplay_pi1.settings.feedUrl: type: string @@ -78,6 +78,13 @@ settings: type: int default: 3600 category: feeddisplay.advanced + plugin.tx_feeddisplay_pi1.settings.fetchMode: + type: string + default: 'live' + enum: + live: 'Live' + scheduler: 'Scheduler' + category: feeddisplay.advanced plugin.tx_feeddisplay_pi1.settings.getFields.feed: type: string default: 'author,authors,contributor,contributors,copyright,description,encoding,items,item_quantity,language,link,links,permalink,title,type,subscribe_url,latitude,longitude,image_height,image_link,image_title,image_url,image_width,all_discovered_feeds,base' diff --git a/Configuration/Sets/FeedDisplay/setup.typoscript b/Configuration/Sets/FeedDisplay/setup.typoscript index 42fd6171..0cf02825 100644 --- a/Configuration/Sets/FeedDisplay/setup.typoscript +++ b/Configuration/Sets/FeedDisplay/setup.typoscript @@ -35,6 +35,7 @@ plugin.tx_feeddisplay_pi1 { linkTarget = {$plugin.tx_feeddisplay_pi1.settings.linkTarget} errorMessage = {$plugin.tx_feeddisplay_pi1.settings.errorMessage} cacheDuration = {$plugin.tx_feeddisplay_pi1.settings.cacheDuration} + fetchMode = {$plugin.tx_feeddisplay_pi1.settings.fetchMode} } ignoreFlexFormSettingsIfEmpty = {$plugin.tx_feeddisplay_pi1.ignoreFlexFormSettingsIfEmpty} diff --git a/Resources/Private/Language/locallang_be.xlf b/Resources/Private/Language/locallang_be.xlf index 9ad9ec9f..76f920f8 100644 --- a/Resources/Private/Language/locallang_be.xlf +++ b/Resources/Private/Language/locallang_be.xlf @@ -118,7 +118,19 @@ Feed cache - Time in seconds in which the data is to be read from the cache until the next renewal of the cache. (0 = no cache, empty = Defined by TypoScript) + Minimum interval between live refresh attempts and lifetime of prepared display data, in seconds. Zero or negative: refresh on each plugin execution. Successful responses remain stored permanently. In scheduler mode this controls display caching only. Empty: inherit TypoScript. + + + Fetch mode + + + Live refreshes the shared feed source as needed during page rendering and retains the last successful response permanently. Scheduler loads a missing feed once during page rendering; afterwards only feeddisplay:refresh updates it. Configure a regular refresh job. Failed first attempts are retried after 60 seconds. Empty: inherit TypoScript or Site Set settings. + + + Live + + + Scheduler Defined by Feed or TypoScript (default) @@ -367,7 +379,10 @@ Strip tags:Remove HTML tags from the feed content - Feed cache:Time in seconds in which the data is to be read from the cache until the next renewal of the cache. (0 = no cache) + Feed cache:Live refresh interval and display cache lifetime in seconds. Zero or negative bypasses these intervals, but successful responses remain stored permanently. + + + Fetch mode:Fetch the feed during page rendering or through the refresh command. Ignore FlexForm Settings if empty:Comma separated list of FlexForm settings that will be ignored in the extension settings merge process, if their value is considered empty (either an empty string or a string containing 0). diff --git a/Tests/Functional/Fixtures/FeedHttp.php b/Tests/Functional/Fixtures/FeedHttp.php index 0b12fc12..6cf24e7b 100644 --- a/Tests/Functional/Fixtures/FeedHttp.php +++ b/Tests/Functional/Fixtures/FeedHttp.php @@ -28,12 +28,17 @@ final class FeedHttp public static array $requests = []; /** @var array */ public static array $responses = []; + /** @var array */ + public static array $beforeResponse = []; public static function middleware(callable $next): callable { return static function (RequestInterface $request, array $options) { $uri = (string)$request->getUri(); self::$requests[] = $uri; + $callback = self::$beforeResponse[$uri] ?? null; + unset(self::$beforeResponse[$uri]); + $callback?->__invoke(); $response = self::$responses[$uri] ?? new Response(503, [], 'Unavailable'); // The recorder closes consumed streams. Every request needs a fresh body. return Create::promiseFor(new Response($response->getStatusCode(), $response->getHeaders(), (string)$response->getBody())); diff --git a/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php b/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php index 4013e65d..69d38afc 100644 --- a/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php +++ b/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php @@ -30,6 +30,7 @@ protected function setUp(): void parent::setUp(); FeedHttp::$requests = []; FeedHttp::$responses = []; + FeedHttp::$beforeResponse = []; // Frontend Bootstrap::init() reloads HTTP configuration from the test instance. file_put_contents($this->instancePath . '/typo3conf/system/additional.php', 'configurationToUseInTestInstance['SYS']['caching']['cacheConfigurations']['pages']['backend'] = Typo3DatabaseBackend::class; + parent::setUp(); + } + + #[Test] + public function firstRequestBootstrapsAndLaterRequestsOnlyUseTheStoredSource(): void + { + $url = $this->writeFeedFixture('First response', ['First item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + self::assertStringContainsString('First response', $this->requestPage()); + $storage = $this->get(FeedStorage::class); + $first = $storage->find($url); + self::assertNotNull($first?->document); + $this->writeFeedFixture('New response', ['Second item']); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('First response', $this->requestPage()); + self::assertSame($first->revision, $storage->find($url)?->revision); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertStringContainsString('updated:', $command->getDisplay()); + self::assertStringContainsString('New response', $this->requestPage()); + unlink($url); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('New response', $this->requestPage()); + } + + #[Test] + public function failedBootstrapIsThrottledAndTheMissingPageExpiresWithinSixtySeconds(): void + { + $url = $this->instancePath . '/fileadmin/missing.xml'; + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler']); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'config.no_cache = 0'); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + $pages = $this->taggedPages($url); + self::assertCount(1, $pages); + $expires = $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['expires'], 'cache_pages', ['identifier' => $pages[0]])->fetchOne(); + self::assertLessThanOrEqual(time() + 60, (int)$expires); + $this->writeFeedXmlFixture('RecoveredTest', 'missing.xml'); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertSame([], $this->taggedPages($url)); + self::assertStringContainsString('Recovered', $this->requestPage()); + } + + #[Test] + public function commandDeduplicatesAllSourcesAndContinuesAfterOneFails(): void + { + $url = $this->writeFeedFixture('Selected', ['Item']); + $this->setFlexFormValues(['general' => ['settings.feedUrl' => $url]]); + $storage = $this->get(FeedStorage::class); + $storage->storeFailure($url, 'Previous failure', 1); + $missing = $this->instancePath . '/fileadmin/absent.xml'; + self::assertSame([$url, $missing], $this->get(ConfiguredFeedUrls::class)->collect([$missing, $url])); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::FAILURE, $command->execute(['--url' => [$missing, $url]])); + self::assertSame(1, substr_count($command->getDisplay(), 'updated:')); + self::assertStringContainsString('failed:', $command->getDisplay()); + self::assertNotNull($storage->find($url)?->document); + } + + #[Test] + public function changedSourceInvalidatesPagesWithoutShorteningTheirNormalLifetime(): void + { + $url = $this->writeFeedFixture('Original', ['Item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'cacheDuration' => -1]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nconfig.cache_period = 7200"); + self::assertStringContainsString('Original', $this->requestPage()); + $pages = $this->taggedPages($url); + self::assertCount(1, $pages); + $expires = $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['expires'], 'cache_pages', ['identifier' => $pages[0]])->fetchOne(); + self::assertGreaterThan(time() + 3600, (int)$expires, json_encode([$pages, $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative()], JSON_THROW_ON_ERROR)); + $this->writeFeedFixture('Changed', ['Item']); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertSame([], $this->taggedPages($url)); + self::assertStringContainsString('Changed', $this->requestPage()); + } + + #[Test] + public function emptySourceListIsSuccessful(): void + { + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertStringContainsString('No feed sources found', $command->getDisplay()); + } + + #[Test] + public function twoPluginsShareOneSourceButKeepSeparateDisplayArrays(): void + { + $url = $this->writeFeedFixture('Shared source', ['One', 'Two']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'cacheDuration' => 600, 'maxFeedCount' => 2]); + $this->setFlexFormValues(['general' => ['settings.maxFeedCount' => 1]]); + $connection = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $record = $connection->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($record); + $record['uid'] = self::CONTENT_ELEMENT_UID + 1; + $record['sorting'] = 256; + $record['pi_flexform'] = ''; + $connection->insert('tt_content', $record); + $body = $this->requestPage(); + self::assertSame(2, substr_count($body, 'Shared source')); + self::assertSame(3, substr_count($body, 'class="item-title"')); + self::assertSame(1, $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->count('*', FeedStorage::TABLE, [])); + self::assertSame(2, $this->getConnectionPool()->getConnectionForTable('cache_feeddisplay')->count('*', 'cache_feeddisplay', [])); + $this->writeFeedFixture('Shared update', ['Three', 'Four']); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + $body = $this->requestPage(); + self::assertSame(2, substr_count($body, 'Shared update')); + self::assertSame(3, substr_count($body, 'class="item-title"')); + self::assertStringNotContainsString('Shared source', $body); + } + + #[Test] + public function schedulerWithoutTheSchemaShowsTheConfiguredError(): void + { + $url = $this->writeFeedFixture('Not fetched', ['Item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'errorMessage' => 'Schema not ready']); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + try { + self::assertStringContainsString('Schema not ready', $this->requestPage()); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function busyFirstBootstrapCanBeRetriedAfterTheOtherWorkerReleasesItsLock(): void + { + $url = $this->writeFeedFixture('Available after lock', ['Item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler']); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + self::assertTrue($lock->acquire($mode)); + try { + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->find($url)); + } finally { + $lock->release(); + } + self::assertStringContainsString('Available after lock', $this->requestPage()); + } + + #[Test] + public function sourceSelectionIncludesHiddenContentButSkipsDeletedWorkspaceAndMalformedRecords(): void + { + $url = $this->instancePath . '/fileadmin/selected.xml'; + $this->setFlexFormValues(['advanced' => ['settings.feedUrl' => $url]]); + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $record = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($record); + $content->update('tt_content', ['hidden' => 1], ['uid' => self::CONTENT_ELEMENT_UID]); + foreach (['deleted' => 1, 't3ver_wsid' => 1] as $field => $value) { + ++$record['uid']; + $copy = $record; + $copy[$field] = $value; + $copy['pi_flexform'] = str_replace('selected.xml', $field . '.xml', (string)$copy['pi_flexform']); + $content->insert('tt_content', $copy); + } + ++$record['uid']; + $record['pi_flexform'] = 'insert('tt_content', $record); + $additional = 'https://example.test/feed?ids=1,2'; + self::assertSame([$url, $additional], $this->get(ConfiguredFeedUrls::class)->collect([' ' . $additional . ' ', $url, $additional])); + } + + #[Test] + #[DataProvider('emptyFlexForms')] + public function emptyFlexFormsAreIgnoredWhileMalformedXmlIsDiagnosed(?string $flexForm): void + { + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $logger = $this->createMock(LoggerInterface::class); + $logger->expects(self::once())->method('warning')->with('Could not read feed FlexForm.', self::anything()); + $collector = new ConfiguredFeedUrls($this->getConnectionPool(), $this->get(FeedStorage::class), $logger); + $content->update('tt_content', ['pi_flexform' => $flexForm], ['uid' => self::CONTENT_ELEMENT_UID]); + self::assertSame([], $collector->collect([])); + $content->update('tt_content', ['pi_flexform' => ' self::CONTENT_ELEMENT_UID]); + self::assertSame([], $collector->collect([])); + } + + /** @return iterable */ + public static function emptyFlexForms(): iterable + { + yield 'null' => [null]; + yield 'empty' => ['']; + yield 'whitespace' => [" \t\n"]; + } + + /** @return list */ + private function taggedPages(string $url): array + { + $backend = $this->get(CacheManager::class)->getCache('pages')->getBackend(); + self::assertInstanceOf(TaggableBackendInterface::class, $backend); + return array_values($backend->findIdentifiersByTag(FeedStorage::cacheTag($url))); + } +} diff --git a/Tests/Functional/Frontend/ScheduledHttpFeedTest.php b/Tests/Functional/Frontend/ScheduledHttpFeedTest.php new file mode 100644 index 00000000..ac3b7262 --- /dev/null +++ b/Tests/Functional/Frontend/ScheduledHttpFeedTest.php @@ -0,0 +1,121 @@ +initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler']); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'config.no_cache = 0'); + self::assertStringContainsString('Original', $this->requestPage()); + $backend = $this->get(CacheManager::class)->getCache('pages')->getBackend(); + self::assertInstanceOf(TaggableBackendInterface::class, $backend); + self::assertCount(1, $backend->findIdentifiersByTag(FeedStorage::cacheTag(self::URL))); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Live change'); + $this->get(FeedProvider::class)->getData(['feedUrl' => self::URL, 'cacheDuration' => 0, 'getFields' => ['feed' => 'title']]); + self::assertSame([], $backend->findIdentifiersByTag(FeedStorage::cacheTag(self::URL))); + self::assertStringContainsString('Live change', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } + + #[Test] + public function unreplayableDocumentIsRepairedOnce(): void + { + $this->storeHttpDocument(self::URL, 'No XML feed'); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Repaired'); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 600]); + self::assertStringContainsString('Repaired', $this->requestPage()); + self::assertStringContainsString('Repaired', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertGreaterThan(1, $this->get(FeedStorage::class)->findState(self::URL)?->lastAttemptAt); + } + + #[Test] + public function failedRepairIsThrottledAndDoesNotReplaceTheStoredResponse(): void + { + $this->storeHttpDocument(self::URL, 'No XML feed'); + $storage = $this->get(FeedStorage::class); + $before = $storage->findState(self::URL); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Available later'); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertSame($before?->revision, $storage->findState(self::URL)?->revision); + self::assertSame($before?->fetchedAt, $storage->findState(self::URL)?->fetchedAt); + $now = (int)$this->get(Context::class)->getPropertyFromAspect('date', 'timestamp'); + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['last_attempt_at' => $now - 61], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + self::assertStringContainsString('Available later', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } + + #[Test] + public function repeatedSchedulerRenderingsReuseDownloadedImages(): void + { + $image = 'https://image.example.test/logo.png'; + FeedHttp::$responses[self::URL] = FeedHttp::rss('Image', '' . $image . 'Imagehttps://example.test/'); + FeedHttp::$responses[$image] = new Response(200, ['Content-Type' => 'image/png'], 'PNGDATA'); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'plugin.tx_feeddisplay_pi1.settings.getFields.feed = title,image_url'); + for ($request = 0; $request < 3; ++$request) { + self::assertStringContainsString('Image', $this->requestPage()); + } + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertSame(1, FeedHttp::countRequests($image)); + } + + #[Test] + public function repairSkipsABusySourceAndRetriesAfterRelease(): void + { + $this->storeHttpDocument(self::URL, 'No XML feed'); + FeedHttp::$responses[self::URL] = FeedHttp::rss('After lock'); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl(self::URL), $mode); + self::assertTrue($lock->acquire($mode)); + try { + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame([], FeedHttp::$requests); + } finally { + $lock->release(); + } + self::assertStringContainsString('After lock', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + } + + private function storeHttpDocument(string $uri, string $body): void + { + $document = StoredDocument::create($body, $uri); + $storage = $this->get(FeedStorage::class); + $storage->storeSuccess(self::URL, $document, 1); + $storage->markInvalidated(self::URL, $document->revision()); + } +} diff --git a/Tests/Functional/Frontend/ScheduledPageCacheTest.php b/Tests/Functional/Frontend/ScheduledPageCacheTest.php new file mode 100644 index 00000000..26e93f0f --- /dev/null +++ b/Tests/Functional/Frontend/ScheduledPageCacheTest.php @@ -0,0 +1,191 @@ +preparePage(); + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $scheduler = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($scheduler); + $this->setFlexFormValues(['advanced' => ['settings.fetchMode' => 'live']]); + $live = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($live); + $content->update('tt_content', ['pi_flexform' => $scheduler['pi_flexform']], ['uid' => self::CONTENT_ELEMENT_UID]); + $live['uid'] = self::CONTENT_ELEMENT_UID + 1; + $live['sorting'] = $schedulerFirst ? 256 : -256; + $content->insert('tt_content', $live); + if ($anotherScheduler) { + $scheduler['uid'] = self::CONTENT_ELEMENT_UID + 2; + $scheduler['sorting'] = 512; + $content->insert('tt_content', $scheduler); + } + FeedHttp::$responses[self::URL] = FeedHttp::rss('NEW_SOURCE'); + $first = $this->requestPage(); + self::assertStringContainsString('NEW_SOURCE', $first); + if ($schedulerFirst) { + self::assertStringContainsString('OLD_SOURCE', $first); + self::assertSame([], $this->pageEntries()); + } else { + self::assertStringNotContainsString('OLD_SOURCE', $first); + self::assertCount(1, $this->pageEntries()); + } + self::assertSame('unchanged', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + self::assertStringNotContainsString('OLD_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + } + + /** @return iterable */ + public static function pluginOrders(): iterable + { + yield 'scheduler before live' => [true, false]; + yield 'live before scheduler' => [false, false]; + yield 'both revisions of one source' => [true, true]; + } + + #[Test] + public function refreshDuringRenderingDoesNotLeaveAnObsoletePageCached(): void + { + $image = $this->preparePage(true); + FeedHttp::$beforeResponse[$image] = function (): void { + FeedHttp::$responses[self::URL] = FeedHttp::rss('NEW_SOURCE'); + self::assertSame('updated', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + }; + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertSame([], $this->pageEntries()); + $stored = $this->get(FeedStorage::class)->findState(self::URL); + self::assertNotNull($stored); + self::assertFalse($stored->invalidationPending); + self::assertSame('unchanged', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + self::assertStringContainsString('NEW_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + } + + #[Test] + public function unchangedSourcesKeepTheirPageAndAllRevisionTags(): void + { + $this->preparePage(); + $secondUrl = self::URL . '/second'; + FeedHttp::$responses[$secondUrl] = FeedHttp::rss('SECOND_SOURCE'); + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $record = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($record); + $this->setFlexFormValues(['general' => ['settings.feedUrl' => $secondUrl]]); + $second = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($second); + $content->update('tt_content', ['pi_flexform' => $record['pi_flexform']], ['uid' => self::CONTENT_ELEMENT_UID]); + $second['uid'] = self::CONTENT_ELEMENT_UID + 1; + $second['sorting'] = 256; + $content->insert('tt_content', $second); + self::assertStringContainsString('SECOND_SOURCE', $this->requestPage()); + $entries = $this->pageEntries(); + self::assertCount(1, $entries); + self::assertGreaterThan(time() + 3600, (int)$entries[0]['expires']); + $tags = $this->getConnectionPool()->getConnectionForTable('cache_pages_tags')->select(['tag'], 'cache_pages_tags', ['identifier' => $entries[0]['identifier']])->fetchFirstColumn(); + foreach ([self::URL, $secondUrl] as $url) { + $state = $this->get(FeedStorage::class)->findState($url); + self::assertNotNull($state); + self::assertContains(FeedStorage::cacheTag($url), $tags); + self::assertContains('feeddisplay_rev_' . FeedStorage::hashUrl($url) . '_' . $state->revision, $tags); + } + $requests = FeedHttp::$requests; + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertSame($requests, FeedHttp::$requests); + self::assertSame($entries, $this->pageEntries()); + } + + #[Test] + public function revisionReadFailureKeepsThePageAndLogsADiagnosis(): void + { + $image = $this->preparePage(true); + $log = $this->instancePath . '/typo3temp/revision-errors.log'; + $this->configureLog($log); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + FeedHttp::$beforeResponse[$image] = static function () use ($manager): void { + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + }; + try { + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + self::assertFileExists($log); + $diagnosis = (string)file_get_contents($log); + self::assertStringContainsString('Could not verify the cached feed revision.', $diagnosis); + self::assertStringContainsString(FeedStorage::hashUrl(self::URL), $diagnosis); + self::assertStringContainsString('cacheIdentifier', $diagnosis); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function livePagesDoNotVerifySourceRevisions(): void + { + $image = $this->preparePage(true); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'plugin.tx_feeddisplay_pi1.settings.fetchMode = live'); + $log = $this->instancePath . '/typo3temp/live-revision-errors.log'; + $this->configureLog($log); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + FeedHttp::$beforeResponse[$image] = static function () use ($manager): void { + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + }; + try { + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + self::assertStringNotContainsString('Could not verify the cached feed revision.', is_file($log) ? (string)file_get_contents($log) : ''); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + private function preparePage(bool $withImage = false): string + { + $image = 'https://image.example.test/' . md5($this->nameWithDataSet()) . '.svg'; + $extra = $withImage ? '' . $image . 'Imagehttps://example.test/' : ''; + FeedHttp::$responses[self::URL] = FeedHttp::rss('OLD_SOURCE', $extra); + FeedHttp::$responses[$image] = new Response(200, ['Content-Type' => 'image/svg+xml'], ''); + self::assertSame('updated', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nconfig.cache_period = 7200\nplugin.tx_feeddisplay_pi1.settings.getFields.feed = title,image_url"); + return $image; + } + + private function configureLog(string $path): void + { + $writers = [LogLevel::WARNING => [FileWriter::class => ['logFile' => $path]]]; + file_put_contents($this->instancePath . '/typo3conf/system/additional.php', "\n\$GLOBALS['TYPO3_CONF_VARS']['LOG']['writerConfiguration'] = " . var_export($writers, true) . ';', FILE_APPEND); + } + + /** @return list> */ + private function pageEntries(): array + { + return $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative(); + } +} diff --git a/Tests/Unit/Storage/FeedStorageTest.php b/Tests/Unit/Storage/FeedStorageTest.php index 2bf63995..44c8b201 100644 --- a/Tests/Unit/Storage/FeedStorageTest.php +++ b/Tests/Unit/Storage/FeedStorageTest.php @@ -20,6 +20,7 @@ use Doctrine\DBAL\Result; use ErHaWeb\FeedDisplay\Storage\FeedStorage; use ErHaWeb\FeedDisplay\Tests\Fixtures\StoredDocument; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; use Psr\Log\NullLogger; use TYPO3\CMS\Core\Database\Connection; @@ -72,4 +73,26 @@ public function lazyDocumentReadsSupportLobStreamsAndMatchTheRequestedRevision() } } + #[Test] + #[DataProvider('storedRevisions')] + public function revisionLookupDoesNotLoadTheDocument(string|false $value, ?string $expected): void + { + $hash = FeedStorage::hashUrl('https://example.test/feed'); + $result = self::createStub(Result::class); + $result->method('fetchOne')->willReturn($value); + $connection = $this->createMock(Connection::class); + $connection->expects(self::once())->method('select')->with(['revision'], FeedStorage::TABLE, ['url_hash' => $hash])->willReturn($result); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + self::assertSame($expected, (new FeedStorage($pool, new NullLogger()))->findRevisionByHash($hash)); + } + + /** @return iterable */ + public static function storedRevisions(): iterable + { + yield 'successful source' => [str_repeat('a', 64), str_repeat('a', 64)]; + yield 'missing source' => [false, null]; + yield 'failed first attempt' => ['', '']; + } + } From e12ad5a3524af8785c588b57bbd9910e3beaee4b Mon Sep 17 00:00:00 2001 From: Eric Harrer Date: Sun, 8 Nov 2026 01:56:42 +0100 Subject: [PATCH 4/5] [FEATURE] Remove explicitly selected feed sources Remove sources by name instead of by age. Rendered pages can stay cached far longer than any usage window, so a source in use would look unused, and stored responses matter most during outages. Run under the refresh lock and invalidate tagged scheduler pages, including pages rendered concurrently with removal. Keep the command out of Scheduler selections on TYPO3 13 and 14. --- Classes/Command/RemoveFeedSourcesCommand.php | 70 ++++++++++++++ Classes/Service/FeedRefresher.php | 10 ++ .../Frontend/RemoveFeedPageCacheTest.php | 53 +++++++++++ .../Frontend/RemoveFeedSourceTest.php | 91 +++++++++++++++++++ 4 files changed, 224 insertions(+) create mode 100644 Classes/Command/RemoveFeedSourcesCommand.php create mode 100644 Tests/Functional/Frontend/RemoveFeedPageCacheTest.php create mode 100644 Tests/Functional/Frontend/RemoveFeedSourceTest.php diff --git a/Classes/Command/RemoveFeedSourcesCommand.php b/Classes/Command/RemoveFeedSourcesCommand.php new file mode 100644 index 00000000..66b7e4e9 --- /dev/null +++ b/Classes/Command/RemoveFeedSourcesCommand.php @@ -0,0 +1,70 @@ + 'feeddisplay:remove', + 'description' => 'Remove explicitly selected stored feed sources', + 'schedulable' => false, +])] +final class RemoveFeedSourcesCommand extends Command +{ + public function __construct(private readonly FeedRefresher $refresher) + { + parent::__construct(); + } + + protected function configure(): void + { + $this->addOption('url', null, InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Exact source to remove (required, repeatable)'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + /** @var list $arguments */ + $arguments = $input->getOption('url'); + $urls = array_values(array_unique(array_map(static fn(string $url): string => trim(stripslashes($url)), $arguments))); + if ($urls === [] || in_array('', $urls, true)) { + $io->error('Specify each source to remove with --url.'); + return self::INVALID; + } + $status = self::SUCCESS; + foreach ($urls as $url) { + $result = $this->refresher->remove($url); + $io->writeln(OutputFormatter::escape(($result->status === 'updated' ? 'removed' : $result->status) . ': ' . $url)); + if ($result->error !== null) { + $io->error(OutputFormatter::escape($result->error->getMessage())); + } + if ($result->status !== 'updated') { + $status = self::FAILURE; + } + } + return $status; + } +} diff --git a/Classes/Service/FeedRefresher.php b/Classes/Service/FeedRefresher.php index 68c5c68c..9ebffa37 100644 --- a/Classes/Service/FeedRefresher.php +++ b/Classes/Service/FeedRefresher.php @@ -77,6 +77,16 @@ public function invalidate(StoredFeed $stored): void $this->storage->markInvalidated($stored->url, $stored->revision); } + public function remove(string $url): RefreshResult + { + return $this->withLock($url, 'remove', function () use ($url): RefreshResult { + $this->storage->remove($url); + // Also retry this after a previous deletion whose invalidation failed. + $this->cacheManager->flushCachesInGroupByTag('pages', FeedStorage::cacheTag($url)); + return new RefreshResult('updated'); + }); + } + /** @param callable(): RefreshResult $operation */ private function withLock(string $url, string $name, callable $operation): RefreshResult { diff --git a/Tests/Functional/Frontend/RemoveFeedPageCacheTest.php b/Tests/Functional/Frontend/RemoveFeedPageCacheTest.php new file mode 100644 index 00000000..83399939 --- /dev/null +++ b/Tests/Functional/Frontend/RemoveFeedPageCacheTest.php @@ -0,0 +1,53 @@ +' . $image . 'Ihttps://example.test/'); + FeedHttp::$responses[$image] = new Response(200, ['Content-Type' => 'image/svg+xml'], ''); + self::assertSame('updated', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nplugin.tx_feeddisplay_pi1.settings.getFields.feed = title,image_url"); + FeedHttp::$beforeResponse[$image] = function (): void { + $command = new CommandTester($this->get(RemoveFeedSourcesCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [self::URL]])); + }; + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->findState(self::URL)); + $pages = $this->getConnectionPool()->getConnectionForTable('cache_pages'); + self::assertSame(0, $pages->count('*', 'cache_pages', [])); + FeedHttp::$responses[self::URL] = FeedHttp::rss('RECREATED_SOURCE'); + self::assertStringContainsString('RECREATED_SOURCE', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->findState(self::URL)); + self::assertSame(1, $pages->count('*', 'cache_pages', [])); + } +} diff --git a/Tests/Functional/Frontend/RemoveFeedSourceTest.php b/Tests/Functional/Frontend/RemoveFeedSourceTest.php new file mode 100644 index 00000000..c5e59938 --- /dev/null +++ b/Tests/Functional/Frontend/RemoveFeedSourceTest.php @@ -0,0 +1,91 @@ +writeFeedFixture('Kept source', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 0]); + self::assertStringContainsString('Kept source', $this->requestPage()); + $command = new CommandTester($this->get(RemoveFeedSourcesCommand::class)); + self::assertSame(Command::INVALID, $command->execute([])); + self::assertNotNull($this->get(FeedStorage::class)->find($url)); + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [$url]])); + self::assertNull($this->get(FeedStorage::class)->find($url)); + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [$url]])); + self::assertStringContainsString('Kept source', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } + + #[Test] + public function removalIsExcludedFromTheScheduler(): void + { + $commands = iterator_to_array($this->get(CommandRegistry::class)->getSchedulableCommands()); + self::assertArrayHasKey('feeddisplay:refresh', $commands); + self::assertArrayNotHasKey('feeddisplay:remove', $commands); + } + + #[Test] + public function missingSourceStillInvalidatesPagesOnEveryRetry(): void + { + $url = 'https://example.test/removed.xml'; + $calls = 0; + $cache = $this->createMock(CacheManager::class); + $cache->expects(self::exactly(2))->method('flushCachesInGroupByTag')->with('pages', FeedStorage::cacheTag($url))->willReturnCallback(static function () use (&$calls): void { + if (++$calls === 1) { + throw new \RuntimeException('Temporary cache failure'); + } + }); + $subject = new FeedRefresher($this->get(FeedStorage::class), $this->get(\ErHaWeb\FeedDisplay\Service\FeedFetcher::class), $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class), $cache, $this->get(\TYPO3\CMS\Core\Context\Context::class), new \Psr\Log\NullLogger()); + self::assertSame('failed', $subject->remove($url)->status); + self::assertSame('updated', $subject->remove($url)->status); + } + + #[Test] + public function busySourcesAreNotRemovedAndCanBeRetried(): void + { + $url = $this->writeFeedFixture('Locked source', ['Item']); + $this->initializeFrontendRootPage($url); + $this->requestPage(); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + self::assertTrue($lock->acquire($mode)); + $command = new CommandTester($this->get(RemoveFeedSourcesCommand::class)); + try { + self::assertSame(Command::FAILURE, $command->execute(['--url' => [$url]])); + self::assertStringContainsString('locked:', $command->getDisplay()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } finally { + $lock->release(); + } + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [$url]])); + self::assertNull($this->get(FeedStorage::class)->find($url)); + } +} From 4a76b00569140198e610e28e296015e2abf8bab7 Mon Sep 17 00:00:00 2001 From: Eric Harrer Date: Sun, 8 Nov 2026 02:02:33 +0100 Subject: [PATCH 5/5] [DOCS] Document persistent feeds and scheduler setup Co-authored-by: grs --- Documentation/Configuration/Index.rst | 1 + Documentation/Configuration/Scheduler.rst | 198 ++++++++++++++++ .../TypoScriptReference/Constants.rst | 25 +- Documentation/Installation/Index.rst | 31 ++- Documentation/KnownProblems/Index.rst | 220 ++++++++++++++++-- README.md | 20 +- 6 files changed, 466 insertions(+), 29 deletions(-) create mode 100644 Documentation/Configuration/Scheduler.rst diff --git a/Documentation/Configuration/Index.rst b/Documentation/Configuration/Index.rst index 8bcae675..ba9a5285 100644 --- a/Documentation/Configuration/Index.rst +++ b/Documentation/Configuration/Index.rst @@ -12,5 +12,6 @@ Configuration SiteSet TypoScriptReference/Index + Scheduler TSconfig Events diff --git a/Documentation/Configuration/Scheduler.rst b/Documentation/Configuration/Scheduler.rst new file mode 100644 index 00000000..4637cd9a --- /dev/null +++ b/Documentation/Configuration/Scheduler.rst @@ -0,0 +1,198 @@ +.. include:: /Includes.rst.txt + +.. _configuration-scheduler: + +Persistent feeds and scheduled refresh +===================================== + +Last successful response +------------------------ + +Feed Display stores the last successfully parsed document for each source URL +in ``tx_feeddisplay_feed``. It has no expiry and survives every TYPO3 cache +flush. Only a successful fetch replaces it; a valid RSS or Atom feed without +items also counts as success. Download, parse and storage errors preserve the +previous response. Include this table in normal database backups. + +The source is shared by plugins using the same URL. Display options and prepared +arrays remain separate per plugin configuration. A refresh by one plugin or the +command updates that shared source for all plugins. The spelling of the URL +matters: redirects do not merge two different configured URLs. + +The TYPO3 ``feeddisplay`` cache contains only disposable display data. Flushing +it rebuilds these arrays from the database; it does not force a download or +delete the persistent response. Use ``feeddisplay:refresh`` to force a refresh. +Only scheduler-mode plugins tag their pages for source invalidation. A new +revision saved by a live fetch or the command invalidates those scheduler pages. +Live pages retain their normal TYPO3 cache lifetime and are not invalidated by +feed changes. ``cacheDuration`` does not disable the TYPO3 page cache. + +Scheduler pages also carry the source revision actually used during rendering. +After TYPO3 writes the page cache entry, Feed Display compares those revisions +with the database. If a source changed or was removed while the page rendered, +the newly written entry is discarded. The running response can still show the +older content; the next request rebuilds it. Matching revisions keep the page's +normal lifetime. No additional source query runs on a page-cache hit. + +Avoid combining an uncached live plugin and a scheduler plugin for the same URL +on one page when the source changes on every fetch. If the scheduler plugin +renders first and the live plugin then stores a new revision, that page contains +two different source versions and is discarded after every rendering. Mixed +mode alone does not disable caching: a live plugin rendered first can update +the source before the scheduler plugin reads the same revision. + +A revision read failure is logged and leaves that source unverified; another +source with a confirmed mismatch can still cause the page to be discarded. +Failures removing an obsolete entry are also logged. This check concerns TYPO3's +page cache, not reverse-proxy, CDN or third-party static caches. + +Live mode (default) +------------------- + +Existing plugins remain in live mode without configuration changes. When the +plugin runs, a positive ``cacheDuration`` limits refresh attempts, including +retries after failure. Zero and negative durations attempt a new fetch on every +uncached rendering, still persist successes and fall back to the last success. +Failures do not change the last successful payload or its success timestamp. +There is no seven-day fallback deadline. If storing a fresh response fails, +positive cache durations retain its prepared display result for that duration. +This is temporary cache protection, not durable storage; zero and negative +durations continue attempting a fetch on each rendering. + +Protection starts with the first successful fetch after the update. Existing +3.0.2 display-cache entries remain usable for their original lifetime until a +persistent response exists; incomplete display arrays are never converted to +source documents. + +Scheduler mode +-------------- + +Select ``scheduler`` through the content element's Fetch mode field, Site +Settings, or TypoScript: + +.. code-block:: typoscript + + plugin.tx_feeddisplay_pi1.settings.fetchMode = scheduler + +In a site's ``settings.yaml`` use YAML syntax: + +.. code-block:: yaml + + plugin.tx_feeddisplay_pi1.settings.fetchMode: scheduler + +A nonempty FlexForm value overrides inherited configuration. Keep ``fetchMode`` +in ``ignoreFlexFormSettingsIfEmpty`` when an empty field should inherit. + +If there is no usable stored document, the first frontend request tries to fetch +one. A failed first attempt is retried no more than once every 60 seconds. +Without data, the configured error message is shown and the page's maximum cache +lifetime is 60 seconds. Once a usable response exists, frontend rendering uses +it regardless of age and does not fetch the feed document again. A recurring +job is required to update it. + +New documents retain the parser input selected by SimplePie, including response +bytes, headers, MIME type and original source/base URIs. Rebuilding display data +does not repeat autodiscovery and keeps the originally selected feed when a page +offers several. + +If a stored document cannot be reconstructed, the frontend attempts repair at +most once every 60 seconds under the same source lock and scheduler HTTP limits. +Failed repair preserves the stored payload. In scheduler mode, ``cacheDuration`` +controls only prepared arrays; zero and negative values do not trigger ordinary +frontend downloads. + +Run the refresh command +----------------------- + +From the deployed installation root: + +.. code-block:: bash + + # Composer + vendor/bin/typo3 feeddisplay:refresh + + # Classic + php typo3/sysext/core/bin/typo3 feeddisplay:refresh + +The command combines non-deleted live-workspace feed plugins' explicit FlexForm +URLs, all stored source rows, and additional repeatable ``--url`` arguments. +Legacy ``list_type = feeddisplay_pi1`` records are included. Hidden records are +included, deleted records and workspace versions are excluded from this scan. +Stored sources are intentionally retained even when no content element uses +them anymore, including failed first attempts. Both live and scheduler sources +are refreshed. There is no usage tracking or automatic pruning. + +It does not resolve full page TypoScript. A URL inherited through TypoScript or +Site Settings becomes discoverable after its first frontend attempt, or can be +provided explicitly: + +.. code-block:: bash + + vendor/bin/typo3 feeddisplay:refresh --url='https://example.org/rss' --url='https://example.org/atom' + +An empty source list succeeds with an explanatory message. ``updated`` means a +new document revision was saved; ``unchanged`` means a successful fetch renewed +freshness without changing the revision. ``locked`` means the source was skipped +because another operation holds its lock. A failed source is diagnosed while +other sources continue; the command exits unsuccessfully if any source fails. + +In EXT:scheduler create an **Execute console commands** task for +``feeddisplay:refresh``. No additional extension-specific Scheduler task is +necessary, and EXT:scheduler is optional. Ensure cron actually invokes TYPO3's +Scheduler. With automatic selection, no array option is needed. TYPO3's Scheduler +form splits array options on commas; for URLs containing literal commas use +cron with quoted, repeated ``--url`` arguments instead. + +Source types and operational limits +----------------------------------- + +HTTP(S) feeds retain TYPO3's HTTP configuration. Live fetches keep SimplePie's +feed autodiscovery. Scheduled fetches, including first frontend fetches in +scheduler mode, read the configured URL directly, so configure the feed itself +for this mode. The selected parser input is stored as binary-safe scalar +data and replayed with its original bytes, selected headers and URL context. +Local RSS/Atom files are also supported; +use absolute paths for jobs and ensure CLI and web PHP see the same filesystem +and run with equivalent read permissions. Relative paths use the process's +working directory; do not share relative sources between different directories. +Run jobs as the application user, never with broader filesystem privileges. + +Scheduled HTTP fetches, including first frontend fetches in scheduler mode, limit +recorded response bodies to 2 MiB in total and cap each HTTP request's configured +timeouts at ten seconds. Reading into PHP strings is bounded; the transport may +buffer data first. This is not a guaranteed network-download limit. Redirects +can cause several requests, so ten seconds is not a total source or command +deadline. Guzzle 7's PHP stream timeout can be an idle timeout. +These new limits are not applied to normal live fetches. A large live source can +therefore render successfully but fail in the refresh job. Local sources of +scheduled fetches must be regular files of at most 2 MiB. + +Database writes are atomic per response, and failed page invalidation remains +pending for a later retry, including an unchanged refresh. The default TYPO3 +locks coordinate processes on one host. Multi-node deployments need a shared +locking strategy and shared page-cache invalidation; the extension does not +introduce a distributed lock service. Replay of the feed document does not +prevent separate HTTP requests by custom getters or event listeners. Feed images +are reused while their file exists in ``typo3temp``. An absent image is downloaded +during rendering, including in scheduler mode. Clearing temporary files can +trigger this download again. + +Remove unused sources +--------------------- + +Only explicitly selected URLs are removed: + +.. code-block:: bash + + vendor/bin/typo3 feeddisplay:remove --url='https://example.org/unused.xml' + +For Classic use the Classic executable shown above. Repeat ``--url`` for several +sources. The command is intentionally unavailable in Scheduler selections. +It acquires the same source lock, deletes the stored row and invalidates tagged +scheduler pages even when the row is already missing. Repeating it is safe, +including after a failed page flush. A busy lock or failure is reported with a failing exit code. + +Remove remaining plugin configuration as well when retiring a source. Otherwise +a later frontend request or refresh scan can recreate it. Manual deletion is +not a permanent exclusion rule. There is no automatic deletion or retention +policy; administrators control when durable data may be removed. diff --git a/Documentation/Configuration/TypoScriptReference/Constants.rst b/Documentation/Configuration/TypoScriptReference/Constants.rst index 38449e1b..d0051f1d 100644 --- a/Documentation/Configuration/TypoScriptReference/Constants.rst +++ b/Documentation/Configuration/TypoScriptReference/Constants.rst @@ -257,8 +257,27 @@ Cache duration :Default: 3600 :Path: plugin.tx_feeddisplay_pi1.settings - Time in seconds in which the data is to be read from the - cache until the next renewal of the cache. (0 = no cache) + In live mode, the minimum interval between source refresh attempts and the + lifetime of prepared display arrays. Zero and negative values attempt a + refresh on every plugin execution. Successful responses remain stored + permanently even when this value is zero. In scheduler mode this value + affects prepared arrays only. TYPO3's page cache is independent. + +.. _configuration-typoscript-constants-settings-fetchmode: + +Fetch mode +~~~~~~~~~~ + +.. confval:: fetchMode + + :type: string + :Default: live + :Path: plugin.tx_feeddisplay_pi1.settings + + ``live`` refreshes during page rendering as needed. ``scheduler`` fetches a + missing source once, then displays its last successful response until a job + refreshes it. Unknown or empty values use live mode. See + :ref:`configuration-scheduler` for first-fetch retries and job setup. .. _configuration-typoscript-constants-settings-getfields: @@ -341,7 +360,7 @@ Ignore Flexform Settings if empty .. confval:: ignoreFlexFormSettingsIfEmpty :type: list of strings, separated by comma - :Default: feedUrl, maxFeedCount, maxContentLength, maxHeaderLength, logoMaxWidth, logoMaxHeight, feedIconMaxWidth, feedIconMaxHeight, dateFormat, stripTags, linkTarget, errorMessage, cacheDuration, getFields.feed, getFields.items + :Default: feedUrl, maxFeedCount, maxContentLength, maxHeaderLength, logoMaxWidth, logoMaxHeight, feedIconMaxWidth, feedIconMaxHeight, dateFormat, stripTags, linkTarget, errorMessage, cacheDuration, fetchMode, getFields.feed, getFields.items :Path: plugin.tx_feeddisplay_pi1 Comma separated list of settings to be overridden by TypoScript if the diff --git a/Documentation/Installation/Index.rst b/Documentation/Installation/Index.rst index fd0764c9..468b5157 100644 --- a/Documentation/Installation/Index.rst +++ b/Documentation/Installation/Index.rst @@ -24,9 +24,9 @@ TYPO3 v14 backend module paths. #. Apply database changes - This extension uses the caching framework to cache feed data and - plugin configuration. For this, the tables `cache_feeddisplay` - and `cache_feeddisplay_tags` must be created. + Apply the regular schema update, including ``tx_feeddisplay_feed`` + for durable responses and ``cache_feeddisplay`` / + ``cache_feeddisplay_tags`` for disposable display data. These tables can be created in the Maintenance module under :guilabel:`Analyze Database Structure` → :guilabel:`Apply selected changes`. @@ -53,9 +53,9 @@ TYPO3 v14 backend module paths. #. Apply database changes - This extension uses the caching framework to cache feed data and - plugin configuration. For this, the tables `cache_feeddisplay` - and `cache_feeddisplay_tags` must be created. + Apply the regular schema update, including ``tx_feeddisplay_feed`` + for durable responses and ``cache_feeddisplay`` / + ``cache_feeddisplay_tags`` for disposable display data. These tables can be created in the Maintenance module under :guilabel:`Analyze Database Structure` → :guilabel:`Apply selected changes`. @@ -121,3 +121,22 @@ Composer installations obtain SimplePie through their normal project dependency and do not require this additional build. Classic installations receive library updates with a new extension release; the extension never downloads libraries during a request. + +Updating from 3.0.2 +================== + +Apply the normal database schema update after replacing the extension files, +for example with ``vendor/bin/typo3 extension:setup`` (Composer) or +``php typo3/sysext/core/bin/typo3 extension:setup`` (Classic). This creates +``tx_feeddisplay_feed``. Rebuild deployment/DI caches as usual. Existing plugins +keep live mode; no Scheduler job or new plugin setting is required. + +Without the table, live rendering falls back to fetching and its existing +display cache, and logs a schema warning. Scheduler rendering cannot persist a +first response and shows its configured error message. Complete the schema +update to enable durable protection. No tables are created during requests. + +There is no upgrade wizard for converting old display-cache entries: they lack +the complete feed document. Protection begins with the first successful fetch +on the new version. See :ref:`configuration-scheduler` for storage and refresh +semantics. Existing Site Set dependencies remain unchanged. diff --git a/Documentation/KnownProblems/Index.rst b/Documentation/KnownProblems/Index.rst index f6aa6ddd..7c1e3067 100644 --- a/Documentation/KnownProblems/Index.rst +++ b/Documentation/KnownProblems/Index.rst @@ -6,31 +6,219 @@ Known problems ============== -The following issues are known problems. However those are not fixable inside `EXT:feed_display`. +Check the effective settings of the affected content element, the TYPO3 log and +the refresh command's output. A feed that opens in a desktop browser can still +fail on the application server. Persistent fallback can conceal an outage. -If you notice anything else feel free to open an issue on `Github `__. +.. _known-problems-scheduler-empty: + +Scheduler mode has no output or shows the error message +======================================================= + +Selecting ``fetchMode = scheduler`` does not create a job. With no stored +response, the first frontend rendering attempts a fetch. On failure the plugin +shows its configured ``errorMessage`` or the default error and limits the page +cache to 60 seconds. Further initial attempts are throttled for 60 seconds, +including across cache flushes. A valid stored response has no expiry and +survives every cache flush. + +#. Apply the normal schema update: ``tx_feeddisplay_feed`` must exist. Without + it, scheduler rendering cannot store a first response. Check the log for + the schema warning and run ``extension:setup`` in the deployed installation. +#. Verify the effective ``feedUrl`` and ``fetchMode``, including FlexForm + overrides. Check that custom templates actually display the error message. +#. Run the command from the installation root with the same URL: + + .. code-block:: bash + + # Composer + vendor/bin/typo3 feeddisplay:refresh --url='https://example.org/feed.xml' + + # Classic + php typo3/sysext/core/bin/typo3 feeddisplay:refresh --url='https://example.org/feed.xml' + +#. Expect ``updated`` or ``unchanged``. Investigate failures in the command + output and TYPO3 log, then configure recurring execution. Saving an + **Execute console commands** task alone does not start TYPO3's Scheduler. + +See :ref:`configuration-scheduler` for first-fetch and refresh behavior. A URL +inherited only through TypoScript or Site Settings is not discovered by the +command until its first frontend attempt, unless passed explicitly with +``--url``. It does not resolve whole pages' TypoScript. + +A job succeeds but the plugin remains stale +=========================================== + +* **Different source spellings:** URL hashes distinguish schemes, host spelling, + query order, trailing slashes and encoding. Redirects do not merge source + rows. Use the same trimmed URL in the plugin and command. +* **Different environments:** CLI and web must use the same database, deployed + code and filesystem. Do not invoke the extension's test installation by + mistake. In DDEV run the command inside the web container. Match proxy, + credentials, DNS and TLS configuration between CLI and web. +* **Wrong job or inactive cron:** Confirm that Scheduler runs and invokes the + intended command. A scheduler-mode plugin never refreshes an existing + usable successful response by itself, regardless of ``cacheDuration``. An + unusable document is an exception: see the replay guidance below. +* **Multiple nodes or external caches:** Host-local TYPO3 locks do not + coordinate nodes. Configure shared locks and page invalidation across the + deployment. CDNs and reverse proxies may need their own invalidation. +* **Changing metadata:** A revision includes response bytes and selected + headers, not only visible items. A changing build date or redirect can + invalidate scheduler pages even when visible content looks identical. Live + pages are not tagged and keep their usual lifetime, even with a zero or + negative feed cache duration. + +The refresh command fails, skips sources or has no sources +========================================================== + +* **Command unavailable:** Verify extension activation, complete files and + rebuilt DI caches. Classic Git checkouts need their library build; TER + packages already include it. See :ref:`installation`. +* **No sources found:** An empty set succeeds. The command uses explicit + FlexForm URLs, stored source rows and additional ``--url`` arguments. It + excludes deleted records and non-live workspace versions, but includes + hidden content and legacy feed plugin records. Load inherited URLs once in + the frontend or provide them explicitly. +* **Retired or mistyped sources keep failing:** Stored rows, including failed + first attempts, remain selected indefinitely. Both live and scheduler + sources are included. Correct the configuration and remove unused rows + with ``feeddisplay:remove``; nothing is automatically pruned. +* **Array options and commas:** Repeat quoted ``--url`` options on the command + line. The Scheduler form splits array options at commas. Automatic source + selection avoids this form issue; use cron when an explicit URL contains a + literal comma. Percent-encoding only helps if the source supports it and + the plugin uses exactly the same spelling. +* **A source is busy:** A result of ``locked`` means skipped because another + operation holds its lock. It is not an HTTP failure. Other sources continue. + Repeated skips suggest overlapping or hanging processes. Check those + processes before scheduling more frequent runs. +* **Oversized or slow response:** Scheduled HTTP fetches, including first + frontend fetches in that mode, cap stored response bodies at 2 MiB in total and + each HTTP request's timeouts at ten seconds. Redirects can cause several + requests; this is not a total time or transport download + limit. Normal live fetches retain their previous limits. A source that + works live can therefore fail in the job. +* **HTTP or parse error:** Login pages, bot challenges, invalid XML, redirect + loops and intermittent 4xx/5xx responses can fail. Live fetches keep HTTP + feed autodiscovery. Scheduled fetches read the configured URL directly, so + configure the feed itself for that mode. +* **Storage, locks or invalidation fail:** Check the concrete exception and URL + in the TYPO3 log, database permissions/capacity and filesystem permissions. + Failed writes preserve the previous response; fresh live data can still + appear in that request. With a positive live ``cacheDuration``, its prepared + result is cached for that duration despite the write error; a cache flush + removes this temporary protection. Zero and negative values keep fetching. + Failed page invalidation remains pending and is retried, including after + an unchanged refresh. + +A mixed run continues after a failed source but returns a failure status overall. +Do not assume that no source updated just because the task is marked failed. .. _known-problems-no-feed-items-displayed: No feed items are displayed =========================== -**Question:** +* **No successful response:** The configured error message is shown when no + usable source exists. Live mode can continue using its display cache while + the new schema is missing, but permanent protection requires the table. +* **The feed is empty:** A valid RSS/Atom feed without items is a successful + replacement, not an error. The no-items message can appear instead of the + fetch-error message. +* **Server access differs from browser access:** Check application-server DNS, + firewall, proxy, authentication and certificate trust through TYPO3's HTTP + configuration. Disabling TLS verification is not a correction for missing + certificates. +* **Local file inaccessible:** Use a path visible inside the PHP container, + including readable symlink targets. For jobs use absolute paths and the + application user with equivalent read permissions. Relative paths depend + on the process's working directory and can point to different files in CLI. +* **Customization suppresses output:** Check getter fields, ``maxFeedCount``, + templates, conditions and ``SingleFeedDataEvent`` listeners. Listener errors + propagate normally; they are not disguised as feed download errors. +* **Broken stored document:** A valid prepared display-cache hit remains usable + without reading the BLOB. On a miss, checksum, structure and header errors + are diagnosed. New documents restore the selected parser input without + repeating discovery. If reconstruction fails, the + frontend attempts repair no more than once every 60 seconds; failed repair + keeps the payload and success time. A manual refresh can replace the document + immediately. No SimplePie PHP objects are persisted. + +Images trigger requests +======================= + +Feed images are downloaded only when their temporary file is missing. Rendering +can therefore still make an image HTTP request in scheduler mode, after a +temporary-file cleanup or when an image is used for the first time. Custom +getters and event listeners may also make their own requests. + +The plugin is missing or displays the example feed +================================================== + +* Include the ``erhaweb/feed-display`` Site Set in the active site's + dependencies, or static TypoScript for a template-record setup. Check the + actual site's rootline. See :ref:`configuration-site-set`. +* Replace the default ``https://typo3.org/rss`` URL in the relevant settings. + A saved FlexForm can override the value you changed site-wide. +* Check content/page visibility, language, access restrictions and the rendered + column. These can prevent plugin execution entirely. +* For legacy ``CType = list`` / ``list_type = feeddisplay_pi1`` records, run + ``feedDisplayCTypeMigration`` when required. The current content type is + ``feeddisplay_pi1``. Also check backend-group permissions for editors. -I get the message "Sorry, no items could be fetched." in the frontend although the feed is available when I call the -feed URL directly. What could be the cause of this problem? +Settings seem ignored +====================== -**Answer:** +* Inspect the content element's FlexForm as well as Site Settings and later + TypoScript assignments. Use YAML ``fetchMode: scheduler`` in site settings + and TypoScript ``fetchMode = scheduler`` in setup, with the full plugin path. +* ``ignoreFlexFormSettingsIfEmpty`` belongs to the plugin, not ``settings``. + Keep ``fetchMode`` in that list for inheritance. An unignored empty or + unknown mode uses live behavior. +* The default ignore list can treat FlexForm ``0`` as inherited. To explicitly + select zero, adjust that setting's inheritance rule or the inherited value. +* ``getFields.feed`` and ``getFields.items`` select SimplePie getters. Check + exact names, supported arguments and the data structure expected by custom + templates. Cropping/HTML settings affect display, not the stored document. -Please verify that the feed is indeed being delivered reliably. I know of cases where the feed seemed to work initially, -but an error 500 occurred on some calls. If this error occurs sporadically, it could be that the memory size on the -system delivering the feed has been exceeded. +Content is stale or changes at unexpected times +================================================ -In this context, please note that the Feed Display extension itself caches the return. This means that in case of doubt -a faulty feed may persist for a longer period of time until the TYPO3 cache is cleared again, although a direct call to -the feed URL will work again. +* A positive live ``cacheDuration`` limits attempts per shared source, + including after failures. Different plugin options do not keep independent + old copies of the feed. A plugin with a shorter interval can update all + plugins of that URL. +* Cache flushes rebuild display arrays from the persistent source without + forcing a download. Use ``feeddisplay:refresh`` to force an attempt. The + last successful response has no expiry; monitor failures in logs/jobs. +* ``cacheDuration = 0`` or a negative value does not disable TYPO3 page caching. + A cached page may avoid plugin execution altogether. Live page-cache + lifetimes remain unchanged; live pages are not invalidated by source changes. + Only scheduler pages carry source tags. A successful changed revision from + either a live plugin or the job invalidates those scheduler pages. +* A scheduler page whose source changes or is removed during rendering is + discarded immediately after TYPO3 writes its cache entry. The running + response can still contain the older revision; the next request rebuilds it. + If checking a source revision fails, the warning identifies its hash and the + page-cache identifier. That source remains unverified. A failed removal is + diagnosed as well. Check database/cache availability; reverse-proxy, CDN + and third-party static caches require their own handling. +* Mixed live/scheduler plugins of the same URL can prevent effective page + caching. In particular, if a scheduler plugin renders before a live plugin + with ``cacheDuration <= 0`` and every fetch changes the source revision, + every rendering creates an inconsistent page that is discarded. Use one + mode for that source on the page. Mixed mode alone is not sufficient to + cause this: live first, followed by scheduler using the same revision, can + remain cacheable. +* Existing 3.0.2 display entries remain valid for their original lifetime until + the first durable response exists. They cannot protect against a full + cache flush before that first successful fetch. +* Removing a source does not disable configured plugins. Another frontend + attempt or refresh scan can recreate it. Remove its configuration as well + when retiring it. ``feeddisplay:remove`` requires explicit URLs, is safe to + repeat and reports lock conflicts or failed invalidation for manual retry. -If the feed is loaded from a remote ``http`` or ``https`` URL, also verify the -TYPO3 HTTP client configuration of the instance. Proxy requirements and -outbound network restrictions apply through TYPO3's PSR-17/PSR-18 interfaces, -so a blocked or incomplete TYPO3 HTTP setup can prevent feed retrieval. +If a problem remains, report its effective configuration, TYPO3/PHP versions, +command output and relevant log entry in the +`issue tracker `__. diff --git a/README.md b/README.md index ffb2d421..65f4fe60 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,10 @@ to Fluid templates. Remote `http` and `https` feeds are fetched through TYPO3's PSR-17/PSR-18 HTTP client, so instance-wide HTTP settings such as proxy configuration are applied -automatically. Parsed results are cached through TYPO3's caching framework and -refreshed when plugin configuration changes or the configured cache lifetime -expires. +automatically. The last successful response per source is stored permanently +in the database and survives TYPO3 cache flushes. Prepared display data stays +separate per plugin configuration. Live refresh is the default; an optional +`feeddisplay:refresh` job can keep scheduler-mode sources current. ## At a glance @@ -36,7 +37,8 @@ expires. - Configurable feed and item fields for Fluid templates. - Site Set support for TYPO3 v13 and v14, with a static TypoScript fallback. - Content-element FlexForm settings plus Site Settings and TypoScript constants. -- Dedicated cache for parsed feed data and plugin configuration. +- Durable last-successful responses with disposable display caches. +- Optional scheduled refresh and explicit removal of unused sources. ## Installation @@ -44,6 +46,16 @@ expires. composer require erhaweb/feed-display ``` +After installation or an update from 3.0.2, apply the regular database schema +update (`vendor/bin/typo3 extension:setup` in Composer installations). Existing +plugins remain in live mode; no Scheduler job is required. Durable protection +starts with the first successful fetch. Cache flushes rebuild display arrays +without forcing a download. Use `feeddisplay:refresh` to refresh sources and +`feeddisplay:remove --url='…'` to remove explicitly selected ones. Configured +sources can be created again afterwards. + +See [persistent feeds and Scheduler setup](Documentation/Configuration/Scheduler.rst). + Continue with the [Composer installation guide](https://docs.typo3.org/p/erhaweb/feed-display/main/en-us/Installation/) and the