diff --git a/Build/Scripts/classicFixture.php b/Build/Scripts/classicFixture.php index 42e30cf3..011bb7cb 100644 --- a/Build/Scripts/classicFixture.php +++ b/Build/Scripts/classicFixture.php @@ -27,6 +27,16 @@ echo 'Classic autoloading resolved both SimplePie APIs from ' . $expectedPath . PHP_EOL; exit(0); } +if ($mode === 'verifyStored') { + $pool = $container->get(\TYPO3\CMS\Core\Database\ConnectionPool::class); + $row = $pool->getConnectionForTable('tx_feeddisplay_feed')->select(['response_data', 'revision'], 'tx_feeddisplay_feed')->fetchAssociative(); + if (!is_array($row) || !is_string($row['response_data']) || hash('sha256', $row['response_data']) !== $row['revision']) { + throw new RuntimeException('Classic rendering did not persist a complete feed document.'); + } + \ErHaWeb\FeedDisplay\Storage\FeedDocument::decode($row['response_data']); + echo 'Verified the persistent Classic source document.' . PHP_EOL; + exit(0); +} if ($mode !== 'seed') { throw new InvalidArgumentException('Expected seed or probe.'); } diff --git a/Build/Scripts/testClassic.sh b/Build/Scripts/testClassic.sh index 6c958636..c9f5cc7d 100644 --- a/Build/Scripts/testClassic.sh +++ b/Build/Scripts/testClassic.sh @@ -76,6 +76,17 @@ for mode in linked packaged; do exit 1 fi done + TYPO3_CONTEXT=Development php Build/Scripts/classicFixture.php "${instance}" verifyStored + rm "${instance}/fileadmin/feed.xml" + ( + cd "${instance}" + TYPO3_CONTEXT=Development php typo3/sysext/core/bin/typo3 cache:flush + ) + curl --fail --silent --show-error "${base}" -o "${instance}/persistent.html" + if ! grep -q 'Classic bundle integration item' "${instance}/persistent.html"; then + cat "${instance}/server.log" >&2 + exit 1 + fi kill "${serverPid}" wait "${serverPid}" 2>/dev/null || true serverPid='' diff --git a/Build/Scripts/verifyTerArtefact.php b/Build/Scripts/verifyTerArtefact.php index b89d2d87..c6c479d2 100644 --- a/Build/Scripts/verifyTerArtefact.php +++ b/Build/Scripts/verifyTerArtefact.php @@ -10,6 +10,9 @@ $manifest = json_decode($zip->getFromName('composer.json'), true, 512, JSON_THROW_ON_ERROR); $bundlePath = $manifest['extra']['typo3/cms']['Package']['providesPackages']['simplepie/simplepie'] ?? null; $isBundle = $bundlePath === 'Resources/Private/Php/ComposerVendor'; +if (is_file($sourceRoot . '/ext_tables.sql') && $zip->locateName('ext_tables.sql') === false) { + throw new RuntimeException('The TER package is missing the extension database schema.'); +} $temporary = sys_get_temp_dir() . '/feed-display-artefact-' . bin2hex(random_bytes(8)); mkdir($temporary); try { diff --git a/Classes/Command/RefreshFeedsCommand.php b/Classes/Command/RefreshFeedsCommand.php new file mode 100644 index 00000000..ef37a6f0 --- /dev/null +++ b/Classes/Command/RefreshFeedsCommand.php @@ -0,0 +1,69 @@ +addOption('url', null, InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Additional feed source (repeatable; commas remain part of the URL)'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + /** @var list $additional */ + $additional = $input->getOption('url'); + try { + $urls = $this->urls->collect($additional); + } catch (\Throwable $exception) { + $io->error(OutputFormatter::escape('Could not collect feed sources: ' . $exception->getMessage())); + return self::FAILURE; + } + if ($urls === []) { + $io->success('No feed sources found. Explicit FlexForm URLs, stored sources and --url are considered.'); + return self::SUCCESS; + } + $status = self::SUCCESS; + foreach ($urls as $url) { + $result = $this->refresher->refresh($url, true); + $io->writeln(OutputFormatter::escape($result->status . ': ' . $url)); + if ($result->error !== null) { + $io->error(OutputFormatter::escape($result->error->getMessage())); + $status = self::FAILURE; + } + } + return $status; + } +} diff --git a/Classes/Command/RemoveFeedSourcesCommand.php b/Classes/Command/RemoveFeedSourcesCommand.php new file mode 100644 index 00000000..66b7e4e9 --- /dev/null +++ b/Classes/Command/RemoveFeedSourcesCommand.php @@ -0,0 +1,70 @@ + 'feeddisplay:remove', + 'description' => 'Remove explicitly selected stored feed sources', + 'schedulable' => false, +])] +final class RemoveFeedSourcesCommand extends Command +{ + public function __construct(private readonly FeedRefresher $refresher) + { + parent::__construct(); + } + + protected function configure(): void + { + $this->addOption('url', null, InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Exact source to remove (required, repeatable)'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + /** @var list $arguments */ + $arguments = $input->getOption('url'); + $urls = array_values(array_unique(array_map(static fn(string $url): string => trim(stripslashes($url)), $arguments))); + if ($urls === [] || in_array('', $urls, true)) { + $io->error('Specify each source to remove with --url.'); + return self::INVALID; + } + $status = self::SUCCESS; + foreach ($urls as $url) { + $result = $this->refresher->remove($url); + $io->writeln(OutputFormatter::escape(($result->status === 'updated' ? 'removed' : $result->status) . ': ' . $url)); + if ($result->error !== null) { + $io->error(OutputFormatter::escape($result->error->getMessage())); + } + if ($result->status !== 'updated') { + $status = self::FAILURE; + } + } + return $status; + } +} diff --git a/Classes/Controller/FeedController.php b/Classes/Controller/FeedController.php index 35503918..253da1a2 100644 --- a/Classes/Controller/FeedController.php +++ b/Classes/Controller/FeedController.php @@ -17,38 +17,42 @@ namespace ErHaWeb\FeedDisplay\Controller; -use ErHaWeb\FeedDisplay\Service\FeedDataService; +use ErHaWeb\FeedDisplay\Service\FeedProvider; +use ErHaWeb\FeedDisplay\Storage\FeedStorage; use Psr\Http\Message\ResponseInterface; -use TYPO3\CMS\Core\Cache\Frontend\FrontendInterface; +use TYPO3\CMS\Core\Cache\CacheDataCollector; +use TYPO3\CMS\Core\Cache\CacheTag; use TYPO3\CMS\Extbase\Mvc\Controller\ActionController; class FeedController extends ActionController { - public function __construct( - private readonly FrontendInterface $cache, - private readonly FeedDataService $feedDataService - ) {} + public function __construct(private readonly FeedProvider $feedProvider) {} public function displayAction(): ResponseInterface { + $scheduler = trim((string)($this->settings['fetchMode'] ?? 'live')) === 'scheduler'; + if (array_key_exists('fetchMode', $this->settings)) { + unset($this->settings['fetchMode']); + $this->view->assign('settings', $this->settings); + } if ($this->settings === []) { return $this->htmlResponse(); } - $cacheIdentifier = 'feeddisplay_' . md5(serialize($this->settings)); - $data = $this->cache->get($cacheIdentifier); - $cacheDuration = (int)$this->settings['cacheDuration']; - - if ($cacheDuration === 0) { - $data = $this->feedDataService->buildData($this->settings); - $this->cache->remove($cacheIdentifier); - } elseif ($data === false || $data['settings'] !== $this->settings) { - $data = $this->feedDataService->buildData($this->settings); - $this->cache->set($cacheIdentifier, $data, [], $cacheDuration); + $url = trim(stripslashes((string)($this->settings['feedUrl'] ?? ''))); + $collector = $this->request->getAttribute('frontend.cache.collector'); + if ($scheduler && $url !== '' && $collector instanceof CacheDataCollector) { + $collector->addCacheTags(new CacheTag(FeedStorage::cacheTag($url))); + } + $provided = $this->feedProvider->provide($this->settings, $scheduler); + $data = $provided->data; + if ($scheduler && $url !== '' && $provided->revision !== null && $collector instanceof CacheDataCollector) { + $collector->addCacheTags(new CacheTag(FeedStorage::revisionTag($url, $provided->revision))); + } + if ($scheduler && !isset($data['feed']) && !isset($data['items']) && $collector instanceof CacheDataCollector) { + $collector->restrictMaximumLifetime(60); } - $this->view->assign('data', $data); - return $this->htmlResponse(); } } diff --git a/Classes/EventListener/VerifyPageFeedRevisions.php b/Classes/EventListener/VerifyPageFeedRevisions.php new file mode 100644 index 00000000..3daf78e5 --- /dev/null +++ b/Classes/EventListener/VerifyPageFeedRevisions.php @@ -0,0 +1,68 @@ +getRequest()->getAttribute('frontend.cache.collector'); + if (!$collector instanceof CacheDataCollector) { + return; + } + $revisions = []; + foreach ($collector->getCacheTags() as $tag) { + if (preg_match('/^feeddisplay_rev_([0-9a-f]{64})_([0-9a-f]{64})$/D', $tag->name, $match) === 1) { + $revisions[$match[1]][$match[2]] = true; + } + } + foreach ($revisions as $urlHash => $usedRevisions) { + $context = ['urlHash' => $urlHash, 'cacheIdentifier' => $event->getCacheIdentifier()]; + try { + $current = $this->storage->findRevisionByHash($urlHash); + } catch (\Throwable $exception) { + $this->logger->warning('Could not verify the cached feed revision.', $context + ['exception' => $exception]); + continue; + } + if ($current !== null && count($usedRevisions) === 1 && isset($usedRevisions[$current])) { + continue; + } + try { + $this->cacheManager->getCache('pages')->remove($event->getCacheIdentifier()); + } catch (\Throwable $exception) { + $this->logger->warning('Could not remove an obsolete feed page.', $context + ['exception' => $exception]); + } + return; + } + } +} diff --git a/Classes/Service/ConfiguredFeedUrls.php b/Classes/Service/ConfiguredFeedUrls.php new file mode 100644 index 00000000..effdac8a --- /dev/null +++ b/Classes/Service/ConfiguredFeedUrls.php @@ -0,0 +1,76 @@ + $additional + * @return list + */ + public function collect(array $additional): array + { + $urls = $this->storage->findUrls(); + $query = $this->connections->getQueryBuilderForTable('tt_content'); + $query->getRestrictions()->removeAll(); + $records = $query->select('*')->from('tt_content')->where( + $query->expr()->eq('deleted', 0), + $query->expr()->eq('t3ver_wsid', 0), + $query->expr()->or( + $query->expr()->eq('CType', $query->createNamedParameter('feeddisplay_pi1')), + $query->expr()->eq('CType', $query->createNamedParameter('list')), + ), + )->executeQuery()->fetchAllAssociative(); + foreach ($records as $record) { + if ($record['CType'] !== 'feeddisplay_pi1' && !($record['CType'] === 'list' && ($record['list_type'] ?? '') === 'feeddisplay_pi1')) { + continue; + } + if (trim((string)($record['pi_flexform'] ?? '')) === '') { + continue; + } + $form = GeneralUtility::xml2array((string)$record['pi_flexform']); + if (!is_array($form)) { + $this->logger->warning('Could not read feed FlexForm.', ['contentElementUid' => (int)$record['uid'], 'error' => $form]); + continue; + } + $sheets = $form['data'] ?? []; + if (!is_array($sheets)) { + $this->logger->warning('Invalid feed FlexForm structure.', ['contentElementUid' => (int)$record['uid']]); + continue; + } + foreach ($sheets as $sheet) { + if (!is_array($sheet)) { + continue; + } + $url = $sheet['lDEF']['settings.feedUrl']['vDEF'] ?? null; + if (is_string($url)) { + $urls[] = $url; + } + } + } + $urls = array_map(static fn(string $url): string => trim(stripslashes($url)), [...$urls, ...$additional]); + return array_values(array_unique(array_filter($urls, static fn(string $url): bool => $url !== ''))); + } +} diff --git a/Classes/Service/FeedDataService.php b/Classes/Service/FeedDataService.php index 5b19a140..22133bec 100644 --- a/Classes/Service/FeedDataService.php +++ b/Classes/Service/FeedDataService.php @@ -49,18 +49,25 @@ public function __construct( */ public function buildData(array $settings): array { - $data = [ - 'settings' => $settings, - ]; - if (!SimplePieDeprecationHandler::run( fn(): bool => $this->initializer->initializeFeed($this->feed, $settings) )) { - return $data; + return ['settings' => $settings]; } - $this->appendFeedProperties($data, $settings); - $this->appendItemProperties($data, $settings); + return $this->buildInitializedData($this->feed, $settings); + } + + /** + * @param FeedSettings $settings + * @return FeedData + */ + public function buildInitializedData(SimplePie $feed, array $settings): array + { + $data = ['settings' => $settings]; + + $this->appendFeedProperties($feed, $data, $settings); + $this->appendItemProperties($feed, $data, $settings); return $data; } @@ -69,7 +76,7 @@ public function buildData(array $settings): array * @param FeedSettings $settings * @param FeedData $data */ - private function appendFeedProperties(array &$data, array $settings): void + private function appendFeedProperties(SimplePie $feed, array &$data, array $settings): void { $getFeedFields = GeneralUtility::trimExplode(',', (string)($settings['getFields']['feed'] ?? '')); foreach ($getFeedFields as $getFeedField) { @@ -79,7 +86,7 @@ private function appendFeedProperties(array &$data, array $settings): void $fieldParts = GeneralUtility::trimExplode('|', $getFeedField); $field = GeneralUtility::underscoredToLowerCamelCase($fieldParts[0]); - $value = $this->getFeedFieldValue($getFeedField, $fieldParts); + $value = $this->getFeedFieldValue($feed, $getFeedField, $fieldParts); if ($getFeedField === 'image_url') { $data['feed']['image'] = $this->getImage($value); @@ -95,19 +102,19 @@ private function appendFeedProperties(array &$data, array $settings): void * @param FeedSettings $settings * @param FeedData $data */ - private function appendItemProperties(array &$data, array $settings): void + private function appendItemProperties(SimplePie $feed, array &$data, array $settings): void { $maxFeedCount = (int)($settings['maxFeedCount'] ?? 0); $getItemsFields = GeneralUtility::trimExplode(',', (string)($settings['getFields']['items'] ?? '')); /** @var list $items */ $items = SimplePieDeprecationHandler::run( - fn(): array => $this->feed->get_items(0, $maxFeedCount) + fn(): array => $feed->get_items(0, $maxFeedCount) ); foreach ($items as $item) { $itemProperties = $this->buildNormalizedItemProperties($item, $getItemsFields); - $itemProperties = $this->dispatchSingleFeedDataEvent($itemProperties, $item, $settings); + $itemProperties = $this->dispatchSingleFeedDataEvent($feed, $itemProperties, $item, $settings); if ($itemProperties !== []) { $data['items'][] = $itemProperties; @@ -118,12 +125,12 @@ private function appendItemProperties(array &$data, array $settings): void /** * @param list $fieldParts */ - private function getFeedFieldValue(string $getFeedField, array $fieldParts): mixed + private function getFeedFieldValue(SimplePie $feed, string $getFeedField, array $fieldParts): mixed { $fieldName = $fieldParts[0] ?? $getFeedField; if ($fieldName === 'subscribe_url') { - return $this->callSimplePieGetter($this->feed->subscribe_url(...)); + return $this->callSimplePieGetter($feed->subscribe_url(...)); } // Avoid calling SimplePie's deprecated favicon handling for legacy configurations. @@ -131,7 +138,7 @@ private function getFeedFieldValue(string $getFeedField, array $fieldParts): mix return null; } - return $this->getValue($this->feed, $fieldParts); + return $this->getValue($feed, $fieldParts); } /** @@ -164,10 +171,10 @@ private function buildNormalizedItemProperties(Item $item, array $getItemsFields * @param array $itemProperties * @return array */ - private function dispatchSingleFeedDataEvent(array $itemProperties, Item $item, array $settings): array + private function dispatchSingleFeedDataEvent(SimplePie $feed, array $itemProperties, Item $item, array $settings): array { $event = $this->eventDispatcher->dispatch( - new SingleFeedDataEvent($itemProperties, $item, $settings, $this->feed) + new SingleFeedDataEvent($itemProperties, $item, $settings, $feed) ); /** @var SingleFeedDataEvent $event */ $normalizedProps = SimplePieDeprecationHandler::run( @@ -225,16 +232,16 @@ private function getImage(mixed $fileUrl): ?string } $temporaryFileName = md5($filename) . '.' . $extension; + $tempFilePath = 'typo3temp/assets/images/' . $temporaryFileName; + $absoluteTempFilePath = Environment::getPublicPath() . '/' . $tempFilePath; + if (@is_file($absoluteTempFilePath)) { + return $tempFilePath; + } $content = GeneralUtility::getUrl($fileUrl); if ($content === false) { return null; } - - $tempFilePath = 'typo3temp/assets/images/' . $temporaryFileName; - $absoluteTempFilePath = Environment::getPublicPath() . '/' . $tempFilePath; - if (!@is_file($absoluteTempFilePath)) { - GeneralUtility::writeFileToTypo3tempDir($absoluteTempFilePath, $content); - } + GeneralUtility::writeFileToTypo3tempDir($absoluteTempFilePath, $content); return $tempFilePath; } diff --git a/Classes/Service/FeedFetcher.php b/Classes/Service/FeedFetcher.php new file mode 100644 index 00000000..921af9be --- /dev/null +++ b/Classes/Service/FeedFetcher.php @@ -0,0 +1,88 @@ + Misc::fix_protocol($url, 1)); + if (Misc::is_remote_uri($target)) { + $url = $target; + $recorder = new FeedHttpRecorder( + $this->clientFactory->getClient(), + $bounded ? $this->timeout('timeout') : null, + $bounded ? $this->timeout('connect_timeout') : null, + $bounded ? FeedHttpRecorder::MAX_BODY_SIZE : null, + ); + } + if ($bounded) { + // Scheduled fetches read one direct source, which keeps their limits predictable. + $feed->set_autodiscovery_level(SimplePie::LOCATOR_NONE); + if ($recorder === null && (!is_file($url) || filesize($url) > FeedHttpRecorder::MAX_BODY_SIZE)) { + throw new \RuntimeException('The local feed source must be a file of at most 2 MiB.'); + } + } + $initialized = SimplePieDeprecationHandler::run( + fn(): bool => $this->initializer->initializeFeed($feed, ['feedUrl' => $url], $recorder) + ); + if (!$initialized) { + $error = $feed->error() ?? ''; + if (is_array($error)) { + $error = implode('; ', $error); + } + throw new \RuntimeException($error !== '' ? $error : 'The source did not return a usable feed.'); + } + $snapshot = $feed->getSnapshot(); + // Every stored byte must have passed the recorder's size accounting. + if ($recorder !== null && !$recorder->received($snapshot['response']['body'])) { + throw new \RuntimeException('The selected feed is not part of the recorded responses.'); + } + $document = new FeedDocument(['version' => 2, 'kind' => 'snapshot', 'snapshot' => $snapshot]); + return new FetchedFeed($feed, $document); + } + + public function restore(FeedDocument $document): SimplePie + { + $feed = new FeedParser($document->payload['snapshot']); + if (!SimplePieDeprecationHandler::run($feed->init(...))) { + throw new \UnexpectedValueException('The stored feed could not be parsed.'); + } + return $feed; + } + + private function timeout(string $key): float + { + $configured = (float)($GLOBALS['TYPO3_CONF_VARS']['HTTP'][$key] ?? 0); + return $configured > 0 ? min($configured, 10.0) : 10.0; + } +} diff --git a/Classes/Service/FeedHttpRecorder.php b/Classes/Service/FeedHttpRecorder.php new file mode 100644 index 00000000..2faa5235 --- /dev/null +++ b/Classes/Service/FeedHttpRecorder.php @@ -0,0 +1,92 @@ +>, body: string} + */ +final class FeedHttpRecorder implements ClientInterface +{ + public const MAX_BODY_SIZE = 2 * 1024 * 1024; + private const HEADERS = ['content-type', 'content-encoding', 'location', 'link', 'content-language']; + + /** @var list */ + private array $steps = []; + private int $bodySize = 0; + + public function __construct( + private readonly GuzzleClientInterface $client, + private readonly ?float $timeout = null, + private readonly ?float $connectTimeout = null, + private readonly ?int $maxBodySize = self::MAX_BODY_SIZE, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $uri = (string)$request->getUri(); + if ($request->getMethod() !== 'GET' || !in_array(strtolower($request->getUri()->getScheme()), ['http', 'https'], true)) { + throw new RequestException('Expected an HTTP(S) GET feed request.', $request); + } + + $options = ['allow_redirects' => false, 'http_errors' => false]; + if ($this->timeout !== null) { + $options['timeout'] = $this->timeout; + $options['connect_timeout'] = $this->connectTimeout ?? $this->timeout; + } + $response = $this->client->send($request, $options); + $stream = $response->getBody(); + try { + if ($stream->isSeekable()) { + $stream->rewind(); + } + $body = Utils::copyToString($stream, $this->maxBodySize === null ? -1 : $this->maxBodySize - $this->bodySize + 1); + } finally { + $stream->close(); + } + $this->bodySize += strlen($body); + if ($this->maxBodySize !== null && $this->bodySize > $this->maxBodySize) { + throw new RequestException('The feed responses exceed the 2 MiB response limit.', $request); + } + + $headers = []; + foreach ($response->getHeaders() as $name => $values) { + if (in_array(strtolower($name), self::HEADERS, true)) { + $headers[$name] = array_values($values); + } + } + + $status = $response->getStatusCode(); + $this->steps[] = ['uri' => $uri, 'status' => $status, 'headers' => $headers, 'body' => $body]; + + return new Response($status, $headers, $body); + } + + public function received(string $body): bool + { + return in_array($body, array_column($this->steps, 'body'), true); + } +} diff --git a/Classes/Service/FeedParser.php b/Classes/Service/FeedParser.php new file mode 100644 index 00000000..c2225763 --- /dev/null +++ b/Classes/Service/FeedParser.php @@ -0,0 +1,76 @@ +>, body: string} + * @phpstan-type SourceSnapshot array{response: SourceResponse, mime: string} + */ +final class FeedParser extends SimplePie +{ + /** @param SourceSnapshot|null $snapshot */ + public function __construct(private ?array $snapshot = null) + { + parent::__construct(); + if ($snapshot !== null) { + $this->feed_url = $snapshot['response']['uri']; + $this->permanent_url = $snapshot['response']['base']; + $this->enable_cache(false); + } + } + + /** @return SourceSnapshot */ + public function getSnapshot(): array + { + return $this->snapshot ?? throw new \LogicException('No successfully fetched parser input.'); + } + + /** + * @param \SimplePie\Cache\Base|\SimplePie\Cache\DataCache|false $cache + * @return array{array, string}|bool + */ + protected function fetch_data(&$cache) + { + if ($this->snapshot === null) { + $result = parent::fetch_data($cache); + if (is_array($result)) { + $this->snapshot = [ + 'response' => [ + 'uri' => $this->feed_url, + 'base' => $this->permanent_url ?? $this->feed_url, + 'status' => $this->status_code, + 'headers' => array_map(static fn(string $value): array => [$value], $result[0]), + 'body' => (string)$this->get_raw_data(), + ], + 'mime' => $result[1], + ]; + } + return $result; + } + + $response = $this->snapshot['response']; + $this->raw_data = $response['body']; + $this->feed_url = $response['uri']; + $this->permanent_url = $response['base']; + $this->status_code = $response['status']; + return [array_map(static fn(array $values): string => implode(', ', $values), $response['headers']), $this->snapshot['mime']]; + } +} diff --git a/Classes/Service/FeedProvider.php b/Classes/Service/FeedProvider.php new file mode 100644 index 00000000..e6cd5122 --- /dev/null +++ b/Classes/Service/FeedProvider.php @@ -0,0 +1,243 @@ + $settings + * @return array + */ + public function getData(array $settings, bool $scheduler = false): array + { + return $this->provide($settings, $scheduler)->data; + } + + /** @param array $settings */ + public function provide(array $settings, bool $scheduler = false): ProvidedFeedData + { + $url = trim(stripslashes((string)($settings['feedUrl'] ?? ''))); + $duration = (int)($settings['cacheDuration'] ?? 0); + $identifier = 'feeddisplay_' . md5(serialize($settings)); + $cached = $this->readCache($identifier, $settings); + if ($url === '') { + return new ProvidedFeedData(['settings' => $settings]); + } + if (!$scheduler && $duration > 0 && $cached !== false + && is_string($cached[self::REVISION] ?? null) && str_starts_with($cached[self::REVISION], 'unstored:') + ) { + // Persistence failed: retain the successful display result for its original TTL. + return new ProvidedFeedData($this->displayData($cached)); + } + try { + $stored = $this->storage->findState($url); + } catch (\Throwable $exception) { + $this->logger->warning('Could not read the persistent feed. Apply the extension database schema.', ['url' => $url, 'exception' => $exception]); + if (!$scheduler && $duration > 0 && $cached !== false) { + return new ProvidedFeedData($this->displayData($cached)); + } + $fetched = $scheduler ? null : $this->fetchWithoutStorage($url); + if ($fetched === null) { + return new ProvidedFeedData($this->cacheData($identifier, ['settings' => $settings], '', $duration)); + } + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler); + } + // A pre-upgrade cache hit remains usable until its original TTL expires. + if (!$scheduler && $duration > 0 && $cached !== false + && !array_key_exists(self::REVISION, $cached) && !($stored->hasDocument ?? false) + ) { + return new ProvidedFeedData($cached); + } + if ($stored !== null && $stored->invalidationPending) { + try { + $this->refresher->invalidate($stored); + } catch (\Throwable $exception) { + $this->logger->warning('Could not invalidate feed pages; the operation remains pending.', ['url' => $url, 'exception' => $exception]); + } + } + $now = (int)$this->context->getPropertyFromAspect('date', 'timestamp'); + $lastAttempt = $stored->lastAttemptAt ?? 0; + $needsFetch = $scheduler + ? !($stored->hasDocument ?? false) && ($lastAttempt === 0 || $lastAttempt + 60 <= $now) + : $duration <= 0 || $lastAttempt === 0 || $lastAttempt + $duration <= $now; + $fetched = null; + $attempted = false; + // The second pass repairs an unusable stored document, at most once per request. + while (true) { + if ($needsFetch) { + $attempted = true; + $result = $this->refresher->refresh($url, $scheduler); + $fetched = $result->fetched; + if ($result->status === 'failed' && $fetched !== null && !$result->persisted) { + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler); + } + try { + // Re-read after a busy lock, too: another worker may have finished. + $stored = $this->storage->findState($url); + } catch (\Throwable $exception) { + $this->logger->warning('Could not read the refreshed feed.', ['url' => $url, 'exception' => $exception]); + if ($fetched !== null) { + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler, $result->persisted); + } + $stored = null; + } + if ($fetched !== null && !($stored->hasDocument ?? false)) { + return $this->fetchedData($identifier, $fetched, $settings, $duration, $scheduler, $result->persisted); + } + if (!$scheduler && $result->status === 'locked' && !($stored->hasDocument ?? false)) { + $fresh = $this->fetchWithoutStorage($url); + if ($fresh !== null) { + return new ProvidedFeedData($this->dataService->buildInitializedData($fresh->feed, $settings)); + } + } + } + if ($duration > 0 && $cached !== false && $stored !== null + && $stored->hasDocument && ($cached[self::REVISION] ?? null) === $stored->revision + ) { + return new ProvidedFeedData($this->displayData($cached), $stored->revision); + } + $feed = $this->restoreFeed($url, $stored, $fetched); + if ($feed !== null && $stored !== null) { + // Public events and getters remain outside transport/storage catches. + $data = $this->dataService->buildInitializedData($feed, $settings); + return new ProvidedFeedData($this->cacheData($identifier, $data, $stored->revision, $duration), $stored->revision); + } + $lastAttempt = $stored->lastAttemptAt ?? 0; + if ($attempted || !($stored->hasDocument ?? false) || ($lastAttempt !== 0 && $lastAttempt + 60 > $now)) { + break; + } + // A successful revision whose payload cannot be used follows bootstrap retry rules. + $needsFetch = true; + } + if (!$scheduler && $duration > 0 && $cached !== false && (isset($cached['feed']) || isset($cached['items']))) { + return new ProvidedFeedData($this->displayData($cached)); + } + return new ProvidedFeedData($this->cacheData($identifier, ['settings' => $settings], '', $duration)); + } + + private function restoreFeed(string $url, ?StoredFeed &$stored, ?FetchedFeed $fetched): ?SimplePie + { + try { + // Retry a concurrently replaced revision once, never spin in the frontend. + for ($attempt = 0; $attempt < 2 && $stored !== null && $stored->hasDocument; ++$attempt) { + if ($fetched !== null && $fetched->document->revision() === $stored->revision) { + return $fetched->feed; + } + $document = $this->storage->loadDocument($url, $stored->revision); + if ($document !== null) { + return $this->fetcher->restore($document); + } + $current = $this->storage->findState($url); + if ($current?->revision === $stored->revision) { + return null; + } + $stored = $current; + } + } catch (\Throwable $exception) { + $this->logger->warning('Could not replay the persistent feed.', ['url' => $url, 'exception' => $exception]); + } + return null; + } + + /** @param array $settings */ + private function fetchedData(string $identifier, FetchedFeed $fetched, array $settings, int $duration, bool $scheduler, bool $persisted = false): ProvidedFeedData + { + $data = $this->dataService->buildInitializedData($fetched->feed, $settings); + $revision = $fetched->document->revision(); + $cacheRevision = $persisted ? $revision : 'unstored:' . $revision; + if (!$scheduler) { + $data = $this->cacheData($identifier, $data, $cacheRevision, $duration); + } + return new ProvidedFeedData($data, $persisted ? $revision : null); + } + + private function fetchWithoutStorage(string $url): ?FetchedFeed + { + try { + return $this->fetcher->fetch($url); + } catch (\Throwable $exception) { + $this->logger->warning('Could not fetch the live feed.', ['url' => $url, 'exception' => $exception]); + return null; + } + } + + /** + * @param array $settings + * @return array|false + */ + private function readCache(string $identifier, array $settings): array|false + { + try { + $cached = $this->cache->get($identifier); + return is_array($cached) && ($cached['settings'] ?? null) === $settings ? $cached : false; + } catch (\Throwable $exception) { + $this->logger->warning('Could not read the feed display cache.', ['exception' => $exception]); + return false; + } + } + + /** + * @param array $data + * @return array + */ + private function cacheData(string $identifier, array $data, string $revision, int $duration): array + { + try { + if ($duration > 0) { + $this->cache->set($identifier, $data + [self::REVISION => $revision], [], $duration); + } else { + $this->cache->remove($identifier); + } + } catch (\Throwable $exception) { + $this->logger->warning('Could not update the feed display cache.', ['exception' => $exception]); + } + return $data; + } + + /** + * @param array $cached + * @return array + */ + private function displayData(array $cached): array + { + unset($cached[self::REVISION]); + return $cached; + } +} diff --git a/Classes/Service/FeedRefresher.php b/Classes/Service/FeedRefresher.php new file mode 100644 index 00000000..9ebffa37 --- /dev/null +++ b/Classes/Service/FeedRefresher.php @@ -0,0 +1,123 @@ +withLock($url, 'refresh', function () use ($url, $bounded): RefreshResult { + $now = (int)$this->context->getPropertyFromAspect('date', 'timestamp'); + try { + $fetched = $this->fetcher->fetch($url, $bounded); + } catch (\Throwable $exception) { + try { + $this->storage->storeFailure($url, $exception->getMessage(), $now); + } catch (\Throwable $storageException) { + $this->logger->warning('Could not record feed fetch failure.', ['url' => $url, 'exception' => $storageException]); + } + throw $exception; + } + $persisted = false; + try { + $changed = $this->storage->storeSuccess($url, $fetched->document, $now); + $persisted = true; + $stored = $this->storage->findState($url); + if ($stored !== null) { + $this->invalidate($stored); + } + } catch (\Throwable $exception) { + // Keep the fresh parser usable for this request even when persistence fails. + return new RefreshResult('failed', $fetched, $exception, $persisted); + } + return new RefreshResult($changed ? 'updated' : 'unchanged', $fetched, persisted: true); + }); + } + + public function invalidate(StoredFeed $stored): void + { + if (!$stored->invalidationPending) { + return; + } + $this->cacheManager->flushCachesInGroupByTag('pages', FeedStorage::cacheTag($stored->url)); + $this->storage->markInvalidated($stored->url, $stored->revision); + } + + public function remove(string $url): RefreshResult + { + return $this->withLock($url, 'remove', function () use ($url): RefreshResult { + $this->storage->remove($url); + // Also retry this after a previous deletion whose invalidation failed. + $this->cacheManager->flushCachesInGroupByTag('pages', FeedStorage::cacheTag($url)); + return new RefreshResult('updated'); + }); + } + + /** @param callable(): RefreshResult $operation */ + private function withLock(string $url, string $name, callable $operation): RefreshResult + { + $lock = null; + $acquired = false; + $result = new RefreshResult('locked'); + $mode = LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + try { + $lock = $this->lockFactory->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + try { + $acquired = $lock->acquire($mode); + } catch (LockAcquireWouldBlockException) { + $acquired = false; + } + if ($acquired) { + $result = $operation(); + } + } catch (\Throwable $exception) { + $result = new RefreshResult('failed', null, $exception); + } finally { + if ($lock !== null && $acquired) { + try { + $lock->release(); + } catch (\Throwable $exception) { + $result = new RefreshResult('failed', $result->fetched, $exception, $result->persisted); + } + } + } + if ($result->error !== null) { + $this->logger->warning('Feed source operation failed.', ['url' => $url, 'operation' => $name, 'exception' => $result->error]); + } + return $result; + } +} diff --git a/Classes/Service/FeedRuntimeInitializer.php b/Classes/Service/FeedRuntimeInitializer.php index 15fa2308..383c28ad 100644 --- a/Classes/Service/FeedRuntimeInitializer.php +++ b/Classes/Service/FeedRuntimeInitializer.php @@ -17,6 +17,7 @@ namespace ErHaWeb\FeedDisplay\Service; +use Psr\Http\Client\ClientInterface; use Psr\Http\Message\RequestFactoryInterface; use Psr\Http\Message\UriFactoryInterface; use SimplePie\SimplePie; @@ -37,7 +38,7 @@ public function __construct( /** * @param FeedSettings $settings */ - public function initializeFeed(SimplePie $feed, array $settings): bool + public function initializeFeed(SimplePie $feed, array $settings, ?ClientInterface $httpClient = null): bool { $feedUrl = $settings['feedUrl'] ?? ''; if ($feedUrl === '') { @@ -47,15 +48,13 @@ public function initializeFeed(SimplePie $feed, array $settings): bool $feedUrl = stripslashes((string)$feedUrl); $feed->set_feed_url($feedUrl); if ($this->shouldUseTypo3HttpClient($feedUrl)) { - $httpClient = $this->guzzleClientFactory->getClient(); + $httpClient ??= $this->guzzleClientFactory->getClient(); assert($httpClient instanceof \Psr\Http\Client\ClientInterface); $feed->set_http_client($httpClient, $this->requestFactory, $this->uriFactory); } $feed->enable_cache(false); // Feed autodiscovery can hit SimplePie's IRI handling before getters run. - SimplePieDeprecationHandler::run($feed->init(...)); - - return true; + return SimplePieDeprecationHandler::run($feed->init(...)) === true; } private function shouldUseTypo3HttpClient(string $feedUrl): bool diff --git a/Classes/Service/FetchedFeed.php b/Classes/Service/FetchedFeed.php new file mode 100644 index 00000000..cef4de44 --- /dev/null +++ b/Classes/Service/FetchedFeed.php @@ -0,0 +1,27 @@ + $data */ + public function __construct(public array $data, public ?string $revision = null) {} +} diff --git a/Classes/Service/RefreshResult.php b/Classes/Service/RefreshResult.php new file mode 100644 index 00000000..02ae1866 --- /dev/null +++ b/Classes/Service/RefreshResult.php @@ -0,0 +1,30 @@ +payload); + } + + public function revision(): string + { + return hash('sha256', $this->encode()); + } + + public static function decode(string $bytes): self + { + // Corrupt internal data is rejected below; never instantiate serialized objects. + $value = @unserialize($bytes, ['allowed_classes' => false]); + if (!is_array($value) || ($value['version'] ?? null) !== 2 || ($value['kind'] ?? null) !== 'snapshot') { + throw new \UnexpectedValueException('Unsupported stored feed format.'); + } + $snapshot = $value['snapshot'] ?? null; + if (!is_array($snapshot) || !is_string($snapshot['mime'] ?? null)) { + throw new \UnexpectedValueException('Invalid stored parser input.'); + } + return new self(['version' => 2, 'kind' => 'snapshot', 'snapshot' => [ + 'response' => self::decodeResponse($snapshot['response'] ?? null), + 'mime' => $snapshot['mime'], + ]]); + } + + /** @return SourceResponse */ + private static function decodeResponse(mixed $value): array + { + if (!is_array($value) || !is_string($value['uri'] ?? null) || !is_string($value['base'] ?? null) + || !is_int($value['status'] ?? null) || !is_string($value['body'] ?? null) || !self::validHeaders($value['headers'] ?? null) + ) { + throw new \UnexpectedValueException('Invalid stored source response.'); + } + // Validate PSR-7 header and status semantics, not just PHP types. + new Response($value['status'], $value['headers'], $value['body']); + return [ + 'uri' => $value['uri'], + 'base' => $value['base'], + 'status' => $value['status'], + 'headers' => $value['headers'], + 'body' => $value['body'], + ]; + } + + /** @phpstan-assert-if-true array> $headers */ + private static function validHeaders(mixed $headers): bool + { + if (!is_array($headers)) { + return false; + } + foreach ($headers as $name => $values) { + if (!is_string($name) || !is_array($values) || !array_is_list($values)) { + return false; + } + foreach ($values as $value) { + if (!is_string($value)) { + return false; + } + } + } + return true; + } +} diff --git a/Classes/Storage/FeedStorage.php b/Classes/Storage/FeedStorage.php new file mode 100644 index 00000000..f8e85f8b --- /dev/null +++ b/Classes/Storage/FeedStorage.php @@ -0,0 +1,174 @@ +connection()->select(['revision'], self::TABLE, ['url_hash' => $urlHash])->fetchOne(); + return $revision === false ? null : (string)$revision; + } + + public function find(string $url): ?StoredFeed + { + $row = $this->connection()->select(['*'], self::TABLE, ['url_hash' => self::hashUrl($url)])->fetchAssociative(); + if (!is_array($row)) { + return null; + } + $document = (string)$row['revision'] === '' ? null : $this->decodeDocument($url, (string)$row['revision'], $row['response_data']); + return $this->stateFromRow($row, $document); + } + + public function findState(string $url): ?StoredFeed + { + $row = $this->connection()->select( + ['url', 'revision', 'fetched_at', 'last_attempt_at', 'last_error', 'invalidation_pending'], + self::TABLE, + ['url_hash' => self::hashUrl($url)], + )->fetchAssociative(); + return is_array($row) ? $this->stateFromRow($row) : null; + } + + public function loadDocument(string $url, string $revision): ?FeedDocument + { + $row = $this->connection()->select(['response_data'], self::TABLE, [ + 'url_hash' => self::hashUrl($url), 'revision' => $revision, + ])->fetchAssociative(); + // A different revision is a concurrent replacement, not corrupt data. + return is_array($row) ? $this->decodeDocument($url, $revision, $row['response_data']) : null; + } + + private function decodeDocument(string $url, string $revision, mixed $bytes): ?FeedDocument + { + try { + if (is_resource($bytes)) { + $bytes = stream_get_contents($bytes); + } + if (!is_string($bytes) || !hash_equals($revision, hash('sha256', $bytes))) { + throw new \UnexpectedValueException('Stored feed checksum mismatch.'); + } + return FeedDocument::decode($bytes); + } catch (\Throwable $exception) { + $this->logger->warning('Could not decode the stored feed.', ['url' => $url, 'exception' => $exception]); + return null; + } + } + + /** @param array $row */ + private function stateFromRow(array $row, ?FeedDocument $document = null): StoredFeed + { + return new StoredFeed( + (string)$row['url'], + $document, + (string)$row['revision'], + (int)$row['fetched_at'], + (int)$row['last_attempt_at'], + (string)$row['last_error'], + (bool)$row['invalidation_pending'], + ); + } + + public function storeSuccess(string $url, FeedDocument $document, int $now): bool + { + $previous = $this->find($url); + $revision = $document->revision(); + $changed = $previous === null || $previous->document === null || $previous->revision !== $revision; + $fields = ['fetched_at' => $now, 'last_attempt_at' => $now, 'last_error' => '']; + if ($changed) { + $fields += ['response_data' => $document->encode(), 'revision' => $revision, 'invalidation_pending' => 1]; + } + $this->write($url, $fields, $previous !== null); + return $changed; + } + + public function storeFailure(string $url, string $error, int $now): void + { + $this->write($url, [ + 'last_attempt_at' => $now, + 'last_error' => mb_convert_encoding($error, 'UTF-8', 'UTF-8'), + ], $this->findState($url) !== null); + } + + public function markInvalidated(string $url, string $revision): void + { + // Never clear a pending invalidation belonging to a concurrently newer revision. + $this->connection()->update(self::TABLE, ['invalidation_pending' => 0], [ + 'url_hash' => self::hashUrl($url), 'revision' => $revision, + ]); + } + + /** @return list */ + public function findUrls(): array + { + return array_map(strval(...), $this->connection()->select(['url'], self::TABLE, [], [], ['uid' => 'ASC'])->fetchFirstColumn()); + } + + public function remove(string $url): void + { + $this->connection()->delete(self::TABLE, ['url_hash' => self::hashUrl($url)]); + } + + /** @param array $fields */ + private function write(string $url, array $fields, bool $exists): void + { + $connection = $this->connection(); + $identifier = ['url_hash' => self::hashUrl($url)]; + $types = isset($fields['response_data']) ? ['response_data' => Connection::PARAM_LOB] : []; + if ($exists) { + // All successful-response fields change in one statement; no delete/insert gap. + $connection->update(self::TABLE, $fields, $identifier, $types); + return; + } + try { + $connection->insert(self::TABLE, $identifier + ['url' => $url] + $fields, $types); + } catch (UniqueConstraintViolationException) { + $connection->update(self::TABLE, $fields, $identifier, $types); + } + } + + private function connection(): Connection + { + return $this->connectionPool->getConnectionForTable(self::TABLE); + } +} diff --git a/Classes/Storage/StoredFeed.php b/Classes/Storage/StoredFeed.php new file mode 100644 index 00000000..0eccfe42 --- /dev/null +++ b/Classes/Storage/StoredFeed.php @@ -0,0 +1,37 @@ +hasDocument = $revision !== ''; + } +} diff --git a/Configuration/FlexForms/FeedDisplay.xml b/Configuration/FlexForms/FeedDisplay.xml index 0c40cbeb..7d8d2041 100644 --- a/Configuration/FlexForms/FeedDisplay.xml +++ b/Configuration/FlexForms/FeedDisplay.xml @@ -226,6 +226,28 @@ number + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.fetchMode.description + + select + selectSingle + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.placeholder.typoscript + + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.fetchMode.live + live + + + LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:flexform.fetchMode.scheduler + scheduler + + + + diff --git a/Configuration/Services.yaml b/Configuration/Services.yaml index 392b4ca6..266b1a85 100644 --- a/Configuration/Services.yaml +++ b/Configuration/Services.yaml @@ -8,10 +8,6 @@ services: resource: '../Classes/*' exclude: '../Classes/Domain/Model/*' - ErHaWeb\FeedDisplay\Controller\FeedController: - arguments: - $cache: '@cache.feeddisplay' - # Register the external dependency explicitly so FeedDataService can autowire it. SimplePie\SimplePie: ~ diff --git a/Configuration/Sets/FeedDisplay/constants.typoscript b/Configuration/Sets/FeedDisplay/constants.typoscript index 5c2831e7..86e89726 100644 --- a/Configuration/Sets/FeedDisplay/constants.typoscript +++ b/Configuration/Sets/FeedDisplay/constants.typoscript @@ -40,6 +40,8 @@ plugin.tx_feeddisplay_pi1 { errorMessage = # cat=plugin.tx_feeddisplay_pi1/200_advanced/340; type=int+; label=LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:constants.settings.cacheDuration cacheDuration = 3600 + # cat=plugin.tx_feeddisplay_pi1/200_advanced/345; type=options[live,scheduler]; label=LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:constants.settings.fetchMode + fetchMode = live getFields { # Each field is internally converted to a SimplePie getter method. @@ -95,5 +97,5 @@ plugin.tx_feeddisplay_pi1 { } # cat=plugin.tx_feeddisplay_pi1/200_advanced/350; type=string; label=LLL:EXT:feed_display/Resources/Private/Language/locallang_be.xlf:constants.settings.ignoreFlexFormSettingsIfEmpty - ignoreFlexFormSettingsIfEmpty = feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,getFields.feed,getFields.items + ignoreFlexFormSettingsIfEmpty = feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,fetchMode,getFields.feed,getFields.items } diff --git a/Configuration/Sets/FeedDisplay/labels.xlf b/Configuration/Sets/FeedDisplay/labels.xlf index 5f21770f..dd44d0cb 100644 --- a/Configuration/Sets/FeedDisplay/labels.xlf +++ b/Configuration/Sets/FeedDisplay/labels.xlf @@ -123,6 +123,12 @@ Time in seconds in which the data is to be read from the cache until the next renewal of the cache. (0 = no cache) + + Fetch mode + + + Fetch the feed during page rendering or through the refresh command. + Ignore FlexForm Settings if empty diff --git a/Configuration/Sets/FeedDisplay/settings.definitions.yaml b/Configuration/Sets/FeedDisplay/settings.definitions.yaml index c6be1954..3eeaf647 100644 --- a/Configuration/Sets/FeedDisplay/settings.definitions.yaml +++ b/Configuration/Sets/FeedDisplay/settings.definitions.yaml @@ -24,7 +24,7 @@ settings: category: feeddisplay.templates plugin.tx_feeddisplay_pi1.settings.ignoreFlexFormSettingsIfEmpty: type: string - default: 'feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,getFields.feed,getFields.items' + default: 'feedUrl,maxFeedCount,maxContentLength,maxHeaderLength,logoMaxWidth,logoMaxHeight,feedIconMaxWidth,feedIconMaxHeight,dateFormat,stripTags,linkTarget,errorMessage,cacheDuration,fetchMode,getFields.feed,getFields.items' category: feeddisplay.general plugin.tx_feeddisplay_pi1.settings.feedUrl: type: string @@ -78,6 +78,13 @@ settings: type: int default: 3600 category: feeddisplay.advanced + plugin.tx_feeddisplay_pi1.settings.fetchMode: + type: string + default: 'live' + enum: + live: 'Live' + scheduler: 'Scheduler' + category: feeddisplay.advanced plugin.tx_feeddisplay_pi1.settings.getFields.feed: type: string default: 'author,authors,contributor,contributors,copyright,description,encoding,items,item_quantity,language,link,links,permalink,title,type,subscribe_url,latitude,longitude,image_height,image_link,image_title,image_url,image_width,all_discovered_feeds,base' diff --git a/Configuration/Sets/FeedDisplay/setup.typoscript b/Configuration/Sets/FeedDisplay/setup.typoscript index 42fd6171..0cf02825 100644 --- a/Configuration/Sets/FeedDisplay/setup.typoscript +++ b/Configuration/Sets/FeedDisplay/setup.typoscript @@ -35,6 +35,7 @@ plugin.tx_feeddisplay_pi1 { linkTarget = {$plugin.tx_feeddisplay_pi1.settings.linkTarget} errorMessage = {$plugin.tx_feeddisplay_pi1.settings.errorMessage} cacheDuration = {$plugin.tx_feeddisplay_pi1.settings.cacheDuration} + fetchMode = {$plugin.tx_feeddisplay_pi1.settings.fetchMode} } ignoreFlexFormSettingsIfEmpty = {$plugin.tx_feeddisplay_pi1.ignoreFlexFormSettingsIfEmpty} diff --git a/Documentation/Configuration/Index.rst b/Documentation/Configuration/Index.rst index 8bcae675..ba9a5285 100644 --- a/Documentation/Configuration/Index.rst +++ b/Documentation/Configuration/Index.rst @@ -12,5 +12,6 @@ Configuration SiteSet TypoScriptReference/Index + Scheduler TSconfig Events diff --git a/Documentation/Configuration/Scheduler.rst b/Documentation/Configuration/Scheduler.rst new file mode 100644 index 00000000..4637cd9a --- /dev/null +++ b/Documentation/Configuration/Scheduler.rst @@ -0,0 +1,198 @@ +.. include:: /Includes.rst.txt + +.. _configuration-scheduler: + +Persistent feeds and scheduled refresh +===================================== + +Last successful response +------------------------ + +Feed Display stores the last successfully parsed document for each source URL +in ``tx_feeddisplay_feed``. It has no expiry and survives every TYPO3 cache +flush. Only a successful fetch replaces it; a valid RSS or Atom feed without +items also counts as success. Download, parse and storage errors preserve the +previous response. Include this table in normal database backups. + +The source is shared by plugins using the same URL. Display options and prepared +arrays remain separate per plugin configuration. A refresh by one plugin or the +command updates that shared source for all plugins. The spelling of the URL +matters: redirects do not merge two different configured URLs. + +The TYPO3 ``feeddisplay`` cache contains only disposable display data. Flushing +it rebuilds these arrays from the database; it does not force a download or +delete the persistent response. Use ``feeddisplay:refresh`` to force a refresh. +Only scheduler-mode plugins tag their pages for source invalidation. A new +revision saved by a live fetch or the command invalidates those scheduler pages. +Live pages retain their normal TYPO3 cache lifetime and are not invalidated by +feed changes. ``cacheDuration`` does not disable the TYPO3 page cache. + +Scheduler pages also carry the source revision actually used during rendering. +After TYPO3 writes the page cache entry, Feed Display compares those revisions +with the database. If a source changed or was removed while the page rendered, +the newly written entry is discarded. The running response can still show the +older content; the next request rebuilds it. Matching revisions keep the page's +normal lifetime. No additional source query runs on a page-cache hit. + +Avoid combining an uncached live plugin and a scheduler plugin for the same URL +on one page when the source changes on every fetch. If the scheduler plugin +renders first and the live plugin then stores a new revision, that page contains +two different source versions and is discarded after every rendering. Mixed +mode alone does not disable caching: a live plugin rendered first can update +the source before the scheduler plugin reads the same revision. + +A revision read failure is logged and leaves that source unverified; another +source with a confirmed mismatch can still cause the page to be discarded. +Failures removing an obsolete entry are also logged. This check concerns TYPO3's +page cache, not reverse-proxy, CDN or third-party static caches. + +Live mode (default) +------------------- + +Existing plugins remain in live mode without configuration changes. When the +plugin runs, a positive ``cacheDuration`` limits refresh attempts, including +retries after failure. Zero and negative durations attempt a new fetch on every +uncached rendering, still persist successes and fall back to the last success. +Failures do not change the last successful payload or its success timestamp. +There is no seven-day fallback deadline. If storing a fresh response fails, +positive cache durations retain its prepared display result for that duration. +This is temporary cache protection, not durable storage; zero and negative +durations continue attempting a fetch on each rendering. + +Protection starts with the first successful fetch after the update. Existing +3.0.2 display-cache entries remain usable for their original lifetime until a +persistent response exists; incomplete display arrays are never converted to +source documents. + +Scheduler mode +-------------- + +Select ``scheduler`` through the content element's Fetch mode field, Site +Settings, or TypoScript: + +.. code-block:: typoscript + + plugin.tx_feeddisplay_pi1.settings.fetchMode = scheduler + +In a site's ``settings.yaml`` use YAML syntax: + +.. code-block:: yaml + + plugin.tx_feeddisplay_pi1.settings.fetchMode: scheduler + +A nonempty FlexForm value overrides inherited configuration. Keep ``fetchMode`` +in ``ignoreFlexFormSettingsIfEmpty`` when an empty field should inherit. + +If there is no usable stored document, the first frontend request tries to fetch +one. A failed first attempt is retried no more than once every 60 seconds. +Without data, the configured error message is shown and the page's maximum cache +lifetime is 60 seconds. Once a usable response exists, frontend rendering uses +it regardless of age and does not fetch the feed document again. A recurring +job is required to update it. + +New documents retain the parser input selected by SimplePie, including response +bytes, headers, MIME type and original source/base URIs. Rebuilding display data +does not repeat autodiscovery and keeps the originally selected feed when a page +offers several. + +If a stored document cannot be reconstructed, the frontend attempts repair at +most once every 60 seconds under the same source lock and scheduler HTTP limits. +Failed repair preserves the stored payload. In scheduler mode, ``cacheDuration`` +controls only prepared arrays; zero and negative values do not trigger ordinary +frontend downloads. + +Run the refresh command +----------------------- + +From the deployed installation root: + +.. code-block:: bash + + # Composer + vendor/bin/typo3 feeddisplay:refresh + + # Classic + php typo3/sysext/core/bin/typo3 feeddisplay:refresh + +The command combines non-deleted live-workspace feed plugins' explicit FlexForm +URLs, all stored source rows, and additional repeatable ``--url`` arguments. +Legacy ``list_type = feeddisplay_pi1`` records are included. Hidden records are +included, deleted records and workspace versions are excluded from this scan. +Stored sources are intentionally retained even when no content element uses +them anymore, including failed first attempts. Both live and scheduler sources +are refreshed. There is no usage tracking or automatic pruning. + +It does not resolve full page TypoScript. A URL inherited through TypoScript or +Site Settings becomes discoverable after its first frontend attempt, or can be +provided explicitly: + +.. code-block:: bash + + vendor/bin/typo3 feeddisplay:refresh --url='https://example.org/rss' --url='https://example.org/atom' + +An empty source list succeeds with an explanatory message. ``updated`` means a +new document revision was saved; ``unchanged`` means a successful fetch renewed +freshness without changing the revision. ``locked`` means the source was skipped +because another operation holds its lock. A failed source is diagnosed while +other sources continue; the command exits unsuccessfully if any source fails. + +In EXT:scheduler create an **Execute console commands** task for +``feeddisplay:refresh``. No additional extension-specific Scheduler task is +necessary, and EXT:scheduler is optional. Ensure cron actually invokes TYPO3's +Scheduler. With automatic selection, no array option is needed. TYPO3's Scheduler +form splits array options on commas; for URLs containing literal commas use +cron with quoted, repeated ``--url`` arguments instead. + +Source types and operational limits +----------------------------------- + +HTTP(S) feeds retain TYPO3's HTTP configuration. Live fetches keep SimplePie's +feed autodiscovery. Scheduled fetches, including first frontend fetches in +scheduler mode, read the configured URL directly, so configure the feed itself +for this mode. The selected parser input is stored as binary-safe scalar +data and replayed with its original bytes, selected headers and URL context. +Local RSS/Atom files are also supported; +use absolute paths for jobs and ensure CLI and web PHP see the same filesystem +and run with equivalent read permissions. Relative paths use the process's +working directory; do not share relative sources between different directories. +Run jobs as the application user, never with broader filesystem privileges. + +Scheduled HTTP fetches, including first frontend fetches in scheduler mode, limit +recorded response bodies to 2 MiB in total and cap each HTTP request's configured +timeouts at ten seconds. Reading into PHP strings is bounded; the transport may +buffer data first. This is not a guaranteed network-download limit. Redirects +can cause several requests, so ten seconds is not a total source or command +deadline. Guzzle 7's PHP stream timeout can be an idle timeout. +These new limits are not applied to normal live fetches. A large live source can +therefore render successfully but fail in the refresh job. Local sources of +scheduled fetches must be regular files of at most 2 MiB. + +Database writes are atomic per response, and failed page invalidation remains +pending for a later retry, including an unchanged refresh. The default TYPO3 +locks coordinate processes on one host. Multi-node deployments need a shared +locking strategy and shared page-cache invalidation; the extension does not +introduce a distributed lock service. Replay of the feed document does not +prevent separate HTTP requests by custom getters or event listeners. Feed images +are reused while their file exists in ``typo3temp``. An absent image is downloaded +during rendering, including in scheduler mode. Clearing temporary files can +trigger this download again. + +Remove unused sources +--------------------- + +Only explicitly selected URLs are removed: + +.. code-block:: bash + + vendor/bin/typo3 feeddisplay:remove --url='https://example.org/unused.xml' + +For Classic use the Classic executable shown above. Repeat ``--url`` for several +sources. The command is intentionally unavailable in Scheduler selections. +It acquires the same source lock, deletes the stored row and invalidates tagged +scheduler pages even when the row is already missing. Repeating it is safe, +including after a failed page flush. A busy lock or failure is reported with a failing exit code. + +Remove remaining plugin configuration as well when retiring a source. Otherwise +a later frontend request or refresh scan can recreate it. Manual deletion is +not a permanent exclusion rule. There is no automatic deletion or retention +policy; administrators control when durable data may be removed. diff --git a/Documentation/Configuration/TypoScriptReference/Constants.rst b/Documentation/Configuration/TypoScriptReference/Constants.rst index 38449e1b..d0051f1d 100644 --- a/Documentation/Configuration/TypoScriptReference/Constants.rst +++ b/Documentation/Configuration/TypoScriptReference/Constants.rst @@ -257,8 +257,27 @@ Cache duration :Default: 3600 :Path: plugin.tx_feeddisplay_pi1.settings - Time in seconds in which the data is to be read from the - cache until the next renewal of the cache. (0 = no cache) + In live mode, the minimum interval between source refresh attempts and the + lifetime of prepared display arrays. Zero and negative values attempt a + refresh on every plugin execution. Successful responses remain stored + permanently even when this value is zero. In scheduler mode this value + affects prepared arrays only. TYPO3's page cache is independent. + +.. _configuration-typoscript-constants-settings-fetchmode: + +Fetch mode +~~~~~~~~~~ + +.. confval:: fetchMode + + :type: string + :Default: live + :Path: plugin.tx_feeddisplay_pi1.settings + + ``live`` refreshes during page rendering as needed. ``scheduler`` fetches a + missing source once, then displays its last successful response until a job + refreshes it. Unknown or empty values use live mode. See + :ref:`configuration-scheduler` for first-fetch retries and job setup. .. _configuration-typoscript-constants-settings-getfields: @@ -341,7 +360,7 @@ Ignore Flexform Settings if empty .. confval:: ignoreFlexFormSettingsIfEmpty :type: list of strings, separated by comma - :Default: feedUrl, maxFeedCount, maxContentLength, maxHeaderLength, logoMaxWidth, logoMaxHeight, feedIconMaxWidth, feedIconMaxHeight, dateFormat, stripTags, linkTarget, errorMessage, cacheDuration, getFields.feed, getFields.items + :Default: feedUrl, maxFeedCount, maxContentLength, maxHeaderLength, logoMaxWidth, logoMaxHeight, feedIconMaxWidth, feedIconMaxHeight, dateFormat, stripTags, linkTarget, errorMessage, cacheDuration, fetchMode, getFields.feed, getFields.items :Path: plugin.tx_feeddisplay_pi1 Comma separated list of settings to be overridden by TypoScript if the diff --git a/Documentation/Installation/Index.rst b/Documentation/Installation/Index.rst index fd0764c9..468b5157 100644 --- a/Documentation/Installation/Index.rst +++ b/Documentation/Installation/Index.rst @@ -24,9 +24,9 @@ TYPO3 v14 backend module paths. #. Apply database changes - This extension uses the caching framework to cache feed data and - plugin configuration. For this, the tables `cache_feeddisplay` - and `cache_feeddisplay_tags` must be created. + Apply the regular schema update, including ``tx_feeddisplay_feed`` + for durable responses and ``cache_feeddisplay`` / + ``cache_feeddisplay_tags`` for disposable display data. These tables can be created in the Maintenance module under :guilabel:`Analyze Database Structure` → :guilabel:`Apply selected changes`. @@ -53,9 +53,9 @@ TYPO3 v14 backend module paths. #. Apply database changes - This extension uses the caching framework to cache feed data and - plugin configuration. For this, the tables `cache_feeddisplay` - and `cache_feeddisplay_tags` must be created. + Apply the regular schema update, including ``tx_feeddisplay_feed`` + for durable responses and ``cache_feeddisplay`` / + ``cache_feeddisplay_tags`` for disposable display data. These tables can be created in the Maintenance module under :guilabel:`Analyze Database Structure` → :guilabel:`Apply selected changes`. @@ -121,3 +121,22 @@ Composer installations obtain SimplePie through their normal project dependency and do not require this additional build. Classic installations receive library updates with a new extension release; the extension never downloads libraries during a request. + +Updating from 3.0.2 +================== + +Apply the normal database schema update after replacing the extension files, +for example with ``vendor/bin/typo3 extension:setup`` (Composer) or +``php typo3/sysext/core/bin/typo3 extension:setup`` (Classic). This creates +``tx_feeddisplay_feed``. Rebuild deployment/DI caches as usual. Existing plugins +keep live mode; no Scheduler job or new plugin setting is required. + +Without the table, live rendering falls back to fetching and its existing +display cache, and logs a schema warning. Scheduler rendering cannot persist a +first response and shows its configured error message. Complete the schema +update to enable durable protection. No tables are created during requests. + +There is no upgrade wizard for converting old display-cache entries: they lack +the complete feed document. Protection begins with the first successful fetch +on the new version. See :ref:`configuration-scheduler` for storage and refresh +semantics. Existing Site Set dependencies remain unchanged. diff --git a/Documentation/KnownProblems/Index.rst b/Documentation/KnownProblems/Index.rst index f6aa6ddd..7c1e3067 100644 --- a/Documentation/KnownProblems/Index.rst +++ b/Documentation/KnownProblems/Index.rst @@ -6,31 +6,219 @@ Known problems ============== -The following issues are known problems. However those are not fixable inside `EXT:feed_display`. +Check the effective settings of the affected content element, the TYPO3 log and +the refresh command's output. A feed that opens in a desktop browser can still +fail on the application server. Persistent fallback can conceal an outage. -If you notice anything else feel free to open an issue on `Github `__. +.. _known-problems-scheduler-empty: + +Scheduler mode has no output or shows the error message +======================================================= + +Selecting ``fetchMode = scheduler`` does not create a job. With no stored +response, the first frontend rendering attempts a fetch. On failure the plugin +shows its configured ``errorMessage`` or the default error and limits the page +cache to 60 seconds. Further initial attempts are throttled for 60 seconds, +including across cache flushes. A valid stored response has no expiry and +survives every cache flush. + +#. Apply the normal schema update: ``tx_feeddisplay_feed`` must exist. Without + it, scheduler rendering cannot store a first response. Check the log for + the schema warning and run ``extension:setup`` in the deployed installation. +#. Verify the effective ``feedUrl`` and ``fetchMode``, including FlexForm + overrides. Check that custom templates actually display the error message. +#. Run the command from the installation root with the same URL: + + .. code-block:: bash + + # Composer + vendor/bin/typo3 feeddisplay:refresh --url='https://example.org/feed.xml' + + # Classic + php typo3/sysext/core/bin/typo3 feeddisplay:refresh --url='https://example.org/feed.xml' + +#. Expect ``updated`` or ``unchanged``. Investigate failures in the command + output and TYPO3 log, then configure recurring execution. Saving an + **Execute console commands** task alone does not start TYPO3's Scheduler. + +See :ref:`configuration-scheduler` for first-fetch and refresh behavior. A URL +inherited only through TypoScript or Site Settings is not discovered by the +command until its first frontend attempt, unless passed explicitly with +``--url``. It does not resolve whole pages' TypoScript. + +A job succeeds but the plugin remains stale +=========================================== + +* **Different source spellings:** URL hashes distinguish schemes, host spelling, + query order, trailing slashes and encoding. Redirects do not merge source + rows. Use the same trimmed URL in the plugin and command. +* **Different environments:** CLI and web must use the same database, deployed + code and filesystem. Do not invoke the extension's test installation by + mistake. In DDEV run the command inside the web container. Match proxy, + credentials, DNS and TLS configuration between CLI and web. +* **Wrong job or inactive cron:** Confirm that Scheduler runs and invokes the + intended command. A scheduler-mode plugin never refreshes an existing + usable successful response by itself, regardless of ``cacheDuration``. An + unusable document is an exception: see the replay guidance below. +* **Multiple nodes or external caches:** Host-local TYPO3 locks do not + coordinate nodes. Configure shared locks and page invalidation across the + deployment. CDNs and reverse proxies may need their own invalidation. +* **Changing metadata:** A revision includes response bytes and selected + headers, not only visible items. A changing build date or redirect can + invalidate scheduler pages even when visible content looks identical. Live + pages are not tagged and keep their usual lifetime, even with a zero or + negative feed cache duration. + +The refresh command fails, skips sources or has no sources +========================================================== + +* **Command unavailable:** Verify extension activation, complete files and + rebuilt DI caches. Classic Git checkouts need their library build; TER + packages already include it. See :ref:`installation`. +* **No sources found:** An empty set succeeds. The command uses explicit + FlexForm URLs, stored source rows and additional ``--url`` arguments. It + excludes deleted records and non-live workspace versions, but includes + hidden content and legacy feed plugin records. Load inherited URLs once in + the frontend or provide them explicitly. +* **Retired or mistyped sources keep failing:** Stored rows, including failed + first attempts, remain selected indefinitely. Both live and scheduler + sources are included. Correct the configuration and remove unused rows + with ``feeddisplay:remove``; nothing is automatically pruned. +* **Array options and commas:** Repeat quoted ``--url`` options on the command + line. The Scheduler form splits array options at commas. Automatic source + selection avoids this form issue; use cron when an explicit URL contains a + literal comma. Percent-encoding only helps if the source supports it and + the plugin uses exactly the same spelling. +* **A source is busy:** A result of ``locked`` means skipped because another + operation holds its lock. It is not an HTTP failure. Other sources continue. + Repeated skips suggest overlapping or hanging processes. Check those + processes before scheduling more frequent runs. +* **Oversized or slow response:** Scheduled HTTP fetches, including first + frontend fetches in that mode, cap stored response bodies at 2 MiB in total and + each HTTP request's timeouts at ten seconds. Redirects can cause several + requests; this is not a total time or transport download + limit. Normal live fetches retain their previous limits. A source that + works live can therefore fail in the job. +* **HTTP or parse error:** Login pages, bot challenges, invalid XML, redirect + loops and intermittent 4xx/5xx responses can fail. Live fetches keep HTTP + feed autodiscovery. Scheduled fetches read the configured URL directly, so + configure the feed itself for that mode. +* **Storage, locks or invalidation fail:** Check the concrete exception and URL + in the TYPO3 log, database permissions/capacity and filesystem permissions. + Failed writes preserve the previous response; fresh live data can still + appear in that request. With a positive live ``cacheDuration``, its prepared + result is cached for that duration despite the write error; a cache flush + removes this temporary protection. Zero and negative values keep fetching. + Failed page invalidation remains pending and is retried, including after + an unchanged refresh. + +A mixed run continues after a failed source but returns a failure status overall. +Do not assume that no source updated just because the task is marked failed. .. _known-problems-no-feed-items-displayed: No feed items are displayed =========================== -**Question:** +* **No successful response:** The configured error message is shown when no + usable source exists. Live mode can continue using its display cache while + the new schema is missing, but permanent protection requires the table. +* **The feed is empty:** A valid RSS/Atom feed without items is a successful + replacement, not an error. The no-items message can appear instead of the + fetch-error message. +* **Server access differs from browser access:** Check application-server DNS, + firewall, proxy, authentication and certificate trust through TYPO3's HTTP + configuration. Disabling TLS verification is not a correction for missing + certificates. +* **Local file inaccessible:** Use a path visible inside the PHP container, + including readable symlink targets. For jobs use absolute paths and the + application user with equivalent read permissions. Relative paths depend + on the process's working directory and can point to different files in CLI. +* **Customization suppresses output:** Check getter fields, ``maxFeedCount``, + templates, conditions and ``SingleFeedDataEvent`` listeners. Listener errors + propagate normally; they are not disguised as feed download errors. +* **Broken stored document:** A valid prepared display-cache hit remains usable + without reading the BLOB. On a miss, checksum, structure and header errors + are diagnosed. New documents restore the selected parser input without + repeating discovery. If reconstruction fails, the + frontend attempts repair no more than once every 60 seconds; failed repair + keeps the payload and success time. A manual refresh can replace the document + immediately. No SimplePie PHP objects are persisted. + +Images trigger requests +======================= + +Feed images are downloaded only when their temporary file is missing. Rendering +can therefore still make an image HTTP request in scheduler mode, after a +temporary-file cleanup or when an image is used for the first time. Custom +getters and event listeners may also make their own requests. + +The plugin is missing or displays the example feed +================================================== + +* Include the ``erhaweb/feed-display`` Site Set in the active site's + dependencies, or static TypoScript for a template-record setup. Check the + actual site's rootline. See :ref:`configuration-site-set`. +* Replace the default ``https://typo3.org/rss`` URL in the relevant settings. + A saved FlexForm can override the value you changed site-wide. +* Check content/page visibility, language, access restrictions and the rendered + column. These can prevent plugin execution entirely. +* For legacy ``CType = list`` / ``list_type = feeddisplay_pi1`` records, run + ``feedDisplayCTypeMigration`` when required. The current content type is + ``feeddisplay_pi1``. Also check backend-group permissions for editors. -I get the message "Sorry, no items could be fetched." in the frontend although the feed is available when I call the -feed URL directly. What could be the cause of this problem? +Settings seem ignored +====================== -**Answer:** +* Inspect the content element's FlexForm as well as Site Settings and later + TypoScript assignments. Use YAML ``fetchMode: scheduler`` in site settings + and TypoScript ``fetchMode = scheduler`` in setup, with the full plugin path. +* ``ignoreFlexFormSettingsIfEmpty`` belongs to the plugin, not ``settings``. + Keep ``fetchMode`` in that list for inheritance. An unignored empty or + unknown mode uses live behavior. +* The default ignore list can treat FlexForm ``0`` as inherited. To explicitly + select zero, adjust that setting's inheritance rule or the inherited value. +* ``getFields.feed`` and ``getFields.items`` select SimplePie getters. Check + exact names, supported arguments and the data structure expected by custom + templates. Cropping/HTML settings affect display, not the stored document. -Please verify that the feed is indeed being delivered reliably. I know of cases where the feed seemed to work initially, -but an error 500 occurred on some calls. If this error occurs sporadically, it could be that the memory size on the -system delivering the feed has been exceeded. +Content is stale or changes at unexpected times +================================================ -In this context, please note that the Feed Display extension itself caches the return. This means that in case of doubt -a faulty feed may persist for a longer period of time until the TYPO3 cache is cleared again, although a direct call to -the feed URL will work again. +* A positive live ``cacheDuration`` limits attempts per shared source, + including after failures. Different plugin options do not keep independent + old copies of the feed. A plugin with a shorter interval can update all + plugins of that URL. +* Cache flushes rebuild display arrays from the persistent source without + forcing a download. Use ``feeddisplay:refresh`` to force an attempt. The + last successful response has no expiry; monitor failures in logs/jobs. +* ``cacheDuration = 0`` or a negative value does not disable TYPO3 page caching. + A cached page may avoid plugin execution altogether. Live page-cache + lifetimes remain unchanged; live pages are not invalidated by source changes. + Only scheduler pages carry source tags. A successful changed revision from + either a live plugin or the job invalidates those scheduler pages. +* A scheduler page whose source changes or is removed during rendering is + discarded immediately after TYPO3 writes its cache entry. The running + response can still contain the older revision; the next request rebuilds it. + If checking a source revision fails, the warning identifies its hash and the + page-cache identifier. That source remains unverified. A failed removal is + diagnosed as well. Check database/cache availability; reverse-proxy, CDN + and third-party static caches require their own handling. +* Mixed live/scheduler plugins of the same URL can prevent effective page + caching. In particular, if a scheduler plugin renders before a live plugin + with ``cacheDuration <= 0`` and every fetch changes the source revision, + every rendering creates an inconsistent page that is discarded. Use one + mode for that source on the page. Mixed mode alone is not sufficient to + cause this: live first, followed by scheduler using the same revision, can + remain cacheable. +* Existing 3.0.2 display entries remain valid for their original lifetime until + the first durable response exists. They cannot protect against a full + cache flush before that first successful fetch. +* Removing a source does not disable configured plugins. Another frontend + attempt or refresh scan can recreate it. Remove its configuration as well + when retiring it. ``feeddisplay:remove`` requires explicit URLs, is safe to + repeat and reports lock conflicts or failed invalidation for manual retry. -If the feed is loaded from a remote ``http`` or ``https`` URL, also verify the -TYPO3 HTTP client configuration of the instance. Proxy requirements and -outbound network restrictions apply through TYPO3's PSR-17/PSR-18 interfaces, -so a blocked or incomplete TYPO3 HTTP setup can prevent feed retrieval. +If a problem remains, report its effective configuration, TYPO3/PHP versions, +command output and relevant log entry in the +`issue tracker `__. diff --git a/README.md b/README.md index ffb2d421..65f4fe60 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,10 @@ to Fluid templates. Remote `http` and `https` feeds are fetched through TYPO3's PSR-17/PSR-18 HTTP client, so instance-wide HTTP settings such as proxy configuration are applied -automatically. Parsed results are cached through TYPO3's caching framework and -refreshed when plugin configuration changes or the configured cache lifetime -expires. +automatically. The last successful response per source is stored permanently +in the database and survives TYPO3 cache flushes. Prepared display data stays +separate per plugin configuration. Live refresh is the default; an optional +`feeddisplay:refresh` job can keep scheduler-mode sources current. ## At a glance @@ -36,7 +37,8 @@ expires. - Configurable feed and item fields for Fluid templates. - Site Set support for TYPO3 v13 and v14, with a static TypoScript fallback. - Content-element FlexForm settings plus Site Settings and TypoScript constants. -- Dedicated cache for parsed feed data and plugin configuration. +- Durable last-successful responses with disposable display caches. +- Optional scheduled refresh and explicit removal of unused sources. ## Installation @@ -44,6 +46,16 @@ expires. composer require erhaweb/feed-display ``` +After installation or an update from 3.0.2, apply the regular database schema +update (`vendor/bin/typo3 extension:setup` in Composer installations). Existing +plugins remain in live mode; no Scheduler job is required. Durable protection +starts with the first successful fetch. Cache flushes rebuild display arrays +without forcing a download. Use `feeddisplay:refresh` to refresh sources and +`feeddisplay:remove --url='…'` to remove explicitly selected ones. Configured +sources can be created again afterwards. + +See [persistent feeds and Scheduler setup](Documentation/Configuration/Scheduler.rst). + Continue with the [Composer installation guide](https://docs.typo3.org/p/erhaweb/feed-display/main/en-us/Installation/) and the diff --git a/Resources/Private/Language/locallang_be.xlf b/Resources/Private/Language/locallang_be.xlf index 9ad9ec9f..76f920f8 100644 --- a/Resources/Private/Language/locallang_be.xlf +++ b/Resources/Private/Language/locallang_be.xlf @@ -118,7 +118,19 @@ Feed cache - Time in seconds in which the data is to be read from the cache until the next renewal of the cache. (0 = no cache, empty = Defined by TypoScript) + Minimum interval between live refresh attempts and lifetime of prepared display data, in seconds. Zero or negative: refresh on each plugin execution. Successful responses remain stored permanently. In scheduler mode this controls display caching only. Empty: inherit TypoScript. + + + Fetch mode + + + Live refreshes the shared feed source as needed during page rendering and retains the last successful response permanently. Scheduler loads a missing feed once during page rendering; afterwards only feeddisplay:refresh updates it. Configure a regular refresh job. Failed first attempts are retried after 60 seconds. Empty: inherit TypoScript or Site Set settings. + + + Live + + + Scheduler Defined by Feed or TypoScript (default) @@ -367,7 +379,10 @@ Strip tags:Remove HTML tags from the feed content - Feed cache:Time in seconds in which the data is to be read from the cache until the next renewal of the cache. (0 = no cache) + Feed cache:Live refresh interval and display cache lifetime in seconds. Zero or negative bypasses these intervals, but successful responses remain stored permanently. + + + Fetch mode:Fetch the feed during page rendering or through the refresh command. Ignore FlexForm Settings if empty:Comma separated list of FlexForm settings that will be ignored in the extension settings merge process, if their value is considered empty (either an empty string or a string containing 0). diff --git a/Tests/Fixtures/StoredDocument.php b/Tests/Fixtures/StoredDocument.php new file mode 100644 index 00000000..901932ed --- /dev/null +++ b/Tests/Fixtures/StoredDocument.php @@ -0,0 +1,31 @@ + 2, 'kind' => 'snapshot', 'snapshot' => [ + 'response' => ['uri' => $uri, 'base' => $uri, 'status' => 200, 'headers' => ['content-type' => ['application/rss+xml']], 'body' => $body], + 'mime' => 'application/rss+xml', + ]]); + } +} diff --git a/Tests/Functional/Fixtures/FeedHttp.php b/Tests/Functional/Fixtures/FeedHttp.php new file mode 100644 index 00000000..6cf24e7b --- /dev/null +++ b/Tests/Functional/Fixtures/FeedHttp.php @@ -0,0 +1,59 @@ + */ + public static array $requests = []; + /** @var array */ + public static array $responses = []; + /** @var array */ + public static array $beforeResponse = []; + + public static function middleware(callable $next): callable + { + return static function (RequestInterface $request, array $options) { + $uri = (string)$request->getUri(); + self::$requests[] = $uri; + $callback = self::$beforeResponse[$uri] ?? null; + unset(self::$beforeResponse[$uri]); + $callback?->__invoke(); + $response = self::$responses[$uri] ?? new Response(503, [], 'Unavailable'); + // The recorder closes consumed streams. Every request needs a fresh body. + return Create::promiseFor(new Response($response->getStatusCode(), $response->getHeaders(), (string)$response->getBody())); + }; + } + + public static function rss(string $title, string $extra = ''): Response + { + return new Response(200, ['Content-Type' => 'application/rss+xml'], + '' . $title . 'https://example.test/Test' + . $extra . 'Itemhttps://example.test/item'); + } + + public static function countRequests(string $url): int + { + return count(array_filter(self::$requests, static fn(string $request): bool => $request === $url)); + } +} diff --git a/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php b/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php new file mode 100644 index 00000000..69d38afc --- /dev/null +++ b/Tests/Functional/Frontend/AbstractHttpFeedTestCase.php @@ -0,0 +1,38 @@ +configurationToUseInTestInstance['SYS']['caching']['cacheConfigurations']['pages']['backend'] = Typo3DatabaseBackend::class; + parent::setUp(); + FeedHttp::$requests = []; + FeedHttp::$responses = []; + FeedHttp::$beforeResponse = []; + // Frontend Bootstrap::init() reloads HTTP configuration from the test instance. + file_put_contents($this->instancePath . '/typo3conf/system/additional.php', 'requestPage(); self::assertSame(2, substr_count($body, 'class="item-title"')); - self::assertStringContainsString('Updated feed title', $body); + self::assertStringContainsString('Original feed title', $body); self::assertStringContainsString('First item', $body); self::assertStringContainsString('Second item', $body); @@ -68,6 +68,9 @@ public function flexFormOverridesTypoScriptAndSelectsSeparateCacheEntries(): voi self::assertSame(1, substr_count($body, 'class="item-title"')); self::assertStringContainsString('Original feed title', $body); self::assertStringNotContainsString('Updated feed title', $body); + // Both settings use one source; a refresh invalidates both display revisions. + $this->get(\ErHaWeb\FeedDisplay\Service\FeedRefresher::class)->refresh($feedUrl); + self::assertStringContainsString('Updated feed title', $this->requestPage()); } #[Test] @@ -146,13 +149,13 @@ public function feedImageGetsPublishedToTemporaryAssets(): void #[Test] public function invalidFeedDisplaysTheDefaultErrorMessage(): void { - $this->initializeFrontendRootPage($this->instancePath . '/fileadmin/missing-feed.xml', [ - 'getFields.feed' => '', - ]); + $this->initializeFrontendRootPage($this->instancePath . '/fileadmin/missing-feed.xml'); $body = $this->requestPage(); self::assertStringContainsString('Sorry, the feed could not be fetched.', $body); + self::assertStringNotContainsString('class="feed-title"', $body); + self::assertStringNotContainsString('class="feed-items"', $body); } #[Test] diff --git a/Tests/Functional/Frontend/PersistentFeedTest.php b/Tests/Functional/Frontend/PersistentFeedTest.php new file mode 100644 index 00000000..5ecc20e2 --- /dev/null +++ b/Tests/Functional/Frontend/PersistentFeedTest.php @@ -0,0 +1,123 @@ +writeFeedFixture('Persisted title', ['Original item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + self::assertStringContainsString('Persisted title', $this->requestPage()); + $storage = $this->get(FeedStorage::class); + $stored = $storage->find($url); + self::assertNotNull($stored); + self::assertNotNull($stored->document); + unlink($url); + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['last_attempt_at' => 0], ['url_hash' => FeedStorage::hashUrl($url)]); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('Original item', $this->requestPage()); + self::assertSame($stored->revision, $storage->find($url)?->revision); + self::assertSame($stored->fetchedAt, $storage->find($url)->fetchedAt); + self::assertNotSame('', $storage->find($url)->lastError); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('Original item', $this->requestPage()); + } + + #[Test] + #[DataProvider('uncachedDurations')] + public function uncachedLiveFetchesStillPersistAndFallBack(int $duration): void + { + $url = $this->writeFeedFixture('Initial', ['Original item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => $duration]); + self::assertStringContainsString('Initial', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + $this->writeFeedFixture('Updated', []); + self::assertStringContainsString('Updated', $this->requestPage()); + unlink($url); + $body = $this->requestPage(); + self::assertStringContainsString('Updated', $body); + self::assertStringNotContainsString('Original item', $body); + } + + /** @return iterable */ + public static function uncachedDurations(): iterable + { + yield 'zero' => [0]; + yield 'negative' => [-1]; + } + + #[Test] + public function legacyDisplayCacheRemainsUsableBeforeTheFirstNewFetch(): void + { + $url = $this->writeFeedFixture('Old cache', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + self::assertStringContainsString('Old cache', $this->requestPage()); + $cached = $this->getCachedFeedData(); + self::assertIsArray($cached); + unset($cached['_feeddisplay_revision']); + $this->setCachedFeedData($cached); + $this->get(FeedStorage::class)->remove($url); + $this->writeFeedFixture('New source', ['Item']); + self::assertStringContainsString('Old cache', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->find($url)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + self::assertStringContainsString('New source', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } + + #[Test] + public function liveRenderingKeepsWorkingWhenTheSchemaUpdateWasMissed(): void + { + $url = $this->writeFeedFixture('Without schema', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + try { + self::assertStringContainsString('Without schema', $this->requestPage()); + unlink($url); + self::assertStringContainsString('Without schema', $this->requestPage()); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function liveFirstRequestCanReadWithoutWritingWhileAnotherWorkerHoldsTheLock(): void + { + $url = $this->writeFeedFixture('Parallel live', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 600]); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + self::assertTrue($lock->acquire($mode)); + try { + self::assertStringContainsString('Parallel live', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->find($url)); + } finally { + $lock->release(); + } + self::assertStringContainsString('Parallel live', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } +} diff --git a/Tests/Functional/Frontend/PersistentHttpFeedTest.php b/Tests/Functional/Frontend/PersistentHttpFeedTest.php new file mode 100644 index 00000000..c7969601 --- /dev/null +++ b/Tests/Functional/Frontend/PersistentHttpFeedTest.php @@ -0,0 +1,258 @@ + 'text/html'], ''); + FeedHttp::$responses[$feedUrl] = FeedHttp::rss('DISCOVERED_ORIGINAL'); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 3600]); + self::assertStringContainsString('DISCOVERED_ORIGINAL', $this->requestPage()); + $before = $this->get(FeedStorage::class)->find(self::URL); + self::assertNotNull($before?->document); + FeedHttp::$responses = []; + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('DISCOVERED_ORIGINAL', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertSame(1, FeedHttp::countRequests($feedUrl)); + self::assertSame($before->revision, $this->get(FeedStorage::class)->find(self::URL)?->revision); + } + + #[Test] + public function changedLiveSourcesLeaveExistingLivePagesCached(): void + { + FeedHttp::$responses[self::URL] = FeedHttp::rss('Original'); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 3600]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nconfig.cache_period = 7200"); + self::assertStringContainsString('Original', $this->requestPage()); + $pages = $this->getConnectionPool()->getConnectionForTable('cache_pages'); + $identifiers = $pages->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative(); + self::assertCount(1, $identifiers); + self::assertGreaterThan(time() + 3600, (int)$identifiers[0]['expires']); + foreach ([3600, 0, -1] as $duration) { + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['last_attempt_at' => 1], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Changed ' . $duration); + $data = $this->get(FeedProvider::class)->getData(['feedUrl' => self::URL, 'cacheDuration' => $duration, 'getFields' => ['feed' => 'title']]); + self::assertSame('Changed ' . $duration, $data['feed']['title']); + self::assertSame($identifiers, $pages->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative()); + self::assertStringContainsString('Original', $this->requestPage()); + } + } + + #[Test] + public function validDisplayCacheDoesNotReadOrDecodeACorruptBlob(): void + { + FeedHttp::$responses[self::URL] = FeedHttp::rss('Cached success'); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 600]); + self::assertStringContainsString('Cached success', $this->requestPage()); + $connection = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $connection->update(FeedStorage::TABLE, ['response_data' => 'corrupt'], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + self::assertStringContainsString('Cached success', $this->requestPage()); + self::assertStringContainsString('Cached success', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + // A miss diagnoses and repairs the unusable document despite a positive live TTL. + $now = (int)$this->get(Context::class)->getPropertyFromAspect('date', 'timestamp'); + $connection->update(FeedStorage::TABLE, ['last_attempt_at' => $now - 61], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Repaired'); + self::assertStringContainsString('Repaired', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } + + #[Test] + #[DataProvider('writeFailureCases')] + public function failedWritesKeepTheSuccessfulDisplayResult(int $duration, bool $existing): void + { + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + if ($existing) { + $this->get(FeedStorage::class)->storeSuccess(self::URL, StoredDocument::create('OldTest'), 1); + } + $before = $this->get(FeedStorage::class)->find(self::URL); + $connection = $this->createMock(Connection::class); + $connection->method('select')->willReturnCallback($real->select(...)); + $connection->method('update')->willThrowException(new \RuntimeException('Simulated packet limit')); + $connection->method('insert')->willThrowException(new \RuntimeException('Simulated packet limit')); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $storage = new FeedStorage($pool, new NullLogger()); + $fetcher = $this->get(FeedFetcher::class); + $context = $this->get(Context::class); + $refresher = new FeedRefresher($storage, $fetcher, $this->get(LockFactory::class), $this->get(CacheManager::class), $context, new NullLogger()); + $provider = new FeedProvider($storage, $refresher, $fetcher, $this->get(FeedDataService::class), $this->get(CacheManager::class)->getCache('feeddisplay'), $context, new NullLogger()); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Fresh but not stored'); + for ($request = 0; $request < 3; ++$request) { + $data = $provider->getData(['feedUrl' => self::URL, 'cacheDuration' => $duration, 'getFields' => ['feed' => 'title']]); + self::assertSame('Fresh but not stored', $data['feed']['title']); + self::assertArrayNotHasKey('_feeddisplay_revision', $data); + } + self::assertSame($duration > 0 ? 1 : 3, FeedHttp::countRequests(self::URL)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + self::assertSame('Fresh but not stored', $provider->getData(['feedUrl' => self::URL, 'cacheDuration' => $duration, 'getFields' => ['feed' => 'title']])['feed']['title']); + self::assertSame($duration > 0 ? 2 : 4, FeedHttp::countRequests(self::URL)); + self::assertSame($before?->revision, $this->get(FeedStorage::class)->find(self::URL)?->revision); + self::assertSame($before?->lastAttemptAt, $this->get(FeedStorage::class)->find(self::URL)?->lastAttemptAt); + } + + /** @return iterable */ + public static function writeFailureCases(): iterable + { + foreach ([600, 0, -1] as $duration) { + foreach ([false, true] as $existing) { + yield $duration . ($existing ? ' old row' : ' no row') => [$duration, $existing]; + } + } + } + + #[Test] + public function missingSchemaStillCachesLiveErrorsForTheConfiguredDuration(): void + { + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 600]); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + try { + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + $this->get(CacheManager::class)->getCache('feeddisplay')->flush(); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function liveOutageKeepsTheStoredResponse(): void + { + $document = StoredDocument::create((string)FeedHttp::rss('Stored live')->getBody(), self::URL . '/'); + $storage = $this->get(FeedStorage::class); + $storage->storeSuccess(self::URL, $document, 1); + $storage->markInvalidated(self::URL, $document->revision()); + $this->initializeFrontendRootPage(self::URL, ['cacheDuration' => 3600]); + self::assertStringContainsString('Stored live', $this->requestPage()); + self::assertStringContainsString('Stored live', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + $after = $storage->findState(self::URL); + self::assertNotNull($after); + self::assertSame($document->revision(), $after->revision); + self::assertSame(1, $after->fetchedAt); + self::assertNotSame('', $after->lastError); + } + + #[Test] + #[DataProvider('readFailuresAfterPersistence')] + public function persistedResultsKeepTheirRevisionAndRetryInvalidation(string $failure): void + { + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $reads = 0; + $connection = self::createStub(Connection::class); + $connection->method('select')->willReturnCallback(static function (array $columns, string $table, array $where = []) use ($real, &$reads, $failure): \Doctrine\DBAL\Result { + if (in_array('last_attempt_at', $columns, true) && ++$reads === 3) { + if ($failure === 'exception') { + throw new \RuntimeException('Read failed after successful write'); + } + $empty = self::createStub(\Doctrine\DBAL\Result::class); + $empty->method('fetchAssociative')->willReturn(false); + return $empty; + } + return $real->select($columns, $table, $where); + }); + $connection->method('insert')->willReturnCallback($real->insert(...)); + $connection->method('update')->willReturnCallback($real->update(...)); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $storage = new FeedStorage($pool, new NullLogger()); + $flushes = 0; + $manager = $this->createMock(CacheManager::class); + $manager->expects(self::exactly(2))->method('flushCachesInGroupByTag')->willReturnCallback(static function () use (&$flushes): void { + if (++$flushes === 1) { + throw new \RuntimeException('Page invalidation temporarily unavailable'); + } + }); + $context = $this->get(Context::class); + $fetcher = $this->get(FeedFetcher::class); + $refresher = new FeedRefresher($storage, $fetcher, $this->get(LockFactory::class), $manager, $context, new NullLogger()); + $cache = $this->get(CacheManager::class)->getCache('feeddisplay'); + $provider = new FeedProvider($storage, $refresher, $fetcher, $this->get(FeedDataService::class), $cache, $context, new NullLogger()); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Persisted'); + $settings = ['feedUrl' => self::URL, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + self::assertSame('Persisted', $provider->getData($settings)['feed']['title']); + $state = $this->get(FeedStorage::class)->findState(self::URL); + self::assertNotNull($state); + self::assertTrue($state->invalidationPending); + $cached = $cache->get('feeddisplay_' . md5(serialize($settings))); + self::assertIsArray($cached); + self::assertSame($state->revision, $cached['_feeddisplay_revision']); + self::assertSame('Persisted', $provider->getData($settings)['feed']['title']); + self::assertFalse($this->get(FeedStorage::class)->findState(self::URL)?->invalidationPending); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + } + + /** @return iterable */ + public static function readFailuresAfterPersistence(): iterable + { + yield 'exception' => ['exception']; + yield 'missing state' => ['missing']; + } + + #[Test] + public function cacheHitsOnlyReadMetadataAndDoNotLogCorruptPayloads(): void + { + FeedHttp::$responses[self::URL] = FeedHttp::rss('Large'); + $settings = ['feedUrl' => self::URL, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + $this->get(FeedProvider::class)->getData($settings); + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $real->update(FeedStorage::TABLE, ['response_data' => 'corrupt'], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + $connection = $this->createMock(Connection::class); + $connection->expects(self::exactly(3))->method('select')->with( + ['url', 'revision', 'fetched_at', 'last_attempt_at', 'last_error', 'invalidation_pending'], + FeedStorage::TABLE, + ['url_hash' => FeedStorage::hashUrl(self::URL)], + )->willReturnCallback($real->select(...)); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $logger = $this->createMock(LoggerInterface::class); + $logger->expects(self::never())->method('warning'); + $storage = new FeedStorage($pool, $logger); + $provider = new FeedProvider($storage, $this->get(FeedRefresher::class), $this->get(FeedFetcher::class), $this->get(FeedDataService::class), $this->get(CacheManager::class)->getCache('feeddisplay'), $this->get(Context::class), $logger); + for ($i = 0; $i < 3; ++$i) { + self::assertSame('Large', $provider->getData($settings)['feed']['title']); + } + } +} diff --git a/Tests/Functional/Frontend/RemoveFeedPageCacheTest.php b/Tests/Functional/Frontend/RemoveFeedPageCacheTest.php new file mode 100644 index 00000000..83399939 --- /dev/null +++ b/Tests/Functional/Frontend/RemoveFeedPageCacheTest.php @@ -0,0 +1,53 @@ +' . $image . 'Ihttps://example.test/'); + FeedHttp::$responses[$image] = new Response(200, ['Content-Type' => 'image/svg+xml'], ''); + self::assertSame('updated', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nplugin.tx_feeddisplay_pi1.settings.getFields.feed = title,image_url"); + FeedHttp::$beforeResponse[$image] = function (): void { + $command = new CommandTester($this->get(RemoveFeedSourcesCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [self::URL]])); + }; + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->findState(self::URL)); + $pages = $this->getConnectionPool()->getConnectionForTable('cache_pages'); + self::assertSame(0, $pages->count('*', 'cache_pages', [])); + FeedHttp::$responses[self::URL] = FeedHttp::rss('RECREATED_SOURCE'); + self::assertStringContainsString('RECREATED_SOURCE', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->findState(self::URL)); + self::assertSame(1, $pages->count('*', 'cache_pages', [])); + } +} diff --git a/Tests/Functional/Frontend/RemoveFeedSourceTest.php b/Tests/Functional/Frontend/RemoveFeedSourceTest.php new file mode 100644 index 00000000..c5e59938 --- /dev/null +++ b/Tests/Functional/Frontend/RemoveFeedSourceTest.php @@ -0,0 +1,91 @@ +writeFeedFixture('Kept source', ['Item']); + $this->initializeFrontendRootPage($url, ['cacheDuration' => 0]); + self::assertStringContainsString('Kept source', $this->requestPage()); + $command = new CommandTester($this->get(RemoveFeedSourcesCommand::class)); + self::assertSame(Command::INVALID, $command->execute([])); + self::assertNotNull($this->get(FeedStorage::class)->find($url)); + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [$url]])); + self::assertNull($this->get(FeedStorage::class)->find($url)); + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [$url]])); + self::assertStringContainsString('Kept source', $this->requestPage()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } + + #[Test] + public function removalIsExcludedFromTheScheduler(): void + { + $commands = iterator_to_array($this->get(CommandRegistry::class)->getSchedulableCommands()); + self::assertArrayHasKey('feeddisplay:refresh', $commands); + self::assertArrayNotHasKey('feeddisplay:remove', $commands); + } + + #[Test] + public function missingSourceStillInvalidatesPagesOnEveryRetry(): void + { + $url = 'https://example.test/removed.xml'; + $calls = 0; + $cache = $this->createMock(CacheManager::class); + $cache->expects(self::exactly(2))->method('flushCachesInGroupByTag')->with('pages', FeedStorage::cacheTag($url))->willReturnCallback(static function () use (&$calls): void { + if (++$calls === 1) { + throw new \RuntimeException('Temporary cache failure'); + } + }); + $subject = new FeedRefresher($this->get(FeedStorage::class), $this->get(\ErHaWeb\FeedDisplay\Service\FeedFetcher::class), $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class), $cache, $this->get(\TYPO3\CMS\Core\Context\Context::class), new \Psr\Log\NullLogger()); + self::assertSame('failed', $subject->remove($url)->status); + self::assertSame('updated', $subject->remove($url)->status); + } + + #[Test] + public function busySourcesAreNotRemovedAndCanBeRetried(): void + { + $url = $this->writeFeedFixture('Locked source', ['Item']); + $this->initializeFrontendRootPage($url); + $this->requestPage(); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + self::assertTrue($lock->acquire($mode)); + $command = new CommandTester($this->get(RemoveFeedSourcesCommand::class)); + try { + self::assertSame(Command::FAILURE, $command->execute(['--url' => [$url]])); + self::assertStringContainsString('locked:', $command->getDisplay()); + self::assertNotNull($this->get(FeedStorage::class)->find($url)?->document); + } finally { + $lock->release(); + } + self::assertSame(Command::SUCCESS, $command->execute(['--url' => [$url]])); + self::assertNull($this->get(FeedStorage::class)->find($url)); + } +} diff --git a/Tests/Functional/Frontend/ScheduledFeedTest.php b/Tests/Functional/Frontend/ScheduledFeedTest.php new file mode 100644 index 00000000..53bb9978 --- /dev/null +++ b/Tests/Functional/Frontend/ScheduledFeedTest.php @@ -0,0 +1,234 @@ +configurationToUseInTestInstance['SYS']['caching']['cacheConfigurations']['pages']['backend'] = Typo3DatabaseBackend::class; + parent::setUp(); + } + + #[Test] + public function firstRequestBootstrapsAndLaterRequestsOnlyUseTheStoredSource(): void + { + $url = $this->writeFeedFixture('First response', ['First item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + self::assertStringContainsString('First response', $this->requestPage()); + $storage = $this->get(FeedStorage::class); + $first = $storage->find($url); + self::assertNotNull($first?->document); + $this->writeFeedFixture('New response', ['Second item']); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('First response', $this->requestPage()); + self::assertSame($first->revision, $storage->find($url)?->revision); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertStringContainsString('updated:', $command->getDisplay()); + self::assertStringContainsString('New response', $this->requestPage()); + unlink($url); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('New response', $this->requestPage()); + } + + #[Test] + public function failedBootstrapIsThrottledAndTheMissingPageExpiresWithinSixtySeconds(): void + { + $url = $this->instancePath . '/fileadmin/missing.xml'; + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler']); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'config.no_cache = 0'); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + $pages = $this->taggedPages($url); + self::assertCount(1, $pages); + $expires = $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['expires'], 'cache_pages', ['identifier' => $pages[0]])->fetchOne(); + self::assertLessThanOrEqual(time() + 60, (int)$expires); + $this->writeFeedXmlFixture('RecoveredTest', 'missing.xml'); + $this->get(CacheManager::class)->flushCaches(); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertSame([], $this->taggedPages($url)); + self::assertStringContainsString('Recovered', $this->requestPage()); + } + + #[Test] + public function commandDeduplicatesAllSourcesAndContinuesAfterOneFails(): void + { + $url = $this->writeFeedFixture('Selected', ['Item']); + $this->setFlexFormValues(['general' => ['settings.feedUrl' => $url]]); + $storage = $this->get(FeedStorage::class); + $storage->storeFailure($url, 'Previous failure', 1); + $missing = $this->instancePath . '/fileadmin/absent.xml'; + self::assertSame([$url, $missing], $this->get(ConfiguredFeedUrls::class)->collect([$missing, $url])); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::FAILURE, $command->execute(['--url' => [$missing, $url]])); + self::assertSame(1, substr_count($command->getDisplay(), 'updated:')); + self::assertStringContainsString('failed:', $command->getDisplay()); + self::assertNotNull($storage->find($url)?->document); + } + + #[Test] + public function changedSourceInvalidatesPagesWithoutShorteningTheirNormalLifetime(): void + { + $url = $this->writeFeedFixture('Original', ['Item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'cacheDuration' => -1]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nconfig.cache_period = 7200"); + self::assertStringContainsString('Original', $this->requestPage()); + $pages = $this->taggedPages($url); + self::assertCount(1, $pages); + $expires = $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['expires'], 'cache_pages', ['identifier' => $pages[0]])->fetchOne(); + self::assertGreaterThan(time() + 3600, (int)$expires, json_encode([$pages, $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative()], JSON_THROW_ON_ERROR)); + $this->writeFeedFixture('Changed', ['Item']); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertSame([], $this->taggedPages($url)); + self::assertStringContainsString('Changed', $this->requestPage()); + } + + #[Test] + public function emptySourceListIsSuccessful(): void + { + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + self::assertStringContainsString('No feed sources found', $command->getDisplay()); + } + + #[Test] + public function twoPluginsShareOneSourceButKeepSeparateDisplayArrays(): void + { + $url = $this->writeFeedFixture('Shared source', ['One', 'Two']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'cacheDuration' => 600, 'maxFeedCount' => 2]); + $this->setFlexFormValues(['general' => ['settings.maxFeedCount' => 1]]); + $connection = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $record = $connection->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($record); + $record['uid'] = self::CONTENT_ELEMENT_UID + 1; + $record['sorting'] = 256; + $record['pi_flexform'] = ''; + $connection->insert('tt_content', $record); + $body = $this->requestPage(); + self::assertSame(2, substr_count($body, 'Shared source')); + self::assertSame(3, substr_count($body, 'class="item-title"')); + self::assertSame(1, $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->count('*', FeedStorage::TABLE, [])); + self::assertSame(2, $this->getConnectionPool()->getConnectionForTable('cache_feeddisplay')->count('*', 'cache_feeddisplay', [])); + $this->writeFeedFixture('Shared update', ['Three', 'Four']); + $command = new CommandTester($this->get(RefreshFeedsCommand::class)); + self::assertSame(Command::SUCCESS, $command->execute([])); + $body = $this->requestPage(); + self::assertSame(2, substr_count($body, 'Shared update')); + self::assertSame(3, substr_count($body, 'class="item-title"')); + self::assertStringNotContainsString('Shared source', $body); + } + + #[Test] + public function schedulerWithoutTheSchemaShowsTheConfiguredError(): void + { + $url = $this->writeFeedFixture('Not fetched', ['Item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler', 'errorMessage' => 'Schema not ready']); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + try { + self::assertStringContainsString('Schema not ready', $this->requestPage()); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function busyFirstBootstrapCanBeRetriedAfterTheOtherWorkerReleasesItsLock(): void + { + $url = $this->writeFeedFixture('Available after lock', ['Item']); + $this->initializeFrontendRootPage($url, ['fetchMode' => 'scheduler']); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl($url), $mode); + self::assertTrue($lock->acquire($mode)); + try { + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertNull($this->get(FeedStorage::class)->find($url)); + } finally { + $lock->release(); + } + self::assertStringContainsString('Available after lock', $this->requestPage()); + } + + #[Test] + public function sourceSelectionIncludesHiddenContentButSkipsDeletedWorkspaceAndMalformedRecords(): void + { + $url = $this->instancePath . '/fileadmin/selected.xml'; + $this->setFlexFormValues(['advanced' => ['settings.feedUrl' => $url]]); + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $record = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($record); + $content->update('tt_content', ['hidden' => 1], ['uid' => self::CONTENT_ELEMENT_UID]); + foreach (['deleted' => 1, 't3ver_wsid' => 1] as $field => $value) { + ++$record['uid']; + $copy = $record; + $copy[$field] = $value; + $copy['pi_flexform'] = str_replace('selected.xml', $field . '.xml', (string)$copy['pi_flexform']); + $content->insert('tt_content', $copy); + } + ++$record['uid']; + $record['pi_flexform'] = 'insert('tt_content', $record); + $additional = 'https://example.test/feed?ids=1,2'; + self::assertSame([$url, $additional], $this->get(ConfiguredFeedUrls::class)->collect([' ' . $additional . ' ', $url, $additional])); + } + + #[Test] + #[DataProvider('emptyFlexForms')] + public function emptyFlexFormsAreIgnoredWhileMalformedXmlIsDiagnosed(?string $flexForm): void + { + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $logger = $this->createMock(LoggerInterface::class); + $logger->expects(self::once())->method('warning')->with('Could not read feed FlexForm.', self::anything()); + $collector = new ConfiguredFeedUrls($this->getConnectionPool(), $this->get(FeedStorage::class), $logger); + $content->update('tt_content', ['pi_flexform' => $flexForm], ['uid' => self::CONTENT_ELEMENT_UID]); + self::assertSame([], $collector->collect([])); + $content->update('tt_content', ['pi_flexform' => ' self::CONTENT_ELEMENT_UID]); + self::assertSame([], $collector->collect([])); + } + + /** @return iterable */ + public static function emptyFlexForms(): iterable + { + yield 'null' => [null]; + yield 'empty' => ['']; + yield 'whitespace' => [" \t\n"]; + } + + /** @return list */ + private function taggedPages(string $url): array + { + $backend = $this->get(CacheManager::class)->getCache('pages')->getBackend(); + self::assertInstanceOf(TaggableBackendInterface::class, $backend); + return array_values($backend->findIdentifiersByTag(FeedStorage::cacheTag($url))); + } +} diff --git a/Tests/Functional/Frontend/ScheduledHttpFeedTest.php b/Tests/Functional/Frontend/ScheduledHttpFeedTest.php new file mode 100644 index 00000000..ac3b7262 --- /dev/null +++ b/Tests/Functional/Frontend/ScheduledHttpFeedTest.php @@ -0,0 +1,121 @@ +initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler']); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'config.no_cache = 0'); + self::assertStringContainsString('Original', $this->requestPage()); + $backend = $this->get(CacheManager::class)->getCache('pages')->getBackend(); + self::assertInstanceOf(TaggableBackendInterface::class, $backend); + self::assertCount(1, $backend->findIdentifiersByTag(FeedStorage::cacheTag(self::URL))); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Live change'); + $this->get(FeedProvider::class)->getData(['feedUrl' => self::URL, 'cacheDuration' => 0, 'getFields' => ['feed' => 'title']]); + self::assertSame([], $backend->findIdentifiersByTag(FeedStorage::cacheTag(self::URL))); + self::assertStringContainsString('Live change', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } + + #[Test] + public function unreplayableDocumentIsRepairedOnce(): void + { + $this->storeHttpDocument(self::URL, 'No XML feed'); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Repaired'); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 600]); + self::assertStringContainsString('Repaired', $this->requestPage()); + self::assertStringContainsString('Repaired', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertGreaterThan(1, $this->get(FeedStorage::class)->findState(self::URL)?->lastAttemptAt); + } + + #[Test] + public function failedRepairIsThrottledAndDoesNotReplaceTheStoredResponse(): void + { + $this->storeHttpDocument(self::URL, 'No XML feed'); + $storage = $this->get(FeedStorage::class); + $before = $storage->findState(self::URL); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + FeedHttp::$responses[self::URL] = FeedHttp::rss('Available later'); + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertSame($before?->revision, $storage->findState(self::URL)?->revision); + self::assertSame($before?->fetchedAt, $storage->findState(self::URL)?->fetchedAt); + $now = (int)$this->get(Context::class)->getPropertyFromAspect('date', 'timestamp'); + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['last_attempt_at' => $now - 61], ['url_hash' => FeedStorage::hashUrl(self::URL)]); + self::assertStringContainsString('Available later', $this->requestPage()); + self::assertSame(2, FeedHttp::countRequests(self::URL)); + } + + #[Test] + public function repeatedSchedulerRenderingsReuseDownloadedImages(): void + { + $image = 'https://image.example.test/logo.png'; + FeedHttp::$responses[self::URL] = FeedHttp::rss('Image', '' . $image . 'Imagehttps://example.test/'); + FeedHttp::$responses[$image] = new Response(200, ['Content-Type' => 'image/png'], 'PNGDATA'); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'plugin.tx_feeddisplay_pi1.settings.getFields.feed = title,image_url'); + for ($request = 0; $request < 3; ++$request) { + self::assertStringContainsString('Image', $this->requestPage()); + } + self::assertSame(1, FeedHttp::countRequests(self::URL)); + self::assertSame(1, FeedHttp::countRequests($image)); + } + + #[Test] + public function repairSkipsABusySourceAndRetriesAfterRelease(): void + { + $this->storeHttpDocument(self::URL, 'No XML feed'); + FeedHttp::$responses[self::URL] = FeedHttp::rss('After lock'); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $mode = \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_EXCLUSIVE | \TYPO3\CMS\Core\Locking\LockingStrategyInterface::LOCK_CAPABILITY_NOBLOCK; + $lock = $this->get(\TYPO3\CMS\Core\Locking\LockFactory::class)->createLocker('feed_display_' . FeedStorage::hashUrl(self::URL), $mode); + self::assertTrue($lock->acquire($mode)); + try { + self::assertStringContainsString('error-no-feed', $this->requestPage()); + self::assertSame([], FeedHttp::$requests); + } finally { + $lock->release(); + } + self::assertStringContainsString('After lock', $this->requestPage()); + self::assertSame(1, FeedHttp::countRequests(self::URL)); + } + + private function storeHttpDocument(string $uri, string $body): void + { + $document = StoredDocument::create($body, $uri); + $storage = $this->get(FeedStorage::class); + $storage->storeSuccess(self::URL, $document, 1); + $storage->markInvalidated(self::URL, $document->revision()); + } +} diff --git a/Tests/Functional/Frontend/ScheduledPageCacheTest.php b/Tests/Functional/Frontend/ScheduledPageCacheTest.php new file mode 100644 index 00000000..26e93f0f --- /dev/null +++ b/Tests/Functional/Frontend/ScheduledPageCacheTest.php @@ -0,0 +1,191 @@ +preparePage(); + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $scheduler = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($scheduler); + $this->setFlexFormValues(['advanced' => ['settings.fetchMode' => 'live']]); + $live = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($live); + $content->update('tt_content', ['pi_flexform' => $scheduler['pi_flexform']], ['uid' => self::CONTENT_ELEMENT_UID]); + $live['uid'] = self::CONTENT_ELEMENT_UID + 1; + $live['sorting'] = $schedulerFirst ? 256 : -256; + $content->insert('tt_content', $live); + if ($anotherScheduler) { + $scheduler['uid'] = self::CONTENT_ELEMENT_UID + 2; + $scheduler['sorting'] = 512; + $content->insert('tt_content', $scheduler); + } + FeedHttp::$responses[self::URL] = FeedHttp::rss('NEW_SOURCE'); + $first = $this->requestPage(); + self::assertStringContainsString('NEW_SOURCE', $first); + if ($schedulerFirst) { + self::assertStringContainsString('OLD_SOURCE', $first); + self::assertSame([], $this->pageEntries()); + } else { + self::assertStringNotContainsString('OLD_SOURCE', $first); + self::assertCount(1, $this->pageEntries()); + } + self::assertSame('unchanged', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + self::assertStringNotContainsString('OLD_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + } + + /** @return iterable */ + public static function pluginOrders(): iterable + { + yield 'scheduler before live' => [true, false]; + yield 'live before scheduler' => [false, false]; + yield 'both revisions of one source' => [true, true]; + } + + #[Test] + public function refreshDuringRenderingDoesNotLeaveAnObsoletePageCached(): void + { + $image = $this->preparePage(true); + FeedHttp::$beforeResponse[$image] = function (): void { + FeedHttp::$responses[self::URL] = FeedHttp::rss('NEW_SOURCE'); + self::assertSame('updated', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + }; + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertSame([], $this->pageEntries()); + $stored = $this->get(FeedStorage::class)->findState(self::URL); + self::assertNotNull($stored); + self::assertFalse($stored->invalidationPending); + self::assertSame('unchanged', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + self::assertStringContainsString('NEW_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + } + + #[Test] + public function unchangedSourcesKeepTheirPageAndAllRevisionTags(): void + { + $this->preparePage(); + $secondUrl = self::URL . '/second'; + FeedHttp::$responses[$secondUrl] = FeedHttp::rss('SECOND_SOURCE'); + $content = $this->getConnectionPool()->getConnectionForTable('tt_content'); + $record = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($record); + $this->setFlexFormValues(['general' => ['settings.feedUrl' => $secondUrl]]); + $second = $content->select(['*'], 'tt_content', ['uid' => self::CONTENT_ELEMENT_UID])->fetchAssociative(); + self::assertIsArray($second); + $content->update('tt_content', ['pi_flexform' => $record['pi_flexform']], ['uid' => self::CONTENT_ELEMENT_UID]); + $second['uid'] = self::CONTENT_ELEMENT_UID + 1; + $second['sorting'] = 256; + $content->insert('tt_content', $second); + self::assertStringContainsString('SECOND_SOURCE', $this->requestPage()); + $entries = $this->pageEntries(); + self::assertCount(1, $entries); + self::assertGreaterThan(time() + 3600, (int)$entries[0]['expires']); + $tags = $this->getConnectionPool()->getConnectionForTable('cache_pages_tags')->select(['tag'], 'cache_pages_tags', ['identifier' => $entries[0]['identifier']])->fetchFirstColumn(); + foreach ([self::URL, $secondUrl] as $url) { + $state = $this->get(FeedStorage::class)->findState($url); + self::assertNotNull($state); + self::assertContains(FeedStorage::cacheTag($url), $tags); + self::assertContains('feeddisplay_rev_' . FeedStorage::hashUrl($url) . '_' . $state->revision, $tags); + } + $requests = FeedHttp::$requests; + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertSame($requests, FeedHttp::$requests); + self::assertSame($entries, $this->pageEntries()); + } + + #[Test] + public function revisionReadFailureKeepsThePageAndLogsADiagnosis(): void + { + $image = $this->preparePage(true); + $log = $this->instancePath . '/typo3temp/revision-errors.log'; + $this->configureLog($log); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + FeedHttp::$beforeResponse[$image] = static function () use ($manager): void { + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + }; + try { + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + self::assertFileExists($log); + $diagnosis = (string)file_get_contents($log); + self::assertStringContainsString('Could not verify the cached feed revision.', $diagnosis); + self::assertStringContainsString(FeedStorage::hashUrl(self::URL), $diagnosis); + self::assertStringContainsString('cacheIdentifier', $diagnosis); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + #[Test] + public function livePagesDoNotVerifySourceRevisions(): void + { + $image = $this->preparePage(true); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, 'plugin.tx_feeddisplay_pi1.settings.fetchMode = live'); + $log = $this->instancePath . '/typo3temp/live-revision-errors.log'; + $this->configureLog($log); + $manager = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->createSchemaManager(); + FeedHttp::$beforeResponse[$image] = static function () use ($manager): void { + $manager->renameTable(FeedStorage::TABLE, FeedStorage::TABLE . '_unavailable'); + }; + try { + self::assertStringContainsString('OLD_SOURCE', $this->requestPage()); + self::assertCount(1, $this->pageEntries()); + self::assertStringNotContainsString('Could not verify the cached feed revision.', is_file($log) ? (string)file_get_contents($log) : ''); + } finally { + $manager->renameTable(FeedStorage::TABLE . '_unavailable', FeedStorage::TABLE); + } + } + + private function preparePage(bool $withImage = false): string + { + $image = 'https://image.example.test/' . md5($this->nameWithDataSet()) . '.svg'; + $extra = $withImage ? '' . $image . 'Imagehttps://example.test/' : ''; + FeedHttp::$responses[self::URL] = FeedHttp::rss('OLD_SOURCE', $extra); + FeedHttp::$responses[$image] = new Response(200, ['Content-Type' => 'image/svg+xml'], ''); + self::assertSame('updated', $this->get(FeedRefresher::class)->refresh(self::URL, true)->status); + $this->initializeFrontendRootPage(self::URL, ['fetchMode' => 'scheduler', 'cacheDuration' => 0]); + $this->addTypoScriptToTemplateRecord(self::ROOT_PAGE_ID, "config.no_cache = 0\nconfig.cache_period = 7200\nplugin.tx_feeddisplay_pi1.settings.getFields.feed = title,image_url"); + return $image; + } + + private function configureLog(string $path): void + { + $writers = [LogLevel::WARNING => [FileWriter::class => ['logFile' => $path]]]; + file_put_contents($this->instancePath . '/typo3conf/system/additional.php', "\n\$GLOBALS['TYPO3_CONF_VARS']['LOG']['writerConfiguration'] = " . var_export($writers, true) . ';', FILE_APPEND); + } + + /** @return list> */ + private function pageEntries(): array + { + return $this->getConnectionPool()->getConnectionForTable('cache_pages')->select(['identifier', 'expires'], 'cache_pages')->fetchAllAssociative(); + } +} diff --git a/Tests/Functional/Storage/FeedFailureTest.php b/Tests/Functional/Storage/FeedFailureTest.php new file mode 100644 index 00000000..6498cf55 --- /dev/null +++ b/Tests/Functional/Storage/FeedFailureTest.php @@ -0,0 +1,200 @@ +instancePath . '/new.xml'; + file_put_contents($path, 'FreshTestItem'); + return $path; + } + + #[Test] + public function failedDatabaseWritePreservesTheOldDocumentAndReturnsTheFreshParser(): void + { + $url = $this->source(); + $storage = $this->get(FeedStorage::class); + $original = StoredDocument::create('Old response'); + $storage->storeSuccess($url, $original, 1000); + $row = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->select(['*'], FeedStorage::TABLE)->fetchAssociative(); + $result = self::createStub(Result::class); + $result->method('fetchAssociative')->willReturn($row); + $connection = $this->createMock(Connection::class); + $connection->method('select')->willReturn($result); + $connection->expects(self::once())->method('update')->willThrowException(new \RuntimeException('Simulated write failure')); + $connection->expects(self::never())->method('delete'); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $failing = new FeedStorage($pool, new NullLogger()); + $refresher = new FeedRefresher($failing, $this->get(FeedFetcher::class), $this->get(LockFactory::class), $this->get(CacheManager::class), $this->get(Context::class), new NullLogger()); + $refreshed = $refresher->refresh($url); + self::assertSame('failed', $refreshed->status); + self::assertSame('Fresh', \ErHaWeb\FeedDisplay\Service\SimplePieDeprecationHandler::run(static fn(): ?string => $refreshed->fetched?->feed->get_title())); + self::assertSame($original->revision(), $storage->find($url)?->revision); + self::assertSame(1000, $storage->find($url)->fetchedAt); + } + + #[Test] + public function corruptDocumentsProduceADiagnosis(): void + { + $logger = $this->createMock(LoggerInterface::class); + $logger->expects(self::once())->method('warning')->with('Could not decode the stored feed.', self::callback(static fn(array $context): bool => $context['url'] === 'broken')); + $storage = new FeedStorage($this->getConnectionPool(), $logger); + $storage->storeSuccess('broken', StoredDocument::create('Original'), 1000); + $bytes = serialize(['version' => 2, 'kind' => 'snapshot', 'snapshot' => ['response' => ['uri' => 'https://example.test', 'base' => 'https://example.test', 'status' => 200, 'body' => 'Body', 'headers' => ['Bad Header' => ['x']]], 'mime' => 'text/xml']]); + $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE)->update(FeedStorage::TABLE, ['response_data' => $bytes, 'revision' => hash('sha256', $bytes)], ['url_hash' => FeedStorage::hashUrl('broken')]); + self::assertNull($storage->find('broken')?->document); + } + + #[Test] + #[DataProvider('lockFailures')] + public function lockFailuresAreDiagnosedAndBusySourcesAreSkipped(string $stage, string $expected): void + { + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($expected === 'failed' ? self::once() : self::never())->method('warning'); + $url = $this->source(); + $locks = $this->createMock(LockFactory::class); + $lock = $this->createMock(LockingStrategyInterface::class); + if ($stage === 'create') { + $locks->method('createLocker')->willThrowException(new \RuntimeException('Create failed')); + } else { + $locks->method('createLocker')->willReturn($lock); + } + if ($stage === 'acquire' || $stage === 'would-block') { + $lock->method('acquire')->willThrowException($stage === 'acquire' ? new \RuntimeException('Acquire failed') : new LockAcquireWouldBlockException('Busy')); + } else { + $lock->method('acquire')->willReturn($stage !== 'busy'); + } + if ($stage === 'release') { + $lock->expects(self::once())->method('release')->willThrowException(new \RuntimeException('Release failed')); + } else { + $lock->expects(self::never())->method('release'); + } + $refresher = new FeedRefresher($this->get(FeedStorage::class), $this->get(FeedFetcher::class), $locks, $this->get(CacheManager::class), $this->get(Context::class), $logger); + self::assertSame($expected, $refresher->refresh($url)->status); + } + + /** @return iterable */ + public static function lockFailures(): iterable + { + yield 'creation' => ['create', 'failed']; + yield 'acquisition' => ['acquire', 'failed']; + yield 'release' => ['release', 'failed']; + yield 'busy return' => ['busy', 'locked']; + yield 'busy exception' => ['would-block', 'locked']; + } + + #[Test] + public function publicEventFailuresAreNotSwallowedByTheProvider(): void + { + $dispatcher = self::createStub(EventDispatcherInterface::class); + $dispatcher->method('dispatch')->willThrowException(new \DomainException('Listener failed')); + $data = new FeedDataService(new SimplePie(), $dispatcher, $this->get(FeedRuntimeInitializer::class)); + $provider = new FeedProvider($this->get(FeedStorage::class), $this->get(FeedRefresher::class), $this->get(FeedFetcher::class), $data, $this->get(CacheManager::class)->getCache('feeddisplay'), $this->get(Context::class), new NullLogger()); + $this->expectException(\DomainException::class); + $this->expectExceptionMessage('Listener failed'); + $provider->getData(['feedUrl' => $this->source(), 'cacheDuration' => 0, 'getFields' => ['items' => 'title']]); + } + #[Test] + public function concurrentRevisionIsNotLabelledWithAnEarlierParsersData(): void + { + $url = $this->source(); + $storage = $this->get(FeedStorage::class); + $next = StoredDocument::create('Concurrent resultTest'); + $lock = $this->createMock(LockingStrategyInterface::class); + $lock->method('acquire')->willReturn(true); + $lock->method('release')->willReturnCallback(static function () use ($storage, $url, $next): bool { + // Model a second writer completing after our lock release, before re-reading. + $storage->storeSuccess($url, $next, time()); + return true; + }); + $locks = self::createStub(LockFactory::class); + $locks->method('createLocker')->willReturn($lock); + $cache = $this->get(CacheManager::class)->getCache('feeddisplay'); + $fetcher = $this->get(FeedFetcher::class); + $context = $this->get(Context::class); + $refresher = new FeedRefresher($storage, $fetcher, $locks, $this->get(CacheManager::class), $context, new NullLogger()); + $provider = new FeedProvider($storage, $refresher, $fetcher, $this->get(FeedDataService::class), $cache, $context, new NullLogger()); + $settings = ['feedUrl' => $url, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + $data = $provider->getData($settings); + self::assertSame('Concurrent result', $data['feed']['title']); + self::assertArrayNotHasKey('_feeddisplay_revision', $data); + $cached = $cache->get('feeddisplay_' . md5(serialize($settings))); + self::assertIsArray($cached); + self::assertSame('Concurrent result', $cached['feed']['title']); + self::assertSame($next->revision(), $cached['_feeddisplay_revision']); + } + + #[Test] + public function concurrentReplacementDuringLazyLoadingIsRetriedWithItsOwnRevision(): void + { + $url = $this->source(); + $storage = $this->get(FeedStorage::class); + $fetcher = $this->get(FeedFetcher::class); + $storage->storeSuccess($url, $fetcher->fetch($url)->document, time()); + $next = StoredDocument::create('Concurrent lazy resultTest'); + $real = $this->getConnectionPool()->getConnectionForTable(FeedStorage::TABLE); + $swapped = false; + $connection = self::createStub(Connection::class); + $connection->method('select')->willReturnCallback(static function (array $columns, string $table, array $where = []) use ($storage, $url, $next, $real, &$swapped): Result { + if ($columns === ['response_data'] && !$swapped) { + $swapped = true; + $storage->storeSuccess($url, $next, time()); + } + return $real->select($columns, $table, $where); + }); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + $lazy = new FeedStorage($pool, new NullLogger()); + $cache = $this->get(CacheManager::class)->getCache('feeddisplay'); + $provider = new FeedProvider($lazy, $this->get(FeedRefresher::class), $fetcher, $this->get(FeedDataService::class), $cache, $this->get(Context::class), new NullLogger()); + $settings = ['feedUrl' => $url, 'cacheDuration' => 600, 'getFields' => ['feed' => 'title']]; + self::assertSame('Concurrent lazy result', $provider->getData($settings)['feed']['title']); + $cached = $cache->get('feeddisplay_' . md5(serialize($settings))); + self::assertIsArray($cached); + self::assertSame($next->revision(), $cached['_feeddisplay_revision']); + } + +} diff --git a/Tests/Functional/Storage/FeedStorageTest.php b/Tests/Functional/Storage/FeedStorageTest.php new file mode 100644 index 00000000..2ce07265 --- /dev/null +++ b/Tests/Functional/Storage/FeedStorageTest.php @@ -0,0 +1,137 @@ +Stra\xDFeTest'; + + #[Test] + public function binaryDocumentSurvivesEveryCacheFlushAndFailedRefresh(): void + { + $storage = $this->get(FeedStorage::class); + $document = StoredDocument::create("\x00Stra\xDFe\xff"); + self::assertTrue($storage->storeSuccess(self::URL, $document, 1000)); + $this->get(CacheManager::class)->flushCaches(); + $storage->storeFailure(self::URL, 'HTTP 503', 2000); + $stored = $storage->find(self::URL); + self::assertNotNull($stored); + self::assertSame($document->payload, $stored->document?->payload); + self::assertSame(1000, $stored->fetchedAt); + self::assertSame(2000, $stored->lastAttemptAt); + self::assertSame('HTTP 503', $stored->lastError); + } + + #[Test] + public function unchangedSuccessRenewsFreshnessAndPreservesPendingInvalidation(): void + { + $storage = $this->get(FeedStorage::class); + $one = StoredDocument::create('one'); + $two = StoredDocument::create('two'); + $storage->storeSuccess(self::URL, $one, 1000); + $storage->markInvalidated(self::URL, $one->revision()); + self::assertFalse($storage->storeSuccess(self::URL, $one, 2000)); + self::assertSame(2000, $storage->find(self::URL)?->fetchedAt); + self::assertFalse($storage->find(self::URL)->invalidationPending); + $storage->storeSuccess(self::URL, $two, 3000); + $storage->markInvalidated(self::URL, $one->revision()); + self::assertTrue($storage->find(self::URL)?->invalidationPending); + self::assertFalse($storage->storeSuccess(self::URL, $two, 4000)); + self::assertTrue($storage->find(self::URL)?->invalidationPending); + } + + #[Test] + public function unchangedRefreshRetriesAFailedPageInvalidation(): void + { + $calls = 0; + $manager = $this->createMock(CacheManager::class); + $manager->expects(self::exactly(2))->method('flushCachesInGroupByTag')->willReturnCallback( + static function () use (&$calls): void { + if (++$calls === 1) { + throw new \RuntimeException('Page cache temporarily unavailable'); + } + } + ); + $body = str_replace('Stra\\xDFe', "Stra\xDFe", self::BODY); + $client = new Client(['handler' => HandlerStack::create(new MockHandler([ + new Response(200, ['Content-Type' => 'application/rss+xml'], $body), + new Response(200, ['Content-Type' => 'application/rss+xml'], $body), + ]))]); + $factory = self::createStub(GuzzleClientFactory::class); + $factory->method('getClient')->willReturn($client); + $http = new HttpFactory(); + $initializer = new FeedRuntimeInitializer($factory, $http, $http); + $lock = self::createStub(LockingStrategyInterface::class); + $lock->method('acquire')->willReturn(true); + $locks = self::createStub(LockFactory::class); + $locks->method('createLocker')->willReturn($lock); + $context = new Context(); + $context->setAspect('date', new DateTimeAspect(new \DateTimeImmutable('@1000'))); + $storage = $this->get(FeedStorage::class); + $subject = new FeedRefresher($storage, new FeedFetcher($initializer, $factory), $locks, $manager, $context, self::createStub(LoggerInterface::class)); + $first = $subject->refresh(self::URL); + self::assertSame('failed', $first->status); + self::assertTrue($first->persisted); + self::assertTrue($storage->find(self::URL)?->invalidationPending); + $context->setAspect('date', new DateTimeAspect(new \DateTimeImmutable('@2000'))); + self::assertSame('unchanged', $subject->refresh(self::URL)->status); + self::assertFalse($storage->find(self::URL)?->invalidationPending); + self::assertSame(2000, $storage->find(self::URL)->fetchedAt); + } + #[Test] + public function lazyReadsDoNotReturnAPayloadFromADifferentRevision(): void + { + $storage = $this->get(FeedStorage::class); + $one = StoredDocument::create("one\xFF\x00"); + $two = StoredDocument::create("two\xDF\x00"); + $storage->storeSuccess(self::URL, $one, 1000); + $state = $storage->findState(self::URL); + self::assertNotNull($state); + self::assertTrue($state->hasDocument); + self::assertNull($state->document); + self::assertSame($one->payload, $storage->loadDocument(self::URL, $state->revision)?->payload); + $storage->storeSuccess(self::URL, $two, 2000); + self::assertNull($storage->loadDocument(self::URL, $state->revision)); + $current = $storage->findState(self::URL); + self::assertNotNull($current); + self::assertSame($two->payload, $storage->loadDocument(self::URL, $current->revision)?->payload); + } + +} diff --git a/Tests/Unit/Controller/FeedControllerTest.php b/Tests/Unit/Controller/FeedControllerTest.php deleted file mode 100644 index 9aef6e56..00000000 --- a/Tests/Unit/Controller/FeedControllerTest.php +++ /dev/null @@ -1,239 +0,0 @@ -createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - - $cache->expects(self::never())->method('get'); - $feedDataService->expects(self::never())->method('buildData'); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings([]); - $subject->setView(new RecordingView()); - - $subject->displayAction(); - } - - #[Test] - public function displayActionBypassesCacheIfCacheDurationIsZero(): void - { - $settings = [ - 'cacheDuration' => 0, - ]; - $data = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'First item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn(false); - $cache->expects(self::once())->method('remove')->with($cacheIdentifier); - $cache->expects(self::never())->method('set'); - $feedDataService->expects(self::once())->method('buildData')->with($settings)->willReturn($data); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($data, $view->assigned['data']); - } - - #[Test] - public function displayActionCachesBuiltDataOnCacheMiss(): void - { - $settings = [ - 'cacheDuration' => 600, - ]; - $data = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'Cached item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn(false); - $cache->expects(self::once())->method('set')->with($cacheIdentifier, $data, [], 600); - $cache->expects(self::never())->method('remove'); - $feedDataService->expects(self::once())->method('buildData')->with($settings)->willReturn($data); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($data, $view->assigned['data']); - } - - #[Test] - public function displayActionReusesCachedDataWhenSettingsMatch(): void - { - $settings = [ - 'cacheDuration' => 600, - ]; - $cachedData = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'Cached item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn($cachedData); - $cache->expects(self::never())->method('set'); - $cache->expects(self::never())->method('remove'); - $feedDataService->expects(self::never())->method('buildData'); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($cachedData, $view->assigned['data']); - } - - #[Test] - public function displayActionRebuildsCacheWhenCachedSettingsDoNotMatch(): void - { - $settings = [ - 'cacheDuration' => 600, - ]; - $cachedData = [ - 'settings' => [ - 'cacheDuration' => 60, - ], - 'items' => [ - ['title' => 'Outdated item'], - ], - ]; - $rebuiltData = [ - 'settings' => $settings, - 'items' => [ - ['title' => 'Fresh item'], - ], - ]; - $cacheIdentifier = 'feeddisplay_' . md5(serialize($settings)); - $cache = $this->createMock(FrontendInterface::class); - $feedDataService = $this->createMock(FeedDataService::class); - $view = new RecordingView(); - - $cache->expects(self::once())->method('get')->with($cacheIdentifier)->willReturn($cachedData); - $cache->expects(self::once())->method('set')->with($cacheIdentifier, $rebuiltData, [], 600); - $cache->expects(self::never())->method('remove'); - $feedDataService->expects(self::once())->method('buildData')->with($settings)->willReturn($rebuiltData); - - $subject = new TestableFeedController($cache, $feedDataService); - $subject->setSettings($settings); - $subject->setView($view); - $subject->displayAction(); - - self::assertSame($rebuiltData, $view->assigned['data']); - } -} - -final class TestableFeedController extends FeedController -{ - /** - * @param array $settings - */ - public function setSettings(array $settings): void - { - $this->settings = $settings; - } - - public function setView(ViewInterface $view): void - { - $this->view = $view; - } - - protected function htmlResponse(?string $html = null): ResponseInterface - { - return new HtmlResponse($html ?? ''); - } -} - -final class RecordingView implements ViewInterface -{ - /** @var array */ - public array $assigned = []; - - public function assign(string $key, mixed $value): self - { - $this->assigned[$key] = $value; - return $this; - } - - /** - * @param array $values - */ - public function assignMultiple(array $values): self - { - foreach ($values as $key => $value) { - $this->assigned[$key] = $value; - } - return $this; - } - - public function render(string $templateFileName = ''): string - { - return ''; - } - - /** - * @param array $variables - */ - public function renderSection(string $sectionName, array $variables = [], bool $ignoreUnknown = false): string - { - return ''; - } - - /** - * @param array $variables - */ - public function renderPartial(string $partialName, string $sectionName, array $variables, bool $ignoreUnknown = false): string - { - return ''; - } -} diff --git a/Tests/Unit/Service/FeedDataServiceTest.php b/Tests/Unit/Service/FeedDataServiceTest.php index 2808b26a..c7a8f8b5 100644 --- a/Tests/Unit/Service/FeedDataServiceTest.php +++ b/Tests/Unit/Service/FeedDataServiceTest.php @@ -21,6 +21,11 @@ use ErHaWeb\FeedDisplay\Service\FeedDataService; use ErHaWeb\FeedDisplay\Service\FeedRuntimeInitializer; use GuzzleHttp\Client; +use GuzzleHttp\Handler\MockHandler; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Psr7\HttpFactory; +use GuzzleHttp\Psr7\Response; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; use Psr\EventDispatcher\EventDispatcherInterface; use Psr\Http\Message\RequestFactoryInterface; @@ -73,7 +78,7 @@ public function buildDataKeepsSimplePieDefaultTransportForLocalFeedPaths(): void $feed->expects(self::once())->method('set_feed_url')->with('/var/www/public/fileadmin/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -108,7 +113,7 @@ public function buildDataMapsFeedAndItemFieldsAndDispatchesEvent(): void $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_title')->willReturn('Feed title'); $feed->expects(self::once())->method('subscribe_url')->willReturn('https://example.com/subscribe'); $feed->expects(self::once())->method('get_image_url')->willReturn('file://' . $sourceImagePath); @@ -158,7 +163,7 @@ public function buildDataIgnoresLegacyFaviconFeedFieldWithoutCallingSimplePieFav $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::never())->method('get_favicon'); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); @@ -193,7 +198,7 @@ public function buildDataKeepsFeedGetterValueWhenKnownSimplePieIriDeprecationIsT $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_description')->willReturnCallback( static fn(): string => require $deprecationFile ); @@ -275,7 +280,7 @@ public function buildDataSupportsGetterCallsWithThreeAndFourFieldParts(): void $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 1)->willReturn([$item]); $eventDispatcher->expects(self::once())->method('dispatch')->willReturnArgument(0); @@ -311,7 +316,7 @@ public function buildDataDoesNotOverwriteAnAlreadyGeneratedTemporaryImage(): voi $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_image_url')->willReturn('file://' . $sourceImagePath); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -344,7 +349,7 @@ public function buildDataStripsSlashesFromConfiguredFeedUrlBeforeInitializingSim $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -375,7 +380,7 @@ public function buildDataReturnsNullForUnknownGettersAndUnsupportedArgumentCount $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_items')->with(0, 1)->willReturn([$item]); $eventDispatcher->expects(self::once())->method('dispatch')->willReturnArgument(0); @@ -409,7 +414,7 @@ public function buildDataReturnsNullIfFeedImageUrlCannotBeResolvedToATemporaryFi $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_image_url')->willReturn('file:///path/to/non-existing.svg'); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -442,7 +447,7 @@ public function buildDataReturnsNullIfFeedImageUrlHasNoFilenameOrExtension(): vo $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::once())->method('get_image_url')->willReturn('https://example.com/images/'); $feed->expects(self::once())->method('get_items')->with(0, 0)->willReturn([]); $eventDispatcher->expects(self::never())->method('dispatch'); @@ -476,7 +481,7 @@ public function buildDataReturnsSerializableDataForObjectRichFieldsAndEventInjec $feed->expects(self::once())->method('set_feed_url')->with('https://example.com/feed.xml'); $feed->expects(self::once())->method('enable_cache')->with(false); - $feed->expects(self::once())->method('init'); + $feed->expects(self::once())->method('init')->willReturn(true); $feed->expects(self::exactly(2))->method('get_items')->willReturnCallback( static function (int $start, int $end) use ($item): array { self::assertSame(0, $start); @@ -512,6 +517,39 @@ static function (SingleFeedDataEvent $event): SingleFeedDataEvent { self::assertNotSame('', serialize($data)); } + #[Test] + #[DataProvider('httpFeedResponses')] + public function initializerReportsWhetherTheHttpFeedWasParsed(Response $response, bool $expected): void + { + $client = new Client(['handler' => HandlerStack::create(new MockHandler([$response]))]); + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->expects(self::once())->method('getClient')->willReturn($client); + $httpFactory = new HttpFactory(); + $subject = new FeedRuntimeInitializer($factory, $httpFactory, $httpFactory); + + self::assertSame( + $expected, + $subject->initializeFeed(new SimplePie(), ['feedUrl' => 'https://example.test/feed.xml']) + ); + } + + /** + * @return iterable + */ + public static function httpFeedResponses(): iterable + { + yield 'HTTP failure' => [new Response(503), false]; + yield 'invalid XML' => [new Response(200, ['Content-Type' => 'application/rss+xml'], ''), false]; + yield 'valid empty feed' => [ + new Response( + 200, + ['Content-Type' => 'application/rss+xml'], + 'Feedhttps://example.test/Empty' + ), + true, + ]; + } + private function createSourceImage(string $fileName): string { $sourceImagePath = Environment::getPublicPath() . '/typo3temp/var/tests/' . $fileName; diff --git a/Tests/Unit/Service/FeedHttpTest.php b/Tests/Unit/Service/FeedHttpTest.php new file mode 100644 index 00000000..234667fb --- /dev/null +++ b/Tests/Unit/Service/FeedHttpTest.php @@ -0,0 +1,212 @@ + $responses + */ + #[Test] + #[DataProvider('feedResponses')] + public function recordingAndReplayMatchLiveParsing(array $responses): void + { + $liveClient = new Client(['handler' => HandlerStack::create(new MockHandler($responses))]); + $guzzleFactory = $this->createMock(GuzzleClientFactory::class); + $guzzleFactory->expects(self::once())->method('getClient')->willReturn($liveClient); + $httpFactory = new HttpFactory(); + $initializer = new FeedRuntimeInitializer($guzzleFactory, $httpFactory, $httpFactory); + $live = new SimplePie(); + self::assertTrue($initializer->initializeFeed($live, ['feedUrl' => self::URL])); + + $recordingClient = new Client(['handler' => HandlerStack::create(new MockHandler($responses))]); + $recordingFactory = self::createStub(GuzzleClientFactory::class); + $recordingFactory->method('getClient')->willReturn($recordingClient); + $fetcher = new \ErHaWeb\FeedDisplay\Service\FeedFetcher(new FeedRuntimeInitializer($recordingFactory, $httpFactory, $httpFactory), $recordingFactory); + $fetched = $fetcher->fetch(self::URL); + $recorded = $fetched->feed; + $replayed = $fetcher->restore(\ErHaWeb\FeedDisplay\Storage\FeedDocument::decode($fetched->document->encode())); + + $liveData = $this->getFeedValues($live); + self::assertSame($liveData, $this->getFeedValues($recorded)); + self::assertSame($liveData, $this->getFeedValues($replayed)); + + $dispatcher = $this->createMock(EventDispatcherInterface::class); + $dispatcher->expects(self::once())->method('dispatch')->willReturnCallback( + static function (SingleFeedDataEvent $event) use ($replayed): SingleFeedDataEvent { + self::assertSame($replayed, $event->getFeed()); + self::assertSame($replayed->get_item(0), $event->getItem()); + return $event; + } + ); + (new FeedDataService($live, $dispatcher, $initializer))->buildInitializedData($replayed, [ + 'getFields' => ['items' => 'title'], + 'maxFeedCount' => 1, + ]); + } + + /** + * @return iterable}> + */ + public static function feedResponses(): iterable + { + $rss = '%sTest' + . 'Itemarticle.html'; + $atom = 'News' + . 'Itemurn:news:1'; + + foreach ([301, 302] as $status) { + yield 'RSS redirect ' . $status => [[ + new Response($status, ['Location' => 'https://new.example.test/rss/feed.xml']), + new Response(200, ['Content-Type' => 'application/rss+xml'], sprintf($rss, 'News')), + ]]; + yield 'Atom redirect ' . $status => [[ + new Response($status, ['Location' => 'https://new.example.test/atom/feed.xml']), + new Response(200, ['Content-Type' => 'application/atom+xml'], $atom), + ]]; + } + + yield 'HTML feed autodiscovery' => [[ + new Response(200, ['Content-Type' => 'text/html'], 'News'), + new Response(200, ['Content-Type' => 'application/rss+xml'], sprintf($rss, 'Discovered feed')), + ]]; + + yield 'non-UTF-8 body' => [[new Response( + 200, + ['Content-Type' => 'application/rss+xml'], + '' . sprintf($rss, "Stra\xDFe") + )]]; + yield 'encoding declared in lowercase header' => [[new Response( + 200, + ['content-type' => 'application/rss+xml; charset=windows-1252'], + sprintf($rss, "Price 10 \x80") + )]]; + yield 'encoding declared in mixed case header' => [[new Response( + 200, + ['Content-Type' => 'application/rss+xml; charset=windows-1252'], + sprintf($rss, "Price 10 \x80") + )]]; + yield 'encoding contradicts body' => [[new Response( + 200, + ['Content-Type' => 'application/rss+xml; charset=ISO-8859-1'], + sprintf($rss, 'Straße') + )]]; + foreach ([['Link', 'Content-Language'], ['link', 'content-language']] as [$linkHeader, $languageHeader]) { + yield $linkHeader . ' and ' . $languageHeader => [[new Response( + 200, + [ + 'Content-Type' => 'application/rss+xml', + $linkHeader => '; rel="self"', + $languageHeader => 'de', + 'Set-Cookie' => 'not-stored=1', + ], + sprintf($rss, 'News') + )]]; + } + } + + #[Test] + public function recorderReadsOnlyTheRemainingBodyBudgetAndClosesTheStream(): void + { + $readBytes = 0; + $stream = new PumpStream(static function (int $length) use (&$readBytes): string { + $readBytes += $length; + return str_repeat('x', $length); + }); + $client = self::createStub(GuzzleClientInterface::class); + $client->method('send')->willReturn(new Response(200, [], $stream)); + $recorder = new FeedHttpRecorder($client, 10, 10); + $this->expectException(RequestException::class); + $this->expectExceptionMessage('2 MiB'); + + try { + $recorder->sendRequest(new Request('GET', self::URL)); + } finally { + self::assertSame(2 * 1024 * 1024 + 1, $readBytes); + self::assertTrue($stream->eof()); + } + } + + #[Test] + public function recorderAppliesTheBudgetAcrossRedirectResponses(): void + { + $client = new Client(['handler' => HandlerStack::create(new MockHandler([ + new Response(302, ['Location' => self::URL], str_repeat('x', 1024 * 1024)), + new Response(200, [], str_repeat('x', 1024 * 1024 + 1)), + ]))]); + $recorder = new FeedHttpRecorder($client, 10, 10); + $recorder->sendRequest(new Request('GET', self::URL)); + $this->expectException(RequestException::class); + $recorder->sendRequest(new Request('GET', self::URL)); + } + + #[Test] + public function localReplayPreservesSourceContextWithoutReadingTheFileAgain(): void + { + $factory = self::createStub(GuzzleClientFactory::class); + $http = new HttpFactory(); + $fetcher = new \ErHaWeb\FeedDisplay\Service\FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + $path = tempnam(__DIR__ . '/../../../.Build/Web/typo3temp', 'feed-replay-'); + self::assertIsString($path); + $this->testFilesToDelete[] = $path . '.moved'; + file_put_contents($path, 'Stra' . "\xDF" . 'eTestItemarticle.html'); + $live = $fetcher->fetch($path); + rename($path, $path . '.moved'); + $restored = $fetcher->restore(\ErHaWeb\FeedDisplay\Storage\FeedDocument::decode($live->document->encode())); + self::assertSame($this->getFeedValues($live->feed), $this->getFeedValues($restored)); + } + + /** + * @return array + */ + private function getFeedValues(SimplePie $feed): array + { + return SimplePieDeprecationHandler::run(static fn(): array => [ + 'title' => $feed->get_title(), + 'subscription' => $feed->subscribe_url(), + 'permanentSubscription' => $feed->subscribe_url(true), + 'base' => $feed->get_base(), + 'language' => $feed->get_language(), + 'self' => $feed->get_link(0, 'self'), + 'itemLink' => $feed->get_item(0)?->get_permalink(), + ]); + } +} diff --git a/Tests/Unit/Service/FeedSnapshotTest.php b/Tests/Unit/Service/FeedSnapshotTest.php new file mode 100644 index 00000000..733ff3c7 --- /dev/null +++ b/Tests/Unit/Service/FeedSnapshotTest.php @@ -0,0 +1,160 @@ + $responses */ + private function fetcher(array $responses): FeedFetcher + { + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->expects(self::once())->method('getClient')->willReturn(new Client(['handler' => HandlerStack::create(new MockHandler($responses))])); + $http = new HttpFactory(); + return new FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + } + + private static function rss(string $title): string + { + return '' . $title . 'TestItemarticle.html'; + } + + /** @param list $responses */ + #[Test] + #[DataProvider('responses')] + public function snapshotRetainsSelectedFeedAndItsMetadata(array $responses): void + { + $fetcher = $this->fetcher($responses); + $fetched = $fetcher->fetch(self::URL); + $document = FeedDocument::decode($fetched->document->encode()); + $replay = $fetcher->restore($document); + $values = static fn(SimplePie $feed): array => SimplePieDeprecationHandler::run(static fn(): array => [ + $feed->get_title(), $feed->subscribe_url(), $feed->subscribe_url(true), $feed->get_base(), + $feed->get_language(), $feed->get_link(0, 'self'), $feed->get_item(0)?->get_permalink(), + $feed->get_item(0)?->get_title(), (new FeedValueNormalizer())->normalizeValue($feed->get_all_discovered_feeds()), + $feed->data['headers'] ?? [], + ]); + self::assertSame($values($fetched->feed), $values($replay)); + self::assertSame($fetched->document->revision(), $document->revision()); + } + + /** @return iterable}> */ + public static function responses(): iterable + { + yield from FeedHttpTest::feedResponses(); + $html = ''; + yield 'two discovered feeds retain the first selected feed' => [[ + new Response(200, ['Content-Type' => 'text/html'], $html), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('SELECTED_A')), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('ALTERNATIVE_B')), + ]]; + yield 'trailing non-feed success does not replace the selected feed' => [[ + new Response(200, ['Content-Type' => 'text/html'], $html), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('SELECTED_A')), + new Response(200, ['Content-Type' => 'text/html'], 'No feed here'), + ]]; + } + + #[Test] + public function boundedFetchesUseOnlyTheDirectSource(): void + { + $html = ''; + $mock = new MockHandler([ + new Response(200, ['Content-Type' => 'text/html'], $html), + new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('NOT_REQUESTED')), + ]); + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->method('getClient')->willReturn(new Client(['handler' => HandlerStack::create($mock)])); + $http = new HttpFactory(); + $fetcher = new FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + try { + $fetcher->fetch(self::URL, true); + self::fail('A bounded fetch must not continue with another source.'); + } catch (\RuntimeException $exception) { + self::assertStringContainsString('could not be found', $exception->getMessage()); + } + self::assertCount(1, $mock); + } + + #[Test] + public function boundedLocalSourcesMustBeFilesWithinTheLimit(): void + { + $factory = $this->createMock(GuzzleClientFactory::class); + $factory->expects(self::never())->method('getClient'); + $http = new HttpFactory(); + $fetcher = new FeedFetcher(new FeedRuntimeInitializer($factory, $http, $http), $factory); + $path = tempnam(sys_get_temp_dir(), 'feed-snapshot-'); + self::assertIsString($path); + try { + file_put_contents($path, str_repeat(' ', FeedHttpRecorder::MAX_BODY_SIZE + 1)); + foreach ([$path, sys_get_temp_dir()] as $source) { + try { + $fetcher->fetch($source, true); + self::fail('The source must be rejected.'); + } catch (\RuntimeException $exception) { + self::assertStringContainsString('at most 2 MiB', $exception->getMessage()); + } + } + } finally { + unlink($path); + } + } + + /** @return iterable */ + public static function httpSourcesWithoutHttpScheme(): iterable + { + yield 'feed scheme' => ['feed://example.test/feed']; + yield 'host only' => ['example.test']; + } + + #[Test] + #[DataProvider('httpSourcesWithoutHttpScheme')] + public function httpSourcesWithoutHttpSchemeAreRecorded(string $source): void + { + $fetched = $this->fetcher([new Response(200, ['Content-Type' => 'application/rss+xml'], self::rss('RECORDED'))])->fetch($source, true); + self::assertSame('RECORDED', SimplePieDeprecationHandler::run($fetched->feed->get_title(...))); + } + + #[Test] + public function recorderConfirmsOnlyReceivedBodies(): void + { + $http = new HttpFactory(); + $recorder = new FeedHttpRecorder(new Client(['handler' => HandlerStack::create(new MockHandler([new Response(200, [], 'RECEIVED')]))])); + $recorder->sendRequest($http->createRequest('GET', self::URL)); + self::assertTrue($recorder->received('RECEIVED')); + self::assertFalse($recorder->received('OTHER')); + } +} diff --git a/Tests/Unit/Storage/FeedDocumentTest.php b/Tests/Unit/Storage/FeedDocumentTest.php new file mode 100644 index 00000000..24de8b79 --- /dev/null +++ b/Tests/Unit/Storage/FeedDocumentTest.php @@ -0,0 +1,57 @@ +payload, FeedDocument::decode($document->encode())->payload); + } + + #[Test] + #[DataProvider('invalidDocuments')] + public function malformedDocumentsAreRejected(mixed $payload): void + { + $this->expectException(\Exception::class); + FeedDocument::decode(serialize($payload)); + } + + /** @return iterable */ + public static function invalidDocuments(): iterable + { + yield 'incomplete snapshot' => [['version' => 2, 'kind' => 'snapshot', 'snapshot' => []]]; + $response = ['uri' => 'https://example.test', 'base' => 'https://example.test', 'status' => 200, 'headers' => [], 'body' => 'XML']; + foreach ([['base' => null], ['status' => 700], ['status' => '200'], ['headers' => ['Bad Header' => ['x']]], ['body' => new \stdClass()]] as $invalid) { + yield 'snapshot ' . json_encode($invalid) => [['version' => 2, 'kind' => 'snapshot', 'snapshot' => [ + 'response' => array_replace($response, $invalid), 'mime' => 'text/xml', + ]]]; + } + yield 'serialized object' => [(object)['version' => 1]]; + yield 'missing format' => [[]]; + yield 'unsupported format' => [['version' => 1, 'kind' => 'http', 'steps' => []]]; + } +} diff --git a/Tests/Unit/Storage/FeedStorageTest.php b/Tests/Unit/Storage/FeedStorageTest.php new file mode 100644 index 00000000..44c8b201 --- /dev/null +++ b/Tests/Unit/Storage/FeedStorageTest.php @@ -0,0 +1,98 @@ +encode()); + rewind($stream); + try { + $row = ['url' => 'source', 'response_data' => $stream, 'revision' => $document->revision(), 'fetched_at' => 1000, 'last_attempt_at' => 1000, 'last_error' => '', 'invalidation_pending' => 0]; + $result = self::createStub(Result::class); + $result->method('fetchAssociative')->willReturn($row); + $connection = self::createStub(Connection::class); + $connection->method('select')->willReturn($result); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + self::assertSame($document->payload, (new FeedStorage($pool, new NullLogger()))->find('source')?->document?->payload); + } finally { + fclose($stream); + } + } + #[Test] + public function lazyDocumentReadsSupportLobStreamsAndMatchTheRequestedRevision(): void + { + $document = StoredDocument::create("Binary \x00\xFF\xDF"); + $stream = fopen('php://memory', 'w+b'); + self::assertIsResource($stream); + fwrite($stream, $document->encode()); + rewind($stream); + try { + $result = self::createStub(Result::class); + $result->method('fetchAssociative')->willReturn(['response_data' => $stream]); + $connection = $this->createMock(Connection::class); + $connection->expects(self::once())->method('select')->with(['response_data'], FeedStorage::TABLE, [ + 'url_hash' => FeedStorage::hashUrl('source'), 'revision' => $document->revision(), + ])->willReturn($result); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + self::assertSame($document->payload, (new FeedStorage($pool, new NullLogger()))->loadDocument('source', $document->revision())?->payload); + } finally { + fclose($stream); + } + } + + #[Test] + #[DataProvider('storedRevisions')] + public function revisionLookupDoesNotLoadTheDocument(string|false $value, ?string $expected): void + { + $hash = FeedStorage::hashUrl('https://example.test/feed'); + $result = self::createStub(Result::class); + $result->method('fetchOne')->willReturn($value); + $connection = $this->createMock(Connection::class); + $connection->expects(self::once())->method('select')->with(['revision'], FeedStorage::TABLE, ['url_hash' => $hash])->willReturn($result); + $pool = self::createStub(ConnectionPool::class); + $pool->method('getConnectionForTable')->willReturn($connection); + self::assertSame($expected, (new FeedStorage($pool, new NullLogger()))->findRevisionByHash($hash)); + } + + /** @return iterable */ + public static function storedRevisions(): iterable + { + yield 'successful source' => [str_repeat('a', 64), str_repeat('a', 64)]; + yield 'missing source' => [false, null]; + yield 'failed first attempt' => ['', '']; + } + +} diff --git a/ext_tables.sql b/ext_tables.sql new file mode 100644 index 00000000..716598ad --- /dev/null +++ b/ext_tables.sql @@ -0,0 +1,13 @@ +CREATE TABLE tx_feeddisplay_feed ( + uid int(11) unsigned NOT NULL auto_increment, + url_hash char(64) DEFAULT '' NOT NULL, + url text, + response_data longblob, + revision char(64) DEFAULT '' NOT NULL, + fetched_at int(11) unsigned DEFAULT '0' NOT NULL, + last_attempt_at int(11) unsigned DEFAULT '0' NOT NULL, + last_error text, + invalidation_pending smallint(5) unsigned DEFAULT '0' NOT NULL, + PRIMARY KEY (uid), + UNIQUE KEY url_hash (url_hash) +);