Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions Classes/Backend/Controller/PersistenceController.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@
use Psr\Http\Message\ServerRequestInterface;
use RuntimeException;
use TYPO3\CMS\Backend\Attribute\AsController;
use TYPO3\CMS\Core\Crypto\Random;
use TYPO3\CMS\Core\DataHandling\Model\CorrelationId;
use TYPO3\CMS\Core\Http\JsonResponse;
use TYPO3\CMS\VisualEditor\Service\DataHandlerService;
use TYPO3\CMS\VisualEditor\SysHistory\SysHistoryCombiner;

use function array_keys;
use function implode;
Expand All @@ -20,6 +23,8 @@
{
public function __construct(
private DataHandlerService $dataHandlerService,
private SysHistoryCombiner $sysHistoryCombiner,
private Random $randomGenerator,
) {
}

Expand All @@ -44,12 +49,16 @@ public function saveAction(ServerRequestInterface $request): ResponseInterface
}

$GLOBALS['TYPO3_REQUEST'] = $request;
$errorLog = $this->dataHandlerService->run($data, []);
$correlationId = CorrelationId::forScope($this->randomGenerator->generateRandomBase64String(32))
->withAspects(SysHistoryCombiner::CORRELATION_ASPECT);
$errorLog = $this->dataHandlerService->run($data, [], $correlationId);

foreach ($cmdArray as $cmd) {
$errorLog = [...$errorLog, ...$this->dataHandlerService->run([], $cmd)];
$errorLog = [...$errorLog, ...$this->dataHandlerService->run([], $cmd, $correlationId)];
}

$this->sysHistoryCombiner->combine();

if ($errorLog) {
return new JsonResponse(['success' => false, 'errorLog' => $errorLog], 500);
}
Expand Down
5 changes: 4 additions & 1 deletion Classes/Service/DataHandlerService.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

use RuntimeException;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\DataHandling\Model\CorrelationId;
use TYPO3\CMS\Core\Schema\TcaSchemaFactory;
use TYPO3\CMS\Core\Utility\GeneralUtility;

Expand All @@ -26,15 +27,17 @@ public function __construct(
*
* @return list<string>
*/
public function run(array $data, array $cmd): array
public function run(array $data, array $cmd, CorrelationId $correlationId): array
{
$this->validateData($data);
$this->validateCmd($cmd);

$dataHandler = GeneralUtility::makeInstance(DataHandler::class); // never use DataHandler over DI!!
$dataHandler->start($data, $cmd);
$dataHandler->setCorrelationId($correlationId);
$dataHandler->process_datamap();
$dataHandler->process_cmdmap();

return $dataHandler->errorLog;
}

Expand Down
170 changes: 170 additions & 0 deletions Classes/SysHistory/SysHistoryCombiner.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
<?php

declare(strict_types=1);

namespace TYPO3\CMS\VisualEditor\SysHistory;

use InvalidArgumentException;
use TYPO3\CMS\Core\DataHandling\History\RecordHistoryStore;
use TYPO3\CMS\Core\DataHandling\Model\CorrelationId;

final readonly class SysHistoryCombiner
{
public const CORRELATION_ASPECT = 'visual-editor';

public function __construct(
private SysHistoryRepositoryInterface $sysHistoryRepository,
) {
}

/**
* Separate Visual Editor saves within the time window of $timeToCombine seconds are merged.
*
* Combines all sys_history entries that have the same actiontype, usertype, userid, originaluserid, recuid, tablename and workspace
* takes the last entry of the combined entries and updates the history_data with the combined oldRecord and newRecord
* deletes all other entries of the combined entries.
*
* @param int $timeToCombine how many seconds to load and combine entries if possible
*/
public function combine(int $timeToCombine = 60): void
{
$toBeDeletedUids = [];
$mapped = $this->getMappedRows($timeToCombine);
foreach ($mapped as $rowsToCombine) {
$toBeDeletedUids = [...$toBeDeletedUids, ...$this->combineRows($rowsToCombine)];
}

$this->sysHistoryRepository->deleteEntries($toBeDeletedUids);
}

/**
* @return array<string, list<array<string, mixed>>>
*/
private function getMappedRows(int $timeToCombine): array
{
$mapped = [];
$groupedPerRecord = [];
foreach ($this->sysHistoryRepository->fetchSysHistory($timeToCombine) as $row) {
$key = implode('-', [
$row['recuid'],
$row['tablename'],
$row['workspace'],
]);
$groupedPerRecord[$key] ??= [];
$groupedPerRecord[$key][] = $row;
}

foreach ($groupedPerRecord as $parentKey => $rows) {
$lastKey = null;
$counter = 0;
foreach ($rows as $row) {
if (!$this->isVisualEditorModification($row)) {
$lastKey = null;
$counter++;
continue;
}

$key = implode('-', [
$row['actiontype'],
$row['usertype'],
$row['userid'],
$row['originaluserid'],
]);
if ($key !== $lastKey) {
$lastKey = $key;
$counter++;
}

$mapped[$counter . '-' . $parentKey . '-' . $key][] = $row;
}
}

return $mapped;
}

/**
* @param array<string, mixed> $row
*/
private function isVisualEditorModification(array $row): bool
{
if ((int)$row['actiontype'] !== RecordHistoryStore::ACTION_MODIFY) {
return false;
}

try {
return CorrelationId::fromString((string)$row['correlation_id'])->getAspects() === [self::CORRELATION_ASPECT];
} catch (InvalidArgumentException) {
return false;
}
}

/**
* @param list<array<string, mixed>> $rowsToCombine
* @return list<int>
*/
private function combineRows(array $rowsToCombine): array
{
if (count($rowsToCombine) <= 1) {
return [];
}

// first is the Oldest, last is the Newest
$first = $rowsToCombine[0];
$last = $rowsToCombine[array_key_last($rowsToCombine)];
$newHistoryData = json_decode($first['history_data'], true, flags: JSON_THROW_ON_ERROR);
foreach (array_slice($rowsToCombine, 1) as $rowToCombine) {
$currentHistoryData = json_decode($rowToCombine['history_data'], true, flags: JSON_THROW_ON_ERROR);

$newHistoryData['oldRecord'] = [
...$currentHistoryData['oldRecord'],
...$newHistoryData['oldRecord'],
];
$newHistoryData['newRecord'] = [
...$newHistoryData['newRecord'],
...$currentHistoryData['newRecord'],
];
}

// if one field is the same, we can remove it from the diff, because it is not a change
$newHistoryData = $this->reduce($newHistoryData);

// if you change the text and than change it back, the resulting diff will be empty, so we can just delete all history entries
if ($this->isTheSame($newHistoryData['oldRecord'], $newHistoryData['newRecord'])) {
return array_map(intval(...), array_column($rowsToCombine, 'uid'));
}

$this->sysHistoryRepository->updateHistoryData((int)$last['uid'], $newHistoryData);

return array_map(intval(...), array_column(array_slice($rowsToCombine, 0, -1), 'uid'));
}

/**
* @param array<string, mixed> $oldRecord
* @param array<string, mixed> $newRecord
*/
private function isTheSame(array $oldRecord, array $newRecord): bool
{
unset($oldRecord['l18n_diffsource'], $newRecord['l18n_diffsource']);

$normalizeScalarValue = static fn(mixed $value): mixed => is_scalar($value) ? (string)$value : $value;
$oldRecord = array_map($normalizeScalarValue, $oldRecord);
$newRecord = array_map($normalizeScalarValue, $newRecord);

return $oldRecord === $newRecord;
}

/**
* @param array{oldRecord: array<string, mixed>, newRecord: array<string, mixed>} $newHistoryData
* @return array{oldRecord: array<string, mixed>, newRecord: array<string, mixed>}
*/
private function reduce(array $newHistoryData): array
{
foreach ($newHistoryData['oldRecord'] as $field => $value) {
if (isset($newHistoryData['newRecord'][$field]) && $newHistoryData['newRecord'][$field] === $value) {
unset($newHistoryData['oldRecord'][$field], $newHistoryData['newRecord'][$field]);
}
}

return $newHistoryData;
}
}
61 changes: 61 additions & 0 deletions Classes/SysHistory/SysHistoryRepository.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
<?php

declare(strict_types=1);

namespace TYPO3\CMS\VisualEditor\SysHistory;

use DateTimeImmutable;
use Symfony\Component\DependencyInjection\Attribute\AsAlias;
use TYPO3\CMS\Core\Database\ConnectionPool;

use function array_map;

#[AsAlias(SysHistoryRepositoryInterface::class)]
final readonly class SysHistoryRepository implements SysHistoryRepositoryInterface
{
public function __construct(
private ConnectionPool $connectionPool,
) {
}

/**
* @return list<array<string, mixed>>
*/
public function fetchSysHistory(int $timeToFetch): array
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_history');
$result = $queryBuilder
->select('*')
->from('sys_history')
->where($queryBuilder->expr()->gte('tstamp', (new DateTimeImmutable())->getTimestamp() - $timeToFetch))
->orderBy('uid', 'ASC')
->executeQuery();
return $result->fetchAllAssociative();
}

/**
* @param list<int> $toBeDeletedUids
*/
public function deleteEntries(array $toBeDeletedUids): void
{
if ($toBeDeletedUids) {
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_history');
$queryBuilder
->delete('sys_history')
->where($queryBuilder->expr()->in('uid', array_map(intval(...), $toBeDeletedUids)))
->executeStatement();
}
}

/**
* @param array<string, mixed> $newHistoryData
*/
public function updateHistoryData(int $uid, array $newHistoryData): void
{
$this->connectionPool->getConnectionForTable('sys_history')->update(
'sys_history',
['history_data' => json_encode($newHistoryData, JSON_THROW_ON_ERROR)],
['uid' => $uid],
);
}
}
23 changes: 23 additions & 0 deletions Classes/SysHistory/SysHistoryRepositoryInterface.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?php

declare(strict_types=1);

namespace TYPO3\CMS\VisualEditor\SysHistory;

interface SysHistoryRepositoryInterface
{
/**
* @return list<array<string, mixed>>
*/
public function fetchSysHistory(int $timeToFetch): array;

/**
* @param list<int> $toBeDeletedUids
*/
public function deleteEntries(array $toBeDeletedUids): void;

/**
* @param array<string, mixed> $newHistoryData
*/
public function updateHistoryData(int $uid, array $newHistoryData): void;
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ export class VeAutoSaveToggle extends LitElement {
super.connectedCallback();

if (!this.disposeUpdateEditorStateListener) {
this.disposeUpdateEditorStateListener = onMessageDebounced('updateEditorState', this.#onEditorStateMessage.bind(this), 300);
this.disposeUpdateEditorStateListener = onMessageDebounced('updateEditorState', this.#onEditorStateMessage.bind(this), 1000);
}

this.addEventListener('click', this.onClick);
Expand Down
51 changes: 51 additions & 0 deletions Tests/Unit/SysHistory/RecordingSysHistoryRepository.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
<?php

declare(strict_types=1);

namespace TYPO3\CMS\VisualEditor\Tests\Unit\SysHistory;

use TYPO3\CMS\VisualEditor\SysHistory\SysHistoryRepositoryInterface;

/**
* only use this in Tests.
* @test
*/
final class RecordingSysHistoryRepository implements SysHistoryRepositoryInterface
{
/** @var list<list<int>> */
public array $recordedDeleteEntries = [];

/** @var array<int, array<string, mixed>> */
public array $recordedUpdateHistoryData = [];

/**
* @param list<array<string, mixed>> $sysHistoryData
*/
public function __construct(private readonly array $sysHistoryData = [])
{
}

/**
* @return list<array<string, mixed>>
*/
public function fetchSysHistory(int $timeToFetch): array
{
return $this->sysHistoryData;
}

/**
* @param list<int> $toBeDeletedUids
*/
public function deleteEntries(array $toBeDeletedUids): void
{
$this->recordedDeleteEntries[] = $toBeDeletedUids;
}

/**
* @param array<string, mixed> $newHistoryData
*/
public function updateHistoryData(int $uid, array $newHistoryData): void
{
$this->recordedUpdateHistoryData[$uid] = $newHistoryData;
}
}
Loading