additional, improved transformers / skipIf to eliminate records
This commit is contained in:
Reindl David (IT-PTR-CEN2-SL10)
@@ -0,0 +1,414 @@
|
||||
<?php
|
||||
|
||||
namespace UbsCsvTransformer\Tests;
|
||||
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use UbsCsvTransformer\FireflyImporter;
|
||||
use UbsCsvTransformer\DebugLogger;
|
||||
|
||||
/**
|
||||
* Tests for the chunked-import state file / resume feature.
|
||||
*
|
||||
* Strategy: subclass FireflyImporter and override import() so no real HTTP or
|
||||
* CLI call is made. The override is configured per test via a callable queue.
|
||||
*/
|
||||
class FireflyImporterChunkStateTest extends TestCase
|
||||
{
|
||||
/** @var string Temporary directory for CSV and state files */
|
||||
private string $tmpDir;
|
||||
|
||||
/** @var string Path to a throwaway JSON config file */
|
||||
private string $jsonConfig;
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
DebugLogger::reset();
|
||||
|
||||
$this->tmpDir = sys_get_temp_dir() . '/ffi_state_test_' . uniqid('', true);
|
||||
mkdir($this->tmpDir, 0700, true);
|
||||
|
||||
// Minimal Firefly importer config (format v3)
|
||||
$configData = [
|
||||
'version' => 3,
|
||||
'flow' => 'csv',
|
||||
'roles' => ['amount'],
|
||||
'default_account' => 1,
|
||||
];
|
||||
$this->jsonConfig = $this->tmpDir . '/ff-config.json';
|
||||
file_put_contents($this->jsonConfig, json_encode($configData));
|
||||
}
|
||||
|
||||
protected function tearDown(): void
|
||||
{
|
||||
// Remove all temp files
|
||||
foreach (glob($this->tmpDir . '/*') ?: [] as $f) {
|
||||
@unlink($f);
|
||||
}
|
||||
@rmdir($this->tmpDir);
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Creates an importer stub whose import() calls return results from $queue
|
||||
* in order. Each element of the queue is either true (success) or false (failure).
|
||||
*
|
||||
* @param array<bool> $importResultQueue
|
||||
* @param int $chunkSize
|
||||
*/
|
||||
private function makeImporter(array $importResultQueue, int $chunkSize): FireflyImporter
|
||||
{
|
||||
$config = [
|
||||
'mode' => 'http',
|
||||
'importerUrl' => 'https://example.com',
|
||||
'accessToken' => 'test-secret-1234567',
|
||||
'personalSecret' => 'test-pat',
|
||||
'jsonConfig' => $this->jsonConfig,
|
||||
'chunkSize' => $chunkSize,
|
||||
];
|
||||
|
||||
$queue = $importResultQueue;
|
||||
|
||||
return new class ($config, $queue) extends FireflyImporter {
|
||||
/** @var array<bool> */
|
||||
private array $queue;
|
||||
|
||||
/** @param array<bool> $queue */
|
||||
public function __construct(array $config, array $queue)
|
||||
{
|
||||
parent::__construct($config);
|
||||
$this->queue = $queue;
|
||||
}
|
||||
|
||||
public function import(string $csvFile): array
|
||||
{
|
||||
$success = array_shift($this->queue) ?? true;
|
||||
if ($success) {
|
||||
return [
|
||||
'success' => true,
|
||||
'exit_code' => 200,
|
||||
'output' => ['stdout' => '', 'stderr' => ''],
|
||||
'duration' => 1.0,
|
||||
'csv_file' => $csvFile,
|
||||
'summary' => [
|
||||
'completed' => true,
|
||||
'created' => 1,
|
||||
'by_type' => ['withdrawal' => 1],
|
||||
'duplicates' => 0,
|
||||
'errors' => [],
|
||||
],
|
||||
];
|
||||
}
|
||||
return [
|
||||
'success' => false,
|
||||
'error' => 'Simulated failure',
|
||||
'output' => ['stdout' => '', 'stderr' => ''],
|
||||
'exit_code' => 500,
|
||||
];
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a CSV with $dataRows data rows (each row has two columns).
|
||||
*/
|
||||
private function writeCsv(string $path, int $dataRows): void
|
||||
{
|
||||
$fp = fopen($path, 'w');
|
||||
assert($fp !== false);
|
||||
fputcsv($fp, ['col_a', 'col_b'], ',', '"', '\\');
|
||||
for ($i = 1; $i <= $dataRows; $i++) {
|
||||
fputcsv($fp, ["val_a_{$i}", "val_b_{$i}"], ',', '"', '\\');
|
||||
}
|
||||
fclose($fp);
|
||||
}
|
||||
|
||||
private function stateFile(string $csvPath): string
|
||||
{
|
||||
return $csvPath . '.ffi-state.json';
|
||||
}
|
||||
|
||||
// ─── Tests ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* When chunkSize is 0, import() is used directly — no state file should appear.
|
||||
*/
|
||||
public function testNoStateFileWhenChunkingNotUsed(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 5);
|
||||
|
||||
$importer = $this->makeImporter([true], 0);
|
||||
$result = $importer->importChunked($csv);
|
||||
|
||||
$this->assertTrue($result['success']);
|
||||
$this->assertFileDoesNotExist($this->stateFile($csv));
|
||||
}
|
||||
|
||||
/**
|
||||
* When the file has fewer rows than chunkSize, no chunking occurs — no state file.
|
||||
*/
|
||||
public function testNoStateFileWhenRowsBelowChunkSize(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 3);
|
||||
|
||||
$importer = $this->makeImporter([true], 10);
|
||||
$importer->importChunked($csv);
|
||||
|
||||
$this->assertFileDoesNotExist($this->stateFile($csv));
|
||||
}
|
||||
|
||||
/**
|
||||
* After chunk 1 of 3 fails, the state file must exist and record 0 completed chunks.
|
||||
*/
|
||||
public function testStateFileCreatedOnFirstChunkFailure(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 9); // 3 chunks of 3
|
||||
|
||||
// Chunk 1 fails immediately
|
||||
$importer = $this->makeImporter([false], 3);
|
||||
$result = $importer->importChunked($csv);
|
||||
|
||||
$this->assertFalse($result['success']);
|
||||
$this->assertFileExists($this->stateFile($csv));
|
||||
|
||||
/** @var array<string, mixed> $state */
|
||||
$state = json_decode((string) file_get_contents($this->stateFile($csv)), true);
|
||||
$this->assertSame([], $state['completed_chunks']);
|
||||
}
|
||||
|
||||
/**
|
||||
* After chunks 1 and 2 succeed but chunk 3 fails, the state file records [0, 1].
|
||||
*/
|
||||
public function testStateFileRecordsCompletedChunksOnPartialFailure(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 9); // 3 chunks of 3
|
||||
|
||||
// Chunks 0, 1 succeed; chunk 2 fails
|
||||
$importer = $this->makeImporter([true, true, false], 3);
|
||||
$result = $importer->importChunked($csv);
|
||||
|
||||
$this->assertFalse($result['success']);
|
||||
$this->assertFileExists($this->stateFile($csv));
|
||||
|
||||
/** @var array<string, mixed> $state */
|
||||
$state = json_decode((string) file_get_contents($this->stateFile($csv)), true);
|
||||
$this->assertSame([0, 1], $state['completed_chunks']);
|
||||
$this->assertArrayHasKey('0', $state['chunk_results']);
|
||||
$this->assertArrayHasKey('1', $state['chunk_results']);
|
||||
}
|
||||
|
||||
/**
|
||||
* After full success the state file is deleted automatically.
|
||||
*/
|
||||
public function testStateFileDeletedAfterFullSuccess(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 6); // 2 chunks of 3
|
||||
|
||||
$importer = $this->makeImporter([true, true], 3);
|
||||
$result = $importer->importChunked($csv);
|
||||
|
||||
$this->assertTrue($result['success']);
|
||||
$this->assertFileDoesNotExist($this->stateFile($csv));
|
||||
}
|
||||
|
||||
/**
|
||||
* On a second run with an existing state showing [0, 1] done, only chunk 2
|
||||
* (index 2) should call import() — i.e., exactly one call is made.
|
||||
*/
|
||||
public function testResumeSkipsAlreadyCompletedChunks(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 9); // 3 chunks of 3
|
||||
|
||||
// ── First run: chunks 0+1 succeed, chunk 2 fails ────────────────────
|
||||
$run1 = $this->makeImporter([true, true, false], 3);
|
||||
$run1->importChunked($csv);
|
||||
|
||||
$this->assertFileExists($this->stateFile($csv));
|
||||
|
||||
// ── Second run: only chunk 2 should be attempted ────────────────────
|
||||
// We record how many times import() is actually called via a counting wrapper
|
||||
$counter = new \stdClass();
|
||||
$counter->value = 0;
|
||||
|
||||
$config = [
|
||||
'mode' => 'http',
|
||||
'importerUrl' => 'https://example.com',
|
||||
'accessToken' => 'test-secret-1234567',
|
||||
'personalSecret' => 'test-pat',
|
||||
'jsonConfig' => $this->jsonConfig,
|
||||
'chunkSize' => 3,
|
||||
];
|
||||
|
||||
$run2 = new class ($config, $counter) extends FireflyImporter {
|
||||
private \stdClass $counter;
|
||||
|
||||
public function __construct(array $config, \stdClass $counter)
|
||||
{
|
||||
parent::__construct($config);
|
||||
$this->counter = $counter;
|
||||
}
|
||||
|
||||
public function import(string $csvFile): array
|
||||
{
|
||||
$this->counter->value++;
|
||||
return [
|
||||
'success' => true,
|
||||
'exit_code' => 200,
|
||||
'output' => ['stdout' => '', 'stderr' => ''],
|
||||
'duration' => 1.0,
|
||||
'csv_file' => $csvFile,
|
||||
'summary' => [
|
||||
'completed' => true,
|
||||
'created' => 1,
|
||||
'by_type' => ['withdrawal' => 1],
|
||||
'duplicates' => 0,
|
||||
'errors' => [],
|
||||
],
|
||||
];
|
||||
}
|
||||
};
|
||||
|
||||
$result2 = $run2->importChunked($csv);
|
||||
|
||||
$this->assertTrue($result2['success']);
|
||||
$this->assertSame(1, $counter->value, 'Only the 1 remaining chunk (index 2) should be imported');
|
||||
$this->assertFileDoesNotExist($this->stateFile($csv), 'State file must be deleted after full success');
|
||||
}
|
||||
|
||||
/**
|
||||
* A state file whose total_rows does not match the current CSV is silently
|
||||
* discarded and a fresh import is started.
|
||||
*/
|
||||
public function testStaleMismatchedStateIsIgnored(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 9); // 3 chunks of 3
|
||||
|
||||
// Plant a stale state file with a wrong total_rows
|
||||
$staleState = [
|
||||
'csv_file' => realpath($csv) ?: $csv,
|
||||
'total_rows' => 99, // wrong
|
||||
'chunk_size' => 3,
|
||||
'total_chunks' => 3,
|
||||
'completed_chunks' => [0, 1],
|
||||
'chunk_results' => [],
|
||||
'created_at' => '2020-01-01T00:00:00+00:00',
|
||||
'updated_at' => '2020-01-01T00:00:00+00:00',
|
||||
];
|
||||
file_put_contents($this->stateFile($csv), json_encode($staleState));
|
||||
|
||||
// All 3 chunks should be called (fresh start despite stale state)
|
||||
$counter = new \stdClass();
|
||||
$counter->value = 0;
|
||||
|
||||
$config = [
|
||||
'mode' => 'http',
|
||||
'importerUrl' => 'https://example.com',
|
||||
'accessToken' => 'test-secret-1234567',
|
||||
'personalSecret' => 'test-pat',
|
||||
'jsonConfig' => $this->jsonConfig,
|
||||
'chunkSize' => 3,
|
||||
];
|
||||
|
||||
$importer = new class ($config, $counter) extends FireflyImporter {
|
||||
private \stdClass $counter;
|
||||
|
||||
public function __construct(array $config, \stdClass $counter)
|
||||
{
|
||||
parent::__construct($config);
|
||||
$this->counter = $counter;
|
||||
}
|
||||
|
||||
public function import(string $csvFile): array
|
||||
{
|
||||
$this->counter->value++;
|
||||
return [
|
||||
'success' => true,
|
||||
'exit_code' => 200,
|
||||
'output' => ['stdout' => '', 'stderr' => ''],
|
||||
'duration' => 1.0,
|
||||
'csv_file' => $csvFile,
|
||||
'summary' => [
|
||||
'completed' => true,
|
||||
'created' => 1,
|
||||
'by_type' => ['withdrawal' => 1],
|
||||
'duplicates' => 0,
|
||||
'errors' => [],
|
||||
],
|
||||
];
|
||||
}
|
||||
};
|
||||
|
||||
$result = $importer->importChunked($csv);
|
||||
|
||||
$this->assertTrue($result['success']);
|
||||
$this->assertSame(3, $counter->value, 'All 3 chunks must be imported when stale state is discarded');
|
||||
}
|
||||
|
||||
/**
|
||||
* A corrupt (non-JSON) state file is silently discarded; no exception is thrown.
|
||||
*/
|
||||
public function testCorruptStateFileIsIgnored(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 6); // 2 chunks of 3
|
||||
|
||||
file_put_contents($this->stateFile($csv), '{this is not valid json!!!}');
|
||||
|
||||
$importer = $this->makeImporter([true, true], 3);
|
||||
$result = $importer->importChunked($csv);
|
||||
|
||||
$this->assertTrue($result['success']);
|
||||
}
|
||||
|
||||
/**
|
||||
* resetImportState() deletes an existing state file.
|
||||
*/
|
||||
public function testResetImportStateClearsStateFile(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 9);
|
||||
|
||||
// Plant a state file
|
||||
file_put_contents($this->stateFile($csv), '{}');
|
||||
$this->assertFileExists($this->stateFile($csv));
|
||||
|
||||
$importer = $this->makeImporter([], 3);
|
||||
$importer->resetImportState($csv);
|
||||
|
||||
$this->assertFileDoesNotExist($this->stateFile($csv));
|
||||
}
|
||||
|
||||
/**
|
||||
* hasResumeState() returns false when no state file is present.
|
||||
*/
|
||||
public function testHasResumeStateReturnsFalseWithoutStateFile(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 9);
|
||||
|
||||
$importer = $this->makeImporter([], 3);
|
||||
$this->assertFalse($importer->hasResumeState($csv));
|
||||
}
|
||||
|
||||
/**
|
||||
* hasResumeState() returns true after a partial failure creates a valid state file.
|
||||
*/
|
||||
public function testHasResumeStateReturnsTrueAfterPartialFailure(): void
|
||||
{
|
||||
$csv = $this->tmpDir . '/test.csv';
|
||||
$this->writeCsv($csv, 9); // 3 chunks of 3
|
||||
|
||||
$importer = $this->makeImporter([true, false], 3); // chunk 2 (index 1) fails
|
||||
$importer->importChunked($csv);
|
||||
|
||||
$importer2 = $this->makeImporter([], 3);
|
||||
$this->assertTrue($importer2->hasResumeState($csv));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user