additional, improved transformers / skipIf to eliminate records
This commit is contained in:
Reindl David (IT-PTR-CEN2-SL10)
+164
-10
@@ -16,6 +16,7 @@ require_once __DIR__ . '/../vendor/autoload.php';
|
||||
use UbsCsvTransformer\TransformerEngine;
|
||||
use UbsCsvTransformer\ConfigurationLoader;
|
||||
use UbsCsvTransformer\FireflyImporter;
|
||||
use UbsCsvTransformer\DebugLogger;
|
||||
|
||||
// ============================================================================
|
||||
// CLI argument processing
|
||||
@@ -99,10 +100,10 @@ KOMMANDOS:
|
||||
Transformiert eine komplette CSV-Datei
|
||||
Optionen:
|
||||
--output=FILE, -o Output-Pfad (Standard: input-transformed.csv)
|
||||
--no-import Nicht automatisch in Firefly III importieren
|
||||
--do-import Nach der Transformation in Firefly III importieren
|
||||
Beispiel:
|
||||
transformer transform ubs-export.csv config.json
|
||||
transformer transform ubs-export.csv config.json -o import.csv
|
||||
transformer transform ubs-export.csv config.json --do-import
|
||||
|
||||
validate [config] [options]
|
||||
Validiert die Konfigurationsdatei
|
||||
@@ -221,10 +222,10 @@ COMMANDS:
|
||||
Transforms a complete CSV file
|
||||
Options:
|
||||
--output=FILE, -o Output path (default: input-transformed.csv)
|
||||
--no-import Do not automatically import into Firefly III
|
||||
--do-import Import into Firefly III after transformation
|
||||
Example:
|
||||
transformer transform ubs-export.csv config.json
|
||||
transformer transform ubs-export.csv config.json -o import.csv
|
||||
transformer transform ubs-export.csv config.json --do-import
|
||||
|
||||
validate [config] [options]
|
||||
Validates the configuration file
|
||||
@@ -427,6 +428,10 @@ function handleTest(int $argc, array $argv): void
|
||||
echo "\n💾 Output-Datei: $outputFile\n";
|
||||
}
|
||||
|
||||
if ($debug) {
|
||||
echo DebugLogger::format(true);
|
||||
}
|
||||
|
||||
echo "\n✅ Test erfolgreich!\n\n";
|
||||
}
|
||||
|
||||
@@ -445,6 +450,8 @@ function handleTransform(int $argc, array $argv): void
|
||||
$debug = isset($options['debug']) || isset($options['d']);
|
||||
|
||||
$outputFile = $options['output'] ?? $options['o'] ?? null;
|
||||
$doImport = isset($options['do-import']);
|
||||
$resetImport = isset($options['reset-import']);
|
||||
|
||||
if (!file_exists($inputFile)) {
|
||||
throw new Exception("Input file not found: $inputFile");
|
||||
@@ -453,11 +460,11 @@ function handleTransform(int $argc, array $argv): void
|
||||
throw new Exception("Configuration file not found: $configFile");
|
||||
}
|
||||
|
||||
echo "\n🚀 TRANSFORMATION STARTEN\n";
|
||||
echo "\n🚀 TRANSFORMATION\n";
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
|
||||
|
||||
$configLoader = new ConfigurationLoader($configFile);
|
||||
$configLoader->load();
|
||||
$config = $configLoader->load();
|
||||
|
||||
// --output overrides target directory and filename from configuration
|
||||
if ($outputFile !== null) {
|
||||
@@ -469,11 +476,158 @@ function handleTransform(int $argc, array $argv): void
|
||||
$engine = new TransformerEngine($configLoader, $debug);
|
||||
$result = $engine->transform($inputFile);
|
||||
|
||||
echo "✅ Transformation erfolgreich!\n";
|
||||
echo " Output-Datei: " . ($result['outputFile'] ?? 'N/A') . "\n";
|
||||
echo " Zeilen transformiert: " . ($result['rowsProcessed'] ?? 0) . "\n";
|
||||
echo "✅ Transformation complete!\n";
|
||||
echo " Output file: " . ($result['outputFile'] ?? 'N/A') . "\n";
|
||||
echo " Rows transformed: " . ($result['rowsProcessed'] ?? 0) . "\n";
|
||||
|
||||
echo "\n✅ Fertig!\n\n";
|
||||
if ($doImport) {
|
||||
if (!empty($config['fireflyImport'])) {
|
||||
echo "\n🚀 FIREFLY III IMPORT\n";
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
|
||||
|
||||
$fireflyConfig = $config['fireflyImport'];
|
||||
$importer = new FireflyImporter($fireflyConfig);
|
||||
|
||||
$outputCsv = $result['outputFile'] ?? '';
|
||||
|
||||
if ($resetImport) {
|
||||
$importer->resetImportState($outputCsv);
|
||||
echo " ℹ️ Import state cleared — starting fresh.\n";
|
||||
} elseif ($importer->hasResumeState($outputCsv)) {
|
||||
$stateRaw = @file_get_contents($outputCsv . '.ffi-state.json');
|
||||
$stateData = is_string($stateRaw) ? json_decode($stateRaw, true) : null;
|
||||
if (is_array($stateData)) {
|
||||
$doneSoFar = count((array) ($stateData['completed_chunks'] ?? []));
|
||||
$totalSoFar = (int) ($stateData['total_chunks'] ?? 0);
|
||||
echo " ℹ️ Resuming previous import: {$doneSoFar}/{$totalSoFar} chunks already completed.\n";
|
||||
echo " Add --reset-import to start over from scratch.\n";
|
||||
}
|
||||
}
|
||||
|
||||
$inChunkedMode = false;
|
||||
|
||||
// Detect the system timezone: PHP CLI often defaults to UTC even when the OS
|
||||
// is configured otherwise. Read /etc/localtime symlink to get the real zone.
|
||||
$localTzName = date_default_timezone_get();
|
||||
if (is_link('/etc/localtime')) {
|
||||
$link = (string) readlink('/etc/localtime');
|
||||
if (preg_match('#zoneinfo/(.+)$#', $link, $tzMatch) === 1) {
|
||||
$localTzName = $tzMatch[1];
|
||||
}
|
||||
}
|
||||
$localTz = new \DateTimeZone($localTzName);
|
||||
|
||||
$importer->setProgressCallback(
|
||||
function (string $event, array $data) use (&$inChunkedMode, $localTz): void {
|
||||
static $chunkHadRetry = false;
|
||||
$ts = '[' . (new \DateTimeImmutable('now', $localTz))->format('H:i:s') . ']';
|
||||
if ($event === 'chunk_start') {
|
||||
$inChunkedMode = true;
|
||||
$chunkHadRetry = false;
|
||||
echo " ⏳ {$ts} Chunk {$data['chunk']}/{$data['total']} ({$data['rows']} rows)...";
|
||||
flush();
|
||||
} elseif ($event === 'chunk_done') {
|
||||
$d = round((float) ($data['result']['duration'] ?? 0), 1);
|
||||
$status = $data['result']['success'] ? 'done' : 'failed';
|
||||
if ($chunkHadRetry) {
|
||||
// After retries the line is already terminated — print a full new line
|
||||
echo " ✅ {$ts} Chunk {$data['chunk']}/{$data['total']}: {$status} ({$d}s)\n";
|
||||
} else {
|
||||
echo " {$status} ({$d}s)\n";
|
||||
}
|
||||
flush();
|
||||
} elseif ($event === 'chunk_retry') {
|
||||
$chunkHadRetry = true;
|
||||
$err = (string) ($data['error'] ?? '');
|
||||
$msg = $err !== '' ? " — {$err}" : '';
|
||||
echo "\n 🔄 {$ts} Chunk {$data['chunk']}/{$data['total']}: attempt {$data['attempt']}/{$data['max_attempts']} failed{$msg}\n";
|
||||
flush();
|
||||
} elseif ($event === 'chunk_delay') {
|
||||
$ctx = ($data['context'] ?? '') === 'retry' ? 'retry' : 'next chunk';
|
||||
echo " ⏸ {$ts} Waiting {$data['seconds']}s before {$ctx}...\n";
|
||||
flush();
|
||||
} elseif ($event === 'chunk_skip') {
|
||||
echo " ⏭ {$ts} Chunk {$data['chunk']}/{$data['total']} already completed — skipping\n";
|
||||
flush();
|
||||
} elseif ($event === 'request_start' && !$inChunkedMode) {
|
||||
echo " ⏳ {$ts} Sending to importer...\n";
|
||||
flush();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
$outputDelimiter = (string) ($config['csvStructure']['outputDelimiter'] ?? ',');
|
||||
$importResult = $importer->importChunked($outputCsv, $outputDelimiter);
|
||||
|
||||
$duration = $importResult['duration'] ?? null;
|
||||
$chunks = $importResult['chunks'] ?? null;
|
||||
$summary = $importResult['summary'] ?? null;
|
||||
|
||||
if ($importResult['success']) {
|
||||
if (is_array($summary)) {
|
||||
$created = $summary['created'] ?? 0;
|
||||
$byType = $summary['by_type'] ?? [];
|
||||
$completed = $summary['completed'] ?? false;
|
||||
$duplicates = $summary['duplicates'] ?? 0;
|
||||
$errors = $summary['errors'] ?? [];
|
||||
|
||||
$status = $completed ? '✅ Import complete!' : '⚠️ Import finished (no "Done!" marker received)';
|
||||
echo $status . ($duration !== null ? " ({$duration}s)" : '') . "\n";
|
||||
echo " Transactions created: {$created}\n";
|
||||
|
||||
$typeLabels = ['deposit' => 'Deposits', 'withdrawal' => 'Withdrawals', 'transfer' => 'Transfers'];
|
||||
foreach ($byType as $type => $count) {
|
||||
$label = $typeLabels[$type] ?? ucfirst($type);
|
||||
echo " {$label}: {$count}\n";
|
||||
}
|
||||
|
||||
if ($duplicates > 0) {
|
||||
echo " ⚠️ Duplicates skipped: {$duplicates}\n";
|
||||
}
|
||||
|
||||
if (!empty($errors)) {
|
||||
$errorCount = count($errors);
|
||||
echo " ❌ Errors ({$errorCount}):\n";
|
||||
foreach ($errors as $err) {
|
||||
echo " - {$err}\n";
|
||||
}
|
||||
}
|
||||
} else {
|
||||
echo "✅ Import complete!" . ($duration !== null ? " ({$duration}s)" : '') . "\n";
|
||||
if (!empty($importResult['output']['stdout'])) {
|
||||
echo $importResult['output']['stdout'] . "\n";
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$errorMsg = $importResult['error']
|
||||
?? ('HTTP ' . ($importResult['exit_code'] ?? '?'));
|
||||
$chunksData = $importResult['chunks'] ?? null;
|
||||
if (is_array($chunksData) && $chunksData['total'] > 1) {
|
||||
$failedChunk = $chunksData['done'] + 1;
|
||||
echo "❌ Import failed at chunk {$failedChunk}/{$chunksData['total']}: {$errorMsg}\n";
|
||||
echo " Run the same command again to resume from where it stopped.\n";
|
||||
echo " Add --reset-import to start over from scratch.\n";
|
||||
} else {
|
||||
echo "❌ Import failed: {$errorMsg}\n";
|
||||
}
|
||||
// Only dump the raw response body in debug mode
|
||||
if ($debug && !empty($importResult['output']['stdout'])) {
|
||||
echo $importResult['output']['stdout'] . "\n";
|
||||
}
|
||||
if (!empty($importResult['output']['stderr'])) {
|
||||
echo $importResult['output']['stderr'] . "\n";
|
||||
}
|
||||
}
|
||||
} else {
|
||||
echo "\n⚠️ --do-import specified but no fireflyImport section found in config.\n";
|
||||
}
|
||||
}
|
||||
|
||||
if ($debug) {
|
||||
echo DebugLogger::format(true);
|
||||
}
|
||||
|
||||
echo "\n✅ Done!\n\n";
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user