#!/usr/bin/env php handleTest($argc, $argv), 'transform' => handleTransform($argc, $argv), 'validate' => handleValidate($argc, $argv), 'auto-import' => handleAutoImport($argc, $argv), 'help', '-h', '--help' => showHelp(), default => throw new Exception("Unbekanntes Kommando: $command"), }; } catch (Exception $e) { fwrite(STDERR, "\n❌ ERROR: " . $e->getMessage() . "\n\n"); exit(1); } // ============================================================================ // COMMAND HANDLERS // ============================================================================ /** * Returns true when the active shell locale is German (de_*) */ function isGermanLocale(): bool { foreach (['LANG', 'LC_ALL', 'LC_MESSAGES', 'LANGUAGE'] as $var) { $val = getenv($var); if ($val !== false && $val !== '') { return str_starts_with(strtolower($val), 'de'); } } return false; } /** * Show help and usage instructions */ function showHelp(): void { if (isGermanLocale()) { echo <<<'HELP_DE' ╔════════════════════════════════════════════════════════════════════════════╗ ║ Firefly Import Preprocessor - Kommandozeilen-Tool ║ ║ ║ ║ Ein schlankes PHP 8 Tool zur Transformation von UBS E-Banking Exporten ║ ║ in ein Firefly III kompatibles Format. ║ ╚════════════════════════════════════════════════════════════════════════════╝ VERWENDUNG: transformer [command] [options] KOMMANDOS: test [input] [config] [options] Testet die Transformation mit limitierter Zeilenzahl Optionen: --rows=N Nur N Zeilen verarbeiten (Standard: 10) --output=FILE, -o Ergebnis auch in Datei schreiben Beispiel: transformer test ubs-export.csv config.json --rows=5 transformer test ubs-export.csv config.json -o test-output.csv transform [input] [config] [options] Transformiert eine komplette CSV-Datei Optionen: --output=FILE, -o Output-Pfad (Standard: input-transformed.csv) --do-import Nach der Transformation in Firefly III importieren Beispiel: transformer transform ubs-export.csv config.json transformer transform ubs-export.csv config.json --do-import validate [config] [options] Validiert die Konfigurationsdatei Optionen: --strict Strikte Validierung (empfohlen) Beispiel: transformer validate config.json transformer validate config.json --strict auto-import [config] [options] Überwacht Quellverzeichnis und verarbeitet neue Dateien Optionen: --watch Kontinuierliche Überwachung (Daemon-Modus) --interval=SEC Prüfintervall in Sekunden (Standard: 60) --dry-run Zeigt was gemacht würde (keine echte Verarbeitung) Beispiel: transformer auto-import config.json transformer auto-import config.json --watch --interval=30 help, -h, --help Zeige diese Hilfe GLOBALE OPTIONEN: --debug, -d Aktiviere Debug-Modus (detaillierte Ausgaben) INSTALLATION: 1. PHP 8.1+ muss installiert sein php --version 2. Autoloader-Setup (wähle eins): Option A: Mit Composer (empfohlen) composer install Option B: Manuell - Dateien in Verzeichnisstruktur: ff-imp-preprocessor/ ├── bin/transformer.php ├── src/*.php └── config/config.json 3. Ausführbar machen: chmod +x bin/transformer.php 4. Konfiguration anpassen: cp config/config.example.json config/config.json nano config/config.json BEISPIELE: # Transformation mit Test (erste 5 Zeilen) ./bin/transformer test data/ubs-export.csv config/config.json --rows=5 # Komplette Transformation ./bin/transformer transform data/ubs-export.csv config/config.json \ --output=output/firefly-import.csv # Konfiguration validieren ./bin/transformer validate config/config.json --strict # Auto-Import mit Überwachung starten ./bin/transformer auto-import config/config.json --watch # Nur nächste Datei verarbeiten ./bin/transformer auto-import config/config.json KONFIGURATION: Die config.json muss folgende Struktur haben: { "metadata": { "extractionRules": {...} }, "csvStructure": { "delimiter": ";", ... }, "columnTransformations": { ... }, "fireflyImport": { "apiUrl": "...", "apiKey": "..." }, "directories": { "source": "./import/source", "output": "./import/output", "archive": "./import/archive", "error": "./import/error" } } DOKUMENTATION: Siehe README.md für vollständige Dokumentation LIZENZ: GPL 3 HELP_DE; return; } echo <<<'HELP_EN' ╔════════════════════════════════════════════════════════════════════════════╗ ║ Firefly Import Preprocessor - Command Line Tool ║ ║ ║ ║ A lightweight PHP 8 tool for transforming UBS E-Banking exports ║ ║ into a Firefly III compatible format. ║ ╚════════════════════════════════════════════════════════════════════════════╝ USAGE: transformer [command] [options] COMMANDS: test [input] [config] [options] Tests the transformation with a limited number of rows Options: --rows=N Process only N rows (default: 10) --output=FILE, -o Also write result to file Example: transformer test ubs-export.csv config.json --rows=5 transformer test ubs-export.csv config.json -o test-output.csv transform [input] [config] [options] Transforms a complete CSV file Options: --output=FILE, -o Output path (default: input-transformed.csv) --do-import Import into Firefly III after transformation Example: transformer transform ubs-export.csv config.json transformer transform ubs-export.csv config.json --do-import validate [config] [options] Validates the configuration file Options: --strict Strict validation (recommended) Example: transformer validate config.json transformer validate config.json --strict auto-import [config] [options] Monitors source directory and processes new files Options: --watch Continuous monitoring (daemon mode) --interval=SEC Check interval in seconds (default: 60) --dry-run Show what would be done (no actual processing) Example: transformer auto-import config.json transformer auto-import config.json --watch --interval=30 help, -h, --help Show this help GLOBAL OPTIONS: --debug, -d Enable debug mode (detailed output) INSTALLATION: 1. PHP 8.1+ must be installed php --version 2. Autoloader setup (choose one): Option A: With Composer (recommended) composer install Option B: Manual - files in directory structure: ff-imp-preprocessor/ ├── bin/transformer.php ├── src/*.php └── config/config.json 3. Make executable: chmod +x bin/transformer.php 4. Adjust configuration: cp config/config.example.json config/config.json nano config/config.json EXAMPLES: # Test transformation (first 5 rows) ./bin/transformer test data/ubs-export.csv config/config.json --rows=5 # Full transformation ./bin/transformer transform data/ubs-export.csv config/config.json \ --output=output/firefly-import.csv # Validate configuration ./bin/transformer validate config/config.json --strict # Start auto-import with monitoring ./bin/transformer auto-import config/config.json --watch # Process only next file ./bin/transformer auto-import config/config.json CONFIGURATION: The config.json must have the following structure: { "metadata": { "extractionRules": {...} }, "csvStructure": { "delimiter": ";", ... }, "columnTransformations": { ... }, "fireflyImport": { "apiUrl": "...", "apiKey": "..." }, "directories": { "source": "./import/source", "output": "./import/output", "archive": "./import/archive", "error": "./import/error" } } DOCUMENTATION: See README.md for full documentation LICENSE: GPL 3 HELP_EN; } /** * Expands ~ to absolute home directory and resolves relative paths */ function expandPath(string $path): string { if (str_starts_with($path, '~/') || $path === '~') { $homeEnv = getenv('HOME'); $pwInfo = posix_getpwuid(posix_getuid()); $home = $homeEnv !== false && $homeEnv !== '' ? $homeEnv : ($pwInfo !== false ? $pwInfo['dir'] : '~'); $path = $home . substr($path, 1); } // Resolve relative paths against cwd (without realpath, so non-existent dirs are allowed) if (!str_starts_with($path, '/')) { $path = getcwd() . '/' . $path; } return $path; } /** * Parses CLI options into an associative array */ function parseOptions(array $argv, int $startIndex = 0): array { $options = []; for ($i = $startIndex; $i < count($argv); $i++) { if (strpos($argv[$i], '--') === 0) { $parts = explode('=', substr($argv[$i], 2), 2); $options[$parts[0]] = $parts[1] ?? true; } elseif (strpos($argv[$i], '-') === 0 && strlen($argv[$i]) > 1) { $options[substr($argv[$i], 1)] = true; } } return $options; } /** * Tests transformation with a limited number of rows */ function handleTest(int $argc, array $argv): void { if ($argc < 4) { throw new Exception("Usage: transformer test [input-file] [config-file] [options]"); } $inputFile = $argv[2]; $configFile = $argv[3]; $options = parseOptions($argv, 4); $debug = isset($options['debug']) || isset($options['d']); $maxRows = isset($options['rows']) ? (int)$options['rows'] : 10; $outputFile = $options['output'] ?? $options['o'] ?? null; if (!file_exists($inputFile)) { throw new Exception("Input file not found: $inputFile"); } if (!file_exists($configFile)) { throw new Exception("Configuration file not found: $configFile"); } echo "\n📊 TEST-MODUS: Verarbeite max. $maxRows Zeilen\n"; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"; $configLoader = new ConfigurationLoader($configFile); $config = $configLoader->load(); $engine = new TransformerEngine($configLoader, $debug); $result = $engine->transform($inputFile, $maxRows); // Ausgabe Statistiken echo "\n✅ STATISTIKEN:\n"; echo " Verarbeitete Zeilen: " . $result['rowsProcessed'] . "\n"; echo " Metadaten extrahiert: " . count($result['metadata'] ?? []) . "\n"; echo " Output-Spalten: " . $result['outputColumns'] . "\n"; if (!empty($result['metadata'])) { echo "\n📋 EXTRAHIERTE METADATEN:\n"; foreach ($result['metadata'] as $key => $value) { $display = substr($value, 0, 50); if (strlen($value) > 50) { $display .= "..."; } echo " $key: $display\n"; } } if (!empty($result['sampleData'])) { $sampleCount = min(5, count($result['sampleData'])); echo "\n📝 BEISPIEL-DATEN ($sampleCount Zeilen):\n"; foreach (array_slice($result['sampleData'], 0, $sampleCount) as $index => $row) { echo " Zeile " . ($index + 1) . ": "; foreach ($row as $col => $value) { $val = substr($value, 0, 30); if (strlen($value) > 30) { $val .= "..."; } //if (!is_int($col)) { echo "$col=$val | "; //} } echo "\n"; } } if ($outputFile) { echo "\n💾 Output-Datei: $outputFile\n"; } if ($debug) { echo DebugLogger::format(true); } echo "\n✅ Test erfolgreich!\n\n"; } /** * Transforms a complete CSV file */ function handleTransform(int $argc, array $argv): void { if ($argc < 4) { throw new Exception("Usage: transformer transform [input-file] [config-file] [options]"); } $inputFile = $argv[2]; $configFile = $argv[3]; $options = parseOptions($argv, 4); $debug = isset($options['debug']) || isset($options['d']); $outputFile = $options['output'] ?? $options['o'] ?? null; $doImport = isset($options['do-import']); $resetImport = isset($options['reset-import']); if (!file_exists($inputFile)) { throw new Exception("Input file not found: $inputFile"); } if (!file_exists($configFile)) { throw new Exception("Configuration file not found: $configFile"); } echo "\n🚀 TRANSFORMATION\n"; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"; $configLoader = new ConfigurationLoader($configFile); $config = $configLoader->load(); // --output overrides target directory and filename from configuration if ($outputFile !== null) { $outputFile = expandPath($outputFile); $configLoader->set('directories.output', dirname($outputFile)); $configLoader->set('csvStructure.outputFilename', basename($outputFile)); } $engine = new TransformerEngine($configLoader, $debug); $result = $engine->transform($inputFile); echo "✅ Transformation complete!\n"; echo " Output file: " . ($result['outputFile'] ?? 'N/A') . "\n"; echo " Rows transformed: " . ($result['rowsProcessed'] ?? 0) . "\n"; if ($doImport) { if (!empty($config['fireflyImport'])) { echo "\n🚀 FIREFLY III IMPORT\n"; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"; $fireflyConfig = $config['fireflyImport']; $importer = new FireflyImporter($fireflyConfig); $outputCsv = $result['outputFile'] ?? ''; if ($resetImport) { $importer->resetImportState($outputCsv); echo " ℹ️ Import state cleared — starting fresh.\n"; } elseif ($importer->hasResumeState($outputCsv)) { $stateRaw = @file_get_contents($outputCsv . '.ffi-state.json'); $stateData = is_string($stateRaw) ? json_decode($stateRaw, true) : null; if (is_array($stateData)) { $doneSoFar = count((array) ($stateData['completed_chunks'] ?? [])); $totalSoFar = (int) ($stateData['total_chunks'] ?? 0); echo " ℹ️ Resuming previous import: {$doneSoFar}/{$totalSoFar} chunks already completed.\n"; echo " Add --reset-import to start over from scratch.\n"; } } $inChunkedMode = false; // Detect the system timezone: PHP CLI often defaults to UTC even when the OS // is configured otherwise. Read /etc/localtime symlink to get the real zone. $localTzName = date_default_timezone_get(); if (is_link('/etc/localtime')) { $link = (string) readlink('/etc/localtime'); if (preg_match('#zoneinfo/(.+)$#', $link, $tzMatch) === 1) { $localTzName = $tzMatch[1]; } } $localTz = new \DateTimeZone($localTzName); $importer->setProgressCallback( function (string $event, array $data) use (&$inChunkedMode, $localTz): void { static $chunkHadRetry = false; $ts = '[' . (new \DateTimeImmutable('now', $localTz))->format('H:i:s') . ']'; if ($event === 'chunk_start') { $inChunkedMode = true; $chunkHadRetry = false; echo " ⏳ {$ts} Chunk {$data['chunk']}/{$data['total']} ({$data['rows']} rows)..."; flush(); } elseif ($event === 'chunk_done') { $d = round((float) ($data['result']['duration'] ?? 0), 1); $status = $data['result']['success'] ? 'done' : 'failed'; if ($chunkHadRetry) { // After retries the line is already terminated — print a full new line echo " ✅ {$ts} Chunk {$data['chunk']}/{$data['total']}: {$status} ({$d}s)\n"; } else { echo " {$status} ({$d}s)\n"; } flush(); } elseif ($event === 'chunk_retry') { $chunkHadRetry = true; $err = (string) ($data['error'] ?? ''); $msg = $err !== '' ? " — {$err}" : ''; echo "\n 🔄 {$ts} Chunk {$data['chunk']}/{$data['total']}: attempt {$data['attempt']}/{$data['max_attempts']} failed{$msg}\n"; flush(); } elseif ($event === 'chunk_delay') { $ctx = ($data['context'] ?? '') === 'retry' ? 'retry' : 'next chunk'; echo " ⏸ {$ts} Waiting {$data['seconds']}s before {$ctx}...\n"; flush(); } elseif ($event === 'chunk_skip') { echo " ⏭ {$ts} Chunk {$data['chunk']}/{$data['total']} already completed — skipping\n"; flush(); } elseif ($event === 'request_start' && !$inChunkedMode) { echo " ⏳ {$ts} Sending to importer...\n"; flush(); } } ); $outputDelimiter = (string) ($config['csvStructure']['outputDelimiter'] ?? ','); $importResult = $importer->importChunked($outputCsv, $outputDelimiter); $duration = $importResult['duration'] ?? null; $chunks = $importResult['chunks'] ?? null; $summary = $importResult['summary'] ?? null; if ($importResult['success']) { if (is_array($summary)) { $created = $summary['created'] ?? 0; $byType = $summary['by_type'] ?? []; $completed = $summary['completed'] ?? false; $duplicates = $summary['duplicates'] ?? 0; $errors = $summary['errors'] ?? []; $status = $completed ? '✅ Import complete!' : '⚠️ Import finished (no "Done!" marker received)'; echo $status . ($duration !== null ? " ({$duration}s)" : '') . "\n"; echo " Transactions created: {$created}\n"; $typeLabels = ['deposit' => 'Deposits', 'withdrawal' => 'Withdrawals', 'transfer' => 'Transfers']; foreach ($byType as $type => $count) { $label = $typeLabels[$type] ?? ucfirst($type); echo " {$label}: {$count}\n"; } if ($duplicates > 0) { echo " ⚠️ Duplicates skipped: {$duplicates}\n"; } if (!empty($errors)) { $errorCount = count($errors); echo " ❌ Errors ({$errorCount}):\n"; foreach ($errors as $err) { echo " - {$err}\n"; } } } else { echo "✅ Import complete!" . ($duration !== null ? " ({$duration}s)" : '') . "\n"; if (!empty($importResult['output']['stdout'])) { echo $importResult['output']['stdout'] . "\n"; } } } else { $errorMsg = $importResult['error'] ?? ('HTTP ' . ($importResult['exit_code'] ?? '?')); $chunksData = $importResult['chunks'] ?? null; if (is_array($chunksData) && $chunksData['total'] > 1) { $failedChunk = $chunksData['done'] + 1; echo "❌ Import failed at chunk {$failedChunk}/{$chunksData['total']}: {$errorMsg}\n"; echo " Run the same command again to resume from where it stopped.\n"; echo " Add --reset-import to start over from scratch.\n"; } else { echo "❌ Import failed: {$errorMsg}\n"; } // Only dump the raw response body in debug mode if ($debug && !empty($importResult['output']['stdout'])) { echo $importResult['output']['stdout'] . "\n"; } if (!empty($importResult['output']['stderr'])) { echo $importResult['output']['stderr'] . "\n"; } } } else { echo "\n⚠️ --do-import specified but no fireflyImport section found in config.\n"; } } if ($debug) { echo DebugLogger::format(true); } echo "\n✅ Done!\n\n"; } /** * Validates the configuration file */ function handleValidate(int $argc, array $argv): void { if ($argc < 3) { throw new Exception("Usage: transformer validate [config-file] [options]"); } $configFile = $argv[2]; $options = parseOptions($argv, 3); $strict = isset($options['strict']); if (!file_exists($configFile)) { throw new Exception("Configuration file not found: $configFile"); } echo "\n✔️ KONFIGURATION VALIDIEREN\n"; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"; $configLoader = new ConfigurationLoader($configFile); try { $config = $configLoader->load(); // Basic validation echo "✅ JSON-Format valide\n"; $required = ['metadata', 'csvStructure', 'columnTransformations']; $missing = []; foreach ($required as $key) { if (isset($config[$key])) { echo "✅ Abschnitt '$key' vorhanden\n"; } else { echo "⚠️ Abschnitt '$key' fehlt\n"; $missing[] = $key; } } // Firefly validation if (isset($config['fireflyImport'])) { echo "✅ Firefly III Konfiguration vorhanden\n"; if (empty($config['fireflyImport']['apiUrl'])) { echo "⚠️ Firefly III API-URL fehlt\n"; if ($strict) { throw new Exception("Strikte Validierung: Firefly III API-URL erforderlich"); } } if (empty($config['fireflyImport']['apiKey'])) { echo "⚠️ Firefly III API-Key fehlt\n"; if ($strict) { throw new Exception("Strikte Validierung: Firefly III API-Key erforderlich"); } } } else { echo "⚠️ Firefly III Konfiguration nicht vorhanden (optional)\n"; } // Directory validation if (isset($config['directories'])) { echo "✅ Verzeichnisse konfiguriert\n"; $dirs = ['source', 'output', 'archive', 'error']; foreach ($dirs as $dir) { if (!empty($config['directories'][$dir])) { echo " ✅ $dir: " . $config['directories'][$dir] . "\n"; } } } if (empty($missing)) { echo "\n✅ Konfiguration ist VALIDE!\n\n"; } else { if ($strict) { throw new Exception("Strikte Validierung: " . count($missing) . " erforderliche Abschnitte fehlen"); } echo "\n⚠️ Konfiguration hat Warnungen aber ist funktional\n\n"; } } catch (Exception $e) { throw new Exception("Validation error: " . $e->getMessage()); } } /** * Auto-import with directory monitoring */ function handleAutoImport(int $argc, array $argv): void { if ($argc < 3) { throw new Exception("Usage: transformer auto-import [config-file] [options]"); } $configFile = $argv[2]; $options = parseOptions($argv, 3); $debug = isset($options['debug']) || isset($options['d']); if (!file_exists($configFile)) { throw new Exception("Configuration file not found: $configFile"); } $configLoader = new ConfigurationLoader($configFile); $config = $configLoader->load(); $sourceDir = $config['directories']['source'] ?? './import/source'; $outputDir = $config['directories']['output'] ?? './import/output'; $archiveDir = $config['directories']['archive'] ?? './import/archive'; $errorDir = $config['directories']['error'] ?? './import/error'; $dryRun = isset($options['dry-run']); $watch = isset($options['watch']); $interval = isset($options['interval']) ? (int)$options['interval'] : 60; // Create directories foreach ([$sourceDir, $outputDir, $archiveDir, $errorDir] as $dir) { if (!is_dir($dir)) { mkdir($dir, 0755, true); } } echo "\n🔍 AUTO-IMPORT GESTARTET\n"; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"; echo " Quellverzeichnis: $sourceDir\n"; echo " Output-Verzeichnis: $outputDir\n"; echo " Archiv-Verzeichnis: $archiveDir\n"; if ($watch) { echo " Mode: WATCH (kontinuierlich)\n"; echo " Intervall: {$interval}s\n"; } else { echo " Mode: EINMALIG\n"; } if ($dryRun) { echo " Dry-Run: JA (keine echten Operationen)\n"; } echo "\n"; if ($watch) { echo "⏳ Drücke Ctrl+C zum Beenden.\n\n"; $running = true; /** @phpstan-ignore while.alwaysTrue (intentional infinite loop — terminated only via Ctrl+C / SIGINT) */ while ($running) { processImportDirectory($sourceDir, $outputDir, $archiveDir, $errorDir, $config, $configFile, $dryRun, $debug); sleep($interval); } } else { processImportDirectory($sourceDir, $outputDir, $archiveDir, $errorDir, $config, $configFile, $dryRun, $debug); } } /** * Processes directory containing CSV files */ function processImportDirectory(string $sourceDir, string $outputDir, string $archiveDir, string $errorDir, array $config, string $configFile, bool $dryRun = false, bool $debug = false): void { if (!is_dir($sourceDir)) { return; } $files = glob($sourceDir . '/*.csv'); if (empty($files)) { return; } foreach ($files as $file) { $basename = basename($file); try { echo "📄 Verarbeite: $basename ... "; $configLoader = new ConfigurationLoader($configFile); $config = $configLoader->load(); $engine = new TransformerEngine($configLoader, $debug); $outputFile = $outputDir . '/' . str_replace('.csv', '-transformed.csv', $basename); if (!$dryRun) { $result = $engine->transform($file); $outputFile = $result['outputFile'] ?? $outputFile; // Archive original file $archiveFile = $archiveDir . '/' . $basename; if (!rename($file, $archiveFile)) { throw new Exception("Could not archive file"); } // Firefly Import if (!empty($config['fireflyImport'])) { $importer = new FireflyImporter($config['fireflyImport']); $importer->import($outputFile); } } echo "✅\n"; } catch (Exception $e) { echo "❌ " . $e->getMessage() . "\n"; if (!$dryRun) { // Move to error directory $errorFile = $errorDir . '/' . $basename; @rename($file, $errorFile); } } } }