839 lines
30 KiB
PHP
Executable File
839 lines
30 KiB
PHP
Executable File
#!/usr/bin/env php
|
||
|
||
<?php
|
||
|
||
/**
|
||
* Firefly Import Preprocessor - Kommandozeilen-Einstiegspunkt
|
||
*
|
||
* PHP 8.1+ Tool zur Transformation von UBS E-Banking CSV-Exporten
|
||
* in ein Firefly III kompatibles Format.
|
||
*
|
||
* Voraussetzung: composer install
|
||
*/
|
||
|
||
require_once __DIR__ . '/../vendor/autoload.php';
|
||
|
||
use UbsCsvTransformer\TransformerEngine;
|
||
use UbsCsvTransformer\ConfigurationLoader;
|
||
use UbsCsvTransformer\FireflyImporter;
|
||
use UbsCsvTransformer\DebugLogger;
|
||
|
||
// ============================================================================
|
||
// CLI argument processing
|
||
// ============================================================================
|
||
|
||
$argc = $_SERVER['argc'] ?? 0;
|
||
$argv = $_SERVER['argv'] ?? [];
|
||
|
||
if ($argc < 2) {
|
||
showHelp();
|
||
exit(0);
|
||
}
|
||
|
||
// Debug mode can be enabled
|
||
$debug = in_array('--debug', $argv) || in_array('-d', $argv);
|
||
|
||
// Extract command
|
||
$command = $argv[1];
|
||
|
||
try {
|
||
match ($command) {
|
||
'test' => handleTest($argc, $argv),
|
||
'transform' => handleTransform($argc, $argv),
|
||
'validate' => handleValidate($argc, $argv),
|
||
'auto-import' => handleAutoImport($argc, $argv),
|
||
'help', '-h', '--help' => showHelp(),
|
||
default => throw new Exception("Unbekanntes Kommando: $command"),
|
||
};
|
||
} catch (Exception $e) {
|
||
fwrite(STDERR, "\n❌ ERROR: " . $e->getMessage() . "\n\n");
|
||
exit(1);
|
||
}
|
||
|
||
// ============================================================================
|
||
// COMMAND HANDLERS
|
||
// ============================================================================
|
||
|
||
/**
|
||
* Returns true when the active shell locale is German (de_*)
|
||
*/
|
||
function isGermanLocale(): bool
|
||
{
|
||
foreach (['LANG', 'LC_ALL', 'LC_MESSAGES', 'LANGUAGE'] as $var) {
|
||
$val = getenv($var);
|
||
if ($val !== false && $val !== '') {
|
||
return str_starts_with(strtolower($val), 'de');
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Show help and usage instructions
|
||
*/
|
||
function showHelp(): void
|
||
{
|
||
if (isGermanLocale()) {
|
||
echo <<<'HELP_DE'
|
||
╔════════════════════════════════════════════════════════════════════════════╗
|
||
║ Firefly Import Preprocessor - Kommandozeilen-Tool ║
|
||
║ ║
|
||
║ Ein schlankes PHP 8 Tool zur Transformation von UBS E-Banking Exporten ║
|
||
║ in ein Firefly III kompatibles Format. ║
|
||
╚════════════════════════════════════════════════════════════════════════════╝
|
||
|
||
VERWENDUNG:
|
||
transformer [command] [options]
|
||
|
||
KOMMANDOS:
|
||
|
||
test [input] [config] [options]
|
||
Testet die Transformation mit limitierter Zeilenzahl
|
||
Optionen:
|
||
--rows=N Nur N Zeilen verarbeiten (Standard: 10)
|
||
--output=FILE, -o Ergebnis auch in Datei schreiben
|
||
Beispiel:
|
||
transformer test ubs-export.csv config.json --rows=5
|
||
transformer test ubs-export.csv config.json -o test-output.csv
|
||
|
||
transform [input] [config] [options]
|
||
Transformiert eine komplette CSV-Datei
|
||
Optionen:
|
||
--output=FILE, -o Output-Pfad (Standard: input-transformed.csv)
|
||
--do-import Nach der Transformation in Firefly III importieren
|
||
Beispiel:
|
||
transformer transform ubs-export.csv config.json
|
||
transformer transform ubs-export.csv config.json --do-import
|
||
|
||
validate [config] [options]
|
||
Validiert die Konfigurationsdatei
|
||
Optionen:
|
||
--strict Strikte Validierung (empfohlen)
|
||
Beispiel:
|
||
transformer validate config.json
|
||
transformer validate config.json --strict
|
||
|
||
auto-import [config] [options]
|
||
Überwacht Quellverzeichnis und verarbeitet neue Dateien
|
||
Optionen:
|
||
--watch Kontinuierliche Überwachung (Daemon-Modus)
|
||
--interval=SEC Prüfintervall in Sekunden (Standard: 60)
|
||
--dry-run Zeigt was gemacht würde (keine echte Verarbeitung)
|
||
Beispiel:
|
||
transformer auto-import config.json
|
||
transformer auto-import config.json --watch --interval=30
|
||
|
||
help, -h, --help
|
||
Zeige diese Hilfe
|
||
|
||
GLOBALE OPTIONEN:
|
||
--debug, -d Aktiviere Debug-Modus (detaillierte Ausgaben)
|
||
|
||
INSTALLATION:
|
||
|
||
1. PHP 8.1+ muss installiert sein
|
||
php --version
|
||
|
||
2. Autoloader-Setup (wähle eins):
|
||
Option A: Mit Composer (empfohlen)
|
||
composer install
|
||
Option B: Manuell - Dateien in Verzeichnisstruktur:
|
||
ff-imp-preprocessor/
|
||
├── bin/transformer.php
|
||
├── src/*.php
|
||
└── config/config.json
|
||
|
||
3. Ausführbar machen:
|
||
chmod +x bin/transformer.php
|
||
|
||
4. Konfiguration anpassen:
|
||
cp config/config.example.json config/config.json
|
||
nano config/config.json
|
||
|
||
BEISPIELE:
|
||
|
||
# Transformation mit Test (erste 5 Zeilen)
|
||
./bin/transformer test data/ubs-export.csv config/config.json --rows=5
|
||
|
||
# Komplette Transformation
|
||
./bin/transformer transform data/ubs-export.csv config/config.json \
|
||
--output=output/firefly-import.csv
|
||
|
||
# Konfiguration validieren
|
||
./bin/transformer validate config/config.json --strict
|
||
|
||
# Auto-Import mit Überwachung starten
|
||
./bin/transformer auto-import config/config.json --watch
|
||
|
||
# Nur nächste Datei verarbeiten
|
||
./bin/transformer auto-import config/config.json
|
||
|
||
KONFIGURATION:
|
||
|
||
Die config.json muss folgende Struktur haben:
|
||
{
|
||
"metadata": { "extractionRules": {...} },
|
||
"csvStructure": { "delimiter": ";", ... },
|
||
"columnTransformations": { ... },
|
||
"fireflyImport": { "apiUrl": "...", "apiKey": "..." },
|
||
"directories": {
|
||
"source": "./import/source",
|
||
"output": "./import/output",
|
||
"archive": "./import/archive",
|
||
"error": "./import/error"
|
||
}
|
||
}
|
||
|
||
DOKUMENTATION:
|
||
|
||
Siehe README.md für vollständige Dokumentation
|
||
|
||
LIZENZ:
|
||
|
||
GPL 3
|
||
|
||
HELP_DE;
|
||
return;
|
||
}
|
||
|
||
echo <<<'HELP_EN'
|
||
╔════════════════════════════════════════════════════════════════════════════╗
|
||
║ Firefly Import Preprocessor - Command Line Tool ║
|
||
║ ║
|
||
║ A lightweight PHP 8 tool for transforming UBS E-Banking exports ║
|
||
║ into a Firefly III compatible format. ║
|
||
╚════════════════════════════════════════════════════════════════════════════╝
|
||
|
||
USAGE:
|
||
transformer [command] [options]
|
||
|
||
COMMANDS:
|
||
|
||
test [input] [config] [options]
|
||
Tests the transformation with a limited number of rows
|
||
Options:
|
||
--rows=N Process only N rows (default: 10)
|
||
--output=FILE, -o Also write result to file
|
||
Example:
|
||
transformer test ubs-export.csv config.json --rows=5
|
||
transformer test ubs-export.csv config.json -o test-output.csv
|
||
|
||
transform [input] [config] [options]
|
||
Transforms a complete CSV file
|
||
Options:
|
||
--output=FILE, -o Output path (default: input-transformed.csv)
|
||
--do-import Import into Firefly III after transformation
|
||
Example:
|
||
transformer transform ubs-export.csv config.json
|
||
transformer transform ubs-export.csv config.json --do-import
|
||
|
||
validate [config] [options]
|
||
Validates the configuration file
|
||
Options:
|
||
--strict Strict validation (recommended)
|
||
Example:
|
||
transformer validate config.json
|
||
transformer validate config.json --strict
|
||
|
||
auto-import [config] [options]
|
||
Monitors source directory and processes new files
|
||
Options:
|
||
--watch Continuous monitoring (daemon mode)
|
||
--interval=SEC Check interval in seconds (default: 60)
|
||
--dry-run Show what would be done (no actual processing)
|
||
Example:
|
||
transformer auto-import config.json
|
||
transformer auto-import config.json --watch --interval=30
|
||
|
||
help, -h, --help
|
||
Show this help
|
||
|
||
GLOBAL OPTIONS:
|
||
--debug, -d Enable debug mode (detailed output)
|
||
|
||
INSTALLATION:
|
||
|
||
1. PHP 8.1+ must be installed
|
||
php --version
|
||
|
||
2. Autoloader setup (choose one):
|
||
Option A: With Composer (recommended)
|
||
composer install
|
||
Option B: Manual - files in directory structure:
|
||
ff-imp-preprocessor/
|
||
├── bin/transformer.php
|
||
├── src/*.php
|
||
└── config/config.json
|
||
|
||
3. Make executable:
|
||
chmod +x bin/transformer.php
|
||
|
||
4. Adjust configuration:
|
||
cp config/config.example.json config/config.json
|
||
nano config/config.json
|
||
|
||
EXAMPLES:
|
||
|
||
# Test transformation (first 5 rows)
|
||
./bin/transformer test data/ubs-export.csv config/config.json --rows=5
|
||
|
||
# Full transformation
|
||
./bin/transformer transform data/ubs-export.csv config/config.json \
|
||
--output=output/firefly-import.csv
|
||
|
||
# Validate configuration
|
||
./bin/transformer validate config/config.json --strict
|
||
|
||
# Start auto-import with monitoring
|
||
./bin/transformer auto-import config/config.json --watch
|
||
|
||
# Process only next file
|
||
./bin/transformer auto-import config/config.json
|
||
|
||
CONFIGURATION:
|
||
|
||
The config.json must have the following structure:
|
||
{
|
||
"metadata": { "extractionRules": {...} },
|
||
"csvStructure": { "delimiter": ";", ... },
|
||
"columnTransformations": { ... },
|
||
"fireflyImport": { "apiUrl": "...", "apiKey": "..." },
|
||
"directories": {
|
||
"source": "./import/source",
|
||
"output": "./import/output",
|
||
"archive": "./import/archive",
|
||
"error": "./import/error"
|
||
}
|
||
}
|
||
|
||
DOCUMENTATION:
|
||
|
||
See README.md for full documentation
|
||
|
||
LICENSE:
|
||
|
||
GPL 3
|
||
|
||
HELP_EN;
|
||
}
|
||
|
||
/**
|
||
* Expands ~ to absolute home directory and resolves relative paths
|
||
*/
|
||
function expandPath(string $path): string
|
||
{
|
||
if (str_starts_with($path, '~/') || $path === '~') {
|
||
$homeEnv = getenv('HOME');
|
||
$pwInfo = posix_getpwuid(posix_getuid());
|
||
$home = $homeEnv !== false && $homeEnv !== '' ? $homeEnv : ($pwInfo !== false ? $pwInfo['dir'] : '~');
|
||
$path = $home . substr($path, 1);
|
||
}
|
||
|
||
// Resolve relative paths against cwd (without realpath, so non-existent dirs are allowed)
|
||
if (!str_starts_with($path, '/')) {
|
||
$path = getcwd() . '/' . $path;
|
||
}
|
||
|
||
return $path;
|
||
}
|
||
|
||
/**
|
||
* Parses CLI options into an associative array
|
||
*/
|
||
function parseOptions(array $argv, int $startIndex = 0): array
|
||
{
|
||
$options = [];
|
||
|
||
for ($i = $startIndex; $i < count($argv); $i++) {
|
||
if (strpos($argv[$i], '--') === 0) {
|
||
$parts = explode('=', substr($argv[$i], 2), 2);
|
||
$options[$parts[0]] = $parts[1] ?? true;
|
||
} elseif (strpos($argv[$i], '-') === 0 && strlen($argv[$i]) > 1) {
|
||
$options[substr($argv[$i], 1)] = true;
|
||
}
|
||
}
|
||
|
||
return $options;
|
||
}
|
||
|
||
/**
|
||
* Tests transformation with a limited number of rows
|
||
*/
|
||
function handleTest(int $argc, array $argv): void
|
||
{
|
||
if ($argc < 4) {
|
||
throw new Exception("Usage: transformer test [input-file] [config-file] [options]");
|
||
}
|
||
|
||
$inputFile = $argv[2];
|
||
$configFile = $argv[3];
|
||
$options = parseOptions($argv, 4);
|
||
$debug = isset($options['debug']) || isset($options['d']);
|
||
|
||
$maxRows = isset($options['rows']) ? (int)$options['rows'] : 10;
|
||
$outputFile = $options['output'] ?? $options['o'] ?? null;
|
||
|
||
if (!file_exists($inputFile)) {
|
||
throw new Exception("Input file not found: $inputFile");
|
||
}
|
||
if (!file_exists($configFile)) {
|
||
throw new Exception("Configuration file not found: $configFile");
|
||
}
|
||
|
||
echo "\n📊 TEST-MODUS: Verarbeite max. $maxRows Zeilen\n";
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
|
||
|
||
$configLoader = new ConfigurationLoader($configFile);
|
||
$config = $configLoader->load();
|
||
|
||
$engine = new TransformerEngine($configLoader, $debug);
|
||
$result = $engine->transform($inputFile, $maxRows);
|
||
|
||
// Ausgabe Statistiken
|
||
echo "\n✅ STATISTIKEN:\n";
|
||
echo " Verarbeitete Zeilen: " . $result['rowsProcessed'] . "\n";
|
||
echo " Metadaten extrahiert: " . count($result['metadata'] ?? []) . "\n";
|
||
echo " Output-Spalten: " . $result['outputColumns'] . "\n";
|
||
|
||
if (!empty($result['metadata'])) {
|
||
echo "\n📋 EXTRAHIERTE METADATEN:\n";
|
||
foreach ($result['metadata'] as $key => $value) {
|
||
$display = substr($value, 0, 50);
|
||
if (strlen($value) > 50) {
|
||
$display .= "...";
|
||
}
|
||
echo " $key: $display\n";
|
||
}
|
||
}
|
||
|
||
if (!empty($result['sampleData'])) {
|
||
$sampleCount = min(5, count($result['sampleData']));
|
||
echo "\n📝 BEISPIEL-DATEN ($sampleCount Zeilen):\n";
|
||
foreach (array_slice($result['sampleData'], 0, $sampleCount) as $index => $row) {
|
||
echo " Zeile " . ($index + 1) . ": ";
|
||
foreach ($row as $col => $value) {
|
||
$val = substr($value, 0, 30);
|
||
if (strlen($value) > 30) {
|
||
$val .= "...";
|
||
}
|
||
//if (!is_int($col)) {
|
||
echo "$col=$val | ";
|
||
//}
|
||
}
|
||
echo "\n";
|
||
}
|
||
}
|
||
|
||
if ($outputFile) {
|
||
echo "\n💾 Output-Datei: $outputFile\n";
|
||
}
|
||
|
||
if ($debug) {
|
||
echo DebugLogger::format(true);
|
||
}
|
||
|
||
echo "\n✅ Test erfolgreich!\n\n";
|
||
}
|
||
|
||
/**
|
||
* Transforms a complete CSV file
|
||
*/
|
||
function handleTransform(int $argc, array $argv): void
|
||
{
|
||
if ($argc < 4) {
|
||
throw new Exception("Usage: transformer transform [input-file] [config-file] [options]");
|
||
}
|
||
|
||
$inputFile = $argv[2];
|
||
$configFile = $argv[3];
|
||
$options = parseOptions($argv, 4);
|
||
$debug = isset($options['debug']) || isset($options['d']);
|
||
|
||
$outputFile = $options['output'] ?? $options['o'] ?? null;
|
||
$doImport = isset($options['do-import']);
|
||
$resetImport = isset($options['reset-import']);
|
||
|
||
if (!file_exists($inputFile)) {
|
||
throw new Exception("Input file not found: $inputFile");
|
||
}
|
||
if (!file_exists($configFile)) {
|
||
throw new Exception("Configuration file not found: $configFile");
|
||
}
|
||
|
||
echo "\n🚀 TRANSFORMATION\n";
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
|
||
|
||
$configLoader = new ConfigurationLoader($configFile);
|
||
$config = $configLoader->load();
|
||
|
||
// --output overrides target directory and filename from configuration
|
||
if ($outputFile !== null) {
|
||
$outputFile = expandPath($outputFile);
|
||
$configLoader->set('directories.output', dirname($outputFile));
|
||
$configLoader->set('csvStructure.outputFilename', basename($outputFile));
|
||
}
|
||
|
||
$engine = new TransformerEngine($configLoader, $debug);
|
||
$result = $engine->transform($inputFile);
|
||
|
||
echo "✅ Transformation complete!\n";
|
||
echo " Output file: " . ($result['outputFile'] ?? 'N/A') . "\n";
|
||
echo " Rows transformed: " . ($result['rowsProcessed'] ?? 0) . "\n";
|
||
|
||
if ($doImport) {
|
||
if (!empty($config['fireflyImport'])) {
|
||
echo "\n🚀 FIREFLY III IMPORT\n";
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
|
||
|
||
$fireflyConfig = $config['fireflyImport'];
|
||
$importer = new FireflyImporter($fireflyConfig);
|
||
|
||
$outputCsv = $result['outputFile'] ?? '';
|
||
|
||
if ($resetImport) {
|
||
$importer->resetImportState($outputCsv);
|
||
echo " ℹ️ Import state cleared — starting fresh.\n";
|
||
} elseif ($importer->hasResumeState($outputCsv)) {
|
||
$stateRaw = @file_get_contents($outputCsv . '.ffi-state.json');
|
||
$stateData = is_string($stateRaw) ? json_decode($stateRaw, true) : null;
|
||
if (is_array($stateData)) {
|
||
$doneSoFar = count((array) ($stateData['completed_chunks'] ?? []));
|
||
$totalSoFar = (int) ($stateData['total_chunks'] ?? 0);
|
||
echo " ℹ️ Resuming previous import: {$doneSoFar}/{$totalSoFar} chunks already completed.\n";
|
||
echo " Add --reset-import to start over from scratch.\n";
|
||
}
|
||
}
|
||
|
||
$inChunkedMode = false;
|
||
|
||
// Detect the system timezone: PHP CLI often defaults to UTC even when the OS
|
||
// is configured otherwise. Read /etc/localtime symlink to get the real zone.
|
||
$localTzName = date_default_timezone_get();
|
||
if (is_link('/etc/localtime')) {
|
||
$link = (string) readlink('/etc/localtime');
|
||
if (preg_match('#zoneinfo/(.+)$#', $link, $tzMatch) === 1) {
|
||
$localTzName = $tzMatch[1];
|
||
}
|
||
}
|
||
$localTz = new \DateTimeZone($localTzName);
|
||
|
||
$importer->setProgressCallback(
|
||
function (string $event, array $data) use (&$inChunkedMode, $localTz): void {
|
||
static $chunkHadRetry = false;
|
||
$ts = '[' . (new \DateTimeImmutable('now', $localTz))->format('H:i:s') . ']';
|
||
if ($event === 'chunk_start') {
|
||
$inChunkedMode = true;
|
||
$chunkHadRetry = false;
|
||
echo " ⏳ {$ts} Chunk {$data['chunk']}/{$data['total']} ({$data['rows']} rows)...";
|
||
flush();
|
||
} elseif ($event === 'chunk_done') {
|
||
$d = round((float) ($data['result']['duration'] ?? 0), 1);
|
||
$status = $data['result']['success'] ? 'done' : 'failed';
|
||
if ($chunkHadRetry) {
|
||
// After retries the line is already terminated — print a full new line
|
||
echo " ✅ {$ts} Chunk {$data['chunk']}/{$data['total']}: {$status} ({$d}s)\n";
|
||
} else {
|
||
echo " {$status} ({$d}s)\n";
|
||
}
|
||
flush();
|
||
} elseif ($event === 'chunk_retry') {
|
||
$chunkHadRetry = true;
|
||
$err = (string) ($data['error'] ?? '');
|
||
$msg = $err !== '' ? " — {$err}" : '';
|
||
echo "\n 🔄 {$ts} Chunk {$data['chunk']}/{$data['total']}: attempt {$data['attempt']}/{$data['max_attempts']} failed{$msg}\n";
|
||
flush();
|
||
} elseif ($event === 'chunk_delay') {
|
||
$ctx = ($data['context'] ?? '') === 'retry' ? 'retry' : 'next chunk';
|
||
echo " ⏸ {$ts} Waiting {$data['seconds']}s before {$ctx}...\n";
|
||
flush();
|
||
} elseif ($event === 'chunk_skip') {
|
||
echo " ⏭ {$ts} Chunk {$data['chunk']}/{$data['total']} already completed — skipping\n";
|
||
flush();
|
||
} elseif ($event === 'request_start' && !$inChunkedMode) {
|
||
echo " ⏳ {$ts} Sending to importer...\n";
|
||
flush();
|
||
}
|
||
}
|
||
);
|
||
|
||
$outputDelimiter = (string) ($config['csvStructure']['outputDelimiter'] ?? ',');
|
||
$importResult = $importer->importChunked($outputCsv, $outputDelimiter);
|
||
|
||
$duration = $importResult['duration'] ?? null;
|
||
$chunks = $importResult['chunks'] ?? null;
|
||
$summary = $importResult['summary'] ?? null;
|
||
|
||
if ($importResult['success']) {
|
||
if (is_array($summary)) {
|
||
$created = $summary['created'] ?? 0;
|
||
$byType = $summary['by_type'] ?? [];
|
||
$completed = $summary['completed'] ?? false;
|
||
$duplicates = $summary['duplicates'] ?? 0;
|
||
$errors = $summary['errors'] ?? [];
|
||
|
||
$status = $completed ? '✅ Import complete!' : '⚠️ Import finished (no "Done!" marker received)';
|
||
echo $status . ($duration !== null ? " ({$duration}s)" : '') . "\n";
|
||
echo " Transactions created: {$created}\n";
|
||
|
||
$typeLabels = ['deposit' => 'Deposits', 'withdrawal' => 'Withdrawals', 'transfer' => 'Transfers'];
|
||
foreach ($byType as $type => $count) {
|
||
$label = $typeLabels[$type] ?? ucfirst($type);
|
||
echo " {$label}: {$count}\n";
|
||
}
|
||
|
||
if ($duplicates > 0) {
|
||
echo " ⚠️ Duplicates skipped: {$duplicates}\n";
|
||
}
|
||
|
||
if (!empty($errors)) {
|
||
$errorCount = count($errors);
|
||
echo " ❌ Errors ({$errorCount}):\n";
|
||
foreach ($errors as $err) {
|
||
echo " - {$err}\n";
|
||
}
|
||
}
|
||
} else {
|
||
echo "✅ Import complete!" . ($duration !== null ? " ({$duration}s)" : '') . "\n";
|
||
if (!empty($importResult['output']['stdout'])) {
|
||
echo $importResult['output']['stdout'] . "\n";
|
||
}
|
||
}
|
||
} else {
|
||
$errorMsg = $importResult['error']
|
||
?? ('HTTP ' . ($importResult['exit_code'] ?? '?'));
|
||
$chunksData = $importResult['chunks'] ?? null;
|
||
if (is_array($chunksData) && $chunksData['total'] > 1) {
|
||
$failedChunk = $chunksData['done'] + 1;
|
||
echo "❌ Import failed at chunk {$failedChunk}/{$chunksData['total']}: {$errorMsg}\n";
|
||
echo " Run the same command again to resume from where it stopped.\n";
|
||
echo " Add --reset-import to start over from scratch.\n";
|
||
} else {
|
||
echo "❌ Import failed: {$errorMsg}\n";
|
||
}
|
||
// Only dump the raw response body in debug mode
|
||
if ($debug && !empty($importResult['output']['stdout'])) {
|
||
echo $importResult['output']['stdout'] . "\n";
|
||
}
|
||
if (!empty($importResult['output']['stderr'])) {
|
||
echo $importResult['output']['stderr'] . "\n";
|
||
}
|
||
}
|
||
} else {
|
||
echo "\n⚠️ --do-import specified but no fireflyImport section found in config.\n";
|
||
}
|
||
}
|
||
|
||
if ($debug) {
|
||
echo DebugLogger::format(true);
|
||
}
|
||
|
||
echo "\n✅ Done!\n\n";
|
||
}
|
||
|
||
/**
|
||
* Validates the configuration file
|
||
*/
|
||
function handleValidate(int $argc, array $argv): void
|
||
{
|
||
if ($argc < 3) {
|
||
throw new Exception("Usage: transformer validate [config-file] [options]");
|
||
}
|
||
|
||
$configFile = $argv[2];
|
||
$options = parseOptions($argv, 3);
|
||
$strict = isset($options['strict']);
|
||
|
||
if (!file_exists($configFile)) {
|
||
throw new Exception("Configuration file not found: $configFile");
|
||
}
|
||
|
||
echo "\n✔️ KONFIGURATION VALIDIEREN\n";
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
|
||
|
||
$configLoader = new ConfigurationLoader($configFile);
|
||
|
||
try {
|
||
$config = $configLoader->load();
|
||
|
||
// Basic validation
|
||
echo "✅ JSON-Format valide\n";
|
||
|
||
$required = ['metadata', 'csvStructure', 'columnTransformations'];
|
||
$missing = [];
|
||
|
||
foreach ($required as $key) {
|
||
if (isset($config[$key])) {
|
||
echo "✅ Abschnitt '$key' vorhanden\n";
|
||
} else {
|
||
echo "⚠️ Abschnitt '$key' fehlt\n";
|
||
$missing[] = $key;
|
||
}
|
||
}
|
||
|
||
// Firefly validation
|
||
if (isset($config['fireflyImport'])) {
|
||
echo "✅ Firefly III Konfiguration vorhanden\n";
|
||
if (empty($config['fireflyImport']['apiUrl'])) {
|
||
echo "⚠️ Firefly III API-URL fehlt\n";
|
||
if ($strict) {
|
||
throw new Exception("Strikte Validierung: Firefly III API-URL erforderlich");
|
||
}
|
||
}
|
||
if (empty($config['fireflyImport']['apiKey'])) {
|
||
echo "⚠️ Firefly III API-Key fehlt\n";
|
||
if ($strict) {
|
||
throw new Exception("Strikte Validierung: Firefly III API-Key erforderlich");
|
||
}
|
||
}
|
||
} else {
|
||
echo "⚠️ Firefly III Konfiguration nicht vorhanden (optional)\n";
|
||
}
|
||
|
||
// Directory validation
|
||
if (isset($config['directories'])) {
|
||
echo "✅ Verzeichnisse konfiguriert\n";
|
||
$dirs = ['source', 'output', 'archive', 'error'];
|
||
foreach ($dirs as $dir) {
|
||
if (!empty($config['directories'][$dir])) {
|
||
echo " ✅ $dir: " . $config['directories'][$dir] . "\n";
|
||
}
|
||
}
|
||
}
|
||
|
||
if (empty($missing)) {
|
||
echo "\n✅ Konfiguration ist VALIDE!\n\n";
|
||
} else {
|
||
if ($strict) {
|
||
throw new Exception("Strikte Validierung: " . count($missing) . " erforderliche Abschnitte fehlen");
|
||
}
|
||
echo "\n⚠️ Konfiguration hat Warnungen aber ist funktional\n\n";
|
||
}
|
||
} catch (Exception $e) {
|
||
throw new Exception("Validation error: " . $e->getMessage());
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Auto-import with directory monitoring
|
||
*/
|
||
function handleAutoImport(int $argc, array $argv): void
|
||
{
|
||
if ($argc < 3) {
|
||
throw new Exception("Usage: transformer auto-import [config-file] [options]");
|
||
}
|
||
|
||
$configFile = $argv[2];
|
||
$options = parseOptions($argv, 3);
|
||
$debug = isset($options['debug']) || isset($options['d']);
|
||
|
||
if (!file_exists($configFile)) {
|
||
throw new Exception("Configuration file not found: $configFile");
|
||
}
|
||
|
||
$configLoader = new ConfigurationLoader($configFile);
|
||
$config = $configLoader->load();
|
||
|
||
$sourceDir = $config['directories']['source'] ?? './import/source';
|
||
$outputDir = $config['directories']['output'] ?? './import/output';
|
||
$archiveDir = $config['directories']['archive'] ?? './import/archive';
|
||
$errorDir = $config['directories']['error'] ?? './import/error';
|
||
$dryRun = isset($options['dry-run']);
|
||
$watch = isset($options['watch']);
|
||
$interval = isset($options['interval']) ? (int)$options['interval'] : 60;
|
||
|
||
// Create directories
|
||
foreach ([$sourceDir, $outputDir, $archiveDir, $errorDir] as $dir) {
|
||
if (!is_dir($dir)) {
|
||
mkdir($dir, 0755, true);
|
||
}
|
||
}
|
||
|
||
echo "\n🔍 AUTO-IMPORT GESTARTET\n";
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
|
||
echo " Quellverzeichnis: $sourceDir\n";
|
||
echo " Output-Verzeichnis: $outputDir\n";
|
||
echo " Archiv-Verzeichnis: $archiveDir\n";
|
||
|
||
if ($watch) {
|
||
echo " Mode: WATCH (kontinuierlich)\n";
|
||
echo " Intervall: {$interval}s\n";
|
||
} else {
|
||
echo " Mode: EINMALIG\n";
|
||
}
|
||
|
||
if ($dryRun) {
|
||
echo " Dry-Run: JA (keine echten Operationen)\n";
|
||
}
|
||
|
||
echo "\n";
|
||
|
||
if ($watch) {
|
||
echo "⏳ Drücke Ctrl+C zum Beenden.\n\n";
|
||
$running = true;
|
||
/** @phpstan-ignore while.alwaysTrue (intentional infinite loop — terminated only via Ctrl+C / SIGINT) */
|
||
while ($running) {
|
||
processImportDirectory($sourceDir, $outputDir, $archiveDir, $errorDir, $config, $configFile, $dryRun, $debug);
|
||
sleep($interval);
|
||
}
|
||
} else {
|
||
processImportDirectory($sourceDir, $outputDir, $archiveDir, $errorDir, $config, $configFile, $dryRun, $debug);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Processes directory containing CSV files
|
||
*/
|
||
function processImportDirectory(string $sourceDir, string $outputDir, string $archiveDir, string $errorDir, array $config, string $configFile, bool $dryRun = false, bool $debug = false): void
|
||
{
|
||
if (!is_dir($sourceDir)) {
|
||
return;
|
||
}
|
||
|
||
$files = glob($sourceDir . '/*.csv');
|
||
|
||
if (empty($files)) {
|
||
return;
|
||
}
|
||
|
||
foreach ($files as $file) {
|
||
$basename = basename($file);
|
||
|
||
try {
|
||
echo "📄 Verarbeite: $basename ... ";
|
||
|
||
$configLoader = new ConfigurationLoader($configFile);
|
||
$config = $configLoader->load();
|
||
|
||
$engine = new TransformerEngine($configLoader, $debug);
|
||
$outputFile = $outputDir . '/' . str_replace('.csv', '-transformed.csv', $basename);
|
||
|
||
if (!$dryRun) {
|
||
$result = $engine->transform($file);
|
||
$outputFile = $result['outputFile'] ?? $outputFile;
|
||
|
||
// Archive original file
|
||
$archiveFile = $archiveDir . '/' . $basename;
|
||
if (!rename($file, $archiveFile)) {
|
||
throw new Exception("Could not archive file");
|
||
}
|
||
|
||
// Firefly Import
|
||
if (!empty($config['fireflyImport'])) {
|
||
$importer = new FireflyImporter($config['fireflyImport']);
|
||
$importer->import($outputFile);
|
||
}
|
||
}
|
||
|
||
echo "✅\n";
|
||
} catch (Exception $e) {
|
||
echo "❌ " . $e->getMessage() . "\n";
|
||
|
||
if (!$dryRun) {
|
||
// Move to error directory
|
||
$errorFile = $errorDir . '/' . $basename;
|
||
@rename($file, $errorFile);
|
||
}
|
||
}
|
||
}
|
||
}
|