shahondin1624
d48e2b7d9d
Major features: - Full backup & restore system (JSON snapshots of all 20 tables + settings) - Web UI, REST API, OCC CLI commands, scheduled background job - Self-update from Gitea releases with Ed25519 signature verification - Configurable column visibility on all data tables (persisted via localStorage) Fixes: - NC admin group fallback for PermissionService (IGroupManager) - Bundle import inline error correction (editable error rows) New files: BackupService, BackupSettingsService, BackupController, BackupJob, SelfUpdateService, 4 OCC commands, ColumnPicker component, Backup.vue, Ed25519 signing scripts, signature verification tests (18 tests) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
583 lines
20 KiB
PHP
583 lines
20 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace OCA\Mitgliederverwaltung\Service;
|
|
|
|
use Psr\Log\LoggerInterface;
|
|
|
|
/**
|
|
* Service for importing ZIP bundles containing multiple entity CSVs.
|
|
*
|
|
* Handles:
|
|
* - ZIP extraction and entity type detection by filename or header analysis
|
|
* - Topological sort of entity dependencies for correct import order
|
|
* - ID remapping across entities (old ID -> new ID)
|
|
* - Per-entity import via EntityImportService
|
|
*
|
|
* Part of Issue #151.
|
|
*/
|
|
class BundleImportService {
|
|
|
|
private EntityImportService $entityImportService;
|
|
private EntityExportService $exportService;
|
|
private LoggerInterface $logger;
|
|
|
|
/**
|
|
* Entity dependency graph for topological sort.
|
|
* Key = entity type, Value = array of entity types it depends on.
|
|
*/
|
|
private const DEPENDENCY_GRAPH = [
|
|
'stufen' => [],
|
|
'familien' => [],
|
|
'mitglieder' => ['stufen', 'familien'],
|
|
'adressen' => ['mitglieder'],
|
|
'telefonnummern' => ['mitglieder'],
|
|
'emails' => ['mitglieder'],
|
|
'stufenverlauf' => ['mitglieder', 'stufen'],
|
|
'beitragsregeln' => [],
|
|
'beitraege' => ['mitglieder'],
|
|
'lager' => [],
|
|
'lagerteilnehmer' => ['lager', 'mitglieder'],
|
|
'verletzungen' => ['mitglieder', 'lager'],
|
|
'gespeicherte_abfragen' => [],
|
|
];
|
|
|
|
/**
|
|
* Filename -> entity type mapping (from EntityExportService::ENTITY_TYPES).
|
|
*/
|
|
private array $filenameMap = [];
|
|
|
|
public function __construct(
|
|
EntityImportService $entityImportService,
|
|
EntityExportService $exportService,
|
|
LoggerInterface $logger
|
|
) {
|
|
$this->entityImportService = $entityImportService;
|
|
$this->exportService = $exportService;
|
|
$this->logger = $logger;
|
|
|
|
// Build filename lookup
|
|
foreach (EntityExportService::ENTITY_TYPES as $type => $meta) {
|
|
$this->filenameMap[strtolower($meta['filename'])] = $type;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Analyze a ZIP file: extract CSVs, detect entity types, determine import order.
|
|
*
|
|
* @param string $zipContent Raw ZIP binary content
|
|
* @return array{entities: array[], importOrder: string[], unknown: string[]}
|
|
*/
|
|
public function analyzeBundle(string $zipContent): array {
|
|
$csvFiles = $this->extractCsvFiles($zipContent);
|
|
|
|
$entities = [];
|
|
$unknown = [];
|
|
|
|
foreach ($csvFiles as $filename => $content) {
|
|
$type = $this->detectTypeByFilename($filename);
|
|
|
|
if ($type === null) {
|
|
// Fall back to header-based detection
|
|
try {
|
|
$parsed = $this->entityImportService->parseFile($content, ';', 'UTF-8');
|
|
$detection = $this->entityImportService->detectEntityType($parsed['columns']);
|
|
$type = $detection['type'];
|
|
} catch (\Exception $e) {
|
|
// Cannot parse this file
|
|
$type = null;
|
|
}
|
|
}
|
|
|
|
if ($type !== null) {
|
|
// Count rows
|
|
try {
|
|
$parsed = $this->entityImportService->parseFile($content, ';', 'UTF-8');
|
|
$rowCount = $parsed['totalRows'];
|
|
} catch (\Exception $e) {
|
|
$rowCount = 0;
|
|
}
|
|
|
|
$entities[] = [
|
|
'filename' => $filename,
|
|
'type' => $type,
|
|
'label' => EntityExportService::ENTITY_TYPES[$type]['label'] ?? $type,
|
|
'rowCount' => $rowCount,
|
|
];
|
|
} else {
|
|
$unknown[] = $filename;
|
|
}
|
|
}
|
|
|
|
// Determine import order based on dependency graph
|
|
$detectedTypes = array_map(fn($e) => $e['type'], $entities);
|
|
$importOrder = $this->topologicalSort($detectedTypes);
|
|
|
|
// Sort entities array by import order
|
|
$orderIndex = array_flip($importOrder);
|
|
usort($entities, function ($a, $b) use ($orderIndex) {
|
|
$ia = $orderIndex[$a['type']] ?? PHP_INT_MAX;
|
|
$ib = $orderIndex[$b['type']] ?? PHP_INT_MAX;
|
|
return $ia <=> $ib;
|
|
});
|
|
|
|
return [
|
|
'entities' => $entities,
|
|
'importOrder' => $importOrder,
|
|
'unknown' => $unknown,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Execute bundle import: import all entities in dependency order with ID remapping.
|
|
*
|
|
* @param string $zipContent Raw ZIP binary content
|
|
* @param array $entityOverrides Optional type overrides: {filename: type}
|
|
* @return array{results: array[], idMap: array}
|
|
*/
|
|
public function executeBundle(string $zipContent, array $entityOverrides = [], array $corrections = []): array {
|
|
$csvFiles = $this->extractCsvFiles($zipContent);
|
|
$analysis = $this->analyzeBundle($zipContent);
|
|
|
|
// Apply overrides
|
|
foreach ($analysis['entities'] as &$entity) {
|
|
if (isset($entityOverrides[$entity['filename']])) {
|
|
$entity['type'] = $entityOverrides[$entity['filename']];
|
|
}
|
|
}
|
|
|
|
// ID remapping table: type => {oldId => newId}
|
|
$idMap = [];
|
|
$results = [];
|
|
|
|
foreach ($analysis['importOrder'] as $type) {
|
|
// Find the entity entry for this type
|
|
$entityEntry = null;
|
|
foreach ($analysis['entities'] as $e) {
|
|
if ($e['type'] === $type) {
|
|
$entityEntry = $e;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if ($entityEntry === null) {
|
|
continue; // This type is not in the ZIP
|
|
}
|
|
|
|
$filename = $entityEntry['filename'];
|
|
$content = $csvFiles[$filename] ?? null;
|
|
if ($content === null) {
|
|
$results[] = [
|
|
'type' => $type,
|
|
'label' => $entityEntry['label'],
|
|
'status' => 'error',
|
|
'error' => 'CSV-Datei nicht gefunden',
|
|
'created' => 0,
|
|
'skipped' => 0,
|
|
'errors' => 1,
|
|
];
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
// Apply ID remapping to the CSV content before import
|
|
$remappedContent = $this->applyIdRemapping($type, $content, $idMap);
|
|
|
|
// Parse, auto-map, and execute
|
|
$parsed = $this->entityImportService->parseFile($remappedContent, ';', 'UTF-8');
|
|
$mapping = $this->entityImportService->autoMapHeaders($type, $parsed['columns']);
|
|
|
|
$entityCorrections = $corrections[$type] ?? [];
|
|
|
|
$result = $this->entityImportService->execute(
|
|
$type,
|
|
$remappedContent,
|
|
$mapping,
|
|
';',
|
|
'UTF-8',
|
|
false,
|
|
$entityCorrections
|
|
);
|
|
|
|
// Collect ID mappings from created entities
|
|
$this->collectIdMappings($type, $content, $result, $idMap);
|
|
|
|
$results[] = [
|
|
'type' => $type,
|
|
'label' => $entityEntry['label'],
|
|
'status' => 'success',
|
|
'created' => $result['created'],
|
|
'skipped' => $result['skipped'],
|
|
'errors' => $result['errors'],
|
|
'details' => $result['details'],
|
|
];
|
|
} catch (\Exception $e) {
|
|
$this->logger->error('Bundle import failed for entity type', [
|
|
'type' => $type,
|
|
'exception' => $e,
|
|
'app' => 'mitgliederverwaltung',
|
|
]);
|
|
|
|
$results[] = [
|
|
'type' => $type,
|
|
'label' => $entityEntry['label'],
|
|
'status' => 'error',
|
|
'error' => $e->getMessage(),
|
|
'created' => 0,
|
|
'skipped' => 0,
|
|
'errors' => 1,
|
|
];
|
|
}
|
|
}
|
|
|
|
return [
|
|
'results' => $results,
|
|
'idMap' => $idMap,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Preview bundle import (dry-run for all entities).
|
|
*
|
|
* @param string $zipContent Raw ZIP binary content
|
|
* @param array $corrections Per-entity corrections: { entityType: { rowNum: { fieldKey: value } } }
|
|
* @return array{results: array[]}
|
|
*/
|
|
public function previewBundle(string $zipContent, array $corrections = []): array {
|
|
$csvFiles = $this->extractCsvFiles($zipContent);
|
|
$analysis = $this->analyzeBundle($zipContent);
|
|
$results = [];
|
|
|
|
foreach ($analysis['importOrder'] as $type) {
|
|
$entityEntry = null;
|
|
foreach ($analysis['entities'] as $e) {
|
|
if ($e['type'] === $type) {
|
|
$entityEntry = $e;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if ($entityEntry === null) continue;
|
|
|
|
$filename = $entityEntry['filename'];
|
|
$content = $csvFiles[$filename] ?? null;
|
|
if ($content === null) continue;
|
|
|
|
try {
|
|
$parsed = $this->entityImportService->parseFile($content, ';', 'UTF-8');
|
|
$mapping = $this->entityImportService->autoMapHeaders($type, $parsed['columns']);
|
|
|
|
$entityCorrections = $corrections[$type] ?? [];
|
|
|
|
$preview = $this->entityImportService->preview(
|
|
$type,
|
|
$content,
|
|
$mapping,
|
|
';',
|
|
'UTF-8',
|
|
false,
|
|
$entityCorrections
|
|
);
|
|
|
|
// Include target fields so the frontend can render editable inputs
|
|
$schema = $this->entityImportService->getImportSchema($type);
|
|
|
|
$results[] = [
|
|
'type' => $type,
|
|
'label' => $entityEntry['label'],
|
|
'summary' => $preview['summary'],
|
|
'fkWarnings' => $preview['fkWarnings'],
|
|
'errorCount' => count($preview['errors']),
|
|
'errors' => array_slice($preview['errors'], 0, 10),
|
|
'targetFields' => $schema['targetFields'],
|
|
];
|
|
} catch (\Exception $e) {
|
|
$results[] = [
|
|
'type' => $type,
|
|
'label' => $entityEntry['label'],
|
|
'summary' => ['total' => 0, 'toCreate' => 0, 'duplicates' => 0, 'errors' => 1],
|
|
'fkWarnings' => [],
|
|
'errorCount' => 1,
|
|
'errors' => [['_rowIndex' => 0, '_errors' => [$e->getMessage()]]],
|
|
'targetFields' => [],
|
|
];
|
|
}
|
|
}
|
|
|
|
return ['results' => $results];
|
|
}
|
|
|
|
// ── ZIP extraction ─────────────────────────────────────────────
|
|
|
|
/**
|
|
* Extract all CSV files from a ZIP archive.
|
|
*
|
|
* @param string $zipContent Raw ZIP binary
|
|
* @return array<string, string> Filename => CSV content
|
|
*/
|
|
private function extractCsvFiles(string $zipContent): array {
|
|
if (!class_exists(\ZipArchive::class)) {
|
|
throw new \RuntimeException('ZipArchive ist nicht verfuegbar.');
|
|
}
|
|
|
|
$tmpFile = tempnam(sys_get_temp_dir(), 'mv_import_');
|
|
if ($tmpFile === false) {
|
|
throw new \RuntimeException('Temporaere Datei konnte nicht erstellt werden.');
|
|
}
|
|
|
|
// Restrict permissions to owner-only before writing content
|
|
chmod($tmpFile, 0600);
|
|
|
|
try {
|
|
file_put_contents($tmpFile, $zipContent);
|
|
|
|
$zip = new \ZipArchive();
|
|
$result = $zip->open($tmpFile);
|
|
if ($result !== true) {
|
|
throw new ValidationException('ZIP-Datei konnte nicht geoeffnet werden. Fehlercode: ' . $result);
|
|
}
|
|
|
|
$csvFiles = [];
|
|
for ($i = 0; $i < $zip->numFiles; $i++) {
|
|
$name = $zip->getNameIndex($i);
|
|
if ($name === false) {
|
|
continue;
|
|
}
|
|
|
|
// Reject path traversal: entries containing '..' or absolute paths
|
|
if (str_contains($name, '..') || str_starts_with($name, '/') || str_starts_with($name, '\\')) {
|
|
$this->logger->warning('Skipping ZIP entry with suspicious path', [
|
|
'entry' => $name,
|
|
'app' => 'mitgliederverwaltung',
|
|
]);
|
|
continue;
|
|
}
|
|
|
|
$baseName = basename($name);
|
|
|
|
// Only include CSV files (skip directories, hidden files)
|
|
if (
|
|
!str_starts_with($baseName, '.') &&
|
|
!str_starts_with($baseName, '__') &&
|
|
(str_ends_with(strtolower($baseName), '.csv') || str_ends_with(strtolower($baseName), '.txt'))
|
|
) {
|
|
$content = $zip->getFromIndex($i);
|
|
if ($content !== false) {
|
|
$csvFiles[$baseName] = $content;
|
|
}
|
|
}
|
|
}
|
|
|
|
$zip->close();
|
|
|
|
if (empty($csvFiles)) {
|
|
throw new ValidationException('ZIP-Archiv enthaelt keine CSV-Dateien.');
|
|
}
|
|
|
|
return $csvFiles;
|
|
} finally {
|
|
if (file_exists($tmpFile)) {
|
|
unlink($tmpFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Entity type detection ──────────────────────────────────────
|
|
|
|
/**
|
|
* Detect entity type by filename matching against known export filenames.
|
|
*/
|
|
private function detectTypeByFilename(string $filename): ?string {
|
|
return $this->filenameMap[strtolower($filename)] ?? null;
|
|
}
|
|
|
|
// ── Topological sort ───────────────────────────────────────────
|
|
|
|
/**
|
|
* Topological sort of entity types based on dependency graph.
|
|
* Only includes types present in the $types array.
|
|
*
|
|
* @param string[] $types Entity types to sort
|
|
* @return string[] Sorted import order
|
|
*/
|
|
private function topologicalSort(array $types): array {
|
|
$typeSet = array_flip($types);
|
|
$sorted = [];
|
|
$visited = [];
|
|
|
|
foreach ($types as $type) {
|
|
$this->visit($type, $typeSet, $visited, $sorted);
|
|
}
|
|
|
|
return $sorted;
|
|
}
|
|
|
|
private function visit(string $type, array $typeSet, array &$visited, array &$sorted): void {
|
|
if (isset($visited[$type])) {
|
|
return;
|
|
}
|
|
$visited[$type] = true;
|
|
|
|
$deps = self::DEPENDENCY_GRAPH[$type] ?? [];
|
|
foreach ($deps as $dep) {
|
|
if (isset($typeSet[$dep])) {
|
|
$this->visit($dep, $typeSet, $visited, $sorted);
|
|
}
|
|
}
|
|
|
|
$sorted[] = $type;
|
|
}
|
|
|
|
// ── ID remapping ───────────────────────────────────────────────
|
|
|
|
/**
|
|
* Apply ID remapping to CSV content based on previously imported entities.
|
|
*
|
|
* Replaces old IDs in FK columns with new IDs from the idMap.
|
|
*/
|
|
private function applyIdRemapping(string $type, string $content, array $idMap): string {
|
|
// Determine which columns need remapping for this type
|
|
$fkColumns = $this->getFkColumnsForRemapping($type);
|
|
if (empty($fkColumns)) {
|
|
return $content;
|
|
}
|
|
|
|
// Parse the CSV
|
|
$lines = explode("\n", str_replace("\r\n", "\n", $content));
|
|
if (count($lines) < 2) {
|
|
return $content;
|
|
}
|
|
|
|
// Check for BOM
|
|
$bom = '';
|
|
if (substr($lines[0], 0, 3) === "\xEF\xBB\xBF") {
|
|
$bom = "\xEF\xBB\xBF";
|
|
$lines[0] = substr($lines[0], 3);
|
|
}
|
|
|
|
// Parse header to find column indices
|
|
$headers = str_getcsv($lines[0], ';');
|
|
$headers = array_map('strtolower', array_map('trim', $headers));
|
|
|
|
$remapIndices = [];
|
|
foreach ($fkColumns as $colInfo) {
|
|
$headerIdx = array_search(strtolower($colInfo['header']), $headers);
|
|
if ($headerIdx !== false) {
|
|
$remapIndices[$headerIdx] = $colInfo['source_type'];
|
|
}
|
|
}
|
|
|
|
if (empty($remapIndices)) {
|
|
return $content;
|
|
}
|
|
|
|
// Rebuild the CSV with remapped IDs
|
|
$output = $bom . $lines[0] . "\r\n";
|
|
for ($i = 1; $i < count($lines); $i++) {
|
|
$line = trim($lines[$i]);
|
|
if ($line === '') continue;
|
|
|
|
$row = str_getcsv($line, ';');
|
|
foreach ($remapIndices as $colIdx => $sourceType) {
|
|
if (isset($row[$colIdx]) && $row[$colIdx] !== '' && isset($idMap[$sourceType])) {
|
|
$oldId = $row[$colIdx];
|
|
if (isset($idMap[$sourceType][$oldId])) {
|
|
$row[$colIdx] = (string)$idMap[$sourceType][$oldId];
|
|
}
|
|
}
|
|
}
|
|
|
|
// Re-encode as CSV line
|
|
$escaped = array_map(function ($field) {
|
|
if (str_contains($field, ';') || str_contains($field, '"') || str_contains($field, "\n")) {
|
|
return '"' . str_replace('"', '""', $field) . '"';
|
|
}
|
|
return $field;
|
|
}, $row);
|
|
$output .= implode(';', $escaped) . "\r\n";
|
|
}
|
|
|
|
return $output;
|
|
}
|
|
|
|
/**
|
|
* Get FK columns that need ID remapping for a given entity type.
|
|
*
|
|
* @return array[] Each: {header, source_type}
|
|
*/
|
|
private function getFkColumnsForRemapping(string $type): array {
|
|
return match ($type) {
|
|
'mitglieder' => [
|
|
['header' => 'stufe-id', 'source_type' => 'stufen'],
|
|
['header' => 'familien-id', 'source_type' => 'familien'],
|
|
],
|
|
'adressen', 'telefonnummern', 'emails' => [
|
|
['header' => 'mitglied-id', 'source_type' => 'mitglieder'],
|
|
],
|
|
'stufenverlauf' => [
|
|
['header' => 'mitglied-id', 'source_type' => 'mitglieder'],
|
|
['header' => 'stufe-id', 'source_type' => 'stufen'],
|
|
],
|
|
'beitraege' => [
|
|
['header' => 'mitglied-id', 'source_type' => 'mitglieder'],
|
|
],
|
|
'lagerteilnehmer' => [
|
|
['header' => 'lager-id', 'source_type' => 'lager'],
|
|
['header' => 'mitglied-id', 'source_type' => 'mitglieder'],
|
|
],
|
|
'verletzungen' => [
|
|
['header' => 'mitglied-id', 'source_type' => 'mitglieder'],
|
|
['header' => 'lager-id', 'source_type' => 'lager'],
|
|
],
|
|
default => [],
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Collect ID mappings from import results.
|
|
* Maps old CSV IDs to new database IDs.
|
|
*/
|
|
private function collectIdMappings(string $type, string $originalContent, array $result, array &$idMap): void {
|
|
if (!isset($idMap[$type])) {
|
|
$idMap[$type] = [];
|
|
}
|
|
|
|
// Parse original content to get old IDs
|
|
$lines = explode("\n", str_replace("\r\n", "\n", $originalContent));
|
|
if (count($lines) < 2) return;
|
|
|
|
// Remove BOM
|
|
if (substr($lines[0], 0, 3) === "\xEF\xBB\xBF") {
|
|
$lines[0] = substr($lines[0], 3);
|
|
}
|
|
|
|
$headers = str_getcsv($lines[0], ';');
|
|
$headers = array_map('strtolower', array_map('trim', $headers));
|
|
$idColIdx = array_search('id', $headers);
|
|
if ($idColIdx === false) return;
|
|
|
|
// Build row-to-oldId mapping
|
|
$oldIds = [];
|
|
for ($i = 1; $i < count($lines); $i++) {
|
|
$line = trim($lines[$i]);
|
|
if ($line === '') continue;
|
|
$row = str_getcsv($line, ';');
|
|
if (isset($row[$idColIdx]) && $row[$idColIdx] !== '') {
|
|
$oldIds[$i - 1] = $row[$idColIdx]; // 0-indexed row
|
|
}
|
|
}
|
|
|
|
// Match with created results
|
|
$createdIdx = 0;
|
|
foreach ($result['details'] as $detail) {
|
|
if ($detail['status'] === 'created' && isset($detail['id'])) {
|
|
$rowNum = $detail['row'] - 2; // Convert to 0-indexed
|
|
if (isset($oldIds[$rowNum])) {
|
|
$idMap[$type][$oldIds[$rowNum]] = $detail['id'];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|