From ce485c9b013feef59984f2c99119150e31936363 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 2 Jun 2026 00:23:50 +0700 Subject: [PATCH 1/2] feat(import): let Import choose source format (SQL or JSON) --- CHANGELOG.md | 2 +- .../Plugins/PluginManager+Registration.swift | 25 +++++++++++++ .../Core/Services/Export/ImportRouting.swift | 30 +++++++++++++++ .../MainWindowToolbar+Actions.swift | 6 ++- .../MainWindowToolbar+Buttons.swift | 31 +++++++++++----- .../MainWindowToolbar+Items.swift | 36 ++++++++++++++---- TablePro/TableProApp.swift | 13 +++---- TablePro/Views/Import/ImportDialog.swift | 7 ++++ TablePro/Views/Import/ImportMenuItems.swift | 32 ++++++++++++++++ TablePro/Views/Import/JSONImportSheet.swift | 26 +++++++------ ...ainContentCoordinator+SidebarActions.swift | 26 ++++++------- .../Main/MainContentCommandActions.swift | 8 +++- .../Views/Main/MainContentCoordinator.swift | 8 ++-- TablePro/Views/Main/MainContentView.swift | 14 ++++--- .../Views/Sidebar/SidebarContextMenu.swift | 10 +++-- .../Services/Export/ImportRoutingTests.swift | 37 +++++++++++++++++++ .../Main/CoordinatorSidebarActionsTests.swift | 6 +-- docs/features/import-export.mdx | 4 +- 18 files changed, 248 insertions(+), 73 deletions(-) create mode 100644 TablePro/Core/Services/Export/ImportRouting.swift create mode 100644 TablePro/Views/Import/ImportMenuItems.swift create mode 100644 TableProTests/Core/Services/Export/ImportRoutingTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f460ac9..74d41b412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Import a JSON file into a table. A dedicated sheet accepts an array of objects, newline-delimited JSON, or TablePro's own JSON export, and lets you map each field to a column in an existing table or in a new table with inferred, editable columns. +- Import a JSON file into a table. The Import menu now lets you pick the source, SQL or JSON, instead of going straight to SQL. The JSON flow accepts an array of objects, newline-delimited JSON, or TablePro's own JSON export, and lets you map each field to a column in an existing table or in a new table with inferred, editable columns. - The window title bar shows the open table's name, with its database and schema below, so you can tell which table you're viewing without checking the sidebar. (#1475) - iOS: open DuckDB database files and in-memory DuckDB databases, matching the Mac app. (#1526) - Save the current query as a favorite from a star button in the SQL editor toolbar. diff --git a/TablePro/Core/Plugins/PluginManager+Registration.swift b/TablePro/Core/Plugins/PluginManager+Registration.swift index 19ceb3dc2..6dd4c1039 100644 --- a/TablePro/Core/Plugins/PluginManager+Registration.swift +++ b/TablePro/Core/Plugins/PluginManager+Registration.swift @@ -262,6 +262,31 @@ extension PluginManager { return Array(importPlugins.values) } + func importPlugins(for databaseType: DatabaseType) -> [any ImportFormatPlugin] { + guard supportsImport(for: databaseType) else { return [] } + let typeId = databaseType.rawValue + return allImportPlugins() + .filter { plugin in + let supported = type(of: plugin).supportedDatabaseTypeIds + let excluded = type(of: plugin).excludedDatabaseTypeIds + if !supported.isEmpty && !supported.contains(typeId) { return false } + if excluded.contains(typeId) { return false } + return true + } + .sorted { lhs, rhs in + let lhsRowBased = type(of: lhs).requiresTargetTable + let rhsRowBased = type(of: rhs).requiresTargetTable + if lhsRowBased != rhsRowBased { return !lhsRowBased } + return type(of: lhs).formatDisplayName < type(of: rhs).formatDisplayName + } + } + + func importFormatOptions(for databaseType: DatabaseType) -> [ImportFormatOption] { + importPlugins(for: databaseType).map { + ImportFormatOption(id: type(of: $0).formatId, name: type(of: $0).formatDisplayName) + } + } + /// Returns a temporary plugin driver for query building (buildBrowseQuery), or nil /// if the plugin doesn't implement custom query building (NoSQL hooks). func queryBuildingDriver(for databaseType: DatabaseType) -> (any PluginDatabaseDriver)? { diff --git a/TablePro/Core/Services/Export/ImportRouting.swift b/TablePro/Core/Services/Export/ImportRouting.swift new file mode 100644 index 000000000..2fbd22bfa --- /dev/null +++ b/TablePro/Core/Services/Export/ImportRouting.swift @@ -0,0 +1,30 @@ +// +// ImportRouting.swift +// TablePro +// + +import Foundation + +struct ImportFormatOption: Identifiable, Equatable { + let id: String + let name: String + + var submenuLabel: String { + String(format: String(localized: "From %@\u{2026}"), name) + } + + var standaloneLabel: String { + String(format: String(localized: "Import %@\u{2026}"), name) + } +} + +enum ImportSheetRoute: Equatable { + case statement(formatId: String) + case rowMapping(formatId: String) +} + +enum ImportRouting { + static func route(formatId: String, requiresTargetTable: Bool) -> ImportSheetRoute { + requiresTargetTable ? .rowMapping(formatId: formatId) : .statement(formatId: formatId) + } +} diff --git a/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Actions.swift b/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Actions.swift index 34d6da6be..ac1b71e18 100644 --- a/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Actions.swift +++ b/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Actions.swift @@ -51,7 +51,9 @@ extension MainWindowToolbar { coordinator?.commandActions?.exportTables() } - @objc func performImport(_ sender: Any?) { - coordinator?.commandActions?.importTables() + @objc func performImportFormat(_ sender: Any?) { + guard let menuItem = sender as? NSMenuItem, + let formatId = menuItem.representedObject as? String else { return } + coordinator?.commandActions?.importTables(formatId: formatId) } } diff --git a/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Buttons.swift b/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Buttons.swift index 3f17ccaf6..1456e247c 100644 --- a/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Buttons.swift +++ b/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Buttons.swift @@ -199,16 +199,29 @@ struct ImportToolbarButton: View { var body: some View { let state = coordinator.toolbarState if PluginManager.shared.supportsImport(for: state.databaseType) { - Button { - coordinator.commandActions?.importTables() - } label: { - Label("Import", systemImage: "square.and.arrow.down") + let formats = PluginManager.shared.importFormatOptions(for: state.databaseType) + let isDisabled = state.connectionState != .connected || state.safeModeLevel.blocksAllWrites + if formats.count <= 1 { + Button { + coordinator.commandActions?.importTables(formatId: formats.first?.id ?? "") + } label: { + Label("Import", systemImage: "square.and.arrow.down") + } + .help(String(localized: "Import Data (⌘⇧I)")) + .disabled(isDisabled || formats.isEmpty) + } else { + Menu { + ForEach(formats) { format in + Button(format.submenuLabel) { + coordinator.commandActions?.importTables(formatId: format.id) + } + } + } label: { + Label("Import", systemImage: "square.and.arrow.down") + } + .help(String(localized: "Import Data (⌘⇧I)")) + .disabled(isDisabled) } - .help(String(localized: "Import Data (⌘⇧I)")) - .disabled( - state.connectionState != .connected - || state.safeModeLevel.blocksAllWrites - ) } } } diff --git a/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Items.swift b/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Items.swift index b37458f5a..0bdbaee0d 100644 --- a/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Items.swift +++ b/TablePro/Core/Services/Infrastructure/MainWindowToolbar+Items.swift @@ -65,14 +65,34 @@ extension MainWindowToolbar { } func subitemImport() -> NSToolbarItem { - menuOnlyItem( - id: Self.importTables, - label: String(localized: "Import"), - symbol: "square.and.arrow.down", - action: #selector(performImport(_:)), - keyEquivalent: "i", - modifiers: [.command, .shift] - ) + let label = String(localized: "Import") + let item = NSToolbarItem(itemIdentifier: Self.importTables) + item.label = label + item.paletteLabel = label + item.image = NSImage(systemSymbolName: "square.and.arrow.down", accessibilityDescription: label) + + let menuItem = NSMenuItem(title: label, action: nil, keyEquivalent: "") + menuItem.image = item.image + menuItem.submenu = buildImportSubmenu() + item.menuFormRepresentation = menuItem + + return item + } + + private func buildImportSubmenu() -> NSMenu { + let menu = NSMenu() + guard let databaseType = coordinator?.connection.type else { return menu } + for format in PluginManager.shared.importFormatOptions(for: databaseType) { + let menuItem = NSMenuItem( + title: format.submenuLabel, + action: #selector(performImportFormat(_:)), + keyEquivalent: "" + ) + menuItem.target = self + menuItem.representedObject = format.id + menu.addItem(menuItem) + } + return menu } // MARK: - Helpers diff --git a/TablePro/TableProApp.swift b/TablePro/TableProApp.swift index e81dd41af..a6444ddf9 100644 --- a/TablePro/TableProApp.swift +++ b/TablePro/TableProApp.swift @@ -339,14 +339,11 @@ struct AppMenuCommands: Commands { } .disabled(!(actions?.isConnected ?? false)) - Button("Import...") { - actions?.importTables() - } - .optionalKeyboardShortcut(shortcut(for: .importData)) - .disabled( - !(actions?.isConnected ?? false) - || actions?.isReadOnly ?? false - || !(actions.map { PluginManager.shared.supportsImport(for: $0.currentDatabaseType) } ?? true) + ImportMenuItems( + formats: actions?.availableImportFormats ?? [], + isDisabled: !(actions?.isConnected ?? false) || (actions?.isReadOnly ?? false), + shortcut: shortcut(for: .importData), + action: { formatId in actions?.importTables(formatId: formatId) } ) Button(String(localized: "Backup Dump\u{2026}")) { diff --git a/TablePro/Views/Import/ImportDialog.swift b/TablePro/Views/Import/ImportDialog.swift index 72d6aaeab..fea4fcb77 100644 --- a/TablePro/Views/Import/ImportDialog.swift +++ b/TablePro/Views/Import/ImportDialog.swift @@ -18,6 +18,13 @@ struct ImportDialog: View { let connection: DatabaseConnection let initialFileURL: URL? + init(isPresented: Binding, connection: DatabaseConnection, initialFileURL: URL?, initialFormatId: String) { + self._isPresented = isPresented + self.connection = connection + self.initialFileURL = initialFileURL + self._selectedFormatId = State(initialValue: initialFormatId) + } + // MARK: - State @State private var fileURL: URL? diff --git a/TablePro/Views/Import/ImportMenuItems.swift b/TablePro/Views/Import/ImportMenuItems.swift new file mode 100644 index 000000000..d4cac953c --- /dev/null +++ b/TablePro/Views/Import/ImportMenuItems.swift @@ -0,0 +1,32 @@ +// +// ImportMenuItems.swift +// TablePro +// + +import SwiftUI + +struct ImportMenuItems: View { + let formats: [ImportFormatOption] + let isDisabled: Bool + let shortcut: KeyboardShortcut? + let action: (String) -> Void + + var body: some View { + if formats.isEmpty { + Button("Import\u{2026}") {} + .disabled(true) + } else if formats.count == 1, let only = formats.first { + Button(only.standaloneLabel) { action(only.id) } + .optionalKeyboardShortcut(shortcut) + .disabled(isDisabled) + } else { + Menu("Import") { + ForEach(formats) { format in + Button(format.submenuLabel) { action(format.id) } + .optionalKeyboardShortcut(format.id == formats.first?.id ? shortcut : nil) + } + } + .disabled(isDisabled) + } + } +} diff --git a/TablePro/Views/Import/JSONImportSheet.swift b/TablePro/Views/Import/JSONImportSheet.swift index d52879488..407c10f56 100644 --- a/TablePro/Views/Import/JSONImportSheet.swift +++ b/TablePro/Views/Import/JSONImportSheet.swift @@ -18,6 +18,7 @@ struct JSONImportSheet: View { @Binding var isPresented: Bool let connection: DatabaseConnection let fileURL: URL + let formatId: String private enum Destination: Hashable { case existingTable @@ -295,12 +296,22 @@ struct JSONImportSheet: View { .textFieldStyle(.roundedBorder) .frame(width: 150) .disabled(!row.include) - Picker("", selection: columnBinding(row).type) { + Menu { ForEach(typeOptions(including: row.type), id: \.self) { type in - Text(type).tag(type) + Button { + columnBinding(row).type.wrappedValue = type + } label: { + if type.caseInsensitiveCompare(row.type) == .orderedSame { + Label(type, systemImage: "checkmark") + } else { + Text(type) + } + } } + } label: { + Text(row.type) + .frame(maxWidth: .infinity, alignment: .leading) } - .labelsHidden() .frame(width: 150) .disabled(!row.include) Toggle("", isOn: columnBinding(row).isPrimaryKey).labelsHidden().disabled(!row.include) @@ -387,14 +398,7 @@ struct JSONImportSheet: View { // MARK: - Plugin private var currentPlugin: (any ImportFormatPlugin)? { - let ext = fileURL.pathExtension.lowercased() - return PluginManager.shared.allImportPlugins().first { - type(of: $0).requiresTargetTable && type(of: $0).acceptedFileExtensions.contains(ext) - } - } - - private var formatId: String { - currentPlugin.map { type(of: $0).formatId } ?? "json" + PluginManager.shared.importPlugin(forFormat: formatId) } private var canImport: Bool { diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift index 0ee7a0292..21519c3f9 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift @@ -137,43 +137,43 @@ extension MainContentCoordinator { activeSheet = .exportQueryResults } - func openImportDialog() { + func openImportDialog(formatId: String) { guard !safeModeLevel.blocksAllWrites else { return } guard PluginManager.shared.supportsImport(for: connection.type) else { AlertHelper.showErrorSheet( title: String(localized: "Import Not Supported"), - message: String(format: String(localized: "SQL import is not supported for %@ connections."), connection.type.rawValue), + message: String(format: String(localized: "Import is not supported for %@ connections."), connection.type.rawValue), window: nil ) return } + guard let plugin = PluginManager.shared.importPlugin(forFormat: formatId) else { return } + let pluginType = type(of: plugin) + let panel = NSOpenPanel() var contentTypes: [UTType] = [] - for plugin in PluginManager.shared.allImportPlugins() { - for ext in type(of: plugin).acceptedFileExtensions { - if let utType = UTType(filenameExtension: ext) { - contentTypes.append(utType) - } + for ext in pluginType.acceptedFileExtensions { + if let utType = UTType(filenameExtension: ext) { + contentTypes.append(utType) } } - if let gzType = UTType(filenameExtension: "gz") { + if !pluginType.requiresTargetTable, let gzType = UTType(filenameExtension: "gz") { contentTypes.append(gzType) } if !contentTypes.isEmpty { panel.allowedContentTypes = contentTypes } panel.allowsMultipleSelection = false - panel.message = "Select SQL file to import" + panel.message = String(format: String(localized: "Select %@ file to import"), pluginType.formatDisplayName) guard let window = contentWindow else { return } panel.beginSheetModal(for: window) { [weak self] response in guard response == .OK, let url = panel.url else { return } self?.importFileURL = url - let ext = url.pathExtension.lowercased() - let isRowBased = PluginManager.shared.allImportPlugins().contains { - type(of: $0).requiresTargetTable && type(of: $0).acceptedFileExtensions.contains(ext) + switch ImportRouting.route(formatId: formatId, requiresTargetTable: pluginType.requiresTargetTable) { + case .statement(let id): self?.activeSheet = .importDialog(formatId: id) + case .rowMapping(let id): self?.activeSheet = .rowImport(formatId: id) } - self?.activeSheet = isRowBased ? .jsonImport : .importDialog } } diff --git a/TablePro/Views/Main/MainContentCommandActions.swift b/TablePro/Views/Main/MainContentCommandActions.swift index b2008d97c..fef03b317 100644 --- a/TablePro/Views/Main/MainContentCommandActions.swift +++ b/TablePro/Views/Main/MainContentCommandActions.swift @@ -677,8 +677,12 @@ final class MainContentCommandActions { coordinator?.openExportQueryResultsDialog() } - func importTables() { - coordinator?.openImportDialog() + func importTables(formatId: String) { + coordinator?.openImportDialog(formatId: formatId) + } + + var availableImportFormats: [ImportFormatOption] { + PluginManager.shared.importFormatOptions(for: currentDatabaseType) } func backupDatabase() { diff --git a/TablePro/Views/Main/MainContentCoordinator.swift b/TablePro/Views/Main/MainContentCoordinator.swift index 1fbf23681..cabfda36d 100644 --- a/TablePro/Views/Main/MainContentCoordinator.swift +++ b/TablePro/Views/Main/MainContentCoordinator.swift @@ -45,8 +45,8 @@ enum ActiveSheet: Identifiable { case quickSwitcher case sqlPreview case exportDialog - case importDialog - case jsonImport + case importDialog(formatId: String) + case rowImport(formatId: String) case exportQueryResults case backupDatabase case restoreDatabase(fileURL: URL) @@ -58,8 +58,8 @@ enum ActiveSheet: Identifiable { case .quickSwitcher: "quickSwitcher" case .sqlPreview: "sqlPreview" case .exportDialog: "exportDialog" - case .importDialog: "importDialog" - case .jsonImport: "jsonImport" + case .importDialog(let formatId): "importDialog-\(formatId)" + case .rowImport(let formatId): "rowImport-\(formatId)" case .exportQueryResults: "exportQueryResults" case .backupDatabase: "backupDatabase" case .restoreDatabase(let fileURL): "restoreDatabase-\(fileURL.path)" diff --git a/TablePro/Views/Main/MainContentView.swift b/TablePro/Views/Main/MainContentView.swift index 5972adc80..00647bdff 100644 --- a/TablePro/Views/Main/MainContentView.swift +++ b/TablePro/Views/Main/MainContentView.swift @@ -216,7 +216,7 @@ struct MainContentView: View { ) } } - case .importDialog: + case .importDialog(let formatId): let importDismiss = Binding( get: { coordinator.activeSheet != nil }, set: { if !$0 { @@ -228,10 +228,11 @@ struct MainContentView: View { ImportDialog( isPresented: importDismiss, connection: connection, - initialFileURL: coordinator.importFileURL + initialFileURL: coordinator.importFileURL, + initialFormatId: formatId ) - case .jsonImport: - let jsonDismiss = Binding( + case .rowImport(let formatId): + let rowDismiss = Binding( get: { coordinator.activeSheet != nil }, set: { if !$0 { coordinator.activeSheet = nil @@ -241,9 +242,10 @@ struct MainContentView: View { ) if let url = coordinator.importFileURL { JSONImportSheet( - isPresented: jsonDismiss, + isPresented: rowDismiss, connection: connection, - fileURL: url + fileURL: url, + formatId: formatId ) } case .backupDatabase: diff --git a/TablePro/Views/Sidebar/SidebarContextMenu.swift b/TablePro/Views/Sidebar/SidebarContextMenu.swift index 997f4c619..af7248f72 100644 --- a/TablePro/Views/Sidebar/SidebarContextMenu.swift +++ b/TablePro/Views/Sidebar/SidebarContextMenu.swift @@ -138,10 +138,12 @@ struct SidebarContextMenu: View { for: coordinator?.connection.type ?? .mysql ) ) { - Button("Import...") { - perform { coordinator?.openImportDialog() } - } - .disabled(isReadOnly) + ImportMenuItems( + formats: PluginManager.shared.importFormatOptions(for: coordinator?.connection.type ?? .mysql), + isDisabled: isReadOnly, + shortcut: nil, + action: { formatId in perform { coordinator?.openImportDialog(formatId: formatId) } } + ) } let maintenanceOps = coordinator?.supportedMaintenanceOperations() ?? [] diff --git a/TableProTests/Core/Services/Export/ImportRoutingTests.swift b/TableProTests/Core/Services/Export/ImportRoutingTests.swift new file mode 100644 index 000000000..a72e358c9 --- /dev/null +++ b/TableProTests/Core/Services/Export/ImportRoutingTests.swift @@ -0,0 +1,37 @@ +// +// ImportRoutingTests.swift +// TableProTests +// + +@testable import TablePro +import Testing + +struct ImportRoutingTests { + @Test("Statement-based formats route to the SQL import dialog") + func statementFormatRoutesToImportDialog() { + #expect(ImportRouting.route(formatId: "sql", requiresTargetTable: false) == .statement(formatId: "sql")) + } + + @Test("Row-based formats route to the row-mapping sheet") + func rowFormatRoutesToRowMapping() { + #expect(ImportRouting.route(formatId: "json", requiresTargetTable: true) == .rowMapping(formatId: "json")) + } + + @Test("Routing carries the chosen format id through unchanged") + func routingPreservesFormatId() { + #expect(ImportRouting.route(formatId: "ndjson", requiresTargetTable: true) == .rowMapping(formatId: "ndjson")) + #expect(ImportRouting.route(formatId: "csv", requiresTargetTable: false) == .statement(formatId: "csv")) + } + + @Test("Submenu label reads From ") + func submenuLabelFormat() { + let option = ImportFormatOption(id: "sql", name: "SQL") + #expect(option.submenuLabel == "From SQL\u{2026}") + } + + @Test("Standalone label reads Import ") + func standaloneLabelFormat() { + let option = ImportFormatOption(id: "json", name: "JSON") + #expect(option.standaloneLabel == "Import JSON\u{2026}") + } +} diff --git a/TableProTests/Views/Main/CoordinatorSidebarActionsTests.swift b/TableProTests/Views/Main/CoordinatorSidebarActionsTests.swift index 3ac8a5981..9d19f5e25 100644 --- a/TableProTests/Views/Main/CoordinatorSidebarActionsTests.swift +++ b/TableProTests/Views/Main/CoordinatorSidebarActionsTests.swift @@ -71,7 +71,7 @@ struct CoordinatorSidebarActionsTests { let (coordinator, _) = makeCoordinator(safeModeLevel: .readOnly) defer { coordinator.teardown() } - coordinator.openImportDialog() + coordinator.openImportDialog(formatId: "sql") } @Test("openImportDialog with MongoDB returns early at type guard") @@ -82,7 +82,7 @@ struct CoordinatorSidebarActionsTests { // Hits the MongoDB/Redis guard; shows an alert as side effect // but should not crash. - coordinator.openImportDialog() + coordinator.openImportDialog(formatId: "sql") } @Test("openImportDialog with Redis returns early at type guard") @@ -91,7 +91,7 @@ struct CoordinatorSidebarActionsTests { let (coordinator, _) = makeCoordinator(type: .redis) defer { coordinator.teardown() } - coordinator.openImportDialog() + coordinator.openImportDialog(formatId: "sql") } // MARK: - openExportDialog diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index e10bc622e..0c7cf3076 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -208,7 +208,7 @@ Import `.sql` and `.sql.gz` files (statements execute directly against your data - Click **File** > **Import** (`Cmd+Shift+I`), or drag and drop a `.sql` / `.sql.gz` file onto the app. + Click **File** > **Import** > **From SQL** (`Cmd+Shift+I`), or drag and drop a `.sql` / `.sql.gz` file onto the app. Set encoding, transaction wrapping, and foreign key check options. @@ -259,7 +259,7 @@ In **Skip and Continue** mode, failed statements are collected (up to 1,000) wit ### Import JSON Data -Pick a `.json`, `.jsonl`, or `.ndjson` file in **File** > **Import** and TablePro opens a dedicated JSON import sheet. It accepts an array of objects `[{...}, {...}]`, newline-delimited JSON (one object per line, streamed for large files), and TablePro's own JSON export shape `{ "table": [ {...} ] }`, so an export round-trips back in. +Choose **File** > **Import** > **From JSON** and pick a `.json`, `.jsonl`, or `.ndjson` file, and TablePro opens a dedicated JSON import sheet. It accepts an array of objects `[{...}, {...}]`, newline-delimited JSON (one object per line, streamed for large files), and TablePro's own JSON export shape `{ "table": [ {...} ] }`, so an export round-trips back in. Choose a destination: From fbafddc3370f6e63b5366add7c0b2ed1f31b1956 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 2 Jun 2026 00:24:01 +0700 Subject: [PATCH 2/2] perf(plugin-json): batch row inserts and report the exact import total --- CHANGELOG.md | 2 +- .../PluginImportDataSink.swift | 6 ++ .../PluginImportProgress.swift | 12 ++++ .../JSONImportPlugin/JSONImportPlugin.swift | 59 ++++++++++++++++--- .../PluginImportDataSink.swift | 6 ++ .../PluginImportProgress.swift | 12 ++++ .../SQLStatementGenerator.swift | 30 ++++++++++ .../Core/Plugins/ImportDataSinkAdapter.swift | 49 +++++++++++++++ .../SQLStatementGeneratorImportTests.swift | 38 ++++++++++++ .../Export/PluginImportProgressTests.swift | 37 ++++++++++++ 10 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 TableProTests/Core/Services/Export/PluginImportProgressTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 74d41b412..ee8eef718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Import a JSON file into a table. The Import menu now lets you pick the source, SQL or JSON, instead of going straight to SQL. The JSON flow accepts an array of objects, newline-delimited JSON, or TablePro's own JSON export, and lets you map each field to a column in an existing table or in a new table with inferred, editable columns. +- Import a JSON file into a table. The Import menu now lets you pick the source, SQL or JSON, instead of going straight to SQL. The JSON flow accepts an array of objects, newline-delimited JSON, or TablePro's own JSON export, and lets you map each field to a column in an existing table or in a new table with inferred, editable columns. Rows insert in batched multi-row statements, so large imports over a remote connection finish in seconds instead of minutes. - The window title bar shows the open table's name, with its database and schema below, so you can tell which table you're viewing without checking the sidebar. (#1475) - iOS: open DuckDB database files and in-memory DuckDB databases, matching the Mac app. (#1526) - Save the current query as a favorite from a star button in the SQL editor toolbar. diff --git a/Packages/TableProCore/Sources/TableProPluginKit/PluginImportDataSink.swift b/Packages/TableProCore/Sources/TableProPluginKit/PluginImportDataSink.swift index c004b7315..ef983d39c 100644 --- a/Packages/TableProCore/Sources/TableProPluginKit/PluginImportDataSink.swift +++ b/Packages/TableProCore/Sources/TableProPluginKit/PluginImportDataSink.swift @@ -10,6 +10,7 @@ public protocol PluginImportDataSink: AnyObject, Sendable { var targetTable: String? { get } func execute(statement: String) async throws func insertRow(_ values: [String: PluginCellValue]) async throws + func insertRows(_ rows: [[String: PluginCellValue]]) async throws func deleteAllRowsFromTargetTable() async throws func beginTransaction() async throws func commitTransaction() async throws @@ -23,6 +24,11 @@ public extension PluginImportDataSink { func insertRow(_ values: [String: PluginCellValue]) async throws { throw PluginImportError.importFailed("Row-based import is not supported by this connection") } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { + for row in rows { + try await insertRow(row) + } + } func deleteAllRowsFromTargetTable() async throws { throw PluginImportError.importFailed("Clearing the target table is not supported by this connection") } diff --git a/Packages/TableProCore/Sources/TableProPluginKit/PluginImportProgress.swift b/Packages/TableProCore/Sources/TableProPluginKit/PluginImportProgress.swift index 043b87e4e..1693cef68 100644 --- a/Packages/TableProCore/Sources/TableProPluginKit/PluginImportProgress.swift +++ b/Packages/TableProCore/Sources/TableProPluginKit/PluginImportProgress.swift @@ -27,6 +27,18 @@ public final class PluginImportProgress: @unchecked Sendable { } } + public func incrementStatement(by amount: Int) { + guard amount > 0 else { return } + lock.lock() + let previous = internalCount + internalCount += amount + let count = internalCount + lock.unlock() + if count / updateInterval != previous / updateInterval { + progress.completedUnitCount = Int64(count) + } + } + public func setStatus(_ message: String) { progress.localizedAdditionalDescription = message } diff --git a/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/Plugins/JSONImportPlugin/JSONImportPlugin.swift index 5ac86569a..47705f3ab 100644 --- a/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -43,12 +43,12 @@ final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { let url = source.fileURL() let useTransaction = settings.wrapInTransaction && settings.errorHandling != .skipAndContinue - progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) - + let batchSize = 500 var inserted = 0 var skipped = 0 var errors: [PluginImportResult.ImportStatementError] = [] let maxErrors = 1_000 + var batch: [(line: Int, row: [String: PluginCellValue])] = [] do { if settings.deleteExistingRows { @@ -59,25 +59,35 @@ final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { } if JSONImportParsing.isLineDelimited(url) { + progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) var lineNumber = 0 for try await line in url.lines { try progress.checkCancellation() lineNumber += 1 let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { continue } - let row = try JSONImportParsing.parseRow(fromLine: trimmed) - try await insert(row, into: sink, at: lineNumber, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) + batch.append((lineNumber, try JSONImportParsing.parseRow(fromLine: trimmed))) + if batch.count >= batchSize { + try await flush(&batch, into: sink, progress: progress, + inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) + } } } else { let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) + progress.setEstimatedTotal(rawRows.count) for (index, rawRow) in rawRows.enumerated() { try progress.checkCancellation() - try await insert(JSONImportParsing.convertRow(rawRow), into: sink, at: index + 1, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) + batch.append((index + 1, JSONImportParsing.convertRow(rawRow))) + if batch.count >= batchSize { + try await flush(&batch, into: sink, progress: progress, + inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) + } } } + try await flush(&batch, into: sink, progress: progress, + inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) + if useTransaction { try await sink.commitTransaction() } @@ -131,6 +141,41 @@ final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { } } + private func flush( + _ batch: inout [(line: Int, row: [String: PluginCellValue])], + into sink: any PluginImportDataSink, + progress: PluginImportProgress, + inserted: inout Int, + skipped: inout Int, + errors: inout [PluginImportResult.ImportStatementError], + maxErrors: Int + ) async throws { + guard !batch.isEmpty else { return } + let entries = batch + batch.removeAll(keepingCapacity: true) + + do { + try await sink.insertRows(entries.map(\.row)) + inserted += entries.count + progress.incrementStatement(by: entries.count) + } catch { + switch settings.errorHandling { + case .stopAndRollback, .stopAndCommit: + let firstLine = entries.first?.line ?? 0 + throw PluginImportError.statementFailed( + statement: "rows \(firstLine)-\(entries.last?.line ?? firstLine)", + line: firstLine, + underlyingError: error + ) + case .skipAndContinue: + for entry in entries { + try await insert(entry.row, into: sink, at: entry.line, progress: progress, + inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) + } + } + } + } + // MARK: - Source introspection func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { diff --git a/Plugins/TableProPluginKit/PluginImportDataSink.swift b/Plugins/TableProPluginKit/PluginImportDataSink.swift index c004b7315..ef983d39c 100644 --- a/Plugins/TableProPluginKit/PluginImportDataSink.swift +++ b/Plugins/TableProPluginKit/PluginImportDataSink.swift @@ -10,6 +10,7 @@ public protocol PluginImportDataSink: AnyObject, Sendable { var targetTable: String? { get } func execute(statement: String) async throws func insertRow(_ values: [String: PluginCellValue]) async throws + func insertRows(_ rows: [[String: PluginCellValue]]) async throws func deleteAllRowsFromTargetTable() async throws func beginTransaction() async throws func commitTransaction() async throws @@ -23,6 +24,11 @@ public extension PluginImportDataSink { func insertRow(_ values: [String: PluginCellValue]) async throws { throw PluginImportError.importFailed("Row-based import is not supported by this connection") } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { + for row in rows { + try await insertRow(row) + } + } func deleteAllRowsFromTargetTable() async throws { throw PluginImportError.importFailed("Clearing the target table is not supported by this connection") } diff --git a/Plugins/TableProPluginKit/PluginImportProgress.swift b/Plugins/TableProPluginKit/PluginImportProgress.swift index 043b87e4e..1693cef68 100644 --- a/Plugins/TableProPluginKit/PluginImportProgress.swift +++ b/Plugins/TableProPluginKit/PluginImportProgress.swift @@ -27,6 +27,18 @@ public final class PluginImportProgress: @unchecked Sendable { } } + public func incrementStatement(by amount: Int) { + guard amount > 0 else { return } + lock.lock() + let previous = internalCount + internalCount += amount + let count = internalCount + lock.unlock() + if count / updateInterval != previous / updateInterval { + progress.completedUnitCount = Int64(count) + } + } + public func setStatus(_ message: String) { progress.localizedAdditionalDescription = message } diff --git a/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 681b63c56..32cf563fc 100644 --- a/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -196,6 +196,36 @@ struct SQLStatementGenerator { return ParameterizedStatement(sql: sql, parameters: bindParameters) } + func insertStatement(columns insertColumns: [String], rows: [[PluginCellValue]]) + -> ParameterizedStatement? + { + guard !insertColumns.isEmpty, !rows.isEmpty, + rows.allSatisfy({ $0.count == insertColumns.count }) else { return nil } + + var bindParameters: [Any?] = [] + let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") + let rowTuples = rows.map { values -> String in + let placeholders = values.map { value -> String in + bindParameters.append(value.asAny) + return placeholder(at: bindParameters.count - 1) + }.joined(separator: ", ") + return "(\(placeholders))" + }.joined(separator: ", ") + + let sql = + "INSERT INTO \(quoteIdentifierFn(tableName)) (\(columnList)) VALUES \(rowTuples)" + + return ParameterizedStatement(sql: sql, parameters: bindParameters) + } + + var maxBindParameters: Int { + switch databaseType { + case .sqlite: 32_766 + case .mssql: 2_100 + default: 65_535 + } + } + func deleteAllRowsStatement() -> String { "DELETE FROM \(quoteIdentifierFn(tableName))" } diff --git a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index 784113755..5f087d70c 100644 --- a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -72,6 +72,55 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { _ = try await driver.executeParameterized(query: statement.sql, parameters: statement.parameters) } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { + guard targetTable != nil else { + throw PluginImportError.importFailed("No target table configured for row import") + } + guard let rowGenerator else { + throw PluginImportError.importFailed("Could not resolve SQL dialect for row import") + } + + var index = 0 + while index < rows.count { + let (columns, values) = mappedColumnsAndValues(rows[index]) + guard !columns.isEmpty else { + index += 1 + continue + } + + var groupValues: [[PluginCellValue]] = [values] + var next = index + 1 + while next < rows.count { + let (nextColumns, nextValues) = mappedColumnsAndValues(rows[next]) + guard nextColumns == columns else { break } + groupValues.append(nextValues) + next += 1 + } + index = next + + let chunkSize = max(1, rowGenerator.maxBindParameters / columns.count) + var offset = 0 + while offset < groupValues.count { + let end = min(offset + chunkSize, groupValues.count) + let chunk = Array(groupValues[offset.. ([String], [PluginCellValue]) { + var pairs: [(column: String, value: PluginCellValue)] = [] + for (field, value) in values { + guard let column = columnMapping[field.lowercased()] else { continue } + pairs.append((column, value)) + } + pairs.sort { $0.column < $1.column } + return (pairs.map(\.column), pairs.map(\.value)) + } + func deleteAllRowsFromTargetTable() async throws { guard targetTable != nil, let rowGenerator else { throw PluginImportError.importFailed("No target table configured for row import") diff --git a/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift b/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift index a6934b9e8..908e19306 100644 --- a/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift +++ b/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift @@ -65,6 +65,44 @@ struct SQLStatementGeneratorImportTests { #expect(generator.insertStatement(columns: ["a"], values: ["1", "2"]) == nil) } + @Test("multi-row insertStatement emits one VALUES tuple per row with shared columns (MySQL)") + func testMultiRowInsertMySQL() throws { + let generator = try makeGenerator() + let stmt = try #require(generator.insertStatement( + columns: ["id", "name"], + rows: [["1", .text("a")], ["2", .text("b")]] + )) + #expect(stmt.sql == "INSERT INTO `users` (`id`, `name`) VALUES (?, ?), (?, ?)") + #expect(stmt.parameters.count == 4) + #expect(stmt.parameters[2] as? String == "2") + #expect(stmt.parameters[3] as? String == "b") + } + + @Test("multi-row insertStatement numbers placeholders across rows for PostgreSQL") + func testMultiRowInsertPostgres() throws { + let generator = try makeGenerator(databaseType: .postgresql) + let stmt = try #require(generator.insertStatement( + columns: ["id", "name"], + rows: [["1", .text("a")], ["2", .text("b")]] + )) + #expect(stmt.sql == "INSERT INTO \"users\" (\"id\", \"name\") VALUES ($1, $2), ($3, $4)") + } + + @Test("multi-row insertStatement returns nil for empty rows or arity mismatch") + func testMultiRowInsertGuards() throws { + let generator = try makeGenerator() + #expect(generator.insertStatement(columns: ["a"], rows: []) == nil) + #expect(generator.insertStatement(columns: ["a", "b"], rows: [["1"]]) == nil) + } + + @Test("maxBindParameters reflects each engine's protocol limit") + func testMaxBindParameters() throws { + #expect(try makeGenerator(databaseType: .postgresql).maxBindParameters == 65_535) + #expect(try makeGenerator(databaseType: .sqlite).maxBindParameters == 32_766) + #expect(try makeGenerator(databaseType: .mssql).maxBindParameters == 2_100) + #expect(try makeGenerator(databaseType: .mysql).maxBindParameters == 65_535) + } + @Test("deleteAllRowsStatement quotes the table identifier per dialect") func testDeleteAllRows() throws { #expect(try makeGenerator(databaseType: .mysql).deleteAllRowsStatement() == "DELETE FROM `users`") diff --git a/TableProTests/Core/Services/Export/PluginImportProgressTests.swift b/TableProTests/Core/Services/Export/PluginImportProgressTests.swift new file mode 100644 index 000000000..7d7413407 --- /dev/null +++ b/TableProTests/Core/Services/Export/PluginImportProgressTests.swift @@ -0,0 +1,37 @@ +// +// PluginImportProgressTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct PluginImportProgressTests { + @Test("Exact total set once is reported as the estimated total") + func exactTotalReported() { + let progress = PluginImportProgress(progress: Progress(totalUnitCount: 0)) + progress.setEstimatedTotal(3_503) + #expect(progress.estimatedTotalStatements == 3_503) + } + + @Test("Batched increments accumulate the exact processed count without overshoot") + func batchedIncrementsCountExactly() { + let progress = PluginImportProgress(progress: Progress(totalUnitCount: 0)) + progress.setEstimatedTotal(3_503) + for _ in 0..<7 { progress.incrementStatement(by: 500) } + progress.incrementStatement(by: 3) + #expect(progress.processedStatements == 3_503) + #expect(progress.processedStatements == progress.estimatedTotalStatements) + } + + @Test("finalize flushes the live completed count to the underlying Progress") + func finalizeFlushesCount() { + let nsProgress = Progress(totalUnitCount: 0) + let progress = PluginImportProgress(progress: nsProgress) + progress.setEstimatedTotal(10) + progress.incrementStatement(by: 7) + progress.finalize() + #expect(nsProgress.completedUnitCount == 7) + } +}