From 5826fec519aa33496718a55cbb518a26855de3f5 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sat, 6 Aug 2016 06:21:25 +0200 Subject: [PATCH] Some new import stuff. --- app/Console/Commands/Import.php | 3 +- app/Http/Controllers/ImportController.php | 17 +- app/Import/ImportEntry.php | 161 ++++++- app/Import/ImportResult.php | 168 +++++++ app/Import/Importer/CsvImporter.php | 482 -------------------- app/Import/Importer/ImporterInterface.php | 74 --- app/Import/Setup/CsvSetup.php | 528 ++++++++++++++++++++++ app/Import/Setup/SetupInterface.php | 92 ++++ 8 files changed, 956 insertions(+), 569 deletions(-) create mode 100644 app/Import/ImportResult.php create mode 100644 app/Import/Setup/CsvSetup.php create mode 100644 app/Import/Setup/SetupInterface.php diff --git a/app/Console/Commands/Import.php b/app/Console/Commands/Import.php index 8a139083df..33be3c8e06 100644 --- a/app/Console/Commands/Import.php +++ b/app/Console/Commands/Import.php @@ -12,6 +12,7 @@ declare(strict_types = 1); namespace FireflyIII\Console\Commands; use FireflyIII\Import\Importer\ImporterInterface; +use FireflyIII\Import\Setup\SetupInterface; use FireflyIII\Import\Logging\CommandHandler; use FireflyIII\Models\ImportJob; use Illuminate\Console\Command; @@ -70,6 +71,7 @@ class Import extends Command $this->line('Going to import job with key "' . $job->key . '" of type ' . $job->file_type); $class = config('firefly.import_formats.' . $job->file_type); + /** @var ImporterInterface $importer */ $importer = app($class); $importer->setJob($job); @@ -79,7 +81,6 @@ class Import extends Command $monolog->pushHandler($handler); $importer->start(); - $this->line('Something something import: ' . $jobKey); } } diff --git a/app/Http/Controllers/ImportController.php b/app/Http/Controllers/ImportController.php index 40a636deaf..03a39c2f49 100644 --- a/app/Http/Controllers/ImportController.php +++ b/app/Http/Controllers/ImportController.php @@ -1,4 +1,11 @@ file_type; - /** @var ImporterInterface $importer */ - $importer = app('FireflyIII\Import\Importer\\' . ucfirst($type) . 'Importer'); + /** @var SetupInterface $importer */ + $importer = app('FireflyIII\Import\Setup\\' . ucfirst($type) . 'Setup'); $importer->setJob($job); return $importer; diff --git a/app/Import/ImportEntry.php b/app/Import/ImportEntry.php index bdae2d8398..3e75f643c0 100644 --- a/app/Import/ImportEntry.php +++ b/app/Import/ImportEntry.php @@ -13,6 +13,13 @@ namespace FireflyIII\Import; use Carbon\Carbon; use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Models\Account; +use FireflyIII\Models\TransactionCurrency; +use FireflyIII\Models\TransactionJournal; +use FireflyIII\Models\TransactionType; +use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface; +use FireflyIII\User; +use Illuminate\Support\Collection; use Log; /** @@ -27,6 +34,12 @@ class ImportEntry /** @var array */ public $fields = []; + /** @var Account */ + public $defaultImportAccount; + + /** @var User */ + public $user; + /** @var array */ private $validFields = ['amount', @@ -35,14 +48,14 @@ class ImportEntry 'date-book', 'description', 'date-process', - 'currency', 'asset-account', 'opposing-account', 'bill', 'budget', 'category']; + 'currency', 'asset-account', 'opposing-account', 'bill', 'budget', 'category', 'tags']; /** * ImportEntry constructor. */ public function __construct() { - + $this->defaultImportAccount = new Account; /** @var string $value */ foreach ($this->validFields as $value) { $this->fields[$value] = null; @@ -50,6 +63,21 @@ class ImportEntry } } + /** + * @return ImportResult + */ + public function import(): ImportResult + { + + $validation = $this->validate(); + + if ($validation->valid()) { + return $this->doImport(); + } + + return $validation; + } + /** * @param string $role * @param string $value @@ -81,10 +109,9 @@ class ImportEntry $this->setObject('asset-account', $convertedValue, $certainty); break; case 'opposing-number': - case 'opposing-iban': - case 'opposing-id': - case 'opposing-number': - case 'opposing-name': + case 'opposing-iban': + case 'opposing-id': + case 'opposing-name': $this->setObject('opposing-account', $convertedValue, $certainty); break; case 'bill-id': @@ -124,12 +151,102 @@ class ImportEntry case '_ignore': break; case 'ing-debet-credit': - case 'rabo-debet-credit': + case 'rabo-debet-credit': $this->manipulateFloat('amount', 'multiply', $convertedValue); break; + case 'tags-comma': + case 'tags-space': + $this->appendCollection('tags', $convertedValue); + } } + /** + * @param User $user + */ + public function setUser(User $user) + { + $this->user = $user; + } + + /** + * @param string $field + * @param Collection $convertedValue + */ + private function appendCollection(string $field, Collection $convertedValue) + { + if (is_null($this->fields[$field])) { + $this->fields[$field] = new Collection; + } + $this->fields[$field] = $this->fields[$field]->merge($convertedValue); + } + + + /** + * @return ImportResult + */ + private function doImport(): ImportResult + { + $result = new ImportResult; + + // here we go! + $journal = new TransactionJournal; + $journal->user()->associate($this->user); + $journal->transactionType()->associate($this->getTransactionType()); + $journal->transactionCurrency()->associate($this->getTransactionCurrency()); + $journal->description = $this->fields['description'] ?? '(empty transaction description)'; + $journal->date = $this->fields['date-transaction'] ?? new Carbon; + $journal->interest_date = $this->fields['date-interest']; + $journal->process_date = $this->fields['date-process']; + $journal->book_date = $this->fields['date-book']; + $journal->completed = 0; + + + + + } + + /** + * @return TransactionCurrency + */ + private function getTransactionCurrency(): TransactionCurrency + { + if (!is_null($this->fields['currency'])) { + return $this->fields['currency']; + } + /** @var CurrencyRepositoryInterface $repository */ + $repository = app(CurrencyRepositoryInterface::class); + + return $repository->findByCode(env('DEFAULT_CURRENCY', 'EUR')); + } + + /** + * @return TransactionType + */ + private function getTransactionType(): TransactionType + { + + + /* + * source: import/asset/expense/revenue/null + * destination: import/asset/expense/revenue/null + * + * */ + + // source and opposing are asset = transfer + // source = asset and dest = import and amount = neg = withdrawal + // source = asset and dest = expense and amount = neg = withdrawal + // source = asset and dest = revenue and amount = pos = deposit + // source = asset and dest = import and amount = pos = deposit + + // source = import + + // source = expense + // + + // source = revenue + } + /** * @param string $field * @param string $action @@ -212,4 +329,34 @@ class ImportEntry } + /** + * Validate the content of the import entry so far. We only need a few things. + * + * @return ImportResult + */ + private function validate(): ImportResult + { + $result = new ImportResult; + $result->validated(); + if ($this->fields['amount'] == 0) { + // false, amount must be above or below zero. + $result->failed(); + $result->appendError('No valid amount found.'); + } + if (is_null($this->fields['date-transaction'])) { + $result->appendWarning('No valid date found.'); + } + if (is_null($this->fields['description']) || (!is_null($this->fields['description']) && strlen($this->fields['description']) == 0)) { + $result->appendWarning('No valid description found.'); + } + if (is_null($this->fields['asset-account'])) { + $result->appendWarning('No valid asset account found. Will use default account.'); + } + if (is_null($this->fields['opposing-account'])) { + $result->appendWarning('No valid asset opposing found. Will use default.'); + } + + return $result; + } + } \ No newline at end of file diff --git a/app/Import/ImportResult.php b/app/Import/ImportResult.php new file mode 100644 index 0000000000..e7be183e85 --- /dev/null +++ b/app/Import/ImportResult.php @@ -0,0 +1,168 @@ +errors = new Collection; + $this->warnings = new Collection; + $this->messages = new Collection; + } + + /** + * @param string $error + * + * @return $this + */ + public function appendError(string $error) + { + $this->errors->push($error); + + return $this; + } + + /** + * @param string $message + * + * @return $this + */ + public function appendMessage(string $message) + { + $this->messages->push($message); + + return $this; + } + + /** + * @param string $title + * + * @return $this + */ + public function appendTitle(string $title) + { + $this->title .= $title; + + return $this; + } + + /** + * @param string $warning + * + * @return $this + */ + public function appendWarning(string $warning) + { + $this->warnings->push($warning); + + return $this; + } + + /** + * @return $this + */ + public function failed() + { + $this->status = self::IMPORT_FAILED; + + return $this; + } + + /** + * @param Collection $errors + */ + public function setErrors(Collection $errors) + { + $this->errors = $errors; + } + + /** + * @param Collection $messages + */ + public function setMessages(Collection $messages) + { + $this->messages = $messages; + } + + /** + * @param string $title + * + * @return $this + */ + public function setTitle(string $title) + { + $this->title = $title; + + return $this; + } + + /** + * @param Collection $warnings + */ + public function setWarnings(Collection $warnings) + { + $this->warnings = $warnings; + } + + /** + * @return $this + */ + public function success() + { + $this->status = self::IMPORT_SUCCESS; + + return $this; + } + + /** + * @return bool + */ + public function valid(): bool + { + return $this->status === self::IMPORT_VALID; + } + + /** + * + */ + public function validated() + { + $this->status = self::IMPORT_VALID; + } + + +} \ No newline at end of file diff --git a/app/Import/Importer/CsvImporter.php b/app/Import/Importer/CsvImporter.php index 731129bdc5..5722cab569 100644 --- a/app/Import/Importer/CsvImporter.php +++ b/app/Import/Importer/CsvImporter.php @@ -11,20 +11,6 @@ declare(strict_types = 1); namespace FireflyIII\Import\Importer; - -use ExpandedForm; -use FireflyIII\Crud\Account\AccountCrud; -use FireflyIII\Import\Converter\ConverterInterface; -use FireflyIII\Import\ImportEntry; -use FireflyIII\Import\Mapper\MapperInterface; -use FireflyIII\Import\MapperPreProcess\PreProcessorInterface; -use FireflyIII\Models\AccountType; -use FireflyIII\Models\ImportJob; -use Illuminate\Http\Request; -use League\Csv\Reader; -use Log; -use Symfony\Component\HttpFoundation\FileBag; - /** * Class CsvImporter * @@ -32,473 +18,5 @@ use Symfony\Component\HttpFoundation\FileBag; */ class CsvImporter implements ImporterInterface { - const EXAMPLE_ROWS = 5; - /** @var ImportJob */ - public $job; - /** - * Create initial (empty) configuration array. - * - * - * - * @return bool - */ - public function configure(): bool - { - if (is_null($this->job->configuration) || (is_array($this->job->configuration) && count($this->job->configuration) === 0)) { - Log::debug('No config detected, will create empty one.'); - - $config = [ - 'has-headers' => false, // assume - 'date-format' => 'Ymd', // assume - 'delimiter' => ',', // assume - 'import-account' => 0, // none, - 'specifics' => [], // none - 'column-count' => 0, // unknown - 'column-roles' => [], // unknown - 'column-do-mapping' => [], // not yet set which columns must be mapped - 'column-roles-complete' => false, // not yet configured roles for columns - 'column-mapping-config' => [], // no mapping made yet. - 'column-mapping-complete' => false, // so mapping is not complete. - ]; - $this->job->configuration = $config; - $this->job->save(); - - return true; - } - - // need to do nothing, for now. - Log::debug('Detected config in upload, will use that one. ', $this->job->configuration); - - return true; - } - - /** - * @return array - */ - public function getConfigurationData(): array - { - $crud = app('FireflyIII\Crud\Account\AccountCrudInterface'); - $accounts = $crud->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]); - $delimiters = [ - ',' => trans('form.csv_comma'), - ';' => trans('form.csv_semicolon'), - 'tab' => trans('form.csv_tab'), - ]; - - $specifics = []; - - // collect specifics. - foreach (config('csv.import_specifics') as $name => $className) { - $specifics[$name] = [ - 'name' => $className::getName(), - 'description' => $className::getDescription(), - ]; - } - - $data = [ - 'accounts' => ExpandedForm::makeSelectList($accounts), - 'specifix' => [], - 'delimiters' => $delimiters, - 'upload_path' => storage_path('upload'), - 'is_upload_possible' => is_writable(storage_path('upload')), - 'specifics' => $specifics, - ]; - - return $data; - } - - /** - * This method returns the data required for the view that will let the user add settings to the import job. - * - * @return array - */ - public function getDataForSettings(): array - { - - if ($this->doColumnRoles()) { - $data = $this->getDataForColumnRoles(); - - return $data; - } - - if ($this->doColumnMapping()) { - $data = $this->getDataForColumnMapping(); - - return $data; - } - - echo 'no settings to do.'; - exit; - - } - - /** - * This method returns the name of the view that will be shown to the user to further configure - * the import job. - * - * @return string - */ - public function getViewForSettings(): string - { - if ($this->doColumnRoles()) { - return 'import.csv.roles'; - } - - if ($this->doColumnMapping()) { - return 'import.csv.map'; - } - - echo 'no view for settings'; - exit; - } - - /** - * This method returns whether or not the user must configure this import - * job further. - * - * @return bool - */ - public function requireUserSettings(): bool - { - Log::debug('doColumnMapping is ' . ($this->doColumnMapping() ? 'true' : 'false')); - Log::debug('doColumnRoles is ' . ($this->doColumnRoles() ? 'true' : 'false')); - if ($this->doColumnMapping() || $this->doColumnRoles()) { - Log::debug('Return true'); - - return true; - } - Log::debug('Return false'); - - return false; - } - - /** - * @param array $data - * - * @return bool - */ - public function saveImportConfiguration(array $data, FileBag $files): bool - { - /** @var AccountCrud $repository */ - $repository = app(AccountCrud::class, [auth()->user()]); - $account = $repository->find(intval($data['csv_import_account'])); - - $hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false; - $config = $this->job->configuration; - $config['has-headers'] = $hasHeaders; - $config['date-format'] = $data['date_format']; - $config['delimiter'] = $data['csv_delimiter']; - - Log::debug('Entered import account.', ['id' => $data['csv_import_account']]); - - - if (!is_null($account->id)) { - Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]); - $config['import-account'] = $account->id; - } else { - Log::error('Could not find anything for csv_import_account.', ['id' => $data['csv_import_account']]); - } - // loop specifics. - if (isset($data['specifics']) && is_array($data['specifics'])) { - foreach ($data['specifics'] as $name => $enabled) { - $config['specifics'][$name] = 1; - } - } - $this->job->configuration = $config; - $this->job->save(); - - return true; - - - } - - /** - * @param ImportJob $job - */ - public function setJob(ImportJob $job) - { - $this->job = $job; - } - - /** - * Run the actual import - * - * @return bool - */ - public function start(): bool - { - $config = $this->job->configuration; - $content = $this->job->uploadFileContents(); - - // create CSV reader. - $reader = Reader::createFromString($content); - $start = $config['has-headers'] ? 1 : 0; - $results = $reader->fetch(); - foreach ($results as $index => $row) { - if ($index >= $start) { - Log::debug(sprintf('Now going to import row %d.', $index)); - $this->importSingleRow($row); - } - } - - Log::debug('This call should be intercepted somehow.'); - - return true; - } - - /** - * Store the settings filled in by the user, if applicable. - * - * @param Request $request - * - */ - public function storeSettings(Request $request) - { - $config = $this->job->configuration; - $all = $request->all(); - if ($request->get('settings') == 'roles') { - $count = $config['column-count']; - - $roleSet = 0; // how many roles have been defined - $mapSet = 0; // how many columns must be mapped - for ($i = 0; $i < $count; $i++) { - $selectedRole = $all['role'][$i] ?? '_ignore'; - $doMapping = isset($all['map'][$i]) && $all['map'][$i] == '1' ? true : false; - if ($selectedRole == '_ignore' && $doMapping === true) { - $doMapping = false; // cannot map ignored columns. - } - if ($selectedRole != '_ignore') { - $roleSet++; - } - if ($doMapping === true) { - $mapSet++; - } - $config['column-roles'][$i] = $selectedRole; - $config['column-do-mapping'][$i] = $doMapping; - } - if ($roleSet > 0) { - $config['column-roles-complete'] = true; - $this->job->configuration = $config; - $this->job->save(); - } - if ($mapSet === 0) { - // skip setting of map: - $config['column-mapping-complete'] = true; - } - } - if ($request->get('settings') == 'map') { - if (isset($all['mapping'])) { - foreach ($all['mapping'] as $index => $data) { - $config['column-mapping-config'][$index] = []; - foreach ($data as $value => $mapId) { - $mapId = intval($mapId); - if ($mapId !== 0) { - $config['column-mapping-config'][$index][$value] = intval($mapId); - } - } - } - } - - // set thing to be completed. - $config['column-mapping-complete'] = true; - $this->job->configuration = $config; - $this->job->save(); - } - } - - /** - * @return bool - */ - private function doColumnMapping(): bool - { - $mapArray = $this->job->configuration['column-do-mapping'] ?? []; - $doMap = false; - foreach ($mapArray as $value) { - if ($value === true) { - $doMap = true; - break; - } - } - - return $this->job->configuration['column-mapping-complete'] === false && $doMap; - } - - /** - * @return bool - */ - private function doColumnRoles(): bool - { - return $this->job->configuration['column-roles-complete'] === false; - } - - /** - * @return array - */ - private function getDataForColumnMapping(): array - { - $config = $this->job->configuration; - $data = []; - $indexes = []; - - foreach ($config['column-do-mapping'] as $index => $mustBeMapped) { - if ($mustBeMapped) { - $column = $config['column-roles'][$index] ?? '_ignore'; - $canBeMapped = config('csv.import_roles.' . $column . '.mappable'); - $preProcessMap = config('csv.import_roles.' . $column . '.pre-process-map'); - if ($canBeMapped) { - $mapperName = '\FireflyIII\Import\Mapper\\' . config('csv.import_roles.' . $column . '.mapper'); - /** @var MapperInterface $mapper */ - $mapper = new $mapperName; - $indexes[] = $index; - $data[$index] = [ - 'name' => $column, - 'mapper' => $mapperName, - 'index' => $index, - 'options' => $mapper->getMap(), - 'preProcessMap' => null, - 'values' => [], - ]; - if ($preProcessMap) { - $data[$index]['preProcessMap'] = '\FireflyIII\Import\MapperPreProcess\\' . - config('csv.import_roles.' . $column . '.pre-process-mapper'); - } - } - - } - } - - // in order to actually map we also need all possible values from the CSV file. - $content = $this->job->uploadFileContents(); - $reader = Reader::createFromString($content); - $results = $reader->fetch(); - - foreach ($results as $rowIndex => $row) { - //do something here - foreach ($indexes as $index) { // this is simply 1, 2, 3, etc. - $value = $row[$index]; - if (strlen($value) > 0) { - - // we can do some preprocessing here, - // which is exclusively to fix the tags: - if (!is_null($data[$index]['preProcessMap'])) { - /** @var PreProcessorInterface $preProcessor */ - $preProcessor = app($data[$index]['preProcessMap']); - $result = $preProcessor->run($value); - $data[$index]['values'] = array_merge($data[$index]['values'], $result); - - Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]); - Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]); - Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $data[$index]['values']]); - - - continue; - } - - $data[$index]['values'][] = $value; - } - } - } - foreach ($data as $index => $entry) { - $data[$index]['values'] = array_unique($data[$index]['values']); - } - - return $data; - } - - /** - * @return array - */ - private function getDataForColumnRoles():array - { - $config = $this->job->configuration; - $data = [ - 'columns' => [], - 'columnCount' => 0, - ]; - - // show user column role configuration. - $content = $this->job->uploadFileContents(); - - // create CSV reader. - $reader = Reader::createFromString($content); - $start = $config['has-headers'] ? 1 : 0; - $end = $start + self::EXAMPLE_ROWS; // first X rows - - // collect example data in $data['columns'] - while ($start < $end) { - $row = $reader->fetchOne($start); - foreach ($row as $index => $value) { - $value = trim($value); - if (strlen($value) > 0) { - $data['columns'][$index][] = $value; - } - } - $start++; - $data['columnCount'] = count($row); - } - - // make unique example data - foreach ($data['columns'] as $index => $values) { - $data['columns'][$index] = array_unique($values); - } - - $data['set_roles'] = []; - // collect possible column roles: - $data['available_roles'] = []; - foreach (array_keys(config('csv.import_roles')) as $role) { - $data['available_roles'][$role] = trans('csv.column_' . $role); - } - - $config['column-count'] = $data['columnCount']; - $this->job->configuration = $config; - $this->job->save(); - - return $data; - - - } - - /** - * @param array $row - * - * @return bool - */ - private function importSingleRow(array $row): bool - { - $object = new ImportEntry; - $config = $this->job->configuration; - - foreach ($row as $index => $value) { - // find the role for this column: - $role = $config['column-roles'][$index] ?? '_ignore'; - $doMap = $config['column-do-mapping'][$index] ?? false; - $converterClass = config('csv.import_roles.' . $role . '.converter'); - $mapping = $config['column-mapping-config'][$index] ?? []; - /** @var ConverterInterface $converter */ - $converter = app('FireflyIII\\Import\\Converter\\' . $converterClass); - // set some useful values for the converter: - $converter->setMapping($mapping); - $converter->setDoMap($doMap); - $converter->setUser($this->job->user); - $converter->setConfig($config); - - // run the converter for this value: - $convertedValue = $converter->convert($value); - $certainty = $converter->getCertainty(); - - // log it. - Log::debug('Value ', ['index' => $index, 'value' => $value, 'role' => $role]); - - // store in import entry: - $object->importValue($role, $value, $certainty, $convertedValue); - // $object->fromRawValue($role, $value); - - - } - - exit; - - return true; - } } \ No newline at end of file diff --git a/app/Import/Importer/ImporterInterface.php b/app/Import/Importer/ImporterInterface.php index 66b72a6f08..f5a4883a9d 100644 --- a/app/Import/Importer/ImporterInterface.php +++ b/app/Import/Importer/ImporterInterface.php @@ -11,82 +11,8 @@ declare(strict_types = 1); namespace FireflyIII\Import\Importer; -use FireflyIII\Import\Role\Map; -use FireflyIII\Models\ImportJob; -use Illuminate\Http\Request; -use Symfony\Component\HttpFoundation\FileBag; -/** - * Interface ImporterInterface - * - * @package FireflyIII\Import\Importer - */ interface ImporterInterface { - /** - * Run the actual import - * - * @return bool - */ - public function start(): bool; - - /** - * After uploading, and after setJob(), prepare anything that is - * necessary for the configure() line. - * - * @return bool - */ - public function configure(): bool; - - /** - * Returns any data necessary to do the configuration. - * - * @return array - */ - public function getConfigurationData(): array; - - /** - * This method returns the data required for the view that will let the user add settings to the import job. - * - * @return array - */ - public function getDataForSettings(): array; - - /** - * Store the settings filled in by the user, if applicable. - * - * @param Request $request - * - */ - public function storeSettings(Request $request); - - /** - * This method returns the name of the view that will be shown to the user to further configure - * the import job. - * - * @return string - */ - public function getViewForSettings(): string; - - /** - * This method returns whether or not the user must configure this import - * job further. - * - * @return bool - */ - public function requireUserSettings(): bool; - - /** - * @param array $data - * - * @return bool - */ - public function saveImportConfiguration(array $data, FileBag $files): bool; - - /** - * @param ImportJob $job - * - */ - public function setJob(ImportJob $job); } \ No newline at end of file diff --git a/app/Import/Setup/CsvSetup.php b/app/Import/Setup/CsvSetup.php new file mode 100644 index 0000000000..4df6f69f6f --- /dev/null +++ b/app/Import/Setup/CsvSetup.php @@ -0,0 +1,528 @@ +defaultImportAccount = new Account; + } + + /** + * Create initial (empty) configuration array. + * + * + * + * @return bool + */ + public function configure(): bool + { + if (is_null($this->job->configuration) || (is_array($this->job->configuration) && count($this->job->configuration) === 0)) { + Log::debug('No config detected, will create empty one.'); + + $config = [ + 'has-headers' => false, // assume + 'date-format' => 'Ymd', // assume + 'delimiter' => ',', // assume + 'import-account' => 0, // none, + 'specifics' => [], // none + 'column-count' => 0, // unknown + 'column-roles' => [], // unknown + 'column-do-mapping' => [], // not yet set which columns must be mapped + 'column-roles-complete' => false, // not yet configured roles for columns + 'column-mapping-config' => [], // no mapping made yet. + 'column-mapping-complete' => false, // so mapping is not complete. + ]; + $this->job->configuration = $config; + $this->job->save(); + + return true; + } + + // need to do nothing, for now. + Log::debug('Detected config in upload, will use that one. ', $this->job->configuration); + + return true; + } + + /** + * @return array + */ + public function getConfigurationData(): array + { + $crud = app('FireflyIII\Crud\Account\AccountCrudInterface'); + $accounts = $crud->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]); + $delimiters = [ + ',' => trans('form.csv_comma'), + ';' => trans('form.csv_semicolon'), + 'tab' => trans('form.csv_tab'), + ]; + + $specifics = []; + + // collect specifics. + foreach (config('csv.import_specifics') as $name => $className) { + $specifics[$name] = [ + 'name' => $className::getName(), + 'description' => $className::getDescription(), + ]; + } + + $data = [ + 'accounts' => ExpandedForm::makeSelectList($accounts), + 'specifix' => [], + 'delimiters' => $delimiters, + 'upload_path' => storage_path('upload'), + 'is_upload_possible' => is_writable(storage_path('upload')), + 'specifics' => $specifics, + ]; + + return $data; + } + + /** + * This method returns the data required for the view that will let the user add settings to the import job. + * + * @return array + */ + public function getDataForSettings(): array + { + + if ($this->doColumnRoles()) { + $data = $this->getDataForColumnRoles(); + + return $data; + } + + if ($this->doColumnMapping()) { + $data = $this->getDataForColumnMapping(); + + return $data; + } + + echo 'no settings to do.'; + exit; + + } + + /** + * This method returns the name of the view that will be shown to the user to further configure + * the import job. + * + * @return string + */ + public function getViewForSettings(): string + { + if ($this->doColumnRoles()) { + return 'import.csv.roles'; + } + + if ($this->doColumnMapping()) { + return 'import.csv.map'; + } + + echo 'no view for settings'; + exit; + } + + /** + * This method returns whether or not the user must configure this import + * job further. + * + * @return bool + */ + public function requireUserSettings(): bool + { + Log::debug('doColumnMapping is ' . ($this->doColumnMapping() ? 'true' : 'false')); + Log::debug('doColumnRoles is ' . ($this->doColumnRoles() ? 'true' : 'false')); + if ($this->doColumnMapping() || $this->doColumnRoles()) { + Log::debug('Return true'); + + return true; + } + Log::debug('Return false'); + + return false; + } + + /** + * @param array $data + * + * @return bool + */ + public function saveImportConfiguration(array $data, FileBag $files): bool + { + /** @var AccountCrud $repository */ + $repository = app(AccountCrud::class, [auth()->user()]); + $account = $repository->find(intval($data['csv_import_account'])); + + $hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false; + $config = $this->job->configuration; + $config['has-headers'] = $hasHeaders; + $config['date-format'] = $data['date_format']; + $config['delimiter'] = $data['csv_delimiter']; + + Log::debug('Entered import account.', ['id' => $data['csv_import_account']]); + + + if (!is_null($account->id)) { + Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]); + $config['import-account'] = $account->id; + } else { + Log::error('Could not find anything for csv_import_account.', ['id' => $data['csv_import_account']]); + } + // loop specifics. + if (isset($data['specifics']) && is_array($data['specifics'])) { + foreach ($data['specifics'] as $name => $enabled) { + $config['specifics'][$name] = 1; + } + } + $this->job->configuration = $config; + $this->job->save(); + + return true; + + + } + + /** + * @param ImportJob $job + */ + public function setJob(ImportJob $job) + { + $this->job = $job; + } + + /** + * Run the actual import + * + * @return bool + */ + public function start(): bool + { + $config = $this->job->configuration; + $content = $this->job->uploadFileContents(); + + if ($config['import-account'] != 0) { + $repository = app(AccountCrud::class, [auth()->user()]); + $this->defaultImportAccount = $repository->find($config['csv_import_account']); + } + + + // create CSV reader. + $reader = Reader::createFromString($content); + $start = $config['has-headers'] ? 1 : 0; + $results = $reader->fetch(); + foreach ($results as $index => $row) { + if ($index >= $start) { + Log::debug(sprintf('Now going to import row %d.', $index)); + $this->importSingleRow($row); + } + } + + Log::debug('This call should be intercepted somehow.'); + + return true; + } + + /** + * Store the settings filled in by the user, if applicable. + * + * @param Request $request + * + */ + public function storeSettings(Request $request) + { + $config = $this->job->configuration; + $all = $request->all(); + if ($request->get('settings') == 'roles') { + $count = $config['column-count']; + + $roleSet = 0; // how many roles have been defined + $mapSet = 0; // how many columns must be mapped + for ($i = 0; $i < $count; $i++) { + $selectedRole = $all['role'][$i] ?? '_ignore'; + $doMapping = isset($all['map'][$i]) && $all['map'][$i] == '1' ? true : false; + if ($selectedRole == '_ignore' && $doMapping === true) { + $doMapping = false; // cannot map ignored columns. + } + if ($selectedRole != '_ignore') { + $roleSet++; + } + if ($doMapping === true) { + $mapSet++; + } + $config['column-roles'][$i] = $selectedRole; + $config['column-do-mapping'][$i] = $doMapping; + } + if ($roleSet > 0) { + $config['column-roles-complete'] = true; + $this->job->configuration = $config; + $this->job->save(); + } + if ($mapSet === 0) { + // skip setting of map: + $config['column-mapping-complete'] = true; + } + } + if ($request->get('settings') == 'map') { + if (isset($all['mapping'])) { + foreach ($all['mapping'] as $index => $data) { + $config['column-mapping-config'][$index] = []; + foreach ($data as $value => $mapId) { + $mapId = intval($mapId); + if ($mapId !== 0) { + $config['column-mapping-config'][$index][$value] = intval($mapId); + } + } + } + } + + // set thing to be completed. + $config['column-mapping-complete'] = true; + $this->job->configuration = $config; + $this->job->save(); + } + } + + /** + * @return bool + */ + private function doColumnMapping(): bool + { + $mapArray = $this->job->configuration['column-do-mapping'] ?? []; + $doMap = false; + foreach ($mapArray as $value) { + if ($value === true) { + $doMap = true; + break; + } + } + + return $this->job->configuration['column-mapping-complete'] === false && $doMap; + } + + /** + * @return bool + */ + private function doColumnRoles(): bool + { + return $this->job->configuration['column-roles-complete'] === false; + } + + /** + * @return array + */ + private function getDataForColumnMapping(): array + { + $config = $this->job->configuration; + $data = []; + $indexes = []; + + foreach ($config['column-do-mapping'] as $index => $mustBeMapped) { + if ($mustBeMapped) { + $column = $config['column-roles'][$index] ?? '_ignore'; + $canBeMapped = config('csv.import_roles.' . $column . '.mappable'); + $preProcessMap = config('csv.import_roles.' . $column . '.pre-process-map'); + if ($canBeMapped) { + $mapperName = '\FireflyIII\Import\Mapper\\' . config('csv.import_roles.' . $column . '.mapper'); + /** @var MapperInterface $mapper */ + $mapper = new $mapperName; + $indexes[] = $index; + $data[$index] = [ + 'name' => $column, + 'mapper' => $mapperName, + 'index' => $index, + 'options' => $mapper->getMap(), + 'preProcessMap' => null, + 'values' => [], + ]; + if ($preProcessMap) { + $data[$index]['preProcessMap'] = '\FireflyIII\Import\MapperPreProcess\\' . + config('csv.import_roles.' . $column . '.pre-process-mapper'); + } + } + + } + } + + // in order to actually map we also need all possible values from the CSV file. + $content = $this->job->uploadFileContents(); + $reader = Reader::createFromString($content); + $results = $reader->fetch(); + + foreach ($results as $rowIndex => $row) { + //do something here + foreach ($indexes as $index) { // this is simply 1, 2, 3, etc. + $value = $row[$index]; + if (strlen($value) > 0) { + + // we can do some preprocessing here, + // which is exclusively to fix the tags: + if (!is_null($data[$index]['preProcessMap'])) { + /** @var PreProcessorInterface $preProcessor */ + $preProcessor = app($data[$index]['preProcessMap']); + $result = $preProcessor->run($value); + $data[$index]['values'] = array_merge($data[$index]['values'], $result); + + Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]); + Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]); + Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $data[$index]['values']]); + + + continue; + } + + $data[$index]['values'][] = $value; + } + } + } + foreach ($data as $index => $entry) { + $data[$index]['values'] = array_unique($data[$index]['values']); + } + + return $data; + } + + /** + * @return array + */ + private function getDataForColumnRoles():array + { + $config = $this->job->configuration; + $data = [ + 'columns' => [], + 'columnCount' => 0, + ]; + + // show user column role configuration. + $content = $this->job->uploadFileContents(); + + // create CSV reader. + $reader = Reader::createFromString($content); + $start = $config['has-headers'] ? 1 : 0; + $end = $start + self::EXAMPLE_ROWS; // first X rows + + // collect example data in $data['columns'] + while ($start < $end) { + $row = $reader->fetchOne($start); + foreach ($row as $index => $value) { + $value = trim($value); + if (strlen($value) > 0) { + $data['columns'][$index][] = $value; + } + } + $start++; + $data['columnCount'] = count($row); + } + + // make unique example data + foreach ($data['columns'] as $index => $values) { + $data['columns'][$index] = array_unique($values); + } + + $data['set_roles'] = []; + // collect possible column roles: + $data['available_roles'] = []; + foreach (array_keys(config('csv.import_roles')) as $role) { + $data['available_roles'][$role] = trans('csv.column_' . $role); + } + + $config['column-count'] = $data['columnCount']; + $this->job->configuration = $config; + $this->job->save(); + + return $data; + + + } + + /** + * @param array $row + * + * @return ImportResult + */ + private function importSingleRow(array $row): ImportResult + { + $object = new ImportEntry; + $object->setUser($this->job->user); + $config = $this->job->configuration; + $result = new ImportResult; + + foreach ($row as $index => $value) { + // find the role for this column: + $role = $config['column-roles'][$index] ?? '_ignore'; + $doMap = $config['column-do-mapping'][$index] ?? false; + $converterClass = config('csv.import_roles.' . $role . '.converter'); + $mapping = $config['column-mapping-config'][$index] ?? []; + /** @var ConverterInterface $converter */ + $converter = app('FireflyIII\\Import\\Converter\\' . $converterClass); + // set some useful values for the converter: + $converter->setMapping($mapping); + $converter->setDoMap($doMap); + $converter->setUser($this->job->user); + $converter->setConfig($config); + + // run the converter for this value: + $convertedValue = $converter->convert($value); + $certainty = $converter->getCertainty(); + + + // log it. + Log::debug('Value ', ['index' => $index, 'value' => $value, 'role' => $role]); + + // store in import entry: + $object->importValue($role, $value, $certainty, $convertedValue); + // $object->fromRawValue($role, $value); + } + + $result = $object->import(); + if ($result->failed()) { + Log::error('Import of row has failed.', $result->errors->toArray()); + } + + exit; + + return true; + } +} \ No newline at end of file diff --git a/app/Import/Setup/SetupInterface.php b/app/Import/Setup/SetupInterface.php new file mode 100644 index 0000000000..66b72a6f08 --- /dev/null +++ b/app/Import/Setup/SetupInterface.php @@ -0,0 +1,92 @@ +