From c9bd72337d00062328078291a358d86a37e32dc6 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sat, 6 Aug 2016 09:31:32 +0200 Subject: [PATCH] Some notes on the import process. --- app/Import/ImportEntry.php | 102 +--------------------------- app/Import/Importer/CsvImporter.php | 14 ++-- app/Import/notes.txt | 46 +++++++++++++ 3 files changed, 54 insertions(+), 108 deletions(-) create mode 100644 app/Import/notes.txt diff --git a/app/Import/ImportEntry.php b/app/Import/ImportEntry.php index 3e75f643c0..05b3d76a10 100644 --- a/app/Import/ImportEntry.php +++ b/app/Import/ImportEntry.php @@ -34,9 +34,6 @@ class ImportEntry /** @var array */ public $fields = []; - /** @var Account */ - public $defaultImportAccount; - /** @var User */ public $user; @@ -181,72 +178,6 @@ class ImportEntry $this->fields[$field] = $this->fields[$field]->merge($convertedValue); } - - /** - * @return ImportResult - */ - private function doImport(): ImportResult - { - $result = new ImportResult; - - // here we go! - $journal = new TransactionJournal; - $journal->user()->associate($this->user); - $journal->transactionType()->associate($this->getTransactionType()); - $journal->transactionCurrency()->associate($this->getTransactionCurrency()); - $journal->description = $this->fields['description'] ?? '(empty transaction description)'; - $journal->date = $this->fields['date-transaction'] ?? new Carbon; - $journal->interest_date = $this->fields['date-interest']; - $journal->process_date = $this->fields['date-process']; - $journal->book_date = $this->fields['date-book']; - $journal->completed = 0; - - - - - } - - /** - * @return TransactionCurrency - */ - private function getTransactionCurrency(): TransactionCurrency - { - if (!is_null($this->fields['currency'])) { - return $this->fields['currency']; - } - /** @var CurrencyRepositoryInterface $repository */ - $repository = app(CurrencyRepositoryInterface::class); - - return $repository->findByCode(env('DEFAULT_CURRENCY', 'EUR')); - } - - /** - * @return TransactionType - */ - private function getTransactionType(): TransactionType - { - - - /* - * source: import/asset/expense/revenue/null - * destination: import/asset/expense/revenue/null - * - * */ - - // source and opposing are asset = transfer - // source = asset and dest = import and amount = neg = withdrawal - // source = asset and dest = expense and amount = neg = withdrawal - // source = asset and dest = revenue and amount = pos = deposit - // source = asset and dest = import and amount = pos = deposit - - // source = import - - // source = expense - // - - // source = revenue - } - /** * @param string $field * @param string $action @@ -328,35 +259,4 @@ class ImportEntry Log::error(sprintf('Will not set %s based on certainty %d (current certainty is %d) or NULL id.', $field, $certainty, $this->certain[$field])); } - - /** - * Validate the content of the import entry so far. We only need a few things. - * - * @return ImportResult - */ - private function validate(): ImportResult - { - $result = new ImportResult; - $result->validated(); - if ($this->fields['amount'] == 0) { - // false, amount must be above or below zero. - $result->failed(); - $result->appendError('No valid amount found.'); - } - if (is_null($this->fields['date-transaction'])) { - $result->appendWarning('No valid date found.'); - } - if (is_null($this->fields['description']) || (!is_null($this->fields['description']) && strlen($this->fields['description']) == 0)) { - $result->appendWarning('No valid description found.'); - } - if (is_null($this->fields['asset-account'])) { - $result->appendWarning('No valid asset account found. Will use default account.'); - } - if (is_null($this->fields['opposing-account'])) { - $result->appendWarning('No valid asset opposing found. Will use default.'); - } - - return $result; - } - -} \ No newline at end of file +} diff --git a/app/Import/Importer/CsvImporter.php b/app/Import/Importer/CsvImporter.php index 85ee7359dc..e989d603a6 100644 --- a/app/Import/Importer/CsvImporter.php +++ b/app/Import/Importer/CsvImporter.php @@ -56,7 +56,6 @@ class CsvImporter implements ImporterInterface $this->defaultImportAccount = $repository->find($config['import-account']); } - // create CSV reader. $reader = Reader::createFromString($content); $start = $config['has-headers'] ? 1 : 0; @@ -64,7 +63,7 @@ class CsvImporter implements ImporterInterface foreach ($results as $index => $row) { if ($index >= $start) { Log::debug(sprintf('Now going to import row %d.', $index)); - $this->importSingleRow($row); + $this->importSingleRow($index, $row); } } @@ -75,16 +74,19 @@ class CsvImporter implements ImporterInterface /** + * @param int $index * @param array $row * * @return ImportResult */ - private function importSingleRow(array $row): ImportResult + private function importSingleRow(int $index, array $row): ImportResult { + // create import object: $object = new ImportEntry; + + // set some vars: $object->setUser($this->job->user); $config = $this->job->configuration; - $result = new ImportResult; foreach ($row as $index => $value) { // find the role for this column: @@ -104,18 +106,16 @@ class CsvImporter implements ImporterInterface $convertedValue = $converter->convert($value); $certainty = $converter->getCertainty(); - // log it. Log::debug('Value ', ['index' => $index, 'value' => $value, 'role' => $role]); // store in import entry: $object->importValue($role, $value, $certainty, $convertedValue); - // $object->fromRawValue($role, $value); } $result = $object->import(); if ($result->failed()) { - Log::error('Import of row has failed.', $result->errors->toArray()); + Log::error(sprintf('Import of row %d has failed.', $index), $result->errors->toArray()); } exit; diff --git a/app/Import/notes.txt b/app/Import/notes.txt new file mode 100644 index 0000000000..045c762e4a --- /dev/null +++ b/app/Import/notes.txt @@ -0,0 +1,46 @@ +The import routine is as follows: + +1. Upload and setup: + +User uploads a file with entries. The Setup/SetupInterface gives the user +the opportunity (in any number of steps) to do the necessary configuration. +This could also be skipped of course. An ImportJob object is created with a +basic and empty configuration. + +Helper classes are as follows, greatly modelled to the CSV importer: + +- The Mapper classes give back lists of Firefly objects. You can show them to the +user in order to help you convert text values to their Firefly counterparts. +For example, the user maps "Gcrsr" to category Groceries. +- The Converter classes exist to help convert text values to their Firely counterparts. +Feed "AB12ABCD897829" to the AssetAccountIban Converter and you should end up with a new +or found asset account. The previously built mapping is used to narrow it down. Submit an empty +mapping if this one is not relevant. + +The mapping and possibly other configuration options are stored in a newly created +ImportJob object, stored in the database. This import job holds a reference to the uploaded file +(placed encrypted in /storage/uploads) and the status of the import. + +2. Actual import + +Using either the command line or the web interface the user can tell Firefly to start the import. + +The ImporterInterface runs and creates an ImportEntry for each line, blob or whatever distinction it +wants. + +For each line, the ImporterInterface should run each field through the selected Converter in order +to convert the text values to their Firefly counterparts. Again, this is modelled to the CSV importer +and may need an update for MT940. + +In any case, this newly minted set of ImportEntries (it cannot be saved or stored atm, +this has to be done in one go) is then fed to the ImportValidator which will reject entries +(almost never) and corrects fields if necessary. + +- Adds a default description if there isn't one present. +- Adds the date (today) if no date is present. +- Determins the type of transaction (withdrawal, deposit, transfer). +- Determins the types of accounts involved (asset, expense, revenue). +- Determins the currency of the transaction. + +This set of corrected ImportEntries is then fed to the ImportStorage class which will generate +TransactionJournals, Transactions and other related objects. \ No newline at end of file