From 7f4feb0cfc5e4e02c3828453c5cd5ee3f48fdd14 Mon Sep 17 00:00:00 2001 From: James Cole Date: Mon, 7 May 2018 20:35:14 +0200 Subject: [PATCH] More code for import routine. --- app/Repositories/Bill/BillRepository.php | 16 +- .../Bill/BillRepositoryInterface.php | 9 + app/Repositories/Budget/BudgetRepository.php | 12 + .../Budget/BudgetRepositoryInterface.php | 10 + .../Category/CategoryRepository.php | 12 + .../Category/CategoryRepositoryInterface.php | 10 + .../Import/Placeholder/ImportTransaction.php | 206 ++++++++++++++---- .../Import/Routine/File/CSVProcessor.php | 155 ++++++++++++- 8 files changed, 380 insertions(+), 50 deletions(-) diff --git a/app/Repositories/Bill/BillRepository.php b/app/Repositories/Bill/BillRepository.php index 670f984f53..e7aabdee2a 100644 --- a/app/Repositories/Bill/BillRepository.php +++ b/app/Repositories/Bill/BillRepository.php @@ -243,6 +243,18 @@ class BillRepository implements BillRepositoryInterface return $sum; } + /** + * Get all bills with these ID's. + * + * @param array $billIds + * + * @return Collection + */ + public function getByIds(array $billIds): Collection + { + return $this->user->bills()->whereIn('id', $billIds)->get(); + } + /** * Get text or return empty string. * @@ -393,11 +405,11 @@ class BillRepository implements BillRepositoryInterface $rules = $this->user->rules() ->leftJoin('rule_actions', 'rule_actions.rule_id', '=', 'rules.id') ->where('rule_actions.action_type', 'link_to_bill') - ->get(['rules.id', 'rules.title', 'rule_actions.action_value','rules.active']); + ->get(['rules.id', 'rules.title', 'rule_actions.action_value', 'rules.active']); $array = []; foreach ($rules as $rule) { $array[$rule->action_value] = $array[$rule->action_value] ?? []; - $array[$rule->action_value][] = ['id' => $rule->id, 'title' => $rule->title,'active' => $rule->active]; + $array[$rule->action_value][] = ['id' => $rule->id, 'title' => $rule->title, 'active' => $rule->active]; } $return = []; foreach ($collection as $bill) { diff --git a/app/Repositories/Bill/BillRepositoryInterface.php b/app/Repositories/Bill/BillRepositoryInterface.php index 37f7e6abd7..3c9e1fa8f9 100644 --- a/app/Repositories/Bill/BillRepositoryInterface.php +++ b/app/Repositories/Bill/BillRepositoryInterface.php @@ -98,6 +98,15 @@ interface BillRepositoryInterface */ public function getBillsUnpaidInRange(Carbon $start, Carbon $end): string; + /** + * Get all bills with these ID's. + * + * @param array $billIds + * + * @return Collection + */ + public function getByIds(array $billIds): Collection; + /** * Get text or return empty string. * diff --git a/app/Repositories/Budget/BudgetRepository.php b/app/Repositories/Budget/BudgetRepository.php index 903c2b0f27..c0ee968441 100644 --- a/app/Repositories/Budget/BudgetRepository.php +++ b/app/Repositories/Budget/BudgetRepository.php @@ -458,6 +458,18 @@ class BudgetRepository implements BudgetRepositoryInterface return $set; } + /** + * Get all budgets with these ID's. + * + * @param array $budgetIds + * + * @return Collection + */ + public function getByIds(array $budgetIds): Collection + { + return $this->user->budgets()->whereIn('id', $budgetIds)->get(); + } + /** * @return Collection */ diff --git a/app/Repositories/Budget/BudgetRepositoryInterface.php b/app/Repositories/Budget/BudgetRepositoryInterface.php index 36879ca327..a929e92484 100644 --- a/app/Repositories/Budget/BudgetRepositoryInterface.php +++ b/app/Repositories/Budget/BudgetRepositoryInterface.php @@ -34,6 +34,16 @@ use Illuminate\Support\Collection; */ interface BudgetRepositoryInterface { + + /** + * Get all budgets with these ID's. + * + * @param array $budgetIds + * + * @return Collection + */ + public function getByIds(array $budgetIds): Collection; + /** * A method that returns the amount of money budgeted per day for this budget, * on average. diff --git a/app/Repositories/Category/CategoryRepository.php b/app/Repositories/Category/CategoryRepository.php index 82964375f8..e9e5a97909 100644 --- a/app/Repositories/Category/CategoryRepository.php +++ b/app/Repositories/Category/CategoryRepository.php @@ -153,6 +153,18 @@ class CategoryRepository implements CategoryRepositoryInterface return $firstJournalDate; } + /** + * Get all categories with ID's. + * + * @param array $categoryIds + * + * @return Collection + */ + public function getByIds(array $categoryIds): Collection + { + return $this->user->categories()->whereIn('id', $categoryIds)->get(); + } + /** * Returns a list of all the categories belonging to a user. * diff --git a/app/Repositories/Category/CategoryRepositoryInterface.php b/app/Repositories/Category/CategoryRepositoryInterface.php index e224ef84e7..8c16732bcb 100644 --- a/app/Repositories/Category/CategoryRepositoryInterface.php +++ b/app/Repositories/Category/CategoryRepositoryInterface.php @@ -32,6 +32,7 @@ use Illuminate\Support\Collection; */ interface CategoryRepositoryInterface { + /** * @param Category $category * @@ -84,6 +85,15 @@ interface CategoryRepositoryInterface */ public function firstUseDate(Category $category): ?Carbon; + /** + * Get all categories with ID's. + * + * @param array $categoryIds + * + * @return Collection + */ + public function getByIds(array $categoryIds): Collection; + /** * Returns a list of all the categories belonging to a user. * diff --git a/app/Support/Import/Placeholder/ImportTransaction.php b/app/Support/Import/Placeholder/ImportTransaction.php index 57ba4522b6..e928b7be2b 100644 --- a/app/Support/Import/Placeholder/ImportTransaction.php +++ b/app/Support/Import/Placeholder/ImportTransaction.php @@ -39,6 +39,8 @@ class ImportTransaction /** @var string */ private $accountName; /** @var string */ + private $accountNumber; + /** @var string */ private $amount; /** @var string */ private $amountCredit; @@ -46,13 +48,25 @@ class ImportTransaction private $amountDebit; /** @var int */ private $billId; + /** @var string */ + private $billName; /** @var int */ private $budgetId; + /** @var string */ + private $budgetName; /** @var int */ private $categoryId; + /** @var string */ + private $categoryName; + /** @var string */ + private $currencyCode; /** @var int */ private $currencyId; /** @var string */ + private $currencyName; + /** @var string */ + private $currencySymbol; + /** @var string */ private $date; /** @var string */ private $description; @@ -60,6 +74,8 @@ class ImportTransaction private $externalId; /** @var string */ private $foreignAmount; + /** @var string */ + private $foreignCurrencyCode; /** @var int */ private $foreignCurrencyId; /** @var array */ @@ -76,9 +92,29 @@ class ImportTransaction private $opposingId; /** @var string */ private $opposingName; + /** @var string */ + private $opposingNumber; /** @var array */ private $tags; + /** + * @return array + */ + public function getMeta(): array + { + return $this->meta; + } + + /** + * @return string + */ + public function getNote(): string + { + return $this->note; + } + + + /** * ImportTransaction constructor. */ @@ -89,8 +125,44 @@ class ImportTransaction $this->meta = []; $this->description = ''; $this->note = ''; + + // mappable items, set to 0: + $this->accountId = 0; + $this->budgetId = 0; + $this->billId = 0; + $this->currencyId = 0; + $this->categoryId = 0; + $this->foreignCurrencyId = 0; + $this->opposingId = 0; + } + /** + * @return string + */ + public function getDescription(): string + { + return $this->description; + } + + /** + * @return int + */ + public function getBillId(): int + { + return $this->billId; + } + + /** + * @return null|string + */ + public function getBillName(): ?string + { + return $this->billName; + } + + + /** * @param ColumnValue $columnValue * @@ -107,9 +179,63 @@ class ImportTransaction // could be the result of a mapping? $this->accountId = $this->getMappedValue($columnValue); break; + case 'account-iban': + $this->accountIban = $columnValue->getValue(); + break; case 'account-name': $this->accountName = $columnValue->getValue(); break; + case 'account-bic': + $this->accountBic = $columnValue->getValue(); + break; + case 'account-number': + $this->accountNumber = $columnValue->getValue(); + break; + case'amount_debit': + $this->amountDebit = $columnValue->getValue(); + break; + case'amount_credit': + $this->amountCredit = $columnValue->getValue(); + break; + case 'amount': + $this->amount = $columnValue->getValue(); + break; + case 'amount_foreign': + $this->foreignAmount = $columnValue->getValue(); + break; + case 'bill-id': + $this->billId = $this->getMappedValue($columnValue); + break; + case 'bill-name': + $this->billName = $columnValue->getValue(); + break; + case 'budget-id': + $this->budgetId = $this->getMappedValue($columnValue); + break; + case 'budget-name': + $this->budgetName = $columnValue->getValue(); + break; + case 'category-id': + $this->categoryId = $this->getMappedValue($columnValue); + break; + case 'category-name': + $this->categoryName = $columnValue->getValue(); + break; + case 'currency-id': + $this->currencyId = $this->getMappedValue($columnValue); + break; + case 'currency-name': + $this->currencyName = $columnValue->getValue(); + break; + case 'currency-code': + $this->currencyCode = $columnValue->getValue(); + break; + case 'currency-symbol': + $this->currencySymbol = $columnValue->getValue(); + break; + case 'external-id': + $this->externalId = $columnValue->getValue(); + break; case 'sepa-ct-id'; case 'sepa-ct-op'; case 'sepa-db'; @@ -126,33 +252,14 @@ class ImportTransaction case 'date-due': $this->meta[$columnValue->getRole()] = $columnValue->getValue(); break; - case'amount_debit': - $this->amountDebit = $columnValue->getValue(); - break; - case'amount_credit': - $this->amountCredit = $columnValue->getValue(); - break; - case 'amount': - $this->amount = $columnValue->getValue(); - break; - case 'amount_foreign': - $this->foreignAmount = $columnValue->getValue(); - break; + case 'foreign-currency-id': $this->foreignCurrencyId = $this->getMappedValue($columnValue); break; - case 'bill-id': - $this->billId = $this->getMappedValue($columnValue); - break; - case 'budget-id': - $this->budgetId = $this->getMappedValue($columnValue); - break; - case 'category-id': - $this->categoryId = $this->getMappedValue($columnValue); - break; - case 'currency-id': - $this->currencyId = $this->getMappedValue($columnValue); + case 'foreign-currency-code': + $this->foreignCurrencyCode = $columnValue->getValue(); break; + case 'date-transaction': $this->date = $columnValue->getValue(); break; @@ -162,22 +269,28 @@ class ImportTransaction case 'note': $this->note .= $columnValue->getValue(); break; - case 'external-id': - $this->externalId = $columnValue->getValue(); - break; - case 'rabo-debit-credit': - case 'ing-debit-credit': - $this->modifiers[$columnValue->getRole()] = $columnValue->getValue(); - break; + case 'opposing-id': $this->opposingId = $this->getMappedValue($columnValue); break; + case 'opposing-iban': + $this->opposingIban = $columnValue->getValue(); + break; case 'opposing-name': $this->opposingName = $columnValue->getValue(); break; case 'opposing-bic': $this->opposingBic = $columnValue->getValue(); break; + case 'opposing-number': + $this->opposingNumber = $columnValue->getValue(); + break; + + case 'rabo-debit-credit': + case 'ing-debit-credit': + $this->modifiers[$columnValue->getRole()] = $columnValue->getValue(); + break; + case 'tags-comma': // todo split using pre-processor. $this->tags = $columnValue->getValue(); @@ -186,25 +299,28 @@ class ImportTransaction // todo split using pre-processor. $this->tags = $columnValue->getValue(); break; - case 'account-iban': - $this->accountIban = $columnValue->getValue(); - break; - case 'opposing-iban': - $this->opposingIban = $columnValue->getValue(); - break; case '_ignore': - case 'bill-name': - case 'currency-name': - case 'currency-code': - case 'foreign-currency-code': - case 'currency-symbol': - case 'budget-name': - case 'category-name': - case 'account-number': - case 'opposing-number': + break; + } } + /** + * @return string + */ + public function getDate(): string + { + return $this->date; + } + + /** + * @return array + */ + public function getTags(): array + { + return $this->tags; + } + /** * Returns the mapped value if it exists in the ColumnValue object. * diff --git a/app/Support/Import/Routine/File/CSVProcessor.php b/app/Support/Import/Routine/File/CSVProcessor.php index e9a0d873d9..0698da05d6 100644 --- a/app/Support/Import/Routine/File/CSVProcessor.php +++ b/app/Support/Import/Routine/File/CSVProcessor.php @@ -23,10 +23,16 @@ declare(strict_types=1); namespace FireflyIII\Support\Import\Routine\File; +use Carbon\Carbon; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Helpers\Attachments\AttachmentHelperInterface; use FireflyIII\Models\Attachment; use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\Account\AccountRepositoryInterface; +use FireflyIII\Repositories\Bill\BillRepositoryInterface; +use FireflyIII\Repositories\Budget\BudgetRepositoryInterface; +use FireflyIII\Repositories\Category\CategoryRepositoryInterface; +use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; use FireflyIII\Support\Import\Placeholder\ColumnValue; use FireflyIII\Support\Import\Placeholder\ImportTransaction; @@ -78,6 +84,12 @@ class CSVProcessor implements FileProcessorInterface // make import objects, according to their role: $importables = $this->processLines($lines); + // now validate all mapped values: + $this->validateMappedValues(); + + + $array = $this->parseImportables($importables); + echo '
';
         print_r($importables);
         print_r($lines);
@@ -215,11 +227,87 @@ class CSVProcessor implements FileProcessorInterface
         Log::debug(sprintf('Role was "%s", but because of mapping, role becomes "%s"', $role, $newRole));
 
         // also store the $mapped values in a "mappedValues" array.
-        $this->mappedValues[$newRole] = $mapped;
+        $this->mappedValues[$newRole][] = $mapped;
 
         return $newRole;
     }
 
+    /**
+     * Each entry is an ImportTransaction that must be converted to an array compatible with the
+     * journal factory. To do so some stuff must still be resolved. See below.
+     *
+     * @param array $importables
+     *
+     * @return array
+     */
+    private function parseImportables(array $importables): array
+    {
+        $array = [];
+        /** @var ImportTransaction $importable */
+        foreach ($importables as $importable) {
+
+            // todo: verify bill mapping
+            // todo: verify currency mapping.
+
+
+            $entry = [
+                'type'               => 'unknown', // todo
+                'date'               => Carbon::createFromFormat($this->config['date-format'] ?? 'Ymd', $importable->getDate()),
+                'tags'               => $importable->getTags(), // todo make sure its filled.
+                'user'               => $this->importJob->user_id,
+                'notes'              => $importable->getNote(),
+
+                // all custom fields:
+                'internal_reference' => $importable->getMeta()['internal-reference'] ?? null,
+                'sepa-cc'            => $importable->getMeta()['sepa-cc'] ?? null,
+                'sepa-ct-op'         => $importable->getMeta()['sepa-ct-op'] ?? null,
+                'sepa-ct-id'         => $importable->getMeta()['sepa-ct-id'] ?? null,
+                'sepa-db'            => $importable->getMeta()['sepa-db'] ?? null,
+                'sepa-country'       => $importable->getMeta()['sepa-countru'] ?? null,
+                'sepa-ep'            => $importable->getMeta()['sepa-ep'] ?? null,
+                'sepa-ci'            => $importable->getMeta()['sepa-ci'] ?? null,
+                'interest_date'      => $importable->getMeta()['date-interest'] ?? null,
+                'book_date'          => $importable->getMeta()['date-book'] ?? null,
+                'process_date'       => $importable->getMeta()['date-process'] ?? null,
+                'due_date'           => $importable->getMeta()['date-due'] ?? null,
+                'payment_date'       => $importable->getMeta()['date-payment'] ?? null,
+                'invoice_date'       => $importable->getMeta()['date-invoice'] ?? null,
+                // todo external ID
+
+                // journal data:
+                'description'        => $importable->getDescription(),
+                'piggy_bank_id'      => null,
+                'piggy_bank_name'    => null,
+                'bill_id'            => $importable->getBillId() === 0 ? null : $importable->getBillId(), //
+                'bill_name'          => $importable->getBillId() !== 0 ? null : $importable->getBillName(),
+
+                // transaction data:
+                'transactions'       => [
+                    [
+                        'currency_id'           => null, // todo find ma
+                        'currency_code'         => 'EUR',
+                        'description'           => null,
+                        'amount'                => random_int(500, 5000) / 100,
+                        'budget_id'             => null,
+                        'budget_name'           => null,
+                        'category_id'           => null,
+                        'category_name'         => null,
+                        'source_id'             => null,
+                        'source_name'           => 'Checking Account',
+                        'destination_id'        => null,
+                        'destination_name'      => 'Random expense account #' . random_int(1, 10000),
+                        'foreign_currency_id'   => null,
+                        'foreign_currency_code' => null,
+                        'foreign_amount'        => null,
+                        'reconciled'            => false,
+                        'identifier'            => 0,
+                    ],
+                ],
+            ];
+        }
+
+        return $array;
+    }
 
     /**
      * Process all lines in the CSV file. Each line is processed separately.
@@ -272,12 +360,73 @@ class CSVProcessor implements FileProcessorInterface
                 $columnValue->setOriginalRole($originalRole);
                 $transaction->addColumnValue($columnValue);
 
-                // create object that parses this column value.
-
                 Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $role, $value));
             }
         }
 
         return $transaction;
     }
+
+    /**
+     * For each value that has been mapped, this method will check if the mapped value(s) are actually existing
+     *
+     * User may indicate that he wants his categories mapped to category #3, #4, #5 but if #5 is owned by another
+     * user it will be removed.
+     *
+     * @throws FireflyException
+     */
+    private function validateMappedValues()
+    {
+        foreach ($this->mappedValues as $role => $values) {
+            $values = array_unique($values);
+            if (count($values) > 0) {
+                switch ($role) {
+                    default:
+                        throw new FireflyException(sprintf('Cannot validate mapped values for role "%s"', $role));
+                    case 'opposing-id':
+                    case 'account-id':
+                        /** @var AccountRepositoryInterface $repository */
+                        $repository = app(AccountRepositoryInterface::class);
+                        $repository->setUser($this->importJob->user);
+                        $set                       = $repository->getAccountsById($values);
+                        $valid                     = $set->pluck('id')->toArray();
+                        $this->mappedValues[$role] = $valid;
+                        break;
+                    case 'currency-id':
+                    case 'foreign-currency-id':
+                        /** @var CurrencyRepositoryInterface $repository */
+                        $repository = app(CurrencyRepositoryInterface::class);
+                        $repository->setUser($this->importJob->user);
+                        $set                       = $repository->getByIds($values);
+                        $valid                     = $set->pluck('id')->toArray();
+                        $this->mappedValues[$role] = $valid;
+                        break;
+                    case 'bill-id':
+                        /** @var BillRepositoryInterface $repository */
+                        $repository = app(BillRepositoryInterface::class);
+                        $repository->setUser($this->importJob->user);
+                        $set                       = $repository->getByIds($values);
+                        $valid                     = $set->pluck('id')->toArray();
+                        $this->mappedValues[$role] = $valid;
+                        break;
+                    case 'budget-id':
+                        /** @var BudgetRepositoryInterface $repository */
+                        $repository = app(BudgetRepositoryInterface::class);
+                        $repository->setUser($this->importJob->user);
+                        $set                       = $repository->getByIds($values);
+                        $valid                     = $set->pluck('id')->toArray();
+                        $this->mappedValues[$role] = $valid;
+                        break;
+                    case 'category-id':
+                        /** @var CategoryRepositoryInterface $repository */
+                        $repository = app(CategoryRepositoryInterface::class);
+                        $repository->setUser($this->importJob->user);
+                        $set                       = $repository->getByIds($values);
+                        $valid                     = $set->pluck('id')->toArray();
+                        $this->mappedValues[$role] = $valid;
+                        break;
+                }
+            }
+        }
+    }
 }
\ No newline at end of file