Import routine can handle new SEPA fields and many new date fields. See #1248

app/Console/Commands/CreateImport.php

@@ -80,7 +80,7 @@ class CreateImport extends Command
     public function handle()
     {
         if (!$this->verifyAccessToken()) {
-            $this->error('Invalid access token.');
+            $this->errorLine('Invalid access token.');
 
             return;
         }
@@ -98,39 +98,34 @@ class CreateImport extends Command
 
         $configurationData = json_decode(file_get_contents($configuration), true);
         if (null === $configurationData) {
-            $this->error(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd));
+            $this->errorLine(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd));
 
             return;
         }
 
-        $this->line(sprintf('Going to create a job to import file: %s', $file));
+        $this->infoLine(sprintf('Going to create a job to import file: %s', $file));
-        $this->line(sprintf('Using configuration file: %s', $configuration));
+        $this->infoLine(sprintf('Using configuration file: %s', $configuration));
-        $this->line(sprintf('Import into user: #%d (%s)', $user->id, $user->email));
+        $this->infoLine(sprintf('Import into user: #%d (%s)', $user->id, $user->email));
-        $this->line(sprintf('Type of import: %s', $type));
+        $this->infoLine(sprintf('Type of import: %s', $type));
 
         /** @var ImportJobRepositoryInterface $jobRepository */
         $jobRepository = app(ImportJobRepositoryInterface::class);
         $jobRepository->setUser($user);
         $job = $jobRepository->create($type);
-        $this->line(sprintf('Created job "%s"', $job->key));
+        $this->infoLine(sprintf('Created job "%s"', $job->key));
 
         Artisan::call('firefly:encrypt-file', ['file' => $file, 'key' => $job->key]);
-        $this->line('Stored import data...');
+        $this->infoLine('Stored import data...');
 
         $jobRepository->setConfiguration($job, $configurationData);
         $jobRepository->updateStatus($job, 'configured');
-        $this->line('Stored configuration...');
+        $this->infoLine('Stored configuration...');
 
         if (true === $this->option('start')) {
-            $this->line('The import will start in a moment. This process is not visible...');
+            $this->infoLine('The import will start in a moment. This process is not visible...');
             Log::debug('Go for import!');
 
             // normally would refer to other firefly:start-import but that doesn't seem to work all to well...
-            $monolog   = Log::getMonolog();
-            $handler   = new CommandHandler($this);
-            $formatter = new LineFormatter(null, null, false, true);
-            $handler->setFormatter($formatter);
-            $monolog->pushHandler($handler);
 
             // start the actual routine:
             $type      = 'csv' === $job->file_type ? 'file' : $job->file_type;
@@ -147,9 +142,9 @@ class CreateImport extends Command
             // give feedback.
             /** @var MessageBag $error */
             foreach ($routine->getErrors() as $index => $error) {
-                $this->error(sprintf('Error importing line #%d: %s', $index, $error));
+                $this->errorLine(sprintf('Error importing line #%d: %s', $index, $error));
             }
-            $this->line(
+            $this->infoLine(
                 sprintf(
                     'The import has finished. %d transactions have been imported out of %d records.', $routine->getJournals()->count(), $routine->getLines()
                 )
@@ -179,29 +174,50 @@ class CreateImport extends Command
         $validTypes     = config('import.options.file.import_formats');
         $type           = strtolower($this->option('type'));
         if (null === $user) {
-            $this->error(sprintf('There is no user with ID %d.', $this->option('user')));
+            $this->errorLine(sprintf('There is no user with ID %d.', $this->option('user')));
 
             return false;
         }
 
         if (!in_array($type, $validTypes)) {
-            $this->error(sprintf('Cannot import file of type "%s"', $type));
+            $this->errorLine(sprintf('Cannot import file of type "%s"', $type));
 
             return false;
         }
 
         if (!file_exists($file)) {
-            $this->error(sprintf('Firefly III cannot find file "%s" (working directory: "%s").', $file, $cwd));
+            $this->errorLine(sprintf('Firefly III cannot find file "%s" (working directory: "%s").', $file, $cwd));
 
             return false;
         }
 
         if (!file_exists($configuration)) {
-            $this->error(sprintf('Firefly III cannot find configuration file "%s" (working directory: "%s").', $configuration, $cwd));
+            $this->errorLine(sprintf('Firefly III cannot find configuration file "%s" (working directory: "%s").', $configuration, $cwd));
 
             return false;
         }
 
         return true;
     }
+
+    /**
+     * @param string     $message
+     * @param array|null $data
+     */
+    private function errorLine(string $message, array $data = null): void
+    {
+        Log::error($message, $data?? []);
+        $this->error($message);
+
+    }
+
+    /**
+     * @param string $message
+     * @param array  $data
+     */
+    private function infoLine(string $message, array $data = null): void
+    {
+        Log::info($message, $data?? []);
+        $this->line($message);
+    }
 }

app/Console/Commands/Import.php

@@ -23,7 +23,6 @@ declare(strict_types=1);
 namespace FireflyIII\Console\Commands;
 
 use FireflyIII\Exceptions\FireflyException;
-use FireflyIII\Import\Logging\CommandHandler;
 use FireflyIII\Import\Routine\RoutineInterface;
 use FireflyIII\Models\ImportJob;
 use Illuminate\Console\Command;
@@ -68,21 +67,17 @@ class Import extends Command
         $jobKey = $this->argument('key');
         $job    = ImportJob::where('key', $jobKey)->first();
         if (null === $job) {
-            $this->error(sprintf('No job found with key "%s"', $jobKey));
+            $this->errorLine(sprintf('No job found with key "%s"', $jobKey));
 
             return;
         }
         if (!$this->isValid($job)) {
-            Log::error('Job is not valid for some reason. Exit.');
+            $this->errorLine('Job is not valid for some reason. Exit.');
 
             return;
         }
 
-        $this->line(sprintf('Going to import job with key "%s" of type "%s"', $job->key, $job->file_type));
+        $this->infoLine(sprintf('Going to import job with key "%s" of type "%s"', $job->key, $job->file_type));
-
-        $monolog = Log::getMonolog();
-        $handler = new CommandHandler($this);
-        $monolog->pushHandler($handler);
 
         // actually start job:
         $type      = 'csv' === $job->file_type ? 'file' : $job->file_type;
@@ -99,16 +94,37 @@ class Import extends Command
 
         /** @var MessageBag $error */
         foreach ($routine->getErrors() as $index => $error) {
-            $this->error(sprintf('Error importing line #%d: %s', $index, $error));
+            $this->errorLine(sprintf('Error importing line #%d: %s', $index, $error));
         }
 
-        $this->line(
+        $this->infoLine(
             sprintf('The import has finished. %d transactions have been imported out of %d records.', $routine->getJournals()->count(), $routine->getLines())
         );
 
         return;
     }
 
+    /**
+     * @param string     $message
+     * @param array|null $data
+     */
+    private function errorLine(string $message, array $data = null): void
+    {
+        Log::error($message, $data ?? []);
+        $this->error($message);
+
+    }
+
+    /**
+     * @param string $message
+     * @param array  $data
+     */
+    private function infoLine(string $message, array $data = null): void
+    {
+        Log::info($message, $data ?? []);
+        $this->line($message);
+    }
+
     /**
      * Check if job is valid to be imported.
      *
@@ -119,15 +135,14 @@ class Import extends Command
     private function isValid(ImportJob $job): bool
     {
         if (null === $job) {
-            Log::error('This job does not seem to exist.');
+            $this->errorLine('This job does not seem to exist.');
-            $this->error('This job does not seem to exist.');
 
             return false;
         }
 
         if ('configured' !== $job->status) {
             Log::error(sprintf('This job is not ready to be imported (status is %s).', $job->status));
-            $this->error('This job is not ready to be imported.');
+            $this->errorLine('This job is not ready to be imported.');
 
             return false;
         }

app/Import/Object/ImportAccount.php

@@ -312,9 +312,9 @@ class ImportAccount
         Log::debug('Finding a mapped account based on', $array);
 
         $search  = intval($array['mapped'] ?? 0);
-        $account = $this->repository->find($search);
+        $account = $this->repository->findNull($search);
 
-        if (null === $account->id) {
+        if (null === $account) {
             Log::error(sprintf('There is no account with id #%d. Invalid mapping will be ignored!', $search));
 
             return null;
@@ -382,7 +382,7 @@ class ImportAccount
 
         // 4: if search for an asset account, fall back to given "default account" (mandatory)
         if (AccountType::ASSET === $this->expectedType) {
-            $this->account = $this->repository->find($this->defaultAccountId);
+            $this->account = $this->repository->findNull($this->defaultAccountId);
             Log::debug(sprintf('Fall back to default account #%d "%s"', $this->account->id, $this->account->name));
 
             return true;

app/Import/Object/ImportJournal.php

@@ -54,6 +54,8 @@ class ImportJournal
     public $hash;
     /** @var array */
     public $metaDates = [];
+    /** @var array */
+    public $metaFields = [];
     /** @var string */
     public $notes = '';
     /** @var ImportAccount */
@@ -192,6 +194,18 @@ class ImportJournal
             case 'account-id':
                 $this->asset->setAccountId($array);
                 break;
+            case 'sepa-cc':
+            case 'sepa-ct-op':
+            case 'sepa-ct-id':
+            case 'sepa-db':
+            case 'sepa-country':
+            case 'sepa-ep':
+            case 'sepa-ci':
+                $value = trim(strval($array['value']));
+                if (strlen($value) > 0) {
+                    $this->metaFields[$array['role']] = $value;
+                }
+                break;
             case 'amount':
                 $this->amount = $array;
                 break;
@@ -252,12 +266,6 @@ class ImportJournal
             case 'description':
                 $this->description .= $array['value'];
                 break;
-            case 'sepa-ct-op':
-            case 'sepa-ct-id':
-            case 'sepa-db':
-                $this->notes .= ' ' . $array['value'];
-                $this->notes = trim($this->notes);
-                break;
             case 'note':
                 $this->notes .= ' ' . $array['value'];
                 $this->notes = trim($this->notes);
@@ -265,6 +273,9 @@ class ImportJournal
             case 'external-id':
                 $this->externalId = $array['value'];
                 break;
+            case 'internal-reference':
+                $this->metaFields['internal_reference'] = $array['value'];
+                break;
             case '_ignore':
                 break;
             case 'ing-debit-credit':
@@ -299,6 +310,15 @@ class ImportJournal
             case 'date-process':
                 $this->metaDates['process_date'] = $array['value'];
                 break;
+            case 'date-due':
+                $this->metaDates['due_date'] = $array['value'];
+                break;
+            case 'date-payment':
+                $this->metaDates['payment_date'] = $array['value'];
+                break;
+            case 'date-invoice':
+                $this->metaDates['invoice_date'] = $array['value'];
+                break;
         }
     }
 

app/Import/Storage/ImportStorage.php

@@ -31,6 +31,7 @@ use FireflyIII\Models\Note;
 use FireflyIII\Models\TransactionType;
 use FireflyIII\Repositories\Bill\BillRepositoryInterface;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
+use FireflyIII\Repositories\Journal\JournalRepositoryInterface;
 use Illuminate\Support\Collection;
 use Log;
 use Preferences;
@@ -61,9 +62,11 @@ class ImportStorage
     /** @var Collection */
     protected $bills;
     /** @var int */
-    protected $defaultCurrencyId = 1; // yes, hard coded
+    protected $defaultCurrencyId = 1;
     /** @var ImportJob */
-    protected $job;
+    protected $job; // yes, hard coded
+    /** @var JournalRepositoryInterface */
+    protected $journalRepository;
     /** @var ImportJobRepositoryInterface */
     protected $repository;
     /** @var Collection */
@@ -104,8 +107,10 @@ class ImportStorage
      */
     public function setJob(ImportJob $job)
     {
-        $this->repository = app(ImportJobRepositoryInterface::class);
+        $this->repository        = app(ImportJobRepositoryInterface::class);
+        $this->journalRepository = app(JournalRepositoryInterface::class);
         $this->repository->setUser($job->user);
+        $this->journalRepository->setUser($job->user);
 
         $config                  = $this->repository->getConfiguration($job);
         $currency                = app('amount')->getDefaultCurrencyByUser($job->user);
@@ -232,9 +237,13 @@ class ImportStorage
         $importJournal->bill->setAmount($amount);
 
         $this->storeBill($journal, $importJournal->bill->getBill());
-        $this->storeMeta($journal, $importJournal->metaDates);
+        $this->storeMetaDates($journal, $importJournal->metaDates);
         $this->storeTags($importJournal->tags, $journal);
 
+        foreach ($importJournal->metaFields as $field => $value) {
+            $this->journalRepository->setMetaString($journal, $field, $value);
+        }
+
         // set notes for journal:
         $dbNote = new Note();
         $dbNote->noteable()->associate($journal);

app/Import/Storage/ImportSupport.php

@@ -39,6 +39,7 @@ use FireflyIII\Models\TransactionJournal;
 use FireflyIII\Models\TransactionJournalMeta;
 use FireflyIII\Models\TransactionType;
 use FireflyIII\Repositories\Bill\BillRepositoryInterface;
+use FireflyIII\Repositories\Journal\JournalRepositoryInterface;
 use FireflyIII\Repositories\Tag\TagRepositoryInterface;
 use FireflyIII\TransactionRules\Processor;
 use Illuminate\Database\Query\JoinClause;
@@ -58,6 +59,8 @@ trait ImportSupport
     protected $defaultCurrencyId = 1;
     /** @var ImportJob */
     protected $job;
+    /** @var JournalRepositoryInterface */
+    protected $journalRepository;
     /** @var Collection */
     protected $rules;
 
@@ -441,7 +444,7 @@ trait ImportSupport
             throw new FireflyException($errorText);
         }
         // save meta data:
-        $journal->setMeta('importHash', $parameters['hash']);
+        $this->journalRepository->setMetaString($journal, 'importHash', $parameters['hash']);
         Log::debug(sprintf('Created journal with ID #%d', $journal->id));
 
         // create transactions:
@@ -472,13 +475,13 @@ trait ImportSupport
      * @param TransactionJournal $journal
      * @param array              $dates
      */
-    private function storeMeta(TransactionJournal $journal, array $dates)
+    private function storeMetaDates(TransactionJournal $journal, array $dates)
     {
         // all other date fields as meta thing:
         foreach ($dates as $name => $value) {
             try {
                 $date = new Carbon($value);
-                $journal->setMeta($name, $date);
+                $this->journalRepository->setMetaDate($journal, $name, $date);
             } catch (Exception $e) {
                 // don't care, ignore:
                 Log::warning(sprintf('Could not parse "%s" into a valid Date object for field %s', $value, $name));