Refactor some code to handle command line imports.

app/Console/Commands/CreateImport.php

@@ -24,13 +24,14 @@ declare(strict_types=1);
 
 namespace FireflyIII\Console\Commands;
 
+use Exception;
 use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\Import\Prerequisites\PrerequisitesInterface;
 use FireflyIII\Import\Routine\RoutineInterface;
+use FireflyIII\Import\Storage\ImportArrayStorage;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
 use FireflyIII\Repositories\User\UserRepositoryInterface;
-use FireflyIII\Services\Internal\File\EncryptService;
 use Illuminate\Console\Command;
-use Illuminate\Support\MessageBag;
 use Log;
 use Preferences;
 
@@ -54,18 +55,17 @@ class CreateImport extends Command
      */
     protected $signature
         = 'firefly:create-import
-                            {file : The file to import.}
+                            {file? : The file to import.}
-                            {configuration : The configuration file to use for the import.}
+                            {configuration? : The configuration file to use for the import.}
                             {--type=csv : The file type of the import.}
-                            {--user= : The user ID that the import should import for.}
+                            {--provider=file : The file type of the import.}
+                            {--user=1 : The user ID that the import should import for.}
                             {--token= : The user\'s access token.}
                             {--start : Starts the job immediately.}';
 
     /**
      * Run the command.
      *
-     * @noinspection MultipleReturnStatementsInspection
-     *
      * @throws FireflyException
      */
     public function handle(): int
@@ -77,76 +77,154 @@ class CreateImport extends Command
         }
         /** @var UserRepositoryInterface $userRepository */
         $userRepository    = app(UserRepositoryInterface::class);
-        $file           = $this->argument('file');
+        $file              = (string)$this->argument('file');
-        $configuration  = $this->argument('configuration');
+        $configuration     = (string)$this->argument('configuration');
         $user              = $userRepository->findNull((int)$this->option('user'));
         $cwd               = getcwd();
-        $type           = strtolower($this->option('type'));
+        $type              = strtolower((string)$this->option('type'));
+        $provider          = strtolower((string)$this->option('provider'));
+        $configurationData = [];
 
         if (!$this->validArguments()) {
             $this->errorLine('Invalid arguments.');
 
             return 1;
         }
-
+        if (\strlen($configuration) > 0) {
             $configurationData = json_decode(file_get_contents($configuration), true);
             if (null === $configurationData) {
                 $this->errorLine(sprintf('Firefly III cannot read the contents of configuration file "%s" (working directory: "%s").', $configuration, $cwd));
 
                 return 1;
             }
+        }
+
 
         $this->infoLine(sprintf('Going to create a job to import file: %s', $file));
         $this->infoLine(sprintf('Using configuration file: %s', $configuration));
         $this->infoLine(sprintf('Import into user: #%d (%s)', $user->id, $user->email));
-        $this->infoLine(sprintf('Type of import: %s', $type));
+        $this->infoLine(sprintf('Type of import: %s', $provider));
 
         /** @var ImportJobRepositoryInterface $jobRepository */
         $jobRepository = app(ImportJobRepositoryInterface::class);
         $jobRepository->setUser($user);
-        $job = $jobRepository->create($type);
+        $importJob = $jobRepository->create($provider);
-        $this->infoLine(sprintf('Created job "%s"', $job->key));
+        $this->infoLine(sprintf('Created job "%s"', $importJob->key));
 
-        /** @var EncryptService $service */
+        // make sure that job has no prerequisites.
-        $service = app(EncryptService::class);
+        if ((bool)config(sprintf('import.has_prereq.%s', $provider))) {
-        $service->encrypt($file, $job->key);
+            // make prerequisites thing.
+            $class = (string)config(sprintf('import.prerequisites.%s', $provider));
+            if (!class_exists($class)) {
+                throw new FireflyException(sprintf('No class to handle configuration for "%s".', $provider)); // @codeCoverageIgnore
+            }
+            /** @var PrerequisitesInterface $object */
+            $object = app($class);
+            $object->setUser($user);
+            if (!$object->isComplete()) {
+                $this->errorLine(sprintf('Import provider "%s" has prerequisites that can only be filled in using the browser.', $provider));
 
-        $this->infoLine('Stored import data...');
+                return 1;
+            }
+        }
+
+        // store file as attachment.
+        if (\strlen($file) > 0) {
+            $messages = $jobRepository->storeCLIUpload($importJob, 'import_file', $file);
+            if ($messages->count() > 0) {
+                $this->errorLine($messages->first());
+
+                return 1;
+            }
+            $this->infoLine('File content saved.');
+        }
+
+        $this->infoLine('Job configuration saved.');
+        $jobRepository->setConfiguration($importJob, $configurationData);
+        $jobRepository->setStatus($importJob, 'ready_to_run');
 
-        $jobRepository->setConfiguration($job, $configurationData);
-        $jobRepository->updateStatus($job, 'configured');
-        $this->infoLine('Stored configuration...');
 
         if (true === $this->option('start')) {
-            $this->infoLine('The import will start in a moment. This process is not visible...');
+            $this->infoLine('The has started. The process is not visible. Please wait.');
             Log::debug('Go for import!');
 
-            // normally would refer to other firefly:start-import but that doesn't seem to work all to well...
+            // run it!
-
+            $key       = sprintf('import.routine.%s', $provider);
-            // start the actual routine:
-            $type      = 'csv' === $job->file_type ? 'file' : $job->file_type;
-            $key       = sprintf('import.routine.%s', $type);
             $className = config($key);
             if (null === $className || !class_exists($className)) {
-                throw new FireflyException(sprintf('Cannot find import routine class for job of type "%s".', $type)); // @codeCoverageIgnore
+                // @codeCoverageIgnoreStart
+                $this->errorLine(sprintf('No routine for provider "%s"', $provider));
+
+                return 1;
+                // @codeCoverageIgnoreEnd
             }
+
+            // keep repeating this call until job lands on "provider_finished"
+            $valid = ['provider_finished'];
+            $count = 0;
+            while (!\in_array($importJob->status, $valid, true) && $count < 6) {
+                Log::debug(sprintf('Now in loop #%d.', $count + 1));
                 /** @var RoutineInterface $routine */
                 $routine = app($className);
-            $routine->setJob($job);
+                $routine->setImportJob($importJob);
+                try {
                     $routine->run();
+                } catch (FireflyException|Exception $e) {
+                    $message = 'The import routine crashed: ' . $e->getMessage();
+                    Log::error($message);
+                    Log::error($e->getTraceAsString());
 
-            // give feedback.
+                    // set job errored out:
-            /** @var MessageBag $error */
+                    $jobRepository->setStatus($importJob, 'error');
-            foreach ($routine->getErrors() as $index => $error) {
+                    $this->errorLine($message);
-                $this->errorLine(sprintf('Error importing line #%d: %s', $index, $error));
+
+                    return 1;
                 }
-            $this->infoLine(
+                $count++;
-                sprintf(
+            }
-                    'The import has finished. %d transactions have been imported out of %d records.', $routine->getJournals()->count(), $routine->getLines()
+            if ($importJob->status === 'provider_finished') {
-                )
+                $this->infoLine('Import has finished. Please wait for storage of data.');
-            );
+                // set job to be storing data:
+                $jobRepository->setStatus($importJob, 'storing_data');
+
+                /** @var ImportArrayStorage $storage */
+                $storage = app(ImportArrayStorage::class);
+                $storage->setImportJob($importJob);
+
+                try {
+                    $storage->store();
+                } catch (FireflyException|Exception $e) {
+                    $message = 'The import routine crashed: ' . $e->getMessage();
+                    Log::error($message);
+                    Log::error($e->getTraceAsString());
+
+                    // set job errored out:
+                    $jobRepository->setStatus($importJob, 'error');
+                    $this->errorLine($message);
+
+                    return 1;
+                }
+                // set storage to be finished:
+                $jobRepository->setStatus($importJob, 'storage_finished');
             }
 
+            // give feedback:
+            $this->infoLine('Job has finished.');
+            if (null !== $importJob->tag) {
+                $this->infoLine(sprintf('%d transaction(s) have been imported.', $importJob->tag->transactionJournals->count()));
+                $this->infoLine(sprintf('You can find your transactions under tag "%s"', $importJob->tag->tag));
+            }
+
+            if (null === $importJob->tag) {
+                $this->errorLine('No transactions have been imported :(.');
+            }
+            if (\count($importJob->errors) > 0) {
+                $this->infoLine(sprintf('%d error(s) occurred:', \count($importJob->errors)));
+                foreach ($importJob->errors as $err) {
+                    $this->errorLine('- ' . $err);
+                }
+            }
+        }
         // clear cache for user:
         Preferences::setForUser($user, 'lastActivity', microtime());
 
@@ -187,20 +265,28 @@ class CreateImport extends Command
         $cwd           = getcwd();
         $validTypes    = config('import.options.file.import_formats');
         $type          = strtolower($this->option('type'));
+        $provider      = strtolower($this->option('provider'));
+        $enabled       = (bool)config(sprintf('import.enabled.%s', $provider));
 
-        if (!\in_array($type, $validTypes, true)) {
+        if (false === $enabled) {
+            $this->errorLine(sprintf('Provider "%s" is not enabled.', $provider));
+
+            return false;
+        }
+
+        if ($provider === 'file' && !\in_array($type, $validTypes, true)) {
             $this->errorLine(sprintf('Cannot import file of type "%s"', $type));
 
             return false;
         }
 
-        if (!file_exists($file)) {
+        if ($provider === 'file' && !file_exists($file)) {
             $this->errorLine(sprintf('Firefly III cannot find file "%s" (working directory: "%s").', $file, $cwd));
 
             return false;
         }
 
-        if (!file_exists($configuration)) {
+        if ($provider === 'file' && !file_exists($configuration)) {
             $this->errorLine(sprintf('Firefly III cannot find configuration file "%s" (working directory: "%s").', $configuration, $cwd));
 
             return false;

app/Factory/TransactionJournalFactory.php

@@ -94,7 +94,7 @@ class TransactionJournalFactory
 
         // store date meta fields (if present):
         $fields = ['sepa-cc', 'sepa-ct-op', 'sepa-ct-id', 'sepa-db', 'sepa-country', 'sepa-ep', 'sepa-ci', 'interest_date', 'book_date', 'process_date',
-                   'due_date', 'payment_date', 'invoice_date', 'internal_reference', 'bunq_payment_id', 'importHash', 'external_id'];
+                   'due_date', 'payment_date', 'invoice_date', 'internal_reference', 'bunq_payment_id', 'importHash','importHashV2', 'external_id'];
 
         foreach ($fields as $field) {
             $this->storeMeta($journal, $data, $field);

app/Http/Controllers/Import/JobStatusController.php

@@ -116,7 +116,7 @@ class JobStatusController extends Controller
     public function start(ImportJob $importJob): JsonResponse
     {
         // catch impossible status:
-        $allowed = ['ready_to_run', 'need_job_config','error','running'];
+        $allowed = ['ready_to_run', 'need_job_config'];
         // todo remove error and running.
 
         if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
@@ -174,7 +174,7 @@ class JobStatusController extends Controller
     public function store(ImportJob $importJob): JsonResponse
     {
         // catch impossible status:
-        $allowed = ['provider_finished', 'storing_data','error'];
+        $allowed = ['provider_finished', 'storing_data'];
         if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
             Log::error('Job is not ready.');
 

app/Import/Routine/FakeRoutine.php

@@ -55,14 +55,15 @@ class FakeRoutine implements RoutineInterface
     public function run(): void
     {
         Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->importJob->status));
-        if ($this->importJob->status !== 'running') {
+        if ($this->importJob->status !== 'ready_to_run') {
-            throw new FireflyException('This fake job should not be started.'); // @codeCoverageIgnore
+            throw new FireflyException(sprintf('Fake job should have status "ready_to_run", not "%s"', $this->importJob->status)); // @codeCoverageIgnore
         }
 
         switch ($this->importJob->stage) {
             default:
                 throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->importJob->stage)); // @codeCoverageIgnore
             case 'new':
+                $this->repository->setStatus($this->importJob, 'running');
                 /** @var StageNewHandler $handler */
                 $handler = app(StageNewHandler::class);
                 $handler->run();
@@ -72,13 +73,15 @@ class FakeRoutine implements RoutineInterface
 
                 return;
             case 'ahoy':
+                $this->repository->setStatus($this->importJob, 'running');
                 /** @var StageAhoyHandler $handler */
                 $handler = app(StageAhoyHandler::class);
                 $handler->run();
-                $this->repository->setStatus($this->importJob, 'need_job_config');
+                $this->repository->setStatus($this->importJob, 'ready_to_run');
                 $this->repository->setStage($this->importJob, 'final');
                 break;
             case 'final':
+                $this->repository->setStatus($this->importJob, 'running');
                 /** @var StageFinalHandler $handler */
                 $handler = app(StageFinalHandler::class);
                 $handler->setImportJob($this->importJob);

app/Import/Routine/FileRoutine.php

@@ -48,10 +48,8 @@ class FileRoutine implements RoutineInterface
     public function run(): void
     {
         Log::debug(sprintf('Now in run() for file routine with status: %s', $this->importJob->status));
-        if ($this->importJob->status !== 'running') {
+        if ($this->importJob->status === 'ready_to_run') {
-            throw new FireflyException('This file import job should not be started.'); // @codeCoverageIgnore
+            $this->repository->setStatus($this->importJob, 'running');
-        }
-        if ($this->importJob->stage === 'ready_to_run') {
             // get processor, depending on file type
             // is just CSV for now.
             $processor = $this->getProcessor();

app/Import/Storage/ImportArrayStorage.php

@@ -163,6 +163,22 @@ class ImportArrayStorage
 
     }
 
+    /**
+     * @param array $transaction
+     *
+     * @throws FireflyException
+     * @return string
+     */
+    private function getHash(array $transaction): string
+    {
+        $json = json_encode($transaction);
+        if ($json === false) {
+            throw new FireflyException('Could not encode import array. Please see the logs.', $transaction); // @codeCoverageIgnore
+        }
+
+        return hash('sha256', $json, false);
+    }
+
     /**
      * Gets the users rules.
      *
@@ -187,6 +203,7 @@ class ImportArrayStorage
     {
         /** @var JournalCollectorInterface $collector */
         $collector = app(JournalCollectorInterface::class);
+        $collector->setUser($this->importJob->user);
         $collector->setAllAssetAccounts()
                   ->setTypes([TransactionType::TRANSFER])
                   ->withOpposingAccount();
@@ -198,20 +215,13 @@ class ImportArrayStorage
     /**
      * Check if the hash exists for the array the user wants to import.
      *
-     * @param array $transaction
+     * @param string $hash
      *
      * @return int|null
      * @throws FireflyException
      */
-    private function hashExists(array $transaction): ?int
+    private function hashExists(string $hash): ?int
     {
-        $json = json_encode($transaction);
-        if ($json === false) {
-            throw new FireflyException('Could not encode import array. Please see the logs.', $transaction); // @codeCoverageIgnore
-        }
-        $hash = hash('sha256', $json, false);
-
-        // find it!
         $entry = $this->journalRepos->findByHash($hash);
         if (null === $entry) {
             return null;
@@ -328,12 +338,13 @@ class ImportArrayStorage
 
         foreach ($array as $index => $transaction) {
             Log::debug(sprintf('Now at item %d out of %d', $index + 1, $count));
-            $existingId = $this->hashExists($transaction);
+            $hash       = $this->getHash($transaction);
+            $existingId = $this->hashExists($hash);
             if (null !== $existingId) {
                 $this->logDuplicateObject($transaction, $existingId);
                 $this->repository->addErrorMessage(
                     $this->importJob, sprintf(
-                                        'Entry #%d ("%s") could not be imported. It already exists.',
+                                        'Row #%d ("%s") could not be imported. It already exists.',
                                         $index, $transaction['description']
                                     )
                 );
@@ -344,7 +355,7 @@ class ImportArrayStorage
                     $this->logDuplicateTransfer($transaction);
                     $this->repository->addErrorMessage(
                         $this->importJob, sprintf(
-                                            'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
+                                            'Row #%d ("%s") could not be imported. Such a transfer already exists.',
                                             $index,
                                             $transaction['description']
                                         )
@@ -352,6 +363,7 @@ class ImportArrayStorage
                     continue;
                 }
             }
+            $transaction['importHashV2'] = $hash;
             $toStore[]                   = $transaction;
         }
         $count = \count($toStore);

app/Repositories/ImportJob/ImportJobRepository.php

@@ -475,6 +475,59 @@ class ImportJobRepository implements ImportJobRepositoryInterface
         $this->user = $user;
     }
 
+    /**
+     * Handle upload for job.
+     *
+     * @param ImportJob    $job
+     * @param string       $name
+     * @param UploadedFile $file
+     *
+     * @return MessageBag
+     * @throws FireflyException
+     */
+    public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag
+    {
+        $messages = new MessageBag;
+
+        if (!file_exists($fileName)) {
+            $messages->add('notfound', sprintf('File not found: %s', $fileName));
+
+            return $messages;
+        }
+
+        $count = $job->attachments()->get()->filter(
+            function (Attachment $att) use ($name) {
+                return $att->filename === $name;
+            }
+        )->count();
+
+        if ($count > 0) {
+            // don't upload, but also don't complain about it.
+            Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name));
+
+            return new MessageBag;
+        }
+        $content    = file_get_contents($fileName);
+        $attachment = new Attachment; // create Attachment object.
+        $attachment->user()->associate($job->user);
+        $attachment->attachable()->associate($job);
+        $attachment->md5      = md5($content);
+        $attachment->filename = $name;
+        $attachment->mime     = 'plain/txt';
+        $attachment->size     = \strlen($content);
+        $attachment->uploaded = 0;
+        $attachment->save();
+        $encrypted = Crypt::encrypt($content);
+
+        // store it:
+        $this->uploadDisk->put($attachment->fileName(), $encrypted);
+        $attachment->uploaded = 1; // update attachment
+        $attachment->save();
+
+        // return it.
+        return new MessageBag;
+    }
+
     /**
      * Handle upload for job.
      *

app/Repositories/ImportJob/ImportJobRepositoryInterface.php

@@ -56,6 +56,17 @@ interface ImportJobRepositoryInterface
      */
     public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag;
 
+    /**
+     * Store file.
+     *
+     * @param ImportJob $job
+     * @param string    $name
+     * @param string    $fileName
+     *
+     * @return MessageBag
+     */
+    public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag;
+
     /**
      * @param ImportJob $job
      * @param array     $transactions

app/Repositories/Journal/JournalRepository.php

@@ -159,7 +159,7 @@ class JournalRepository implements JournalRepositoryInterface
     {
         return TransactionJournalMeta
             ::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
-            ->where('data', $hash)
+            ->where('data', json_encode($hash))
             ->where('name', 'importHashV2')
             ->first(['journal_meta.*']);
     }

app/Support/Import/Routine/File/MappingConverger.php

@@ -168,7 +168,7 @@ class MappingConverger
                 $newRole = 'opposing-id';
                 break;
         }
-        Log::debug(sprintf('Role was "%s", but because of mapping, role becomes "%s"', $role, $newRole));
+        Log::debug(sprintf('Role was "%s", but because of mapping (mapped to #%d), role becomes "%s"', $role, $mapped, $newRole));
 
         // also store the $mapped values in a "mappedValues" array.
         $this->mappedValues[$newRole][] = $mapped;