New code for updated import routine.

app/Http/Controllers/Import/IndexController.php

@@ -58,7 +58,7 @@ class IndexController extends Controller
     }
 
     /**
-     * Creates a new import job for $importProvider with the default (global) job configuration.
+     * Creates a new import job for $importProvider.
      *
      * @param string $importProvider
      *
@@ -73,7 +73,6 @@ class IndexController extends Controller
         // if job provider has no prerequisites:
         if (!(bool)config(sprintf('import.has_prereq.%s', $importProvider))) {
 
-
             // if job provider also has no configuration:
             if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
                 $this->repository->updateStatus($importJob, 'ready_to_run');
@@ -93,11 +92,11 @@ class IndexController extends Controller
         if (!class_exists($class)) {
             throw new FireflyException(sprintf('No class to handle configuration for "%s".', $importProvider)); // @codeCoverageIgnore
         }
-        /** @var PrerequisitesInterface $object */
+        /** @var PrerequisitesInterface $providerPre */
-        $object = app($class);
+        $providerPre = app($class);
-        $object->setUser(auth()->user());
+        $providerPre->setUser(auth()->user());
 
-        if (!$object->isComplete()) {
+        if (!$providerPre->isComplete()) {
             // redirect to global prerequisites
             return redirect(route('import.prerequisites.index', [$importProvider, $importJob->key]));
         }
@@ -110,42 +109,6 @@ class IndexController extends Controller
 
     }
 
-    //    /**
-    //     * Generate a JSON file of the job's configuration and send it to the user.
-    //     *
-    //     * @param ImportJob $job
-    //     *
-    //     * @return LaravelResponse
-    //     */
-    //    public function download(ImportJob $job)
-    //    {
-    //        Log::debug('Now in download()', ['job' => $job->key]);
-    //        $config = $job->configuration;
-    //
-    //        // This is CSV import specific:
-    //        $config['column-roles-complete']   = false;
-    //        $config['column-mapping-complete'] = false;
-    //        $config['initial-config-complete'] = false;
-    //        $config['has-file-upload']         = false;
-    //        $config['delimiter']               = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter'];
-    //        unset($config['stage']);
-    //
-    //        $result = json_encode($config, JSON_PRETTY_PRINT);
-    //        $name   = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\'));
-    //
-    //        /** @var LaravelResponse $response */
-    //        $response = response($result, 200);
-    //        $response->header('Content-disposition', 'attachment; filename=' . $name)
-    //                 ->header('Content-Type', 'application/json')
-    //                 ->header('Content-Description', 'File Transfer')
-    //                 ->header('Connection', 'Keep-Alive')
-    //                 ->header('Expires', '0')
-    //                 ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0')
-    //                 ->header('Pragma', 'public')
-    //                 ->header('Content-Length', \strlen($result));
-    //
-    //        return $response;
-    //    }
 
     /**
      * General import index.
@@ -252,4 +215,42 @@ class IndexController extends Controller
     //
     //        throw new FireflyException('Job did not complete successfully. Please review the log files.');
     //    }
+
+
+    //    /**
+    //     * Generate a JSON file of the job's configuration and send it to the user.
+    //     *
+    //     * @param ImportJob $job
+    //     *
+    //     * @return LaravelResponse
+    //     */
+    //    public function download(ImportJob $job)
+    //    {
+    //        Log::debug('Now in download()', ['job' => $job->key]);
+    //        $config = $job->configuration;
+    //
+    //        // This is CSV import specific:
+    //        $config['column-roles-complete']   = false;
+    //        $config['column-mapping-complete'] = false;
+    //        $config['initial-config-complete'] = false;
+    //        $config['has-file-upload']         = false;
+    //        $config['delimiter']               = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter'];
+    //        unset($config['stage']);
+    //
+    //        $result = json_encode($config, JSON_PRETTY_PRINT);
+    //        $name   = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\'));
+    //
+    //        /** @var LaravelResponse $response */
+    //        $response = response($result, 200);
+    //        $response->header('Content-disposition', 'attachment; filename=' . $name)
+    //                 ->header('Content-Type', 'application/json')
+    //                 ->header('Content-Description', 'File Transfer')
+    //                 ->header('Connection', 'Keep-Alive')
+    //                 ->header('Expires', '0')
+    //                 ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0')
+    //                 ->header('Pragma', 'public')
+    //                 ->header('Content-Length', \strlen($result));
+    //
+    //        return $response;
+    //    }
 }

app/Http/Controllers/Import/JobConfigurationController.php

@@ -61,63 +61,80 @@ class JobConfigurationController extends Controller
     /**
      * Configure the job. This method is returned to until job is deemed "configured".
      *
-     * @param ImportJob $job
+     * @param ImportJob $importJob
      *
      * @return \Illuminate\Contracts\View\Factory|\Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|\Illuminate\View\View
      *
      * @throws FireflyException
      */
-    public function index(ImportJob $job)
+    public function index(ImportJob $importJob)
     {
-        // if provider has no config, just push it through
+        // catch impossible status:
-        $importProvider = $job->provider;
+        $allowed = ['has_prereq', 'need_job_config', 'has_config'];
-        if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
+        if (null !== $importJob && !in_array($importJob->status, $allowed)) {
-            $this->repository->updateStatus($job, 'ready_to_run');
+            Log::error('Job is not new but wants to do prerequisites');
-
+            session()->flash('error', trans('import.bad_job_status'));
-            return redirect(route('import.job.status.index', [$job->key]));
+            return redirect(route('import.index'));
         }
 
+        Log::debug(sprintf('Now in JobConfigurationController::index() with job "%s" and status "%s"', $importJob->key, $importJob->status));
+
+        // if provider has no config, just push it through:
+        $importProvider = $importJob->provider;
+        if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
+            Log::debug('Job needs no config, is ready to run!');
+            $this->repository->updateStatus($importJob ,'ready_to_run');
+
+            return redirect(route('import.job.status.index', [$importProvider->key]));
+        }
 
         // create configuration class:
-        $configurator = $this->makeConfigurator($job);
+        $configurator = $this->makeConfigurator($importJob);
 
         // is the job already configured?
         if ($configurator->configurationComplete()) {
-            $this->repository->updateStatus($job, 'ready_to_run');
+            Log::debug('Config is complete, set status to ready_to_run.');
+            $this->repository->updateStatus($importJob, 'ready_to_run');
 
-            return redirect(route('import.job.status.index', [$job->key]));
+            return redirect(route('import.job.status.index', [$importJob->key]));
         }
 
-        $this->repository->updateStatus($job, 'configuring');
-
         $view         = $configurator->getNextView();
         $data         = $configurator->getNextData();
         $subTitle     = trans('firefly.import_config_bread_crumb');
         $subTitleIcon = 'fa-wrench';
 
-        return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
+        return view($view, compact('data', 'importJob', 'subTitle', 'subTitleIcon'));
     }
 
     /**
      * Store the configuration. Returns to "configure" method until job is configured.
      *
      * @param Request   $request
-     * @param ImportJob $job
+     * @param ImportJob $importJob
      *
      * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
      *
      * @throws FireflyException
      */
-    public function post(Request $request, ImportJob $job)
+    public function post(Request $request, ImportJob $importJob)
     {
-        Log::debug('Now in postConfigure()', ['job' => $job->key]);
+        // catch impossible status:
-        $configurator = $this->makeConfigurator($job);
+        $allowed = ['has_prereq', 'need_job_config', 'has_config'];
+        if (null !== $importJob && !in_array($importJob->status, $allowed)) {
+            Log::error('Job is not new but wants to do prerequisites');
+            session()->flash('error', trans('import.bad_job_status'));
+            return redirect(route('import.index'));
+        }
+
+        Log::debug('Now in postConfigure()', ['job' => $importJob->key]);
+        $configurator = $this->makeConfigurator($importJob);
 
         // is the job already configured?
         if ($configurator->configurationComplete()) {
-            $this->repository->updateStatus($job, 'ready_to_run');
+            $this->repository->updateStatus($importJob, 'ready_to_run');
 
-            return redirect(route('import.job.status.index', [$job->key]));
+            return redirect(route('import.job.status.index', [$importJob->key]));
         }
 
         $data     = $request->all();
@@ -128,27 +145,27 @@ class JobConfigurationController extends Controller
         }
 
         // return to configure
-        return redirect(route('import.job.configuration.index', [$job->key]));
+        return redirect(route('import.job.configuration.index', [$importJob->key]));
     }
 
     /**
-     * @param ImportJob $job
+     * @param ImportJob $importJob
      *
      * @return JobConfiguratorInterface
      *
      * @throws FireflyException
      */
-    private function makeConfigurator(ImportJob $job): JobConfiguratorInterface
+    private function makeConfigurator(ImportJob $importJob): JobConfiguratorInterface
     {
-        $key       = sprintf('import.configuration.%s', $job->provider);
+        $key       = sprintf('import.configuration.%s', $importJob->provider);
         $className = (string)config($key);
         if (null === $className || !class_exists($className)) {
-            throw new FireflyException(sprintf('Cannot find configurator class for job with provider "%s".', $job->provider)); // @codeCoverageIgnore
+            throw new FireflyException(sprintf('Cannot find configurator class for job with provider "%s".', $importJob->provider)); // @codeCoverageIgnore
         }
         Log::debug(sprintf('Going to create class "%s"', $className));
         /** @var JobConfiguratorInterface $configurator */
         $configurator = app($className);
-        $configurator->setJob($job);
+        $configurator->setJob($importJob);
 
         return $configurator;
     }

app/Http/Controllers/Import/JobStatusController.php

@@ -86,10 +86,10 @@ class JobStatusController extends Controller
      *
      * @return JsonResponse
      */
-    public function json(ImportJob $job): JsonResponse
+    public function json(ImportJob $importJob): JsonResponse
     {
         $json = [
-            'status' => $job->status,
+            'status' => $importJob->status,
         ];
 
         return response()->json($json);
@@ -101,20 +101,34 @@ class JobStatusController extends Controller
      * @return JsonResponse
      * @throws FireflyException
      */
-    public function start(ImportJob $job): JsonResponse
+    public function start(ImportJob $importJob): JsonResponse
     {
-        $importProvider = $job->provider;
+        // catch impossible status:
+        $allowed = ['ready_to_run'];
+        if (null !== $importJob && !in_array($importJob->status, $allowed)) {
+            Log::error('Job is not ready.');
+            session()->flash('error', trans('import.bad_job_status'));
+            return redirect(route('import.index'));
+        }
+
+        $importProvider = $importJob->provider;
         $key            = sprintf('import.routine.%s', $importProvider);
         $className      = config($key);
         if (null === $className || !class_exists($className)) {
             return response()->json(['status' => 'NOK', 'message' => sprintf('Cannot find import routine class for job of type "%s".', $importProvider)]);
         }
+
+
+        // if the job is set to "provider_finished", we should be able to store transactions
+        // generated by the provider.
+        // otherwise, just continue.
+
         // set job to be running:
-        $this->repository->setStatus($job, 'running');
+        $this->repository->setStatus($importJob, 'running');
 
         /** @var RoutineInterface $routine */
         $routine = app($className);
-        $routine->setJob($job);
+        $routine->setJob($importJob);
         try {
             $routine->run();
         } catch (FireflyException $e) {
@@ -123,16 +137,13 @@ class JobStatusController extends Controller
             Log::error($e->getTraceAsString());
 
             // set job errored out:
-            $this->repository->setStatus($job, 'errored');
+            $this->repository->setStatus($importJob, 'error');
 
             return response()->json(['status' => 'NOK', 'message' => $message]);
         }
 
-        // set job finished this step:
-        $this->repository->setStatus($job, 'stage_finished');
-
         // expect nothing from routine, just return OK to user.
-        return response()->json(['status' => 'OK', 'message' => 'finished']);
+        return response()->json(['status' => 'OK', 'message' => 'stage_finished']);
     }
 
     //    /**

app/Http/Controllers/Import/PrerequisitesController.php

@@ -63,9 +63,8 @@ class PrerequisitesController extends Controller
     }
 
     /**
-     * Once there are no prerequisites, this method will create an importjob object and
+     * This method will process and store import provider global prerequisites
-     * redirect the user to a view where this object can be used by a bank specific
+     * such as API keys.
-     * class to process.
      *
      * @param string    $importProvider
      * @param ImportJob $importJob
@@ -75,6 +74,14 @@ class PrerequisitesController extends Controller
      */
     public function index(string $importProvider, ImportJob $importJob = null)
     {
+        // catch impossible status:
+        $allowed = ['new'];
+        if (null !== $importJob && !in_array($importJob->status, $allowed)) {
+            Log::error('Job is not new but wants to do prerequisites');
+            session()->flash('error', trans('import.bad_job_status'));
+            return redirect(route('import.index'));
+        }
+
         app('view')->share('subTitle', trans('import.prerequisites_breadcrumb_' . $importProvider));
         $class = (string)config(sprintf('import.prerequisites.%s', $importProvider));
         if (!class_exists($class)) {
@@ -84,9 +91,12 @@ class PrerequisitesController extends Controller
         $object = app($class);
         $object->setUser(auth()->user());
 
-        // TODO if prerequisites have been met and job is not null, just skip this step.
         if (null !== $importJob && $object->isComplete()) {
-            // set job to
+            // update job:
+            $this->repository->setStatus($importJob, 'has_prereq');
+
+            // redirect to job config:
+            return redirect(route('import.job.configuration.index', [$importJob->key]));
         }
 
 
@@ -117,6 +127,15 @@ class PrerequisitesController extends Controller
     {
         Log::debug(sprintf('Now in postPrerequisites for %s', $importProvider));
 
+        // catch impossible status:
+        $allowed = ['new'];
+        if (null !== $importJob && !in_array($importJob->status, $allowed)) {
+            Log::error('Job is not new but wants to do prerequisites');
+            session()->flash('error', trans('import.bad_job_status'));
+            return redirect(route('import.index'));
+        }
+
+
         $class = (string)config(sprintf('import.prerequisites.%s', $importProvider));
         if (!class_exists($class)) {
             throw new FireflyException(sprintf('Cannot find class %s', $class)); // @codeCoverageIgnore

app/Import/Routine/FakeRoutine.php

@@ -26,8 +26,10 @@ namespace FireflyIII\Import\Routine;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Models\ImportJob;
 use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
+use FireflyIII\Support\Import\Routine\Fake\StageAhoyHandler;
+use FireflyIII\Support\Import\Routine\Fake\StageFinalHandler;
 use FireflyIII\Support\Import\Routine\Fake\StageNewHandler;
-
+use Log;
 
 /**
  * Class FakeRoutine
@@ -51,9 +53,9 @@ class FakeRoutine implements RoutineInterface
     /**
      * Fake import routine has three stages:
      *
-     * "new": will quietly log gibberish for 15 seconds, then switch to stage "ahoy"
+     * "new": will quietly log gibberish for 15 seconds, then switch to stage "ahoy".
-     *        unless "ahoy" has been done already. If so, jump to stage "final".
+     *        will also set status to "ready_to_run" so it will arrive here again.
-     * "ahoy": will log some nonsense and then drop job into "need_extra_config" to force it back to the job config routine.
+     * "ahoy": will log some nonsense and then drop job into status:"need_job_config" to force it back to the job config routine.
      * "final": will do some logging, sleep for 10 seconds and then finish. Generates 5 random transactions.
      *
      * @return bool
@@ -61,12 +63,33 @@ class FakeRoutine implements RoutineInterface
      */
     public function run(): void
     {
+        Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->job->status));
+        if ($this->job->status !== 'running') {
+            throw new FireflyException('This fake job should not be started.');
+        }
+
         switch ($this->job->stage) {
             default:
                 throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->job->stage));
             case 'new':
                 $handler = new StageNewHandler;
                 $handler->run();
+                $this->repository->setStage($this->job, 'ahoy');
+                // set job finished this step:
+                $this->repository->setStatus($this->job, 'ready_to_run');
+
+                return;
+            case 'ahoy':
+                $handler = new StageAhoyHandler;
+                $handler->run();
+                $this->repository->setStatus($this->job, 'need_job_config');
+                $this->repository->setStage($this->job, 'final');
+                break;
+            case 'final':
+                $handler = new StageFinalHandler;
+                $transactions = $handler->getTransactions();
+                $this->repository->setStatus($this->job, 'provider_finished');
+                $this->repository->setStage($this->job, 'final');
         }
     }
 

app/Import/Routine/RoutineInterface.php

@@ -31,6 +31,10 @@ use FireflyIII\Models\ImportJob;
 interface RoutineInterface
 {
     /**
+     * At the end of each run(), the import routine must set the job to the expected status.
+     *
+     * The final status of the routine must be "provider_finished".
+     *
      * @return bool
      * @throws FireflyException
      */

app/Repositories/ImportJob/ImportJobRepository.php

@@ -356,6 +356,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
      */
     public function setStatus(ImportJob $job, string $status): ImportJob
     {
+        Log::debug(sprintf('Set status of job "%s" to "%s"', $job->key, $status));
         $job->status = $status;
         $job->save();
 

app/Support/Import/Routine/Fake/StageAhoyHandler.php

@@ -0,0 +1,45 @@
+<?php
+/**
+ * StageAhoyHandler.php
+ * Copyright (c) 2018 thegrumpydictator@gmail.com
+ *
+ * This file is part of Firefly III.
+ *
+ * Firefly III is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Firefly III is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Support\Import\Routine\Fake;
+
+use FireflyIII\Exceptions\FireflyException;
+use Log;
+
+/**
+ * Class StageAhoyHandler
+ */
+class StageAhoyHandler
+{
+    /**
+     * @throws FireflyException
+     */
+    public function run(): void
+    {
+        for ($i = 0; $i < 15; $i++) {
+            Log::debug(sprintf('Am now in stage AHOY hander, sleeping... (%d)', $i));
+            sleep(1);
+        }
+    }
+
+}

app/Support/Import/Routine/Fake/StageFinalHandler.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace FireflyIII\Support\Import\Routine\Fake;
+
+/**
+ * Class StageFinalHandler
+ *
+ * @package FireflyIII\Support\Import\Routine\Fake
+ */
+class StageFinalHandler
+{
+    /**
+     * @return array
+     */
+    public function getTransactions(): array
+    {
+        $transactions = [];
+
+        for ($i = 0; $i < 5; $i++) {
+            $transaction = [];
+
+
+            $transactions[] = $transaction;
+        }
+
+
+        return $transactions;
+
+    }
+
+}

app/Support/Import/Routine/Fake/StageNewHandler.php

@@ -23,6 +23,7 @@ declare(strict_types=1);
 
 namespace FireflyIII\Support\Import\Routine\Fake;
 
+use FireflyIII\Exceptions\FireflyException;
 use Log;
 
 /**
@@ -31,7 +32,7 @@ use Log;
 class StageNewHandler
 {
     /**
-     *
+     * @throws FireflyException
      */
     public function run(): void
     {

config/import.php

@@ -38,7 +38,7 @@ use FireflyIII\Import\Routine\SpectreRoutine;
 
 return [
     'enabled'       => [
-        'fake'    => false,
+        'fake'    => true,
         'file'    => true,
         'bunq'    => true,
         'spectre' => true,
@@ -65,7 +65,7 @@ return [
         'yodlee'  => false,
     ],
     'has_config'    => [
-        'fake'    => false,
+        'fake'    => true,
         'file'    => true,
         'bunq'    => true,
         'spectre' => true,

public/js/ff/import/status_v2.js

@@ -24,8 +24,9 @@ var timeOutId;
 var hasStartedJob = false;
 var checkInitialInterval = 1000;
 var checkNextInterval = 500;
-var maxLoops = 20;
+var maxLoops = 60;
 var totalLoops = 0;
+var startCount = 0;
 
 $(function () {
     "use strict";
@@ -50,6 +51,10 @@ function reportOnJobStatus(data) {
     console.log(data);
     switch (data.status) {
         case "ready_to_run":
+            if (startCount > 0) {
+                hasStartedJob = false;
+            }
+            startCount++;
             startJob();
             checkOnJob();
             break;
@@ -57,8 +62,14 @@ function reportOnJobStatus(data) {
             showProgressBox();
             checkOnJob();
             break;
+
+        case "need_job_config":
+            // redirect user to configuration for this job.
+            window.location.replace(jobConfigurationUri);
+            break;
         default:
             console.error('Cannot handle status ' + data.status);
+
     }
 }
 

resources/lang/en_US/import.php

@@ -180,6 +180,7 @@ return [
     // index of import:
     'general_index_title'                  => 'Import a file',
     'general_index_intro'                  => 'Welcome to Firefly III\'s import routine. There are a few ways of importing data into Firefly III, displayed here as buttons.',
+    'bad_job_status'                       => 'You cannot access this page when the job is at this point. Sorry!',
 
     // import provider strings (index):
     'button_fake'                          => 'Fake an import',

resources/views/import/fake/enter-artist.twig

@@ -20,7 +20,7 @@
         </div>
     </div>
 
-    <form method="POST" action="{{ route('import.job.configuration.post', job.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
+    <form method="POST" action="{{ route('import.job.configuration.post', importJob.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
         <input name="_token" type="hidden" value="{{ csrf_token() }}">
         <div class="row">
             <div class="col-lg-12">

resources/views/import/fake/enter-song.twig

@@ -20,7 +20,7 @@
         </div>
     </div>
 
-    <form method="POST" action="{{ route('import.job.configuration.post', job.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
+    <form method="POST" action="{{ route('import.job.configuration.post', importJob.key) }}" accept-charset="UTF-8" class="form-horizontal" enctype="multipart/form-data">
         <input name="_token" type="hidden" value="{{ csrf_token() }}">
         <div class="row">
             <div class="col-lg-12">

resources/views/import/status.twig

@@ -170,7 +170,7 @@
 
         var jobStatusUri = '{{ route('import.job.status.json', [importJob.key]) }}';
         var jobStartUri = '{{ route('import.job.start', [importJob.key]) }}';
-
+        var jobConfigurationUri = '{{ route('import.job.configuration.index', [importJob.key]) }}';
 
         // some useful translations.
         {#var langImportTimeOutError = '(time out thing)';#}
@@ -178,7 +178,7 @@
         {#var langImportMultiError = '{{ trans('import.status_errors_multi')|escape('js') }}';#}
 
 
-        {#var jobConfigureUri = '#}{#{{ route('import.configure', [job.key]) }}#}{#';#}
+
         var token = '{{ csrf_token() }}';
         var job = {{ job|json_encode|raw }};
     </script>