From ccda71ff8e90f0ca136772434783a0c1b92add93 Mon Sep 17 00:00:00 2001 From: James Cole Date: Tue, 1 May 2018 20:47:38 +0200 Subject: [PATCH] New code for updated import routine. --- .../Controllers/Import/IndexController.php | 85 ++++++++++--------- .../Import/JobConfigurationController.php | 69 +++++++++------ .../Import/JobStatusController.php | 33 ++++--- .../Import/PrerequisitesController.php | 29 +++++-- app/Import/Routine/FakeRoutine.php | 31 ++++++- app/Import/Routine/RoutineInterface.php | 4 + .../ImportJob/ImportJobRepository.php | 1 + .../Import/Routine/Fake/StageAhoyHandler.php | 45 ++++++++++ .../Import/Routine/Fake/StageFinalHandler.php | 31 +++++++ .../Import/Routine/Fake/StageNewHandler.php | 3 +- config/import.php | 4 +- public/js/ff/import/status_v2.js | 13 ++- resources/lang/en_US/import.php | 3 +- resources/views/import/fake/enter-artist.twig | 2 +- resources/views/import/fake/enter-song.twig | 2 +- resources/views/import/status.twig | 4 +- 16 files changed, 262 insertions(+), 97 deletions(-) create mode 100644 app/Support/Import/Routine/Fake/StageAhoyHandler.php create mode 100644 app/Support/Import/Routine/Fake/StageFinalHandler.php diff --git a/app/Http/Controllers/Import/IndexController.php b/app/Http/Controllers/Import/IndexController.php index 9dd0691f07..639b5bb152 100644 --- a/app/Http/Controllers/Import/IndexController.php +++ b/app/Http/Controllers/Import/IndexController.php @@ -58,7 +58,7 @@ class IndexController extends Controller } /** - * Creates a new import job for $importProvider with the default (global) job configuration. + * Creates a new import job for $importProvider. * * @param string $importProvider * @@ -73,7 +73,6 @@ class IndexController extends Controller // if job provider has no prerequisites: if (!(bool)config(sprintf('import.has_prereq.%s', $importProvider))) { - // if job provider also has no configuration: if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) { $this->repository->updateStatus($importJob, 'ready_to_run'); @@ -93,11 +92,11 @@ class IndexController extends Controller if (!class_exists($class)) { throw new FireflyException(sprintf('No class to handle configuration for "%s".', $importProvider)); // @codeCoverageIgnore } - /** @var PrerequisitesInterface $object */ - $object = app($class); - $object->setUser(auth()->user()); + /** @var PrerequisitesInterface $providerPre */ + $providerPre = app($class); + $providerPre->setUser(auth()->user()); - if (!$object->isComplete()) { + if (!$providerPre->isComplete()) { // redirect to global prerequisites return redirect(route('import.prerequisites.index', [$importProvider, $importJob->key])); } @@ -110,42 +109,6 @@ class IndexController extends Controller } - // /** - // * Generate a JSON file of the job's configuration and send it to the user. - // * - // * @param ImportJob $job - // * - // * @return LaravelResponse - // */ - // public function download(ImportJob $job) - // { - // Log::debug('Now in download()', ['job' => $job->key]); - // $config = $job->configuration; - // - // // This is CSV import specific: - // $config['column-roles-complete'] = false; - // $config['column-mapping-complete'] = false; - // $config['initial-config-complete'] = false; - // $config['has-file-upload'] = false; - // $config['delimiter'] = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter']; - // unset($config['stage']); - // - // $result = json_encode($config, JSON_PRETTY_PRINT); - // $name = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\')); - // - // /** @var LaravelResponse $response */ - // $response = response($result, 200); - // $response->header('Content-disposition', 'attachment; filename=' . $name) - // ->header('Content-Type', 'application/json') - // ->header('Content-Description', 'File Transfer') - // ->header('Connection', 'Keep-Alive') - // ->header('Expires', '0') - // ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0') - // ->header('Pragma', 'public') - // ->header('Content-Length', \strlen($result)); - // - // return $response; - // } /** * General import index. @@ -252,4 +215,42 @@ class IndexController extends Controller // // throw new FireflyException('Job did not complete successfully. Please review the log files.'); // } + + + // /** + // * Generate a JSON file of the job's configuration and send it to the user. + // * + // * @param ImportJob $job + // * + // * @return LaravelResponse + // */ + // public function download(ImportJob $job) + // { + // Log::debug('Now in download()', ['job' => $job->key]); + // $config = $job->configuration; + // + // // This is CSV import specific: + // $config['column-roles-complete'] = false; + // $config['column-mapping-complete'] = false; + // $config['initial-config-complete'] = false; + // $config['has-file-upload'] = false; + // $config['delimiter'] = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter']; + // unset($config['stage']); + // + // $result = json_encode($config, JSON_PRETTY_PRINT); + // $name = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\')); + // + // /** @var LaravelResponse $response */ + // $response = response($result, 200); + // $response->header('Content-disposition', 'attachment; filename=' . $name) + // ->header('Content-Type', 'application/json') + // ->header('Content-Description', 'File Transfer') + // ->header('Connection', 'Keep-Alive') + // ->header('Expires', '0') + // ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0') + // ->header('Pragma', 'public') + // ->header('Content-Length', \strlen($result)); + // + // return $response; + // } } diff --git a/app/Http/Controllers/Import/JobConfigurationController.php b/app/Http/Controllers/Import/JobConfigurationController.php index 7261da9bc1..cd27eb7420 100644 --- a/app/Http/Controllers/Import/JobConfigurationController.php +++ b/app/Http/Controllers/Import/JobConfigurationController.php @@ -61,63 +61,80 @@ class JobConfigurationController extends Controller /** * Configure the job. This method is returned to until job is deemed "configured". * - * @param ImportJob $job + * @param ImportJob $importJob * * @return \Illuminate\Contracts\View\Factory|\Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|\Illuminate\View\View * * @throws FireflyException */ - public function index(ImportJob $job) + public function index(ImportJob $importJob) { - // if provider has no config, just push it through - $importProvider = $job->provider; - if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) { - $this->repository->updateStatus($job, 'ready_to_run'); - - return redirect(route('import.job.status.index', [$job->key])); + // catch impossible status: + $allowed = ['has_prereq', 'need_job_config', 'has_config']; + if (null !== $importJob && !in_array($importJob->status, $allowed)) { + Log::error('Job is not new but wants to do prerequisites'); + session()->flash('error', trans('import.bad_job_status')); + return redirect(route('import.index')); } + Log::debug(sprintf('Now in JobConfigurationController::index() with job "%s" and status "%s"', $importJob->key, $importJob->status)); + + // if provider has no config, just push it through: + $importProvider = $importJob->provider; + if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) { + Log::debug('Job needs no config, is ready to run!'); + $this->repository->updateStatus($importJob ,'ready_to_run'); + + return redirect(route('import.job.status.index', [$importProvider->key])); + } // create configuration class: - $configurator = $this->makeConfigurator($job); + $configurator = $this->makeConfigurator($importJob); // is the job already configured? if ($configurator->configurationComplete()) { - $this->repository->updateStatus($job, 'ready_to_run'); + Log::debug('Config is complete, set status to ready_to_run.'); + $this->repository->updateStatus($importJob, 'ready_to_run'); - return redirect(route('import.job.status.index', [$job->key])); + return redirect(route('import.job.status.index', [$importJob->key])); } - $this->repository->updateStatus($job, 'configuring'); - $view = $configurator->getNextView(); $data = $configurator->getNextData(); $subTitle = trans('firefly.import_config_bread_crumb'); $subTitleIcon = 'fa-wrench'; - return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon')); + return view($view, compact('data', 'importJob', 'subTitle', 'subTitleIcon')); } /** * Store the configuration. Returns to "configure" method until job is configured. * * @param Request $request - * @param ImportJob $job + * @param ImportJob $importJob * * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector * * @throws FireflyException */ - public function post(Request $request, ImportJob $job) + public function post(Request $request, ImportJob $importJob) { - Log::debug('Now in postConfigure()', ['job' => $job->key]); - $configurator = $this->makeConfigurator($job); + // catch impossible status: + $allowed = ['has_prereq', 'need_job_config', 'has_config']; + if (null !== $importJob && !in_array($importJob->status, $allowed)) { + Log::error('Job is not new but wants to do prerequisites'); + session()->flash('error', trans('import.bad_job_status')); + return redirect(route('import.index')); + } + + Log::debug('Now in postConfigure()', ['job' => $importJob->key]); + $configurator = $this->makeConfigurator($importJob); // is the job already configured? if ($configurator->configurationComplete()) { - $this->repository->updateStatus($job, 'ready_to_run'); + $this->repository->updateStatus($importJob, 'ready_to_run'); - return redirect(route('import.job.status.index', [$job->key])); + return redirect(route('import.job.status.index', [$importJob->key])); } $data = $request->all(); @@ -128,27 +145,27 @@ class JobConfigurationController extends Controller } // return to configure - return redirect(route('import.job.configuration.index', [$job->key])); + return redirect(route('import.job.configuration.index', [$importJob->key])); } /** - * @param ImportJob $job + * @param ImportJob $importJob * * @return JobConfiguratorInterface * * @throws FireflyException */ - private function makeConfigurator(ImportJob $job): JobConfiguratorInterface + private function makeConfigurator(ImportJob $importJob): JobConfiguratorInterface { - $key = sprintf('import.configuration.%s', $job->provider); + $key = sprintf('import.configuration.%s', $importJob->provider); $className = (string)config($key); if (null === $className || !class_exists($className)) { - throw new FireflyException(sprintf('Cannot find configurator class for job with provider "%s".', $job->provider)); // @codeCoverageIgnore + throw new FireflyException(sprintf('Cannot find configurator class for job with provider "%s".', $importJob->provider)); // @codeCoverageIgnore } Log::debug(sprintf('Going to create class "%s"', $className)); /** @var JobConfiguratorInterface $configurator */ $configurator = app($className); - $configurator->setJob($job); + $configurator->setJob($importJob); return $configurator; } diff --git a/app/Http/Controllers/Import/JobStatusController.php b/app/Http/Controllers/Import/JobStatusController.php index 53307c51da..f97854b807 100644 --- a/app/Http/Controllers/Import/JobStatusController.php +++ b/app/Http/Controllers/Import/JobStatusController.php @@ -86,10 +86,10 @@ class JobStatusController extends Controller * * @return JsonResponse */ - public function json(ImportJob $job): JsonResponse + public function json(ImportJob $importJob): JsonResponse { $json = [ - 'status' => $job->status, + 'status' => $importJob->status, ]; return response()->json($json); @@ -101,20 +101,34 @@ class JobStatusController extends Controller * @return JsonResponse * @throws FireflyException */ - public function start(ImportJob $job): JsonResponse + public function start(ImportJob $importJob): JsonResponse { - $importProvider = $job->provider; + // catch impossible status: + $allowed = ['ready_to_run']; + if (null !== $importJob && !in_array($importJob->status, $allowed)) { + Log::error('Job is not ready.'); + session()->flash('error', trans('import.bad_job_status')); + return redirect(route('import.index')); + } + + $importProvider = $importJob->provider; $key = sprintf('import.routine.%s', $importProvider); $className = config($key); if (null === $className || !class_exists($className)) { return response()->json(['status' => 'NOK', 'message' => sprintf('Cannot find import routine class for job of type "%s".', $importProvider)]); } + + + // if the job is set to "provider_finished", we should be able to store transactions + // generated by the provider. + // otherwise, just continue. + // set job to be running: - $this->repository->setStatus($job, 'running'); + $this->repository->setStatus($importJob, 'running'); /** @var RoutineInterface $routine */ $routine = app($className); - $routine->setJob($job); + $routine->setJob($importJob); try { $routine->run(); } catch (FireflyException $e) { @@ -123,16 +137,13 @@ class JobStatusController extends Controller Log::error($e->getTraceAsString()); // set job errored out: - $this->repository->setStatus($job, 'errored'); + $this->repository->setStatus($importJob, 'error'); return response()->json(['status' => 'NOK', 'message' => $message]); } - // set job finished this step: - $this->repository->setStatus($job, 'stage_finished'); - // expect nothing from routine, just return OK to user. - return response()->json(['status' => 'OK', 'message' => 'finished']); + return response()->json(['status' => 'OK', 'message' => 'stage_finished']); } // /** diff --git a/app/Http/Controllers/Import/PrerequisitesController.php b/app/Http/Controllers/Import/PrerequisitesController.php index ab4607136c..cc44c5b310 100644 --- a/app/Http/Controllers/Import/PrerequisitesController.php +++ b/app/Http/Controllers/Import/PrerequisitesController.php @@ -63,9 +63,8 @@ class PrerequisitesController extends Controller } /** - * Once there are no prerequisites, this method will create an importjob object and - * redirect the user to a view where this object can be used by a bank specific - * class to process. + * This method will process and store import provider global prerequisites + * such as API keys. * * @param string $importProvider * @param ImportJob $importJob @@ -75,6 +74,14 @@ class PrerequisitesController extends Controller */ public function index(string $importProvider, ImportJob $importJob = null) { + // catch impossible status: + $allowed = ['new']; + if (null !== $importJob && !in_array($importJob->status, $allowed)) { + Log::error('Job is not new but wants to do prerequisites'); + session()->flash('error', trans('import.bad_job_status')); + return redirect(route('import.index')); + } + app('view')->share('subTitle', trans('import.prerequisites_breadcrumb_' . $importProvider)); $class = (string)config(sprintf('import.prerequisites.%s', $importProvider)); if (!class_exists($class)) { @@ -84,9 +91,12 @@ class PrerequisitesController extends Controller $object = app($class); $object->setUser(auth()->user()); - // TODO if prerequisites have been met and job is not null, just skip this step. if (null !== $importJob && $object->isComplete()) { - // set job to + // update job: + $this->repository->setStatus($importJob, 'has_prereq'); + + // redirect to job config: + return redirect(route('import.job.configuration.index', [$importJob->key])); } @@ -117,6 +127,15 @@ class PrerequisitesController extends Controller { Log::debug(sprintf('Now in postPrerequisites for %s', $importProvider)); + // catch impossible status: + $allowed = ['new']; + if (null !== $importJob && !in_array($importJob->status, $allowed)) { + Log::error('Job is not new but wants to do prerequisites'); + session()->flash('error', trans('import.bad_job_status')); + return redirect(route('import.index')); + } + + $class = (string)config(sprintf('import.prerequisites.%s', $importProvider)); if (!class_exists($class)) { throw new FireflyException(sprintf('Cannot find class %s', $class)); // @codeCoverageIgnore diff --git a/app/Import/Routine/FakeRoutine.php b/app/Import/Routine/FakeRoutine.php index 6fef517d3c..a79f9ce76f 100644 --- a/app/Import/Routine/FakeRoutine.php +++ b/app/Import/Routine/FakeRoutine.php @@ -26,8 +26,10 @@ namespace FireflyIII\Import\Routine; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use FireflyIII\Support\Import\Routine\Fake\StageAhoyHandler; +use FireflyIII\Support\Import\Routine\Fake\StageFinalHandler; use FireflyIII\Support\Import\Routine\Fake\StageNewHandler; - +use Log; /** * Class FakeRoutine @@ -51,9 +53,9 @@ class FakeRoutine implements RoutineInterface /** * Fake import routine has three stages: * - * "new": will quietly log gibberish for 15 seconds, then switch to stage "ahoy" - * unless "ahoy" has been done already. If so, jump to stage "final". - * "ahoy": will log some nonsense and then drop job into "need_extra_config" to force it back to the job config routine. + * "new": will quietly log gibberish for 15 seconds, then switch to stage "ahoy". + * will also set status to "ready_to_run" so it will arrive here again. + * "ahoy": will log some nonsense and then drop job into status:"need_job_config" to force it back to the job config routine. * "final": will do some logging, sleep for 10 seconds and then finish. Generates 5 random transactions. * * @return bool @@ -61,12 +63,33 @@ class FakeRoutine implements RoutineInterface */ public function run(): void { + Log::debug(sprintf('Now in run() for fake routine with status: %s', $this->job->status)); + if ($this->job->status !== 'running') { + throw new FireflyException('This fake job should not be started.'); + } + switch ($this->job->stage) { default: throw new FireflyException(sprintf('Fake routine cannot handle stage "%s".', $this->job->stage)); case 'new': $handler = new StageNewHandler; $handler->run(); + $this->repository->setStage($this->job, 'ahoy'); + // set job finished this step: + $this->repository->setStatus($this->job, 'ready_to_run'); + + return; + case 'ahoy': + $handler = new StageAhoyHandler; + $handler->run(); + $this->repository->setStatus($this->job, 'need_job_config'); + $this->repository->setStage($this->job, 'final'); + break; + case 'final': + $handler = new StageFinalHandler; + $transactions = $handler->getTransactions(); + $this->repository->setStatus($this->job, 'provider_finished'); + $this->repository->setStage($this->job, 'final'); } } diff --git a/app/Import/Routine/RoutineInterface.php b/app/Import/Routine/RoutineInterface.php index 32ee378874..484b0f5c37 100644 --- a/app/Import/Routine/RoutineInterface.php +++ b/app/Import/Routine/RoutineInterface.php @@ -31,6 +31,10 @@ use FireflyIII\Models\ImportJob; interface RoutineInterface { /** + * At the end of each run(), the import routine must set the job to the expected status. + * + * The final status of the routine must be "provider_finished". + * * @return bool * @throws FireflyException */ diff --git a/app/Repositories/ImportJob/ImportJobRepository.php b/app/Repositories/ImportJob/ImportJobRepository.php index 8bb8af480e..55c8c54cd5 100644 --- a/app/Repositories/ImportJob/ImportJobRepository.php +++ b/app/Repositories/ImportJob/ImportJobRepository.php @@ -356,6 +356,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface */ public function setStatus(ImportJob $job, string $status): ImportJob { + Log::debug(sprintf('Set status of job "%s" to "%s"', $job->key, $status)); $job->status = $status; $job->save(); diff --git a/app/Support/Import/Routine/Fake/StageAhoyHandler.php b/app/Support/Import/Routine/Fake/StageAhoyHandler.php new file mode 100644 index 0000000000..851b61ce06 --- /dev/null +++ b/app/Support/Import/Routine/Fake/StageAhoyHandler.php @@ -0,0 +1,45 @@ +. + */ + +declare(strict_types=1); + +namespace FireflyIII\Support\Import\Routine\Fake; + +use FireflyIII\Exceptions\FireflyException; +use Log; + +/** + * Class StageAhoyHandler + */ +class StageAhoyHandler +{ + /** + * @throws FireflyException + */ + public function run(): void + { + for ($i = 0; $i < 15; $i++) { + Log::debug(sprintf('Am now in stage AHOY hander, sleeping... (%d)', $i)); + sleep(1); + } + } + +} \ No newline at end of file diff --git a/app/Support/Import/Routine/Fake/StageFinalHandler.php b/app/Support/Import/Routine/Fake/StageFinalHandler.php new file mode 100644 index 0000000000..6900573de5 --- /dev/null +++ b/app/Support/Import/Routine/Fake/StageFinalHandler.php @@ -0,0 +1,31 @@ + [ - 'fake' => false, + 'fake' => true, 'file' => true, 'bunq' => true, 'spectre' => true, @@ -65,7 +65,7 @@ return [ 'yodlee' => false, ], 'has_config' => [ - 'fake' => false, + 'fake' => true, 'file' => true, 'bunq' => true, 'spectre' => true, diff --git a/public/js/ff/import/status_v2.js b/public/js/ff/import/status_v2.js index 421627de6f..92ab649dba 100644 --- a/public/js/ff/import/status_v2.js +++ b/public/js/ff/import/status_v2.js @@ -24,8 +24,9 @@ var timeOutId; var hasStartedJob = false; var checkInitialInterval = 1000; var checkNextInterval = 500; -var maxLoops = 20; +var maxLoops = 60; var totalLoops = 0; +var startCount = 0; $(function () { "use strict"; @@ -50,6 +51,10 @@ function reportOnJobStatus(data) { console.log(data); switch (data.status) { case "ready_to_run": + if (startCount > 0) { + hasStartedJob = false; + } + startCount++; startJob(); checkOnJob(); break; @@ -57,8 +62,14 @@ function reportOnJobStatus(data) { showProgressBox(); checkOnJob(); break; + + case "need_job_config": + // redirect user to configuration for this job. + window.location.replace(jobConfigurationUri); + break; default: console.error('Cannot handle status ' + data.status); + } } diff --git a/resources/lang/en_US/import.php b/resources/lang/en_US/import.php index fbfbcdc6b2..46bfd04366 100644 --- a/resources/lang/en_US/import.php +++ b/resources/lang/en_US/import.php @@ -180,6 +180,7 @@ return [ // index of import: 'general_index_title' => 'Import a file', 'general_index_intro' => 'Welcome to Firefly III\'s import routine. There are a few ways of importing data into Firefly III, displayed here as buttons.', + 'bad_job_status' => 'You cannot access this page when the job is at this point. Sorry!', // import provider strings (index): 'button_fake' => 'Fake an import', @@ -217,7 +218,7 @@ return [ 'do_config_quovo' => 'Configuration for imports from Quovo', // job configuration: - 'job_configuration_breadcrumb' => 'Configuration for job ":key"', + 'job_configuration_breadcrumb' => 'Configuration for job ":key"', // import index page: 'index_breadcrumb' => 'Index', diff --git a/resources/views/import/fake/enter-artist.twig b/resources/views/import/fake/enter-artist.twig index dcd25f50ac..824621b1af 100644 --- a/resources/views/import/fake/enter-artist.twig +++ b/resources/views/import/fake/enter-artist.twig @@ -20,7 +20,7 @@ -
+
diff --git a/resources/views/import/fake/enter-song.twig b/resources/views/import/fake/enter-song.twig index 1874b386a5..6c8e168f3e 100644 --- a/resources/views/import/fake/enter-song.twig +++ b/resources/views/import/fake/enter-song.twig @@ -20,7 +20,7 @@
- +
diff --git a/resources/views/import/status.twig b/resources/views/import/status.twig index 288a290f70..d40a9ad4f7 100644 --- a/resources/views/import/status.twig +++ b/resources/views/import/status.twig @@ -170,7 +170,7 @@ var jobStatusUri = '{{ route('import.job.status.json', [importJob.key]) }}'; var jobStartUri = '{{ route('import.job.start', [importJob.key]) }}'; - + var jobConfigurationUri = '{{ route('import.job.configuration.index', [importJob.key]) }}'; // some useful translations. {#var langImportTimeOutError = '(time out thing)';#} @@ -178,7 +178,7 @@ {#var langImportMultiError = '{{ trans('import.status_errors_multi')|escape('js') }}';#} - {#var jobConfigureUri = '#}{#{{ route('import.configure', [job.key]) }}#}{#';#} + var token = '{{ csrf_token() }}'; var job = {{ job|json_encode|raw }};