diff --git a/app/Http/Controllers/Import/FileController.php b/app/Http/Controllers/Import/FileController.php index 5e13cc9ed3..d6ad685aa2 100644 --- a/app/Http/Controllers/Import/FileController.php +++ b/app/Http/Controllers/Import/FileController.php @@ -175,50 +175,7 @@ class FileController extends Controller return redirect(route('import.file.configure', [$job->key])); } - /** - * Show status of import job in JSON. - * - * @param ImportJob $job - * - * @return \Illuminate\Http\JsonResponse - */ - public function json(ImportJob $job) - { - $result = [ - 'started' => false, - 'finished' => false, - 'running' => false, - 'errors' => array_values($job->extended_status['errors']), - 'percentage' => 0, - 'show_percentage' => false, - 'steps' => $job->extended_status['steps'], - 'done' => $job->extended_status['done'], - 'statusText' => trans('firefly.import_status_job_' . $job->status), - 'status' => $job->status, - 'finishedText' => '', - ]; - if (0 !== $job->extended_status['steps']) { - $result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0); - $result['show_percentage'] = true; - } - - if ('finished' === $job->status) { - $tagId = $job->extended_status['tag']; - /** @var TagRepositoryInterface $repository */ - $repository = app(TagRepositoryInterface::class); - $tag = $repository->find($tagId); - $result['finished'] = true; - $result['finishedText'] = trans('firefly.import_status_finished_job', ['link' => route('tags.show', [$tag->id, 'all']), 'tag' => $tag->tag]); - } - - if ('running' === $job->status) { - $result['started'] = true; - $result['running'] = true; - } - - return Response::json($result); - } /** * Step 4. Save the configuration. @@ -269,24 +226,7 @@ class FileController extends Controller return Response::json(['run' => 'ok']); } - throw new FireflyException('Job did not complete succesfully.'); - } - - /** - * @param ImportJob $job - * - * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View - */ - public function status(ImportJob $job) - { - $statuses = ['configured', 'running', 'finished']; - if (!in_array($job->status, $statuses)) { - return redirect(route('import.file.configure', [$job->key])); - } - $subTitle = trans('firefly.import_status_sub_title'); - $subTitleIcon = 'fa-star'; - - return view('import.status', compact('job', 'subTitle', 'subTitleIcon')); + throw new FireflyException('Job did not complete successfully.'); } /** diff --git a/app/Http/Controllers/Import/IndexController.php b/app/Http/Controllers/Import/IndexController.php index 866f0b1557..b644e561e3 100644 --- a/app/Http/Controllers/Import/IndexController.php +++ b/app/Http/Controllers/Import/IndexController.php @@ -6,7 +6,12 @@ namespace FireflyIII\Http\Controllers\Import; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Http\Controllers\Controller; +use FireflyIII\Import\Routine\ImportRoutine; +use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface; +use Illuminate\Http\Response as LaravelResponse; +use Log; +use Response; use View; /** @@ -56,6 +61,42 @@ class IndexController extends Controller } + /** + * Generate a JSON file of the job's configuration and send it to the user. + * + * @param ImportJob $job + * + * @return LaravelResponse + */ + public function download(ImportJob $job) + { + Log::debug('Now in download()', ['job' => $job->key]); + $config = $job->configuration; + + // This is CSV import specific: + $config['column-roles-complete'] = false; + $config['column-mapping-complete'] = false; + $config['initial-config-complete'] = false; + $config['has-file-upload'] = false; + $config['delimiter'] = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter']; + + $result = json_encode($config, JSON_PRETTY_PRINT); + $name = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\')); + + /** @var LaravelResponse $response */ + $response = response($result, 200); + $response->header('Content-disposition', 'attachment; filename=' . $name) + ->header('Content-Type', 'application/json') + ->header('Content-Description', 'File Transfer') + ->header('Connection', 'Keep-Alive') + ->header('Expires', '0') + ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0') + ->header('Pragma', 'public') + ->header('Content-Length', strlen($result)); + + return $response; + } + /** * General import index. * @@ -70,4 +111,34 @@ class IndexController extends Controller return view('import.index', compact('subTitle', 'subTitleIcon', 'routines')); } + /** + * @param ImportJob $job + * + * @return \Illuminate\Http\JsonResponse + * @throws FireflyException + */ + public function start(ImportJob $job) + { + + $type = $job->file_type; + $key = sprintf('import.routine.%s', $type); + $className = config($key); + if (null === $className || !class_exists($className)) { + throw new FireflyException(sprintf('Cannot find import routine class for job of type "%s".', $type)); // @codeCoverageIgnore + } + var_dump($className); + exit; + + /** @var ImportRoutine $routine */ + $routine = app(ImportRoutine::class); + $routine->setJob($job); + $result = $routine->run(); + + if ($result) { + return Response::json(['run' => 'ok']); + } + + throw new FireflyException('Job did not complete successfully. Please review the log files.'); + } + } \ No newline at end of file diff --git a/app/Http/Controllers/Import/StatusController.php b/app/Http/Controllers/Import/StatusController.php index f4f11d9fda..d04d4edb00 100644 --- a/app/Http/Controllers/Import/StatusController.php +++ b/app/Http/Controllers/Import/StatusController.php @@ -5,6 +5,9 @@ namespace FireflyIII\Http\Controllers\Import; use FireflyIII\Http\Controllers\Controller; +use FireflyIII\Models\ImportJob; +use FireflyIII\Repositories\Tag\TagRepositoryInterface; +use Response; /** * Class StatusController @@ -12,4 +15,83 @@ use FireflyIII\Http\Controllers\Controller; class StatusController extends Controller { + /** + * + */ + public function __construct() + { + parent::__construct(); + + $this->middleware( + function ($request, $next) { + app('view')->share('mainTitleIcon', 'fa-archive'); + app('view')->share('title', trans('firefly.import_index_title')); + + return $next($request); + } + ); + } + /** + * @param ImportJob $job + * + * @return \Illuminate\Contracts\View\Factory|\Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|\Illuminate\View\View + */ + public function index(ImportJob $job) + { + $statuses = ['configured', 'running', 'finished', 'errored']; + if (!in_array($job->status, $statuses)) { + return redirect(route('import.file.configure', [$job->key])); + } + $subTitle = trans('firefly.import_status_sub_title'); + $subTitleIcon = 'fa-star'; + + return view('import.status', compact('job', 'subTitle', 'subTitleIcon')); + } + + /** + * Show status of import job in JSON. + * + * @param ImportJob $job + * + * @return \Illuminate\Http\JsonResponse + */ + public function json(ImportJob $job) + { + $result = [ + 'started' => false, + 'finished' => false, + 'running' => false, + 'errors' => array_values($job->extended_status['errors']), + 'percentage' => 0, + 'show_percentage' => false, + 'steps' => $job->extended_status['steps'], + 'done' => $job->extended_status['done'], + 'statusText' => trans('firefly.import_status_job_' . $job->status), + 'status' => $job->status, + 'finishedText' => '', + ]; + + if (0 !== $job->extended_status['steps']) { + $result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0); + $result['show_percentage'] = true; + } + + if ('finished' === $job->status) { + $tagId = $job->extended_status['tag']; + /** @var TagRepositoryInterface $repository */ + $repository = app(TagRepositoryInterface::class); + $tag = $repository->find($tagId); + $result['finished'] = true; + $result['finishedText'] = trans('firefly.import_status_finished_job', ['link' => route('tags.show', [$tag->id, 'all']), 'tag' => $tag->tag]); + } + + if ('running' === $job->status) { + $result['started'] = true; + $result['running'] = true; + } + // TODO cannot handle 'errored' + + return Response::json($result); + } + } \ No newline at end of file diff --git a/app/Import/Routine/FileRoutine.php b/app/Import/Routine/FileRoutine.php index 74dc52150c..eb5ea1cd9c 100644 --- a/app/Import/Routine/FileRoutine.php +++ b/app/Import/Routine/FileRoutine.php @@ -1,6 +1,6 @@ . + */ +declare(strict_types=1); + +namespace FireflyIII\Import\Routine; + +use Carbon\Carbon; +use DB; +use FireflyIII\Import\FileProcessor\FileProcessorInterface; +use FireflyIII\Import\Storage\ImportStorage; +use FireflyIII\Models\ImportJob; +use FireflyIII\Models\Tag; +use FireflyIII\Repositories\Tag\TagRepositoryInterface; +use Illuminate\Support\Collection; +use Log; + +interface RoutineInterface +{ + /** + * @return bool + */ + public function run(): bool; + + /** + * @param ImportJob $job + * + * @return mixed + */ + public function setJob(ImportJob $job); +} diff --git a/config/import.php b/config/import.php index 4181666ea5..877eeb4711 100644 --- a/config/import.php +++ b/config/import.php @@ -2,25 +2,32 @@ declare(strict_types=1); return [ - 'enabled' => [ + 'enabled' => [ 'file' => true, 'bunq' => true, 'spectre' => true, 'plaid' => true, ], - 'prerequisites' => [ + 'prerequisites' => [ 'file' => 'FireflyIII\Import\Prerequisites\FilePrerequisites', 'bunq' => 'FireflyIII\Import\Prerequisites\BunqPrerequisites', 'spectre' => 'FireflyIII\Import\Prerequisites\SpectrePrerequisites', 'plaid' => 'FireflyIII\Import\Prerequisites\PlaidPrerequisites', ], - 'configuration' => [ + 'configuration' => [ 'file' => 'FireflyIII\Import\Configuration\FileConfigurator', 'bunq' => 'FireflyIII\Import\Configuration\BunqConfigurator', 'spectre' => 'FireflyIII\Import\Configuration\SpectreConfigurator', 'plaid' => 'FireflyIII\Import\Configuration\PlaidConfigurator', ], + 'routine' => [ + 'file' => 'FireflyIII\Import\Routine\FileRoutine', + 'bunq' => 'FireflyIII\Import\Routine\BunqRoutine', + 'spectre' => 'FireflyIII\Import\Routine\SpectreRoutine', + 'plaid' => 'FireflyIII\Import\Routine\PlaidRoutine', + ], + 'options' => [ 'file' => [ 'import_formats' => ['csv'], // mt940 diff --git a/public/js/ff/import/status.js b/public/js/ff/import/status.js index 02c3bbd073..d864c2cd14 100644 --- a/public/js/ff/import/status.js +++ b/public/js/ff/import/status.js @@ -18,7 +18,7 @@ * along with Firefly III. If not, see . */ -/** global: jobImportUrl, langImportSingleError, langImportMultiError, jobStartUrl, langImportTimeOutError, langImportFinished, langImportFatalError */ +/** global: langImportSingleError, langImportMultiError, jobStartUrl, langImportTimeOutError, langImportFinished, langImportFatalError */ var timeOutId; var startInterval = 1000; @@ -34,9 +34,10 @@ var knownErrors = 0; $(function () { "use strict"; - timeOutId = setTimeout(checkImportStatus, startInterval); + timeOutId = setTimeout(checkJobStatus, startInterval); + $('.start-job').click(startJob); - if(autoStart) { + if (job.configuration['auto-start']) { startJob(); } }); @@ -44,14 +45,14 @@ $(function () { /** * Downloads some JSON and responds to its content to see what the status is of the current import. */ -function checkImportStatus() { - $.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport); +function checkJobStatus() { + $.getJSON(jobStatusUri).done(reportOnJobStatus).fail(reportFailedJob); } /** * This method is called when the JSON query returns an error. If possible, this error is relayed to the user. */ -function failedJobImport(jqxhr, textStatus, error) { +function reportFailedJob(jqxhr, textStatus, error) { // hide all possible boxes: $('.statusbox').hide(); @@ -70,13 +71,15 @@ function failedJobImport(jqxhr, textStatus, error) { * * @param data */ -function reportOnJobImport(data) { +function reportOnJobStatus(data) { switch (data.status) { case "configured": // job is ready. Do not check again, just show the start-box. Hide the rest. - $('.statusbox').hide(); - $('.status_configured').show(); + if (!job.configuration['auto-start']) { + $('.statusbox').hide(); + $('.status_configured').show(); + } break; case "running": // job is running! Show the running box: @@ -97,7 +100,7 @@ function reportOnJobImport(data) { showStalledBox(); } else { // check again in 500ms - timeOutId = setTimeout(checkImportStatus, interval); + timeOutId = setTimeout(checkJobStatus, interval); } break; case "finished": @@ -106,6 +109,19 @@ function reportOnJobImport(data) { // show text: $('#import-status-more-info').html(data.finishedText); break; + case "errored": + // TODO this view is not yet used. + // hide all possible boxes: + $('.statusbox').hide(); + + // fill in some details: + var errorMessage = data.error_message; + + $('.fatal_error_txt').text(errorMessage); + + // show the fatal error box: + $('.fatal_error').show(); + break; default: break; @@ -147,13 +163,16 @@ function jobIsStalled(data) { function startJob() { // disable the button, add loading thing. $('.start-job').prop('disabled', true).text('...'); - $.post(jobStartUrl).fail(reportOnSubmitError); + $.post(jobStartUri).fail(reportOnSubmitError); // check status, every 500 ms. - timeOutId = setTimeout(checkImportStatus, startInterval); + timeOutId = setTimeout(checkJobStatus, startInterval); } -function reportOnSubmitError() { +/** + * When the start button fails (returns error code) this function reports. It assumes a time out. + */ +function reportOnSubmitError(jqxhr, textStatus, error) { // stop the refresh thing clearTimeout(timeOutId); @@ -161,7 +180,7 @@ function reportOnSubmitError() { $('.statusbox').hide(); // fill in some details: - var errorMessage = "Time out while waiting for job to finish."; + var errorMessage = "Submitting the job returned an error: " + textStatus + ' ' + error; $('.fatal_error_txt').text(errorMessage); diff --git a/resources/lang/en_US/firefly.php b/resources/lang/en_US/firefly.php index 9c4d3e3a49..ef97fb4956 100644 --- a/resources/lang/en_US/firefly.php +++ b/resources/lang/en_US/firefly.php @@ -1085,10 +1085,6 @@ return [ // import status page: 'import_status_bread_crumb' => 'Import status', 'import_status_sub_title' => 'Import status', - 'import_status_wait_title' => 'Please hold...', - 'import_status_wait_text' => 'This box will disappear in a moment.', - 'import_status_ready_title' => 'Import is ready to start', - 'import_status_ready_text' => 'The import is ready to start. All the configuration you needed to do has been done. Please download the configuration file. It will help you with the import should it not go as planned. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.', 'import_status_ready_config' => 'Download configuration', 'import_status_ready_start' => 'Start the import', 'import_status_ready_share' => 'Please consider downloading your configuration and sharing it at the import configuration center. This will allow other users of Firefly III to import their files more easily.', @@ -1097,9 +1093,6 @@ return [ 'import_status_errors_title' => 'Errors during the import', 'import_status_errors_single' => 'An error has occured during the import. It does not appear to be fatal.', 'import_status_errors_multi' => 'Some errors occured during the import. These do not appear to be fatal.', - 'import_status_fatal_title' => 'A fatal error occurred', - 'import_status_fatal_text' => 'A fatal error occurred, which the import-routine cannot recover from. Please see the explanation in red below.', - 'import_status_fatal_more' => 'If the error is a time-out, the import will have stopped half-way. For some server configurations, it is merely the server that stopped while the import keeps running in the background. To verify this, check out the log files. If the problem persists, consider importing over the command line instead.', 'import_status_finished_title' => 'Import routine finished', 'import_status_finished_text' => 'The import routine has imported your file.', 'import_status_finished_job' => 'The transactions imported can be found in tag :tag.', diff --git a/resources/lang/en_US/import.php b/resources/lang/en_US/import.php index d1135a4198..baf69809a6 100644 --- a/resources/lang/en_US/import.php +++ b/resources/lang/en_US/import.php @@ -17,6 +17,15 @@ return [ // // // import configuration routine: + // status of import: + 'status_wait_title' => 'Please hold...', + 'status_wait_text' => 'This box will disappear in a moment.', + 'status_fatal_title' => 'A fatal error occurred', + 'status_fatal_text' => 'A fatal error occurred, which the import-routine cannot recover from. Please see the explanation in red below.', + 'status_fatal_more' => 'If the error is a time-out, the import will have stopped half-way. For some server configurations, it is merely the server that stopped while the import keeps running in the background. To verify this, check out the log files. If the problem persists, consider importing over the command line instead.', + 'status_ready_title' => 'Import is ready to start', + 'status_ready_text' => 'The import is ready to start. All the configuration you needed to do has been done. Please download the configuration file. It will help you with the import should it not go as planned. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.', + 'status_ready_noconfig_text' => 'The import is ready to start. All the configuration you needed to do has been done. To actually run the import, you can either execute the following command in your console, or run the web-based import. Depending on your configuration, the console import will give you more feedback.', // file: upload something: 'file_upload_title' => 'Import setup (1/4) - Upload your file', @@ -61,12 +70,12 @@ return [ 'csv_roles_warning' => 'At the very least, mark one column as the amount-column. It is advisable to also select a column for the description, date and the opposing account.', - // map data + // file: map data 'file_map_title' => 'Import setup (4/4) - Connect import data to Firefly III data', 'file_map_text' => 'In the following tables, the left value shows you information found in your uploaded file. It is your task to map this value, if possible, to a value already present in your database. Firefly will stick to this mapping. If there is no value to map to, or you do not wish to map the specific value, select nothing.', 'file_map_field_value' => 'Field value', 'file_map_field_mapped_to' => 'Mapped to', - 'map_do_not_map' => '(do not map)', + 'map_do_not_map' => '(do not map)', 'file_map_submit' => 'Start the import', // map things. diff --git a/resources/views/import/status.twig b/resources/views/import/status.twig new file mode 100644 index 0000000000..2e69c94506 --- /dev/null +++ b/resources/views/import/status.twig @@ -0,0 +1,158 @@ +{% extends "./layout/default" %} + +{% block breadcrumbs %} + {{ Breadcrumbs.renderIfExists }} +{% endblock %} +{% block content %} + + {# Initial display. Will refresh (and disappear almost immediately. #} +
+
+
+
+

{{ trans('import.status_wait_title') }}

+
+
+

+ {{ trans('import.status_wait_text') }} +

+
+
+
+
+ + {# Fatal error display. Will be shown (duh) when something goes horribly wrong. #} + + + {# Box for when the job is ready to start #} + + + {# Box for when the job is running! #} + + + {# displays the finished status of the import #} + + + {# box to show error information. #} + + +{% endblock %} +{% block scripts %} + + +{% endblock %} +{% block styles %} +{% endblock %} diff --git a/routes/web.php b/routes/web.php index 42ca3e5acc..3daf730bc5 100755 --- a/routes/web.php +++ b/routes/web.php @@ -442,6 +442,14 @@ Route::group( // get status of any job: Route::get('status/{importJob}', ['uses' => 'Import\StatusController@index', 'as' => 'status']); + Route::get('json/{importJob}', ['uses' => 'Import\StatusController@json', 'as' => 'status.json']); + + // start a job + Route::any('start/{importJob}', ['uses' => 'Import\IndexController@start', 'as' => 'start']); + + // download config + Route::get('download/{importJob}', ['uses' => 'Import\IndexController@download', 'as' => 'download']); + // file import // Route::get('file', ['uses' => 'Import\FileController@index', 'as' => 'file.index']); @@ -449,7 +457,7 @@ Route::group( // Route::get('file/download/{importJob}', ['uses' => 'Import\FileController@download', 'as' => 'file.download']); // Route::get('file/status/{importJob}', ['uses' => 'Import\FileController@status', 'as' => 'file.status']); - // Route::get('file/json/{importJob}', ['uses' => 'Import\FileController@json', 'as' => 'file.json']); + // // Route::any('file/start/{importJob}', ['uses' => 'Import\FileController@start', 'as' => 'file.start']); // Route:: get('bank/{bank}/configure/{importJob}', ['uses' => 'Import\BankController@configure', 'as' => 'bank.configure']); // Route::post('bank/{bank}/configure/{importJob}', ['uses' => 'Import\BankController@postConfigure', 'as' => 'bank.configure.post']);