kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-10-12 15:35:15 +00:00
Code Issues Projects Releases Wiki Activity

Improved code for new import + some tests.

Browse Source
This commit is contained in:
James Cole
2018-05-04 20:21:27 +02:00
parent b541f7b944
commit 1c0da454db
22 changed files with 2770 additions and 2281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
app/Http/Controllers/Import/IndexController.php
View File

@@ -24,7 +24,6 @@ namespace FireflyIII\Http\Controllers\Import;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Controllers\Controller;
use FireflyIII\Http\Middleware\IsDemoUser;
use FireflyIII\Import\Prerequisites\PrerequisitesInterface;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use View;
@@ -54,7 +53,6 @@ class IndexController extends Controller
return $next($request);
}
);
$this->middleware(IsDemoUser::class)->except(['index']);
}
/**
@@ -68,11 +66,19 @@ class IndexController extends Controller
*/
public function create(string $importProvider)
{
if (
!(bool)config('app.debug')
&& !(bool)config(sprintf('import.enabled.%s', $importProvider)) === true
&& !\in_array(config('app.env'), ['demo', 'testing'])
) {
throw new FireflyException(sprintf('Import using provider "%s" is currently not available.', $importProvider)); // @codeCoverageIgnore
}
$importJob = $this->repository->create($importProvider);
// if job provider has no prerequisites:
if (!(bool)config(sprintf('import.has_prereq.%s', $importProvider))) {
// @codeCoverageIgnoreStart
// if job provider also has no configuration:
if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
$this->repository->updateStatus($importJob, 'ready_to_run');
@@ -85,6 +91,7 @@ class IndexController extends Controller
// redirect to job configuration.
return redirect(route('import.job.configuration.index', [$importJob->key]));
// @codeCoverageIgnoreEnd
}
// if need to set prerequisites, do that first.
@@ -122,7 +129,7 @@ class IndexController extends Controller
$config = config('import.enabled');
$providers = [];
foreach ($config as $name => $enabled) {
if ($enabled || (bool)config('app.debug')) {
if ($enabled || (bool)config('app.debug') || \in_array(config('app.env'), ['demo', 'testing'])) {
$providers[$name] = [];
}
}
@@ -147,110 +154,4 @@ class IndexController extends Controller
return view('import.index', compact('subTitle', 'subTitleIcon', 'providers'));
}
//
// /**
// * @param Request $request
// * @param string $bank
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
// */
// public function reset(Request $request, string $bank)
// {
// if ($bank === 'bunq') {
// // remove bunq related preferences.
// Preferences::delete('bunq_api_key');
// Preferences::delete('bunq_server_public_key');
// Preferences::delete('bunq_private_key');
// Preferences::delete('bunq_public_key');
// Preferences::delete('bunq_installation_token');
// Preferences::delete('bunq_installation_id');
// Preferences::delete('bunq_device_server_id');
// Preferences::delete('external_ip');
//
// }
//
// if ($bank === 'spectre') {
// // remove spectre related preferences:
// Preferences::delete('spectre_client_id');
// Preferences::delete('spectre_app_secret');
// Preferences::delete('spectre_service_secret');
// Preferences::delete('spectre_app_id');
// Preferences::delete('spectre_secret');
// Preferences::delete('spectre_private_key');
// Preferences::delete('spectre_public_key');
// Preferences::delete('spectre_customer');
// }
//
// Preferences::mark();
// $request->session()->flash('info', (string)trans('firefly.settings_reset_for_' . $bank));
//
// return redirect(route('import.index'));
//
// }
// /**
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\JsonResponse
// *
// * @throws FireflyException
// */
// public function start(ImportJob $job)
// {
// $type = $job->file_type;
// $key = sprintf('import.routine.%s', $type);
// $className = config($key);
// if (null === $className || !class_exists($className)) {
// throw new FireflyException(sprintf('Cannot find import routine class for job of type "%s".', $type)); // @codeCoverageIgnore
// }
//
// /** @var RoutineInterface $routine */
// $routine = app($className);
// $routine->setJob($job);
// $result = $routine->run();
//
// if ($result) {
// return response()->json(['run' => 'ok']);
// }
//
// throw new FireflyException('Job did not complete successfully. Please review the log files.');
// }
// /**
// * Generate a JSON file of the job's configuration and send it to the user.
// *
// * @param ImportJob $job
// *
// * @return LaravelResponse
// */
// public function download(ImportJob $job)
// {
// Log::debug('Now in download()', ['job' => $job->key]);
// $config = $job->configuration;
//
// // This is CSV import specific:
// $config['column-roles-complete'] = false;
// $config['column-mapping-complete'] = false;
// $config['initial-config-complete'] = false;
// $config['has-file-upload'] = false;
// $config['delimiter'] = "\t" === $config['delimiter'] ? 'tab' : $config['delimiter'];
// unset($config['stage']);
//
// $result = json_encode($config, JSON_PRETTY_PRINT);
// $name = sprintf('"%s"', addcslashes('import-configuration-' . date('Y-m-d') . '.json', '"\\'));
//
// /** @var LaravelResponse $response */
// $response = response($result, 200);
// $response->header('Content-disposition', 'attachment; filename=' . $name)
// ->header('Content-Type', 'application/json')
// ->header('Content-Description', 'File Transfer')
// ->header('Connection', 'Keep-Alive')
// ->header('Expires', '0')
// ->header('Cache-Control', 'must-revalidate, post-check=0, pre-check=0')
// ->header('Pragma', 'public')
// ->header('Content-Length', \strlen($result));
//
// return $response;
// }
}

16
app/Http/Controllers/Import/JobConfigurationController.php
View File

@@ -70,10 +70,10 @@ class JobConfigurationController extends Controller
public function index(ImportJob $importJob)
{
// catch impossible status:
$allowed = ['has_prereq', 'need_job_config', 'has_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
$allowed = ['has_prereq', 'need_job_config'];
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
session()->flash('error', trans('import.bad_job_status', ['status' => $importJob->status]));
return redirect(route('import.index'));
}
@@ -82,10 +82,12 @@ class JobConfigurationController extends Controller
// if provider has no config, just push it through:
$importProvider = $importJob->provider;
if (!(bool)config(sprintf('import.has_config.%s', $importProvider))) {
// @codeCoverageIgnoreStart
Log::debug('Job needs no config, is ready to run!');
$this->repository->updateStatus($importJob ,'ready_to_run');
$this->repository->updateStatus($importJob, 'ready_to_run');
return redirect(route('import.job.status.index', [$importJob->key]));
// @codeCoverageIgnoreEnd
}
// create configuration class:
@@ -120,10 +122,10 @@ class JobConfigurationController extends Controller
public function post(Request $request, ImportJob $importJob)
{
// catch impossible status:
$allowed = ['has_prereq', 'need_job_config', 'has_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
Log::error('Job is not new but wants to do prerequisites');
$allowed = ['has_prereq', 'need_job_config'];
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
session()->flash('error', trans('import.bad_job_status', ['status' => $importJob->status]));
return redirect(route('import.index'));
}

169
app/Http/Controllers/Import/JobStatusController.php
View File

@@ -32,7 +32,6 @@ use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Http\JsonResponse;
use Log;
use Symfony\Component\Debug\Exception\FatalThrowableError;
/**
* Class JobStatusController
@@ -68,19 +67,6 @@ class JobStatusController extends Controller
*/
public function index(ImportJob $importJob)
{
// jump away depending on job status:
if ($importJob->status === 'has_prereq') {
// TODO back to configuration.
}
if ($importJob->status === 'errored') {
// TODO to error screen
}
if ($importJob->status === 'finished') {
// TODO to finished screen.
}
$subTitleIcon = 'fa-gear';
$subTitle = trans('import.job_status_breadcrumb', ['key' => $importJob->key]);
@@ -94,42 +80,44 @@ class JobStatusController extends Controller
*/
public function json(ImportJob $importJob): JsonResponse
{
$extendedStatus = $importJob->extended_status;
$count = \count($importJob->transactions);
$json = [
$count = \count($importJob->transactions);
$json = [
'status' => $importJob->status,
'errors' => $importJob->errors,
'count' => $count,
'tag_id' => $importJob->tag_id,
'tag_name' => null === $importJob->tag_id ? null : $importJob->tag->tag,
'journals' => $extendedStatus['count'] ?? 0,
'report_txt' => trans('import.unknown_import_result'),
];
// if count is zero:
if (null !== $importJob->tag_id) {
$count = $importJob->tag->transactionJournals->count();
}
if ($count === 0) {
$json['report_txt'] = trans('import.result_no_transactions');
}
if ($count === 1 && null !== $importJob->tag_id) {
$json['report_txt'] = trans('import.result_one_transaction', ['route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag ]);
if ($count === 1) {
$json['report_txt'] = trans('import.result_one_transaction', ['route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag]);
}
if ($count > 1 && null !== $importJob->tag_id) {
$json['report_txt'] = trans('import.result_many_transactions', ['count' => $count,'route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag ]);
$json['report_txt'] = trans(
'import.result_many_transactions', ['count' => $count, 'route' => route('tags.show', [$importJob->tag_id]), 'tag' => $importJob->tag->tag]
);
}
return response()->json($json);
}
/**
* @param ImportJob $job
* @param ImportJob $importJob
*
* @return JsonResponse
* @throws FireflyException
*/
public function start(ImportJob $importJob): JsonResponse
{
// catch impossible status:
$allowed = ['ready_to_run', 'need_job_config'];
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
Log::error('Job is not ready.');
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "ready_to_run".']);
@@ -139,29 +127,13 @@ class JobStatusController extends Controller
$key = sprintf('import.routine.%s', $importProvider);
$className = config($key);
if (null === $className || !class_exists($className)) {
return response()->json(['status' => 'NOK', 'message' => sprintf('Cannot find import routine class for job of type "%s".', $importProvider)]);
// @codeCoverageIgnoreStart
return response()->json(
['status' => 'NOK', 'message' => sprintf('Cannot find import routine class for job of type "%s".', $importProvider)]
);
// @codeCoverageIgnoreEnd
}
// if the job is set to "provider_finished", we should be able to store transactions
// generated by the provider.
// otherwise, just continue.
if ($importJob->status === 'provider_finished') {
try {
$this->importFromJob($importJob);
} catch (FireflyException $e) {
$message = 'The import storage routine crashed: ' . $e->getMessage();
Log::error($message);
Log::error($e->getTraceAsString());
// set job errored out:
$this->repository->setStatus($importJob, 'error');
return response()->json(['status' => 'NOK', 'message' => $message]);
}
}
// set job to be running:
$this->repository->setStatus($importJob, 'running');
@@ -170,7 +142,7 @@ class JobStatusController extends Controller
$routine->setJob($importJob);
try {
$routine->run();
} catch (FireflyException $e) {
} catch (FireflyException|Exception $e) {
$message = 'The import routine crashed: ' . $e->getMessage();
Log::error($message);
Log::error($e->getTraceAsString());
@@ -186,16 +158,20 @@ class JobStatusController extends Controller
}
/**
* @param ImportJob $job
* Store does three things:
*
* - Store the transactions.
* - Add them to a tag.
*
* @param ImportJob $importJob
*
* @return JsonResponse
* @throws FireflyException
*/
public function store(ImportJob $importJob): JsonResponse
{
// catch impossible status:
$allowed = ['provider_finished', 'storing_data']; // todo remove storing data.
if (null !== $importJob && !in_array($importJob->status, $allowed)) {
$allowed = ['provider_finished', 'storing_data'];
if (null !== $importJob && !\in_array($importJob->status, $allowed, true)) {
Log::error('Job is not ready.');
return response()->json(['status' => 'NOK', 'message' => 'JobStatusController::start expects state "provider_finished".']);
@@ -205,7 +181,7 @@ class JobStatusController extends Controller
$this->repository->setStatus($importJob, 'storing_data');
try {
$this->importFromJob($importJob);
$this->storeTransactions($importJob);
} catch (FireflyException $e) {
$message = 'The import storage routine crashed: ' . $e->getMessage();
Log::error($message);
@@ -216,9 +192,9 @@ class JobStatusController extends Controller
return response()->json(['status' => 'NOK', 'message' => $message]);
}
// set storage to be finished:
$this->repository->setStatus($importJob, 'storage_finished');
// set job to be finished.
$this->repository->setStatus($importJob, 'finished');
// expect nothing from routine, just return OK to user.
return response()->json(['status' => 'OK', 'message' => 'storage_finished']);
@@ -229,90 +205,15 @@ class JobStatusController extends Controller
*
* @throws FireflyException
*/
private function importFromJob(ImportJob $importJob): void
private function storeTransactions(ImportJob $importJob): void
{
/** @var ImportArrayStorage $storage */
$storage = app(ImportArrayStorage::class);
$storage->setJob($importJob);
try {
$storage = new ImportArrayStorage($importJob);
$journals = $storage->store();
$extendedStatus = $importJob->extended_status;
$extendedStatus['count'] = $journals->count();
$this->repository->setExtendedStatus($importJob, $extendedStatus);
} catch (FireflyException|Exception|FatalThrowableError $e) {
$storage->store();
} catch (FireflyException|Exception $e) {
throw new FireflyException($e->getMessage());
}
}
// /**
// * @param ImportJob $job
// *
// * @return \Illuminate\Contracts\View\Factory|\Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|\Illuminate\View\View
// */
// public function index(ImportJob $job)
// {
// $statuses = ['configured', 'running', 'finished', 'error'];
// if (!\in_array($job->status, $statuses)) {
// return redirect(route('import.configure', [$job->key]));
// }
// $subTitle = trans('import.status_sub_title');
// $subTitleIcon = 'fa-star';
//
// return view('import.status', compact('job', 'subTitle', 'subTitleIcon'));
// }
//
// /**
// * Show status of import job in JSON.
// *
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\JsonResponse
// */
// public function json(ImportJob $job)
// {
// $result = [
// 'started' => false,
// 'finished' => false,
// 'running' => false,
// 'errors' => array_values($job->extended_status['errors']),
// 'percentage' => 0,
// 'show_percentage' => false,
// 'steps' => $job->extended_status['steps'],
// 'done' => $job->extended_status['done'],
// 'statusText' => trans('import.status_job_' . $job->status),
// 'status' => $job->status,
// 'finishedText' => '',
// ];
//
// if (0 !== $job->extended_status['steps']) {
// $result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0);
// $result['show_percentage'] = true;
// }
// if ('finished' === $job->status) {
// $result['finished'] = true;
// $tagId = (int)$job->extended_status['tag'];
// if ($tagId !== 0) {
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $tag = $repository->find($tagId);
// $count = $tag->transactionJournals()->count();
// $result['finishedText'] = trans(
// 'import.status_finished_job', ['count' => $count, 'link' => route('tags.show', [$tag->id, 'all']), 'tag' => $tag->tag]
// );
// }
//
// if ($tagId === 0) {
// $result['finishedText'] = trans('import.status_finished_no_tag'); // @codeCoverageIgnore
// }
// }
//
// if ('running' === $job->status) {
// $result['started'] = true;
// $result['running'] = true;
// }
// $result['percentage'] = $result['percentage'] > 100 ? 100 : $result['percentage'];
// Log::debug(sprintf('JOB STATUS: %d/%d', $result['done'], $result['steps']));
//
// return response()->json($result);
// }
}

2
app/Http/Controllers/Import/PrerequisitesController.php
View File

@@ -51,7 +51,6 @@ class PrerequisitesController extends Controller
function ($request, $next) {
app('view')->share('mainTitleIcon', 'fa-archive');
app('view')->share('title', trans('firefly.import_index_title'));
app('view')->share('subTitleIcon', 'fa-check');
$this->repository = app(ImportJobRepositoryInterface::class);
@@ -59,7 +58,6 @@ class PrerequisitesController extends Controller
return $next($request);
}
);
$this->middleware(IsDemoUser::class);
}
/**