kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-10-12 15:35:15 +00:00
Code Issues Projects Releases Wiki Activity

Lots of new code for new importer routine.

Browse Source
This commit is contained in:
James Cole
2017-06-10 15:09:41 +02:00
parent 0b4efe4ae1
commit 091596e80e
25 changed files with 1415 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

561
app/Http/Controllers/ImportController.php
View File

@@ -12,11 +12,10 @@ declare(strict_types=1);
namespace FireflyIII\Http\Controllers;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Requests\ImportUploadRequest;
use FireflyIII\Import\Configurator\ConfiguratorInterface;
use FireflyIII\Import\ImportProcedureInterface;
use FireflyIII\Import\Setup\SetupInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
@@ -25,10 +24,6 @@ use Illuminate\Http\Request;
use Illuminate\Http\Response as LaravelResponse;
use Log;
use Response;
use Session;
use SplFileObject;
use Storage;
use Symfony\Component\HttpFoundation\File\UploadedFile;
use View;
/**
@@ -54,30 +49,28 @@ class ImportController extends Controller
}
);
}
//
// /**
// * This is the last step before the import starts.
// *
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
// */
// public function complete(ImportJob $job)
// {
// Log::debug('Now in complete()', ['job' => $job->key]);
// if (!$this->jobInCorrectStep($job, 'complete')) {
// return $this->redirectToCorrectStep($job);
// }
// $subTitle = trans('firefly.import_complete');
// $subTitleIcon = 'fa-star';
//
// return view('import.complete', compact('job', 'subTitle', 'subTitleIcon'));
// }
/**
* This is the last step before the import starts.
*
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
*/
public function complete(ImportJob $job)
{
Log::debug('Now in complete()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'complete')) {
return $this->redirectToCorrectStep($job);
}
$subTitle = trans('firefly.import_complete');
$subTitleIcon = 'fa-star';
return view('import.complete', compact('job', 'subTitle', 'subTitleIcon'));
}
/**
* This is step 3.
* This is the first step in configuring the job. It can only be executed
* when the job is set to "import_status_never_started".
* This is step 3. This repeats until the job is configured.
*
* @param ImportJob $job
*
@@ -86,19 +79,19 @@ class ImportController extends Controller
*/
public function configure(ImportJob $job)
{
Log::debug('Now at start of configure()');
if (!$this->jobInCorrectStep($job, 'configure')) {
return $this->redirectToCorrectStep($job);
}
// create configuration class:
$configurator = $this->makeConfigurator($job);
// actual code
$importer = $this->makeImporter($job);
$importer->configure();
$data = $importer->getConfigurationData();
// is the job already configured?
if ($configurator->isJobConfigured()) {
return redirect(route('import.status', [$job->key]));
}
$view = $configurator->getNextView();
$data = $configurator->getNextData();
$subTitle = trans('firefly.configure_import');
$subTitleIcon = 'fa-wrench';
return view('import.' . $job->file_type . '.configure', compact('data', 'job', 'subTitle', 'subTitleIcon'));
return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
}
/**
@@ -134,25 +127,25 @@ class ImportController extends Controller
}
/**
* @param ImportJob $job
*
* @return View
*/
public function finished(ImportJob $job)
{
if (!$this->jobInCorrectStep($job, 'finished')) {
return $this->redirectToCorrectStep($job);
}
// if there is a tag (there might not be), we can link to it:
$tagId = $job->extended_status['importTag'] ?? 0;
$subTitle = trans('firefly.import_finished');
$subTitleIcon = 'fa-star';
return view('import.finished', compact('job', 'subTitle', 'subTitleIcon', 'tagId'));
}
// /**
// * @param ImportJob $job
// *
// * @return View
// */
// public function finished(ImportJob $job)
// {
// if (!$this->jobInCorrectStep($job, 'finished')) {
// return $this->redirectToCorrectStep($job);
// }
//
// // if there is a tag (there might not be), we can link to it:
// $tagId = $job->extended_status['importTag'] ?? 0;
//
// $subTitle = trans('firefly.import_finished');
// $subTitleIcon = 'fa-star';
//
// return view('import.finished', compact('job', 'subTitle', 'subTitleIcon', 'tagId'));
// }
/**
* This is step 1. Upload a file.
@@ -161,7 +154,6 @@ class ImportController extends Controller
*/
public function index()
{
Log::debug('Now at index');
$subTitle = trans('firefly.import_data_index');
$subTitleIcon = 'fa-home';
$importFileTypes = [];
@@ -174,6 +166,35 @@ class ImportController extends Controller
return view('import.index', compact('subTitle', 'subTitleIcon', 'importFileTypes', 'defaultImportType'));
}
/**
* This is step 2. It creates an Import Job. Stores the import.
*
* @param ImportUploadRequest $request
* @param ImportJobRepositoryInterface $repository
* @param UserRepositoryInterface $userRepository
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function initialize(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository)
{
Log::debug('Now in initialize()');
// create import job:
$type = $request->get('import_file_type');
$job = $repository->create($type);
Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]);
// process file:
$repository->processFile($job, $request->files->get('import_file'));
// process config, if present:
if ($request->files->has('configuration_file')) {
$repository->processConfiguration($job, $request->files->get('configuration_file'));
}
return redirect(route('import.configure', [$job->key]));
}
/**
* @param ImportJob $job
*
@@ -182,22 +203,22 @@ class ImportController extends Controller
public function json(ImportJob $job)
{
$result = [
'showPercentage' => false,
'started' => false,
'finished' => false,
'running' => false,
'errors' => $job->extended_status['errors'],
'percentage' => 0,
'steps' => $job->extended_status['total_steps'],
'stepsDone' => $job->extended_status['steps_done'],
'statusText' => trans('firefly.import_status_' . $job->status),
'finishedText' => '',
'started' => false,
'finished' => false,
'running' => false,
'errors' => $job->extended_status['errors'],
'percentage' => 0,
'steps' => $job->extended_status['total_steps'],
'stepsDone' => $job->extended_status['steps_done'],
'statusText' => trans('firefly.import_status_' . $job->status),
'status' => $job->status,
'finishedText' => '',
];
$percentage = 0;
if ($job->extended_status['total_steps'] !== 0) {
$percentage = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0);
}
if ($job->status === 'import_complete') {
if ($job->status === 'complete') {
$tagId = $job->extended_status['importTag'];
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
@@ -206,11 +227,10 @@ class ImportController extends Controller
$result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]);
}
if ($job->status === 'import_running') {
$result['started'] = true;
$result['running'] = true;
$result['showPercentage'] = true;
$result['percentage'] = $percentage;
if ($job->status === 'running') {
$result['started'] = true;
$result['running'] = true;
$result['percentage'] = $percentage;
}
return Response::json($result);
@@ -228,87 +248,81 @@ class ImportController extends Controller
public function postConfigure(Request $request, ImportJobRepositoryInterface $repository, ImportJob $job)
{
Log::debug('Now in postConfigure()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'process')) {
return $this->redirectToCorrectStep($job);
$configurator = $this->makeConfigurator($job);
// is the job already configured?
if ($configurator->isJobConfigured()) {
return redirect(route('import.status', [$job->key]));
}
Log::debug('Continue postConfigure()', ['job' => $job->key]);
$data = $request->all();
$configurator->configureJob($data);
// actual code
$importer = $this->makeImporter($job);
$data = $request->all();
$files = $request->files;
$importer->saveImportConfiguration($data, $files);
// update job:
$repository->updateStatus($job, 'import_configuration_saved');
// return redirect to settings.
// this could loop until the user is done.
return redirect(route('import.settings', [$job->key]));
// return to configure
return redirect(route('import.configure', [$job->key]));
}
/**
* This step 6. Depending on the importer, this will process the
* settings given and store them.
*
* @param Request $request
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
* @throws FireflyException
*/
public function postSettings(Request $request, ImportJob $job)
{
Log::debug('Now in postSettings()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'store-settings')) {
return $this->redirectToCorrectStep($job);
}
$importer = $this->makeImporter($job);
$importer->storeSettings($request);
// /**
// * This step 6. Depending on the importer, this will process the
// * settings given and store them.
// *
// * @param Request $request
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
// * @throws FireflyException
// */
// public function postSettings(Request $request, ImportJob $job)
// {
// Log::debug('Now in postSettings()', ['job' => $job->key]);
// if (!$this->jobInCorrectStep($job, 'store-settings')) {
// return $this->redirectToCorrectStep($job);
// }
// $importer = $this->makeImporter($job);
// $importer->storeSettings($request);
//
// // return redirect to settings (for more settings perhaps)
// return redirect(route('import.settings', [$job->key]));
// }
// return redirect to settings (for more settings perhaps)
return redirect(route('import.settings', [$job->key]));
}
/**
* Step 5. Depending on the importer, this will show the user settings to
* fill in.
*
* @param ImportJobRepositoryInterface $repository
* @param ImportJob $job
*
* @return View
*/
public function settings(ImportJobRepositoryInterface $repository, ImportJob $job)
{
Log::debug('Now in settings()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'settings')) {
return $this->redirectToCorrectStep($job);
}
Log::debug('Continue in settings()');
$importer = $this->makeImporter($job);
$subTitle = trans('firefly.settings_for_import');
$subTitleIcon = 'fa-wrench';
// now show settings screen to user.
if ($importer->requireUserSettings()) {
Log::debug('Job requires user config.');
$data = $importer->getDataForSettings();
$view = $importer->getViewForSettings();
return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
}
Log::debug('Job does NOT require user config.');
$repository->updateStatus($job, 'settings_complete');
// if no more settings, save job and continue to process thing.
return redirect(route('import.complete', [$job->key]));
// ask the importer for the requested action.
// for example pick columns or map data.
// depends of course on the data in the job.
}
// /**
// * Step 5. Depending on the importer, this will show the user settings to
// * fill in.
// *
// * @param ImportJobRepositoryInterface $repository
// * @param ImportJob $job
// *
// * @return View
// */
// public function settings(ImportJobRepositoryInterface $repository, ImportJob $job)
// {
// Log::debug('Now in settings()', ['job' => $job->key]);
// if (!$this->jobInCorrectStep($job, 'settings')) {
// return $this->redirectToCorrectStep($job);
// }
// Log::debug('Continue in settings()');
// $importer = $this->makeImporter($job);
// $subTitle = trans('firefly.settings_for_import');
// $subTitleIcon = 'fa-wrench';
//
// // now show settings screen to user.
// if ($importer->requireUserSettings()) {
// Log::debug('Job requires user config.');
// $data = $importer->getDataForSettings();
// $view = $importer->getViewForSettings();
//
// return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
// }
// Log::debug('Job does NOT require user config.');
//
// $repository->updateStatus($job, 'settings_complete');
//
// // if no more settings, save job and continue to process thing.
// return redirect(route('import.complete', [$job->key]));
//
// // ask the importer for the requested action.
// // for example pick columns or map data.
// // depends of course on the data in the job.
// }
/**
* @param ImportProcedureInterface $importProcedure
@@ -316,6 +330,7 @@ class ImportController extends Controller
*/
public function start(ImportProcedureInterface $importProcedure, ImportJob $job)
{
die('TODO here.');
set_time_limit(0);
if ($job->status == 'settings_complete') {
$importProcedure->runImport($job);
@@ -330,175 +345,117 @@ class ImportController extends Controller
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
*/
public function status(ImportJob $job)
{ //
Log::debug('Now in status()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'status')) {
return $this->redirectToCorrectStep($job);
}
{
$subTitle = trans('firefly.import_status');
$subTitleIcon = 'fa-star';
return view('import.status', compact('job', 'subTitle', 'subTitleIcon'));
}
/**
* This is step 2. It creates an Import Job. Stores the import.
*
* @param ImportUploadRequest $request
* @param ImportJobRepositoryInterface $repository
* @param UserRepositoryInterface $userRepository
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function upload(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository)
{
Log::debug('Now in upload()');
// create import job:
$type = $request->get('import_file_type');
$job = $repository->create($type);
Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]);
/** @var UploadedFile $upload */
$upload = $request->files->get('import_file');
$newName = $job->key . '.upload';
$uploaded = new SplFileObject($upload->getRealPath());
$content = $uploaded->fread($uploaded->getSize());
$contentEncrypted = Crypt::encrypt($content);
$disk = Storage::disk('upload');
// user is demo user, replace upload with prepared file.
if ($userRepository->hasRole(auth()->user(), 'demo')) {
$stubsDisk = Storage::disk('stubs');
$content = $stubsDisk->get('demo-import.csv');
$contentEncrypted = Crypt::encrypt($content);
$disk->put($newName, $contentEncrypted);
Log::debug('Replaced upload with demo file.');
// also set up prepared configuration.
$configuration = json_decode($stubsDisk->get('demo-configuration.json'), true);
$repository->setConfiguration($job, $configuration);
Log::debug('Set configuration for demo user', $configuration);
// also flash info
Session::flash('info', trans('demo.import-configure-security'));
}
if (!$userRepository->hasRole(auth()->user(), 'demo')) {
// user is not demo, process original upload:
$disk->put($newName, $contentEncrypted);
Log::debug('Uploaded file', ['name' => $upload->getClientOriginalName(), 'size' => $upload->getSize(), 'mime' => $upload->getClientMimeType()]);
}
// store configuration file's content into the job's configuration thing. Otherwise, leave it empty.
// demo user's configuration upload is ignored completely.
if ($request->files->has('configuration_file') && !auth()->user()->hasRole('demo')) {
/** @var UploadedFile $configFile */
$configFile = $request->files->get('configuration_file');
Log::debug(
'Uploaded configuration file',
['name' => $configFile->getClientOriginalName(), 'size' => $configFile->getSize(), 'mime' => $configFile->getClientMimeType()]
);
$configFileObject = new SplFileObject($configFile->getRealPath());
$configRaw = $configFileObject->fread($configFileObject->getSize());
$configuration = json_decode($configRaw, true);
// @codeCoverageIgnoreStart
if (!is_null($configuration) && is_array($configuration)) {
Log::debug('Found configuration', $configuration);
$repository->setConfiguration($job, $configuration);
}
// @codeCoverageIgnoreEnd
}
return redirect(route('import.configure', [$job->key]));
}
/**
* @param ImportJob $job
* @param string $method
*
* @return bool
*/
private function jobInCorrectStep(ImportJob $job, string $method): bool
{
Log::debug('Now in jobInCorrectStep()', ['job' => $job->key, 'method' => $method]);
switch ($method) {
case 'configure':
case 'process':
return $job->status === 'import_status_never_started';
case 'settings':
case 'store-settings':
Log::debug(sprintf('Job %d with key %s has status %s', $job->id, $job->key, $job->status));
return $job->status === 'import_configuration_saved';
case 'finished':
return $job->status === 'import_complete';
case 'complete':
return $job->status === 'settings_complete';
case 'status':
return ($job->status === 'settings_complete') || ($job->status === 'import_running');
}
return false; // @codeCoverageIgnore
}
// /**
// * @param ImportJob $job
// * @param string $method
// *
// * @return bool
// */
// private function jobInCorrectStep(ImportJob $job, string $method): bool
// {
// Log::debug('Now in jobInCorrectStep()', ['job' => $job->key, 'method' => $method]);
// switch ($method) {
// case 'configure':
// case 'process':
// return $job->status === 'import_status_never_started';
// case 'settings':
// case 'store-settings':
// Log::debug(sprintf('Job %d with key %s has status %s', $job->id, $job->key, $job->status));
//
// return $job->status === 'import_configuration_saved';
// case 'finished':
// return $job->status === 'import_complete';
// case 'complete':
// return $job->status === 'settings_complete';
// case 'status':
// return ($job->status === 'settings_complete') || ($job->status === 'import_running');
// }
//
// return false; // @codeCoverageIgnore
//
// }
/**
* @param ImportJob $job
*
* @return SetupInterface
* @return ConfiguratorInterface
* @throws FireflyException
*/
private function makeImporter(ImportJob $job): SetupInterface
private function makeConfigurator(ImportJob $job): ConfiguratorInterface
{
// create proper importer (depends on job)
$type = strtolower($job->file_type);
// validate type:
$validTypes = array_keys(config('firefly.import_formats'));
if (in_array($type, $validTypes)) {
/** @var SetupInterface $importer */
$importer = app('FireflyIII\Import\Setup\\' . ucfirst($type) . 'Setup');
$importer->setJob($job);
return $importer;
$type = $job->file_type;
$key = sprintf('firefly.import_configurators.%s', $type);
$className = config($key);
if (is_null($className)) {
throw new FireflyException('Cannot find configurator class for this job.');
}
throw new FireflyException(sprintf('"%s" is not a valid file type', $type)); // @codeCoverageIgnore
$configurator = new $className($job);
return $configurator;
}
/**
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
* @throws FireflyException
*/
private function redirectToCorrectStep(ImportJob $job)
{
Log::debug('Now in redirectToCorrectStep()', ['job' => $job->key]);
switch ($job->status) {
case 'import_status_never_started':
Log::debug('Will redirect to configure()');
// /**
// * @param ImportJob $job
// *
// * @return SetupInterface
// * @throws FireflyException
// */
// private function makeImporter(ImportJob $job): SetupInterface
// {
// // create proper importer (depends on job)
// $type = strtolower($job->file_type);
//
// // validate type:
// $validTypes = array_keys(config('firefly.import_formats'));
//
//
// if (in_array($type, $validTypes)) {
// /** @var SetupInterface $importer */
// $importer = app('FireflyIII\Import\Setup\\' . ucfirst($type) . 'Setup');
// $importer->setJob($job);
//
// return $importer;
// }
// throw new FireflyException(sprintf('"%s" is not a valid file type', $type)); // @codeCoverageIgnore
//
// }
return redirect(route('import.configure', [$job->key]));
case 'import_configuration_saved':
Log::debug('Will redirect to settings()');
return redirect(route('import.settings', [$job->key]));
case 'settings_complete':
Log::debug('Will redirect to complete()');
return redirect(route('import.complete', [$job->key]));
case 'import_complete':
Log::debug('Will redirect to finished()');
return redirect(route('import.finished', [$job->key]));
}
throw new FireflyException('Cannot redirect for job state ' . $job->status); // @codeCoverageIgnore
}
// /**
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
// * @throws FireflyException
// */
// private function redirectToCorrectStep(ImportJob $job)
// {
// Log::debug('Now in redirectToCorrectStep()', ['job' => $job->key]);
// switch ($job->status) {
// case 'import_status_never_started':
// Log::debug('Will redirect to configure()');
//
// return redirect(route('import.configure', [$job->key]));
// case 'import_configuration_saved':
// Log::debug('Will redirect to settings()');
//
// return redirect(route('import.settings', [$job->key]));
// case 'settings_complete':
// Log::debug('Will redirect to complete()');
//
// return redirect(route('import.complete', [$job->key]));
// case 'import_complete':
// Log::debug('Will redirect to finished()');
//
// return redirect(route('import.finished', [$job->key]));
// }
//
// throw new FireflyException('Cannot redirect for job state ' . $job->status); // @codeCoverageIgnore
//
// }
}

5
app/Http/Requests/ImportUploadRequest.php
View File

@@ -38,8 +38,9 @@ class ImportUploadRequest extends Request
$types = array_keys(config('firefly.import_formats'));
return [
'import_file' => 'required|file',
'import_file_type' => 'required|in:' . join(',', $types),
'import_file' => 'required|file',
'import_file_type' => 'required|in:' . join(',', $types),
'configuration_file' => 'file',
];
}

60
app/Import/Configurator/ConfiguratorInterface.php Normal file
View File

@@ -0,0 +1,60 @@
<?php
/**
* ConfiguratorInterface.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Configurator;
use FireflyIII\Models\ImportJob;
/**
* Interface ConfiguratorInterface
*
* @package FireflyIII\Import\Configurator
*/
interface ConfiguratorInterface
{
/**
* ConfiguratorInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job);
/**
* Store any data from the $data array into the job.
*
* @param array $data
*
* @return bool
*/
public function configureJob(array $data): bool;
/**
* Return the data required for the next step in the job configuration.
*
* @return array
*/
public function getNextData(): array;
/**
* Returns the view of the next step in the job configuration.
*
* @return string
*/
public function getNextView(): string;
/**
* Returns true when the initial configuration for this job is complete.
*
* @return bool
*/
public function isJobConfigured(): bool;
}

140
app/Import/Configurator/CsvConfigurator.php Normal file
View File

@@ -0,0 +1,140 @@
<?php
/**
* CsvConfigurator.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Configurator;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use Log;
/**
* Class CsvConfigurator
*
* @package FireflyIII\Import\Configurator
*/
class CsvConfigurator implements ConfiguratorInterface
{
private $job;
public function __construct(ImportJob $job)
{
$this->job = $job;
if (is_null($this->job->configuration) || count($this->job->configuration) === 0) {
Log::debug(sprintf('Gave import job %s initial configuration.', $this->job->key));
$this->job->configuration = config('csv.default_config');
$this->job->save();
}
}
/**
* Store any data from the $data array into the job.
*
* @param array $data
*
* @return bool
* @throws FireflyException
*/
public function configureJob(array $data): bool
{
$class = $this->getConfigurationClass();
/** @var ConfigurationInterface $object */
$object = new $class($this->job);
return $object->storeConfiguration($data);
}
/**
* Return the data required for the next step in the job configuration.
*
* @return array
* @throws FireflyException
*/
public function getNextData(): array
{
$class = $this->getConfigurationClass();
/** @var ConfigurationInterface $object */
$object = new $class($this->job);
return $object->getData();
}
/**
* @return string
* @throws FireflyException
*/
public function getNextView(): string
{
if (!$this->job->configuration['initial-config-complete']) {
return 'import.csv.initial';
}
if (!$this->job->configuration['column-roles-complete']) {
return 'import.csv.roles';
}
if (!$this->job->configuration['column-mapping-complete']) {
return 'import.csv.map';
}
throw new FireflyException('No view for state');
}
/**
* @return bool
*/
public function isJobConfigured(): bool
{
if ($this->job->configuration['initial-config-complete']
&& $this->job->configuration['column-roles-complete']
&& $this->job->configuration['column-mapping-complete']
) {
$this->job->status = 'configured';
$this->job->save();
return true;
}
return false;
}
/**
* @return string
* @throws FireflyException
*/
private function getConfigurationClass(): string
{
$class = false;
switch (true) {
case(!$this->job->configuration['initial-config-complete']):
$class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Initial';
break;
case (!$this->job->configuration['column-roles-complete']):
$class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Roles';
break;
case (!$this->job->configuration['column-mapping-complete']):
$class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Map';
break;
default:
break;
}
if ($class === false) {
throw new FireflyException('Cannot handle current job state in getConfigurationClass().');
}
if (!class_exists($class)) {
throw new FireflyException(sprintf('Class %s does not exist in getConfigurationClass().', $class));
}
return $class;
}
}

0
app/Import/Importer/CsvImporter.php → app/Import/Importer/OldCsvImporter.php
View File

0
app/Import/Importer/ImporterInterface.php → app/Import/Importer/OldImporterInterface.php
View File

1
app/Import/Setup/CsvSetup.php
View File

@@ -237,7 +237,6 @@ class CsvSetup implements SetupInterface
$all = $request->all();
if ($request->get('settings') == 'roles') {
$count = $config['column-count'];
$roleSet = 0; // how many roles have been defined
$mapSet = 0; // how many columns must be mapped
for ($i = 0; $i < $count; $i++) {

8
app/Models/ImportJob.php
View File

@@ -42,11 +42,9 @@ class ImportJob extends Model
protected $validStatus
= [
'import_status_never_started', // initial state
'import_configuration_saved', // import configuration saved. This step is going to be obsolete.
'settings_complete', // aka: ready for import.
'import_running', // import currently underway
'import_complete', // done with everything
'new',
'initialized',
'configured',
];
/**

79
app/Repositories/ImportJob/ImportJobRepository.php
View File

@@ -13,10 +13,16 @@ declare(strict_types=1);
namespace FireflyIII\Repositories\ImportJob;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use FireflyIII\User;
use Illuminate\Support\Str;
use Log;
use SplFileObject;
use Storage;
use Symfony\Component\HttpFoundation\File\UploadedFile;
/**
* Class ImportJobRepository
@@ -51,7 +57,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$importJob->user()->associate($this->user);
$importJob->file_type = $fileType;
$importJob->key = Str::random(12);
$importJob->status = 'import_status_never_started';
$importJob->status = 'new';
$importJob->extended_status = [
'total_steps' => 0,
'steps_done' => 0,
@@ -86,6 +92,77 @@ class ImportJobRepository implements ImportJobRepositoryInterface
return $result;
}
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return bool
*/
public function processConfiguration(ImportJob $job, UploadedFile $file): bool
{
/** @var UserRepositoryInterface $repository */
$repository = app(UserRepositoryInterface::class);
// demo user's configuration upload is ignored completely.
if ($repository->hasRole($this->user, 'demo')) {
Log::debug(
'Uploaded configuration file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]
);
$configFileObject = new SplFileObject($file->getRealPath());
$configRaw = $configFileObject->fread($configFileObject->getSize());
$configuration = json_decode($configRaw, true);
if (!is_null($configuration) && is_array($configuration)) {
Log::debug('Found configuration', $configuration);
$this->setConfiguration($job, $configuration);
}
}
return true;
}
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return mixed
*/
public function processFile(ImportJob $job, UploadedFile $file): bool
{
/** @var UserRepositoryInterface $repository */
$repository = app(UserRepositoryInterface::class);
$newName = sprintf('%s.upload', $job->key);
$uploaded = new SplFileObject($file->getRealPath());
$content = $uploaded->fread($uploaded->getSize());
$contentEncrypted = Crypt::encrypt($content);
$disk = Storage::disk('upload');
// user is demo user, replace upload with prepared file.
if ($repository->hasRole($this->user, 'demo')) {
$stubsDisk = Storage::disk('stubs');
$content = $stubsDisk->get('demo-import.csv');
$contentEncrypted = Crypt::encrypt($content);
$disk->put($newName, $contentEncrypted);
Log::debug('Replaced upload with demo file.');
// also set up prepared configuration.
$configuration = json_decode($stubsDisk->get('demo-configuration.json'), true);
$this->setConfiguration($job, $configuration);
Log::debug('Set configuration for demo user', $configuration);
}
if (!$repository->hasRole($this->user, 'demo')) {
// user is not demo, process original upload:
$disk->put($newName, $contentEncrypted);
Log::debug('Uploaded file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]);
}
$job->status = 'initialized';
$job->save();
return true;
}
/**
* @param ImportJob $job
* @param array $configuration

17
app/Repositories/ImportJob/ImportJobRepositoryInterface.php
View File

@@ -15,6 +15,7 @@ namespace FireflyIII\Repositories\ImportJob;
use FireflyIII\Models\ImportJob;
use FireflyIII\User;
use Symfony\Component\HttpFoundation\File\UploadedFile;
/**
* Interface ImportJobRepositoryInterface
@@ -37,6 +38,22 @@ interface ImportJobRepositoryInterface
*/
public function findByKey(string $key): ImportJob;
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return mixed
*/
public function processFile(ImportJob $job, UploadedFile $file): bool;
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return bool
*/
public function processConfiguration(ImportJob $job, UploadedFile $file): bool;
/**
* @param ImportJob $job
* @param array $configuration

46
app/Support/Import/Configuration/ConfigurationInterface.php Normal file
View File

@@ -0,0 +1,46 @@
<?php
/**
* ConfigurationInterface.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration;
use FireflyIII\Models\ImportJob;
/**
* Class ConfigurationInterface
*
* @package FireflyIII\Support\Import\Configuration
*/
interface ConfigurationInterface
{
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job);
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getData(): array;
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool;
}

122
app/Support/Import/Configuration/Csv/Initial.php Normal file
View File

@@ -0,0 +1,122 @@
<?php
/**
* CsvInitial.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\Csv;
use ExpandedForm;
use FireflyIII\Models\AccountType;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use Log;
/**
* Class CsvInitial
*
* @package FireflyIII\Support\Import\Configuration
*/
class Initial implements ConfigurationInterface
{
private $job;
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job)
{
$this->job = $job;
}
/**
* @return array
*/
public function getData(): array
{
/** @var AccountRepositoryInterface $accountRepository */
$accountRepository = app(AccountRepositoryInterface::class);
$accounts = $accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]);
$delimiters = [
',' => trans('form.csv_comma'),
';' => trans('form.csv_semicolon'),
'tab' => trans('form.csv_tab'),
];
$specifics = [];
// collect specifics.
foreach (config('csv.import_specifics') as $name => $className) {
$specifics[$name] = [
'name' => $className::getName(),
'description' => $className::getDescription(),
];
}
$data = [
'accounts' => ExpandedForm::makeSelectList($accounts),
'specifix' => [],
'delimiters' => $delimiters,
'specifics' => $specifics,
];
return $data;
}
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool
{
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$importId = $data['csv_import_account'] ?? 0;
$account = $repository->find(intval($importId));
$hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false;
$config = $this->job->configuration;
$config['initial-config-complete'] = true;
$config['has-headers'] = $hasHeaders;
$config['date-format'] = $data['date_format'];
$config['delimiter'] = $data['csv_delimiter'];
$config['delimiter'] = $config['delimiter'] === 'tab' ? "\t" : $config['delimiter'];
Log::debug('Entered import account.', ['id' => $importId]);
if (!is_null($account->id)) {
Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]);
$config['import-account'] = $account->id;
}
if (is_null($account->id)) {
Log::error('Could not find anything for csv_import_account.', ['id' => $importId]);
}
// loop specifics.
if (isset($data['specifics']) && is_array($data['specifics'])) {
foreach ($data['specifics'] as $name => $enabled) {
// verify their content.
$className = sprintf('FireflyIII\Import\Specifics\%s', $name);
if (class_exists($className)) {
$config['specifics'][$name] = 1;
}
}
}
$this->job->configuration = $config;
$this->job->save();
return true;
}
}

229
app/Support/Import/Configuration/Csv/Map.php Normal file
View File

@@ -0,0 +1,229 @@
<?php
/**
* Mapping.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\Csv;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\Mapper\MapperInterface;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use League\Csv\Reader;
use Log;
/**
* Class Mapping
*
* @package FireflyIII\Support\Import\Configuration\Csv
*/
class Map implements ConfigurationInterface
{
/** @var array that holds each column to be mapped by the user */
private $data = [];
/** @var array that holds the indexes of those columns (ie. 2, 5, 8) */
private $indexes = [];
/** @var ImportJob */
private $job;
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job)
{
$this->job = $job;
}
/**
* @return array
* @throws FireflyException
*/
public function getData(): array
{
$config = $this->job->configuration;
$this->getMappableColumns();
// in order to actually map we also need all possible values from the CSV file.
$content = $this->job->uploadFileContents();
/** @var Reader $reader */
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$results = $reader->fetch();
$validSpecifics = array_keys(config('csv.import_specifics'));
foreach ($results as $rowIndex => $row) {
// skip first row?
if ($rowIndex === 0 && $config['has-headers']) {
continue;
}
// run specifics here:
// and this is the point where the specifix go to work.
foreach ($config['specifics'] as $name => $enabled) {
if (!in_array($name, $validSpecifics)) {
throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
}
$class = config('csv.import_specifics.' . $name);
/** @var SpecificInterface $specific */
$specific = app($class);
// it returns the row, possibly modified:
$row = $specific->run($row);
}
//do something here
foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
if (!isset($row[$index])) {
// don't really know how to handle this. Just skip, for now.
continue;
}
$value = $row[$index];
if (strlen($value) > 0) {
// we can do some preprocessing here,
// which is exclusively to fix the tags:
if (!is_null($data[$index]['preProcessMap'])) {
/** @var PreProcessorInterface $preProcessor */
$preProcessor = app($data[$index]['preProcessMap']);
$result = $preProcessor->run($value);
$data[$index]['values'] = array_merge($data[$index]['values'], $result);
Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $data[$index]['values']]);
continue;
}
$data[$index]['values'][] = $value;
}
}
}
foreach ($data as $index => $entry) {
$data[$index]['values'] = array_unique($data[$index]['values']);
}
return $data;
}
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool
{
return true;
}
/**
* @param string $column
*
* @return MapperInterface
*/
private function createMapper(string $column): MapperInterface
{
$mapperClass = config('csv.import_roles.' . $column . '.mapper');
$mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass);
/** @var MapperInterface $mapper */
$mapper = new $mapperName;
return $mapper;
}
/**
* @return bool
*/
private function getMappableColumns(): bool
{
$config = $this->job->configuration;
/**
* @var int $index
* @var bool $mustBeMapped
*/
foreach ($config['column-do-mapping'] as $index => $mustBeMapped) {
$column = $this->validateColumnName($config['column-roles'][$index] ?? '_ignore');
$shouldMap = $this->shouldMapColumn($column, $mustBeMapped);
if ($shouldMap) {
// create configuration entry for this specific column and add column to $this->data array for later processing.
$this->data[$index] = [
'name' => $column,
'index' => $index,
'options' => $this->createMapper($column)->getMap(),
'preProcessMap' => $this->getPreProcessorName($column),
'values' => [],
];
}
}
return true;
}
/**
* @param string $column
*
* @return string
*/
private function getPreProcessorName(string $column): string
{
$name = '';
$hasPreProcess = config('csv.import_roles.' . $column . '.pre-process-map');
$preProcessClass = config('csv.import_roles.' . $column . '.pre-process-mapper');
if (!is_null($hasPreProcess) && $hasPreProcess === true && !is_null($preProcessClass)) {
$name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
}
return $name;
}
/**
* @param string $column
* @param bool $mustBeMapped
*
* @return bool
*/
private function shouldMapColumn(string $column, bool $mustBeMapped): bool
{
$canBeMapped = config('csv.import_roles.' . $column . '.mappable');
return ($canBeMapped && $mustBeMapped);
}
/**
* @param string $column
*
* @return string
* @throws FireflyException
*/
private function validateColumnName(string $column): string
{
// is valid column?
$validColumns = array_keys(config('csv.import_roles'));
if (!in_array($column, $validColumns)) {
throw new FireflyException(sprintf('"%s" is not a valid column.', $column));
}
return $column;
}
}

261
app/Support/Import/Configuration/Csv/Roles.php Normal file
View File

@@ -0,0 +1,261 @@
<?php
/**
* Roles.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\Csv;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use League\Csv\Reader;
use Log;
/**
* Class Roles
*
* @package FireflyIII\Support\Import\Configuration\Csv
*/
class Roles implements ConfigurationInterface
{
private $data = [];
/** @var ImportJob */
private $job;
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job)
{
$this->job = $job;
}
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getData(): array
{
$config = $this->job->configuration;
$content = $this->job->uploadFileContents();
// create CSV reader.
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$start = $config['has-headers'] ? 1 : 0;
$end = $start + config('csv.example_rows');
// set data:
$this->data = [
'examples' => [],
'roles' => $this->getRoles(),
'total' => 0,
'headers' => $config['has-headers'] ? $reader->fetchOne(0) : [],
];
while ($start < $end) {
$row = $reader->fetchOne($start);
$row = $this->processSpecifics($row);
$count = count($row);
$this->data['total'] = $count > $this->data['total'] ? $count : $this->data['total'];
$this->processRow($row);
$start++;
}
$this->updateColumCount();
$this->makeExamplesUnique();
return $this->data;
}
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool
{
Log::debug('Now in storeConfiguration of Roles.');
$config = $this->job->configuration;
$count = $config['column-count'];
for ($i = 0; $i < $count; $i++) {
$role = $data['role'][$i] ?? '_ignore';
$mapping = isset($data['map'][$i]) && $data['map'][$i] === '1' ? true : false;
$config['column-roles'][$i] = $role;
$config['column-do-mapping'][$i] = $mapping;
Log::debug(sprintf('Column %d has been given role %s', $i, $role));
}
$this->job->configuration = $config;
$this->job->save();
$this->ignoreUnmappableColumns();
$this->setRolesComplete();
$this->isMappingNecessary();
return true;
}
/**
* @return array
*/
private function getRoles(): array
{
$roles = [];
foreach (array_keys(config('csv.import_roles')) as $role) {
$roles[$role] = trans('csv.column_' . $role);
}
return $roles;
}
/**
* @return bool
*/
private function ignoreUnmappableColumns(): bool
{
$config = $this->job->configuration;
$count = $config['column-count'];
for ($i = 0; $i < $count; $i++) {
$role = $config['column-roles'][$i] ?? '_ignore';
$mapping = $config['column-do-mapping'][$i] ?? false;
if ($role === '_ignore' && $mapping === true) {
$mapping = false;
Log::debug(sprintf('Column %d has type %s so it cannot be mapped.', $i, $role));
}
$config['column-do-mapping'][$i] = $mapping;
}
$this->job->configuration = $config;
$this->job->save();
return true;
}
/**
* @return bool
*/
private function isMappingNecessary()
{
$config = $this->job->configuration;
$count = $config['column-count'];
$toBeMapped = 0;
for ($i = 0; $i < $count; $i++) {
$mapping = $config['column-do-mapping'][$i] ?? false;
if ($mapping === true) {
$toBeMapped++;
}
}
Log::debug(sprintf('Found %d columns that need mapping.', $toBeMapped));
if ($toBeMapped === 0) {
// skip setting of map, because none need to be mapped:
$config['column-mapping-complete'] = true;
$this->job->configuration = $config;
$this->job->save();
}
return true;
}
/**
* make unique example data
*/
private function makeExamplesUnique(): bool
{
foreach ($this->data['examples'] as $index => $values) {
$this->data['examples'][$index] = array_unique($values);
}
return true;
}
/**
* @param array $row
*
* @return bool
*/
private function processRow(array $row): bool
{
foreach ($row as $index => $value) {
$value = trim($value);
if (strlen($value) > 0) {
$this->data['examples'][$index][] = $value;
}
}
return true;
}
/**
* run specifics here:
* and this is the point where the specifix go to work.
*
* @param array $row
*
* @return array
*/
private function processSpecifics(array $row): array
{
foreach ($this->job->configuration['specifics'] as $name => $enabled) {
/** @var SpecificInterface $specific */
$specific = app('FireflyIII\Import\Specifics\\' . $name);
$row = $specific->run($row);
}
return $row;
}
/**
* @return bool
*/
private function setRolesComplete(): bool
{
$config = $this->job->configuration;
$count = $config['column-count'];
$assigned = 0;
for ($i = 0; $i < $count; $i++) {
$role = $config['column-roles'][$i] ?? '_ignore';
if ($role !== '_ignore') {
$assigned++;
}
}
if ($assigned > 0) {
$config['column-roles-complete'] = true;
$this->job->configuration = $config;
$this->job->save();
}
return true;
}
/**
* @return bool
*/
private function updateColumCount(): bool
{
$config = $this->job->configuration;
$count = $this->data['total'];
$config['column-count'] = $count;
$this->job->configuration = $config;
$this->job->save();
return true;
}
}