kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-09-18 18:44:16 +00:00
Code Issues Projects Releases Wiki Activity

Lots of new code for new importer routine.

Browse Source
This commit is contained in:
James Cole
2017-06-10 15:09:41 +02:00
parent 0b4efe4ae1
commit 091596e80e
25 changed files with 1415 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

535
app/Http/Controllers/ImportController.php
View File

@@ -12,11 +12,10 @@ declare(strict_types=1);
namespace FireflyIII\Http\Controllers;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Requests\ImportUploadRequest;
use FireflyIII\Import\Configurator\ConfiguratorInterface;
use FireflyIII\Import\ImportProcedureInterface;
use FireflyIII\Import\Setup\SetupInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
@@ -25,10 +24,6 @@ use Illuminate\Http\Request;
use Illuminate\Http\Response as LaravelResponse;
use Log;
use Response;
use Session;
use SplFileObject;
use Storage;
use Symfony\Component\HttpFoundation\File\UploadedFile;
use View;
/**
@@ -54,30 +49,28 @@ class ImportController extends Controller
}
);
}
//
// /**
// * This is the last step before the import starts.
// *
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
// */
// public function complete(ImportJob $job)
// {
// Log::debug('Now in complete()', ['job' => $job->key]);
// if (!$this->jobInCorrectStep($job, 'complete')) {
// return $this->redirectToCorrectStep($job);
// }
// $subTitle = trans('firefly.import_complete');
// $subTitleIcon = 'fa-star';
//
// return view('import.complete', compact('job', 'subTitle', 'subTitleIcon'));
// }
/**
* This is the last step before the import starts.
*
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
*/
public function complete(ImportJob $job)
{
Log::debug('Now in complete()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'complete')) {
return $this->redirectToCorrectStep($job);
}
$subTitle = trans('firefly.import_complete');
$subTitleIcon = 'fa-star';
return view('import.complete', compact('job', 'subTitle', 'subTitleIcon'));
}
/**
* This is step 3.
* This is the first step in configuring the job. It can only be executed
* when the job is set to "import_status_never_started".
* This is step 3. This repeats until the job is configured.
*
* @param ImportJob $job
*
@@ -86,19 +79,19 @@ class ImportController extends Controller
*/
public function configure(ImportJob $job)
{
Log::debug('Now at start of configure()');
if (!$this->jobInCorrectStep($job, 'configure')) {
return $this->redirectToCorrectStep($job);
}
// create configuration class:
$configurator = $this->makeConfigurator($job);
// actual code
$importer = $this->makeImporter($job);
$importer->configure();
$data = $importer->getConfigurationData();
// is the job already configured?
if ($configurator->isJobConfigured()) {
return redirect(route('import.status', [$job->key]));
}
$view = $configurator->getNextView();
$data = $configurator->getNextData();
$subTitle = trans('firefly.configure_import');
$subTitleIcon = 'fa-wrench';
return view('import.' . $job->file_type . '.configure', compact('data', 'job', 'subTitle', 'subTitleIcon'));
return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
}
/**
@@ -134,25 +127,25 @@ class ImportController extends Controller
}
/**
* @param ImportJob $job
*
* @return View
*/
public function finished(ImportJob $job)
{
if (!$this->jobInCorrectStep($job, 'finished')) {
return $this->redirectToCorrectStep($job);
}
// if there is a tag (there might not be), we can link to it:
$tagId = $job->extended_status['importTag'] ?? 0;
$subTitle = trans('firefly.import_finished');
$subTitleIcon = 'fa-star';
return view('import.finished', compact('job', 'subTitle', 'subTitleIcon', 'tagId'));
}
// /**
// * @param ImportJob $job
// *
// * @return View
// */
// public function finished(ImportJob $job)
// {
// if (!$this->jobInCorrectStep($job, 'finished')) {
// return $this->redirectToCorrectStep($job);
// }
//
// // if there is a tag (there might not be), we can link to it:
// $tagId = $job->extended_status['importTag'] ?? 0;
//
// $subTitle = trans('firefly.import_finished');
// $subTitleIcon = 'fa-star';
//
// return view('import.finished', compact('job', 'subTitle', 'subTitleIcon', 'tagId'));
// }
/**
* This is step 1. Upload a file.
@@ -161,7 +154,6 @@ class ImportController extends Controller
*/
public function index()
{
Log::debug('Now at index');
$subTitle = trans('firefly.import_data_index');
$subTitleIcon = 'fa-home';
$importFileTypes = [];
@@ -174,6 +166,35 @@ class ImportController extends Controller
return view('import.index', compact('subTitle', 'subTitleIcon', 'importFileTypes', 'defaultImportType'));
}
/**
* This is step 2. It creates an Import Job. Stores the import.
*
* @param ImportUploadRequest $request
* @param ImportJobRepositoryInterface $repository
* @param UserRepositoryInterface $userRepository
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function initialize(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository)
{
Log::debug('Now in initialize()');
// create import job:
$type = $request->get('import_file_type');
$job = $repository->create($type);
Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]);
// process file:
$repository->processFile($job, $request->files->get('import_file'));
// process config, if present:
if ($request->files->has('configuration_file')) {
$repository->processConfiguration($job, $request->files->get('configuration_file'));
}
return redirect(route('import.configure', [$job->key]));
}
/**
* @param ImportJob $job
*
@@ -182,7 +203,6 @@ class ImportController extends Controller
public function json(ImportJob $job)
{
$result = [
'showPercentage' => false,
'started' => false,
'finished' => false,
'running' => false,
@@ -191,13 +211,14 @@ class ImportController extends Controller
'steps' => $job->extended_status['total_steps'],
'stepsDone' => $job->extended_status['steps_done'],
'statusText' => trans('firefly.import_status_' . $job->status),
'status' => $job->status,
'finishedText' => '',
];
$percentage = 0;
if ($job->extended_status['total_steps'] !== 0) {
$percentage = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0);
}
if ($job->status === 'import_complete') {
if ($job->status === 'complete') {
$tagId = $job->extended_status['importTag'];
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
@@ -206,10 +227,9 @@ class ImportController extends Controller
$result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]);
}
if ($job->status === 'import_running') {
if ($job->status === 'running') {
$result['started'] = true;
$result['running'] = true;
$result['showPercentage'] = true;
$result['percentage'] = $percentage;
}
@@ -228,87 +248,81 @@ class ImportController extends Controller
public function postConfigure(Request $request, ImportJobRepositoryInterface $repository, ImportJob $job)
{
Log::debug('Now in postConfigure()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'process')) {
return $this->redirectToCorrectStep($job);
}
Log::debug('Continue postConfigure()', ['job' => $job->key]);
$configurator = $this->makeConfigurator($job);
// actual code
$importer = $this->makeImporter($job);
// is the job already configured?
if ($configurator->isJobConfigured()) {
return redirect(route('import.status', [$job->key]));
}
$data = $request->all();
$files = $request->files;
$importer->saveImportConfiguration($data, $files);
$configurator->configureJob($data);
// update job:
$repository->updateStatus($job, 'import_configuration_saved');
// return redirect to settings.
// this could loop until the user is done.
return redirect(route('import.settings', [$job->key]));
// return to configure
return redirect(route('import.configure', [$job->key]));
}
/**
* This step 6. Depending on the importer, this will process the
* settings given and store them.
*
* @param Request $request
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
* @throws FireflyException
*/
public function postSettings(Request $request, ImportJob $job)
{
Log::debug('Now in postSettings()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'store-settings')) {
return $this->redirectToCorrectStep($job);
}
$importer = $this->makeImporter($job);
$importer->storeSettings($request);
// /**
// * This step 6. Depending on the importer, this will process the
// * settings given and store them.
// *
// * @param Request $request
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
// * @throws FireflyException
// */
// public function postSettings(Request $request, ImportJob $job)
// {
// Log::debug('Now in postSettings()', ['job' => $job->key]);
// if (!$this->jobInCorrectStep($job, 'store-settings')) {
// return $this->redirectToCorrectStep($job);
// }
// $importer = $this->makeImporter($job);
// $importer->storeSettings($request);
//
// // return redirect to settings (for more settings perhaps)
// return redirect(route('import.settings', [$job->key]));
// }
// return redirect to settings (for more settings perhaps)
return redirect(route('import.settings', [$job->key]));
}
/**
* Step 5. Depending on the importer, this will show the user settings to
* fill in.
*
* @param ImportJobRepositoryInterface $repository
* @param ImportJob $job
*
* @return View
*/
public function settings(ImportJobRepositoryInterface $repository, ImportJob $job)
{
Log::debug('Now in settings()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'settings')) {
return $this->redirectToCorrectStep($job);
}
Log::debug('Continue in settings()');
$importer = $this->makeImporter($job);
$subTitle = trans('firefly.settings_for_import');
$subTitleIcon = 'fa-wrench';
// now show settings screen to user.
if ($importer->requireUserSettings()) {
Log::debug('Job requires user config.');
$data = $importer->getDataForSettings();
$view = $importer->getViewForSettings();
return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
}
Log::debug('Job does NOT require user config.');
$repository->updateStatus($job, 'settings_complete');
// if no more settings, save job and continue to process thing.
return redirect(route('import.complete', [$job->key]));
// ask the importer for the requested action.
// for example pick columns or map data.
// depends of course on the data in the job.
}
// /**
// * Step 5. Depending on the importer, this will show the user settings to
// * fill in.
// *
// * @param ImportJobRepositoryInterface $repository
// * @param ImportJob $job
// *
// * @return View
// */
// public function settings(ImportJobRepositoryInterface $repository, ImportJob $job)
// {
// Log::debug('Now in settings()', ['job' => $job->key]);
// if (!$this->jobInCorrectStep($job, 'settings')) {
// return $this->redirectToCorrectStep($job);
// }
// Log::debug('Continue in settings()');
// $importer = $this->makeImporter($job);
// $subTitle = trans('firefly.settings_for_import');
// $subTitleIcon = 'fa-wrench';
//
// // now show settings screen to user.
// if ($importer->requireUserSettings()) {
// Log::debug('Job requires user config.');
// $data = $importer->getDataForSettings();
// $view = $importer->getViewForSettings();
//
// return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
// }
// Log::debug('Job does NOT require user config.');
//
// $repository->updateStatus($job, 'settings_complete');
//
// // if no more settings, save job and continue to process thing.
// return redirect(route('import.complete', [$job->key]));
//
// // ask the importer for the requested action.
// // for example pick columns or map data.
// // depends of course on the data in the job.
// }
/**
* @param ImportProcedureInterface $importProcedure
@@ -316,6 +330,7 @@ class ImportController extends Controller
*/
public function start(ImportProcedureInterface $importProcedure, ImportJob $job)
{
die('TODO here.');
set_time_limit(0);
if ($job->status == 'settings_complete') {
$importProcedure->runImport($job);
@@ -330,175 +345,117 @@ class ImportController extends Controller
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
*/
public function status(ImportJob $job)
{ //
Log::debug('Now in status()', ['job' => $job->key]);
if (!$this->jobInCorrectStep($job, 'status')) {
return $this->redirectToCorrectStep($job);
}
{
$subTitle = trans('firefly.import_status');
$subTitleIcon = 'fa-star';
return view('import.status', compact('job', 'subTitle', 'subTitleIcon'));
}
/**
* This is step 2. It creates an Import Job. Stores the import.
*
* @param ImportUploadRequest $request
* @param ImportJobRepositoryInterface $repository
* @param UserRepositoryInterface $userRepository
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function upload(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository)
{
Log::debug('Now in upload()');
// create import job:
$type = $request->get('import_file_type');
$job = $repository->create($type);
Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]);
/** @var UploadedFile $upload */
$upload = $request->files->get('import_file');
$newName = $job->key . '.upload';
$uploaded = new SplFileObject($upload->getRealPath());
$content = $uploaded->fread($uploaded->getSize());
$contentEncrypted = Crypt::encrypt($content);
$disk = Storage::disk('upload');
// user is demo user, replace upload with prepared file.
if ($userRepository->hasRole(auth()->user(), 'demo')) {
$stubsDisk = Storage::disk('stubs');
$content = $stubsDisk->get('demo-import.csv');
$contentEncrypted = Crypt::encrypt($content);
$disk->put($newName, $contentEncrypted);
Log::debug('Replaced upload with demo file.');
// also set up prepared configuration.
$configuration = json_decode($stubsDisk->get('demo-configuration.json'), true);
$repository->setConfiguration($job, $configuration);
Log::debug('Set configuration for demo user', $configuration);
// also flash info
Session::flash('info', trans('demo.import-configure-security'));
}
if (!$userRepository->hasRole(auth()->user(), 'demo')) {
// user is not demo, process original upload:
$disk->put($newName, $contentEncrypted);
Log::debug('Uploaded file', ['name' => $upload->getClientOriginalName(), 'size' => $upload->getSize(), 'mime' => $upload->getClientMimeType()]);
}
// store configuration file's content into the job's configuration thing. Otherwise, leave it empty.
// demo user's configuration upload is ignored completely.
if ($request->files->has('configuration_file') && !auth()->user()->hasRole('demo')) {
/** @var UploadedFile $configFile */
$configFile = $request->files->get('configuration_file');
Log::debug(
'Uploaded configuration file',
['name' => $configFile->getClientOriginalName(), 'size' => $configFile->getSize(), 'mime' => $configFile->getClientMimeType()]
);
$configFileObject = new SplFileObject($configFile->getRealPath());
$configRaw = $configFileObject->fread($configFileObject->getSize());
$configuration = json_decode($configRaw, true);
// @codeCoverageIgnoreStart
if (!is_null($configuration) && is_array($configuration)) {
Log::debug('Found configuration', $configuration);
$repository->setConfiguration($job, $configuration);
}
// @codeCoverageIgnoreEnd
}
return redirect(route('import.configure', [$job->key]));
}
/**
* @param ImportJob $job
* @param string $method
*
* @return bool
*/
private function jobInCorrectStep(ImportJob $job, string $method): bool
{
Log::debug('Now in jobInCorrectStep()', ['job' => $job->key, 'method' => $method]);
switch ($method) {
case 'configure':
case 'process':
return $job->status === 'import_status_never_started';
case 'settings':
case 'store-settings':
Log::debug(sprintf('Job %d with key %s has status %s', $job->id, $job->key, $job->status));
return $job->status === 'import_configuration_saved';
case 'finished':
return $job->status === 'import_complete';
case 'complete':
return $job->status === 'settings_complete';
case 'status':
return ($job->status === 'settings_complete') || ($job->status === 'import_running');
}
return false; // @codeCoverageIgnore
}
// /**
// * @param ImportJob $job
// * @param string $method
// *
// * @return bool
// */
// private function jobInCorrectStep(ImportJob $job, string $method): bool
// {
// Log::debug('Now in jobInCorrectStep()', ['job' => $job->key, 'method' => $method]);
// switch ($method) {
// case 'configure':
// case 'process':
// return $job->status === 'import_status_never_started';
// case 'settings':
// case 'store-settings':
// Log::debug(sprintf('Job %d with key %s has status %s', $job->id, $job->key, $job->status));
//
// return $job->status === 'import_configuration_saved';
// case 'finished':
// return $job->status === 'import_complete';
// case 'complete':
// return $job->status === 'settings_complete';
// case 'status':
// return ($job->status === 'settings_complete') || ($job->status === 'import_running');
// }
//
// return false; // @codeCoverageIgnore
//
// }
/**
* @param ImportJob $job
*
* @return SetupInterface
* @return ConfiguratorInterface
* @throws FireflyException
*/
private function makeImporter(ImportJob $job): SetupInterface
private function makeConfigurator(ImportJob $job): ConfiguratorInterface
{
// create proper importer (depends on job)
$type = strtolower($job->file_type);
// validate type:
$validTypes = array_keys(config('firefly.import_formats'));
if (in_array($type, $validTypes)) {
/** @var SetupInterface $importer */
$importer = app('FireflyIII\Import\Setup\\' . ucfirst($type) . 'Setup');
$importer->setJob($job);
return $importer;
$type = $job->file_type;
$key = sprintf('firefly.import_configurators.%s', $type);
$className = config($key);
if (is_null($className)) {
throw new FireflyException('Cannot find configurator class for this job.');
}
throw new FireflyException(sprintf('"%s" is not a valid file type', $type)); // @codeCoverageIgnore
$configurator = new $className($job);
return $configurator;
}
/**
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
* @throws FireflyException
*/
private function redirectToCorrectStep(ImportJob $job)
{
Log::debug('Now in redirectToCorrectStep()', ['job' => $job->key]);
switch ($job->status) {
case 'import_status_never_started':
Log::debug('Will redirect to configure()');
// /**
// * @param ImportJob $job
// *
// * @return SetupInterface
// * @throws FireflyException
// */
// private function makeImporter(ImportJob $job): SetupInterface
// {
// // create proper importer (depends on job)
// $type = strtolower($job->file_type);
//
// // validate type:
// $validTypes = array_keys(config('firefly.import_formats'));
//
//
// if (in_array($type, $validTypes)) {
// /** @var SetupInterface $importer */
// $importer = app('FireflyIII\Import\Setup\\' . ucfirst($type) . 'Setup');
// $importer->setJob($job);
//
// return $importer;
// }
// throw new FireflyException(sprintf('"%s" is not a valid file type', $type)); // @codeCoverageIgnore
//
// }
return redirect(route('import.configure', [$job->key]));
case 'import_configuration_saved':
Log::debug('Will redirect to settings()');
return redirect(route('import.settings', [$job->key]));
case 'settings_complete':
Log::debug('Will redirect to complete()');
return redirect(route('import.complete', [$job->key]));
case 'import_complete':
Log::debug('Will redirect to finished()');
return redirect(route('import.finished', [$job->key]));
}
throw new FireflyException('Cannot redirect for job state ' . $job->status); // @codeCoverageIgnore
}
// /**
// * @param ImportJob $job
// *
// * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
// * @throws FireflyException
// */
// private function redirectToCorrectStep(ImportJob $job)
// {
// Log::debug('Now in redirectToCorrectStep()', ['job' => $job->key]);
// switch ($job->status) {
// case 'import_status_never_started':
// Log::debug('Will redirect to configure()');
//
// return redirect(route('import.configure', [$job->key]));
// case 'import_configuration_saved':
// Log::debug('Will redirect to settings()');
//
// return redirect(route('import.settings', [$job->key]));
// case 'settings_complete':
// Log::debug('Will redirect to complete()');
//
// return redirect(route('import.complete', [$job->key]));
// case 'import_complete':
// Log::debug('Will redirect to finished()');
//
// return redirect(route('import.finished', [$job->key]));
// }
//
// throw new FireflyException('Cannot redirect for job state ' . $job->status); // @codeCoverageIgnore
//
// }
}

1
app/Http/Requests/ImportUploadRequest.php
View File

@@ -40,6 +40,7 @@ class ImportUploadRequest extends Request
return [
'import_file' => 'required|file',
'import_file_type' => 'required|in:' . join(',', $types),
'configuration_file' => 'file',
];
}

60
app/Import/Configurator/ConfiguratorInterface.php Normal file
View File

@@ -0,0 +1,60 @@
<?php
/**
* ConfiguratorInterface.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Configurator;
use FireflyIII\Models\ImportJob;
/**
* Interface ConfiguratorInterface
*
* @package FireflyIII\Import\Configurator
*/
interface ConfiguratorInterface
{
/**
* ConfiguratorInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job);
/**
* Store any data from the $data array into the job.
*
* @param array $data
*
* @return bool
*/
public function configureJob(array $data): bool;
/**
* Return the data required for the next step in the job configuration.
*
* @return array
*/
public function getNextData(): array;
/**
* Returns the view of the next step in the job configuration.
*
* @return string
*/
public function getNextView(): string;
/**
* Returns true when the initial configuration for this job is complete.
*
* @return bool
*/
public function isJobConfigured(): bool;
}

140
app/Import/Configurator/CsvConfigurator.php Normal file
View File

@@ -0,0 +1,140 @@
<?php
/**
* CsvConfigurator.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Configurator;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use Log;
/**
* Class CsvConfigurator
*
* @package FireflyIII\Import\Configurator
*/
class CsvConfigurator implements ConfiguratorInterface
{
private $job;
public function __construct(ImportJob $job)
{
$this->job = $job;
if (is_null($this->job->configuration) || count($this->job->configuration) === 0) {
Log::debug(sprintf('Gave import job %s initial configuration.', $this->job->key));
$this->job->configuration = config('csv.default_config');
$this->job->save();
}
}
/**
* Store any data from the $data array into the job.
*
* @param array $data
*
* @return bool
* @throws FireflyException
*/
public function configureJob(array $data): bool
{
$class = $this->getConfigurationClass();
/** @var ConfigurationInterface $object */
$object = new $class($this->job);
return $object->storeConfiguration($data);
}
/**
* Return the data required for the next step in the job configuration.
*
* @return array
* @throws FireflyException
*/
public function getNextData(): array
{
$class = $this->getConfigurationClass();
/** @var ConfigurationInterface $object */
$object = new $class($this->job);
return $object->getData();
}
/**
* @return string
* @throws FireflyException
*/
public function getNextView(): string
{
if (!$this->job->configuration['initial-config-complete']) {
return 'import.csv.initial';
}
if (!$this->job->configuration['column-roles-complete']) {
return 'import.csv.roles';
}
if (!$this->job->configuration['column-mapping-complete']) {
return 'import.csv.map';
}
throw new FireflyException('No view for state');
}
/**
* @return bool
*/
public function isJobConfigured(): bool
{
if ($this->job->configuration['initial-config-complete']
&& $this->job->configuration['column-roles-complete']
&& $this->job->configuration['column-mapping-complete']
) {
$this->job->status = 'configured';
$this->job->save();
return true;
}
return false;
}
/**
* @return string
* @throws FireflyException
*/
private function getConfigurationClass(): string
{
$class = false;
switch (true) {
case(!$this->job->configuration['initial-config-complete']):
$class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Initial';
break;
case (!$this->job->configuration['column-roles-complete']):
$class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Roles';
break;
case (!$this->job->configuration['column-mapping-complete']):
$class = 'FireflyIII\\Support\\Import\\Configuration\\Csv\\Map';
break;
default:
break;
}
if ($class === false) {
throw new FireflyException('Cannot handle current job state in getConfigurationClass().');
}
if (!class_exists($class)) {
throw new FireflyException(sprintf('Class %s does not exist in getConfigurationClass().', $class));
}
return $class;
}
}

0
app/Import/Importer/CsvImporter.php → app/Import/Importer/OldCsvImporter.php
View File

0
app/Import/Importer/ImporterInterface.php → app/Import/Importer/OldImporterInterface.php
View File

1
app/Import/Setup/CsvSetup.php
View File

@@ -237,7 +237,6 @@ class CsvSetup implements SetupInterface
$all = $request->all();
if ($request->get('settings') == 'roles') {
$count = $config['column-count'];
$roleSet = 0; // how many roles have been defined
$mapSet = 0; // how many columns must be mapped
for ($i = 0; $i < $count; $i++) {

8
app/Models/ImportJob.php
View File

@@ -42,11 +42,9 @@ class ImportJob extends Model
protected $validStatus
= [
'import_status_never_started', // initial state
'import_configuration_saved', // import configuration saved. This step is going to be obsolete.
'settings_complete', // aka: ready for import.
'import_running', // import currently underway
'import_complete', // done with everything
'new',
'initialized',
'configured',
];
/**

79
app/Repositories/ImportJob/ImportJobRepository.php
View File

@@ -13,10 +13,16 @@ declare(strict_types=1);
namespace FireflyIII\Repositories\ImportJob;
use Crypt;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use FireflyIII\User;
use Illuminate\Support\Str;
use Log;
use SplFileObject;
use Storage;
use Symfony\Component\HttpFoundation\File\UploadedFile;
/**
* Class ImportJobRepository
@@ -51,7 +57,7 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$importJob->user()->associate($this->user);
$importJob->file_type = $fileType;
$importJob->key = Str::random(12);
$importJob->status = 'import_status_never_started';
$importJob->status = 'new';
$importJob->extended_status = [
'total_steps' => 0,
'steps_done' => 0,
@@ -86,6 +92,77 @@ class ImportJobRepository implements ImportJobRepositoryInterface
return $result;
}
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return bool
*/
public function processConfiguration(ImportJob $job, UploadedFile $file): bool
{
/** @var UserRepositoryInterface $repository */
$repository = app(UserRepositoryInterface::class);
// demo user's configuration upload is ignored completely.
if ($repository->hasRole($this->user, 'demo')) {
Log::debug(
'Uploaded configuration file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]
);
$configFileObject = new SplFileObject($file->getRealPath());
$configRaw = $configFileObject->fread($configFileObject->getSize());
$configuration = json_decode($configRaw, true);
if (!is_null($configuration) && is_array($configuration)) {
Log::debug('Found configuration', $configuration);
$this->setConfiguration($job, $configuration);
}
}
return true;
}
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return mixed
*/
public function processFile(ImportJob $job, UploadedFile $file): bool
{
/** @var UserRepositoryInterface $repository */
$repository = app(UserRepositoryInterface::class);
$newName = sprintf('%s.upload', $job->key);
$uploaded = new SplFileObject($file->getRealPath());
$content = $uploaded->fread($uploaded->getSize());
$contentEncrypted = Crypt::encrypt($content);
$disk = Storage::disk('upload');
// user is demo user, replace upload with prepared file.
if ($repository->hasRole($this->user, 'demo')) {
$stubsDisk = Storage::disk('stubs');
$content = $stubsDisk->get('demo-import.csv');
$contentEncrypted = Crypt::encrypt($content);
$disk->put($newName, $contentEncrypted);
Log::debug('Replaced upload with demo file.');
// also set up prepared configuration.
$configuration = json_decode($stubsDisk->get('demo-configuration.json'), true);
$this->setConfiguration($job, $configuration);
Log::debug('Set configuration for demo user', $configuration);
}
if (!$repository->hasRole($this->user, 'demo')) {
// user is not demo, process original upload:
$disk->put($newName, $contentEncrypted);
Log::debug('Uploaded file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]);
}
$job->status = 'initialized';
$job->save();
return true;
}
/**
* @param ImportJob $job
* @param array $configuration

17
app/Repositories/ImportJob/ImportJobRepositoryInterface.php
View File

@@ -15,6 +15,7 @@ namespace FireflyIII\Repositories\ImportJob;
use FireflyIII\Models\ImportJob;
use FireflyIII\User;
use Symfony\Component\HttpFoundation\File\UploadedFile;
/**
* Interface ImportJobRepositoryInterface
@@ -37,6 +38,22 @@ interface ImportJobRepositoryInterface
*/
public function findByKey(string $key): ImportJob;
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return mixed
*/
public function processFile(ImportJob $job, UploadedFile $file): bool;
/**
* @param ImportJob $job
* @param UploadedFile $file
*
* @return bool
*/
public function processConfiguration(ImportJob $job, UploadedFile $file): bool;
/**
* @param ImportJob $job
* @param array $configuration

46
app/Support/Import/Configuration/ConfigurationInterface.php Normal file
View File

@@ -0,0 +1,46 @@
<?php
/**
* ConfigurationInterface.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration;
use FireflyIII\Models\ImportJob;
/**
* Class ConfigurationInterface
*
* @package FireflyIII\Support\Import\Configuration
*/
interface ConfigurationInterface
{
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job);
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getData(): array;
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool;
}

122
app/Support/Import/Configuration/Csv/Initial.php Normal file
View File

@@ -0,0 +1,122 @@
<?php
/**
* CsvInitial.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\Csv;
use ExpandedForm;
use FireflyIII\Models\AccountType;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use Log;
/**
* Class CsvInitial
*
* @package FireflyIII\Support\Import\Configuration
*/
class Initial implements ConfigurationInterface
{
private $job;
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job)
{
$this->job = $job;
}
/**
* @return array
*/
public function getData(): array
{
/** @var AccountRepositoryInterface $accountRepository */
$accountRepository = app(AccountRepositoryInterface::class);
$accounts = $accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]);
$delimiters = [
',' => trans('form.csv_comma'),
';' => trans('form.csv_semicolon'),
'tab' => trans('form.csv_tab'),
];
$specifics = [];
// collect specifics.
foreach (config('csv.import_specifics') as $name => $className) {
$specifics[$name] = [
'name' => $className::getName(),
'description' => $className::getDescription(),
];
}
$data = [
'accounts' => ExpandedForm::makeSelectList($accounts),
'specifix' => [],
'delimiters' => $delimiters,
'specifics' => $specifics,
];
return $data;
}
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool
{
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$importId = $data['csv_import_account'] ?? 0;
$account = $repository->find(intval($importId));
$hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false;
$config = $this->job->configuration;
$config['initial-config-complete'] = true;
$config['has-headers'] = $hasHeaders;
$config['date-format'] = $data['date_format'];
$config['delimiter'] = $data['csv_delimiter'];
$config['delimiter'] = $config['delimiter'] === 'tab' ? "\t" : $config['delimiter'];
Log::debug('Entered import account.', ['id' => $importId]);
if (!is_null($account->id)) {
Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]);
$config['import-account'] = $account->id;
}
if (is_null($account->id)) {
Log::error('Could not find anything for csv_import_account.', ['id' => $importId]);
}
// loop specifics.
if (isset($data['specifics']) && is_array($data['specifics'])) {
foreach ($data['specifics'] as $name => $enabled) {
// verify their content.
$className = sprintf('FireflyIII\Import\Specifics\%s', $name);
if (class_exists($className)) {
$config['specifics'][$name] = 1;
}
}
}
$this->job->configuration = $config;
$this->job->save();
return true;
}
}

229
app/Support/Import/Configuration/Csv/Map.php Normal file
View File

@@ -0,0 +1,229 @@
<?php
/**
* Mapping.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\Csv;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\Mapper\MapperInterface;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use League\Csv\Reader;
use Log;
/**
* Class Mapping
*
* @package FireflyIII\Support\Import\Configuration\Csv
*/
class Map implements ConfigurationInterface
{
/** @var array that holds each column to be mapped by the user */
private $data = [];
/** @var array that holds the indexes of those columns (ie. 2, 5, 8) */
private $indexes = [];
/** @var ImportJob */
private $job;
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job)
{
$this->job = $job;
}
/**
* @return array
* @throws FireflyException
*/
public function getData(): array
{
$config = $this->job->configuration;
$this->getMappableColumns();
// in order to actually map we also need all possible values from the CSV file.
$content = $this->job->uploadFileContents();
/** @var Reader $reader */
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$results = $reader->fetch();
$validSpecifics = array_keys(config('csv.import_specifics'));
foreach ($results as $rowIndex => $row) {
// skip first row?
if ($rowIndex === 0 && $config['has-headers']) {
continue;
}
// run specifics here:
// and this is the point where the specifix go to work.
foreach ($config['specifics'] as $name => $enabled) {
if (!in_array($name, $validSpecifics)) {
throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
}
$class = config('csv.import_specifics.' . $name);
/** @var SpecificInterface $specific */
$specific = app($class);
// it returns the row, possibly modified:
$row = $specific->run($row);
}
//do something here
foreach ($indexes as $index) { // this is simply 1, 2, 3, etc.
if (!isset($row[$index])) {
// don't really know how to handle this. Just skip, for now.
continue;
}
$value = $row[$index];
if (strlen($value) > 0) {
// we can do some preprocessing here,
// which is exclusively to fix the tags:
if (!is_null($data[$index]['preProcessMap'])) {
/** @var PreProcessorInterface $preProcessor */
$preProcessor = app($data[$index]['preProcessMap']);
$result = $preProcessor->run($value);
$data[$index]['values'] = array_merge($data[$index]['values'], $result);
Log::debug($rowIndex . ':' . $index . 'Value before preprocessor', ['value' => $value]);
Log::debug($rowIndex . ':' . $index . 'Value after preprocessor', ['value-new' => $result]);
Log::debug($rowIndex . ':' . $index . 'Value after joining', ['value-complete' => $data[$index]['values']]);
continue;
}
$data[$index]['values'][] = $value;
}
}
}
foreach ($data as $index => $entry) {
$data[$index]['values'] = array_unique($data[$index]['values']);
}
return $data;
}
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool
{
return true;
}
/**
* @param string $column
*
* @return MapperInterface
*/
private function createMapper(string $column): MapperInterface
{
$mapperClass = config('csv.import_roles.' . $column . '.mapper');
$mapperName = sprintf('\\FireflyIII\\Import\Mapper\\%s', $mapperClass);
/** @var MapperInterface $mapper */
$mapper = new $mapperName;
return $mapper;
}
/**
* @return bool
*/
private function getMappableColumns(): bool
{
$config = $this->job->configuration;
/**
* @var int $index
* @var bool $mustBeMapped
*/
foreach ($config['column-do-mapping'] as $index => $mustBeMapped) {
$column = $this->validateColumnName($config['column-roles'][$index] ?? '_ignore');
$shouldMap = $this->shouldMapColumn($column, $mustBeMapped);
if ($shouldMap) {
// create configuration entry for this specific column and add column to $this->data array for later processing.
$this->data[$index] = [
'name' => $column,
'index' => $index,
'options' => $this->createMapper($column)->getMap(),
'preProcessMap' => $this->getPreProcessorName($column),
'values' => [],
];
}
}
return true;
}
/**
* @param string $column
*
* @return string
*/
private function getPreProcessorName(string $column): string
{
$name = '';
$hasPreProcess = config('csv.import_roles.' . $column . '.pre-process-map');
$preProcessClass = config('csv.import_roles.' . $column . '.pre-process-mapper');
if (!is_null($hasPreProcess) && $hasPreProcess === true && !is_null($preProcessClass)) {
$name = sprintf('\\FireflyIII\\Import\\MapperPreProcess\\%s', $preProcessClass);
}
return $name;
}
/**
* @param string $column
* @param bool $mustBeMapped
*
* @return bool
*/
private function shouldMapColumn(string $column, bool $mustBeMapped): bool
{
$canBeMapped = config('csv.import_roles.' . $column . '.mappable');
return ($canBeMapped && $mustBeMapped);
}
/**
* @param string $column
*
* @return string
* @throws FireflyException
*/
private function validateColumnName(string $column): string
{
// is valid column?
$validColumns = array_keys(config('csv.import_roles'));
if (!in_array($column, $validColumns)) {
throw new FireflyException(sprintf('"%s" is not a valid column.', $column));
}
return $column;
}
}

261
app/Support/Import/Configuration/Csv/Roles.php Normal file
View File

@@ -0,0 +1,261 @@
<?php
/**
* Roles.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Configuration\Csv;
use FireflyIII\Import\Specifics\SpecificInterface;
use FireflyIII\Models\ImportJob;
use FireflyIII\Support\Import\Configuration\ConfigurationInterface;
use League\Csv\Reader;
use Log;
/**
* Class Roles
*
* @package FireflyIII\Support\Import\Configuration\Csv
*/
class Roles implements ConfigurationInterface
{
private $data = [];
/** @var ImportJob */
private $job;
/**
* ConfigurationInterface constructor.
*
* @param ImportJob $job
*/
public function __construct(ImportJob $job)
{
$this->job = $job;
}
/**
* Get the data necessary to show the configuration screen.
*
* @return array
*/
public function getData(): array
{
$config = $this->job->configuration;
$content = $this->job->uploadFileContents();
// create CSV reader.
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$start = $config['has-headers'] ? 1 : 0;
$end = $start + config('csv.example_rows');
// set data:
$this->data = [
'examples' => [],
'roles' => $this->getRoles(),
'total' => 0,
'headers' => $config['has-headers'] ? $reader->fetchOne(0) : [],
];
while ($start < $end) {
$row = $reader->fetchOne($start);
$row = $this->processSpecifics($row);
$count = count($row);
$this->data['total'] = $count > $this->data['total'] ? $count : $this->data['total'];
$this->processRow($row);
$start++;
}
$this->updateColumCount();
$this->makeExamplesUnique();
return $this->data;
}
/**
* Store the result.
*
* @param array $data
*
* @return bool
*/
public function storeConfiguration(array $data): bool
{
Log::debug('Now in storeConfiguration of Roles.');
$config = $this->job->configuration;
$count = $config['column-count'];
for ($i = 0; $i < $count; $i++) {
$role = $data['role'][$i] ?? '_ignore';
$mapping = isset($data['map'][$i]) && $data['map'][$i] === '1' ? true : false;
$config['column-roles'][$i] = $role;
$config['column-do-mapping'][$i] = $mapping;
Log::debug(sprintf('Column %d has been given role %s', $i, $role));
}
$this->job->configuration = $config;
$this->job->save();
$this->ignoreUnmappableColumns();
$this->setRolesComplete();
$this->isMappingNecessary();
return true;
}
/**
* @return array
*/
private function getRoles(): array
{
$roles = [];
foreach (array_keys(config('csv.import_roles')) as $role) {
$roles[$role] = trans('csv.column_' . $role);
}
return $roles;
}
/**
* @return bool
*/
private function ignoreUnmappableColumns(): bool
{
$config = $this->job->configuration;
$count = $config['column-count'];
for ($i = 0; $i < $count; $i++) {
$role = $config['column-roles'][$i] ?? '_ignore';
$mapping = $config['column-do-mapping'][$i] ?? false;
if ($role === '_ignore' && $mapping === true) {
$mapping = false;
Log::debug(sprintf('Column %d has type %s so it cannot be mapped.', $i, $role));
}
$config['column-do-mapping'][$i] = $mapping;
}
$this->job->configuration = $config;
$this->job->save();
return true;
}
/**
* @return bool
*/
private function isMappingNecessary()
{
$config = $this->job->configuration;
$count = $config['column-count'];
$toBeMapped = 0;
for ($i = 0; $i < $count; $i++) {
$mapping = $config['column-do-mapping'][$i] ?? false;
if ($mapping === true) {
$toBeMapped++;
}
}
Log::debug(sprintf('Found %d columns that need mapping.', $toBeMapped));
if ($toBeMapped === 0) {
// skip setting of map, because none need to be mapped:
$config['column-mapping-complete'] = true;
$this->job->configuration = $config;
$this->job->save();
}
return true;
}
/**
* make unique example data
*/
private function makeExamplesUnique(): bool
{
foreach ($this->data['examples'] as $index => $values) {
$this->data['examples'][$index] = array_unique($values);
}
return true;
}
/**
* @param array $row
*
* @return bool
*/
private function processRow(array $row): bool
{
foreach ($row as $index => $value) {
$value = trim($value);
if (strlen($value) > 0) {
$this->data['examples'][$index][] = $value;
}
}
return true;
}
/**
* run specifics here:
* and this is the point where the specifix go to work.
*
* @param array $row
*
* @return array
*/
private function processSpecifics(array $row): array
{
foreach ($this->job->configuration['specifics'] as $name => $enabled) {
/** @var SpecificInterface $specific */
$specific = app('FireflyIII\Import\Specifics\\' . $name);
$row = $specific->run($row);
}
return $row;
}
/**
* @return bool
*/
private function setRolesComplete(): bool
{
$config = $this->job->configuration;
$count = $config['column-count'];
$assigned = 0;
for ($i = 0; $i < $count; $i++) {
$role = $config['column-roles'][$i] ?? '_ignore';
if ($role !== '_ignore') {
$assigned++;
}
}
if ($assigned > 0) {
$config['column-roles-complete'] = true;
$this->job->configuration = $config;
$this->job->save();
}
return true;
}
/**
* @return bool
*/
private function updateColumCount(): bool
{
$config = $this->job->configuration;
$count = $this->data['total'];
$config['column-count'] = $count;
$this->job->configuration = $config;
$this->job->save();
return true;
}
}

1
config/csv.php
View File

@@ -292,6 +292,7 @@ return [
// number of example rows:
'example_rows' => 5,
'default_config' => [
'initial-config-complete' => false,
'has-headers' => false, // assume
'date-format' => 'Ymd', // assume
'delimiter' => ',', // assume

5
config/firefly.php
View File

@@ -31,7 +31,10 @@ return [
'csv' => 'FireflyIII\Export\Exporter\CsvExporter',
],
'import_formats' => [
'csv' => 'FireflyIII\Import\Importer\CsvImporter',
'csv' => 'FireflyIII\Import\Configurator\CsvConfigurator',
],
'import_configurators' => [
'csv' => 'FireflyIII\Import\Configurator\CsvConfigurator',
],
'default_export_format' => 'csv',
'default_import_format' => 'csv',

103
public/js/ff/import/status.js
View File

@@ -10,6 +10,10 @@
/** global: jobImportUrl, langImportSingleError, langImportMultiError, jobStartUrl, langImportTimeOutError, langImportFinished, langImportFatalError */
var displayStatus = 'initial';
var timeOutId;
var startedImport = false;
var startInterval = 2000;
var interval = 500;
@@ -19,20 +23,85 @@ var stepCount = 0;
$(function () {
"use strict";
$('#import-status-intro').hide();
$('#import-status-more-info').hide();
//$('#import-status-intro').hide();
//$('#import-status-more-info').hide();
// check status, every 500 ms.
setTimeout(checkImportStatus, startInterval);
timeOutId = setTimeout(checkImportStatus, startInterval);
// button to start import routine:
$('.start-job').click(startJob);
});
function startJob() {
console.log('Job started.');
$.post(jobStartUrl);
return false;
}
function checkImportStatus() {
"use strict";
$.getJSON(jobImportUrl).done(reportOnJobImport).fail(failedJobImport);
}
function reportOnJobImport(data) {
"use strict";
displayCorrectBox(data.status);
//updateBar(data);
//reportErrors(data);
//reportStatus(data);
//updateTimeout(data);
//if (importJobFinished(data)) {
// finishedJob(data);
// return;
//}
// same number of steps as last time?
//if (currentLimit > timeoutLimit) {
// timeoutError();
// return;
//}
// if the job has not actually started, do so now:
//if (!data.started && !startedImport) {
// kickStartJob();
// return;
//}
// trigger another check.
//timeOutId = setTimeout(checkImportStatus, interval);
}
function displayCorrectBox(status) {
console.log('Current job state is ' + status);
if(status === 'configured' && displayStatus === 'initial') {
// hide some boxes:
$('.status_initial').hide();
return;
}
console.error('CANNOT HANDLE CURRENT STATE');
}
function importComplete() {
"use strict";
var bar = $('#import-status-bar');
@@ -131,35 +200,7 @@ function finishedJob(data) {
}
function reportOnJobImport(data) {
"use strict";
updateBar(data);
reportErrors(data);
reportStatus(data);
updateTimeout(data);
if (importJobFinished(data)) {
finishedJob(data);
return;
}
// same number of steps as last time?
if (currentLimit > timeoutLimit) {
timeoutError();
return;
}
// if the job has not actually started, do so now:
if (!data.started && !startedImport) {
kickStartJob();
return;
}
// trigger another check.
setTimeout(checkImportStatus, interval);
}
function startedTheImport() {
"use strict";

42
resources/lang/en_US/csv.php
View File

@@ -13,26 +13,30 @@ declare(strict_types=1);
return [
'import_configure_title' => 'Configure your import',
'import_configure_intro' => 'There are some options for your CSV import. Please indicate if your CSV file contains headers on the first column, and what the date format of your date-fields is. That might require some experimentation. The field delimiter is usually a ",", but could also be a ";". Check this carefully.',
'import_configure_form' => 'Basic CSV import options',
'header_help' => 'Check this if the first row of your CSV file are the column titles',
'date_help' => 'Date time format in your CSV. Follow the format like <a href="https://secure.php.net/manual/en/datetime.createfromformat.php#refsect1-datetime.createfromformat-parameters">this page</a> indicates. The default value will parse dates that look like this: :dateExample.',
'delimiter_help' => 'Choose the field delimiter that is used in your input file. If not sure, comma is the safest option.',
'import_account_help' => 'If your CSV file does NOT contain information about your asset account(s), use this dropdown to select to which account the transactions in the CSV belong to.',
'upload_not_writeable' => 'The grey box contains a file path. It should be writeable. Please make sure it is.',
// initial config
'initial_config_title' => 'Import configuration (1/3)',
'initial_config_text' => 'To be able to import your file correctly, please validate the options below.',
'initial_config_box' => 'Basic CSV import configuration',
'initial_header_help' => 'Check this box if the first row of your CSV file are the column titles.',
'initial_date_help' => 'Date time format in your CSV. Follow the format like <a href="https://secure.php.net/manual/en/datetime.createfromformat.php#refsect1-datetime.createfromformat-parameters">this page</a> indicates. The default value will parse dates that look like this: :dateExample.',
'initial_delimiter_help' => 'Choose the field delimiter that is used in your input file. If not sure, comma is the safest option.',
'initial_import_account_help' => 'If your CSV file does NOT contain information about your asset account(s), use this dropdown to select to which account the transactions in the CSV belong to.',
// roles
'column_roles_title' => 'Define column roles',
'column_roles_table' => 'Table',
'column_name' => 'Name of column',
'column_example' => 'Column example data',
'column_role' => 'Column data meaning',
'do_map_value' => 'Map these values',
'column' => 'Column',
'no_example_data' => 'No example data available',
'store_column_roles' => 'Continue import',
'do_not_map' => '(do not map)',
// roles config
'roles_title' => 'Define each column\'s role',
'roles_text' => 'Each column in your CSV file contains certain data. Please indicate what kind of data the importer should expect. The option to "map" data means that you will link each entry found in the column to a value in your database. An often mapped column is the column that contains the IBAN of the opposing account. That can be easily matched to IBAN\'s present in your database already.',
'roles_table' => 'Table',
'roles_column_name' => 'Name of column',
'roles_column_example' => 'Column example data',
'roles_column_role' => 'Column data meaning',
'roles_do_map_value' => 'Map these values',
'roles_column' => 'Column',
'roles_no_example_data' => 'No example data available',
'roles_store' => 'Continue import',
'roles_do_not_map' => '(do not map)',
// map data
'map_title' => 'Connect import data to Firefly III data',
'map_text' => 'In the following tables, the left value shows you information found in your uploaded CSV file. It is your task to map this value, if possible, to a value already present in your database. Firefly will stick to this mapping. If there is no value to map to, or you do not wish to map the specific value, select nothing.',

1
resources/lang/en_US/firefly.php
View File

@@ -1003,7 +1003,6 @@ return [
'import_finished_report' => 'The import has finished. Please note any errors in the block above this line. All transactions imported during this particular session have been tagged, and you can check them out below. ',
'import_finished_link' => 'The transactions imported can be found in tag <a href=":link" class="label label-success" style="font-size:100%;font-weight:normal;">:tag</a>.',
'need_at_least_one_account' => 'You need at least one asset account to be able to create piggy banks',
'see_help_top_right' => 'For more information, please check out the help pages using the icon in the top right corner of the page.',
'bread_crumb_import_complete' => 'Import ":key" complete',
'bread_crumb_configure_import' => 'Configure import ":key"',
'bread_crumb_import_finished' => 'Import ":key" finished',

31
resources/views/import/csv/configure.twig → resources/views/import/csv/initial.twig
View File

@@ -10,11 +10,11 @@
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ trans('csv.import_configure_title') }}</h3>
<h3 class="box-title">{{ trans('csv.initial_config_title') }}</h3>
</div>
<div class="box-body">
<p>
{{ trans('csv.import_configure_intro') }}
{{ trans('csv.initial_config_text') }}
</p>
</div>
</div>
@@ -29,16 +29,16 @@
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ trans('csv.import_configure_form') }}</h3>
<h3 class="box-title">{{ trans('csv.initial_config_box') }}</h3>
</div>
<div class="box-body">
<div class="row">
<div class="col-lg-6">
{{ ExpandedForm.checkbox('has_headers',1,job.configuration['has-headers'],{helpText: trans('csv.header_help')}) }}
{{ ExpandedForm.text('date_format',job.configuration['date-format'],{helpText: trans('csv.date_help', {dateExample: phpdate('Ymd')}) }) }}
{{ ExpandedForm.select('csv_delimiter', data.delimiters, job.configuration['delimiter'], {helpText: trans('csv.delimiter_help') } ) }}
{{ ExpandedForm.select('csv_import_account', data.accounts, job.configuration['import-account'], {helpText: trans('csv.import_account_help')} ) }}
{{ ExpandedForm.checkbox('has_headers',1,job.configuration['has-headers'],{helpText: trans('csv.initial_header_help')}) }}
{{ ExpandedForm.text('date_format',job.configuration['date-format'],{helpText: trans('csv.initial_date_help', {dateExample: phpdate('Ymd')}) }) }}
{{ ExpandedForm.select('csv_delimiter', data.delimiters, job.configuration['delimiter'], {helpText: trans('csv.initial_delimiter_help') } ) }}
{{ ExpandedForm.select('csv_import_account', data.accounts, job.configuration['import-account'], {helpText: trans('csv.initial_import_account_help')} ) }}
{% for type, specific in data.specifics %}
<div class="form-group">
@@ -56,28 +56,12 @@
</div>
</div>
{% endfor %}
{% if not data.is_upload_possible %}
<div class="form-group" id="csv_holder">
<div class="col-sm-4">
&nbsp;
</div>
<div class="col-sm-8">
<pre>{{ data.upload_path }}</pre>
<p class="text-danger">
{{ trans('csv.upload_not_writeable') }}
</p>
</div>
</div>
{% endif %}
</div>
</div>
</div>
</div>
</div>
</div>
{% if data.is_upload_possible %}
<div class="row">
<div class="col-lg-12">
<div class="box">
@@ -89,7 +73,6 @@
</div>
</div>
</div>
{% endif %}
</form>

5
resources/views/import/csv/map.twig
View File

@@ -1,7 +1,7 @@
{% extends "./layout/default" %}
{% block breadcrumbs %}
{{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }}
{{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName, job) }}
{% endblock %}
{% block content %}
@@ -22,9 +22,8 @@
</div>
</div>
<form action="{{ route('import.post-settings', job.key) }}" method="post">
<form action="{{ route('import.process-configuration', job.key) }}" method="post">
<input type="hidden" name="_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="settings" value="map"/>
{% for field in data %}
<div class="row">

36
resources/views/import/csv/roles.twig
View File

@@ -1,7 +1,7 @@
{% extends "./layout/default" %}
{% block breadcrumbs %}
{{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName) }}
{{ Breadcrumbs.renderIfExists(Route.getCurrentRoute.getName, job) }}
{% endblock %}
{% block content %}
@@ -10,18 +10,18 @@
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ trans('csv.column_roles_title') }}</h3>
<h3 class="box-title">{{ trans('csv.roles_title') }}</h3>
</div>
<div class="box-body">
<p>
{{ 'see_help_top_right'|_ }}
{{ trans('csv.roles_text') }}
</p>
</div>
</div>
</div>
</div>
<form action="{{ route('import.post-settings', job.key) }}" method="post">
<form action="{{ route('import.process-configuration', job.key) }}" method="post">
<input type="hidden" name="_token" value="{{ csrf_token() }}"/>
<input type="hidden" name="settings" value="roles"/>
@@ -29,41 +29,41 @@
<div class="col-lg-12">
<div class="box">
<div class="box-header with-border">
<h3 class="box-title">{{ trans('csv.column_roles_table') }}</h3>
<h3 class="box-title">{{ trans('csv.roles_table') }}</h3>
</div>
<div class="box-body">
<table class="table">
<thead>
<tr>
<th style="width:20%;">{{ trans('csv.column_name') }}</th>
<th style="width:40%;">{{ trans('csv.column_example') }}</th>
<th style="width:30%;">{{ trans('csv.column_role') }}</th>
<th style="width:10%;">{{ trans('csv.do_map_value') }}</th>
<th style="width:20%;">{{ trans('csv.roles_column_name') }}</th>
<th style="width:40%;">{{ trans('csv.roles_column_example') }}</th>
<th style="width:30%;">{{ trans('csv.roles_column_role') }}</th>
<th style="width:10%;">{{ trans('csv.roles_do_map_value') }}</th>
</tr>
</thead>
{% for i in 0..(data.columnCount-1) %}
{% for i in 0..(data.total -1) %}
<tr>
<td>
{% if data.columnHeaders[i] == '' %}
{{ trans('csv.column') }} #{{ loop.index }}
{% if data.headers[i] == '' %}
{{ trans('csv.roles_column') }} #{{ loop.index }}
{% else %}
{{ data.columnHeaders[i] }}
{{ data.headers[i] }}
{% endif %}
</td>
<td>
{% if data.columns[i]|length == 0 %}
<em>{{ trans('csv.no_example_data') }}</em>
{% if data.examples[i]|length == 0 %}
<em>{{ trans('csv.roles_no_example_data') }}</em>
{% else %}
{% for example in data.columns[i] %}
{% for example in data.examples[i] %}
<code>{{ example }}</code><br/>
{% endfor %}
{% endif %}
<td>
{{ Form.select(('role['~loop.index0~']'),
data.available_roles,
data.roles,
job.configuration['column-roles'][loop.index0],
{class: 'form-control'}) }}
</td>
@@ -91,7 +91,7 @@
<div class="box">
<div class="box-body">
<button type="submit" class="btn btn-success pull-right">
{{ trans('csv.store_column_roles') }} <i class="fa fa-arrow-right"></i>
{{ trans('csv.roles_store') }} <i class="fa fa-arrow-right"></i>
</button>
</div>
</div>

5
resources/views/import/index.twig
View File

@@ -23,20 +23,19 @@
</div>
<div class="row">
<form method="POST" action="{{ route('import.upload') }}" accept-charset="UTF-8" class="form-horizontal" id="update"
<form method="POST" action="{{ route('import.initialize') }}" accept-charset="UTF-8" class="form-horizontal" id="initialize"
enctype="multipart/form-data">
<input name="_token" type="hidden" value="{{ csrf_token() }}">
<div class="col-lg-8 col-md-8 col-sm-12 col-xs-12">
{{ ExpandedForm.file('import_file', {helpText: 'import_file_help'|_}) }}
{{ ExpandedForm.file('configuration_file', {helpText: 'configuration_file_help'|_|raw}) }}
{{ ExpandedForm.select('import_file_type', importFileTypes, defaultImportType, {'helpText' : 'import_file_type_help'|_}) }}
<div class="form-group" id="import_file_holder">
<label for="ffInput_submit" class="col-sm-4 control-label">&nbsp;</label>
<div class="col-sm-8">
<button type="submit" class="btn pull-right btn-success">
{{ ('import_start')|_ }}

55
resources/views/import/status.twig
View File

@@ -4,6 +4,59 @@
{{ Breadcrumbs.renderIfExists }}
{% endblock %}
{% block content %}
{# Initial display. Will refresh (and disappear almost immediately. #}
<div class="row status_initial">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-primary">
<div class="box-header with-border">
<h3 class="box-title">Nothing to see here...</h3>
</div>
<div class="box-body">
<p>This box will be replaced in a moment with the status of your import.</p>
</div>
</div>
</div>
</div>
<div class="row status_configured">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-primary">
<div class="box-header with-border">
<h3 class="box-title">{{ 'import_complete'|_ }}</h3>
</div>
<div class="box-body">
<p>
{{ 'import_complete_text'|_ }}
</p>
<p>
<code>php artisan firefly:start-import {{ job.key }}</code>
</p>
<div class="row">
<div class="col-lg-4">
<a href="{{ route('import.download', [job.key]) }}" class="btn btn-default"><i
class="fa fa-fw fa-download"></i> {{ 'import_download_config'|_ }}</a>
</div>
<div class="col-lg-4">
<a href="#" class="btn btn-success start-job"><i
class="fa fa-fw fa-gears"></i> {{ 'import_start_import'|_ }}</a>
</div>
</div>
<p>
&nbsp;
</p>
<p class="text-info">
{{ 'import_share_configuration'|_ }}
</p>
</div>
</div>
</div>
</div>
{#
<div class="row">
<div class="col-lg-8 col-lg-offset-2 col-md-12 col-sm-12">
<div class="box box-primary">
@@ -54,7 +107,7 @@
</div>
</div>
</div>
#}
{% endblock %}
{% block scripts %}
<script type="text/javascript">

5
routes/web.php
View File

@@ -380,7 +380,10 @@ Route::group(
*/
Route::group(
['middleware' => 'user-full-auth', 'prefix' => 'import', 'as' => 'import.'], function () {
Route::get('', ['uses' => 'ImportController@index', 'as' => 'index']);
Route::post('initialize', ['uses' => 'ImportController@initialize', 'as' => 'initialize']);
Route::get('configure/{importJob}', ['uses' => 'ImportController@configure', 'as' => 'configure']);
Route::get('settings/{importJob}', ['uses' => 'ImportController@settings', 'as' => 'settings']);
Route::get('complete/{importJob}', ['uses' => 'ImportController@complete', 'as' => 'complete']);
@@ -389,7 +392,7 @@ Route::group(
Route::get('json/{importJob}', ['uses' => 'ImportController@json', 'as' => 'json']);
Route::get('finished/{importJob}', ['uses' => 'ImportController@finished', 'as' => 'finished']);
Route::post('upload', ['uses' => 'ImportController@upload', 'as' => 'upload']);
Route::post('configure/{importJob}', ['uses' => 'ImportController@postConfigure', 'as' => 'process-configuration']);
Route::post('settings/{importJob}', ['uses' => 'ImportController@postSettings', 'as' => 'post-settings']);
Route::post('start/{importJob}', ['uses' => 'ImportController@start', 'as' => 'start']);