kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-10-12 15:35:15 +00:00
Code Issues Projects Releases Wiki Activity

Import routine cleanup.

Browse Source
This commit is contained in:
James Cole
2017-06-24 06:57:24 +02:00
parent 445dbf8779
commit e525e673a8
33 changed files with 245 additions and 583 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
app/Http/Controllers/ImportController.php
View File

@@ -17,8 +17,8 @@ use FireflyIII\Http\Requests\ImportUploadRequest;
use FireflyIII\Import\Configurator\ConfiguratorInterface;
use FireflyIII\Import\Routine\ImportRoutine;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use Illuminate\Http\Request;
use Illuminate\Http\Response as LaravelResponse;
use Log;
@@ -45,7 +45,7 @@ class ImportController extends Controller
$this->middleware(
function ($request, $next) {
View::share('mainTitleIcon', 'fa-archive');
View::share('title', trans('firefly.import_data_full'));
View::share('title', trans('firefly.import_index_title'));
$this->repository = app(ImportJobRepositoryInterface::class);
return $next($request);
@@ -74,7 +74,7 @@ class ImportController extends Controller
}
$view = $configurator->getNextView();
$data = $configurator->getNextData();
$subTitle = trans('firefly.configure_import');
$subTitle = trans('firefly.import_config_bread_crumb');
$subTitleIcon = 'fa-wrench';
return view($view, compact('data', 'job', 'subTitle', 'subTitleIcon'));
@@ -124,7 +124,7 @@ class ImportController extends Controller
*/
public function index()
{
$subTitle = trans('firefly.import_data_index');
$subTitle = trans('firefly.import_index_sub_title');
$subTitleIcon = 'fa-home';
$importFileTypes = [];
$defaultImportType = config('firefly.default_import_format');
@@ -176,30 +176,31 @@ class ImportController extends Controller
public function json(ImportJob $job)
{
$result = [
'started' => false,
'finished' => false,
'running' => false,
'errors' => array_values($job->extended_status['errors']),
'percentage' => 0,
'steps' => $job->extended_status['steps'],
'done' => $job->extended_status['done'],
'statusText' => trans('firefly.import_status_' . $job->status),
'status' => $job->status,
'finishedText' => '',
'started' => false,
'finished' => false,
'running' => false,
'errors' => array_values($job->extended_status['errors']),
'percentage' => 0,
'show_percentage' => false,
'steps' => $job->extended_status['steps'],
'done' => $job->extended_status['done'],
'statusText' => trans('firefly.import_status_job_' . $job->status),
'status' => $job->status,
'finishedText' => '',
];
if ($job->extended_status['steps'] !== 0) {
$result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0);
$result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0);
$result['show_percentage'] = true;
}
if ($job->status === 'finished') {
// $tagId = $job->extended_status['importTag'];
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $tag = $repository->find($tagId);
$tag = new Tag;
$tagId = $job->extended_status['tag'];
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$tag = $repository->find($tagId);
$result['finished'] = true;
$result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]);
$result['finishedText'] = trans('firefly.import_status_finished_job', ['link' => route('tags.show', [$tag->id, 'all']), 'tag' => $tag->tag]);
}
if ($job->status === 'running') {
@@ -262,7 +263,7 @@ class ImportController extends Controller
if (!in_array($job->status, $statuses)) {
return redirect(route('import.configure', [$job->key]));
}
$subTitle = trans('firefly.import_status');
$subTitle = trans('firefly.import_status_sub_title');
$subTitleIcon = 'fa-star';
return view('import.status', compact('job', 'subTitle', 'subTitleIcon'));

14
app/Http/breadcrumbs.php
View File

@@ -469,26 +469,20 @@ Breadcrumbs::register(
$breadcrumbs->push(trans('firefly.import'), route('import.index'));
}
);
Breadcrumbs::register(
'import.complete', function (BreadCrumbGenerator $breadcrumbs, ImportJob $job) {
$breadcrumbs->parent('import.index');
$breadcrumbs->push(trans('firefly.bread_crumb_import_complete', ['key' => $job->key]), route('import.complete', [$job->key]));
}
);
Breadcrumbs::register(
'import.configure', function (BreadCrumbGenerator $breadcrumbs, ImportJob $job) {
$breadcrumbs->parent('import.index');
$breadcrumbs->push(trans('firefly.bread_crumb_configure_import', ['key' => $job->key]), route('import.configure', [$job->key]));
$breadcrumbs->push(trans('firefly.import_config_sub_title', ['key' => $job->key]), route('import.configure', [$job->key]));
}
);
Breadcrumbs::register(
'import.finished', function (BreadCrumbGenerator $breadcrumbs, ImportJob $job) {
'import.status', function (BreadCrumbGenerator $breadcrumbs, ImportJob $job) {
$breadcrumbs->parent('import.index');
$breadcrumbs->push(trans('firefly.bread_crumb_import_finished', ['key' => $job->key]), route('import.finished', [$job->key]));
$breadcrumbs->push(trans('firefly.import_status_bread_crumb', ['key' => $job->key]), route('import.status', [$job->key]));
}
);
/**
* PREFERENCES
*/

11
app/Import/Converter/Amount.php
View File

@@ -18,7 +18,7 @@ namespace FireflyIII\Import\Converter;
*
* @package FireflyIII\Import\Converter
*/
class Amount extends BasicConverter implements ConverterInterface
class Amount implements ConverterInterface
{
/**
@@ -28,9 +28,9 @@ class Amount extends BasicConverter implements ConverterInterface
*
* @param $value
*
* @return float
* @return string
*/
public function convert($value): float
public function convert($value): string
{
$len = strlen($value);
$decimalPosition = $len - 3;
@@ -59,10 +59,7 @@ class Amount extends BasicConverter implements ConverterInterface
$value = str_replace($search, '', $value);
}
$this->setCertainty(90);
return round(floatval($value), 12);
return strval(round(floatval($value), 12));
}
}

85
app/Import/Converter/BasicConverter.php
View File

@@ -1,85 +0,0 @@
<?php
/**
* BasicConverter.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\User;
/**
* Class BasicConverter
*
* @package FireflyIII\Import\Converter
*/
class BasicConverter
{
/** @var int */
public $certainty = 50;
/** @var array */
public $config;
/** @var bool */
public $doMap;
/** @var array */
public $mapping = [];
/** @var User */
public $user;
/**
* @return int
*/
public function getCertainty(): int
{
return $this->certainty;
}
/**
* @param int $certainty
*/
protected function setCertainty(int $certainty)
{
$this->certainty = $certainty;
}
/**
* @param array $config
*/
public function setConfig(array $config)
{
$this->config = $config;
}
/**
* @param mixed $doMap
*/
public function setDoMap(bool $doMap)
{
$this->doMap = $doMap;
}
/**
* @param array $mapping
*
*/
public function setMapping(array $mapping)
{
$this->mapping = $mapping;
}
/**
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
}
}

28
app/Import/Converter/ConverterInterface.php
View File

@@ -13,8 +13,6 @@ declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\User;
/**
* Interface ConverterInterface
*
@@ -27,30 +25,4 @@ interface ConverterInterface
*
*/
public function convert($value);
/**
* @return int
*/
public function getCertainty(): int;
/**
* @param array $config
*/
public function setConfig(array $config);
/**
* @param bool $doMap
*/
public function setDoMap(bool $doMap);
/**
* @param array $mapping
*
*/
public function setMapping(array $mapping);
/**
* @param User $user
*/
public function setUser(User $user);
}

4
app/Import/Converter/INGDebetCredit.php
View File

@@ -20,7 +20,7 @@ use Log;
*
* @package FireflyIII\Import\Converter
*/
class INGDebetCredit extends BasicConverter implements ConverterInterface
class INGDebetCredit implements ConverterInterface
{
/**
@@ -34,12 +34,10 @@ class INGDebetCredit extends BasicConverter implements ConverterInterface
if ($value === 'Af') {
Log::debug('Return -1');
$this->setCertainty(100);
return -1;
}
$this->setCertainty(100);
Log::debug('Return 1');
return 1;

4
app/Import/Converter/RabobankDebetCredit.php
View File

@@ -20,7 +20,7 @@ use Log;
*
* @package FireflyIII\Import\Converter
*/
class RabobankDebetCredit extends BasicConverter implements ConverterInterface
class RabobankDebetCredit implements ConverterInterface
{
/**
@@ -34,13 +34,11 @@ class RabobankDebetCredit extends BasicConverter implements ConverterInterface
if ($value === 'D') {
Log::debug('Return -1');
$this->setCertainty(100);
return -1;
}
Log::debug('Return 1');
$this->setCertainty(100);
return 1;
}

86
app/Import/Converter/TagSplit.php
View File

@@ -1,86 +0,0 @@
<?php
/**
* TagSplit.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use FireflyIII\User;
use Illuminate\Support\Collection;
use Log;
/**
* Class TagSplit
*
* @package FireflyIII\Import\Converter
*/
class TagSplit
{
/**
* @param User $user
* @param array $mapping
* @param array $parts
*
* @return Collection
*/
public static function createSetFromSplits(User $user, array $mapping, array $parts): Collection
{
$set = new Collection;
Log::debug('Exploded parts.', $parts);
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($user);
/** @var string $part */
foreach ($parts as $part) {
if (isset($mapping[$part])) {
Log::debug('Found tag in mapping. Should exist.', ['value' => $part, 'map' => $mapping[$part]]);
$tag = $repository->find(intval($mapping[$part]));
if (!is_null($tag->id)) {
Log::debug('Found tag by ID', ['id' => $tag->id]);
$set->push($tag);
continue;
}
}
// not mapped? Still try to find it first:
$tag = $repository->findByTag($part);
if (!is_null($tag->id)) {
Log::debug('Found tag by name ', ['id' => $tag->id]);
$set->push($tag);
}
if (is_null($tag->id)) {
// create new tag
$tag = $repository->store(
[
'tag' => $part,
'date' => null,
'description' => $part,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
]
);
Log::debug('Created new tag', ['name' => $part, 'id' => $tag->id]);
$set->push($tag);
}
}
return $set;
}
}

48
app/Import/Converter/TagsComma.php
View File

@@ -1,48 +0,0 @@
<?php
/**
* TagsComma.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use Illuminate\Support\Collection;
use Log;
/**
* Class TagsComma
*
* @package FireflyIII\Import\Converter
*/
class TagsComma extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Collection
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using TagsComma', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Collection;
}
$parts = array_unique(explode(',', $value));
$set = TagSplit::createSetFromSplits($this->user, $this->mapping, $parts);
$this->setCertainty(100);
return $set;
}
}

49
app/Import/Converter/TagsSpace.php
View File

@@ -1,49 +0,0 @@
<?php
/**
* TagsSpace.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use Illuminate\Support\Collection;
use Log;
/**
* Class TagsSpace
*
* @package FireflyIII\Import\Converter
*/
class TagsSpace extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Collection
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using TagsSpace', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Collection;
}
$parts = array_unique(explode(' ', $value));
$set = TagSplit::createSetFromSplits($this->user, $this->mapping, $parts);
$this->setCertainty(100);
return $set;
}
}

1
app/Import/FileProcessor/CsvProcessor.php
View File

@@ -70,6 +70,7 @@ class CsvProcessor implements FileProcessorInterface
$index = 0;
Log::notice('Building importable objects from CSV file.');
foreach ($entries as $index => $row) {
sleep(1);
// verify if not exists already:
if ($this->rowAlreadyImported($row)) {
$message = sprintf('Row #%d has already been imported.', $index);

2
app/Import/Mapper/AssetAccountIbans.php
View File

@@ -50,7 +50,7 @@ class AssetAccountIbans implements MapperInterface
asort($list);
$list = $topList + $list;
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Mapper/AssetAccounts.php
View File

@@ -47,7 +47,7 @@ class AssetAccounts implements MapperInterface
asort($list);
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Mapper/Bills.php
View File

@@ -40,7 +40,7 @@ class Bills implements MapperInterface
}
asort($list);
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Mapper/Budgets.php
View File

@@ -41,7 +41,7 @@ class Budgets implements MapperInterface
}
asort($list);
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Mapper/Categories.php
View File

@@ -41,7 +41,7 @@ class Categories implements MapperInterface
}
asort($list);
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Mapper/OpposingAccountIbans.php
View File

@@ -56,7 +56,7 @@ class OpposingAccountIbans implements MapperInterface
asort($list);
$list = $topList + $list;
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Mapper/OpposingAccounts.php
View File

@@ -53,7 +53,7 @@ class OpposingAccounts implements MapperInterface
asort($list);
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;
}

2
app/Import/Mapper/Tags.php
View File

@@ -40,7 +40,7 @@ class Tags implements MapperInterface
}
asort($list);
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Mapper/TransactionCurrencies.php
View File

@@ -36,7 +36,7 @@ class TransactionCurrencies implements MapperInterface
asort($list);
$list = [0 => trans('csv.do_not_map')] + $list;
$list = [0 => trans('csv.map_do_not_map')] + $list;
return $list;

2
app/Import/Object/ImportJournal.php
View File

@@ -104,7 +104,7 @@ class ImportJournal
/** @var ConverterInterface $amountConverter */
$amountConverter = app(Amount::class);
$this->amount = strval($amountConverter->convert($this->amount));
$this->amount = $amountConverter->convert($this->amount);
// modify
foreach ($this->modifiers as $modifier) {
$class = sprintf('FireflyIII\Import\Converter\%s', config(sprintf('csv.import_roles.%s.converter', $modifier['role'])));

96
app/Import/Routine/ImportRoutine.php
View File

@@ -12,9 +12,13 @@ declare(strict_types=1);
namespace FireflyIII\Import\Routine;
use Carbon\Carbon;
use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Models\TransactionJournal;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use Illuminate\Support\Collection;
use Log;
@@ -53,8 +57,37 @@ class ImportRoutine
return false;
}
set_time_limit(0);
Log::debug(sprintf('Start with import job %s', $this->job->key));
$importObjects = $this->getImportObjects();
$this->lines = $importObjects->count();
// once done, use storage thing to actually store them:
Log::debug(sprintf('Returned %d valid objects from file processor', $this->lines));
$storage = $this->storeObjects($importObjects);
// update job:
$this->job->status = 'finished';
$this->job->save();
$this->journals = $storage->journals;
$this->errors = $storage->errors;
// create tag, link tag to all journals:
$this->createImportTag();
Log::debug(sprintf('Done with import job %s', $this->job->key));
return true;
}
/**
* @return Collection
*/
protected function getImportObjects(): Collection
{
$objects = new Collection;
$type = $this->job->file_type;
$class = config(sprintf('firefly.import_processors.%s', $type));
@@ -62,7 +95,6 @@ class ImportRoutine
$processor = app($class);
$processor->setJob($this->job);
set_time_limit(0);
if ($this->job->status == 'configured') {
// set job as "running"...
@@ -73,28 +105,56 @@ class ImportRoutine
$processor->run();
$objects = $processor->getObjects();
}
$this->lines = $objects->count();
// once done, use storage thing to actually store them:
Log::debug(sprintf('Returned %d valid objects from file processor', $this->lines));
return $objects;
}
/**
*
*/
private function createImportTag(): Tag
{
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->job->user);
$data = [
'tag' => trans('firefly.import_with_key', ['key' => $this->job->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
$extended = $this->job->extended_status;
$extended['tag'] = $tag->id;
$this->job->extended_status = $extended;
$this->job->save();
$this->journals->each(
function (TransactionJournal $journal) use ($tag) {
$journal->tags()->save($tag);
}
);
return $tag;
}
/**
* @param Collection $objects
*
* @return ImportStorage
*/
private function storeObjects(Collection $objects): ImportStorage
{
$storage = new ImportStorage;
$storage->setJob($this->job);
$storage->setDateFormat($this->job->configuration['date-format']);
$storage->setObjects($objects);
$storage->store();
// update job:
$this->job->status = 'finished';
$this->job->save();
$this->journals = $storage->journals;
$this->errors = $storage->errors;
// run rules:
Log::debug(sprintf('Done with import job %s', $this->job->key));
return true;
return $storage;
}
}

2
app/Import/Storage/ImportStorage.php
View File

@@ -99,7 +99,7 @@ class ImportStorage
* @var ImportJournal $object
*/
foreach ($this->objects as $index => $object) {
sleep(4);
sleep(1);
Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $object->description));
$errors = new MessageBag;

8
app/Repositories/ImportJob/ImportJobRepository.php
View File

@@ -60,10 +60,10 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$importJob->status = 'new';
$importJob->configuration = [];
$importJob->extended_status = [
'steps' => 0,
'done' => 0,
'importTag' => 0,
'errors' => [],
'steps' => 0,
'done' => 0,
'tag' => 0,
'errors' => [],
];
$importJob->save();