kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-10-12 15:35:15 +00:00
Code Issues Projects Releases Wiki Activity

More and improved code for the import routine.

Browse Source
This commit is contained in:
James Cole
2017-06-24 05:49:33 +02:00
parent edb355941c
commit 445dbf8779
32 changed files with 662 additions and 1874 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
app/Http/Controllers/ImportController.php
View File

@@ -15,30 +15,26 @@ namespace FireflyIII\Http\Controllers;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Http\Requests\ImportUploadRequest;
use FireflyIII\Import\Configurator\ConfiguratorInterface;
use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\ImportProcedureInterface;
use FireflyIII\Import\Routine\ImportRoutine;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use Illuminate\Http\Request;
use Illuminate\Http\Response as LaravelResponse;
use Illuminate\Support\Collection;
use Log;
use Response;
use Validator;
use View;
/**
* Class ImportController
* Class ImportController.
*
* @package FireflyIII\Http\Controllers
*/
class ImportController extends Controller
{
/** @var ImportJobRepositoryInterface */
public $repository;
/**
*
*/
@@ -50,6 +46,7 @@ class ImportController extends Controller
function ($request, $next) {
View::share('mainTitleIcon', 'fa-archive');
View::share('title', trans('firefly.import_data_full'));
$this->repository = app(ImportJobRepositoryInterface::class);
return $next($request);
}
@@ -71,6 +68,8 @@ class ImportController extends Controller
// is the job already configured?
if ($configurator->isJobConfigured()) {
$this->repository->updateStatus($job, 'configured');
return redirect(route('import.status', [$job->key]));
}
$view = $configurator->getNextView();
@@ -82,11 +81,11 @@ class ImportController extends Controller
}
/**
* Generate a JSON file of the job's config and send it to the user.
* Generate a JSON file of the job's configuration and send it to the user.
*
* @param ImportJob $job
*
* @return mixed
* @return string
*/
public function download(ImportJob $job)
{
@@ -140,70 +139,72 @@ class ImportController extends Controller
/**
* This is step 2. It creates an Import Job. Stores the import.
*
* @param ImportUploadRequest $request
* @param ImportJobRepositoryInterface $repository
* @param UserRepositoryInterface $userRepository
* @param ImportUploadRequest $request
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function initialize(ImportUploadRequest $request, ImportJobRepositoryInterface $repository, UserRepositoryInterface $userRepository)
public function initialize(ImportUploadRequest $request)
{
Log::debug('Now in initialize()');
// create import job:
$type = $request->get('import_file_type');
$job = $repository->create($type);
$job = $this->repository->create($type);
Log::debug('Created new job', ['key' => $job->key, 'id' => $job->id]);
// process file:
$repository->processFile($job, $request->files->get('import_file'));
$this->repository->processFile($job, $request->files->get('import_file'));
// process config, if present:
if ($request->files->has('configuration_file')) {
$repository->processConfiguration($job, $request->files->get('configuration_file'));
$this->repository->processConfiguration($job, $request->files->get('configuration_file'));
}
$this->repository->updateStatus($job, 'initialized');
return redirect(route('import.configure', [$job->key]));
}
/**
*
* Show status of import job in JSON.
*
* @param ImportJob $job
*
* @return \Illuminate\Http\JsonResponse
*/
public function json(ImportJob $job)
{
$result = [
$result = [
'started' => false,
'finished' => false,
'running' => false,
'errors' => $job->extended_status['errors'],
'errors' => array_values($job->extended_status['errors']),
'percentage' => 0,
'steps' => $job->extended_status['total_steps'],
'stepsDone' => $job->extended_status['steps_done'],
'steps' => $job->extended_status['steps'],
'done' => $job->extended_status['done'],
'statusText' => trans('firefly.import_status_' . $job->status),
'status' => $job->status,
'finishedText' => '',
];
$percentage = 0;
if ($job->extended_status['total_steps'] !== 0) {
$percentage = round(($job->extended_status['steps_done'] / $job->extended_status['total_steps']) * 100, 0);
if ($job->extended_status['steps'] !== 0) {
$result['percentage'] = round(($job->extended_status['done'] / $job->extended_status['steps']) * 100, 0);
}
if ($job->status === 'finished') {
// $tagId = $job->extended_status['importTag'];
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $tag = $repository->find($tagId);
$tag = new Tag;
// $tagId = $job->extended_status['importTag'];
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $tag = $repository->find($tagId);
$tag = new Tag;
$result['finished'] = true;
$result['finishedText'] = trans('firefly.import_finished_link', ['link' => route('tags.show', [$tag->id]), 'tag' => $tag->tag]);
}
if ($job->status === 'running') {
$result['started'] = true;
$result['running'] = true;
$result['percentage'] = $percentage;
$result['showPercentage'] = true;
$result['started'] = true;
$result['running'] = true;
}
return Response::json($result);
@@ -212,13 +213,12 @@ class ImportController extends Controller
/**
* Step 4. Save the configuration.
*
* @param Request $request
* @param ImportJobRepositoryInterface $repository
* @param ImportJob $job
* @param Request $request
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function postConfigure(Request $request, ImportJobRepositoryInterface $repository, ImportJob $job)
public function postConfigure(Request $request, ImportJob $job)
{
Log::debug('Now in postConfigure()', ['job' => $job->key]);
$configurator = $this->makeConfigurator($job);
@@ -237,25 +237,31 @@ class ImportController extends Controller
/**
* @param ImportJob $job
*
* @return string
* @return \Illuminate\Http\JsonResponse
* @throws FireflyException
*/
public function start(ImportJob $job)
{
$routine = new ImportRoutine($job);
$routine->run();
$result = $routine->run();
if ($result) {
return Response::json(['run' => 'ok']);
}
return 'done!';
throw new FireflyException('Job did not complete succesfully.');
}
/**
* This is the last step before the import starts.
*
* @param ImportJob $job
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector|View
*/
public function status(ImportJob $job)
{
$statuses = ['configured', 'running', 'finished'];
if (!in_array($job->status, $statuses)) {
return redirect(route('import.configure', [$job->key]));
}
$subTitle = trans('firefly.import_status');
$subTitleIcon = 'fa-star';

3
app/Import/Configurator/CsvConfigurator.php
View File

@@ -103,9 +103,6 @@ class CsvConfigurator implements ConfiguratorInterface
&& $this->job->configuration['column-roles-complete']
&& $this->job->configuration['column-mapping-complete']
) {
$this->job->status = 'configured';
$this->job->save();
return true;
}

69
app/Import/Converter/AccountId.php
View File

@@ -1,69 +0,0 @@
<?php
/**
* AccountId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AccountId
*
* @package FireflyIII\Import\Converter
*/
class AccountId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using AssetAccountId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
$account = $repository->find($value);// not mapped? Still try to find it first:
if (!is_null($account->id)) {
$this->setCertainty(90);
Log::debug('Found account by ID ', ['id' => $account->id]);
return $account;
}
$this->setCertainty(0); // should not really happen. If the ID does not match FF, what is FF supposed to do?
return new Account;
}
}

87
app/Import/Converter/AssetAccountIban.php
View File

@@ -1,87 +0,0 @@
<?php
/**
* AssetAccountIban.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AssetAccountIban
*
* @package FireflyIII\Import\Converter
*/
class AssetAccountIban extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value): Account
{
$value = trim($value);
Log::debug('Going to convert ', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
$this->setCertainty(100);
Log::debug('Found account by ID', ['id' => $account->id]);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByIban($value, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found account by IBAN', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => 'Asset account with IBAN ' . $value, 'iban' => $value, 'user' => $this->user->id, 'accountType' => 'asset', 'virtualBalance' => 0,
'active' => true, 'openingBalance' => 0]
);
if (is_null($account->id)) {
$this->setCertainty(0);
Log::info('Could not store new asset account by IBAN', $account->getErrors()->toArray());
return new Account;
}
$this->setCertainty(100);
return $account;
}
}

90
app/Import/Converter/AssetAccountName.php
View File

@@ -1,90 +0,0 @@
<?php
/**
* AssetAccountName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AssetAccountName
*
* @package FireflyIII\Import\Converter
*/
class AssetAccountName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using AssetAccountName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByName($value, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found asset account by name', ['value' => $value, 'id' => $account->id]);
return $account;
}
$account = $repository->store(
['name' => $value, 'iban' => null, 'openingBalance' => 0, 'user' => $this->user->id, 'accountType' => 'asset', 'virtualBalance' => 0,
'active' => true]
);
if (is_null($account->id)) {
$this->setCertainty(0);
Log::info('Could not store new asset account by name', $account->getErrors()->toArray());
return new Account;
}
$this->setCertainty(100);
Log::debug('Created new asset account ', ['name' => $account->name, 'id' => $account->id]);
return $account;
}
}

96
app/Import/Converter/AssetAccountNumber.php
View File

@@ -1,96 +0,0 @@
<?php
/**
* AssetAccountNumber.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class AssetAccountNumber
*
* @package FireflyIII\Import\Converter
*/
class AssetAccountNumber extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using AssetAccountNumber', ['value' => $value]);
if (strlen($value) === 0) {
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByAccountNumber($value, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
// try to find by the name we would give it:
$accountName = 'Asset account with number ' . e($value);
$account = $repository->findByName($accountName, [AccountType::ASSET]);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => $accountName, 'openingBalance' => 0, 'iban' => null, 'user' => $this->user->id,
'accountType' => 'asset',
'virtualBalance' => 0, 'accountNumber' => $value, 'active' => true]
);
if (is_null($account->id)) {
$this->setCertainty(0);
Log::info('Could not store new asset account by account number', $account->getErrors()->toArray());
return new Account;
}
$this->setCertainty(100);
return $account;
}
}

76
app/Import/Converter/BillId.php
View File

@@ -1,76 +0,0 @@
<?php
/**
* BillId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Bill;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use Log;
/**
* Class BillId
*
* @package FireflyIII\Import\Converter
*/
class BillId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Bill
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using BillId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Bill;
}
/** @var BillRepositoryInterface $repository */
$repository = app(BillRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found bill in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$bill = $repository->find(intval($this->mapping[$value]));
if (!is_null($bill->id)) {
Log::debug('Found bill by ID', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
}
// not mapped? Still try to find it first:
$bill = $repository->find($value);
if (!is_null($bill->id)) {
Log::debug('Found bill by ID ', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
// should not really happen. If the ID does not match FF, what is FF supposed to do?
Log::info(sprintf('Could not find bill with ID %d. Will return NULL', $value));
$this->setCertainty(0);
return new Bill;
}
}

99
app/Import/Converter/BillName.php
View File

@@ -1,99 +0,0 @@
<?php
/**
* BillName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Bill;
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
use Log;
/**
* Class BillName
*
* @package FireflyIII\Import\Converter
*/
class BillName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Bill
* @throws FireflyException
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using BillName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Bill;
}
/** @var BillRepositoryInterface $repository */
$repository = app(BillRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found bill in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$bill = $repository->find(intval($this->mapping[$value]));
if (!is_null($bill->id)) {
Log::debug('Found bill by ID', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
}
// not mapped? Still try to find it first:
$bill = $repository->findByName($value);
if (!is_null($bill->id)) {
Log::debug('Found bill by name ', ['id' => $bill->id]);
$this->setCertainty(100);
return $bill;
}
// create new bill. Use a lot of made up values.
$bill = $repository->store(
[
'name' => $value,
'match' => $value,
'amount_min' => 1,
'user' => $this->user->id,
'amount_max' => 10,
'date' => date('Ymd'),
'repeat_freq' => 'monthly',
'skip' => 0,
'automatch' => 0,
'active' => 1,
]
);
if (is_null($bill->id)) {
$this->setCertainty(0);
Log::info('Could not store new bill by name', $bill->getErrors()->toArray());
return new Bill;
}
$this->setCertainty(100);
return $bill;
}
}

76
app/Import/Converter/BudgetId.php
View File

@@ -1,76 +0,0 @@
<?php
/**
* BudgetId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Budget;
use FireflyIII\Repositories\Budget\BudgetRepositoryInterface;
use Log;
/**
* Class BudgetId
*
* @package FireflyIII\Import\Converter
*/
class BudgetId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Budget
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using BudgetId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Budget;
}
/** @var BudgetRepositoryInterface $repository */
$repository = app(BudgetRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found budget in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$budget = $repository->find(intval($this->mapping[$value]));
if (!is_null($budget->id)) {
Log::debug('Found budget by ID', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
}
// not mapped? Still try to find it first:
$budget = $repository->find($value);
if (!is_null($budget->id)) {
Log::debug('Found budget by ID ', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
// should not really happen. If the ID does not match FF, what is FF supposed to do?
$this->setCertainty(0);
Log::info(sprintf('Could not find budget with ID %d. Will return NULL', $value));
return new Budget;
}
}

80
app/Import/Converter/BudgetName.php
View File

@@ -1,80 +0,0 @@
<?php
/**
* BudgetName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Budget;
use FireflyIII\Repositories\Budget\BudgetRepositoryInterface;
use Log;
/**
* Class BudgetName
*
* @package FireflyIII\Import\Converter
*/
class BudgetName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Budget
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using BudgetName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Budget;
}
/** @var BudgetRepositoryInterface $repository */
$repository = app(BudgetRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found budget in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$budget = $repository->find(intval($this->mapping[$value]));
if (!is_null($budget->id)) {
Log::debug('Found budget by ID', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
}
// not mapped? Still try to find it first:
$budget = $repository->findByName($value);
if (!is_null($budget->id)) {
Log::debug('Found budget by name ', ['id' => $budget->id]);
$this->setCertainty(100);
return $budget;
}
// create new budget. Use a lot of made up values.
$budget = $repository->store(
[
'name' => $value,
'user' => $this->user->id,
]
);
$this->setCertainty(100);
return $budget;
}
}

76
app/Import/Converter/CategoryId.php
View File

@@ -1,76 +0,0 @@
<?php
/**
* CategoryId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Category;
use FireflyIII\Repositories\Category\CategoryRepositoryInterface;
use Log;
/**
* Class CategoryId
*
* @package FireflyIII\Import\Converter
*/
class CategoryId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Category
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using CategoryId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new Category;
}
/** @var CategoryRepositoryInterface $repository */
$repository = app(CategoryRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found category in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$category = $repository->find(intval($this->mapping[$value]));
if (!is_null($category->id)) {
Log::debug('Found category by ID', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
}
// not mapped? Still try to find it first:
$category = $repository->find($value);
if (!is_null($category->id)) {
Log::debug('Found category by ID ', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
// should not really happen. If the ID does not match FF, what is FF supposed to do?
$this->setCertainty(0);
Log::info(sprintf('Could not find category with ID %d. Will return NULL', $value));
return new Category;
}
}

80
app/Import/Converter/CategoryName.php
View File

@@ -1,80 +0,0 @@
<?php
/**
* CategoryName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Category;
use FireflyIII\Repositories\Category\CategoryRepositoryInterface;
use Log;
/**
* Class CategoryName
*
* @package FireflyIII\Import\Converter
*/
class CategoryName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Category
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using CategoryName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Category;
}
/** @var CategoryRepositoryInterface $repository */
$repository = app(CategoryRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found category in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$category = $repository->find(intval($this->mapping[$value]));
if (!is_null($category->id)) {
Log::debug('Found category by ID', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
}
// not mapped? Still try to find it first:
$category = $repository->findByName($value);
if (!is_null($category->id)) {
Log::debug('Found category by name ', ['id' => $category->id]);
$this->setCertainty(100);
return $category;
}
// create new category. Use a lot of made up values.
$category = $repository->store(
[
'name' => $value,
'user' => $this->user->id,
]
);
$this->setCertainty(100);
return $category;
}
}

71
app/Import/Converter/CurrencyCode.php
View File

@@ -1,71 +0,0 @@
<?php
/**
* CurrencyCode.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencyCode
*
* @package FireflyIII\Import\Converter
*/
class CurrencyCode extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value): TransactionCurrency
{
Log::debug('Going to convert currency code', ['value' => $value]);
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->findByCode($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by code', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
$currency = $repository->store(
[
'name' => $value,
'code' => $value,
'symbol' => $value,
]
);
$this->setCertainty(100);
return $currency;
}
}

75
app/Import/Converter/CurrencyId.php
View File

@@ -1,75 +0,0 @@
<?php
/**
* CurrencyId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencyId
*
* @package FireflyIII\Import\Converter
*/
class CurrencyId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value)
{
$value = intval(trim($value));
Log::debug('Going to convert using CurrencyId', ['value' => $value]);
if ($value === 0) {
$this->setCertainty(0);
return new TransactionCurrency;
}
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->find($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by ID ', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
$this->setCertainty(0);
// should not really happen. If the ID does not match FF, what is FF supposed to do?
Log::info(sprintf('Could not find category with ID %d. Will return NULL', $value));
return new TransactionCurrency;
}
}

81
app/Import/Converter/CurrencyName.php
View File

@@ -1,81 +0,0 @@
<?php
/**
* CurrencyName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencyName
*
* @package FireflyIII\Import\Converter
*/
class CurrencyName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using CurrencyName', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new TransactionCurrency;
}
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->findByName($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by name ', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
// create new currency
$currency = $repository->store(
[
'name' => $value,
'code' => strtoupper(substr($value, 0, 3)),
'symbol' => strtoupper(substr($value, 0, 1)),
]
);
$this->setCertainty(100);
return $currency;
}
}

81
app/Import/Converter/CurrencySymbol.php
View File

@@ -1,81 +0,0 @@
<?php
/**
* CurrencySymbol.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
use Log;
/**
* Class CurrencySymbol
*
* @package FireflyIII\Import\Converter
*/
class CurrencySymbol extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return TransactionCurrency
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using CurrencySymbol', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new TransactionCurrency;
}
/** @var CurrencyRepositoryInterface $repository */
$repository = app(CurrencyRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found currency in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$currency = $repository->find(intval($this->mapping[$value]));
if (!is_null($currency->id)) {
Log::debug('Found currency by ID', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
}
// not mapped? Still try to find it first:
$currency = $repository->findBySymbol($value);
if (!is_null($currency->id)) {
Log::debug('Found currency by symbol ', ['id' => $currency->id]);
$this->setCertainty(100);
return $currency;
}
// create new currency
$currency = $repository->store(
[
'name' => 'Currency ' . $value,
'code' => $value,
'symbol' => $value,
]
);
$this->setCertainty(100);
return $currency;
}
}

53
app/Import/Converter/Date.php
View File

@@ -1,53 +0,0 @@
<?php
/**
* Date.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use Carbon\Carbon;
use FireflyIII\Exceptions\FireflyException;
use InvalidArgumentException;
use Log;
/**
* Class Date
*
* @package FireflyIII\Import\Converter
*/
class Date extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Carbon
* @throws FireflyException
*/
public function convert($value): Carbon
{
Log::debug('Going to convert date', ['value' => $value]);
Log::debug('Format: ', ['format' => $this->config['date-format']]);
try {
$date = Carbon::createFromFormat($this->config['date-format'], $value);
} catch (InvalidArgumentException $e) {
Log::info($e->getMessage());
Log::info('Cannot convert this string using the given format.', ['value' => $value, 'format' => $this->config['date-format']]);
$this->setCertainty(0);
return new Carbon;
}
Log::debug('Converted date', ['converted' => $date->toAtomString()]);
$this->setCertainty(100);
return $date;
}
}

39
app/Import/Converter/Description.php
View File

@@ -1,39 +0,0 @@
<?php
/**
* Description.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
/**
* Class Description
*
* @package FireflyIII\Import\Converter
*/
class Description extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return string
*/
public function convert($value): string
{
// this should replace all control characters
// but leave utf8 intact:
$value = preg_replace('/[\x00-\x1F\x80-\x9F]/u', '', $value);
$this->setCertainty(100);
return strval($value);
}
}

39
app/Import/Converter/ExternalId.php
View File

@@ -1,39 +0,0 @@
<?php
/**
* ExternalId.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
/**
* Class ExternalId
*
* @package FireflyIII\Import\Converter
*/
class ExternalId extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return string
*/
public function convert($value): string
{
// this should replace all control characters
// but leave utf8 intact:
$value = preg_replace('/[\x00-\x1F\x80-\x9F]/u', '', $value);
$this->setCertainty(100);
return strval(trim($value));
}
}

34
app/Import/Converter/Ignore.php
View File

@@ -1,34 +0,0 @@
<?php
/**
* Ignore.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
/**
* Class Ignore
*
* @package FireflyIII\Import\Converter
*/
class Ignore extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return null
*/
public function convert($value)
{
return null;
}
}

91
app/Import/Converter/OpposingAccountIban.php
View File

@@ -1,91 +0,0 @@
<?php
/**
* OpposingAccountIban.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Account;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class OpposingAccountIban
*
* @package FireflyIII\Import\Converter
*/
class OpposingAccountIban extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value): Account
{
$value = trim($value);
Log::debug('Going to convert opposing IBAN', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByIban($value, []);
if (!is_null($account->id)) {
Log::debug('Found account by IBAN', ['id' => $account->id]);
Log::info(
'The match between IBAN and account is uncertain because the type of transactions may not have been determined.',
['id' => $account->id, 'iban' => $value]
);
$this->setCertainty(50);
return $account;
}
// the IBAN given may not be a valid IBAN. If not, we cannot store by
// iban and we have no opposing account. There should be some kind of fall back
// routine.
try {
$account = $repository->store(
['name' => $value, 'iban' => $value, 'user' => $this->user->id, 'accountType' => 'import', 'virtualBalance' => 0, 'active' => true,
'openingBalance' => 0]
);
$this->setCertainty(100);
} catch (FireflyException $e) {
Log::error($e);
$account = new Account;
}
return $account;
}
}

89
app/Import/Converter/OpposingAccountName.php
View File

@@ -1,89 +0,0 @@
<?php
/**
* OpposingAccountName.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class OpposingAccountName
*
* @package FireflyIII\Import\Converter
*/
class OpposingAccountName extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value): Account
{
$value = trim($value);
Log::debug('Going to convert opposing account name', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByName($value, []);
if (!is_null($account->id)) {
Log::debug('Found opposing account by name', ['id' => $account->id]);
Log::info(
'The match between name and account is uncertain because the type of transactions may not have been determined.',
['id' => $account->id, 'name' => $value]
);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => $value, 'iban' => null, 'user' => $this->user->id, 'accountType' => 'import', 'virtualBalance' => 0, 'active' => true,
'openingBalance' => 0,
]
);
if (is_null($account->id)) {
$this->setCertainty(0);
return new Account;
}
$this->setCertainty(100);
Log::debug('Created new opposing account ', ['name' => $account->name, 'id' => $account->id]);
return $account;
}
}

91
app/Import/Converter/OpposingAccountNumber.php
View File

@@ -1,91 +0,0 @@
<?php
/**
* OpposingAccountNumber.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Converter;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use Log;
/**
* Class OpposingAccountNumber
*
* @package FireflyIII\Import\Converter
*/
class OpposingAccountNumber extends BasicConverter implements ConverterInterface
{
/**
* @param $value
*
* @return Account
*/
public function convert($value)
{
$value = trim($value);
Log::debug('Going to convert using OpposingAccountNumber', ['value' => $value]);
if (strlen($value) === 0) {
$this->setCertainty(0);
return new Account;
}
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->user);
if (isset($this->mapping[$value])) {
Log::debug('Found account in mapping. Should exist.', ['value' => $value, 'map' => $this->mapping[$value]]);
$account = $repository->find(intval($this->mapping[$value]));
if (!is_null($account->id)) {
Log::debug('Found account by ID', ['id' => $account->id]);
$this->setCertainty(100);
return $account;
}
}
// not mapped? Still try to find it first:
$account = $repository->findByAccountNumber($value, []);
if (!is_null($account->id)) {
Log::debug('Found account by number', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
// try to find by the name we would give it:
$accountName = 'Import account with number ' . e($value);
$account = $repository->findByName($accountName, [AccountType::IMPORT]);
if (!is_null($account->id)) {
Log::debug('Found account by name', ['id' => $account->id]);
$this->setCertainty(50);
return $account;
}
$account = $repository->store(
['name' => $accountName, 'openingBalance' => 0, 'iban' => null, 'user' => $this->user->id,
'accountType' => 'import',
'virtualBalance' => 0, 'accountNumber' => $value, 'active' => true]
);
$this->setCertainty(100);
return $account;
}
}

82
app/Import/FileProcessor/CsvProcessor.php
View File

@@ -67,25 +67,28 @@ class CsvProcessor implements FileProcessorInterface
Log::debug('Now in CsvProcessor run(). Job is now running...');
$entries = $this->getImportArray();
$count = 0;
$index = 0;
Log::notice('Building importable objects from CSV file.');
foreach ($entries as $index => $row) {
// verify if not exists already:
if ($this->hashPresent($row)) {
Log::info(sprintf('Row #%d has already been imported.', $index));
if ($this->rowAlreadyImported($row)) {
$message = sprintf('Row #%d has already been imported.', $index);
$this->job->addError($index, $message);
$this->job->addStepsDone(5); // all steps.
Log::info($message);
continue;
}
$this->objects->push($this->importRow($index, $row));
/**
* 1. Build import entry.
* 2. Validate import entry.
* 3. Store journal.
* 4. Run rules.
*/
$this->job->addStepsDone(1);
$count++;
sleep(1);
}
// if job has no step count, set it now:
$extended = $this->job->extended_status;
if ($extended['steps'] === 0) {
$extended['steps'] = $index * 5;
$this->job->extended_status = $extended;
$this->job->save();
}
return true;
}
@@ -148,36 +151,6 @@ class CsvProcessor implements FileProcessorInterface
return $results;
}
/**
* Checks if the row has not been imported before.
*
* TODO for debugging, will always return false.
*
* @param array $array
*
* @noinspection PhpUnreachableStatementInspection
* @return bool
*/
private function hashPresent(array $array): bool
{
$string = json_encode($array);
$hash = hash('sha256', json_encode($string));
$json = json_encode($hash);
$entry = TransactionJournalMeta::
leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
->where('data', $json)
->where('name', 'importHash')
->first();
return false;
if (!is_null($entry)) {
return true;
}
return false;
}
/**
* Take a row, build import journal by annotating each value and storing it in the import journal.
*
@@ -207,6 +180,33 @@ class CsvProcessor implements FileProcessorInterface
return $journal;
}
/**
* Checks if the row has not been imported before.
*
* @param array $array
*
* @return bool
*/
private function rowAlreadyImported(array $array): bool
{
$string = json_encode($array);
$hash = hash('sha256', json_encode($string));
$json = json_encode($hash);
$entry = TransactionJournalMeta::
leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
->where('data', $json)
->where('name', 'importHash')
->first();
return rand(1, 10) === 3;
if (!is_null($entry)) {
return true;
}
return false;
}
/**
* And this is the point where the specifix go to work.
*

3
app/Import/Routine/ImportRoutine.php
View File

@@ -40,6 +40,7 @@ class ImportRoutine
$this->job = $job;
$this->journals = new Collection;
$this->errors = new Collection;
Log::debug(sprintf('Job ID is #%d', $job->id));
}
/**
@@ -48,7 +49,7 @@ class ImportRoutine
public function run(): bool
{
if ($this->job->status !== 'configured') {
Log::error(sprintf('Job %s is in state %s so it cannot be started.', $this->job->key, $this->job->status));
Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->job->status));
return false;
}

40
app/Import/Storage/ImportStorage.php
View File

@@ -58,8 +58,6 @@ class ImportStorage
$this->objects = new Collection;
$this->journals = new Collection;
$this->errors = new Collection;
$this->rules = $this->getUserRules();
}
/**
@@ -75,7 +73,8 @@ class ImportStorage
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
$this->job = $job;
$this->rules = $this->getUserRules();
}
/**
@@ -100,6 +99,7 @@ class ImportStorage
* @var ImportJournal $object
*/
foreach ($this->objects as $index => $object) {
sleep(4);
Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $object->description));
$errors = new MessageBag;
@@ -148,11 +148,20 @@ class ImportStorage
$journal->order = 0;
$journal->tag_count = 0;
$journal->encrypted = 0;
$journal->completed = 0;
$journal->completed = false;
if (rand(1, 10) === 3) {
$journal->date = null;
$journal->description = null;
}
if (!$journal->save()) {
$errorText = join(', ', $journal->getErrors()->all());
$errors->add('no-key', sprintf('Error storing journal: %s', $errorText));
$this->addErrorToJob($index, sprintf('Error storing line #%d: %s', $index, $errorText));
Log::error(sprintf('Could not store line #%d: %s', $index, $errorText));
// add the rest of the steps:
$this->job->addStepsDone(3);
continue;
}
$journal->setMeta('importHash', $object->hash);
@@ -184,7 +193,6 @@ class ImportStorage
Log::debug(sprintf('Created transaction with ID #%d and account #%d', $two->id, $opposing->id));
$this->job->addStepsDone(1);
sleep(1);
// category
$category = $object->category->getCategory();
@@ -222,6 +230,11 @@ class ImportStorage
if (strlen($object->notes) > 0) {
$journal->setMeta('notes', $object->notes);
}
// set journal completed:
$journal->completed = true;
$journal->save();
$this->job->addStepsDone(1);
// run rules:
@@ -230,9 +243,6 @@ class ImportStorage
$this->journals->push($journal);
$this->errors->push($errors);
sleep(1);
}
@@ -262,6 +272,18 @@ class ImportStorage
return true;
}
/**
* @param int $index
* @param string $error
*/
private function addErrorToJob(int $index, string $error)
{
$extended = $this->job->extended_status;
$extended['errors'][$index][] = $error;
$this->job->extended_status = $extended;
$this->job->save();
}
/**
* @return Collection
*/

23
app/Models/ImportJob.php
View File

@@ -66,13 +66,28 @@ class ImportJob extends Model
throw new NotFoundHttpException;
}
/**
* @param int $index
* @param string $message
*
* @return bool
*/
public function addError(int $index, string $message): bool
{
$extended = $this->extended_status;
$extended['errors'][$index][] = $message;
$this->extended_status = $extended;
return true;
}
/**
* @param int $count
*/
public function addStepsDone(int $count)
{
$status = $this->extended_status;
$status['steps_done'] += $count;
$status = $this->extended_status;
$status['done'] += $count;
$this->extended_status = $status;
$this->save();
@@ -84,7 +99,7 @@ class ImportJob extends Model
public function addTotalSteps(int $count)
{
$status = $this->extended_status;
$status['total_steps'] += $count;
$status['steps'] += $count;
$this->extended_status = $status;
$this->save();
@@ -165,7 +180,7 @@ class ImportJob extends Model
$disk = Storage::disk('upload');
$encryptedContent = $disk->get($fileName);
$content = Crypt::decrypt($encryptedContent);
Log::debug(sprintf('Content size is %d bytes.', $content));
Log::debug(sprintf('Content size is %d bytes.', strlen($content)));
return $content;
}

12
app/Repositories/ImportJob/ImportJobRepository.php
View File

@@ -58,12 +58,12 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$importJob->file_type = $fileType;
$importJob->key = Str::random(12);
$importJob->status = 'new';
$importJob->configuration = [];
$importJob->extended_status = [
'total_steps' => 0,
'steps_done' => 0,
'import_count' => 0,
'importTag' => 0,
'errors' => [],
'steps' => 0,
'done' => 0,
'importTag' => 0,
'errors' => [],
];
$importJob->save();
@@ -157,8 +157,6 @@ class ImportJobRepository implements ImportJobRepositoryInterface
$disk->put($newName, $contentEncrypted);
Log::debug('Uploaded file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]);
}
$job->status = 'initialized';
$job->save();
return true;
}

2
app/Support/Import/Configuration/Csv/Map.php
View File

@@ -95,7 +95,7 @@ class Map implements ConfigurationInterface
// save number of rows, thus number of steps, in job:
$steps = $rowIndex * 5;
$extended = $this->job->extended_status;
$extended['total_steps'] = $steps;
$extended['steps'] = $steps;
$this->job->extended_status = $extended;
$this->job->save();