kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-09-29 18:20:01 +00:00
Code Issues Projects Releases Wiki Activity

Improve export routine and cron job for #837

Browse Source
This commit is contained in:
James Cole
2017-09-14 21:17:19 +02:00
parent f54b4c3abc
commit 519ca4b2af
10 changed files with 168 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
app/Console/Commands/CreateExport.php
View File

@@ -13,7 +13,15 @@ declare(strict_types=1);
namespace FireflyIII\Console\Commands;
use Carbon\Carbon;
use FireflyIII\Export\ProcessorInterface;
use FireflyIII\Models\AccountType;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\Repositories\ExportJob\ExportJobRepositoryInterface;
use FireflyIII\Repositories\Journal\JournalRepositoryInterface;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use Illuminate\Console\Command;
use Storage;
/**
@@ -23,18 +31,25 @@ use Illuminate\Console\Command;
*/
class CreateExport extends Command
{
use VerifiesAccessToken;
/**
* The console command description.
*
* @var string
*/
protected $description = 'Used to create an export of your data. This will result in an UNENCRYPTED backup in your storage/export folder.';
protected $description = 'Use this command to create a new import. Your user ID can be found on the /profile page.';
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'firefly:create-export {--with_attachments} {--with_uploads}';
protected $signature
= 'firefly:create-export
{--user= : The user ID that the import should import for.}
{--token= : The user\'s access token.}
{--with_attachments : Include user\'s attachments?}
{--with_uploads : Include user\'s uploads?}';
/**
* Create a new command instance.
@@ -53,10 +68,71 @@ class CreateExport extends Command
*/
public function handle()
{
//$user = User::find(1); // can ony
//$first = '';
//$today = new Carbon;
//
$this->error('Export is under construction.');
if (!$this->verifyAccessToken()) {
$this->error('Invalid access token.');
return;
}
$this->line('Full export is running...');
// make repositories
/** @var UserRepositoryInterface $userRepository */
$userRepository = app(UserRepositoryInterface::class);
/** @var ExportJobRepositoryInterface $jobRepository */
$jobRepository = app(ExportJobRepositoryInterface::class);
/** @var AccountRepositoryInterface $accountRepository */
$accountRepository = app(AccountRepositoryInterface::class);
/** @var JournalRepositoryInterface $journalRepository */
$journalRepository = app(JournalRepositoryInterface::class);
// set user
$user = $userRepository->find(intval($this->option('user')));
$jobRepository->setUser($user);
$journalRepository->setUser($user);
$accountRepository->setUser($user);
// first date
$firstJournal = $journalRepository->first();
$first = new Carbon;
if (!is_null($firstJournal->id)) {
$first = $firstJournal->date;
}
// create job and settings.
$job = $jobRepository->create();
$settings = [
'accounts' => $accountRepository->getAccountsByType([AccountType::ASSET, AccountType::DEFAULT]),
'startDate' => $first,
'endDate' => new Carbon,
'exportFormat' => 'csv',
'includeAttachments' => $this->option('with_attachments'),
'includeOldUploads' => $this->option('with_uploads'),
'job' => $job,
];
/** @var ProcessorInterface $processor */
$processor = app(ProcessorInterface::class);
$processor->setSettings($settings);
$processor->collectJournals();
$processor->convertJournals();
$processor->exportJournals();
if ($settings['includeAttachments']) {
$processor->collectAttachments();
}
if ($settings['includeOldUploads']) {
$processor->collectOldUploads();
}
$processor->createZipFile();
$disk = Storage::disk('export');
$fileName = sprintf('export-%s.zip', date('Y-m-d_H-i-s'));
$disk->move($job->key . '.zip', $fileName);
$this->line('The export has finished! You can find the ZIP file in this location:');
$this->line(storage_path(sprintf('export/%s', $fileName)));
return;
}
}

16
app/Console/Commands/CreateImport.php
View File

@@ -30,6 +30,7 @@ use Monolog\Formatter\LineFormatter;
*/
class CreateImport extends Command
{
use VerifiesAccessToken;
/**
* The console command description.
*
@@ -42,7 +43,13 @@ class CreateImport extends Command
*
* @var string
*/
protected $signature = 'firefly:create-import {file} {configuration} {--user=1} {--type=csv} {--start}';
protected $signature = 'firefly:create-import
{file : The file to import.}
{configuration : The configuration file to use for the import/}
{--type=csv : The file type of the import.}
{--user= : The user ID that the import should import for.}
{--token= : The user\'s access token.}
{--start : Starts the job immediately.}';
/**
* Create a new command instance.
@@ -61,6 +68,11 @@ class CreateImport extends Command
*/
public function handle()
{
if (!$this->verifyAccessToken()) {
$this->error('Invalid access token.');
return;
}
/** @var UserRepositoryInterface $userRepository */
$userRepository = app(UserRepositoryInterface::class);
$file = $this->argument('file');
@@ -150,12 +162,12 @@ class CreateImport extends Command
$cwd = getcwd();
$validTypes = array_keys(config('firefly.import_formats'));
$type = strtolower($this->option('type'));
if (is_null($user->id)) {
$this->error(sprintf('There is no user with ID %d.', $this->option('user')));
return false;
}
if (!in_array($type, $validTypes)) {
$this->error(sprintf('Cannot import file of type "%s"', $type));

52
app/Console/Commands/VerifiesAccessToken.php Normal file
View File

@@ -0,0 +1,52 @@
<?php
/**
* VerifiesAccessToken.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Console\Commands;
use FireflyIII\Repositories\User\UserRepositoryInterface;
use Log;
use Preferences;
trait VerifiesAccessToken
{
/**
* @return bool
*/
protected function verifyAccessToken(): bool
{
$userId = intval($this->option('user'));
$token = strval($this->option('token'));
/** @var UserRepositoryInterface $repository */
$repository = app(UserRepositoryInterface::class);
$user = $repository->find($userId);
if (is_null($user->id)) {
Log::error(sprintf('verifyAccessToken(): no such user for input "%d"', $userId));
return false;
}
$accessToken = Preferences::getForUser($user, 'access_token', null);
if (is_null($accessToken)) {
Log::error(sprintf('User #%d has no access token, so cannot access command line options.', $userId));
return false;
}
if (!($accessToken->data === $token)) {
Log::error(sprintf('Invalid access token for user #%d.', $userId));
return false;
}
return true;
}
}

4
app/Console/Commands/VerifyDatabase.php
View File

@@ -108,6 +108,9 @@ class VerifyDatabase extends Command
}
/**
*
*/
private function createAccessTokens()
{
$users = User::get();
@@ -117,6 +120,7 @@ class VerifyDatabase extends Command
if (is_null($pref)) {
$token = $user->generateAccessToken();
Preferences::setForUser($user, 'access_token', $token);
$this->line(sprintf('Generated access token for user %s', $user->email));
}
}
}

1
app/Export/Collector/AttachmentCollector.php
View File

@@ -122,6 +122,7 @@ class AttachmentCollector extends BasicCollector implements CollectorInterface
*/
private function getAttachments(): Collection
{
$this->repository->setUser($this->user);
$attachments = $this->repository->getBetween($this->start, $this->end);
return $attachments;

12
app/Export/Collector/BasicCollector.php
View File

@@ -15,6 +15,7 @@ namespace FireflyIII\Export\Collector;
use FireflyIII\Models\ExportJob;
use FireflyIII\User;
use Illuminate\Support\Collection;
/**
@@ -26,6 +27,8 @@ class BasicCollector
{
/** @var ExportJob */
protected $job;
/** @var User */
protected $user;
/** @var Collection */
private $entries;
@@ -59,6 +62,15 @@ class BasicCollector
public function setJob(ExportJob $job)
{
$this->job = $job;
$this->user = $job->user;
}
/**
* @param User $user
*/
public function setUser(User $user)
{
$this->user = $user;
}

347
app/Export/Collector/JournalExportCollector.php
View File

@@ -1,347 +0,0 @@
<?php
/**
* JournalExportCollector.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Export\Collector;
use Carbon\Carbon;
use DB;
use FireflyIII\Models\Transaction;
use Illuminate\Database\Query\JoinClause;
use Illuminate\Support\Collection;
use Steam;
/**
* Class JournalExportCollector
*
* @package FireflyIII\Export\Collector
*/
class JournalExportCollector extends BasicCollector implements CollectorInterface
{
/** @var Collection */
private $accounts;
/** @var Carbon */
private $end;
/** @var Carbon */
private $start;
/** @var Collection */
private $workSet;
/**
* @return bool
*/
public function run(): bool
{
/*
* Instead of collecting journals we collect transactions for the given accounts.
* We left join the OPPOSING transaction AND some journal data.
* After that we complement this info with budgets, categories, etc.
*
* This is way more efficient and will also work on split journals.
*/
$this->getWorkSet();
/*
* Extract:
* possible budget ids for journals
* possible category ids journals
* possible budget ids for transactions
* possible category ids for transactions
*
* possible IBAN and account numbers?
*
*/
$journals = $this->extractJournalIds();
$transactions = $this->extractTransactionIds();
// extend work set with category data from journals:
$this->categoryDataForJournals($journals);
// extend work set with category cate from transactions (overrules journals):
$this->categoryDataForTransactions($transactions);
// same for budgets:
$this->budgetDataForJournals($journals);
$this->budgetDataForTransactions($transactions);
$this->setEntries($this->workSet);
return true;
}
/**
* @param Collection $accounts
*/
public function setAccounts(Collection $accounts)
{
$this->accounts = $accounts;
}
/**
* @param Carbon $start
* @param Carbon $end
*/
public function setDates(Carbon $start, Carbon $end)
{
$this->start = $start;
$this->end = $end;
}
/**
* @param array $journals
*
* @return bool
*/
private function budgetDataForJournals(array $journals): bool
{
$set = DB::table('budget_transaction_journal')
->leftJoin('budgets', 'budgets.id', '=', 'budget_transaction_journal.budget_id')
->whereIn('budget_transaction_journal.transaction_journal_id', $journals)
->get(
[
'budget_transaction_journal.budget_id',
'budget_transaction_journal.transaction_journal_id',
'budgets.name',
'budgets.encrypted',
]
);
$set->each(
function ($obj) {
$obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name);
}
);
$array = [];
foreach ($set as $obj) {
$array[$obj->transaction_journal_id] = ['id' => $obj->budget_id, 'name' => $obj->name];
}
$this->workSet->each(
function ($obj) use ($array) {
if (isset($array[$obj->transaction_journal_id])) {
$obj->budget_id = $array[$obj->transaction_journal_id]['id'];
$obj->budget_name = $array[$obj->transaction_journal_id]['name'];
}
}
);
return true;
}
/**
* @param array $transactions
*
* @return bool
*/
private function budgetDataForTransactions(array $transactions): bool
{
$set = DB::table('budget_transaction')
->leftJoin('budgets', 'budgets.id', '=', 'budget_transaction.budget_id')
->whereIn('budget_transaction.transaction_id', $transactions)
->get(
[
'budget_transaction.budget_id',
'budget_transaction.transaction_id',
'budgets.name',
'budgets.encrypted',
]
);
$set->each(
function ($obj) {
$obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name);
}
);
$array = [];
foreach ($set as $obj) {
$array[$obj->transaction_id] = ['id' => $obj->budget_id, 'name' => $obj->name];
}
$this->workSet->each(
function ($obj) use ($array) {
// first transaction
if (isset($array[$obj->id])) {
$obj->budget_id = $array[$obj->id]['id'];
$obj->budget_name = $array[$obj->id]['name'];
}
}
);
return true;
}
/**
* @param array $journals
*
* @return bool
*/
private function categoryDataForJournals(array $journals): bool
{
$set = DB::table('category_transaction_journal')
->leftJoin('categories', 'categories.id', '=', 'category_transaction_journal.category_id')
->whereIn('category_transaction_journal.transaction_journal_id', $journals)
->get(
[
'category_transaction_journal.category_id',
'category_transaction_journal.transaction_journal_id',
'categories.name',
'categories.encrypted',
]
);
$set->each(
function ($obj) {
$obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name);
}
);
$array = [];
foreach ($set as $obj) {
$array[$obj->transaction_journal_id] = ['id' => $obj->category_id, 'name' => $obj->name];
}
$this->workSet->each(
function ($obj) use ($array) {
if (isset($array[$obj->transaction_journal_id])) {
$obj->category_id = $array[$obj->transaction_journal_id]['id'];
$obj->category_name = $array[$obj->transaction_journal_id]['name'];
}
}
);
return true;
}
/**
* @param array $transactions
*
* @return bool
*/
private function categoryDataForTransactions(array $transactions): bool
{
$set = DB::table('category_transaction')
->leftJoin('categories', 'categories.id', '=', 'category_transaction.category_id')
->whereIn('category_transaction.transaction_id', $transactions)
->get(
[
'category_transaction.category_id',
'category_transaction.transaction_id',
'categories.name',
'categories.encrypted',
]
);
$set->each(
function ($obj) {
$obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name);
}
);
$array = [];
foreach ($set as $obj) {
$array[$obj->transaction_id] = ['id' => $obj->category_id, 'name' => $obj->name];
}
$this->workSet->each(
function ($obj) use ($array) {
// first transaction
if (isset($array[$obj->id])) {
$obj->category_id = $array[$obj->id]['id'];
$obj->category_name = $array[$obj->id]['name'];
}
}
);
return true;
}
/**
* @return array
*/
private function extractJournalIds(): array
{
return $this->workSet->pluck('transaction_journal_id')->toArray();
}
/**
* @return array
*/
private function extractTransactionIds()
{
$set = $this->workSet->pluck('id')->toArray();
$opposing = $this->workSet->pluck('opposing_id')->toArray();
$complete = $set + $opposing;
return array_unique($complete);
}
/**
* @SuppressWarnings(PHPMD.ExcessiveMethodLength)
*/
private function getWorkSet()
{
$accountIds = $this->accounts->pluck('id')->toArray();
$this->workSet = Transaction::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->leftJoin(
'transactions AS opposing', function (JoinClause $join) {
$join->on('opposing.transaction_journal_id', '=', 'transactions.transaction_journal_id')
->where('opposing.amount', '=', DB::raw('transactions.amount * -1'))
->where('transactions.identifier', '=', DB::raw('opposing.identifier'));
}
)
->leftJoin('accounts', 'transactions.account_id', '=', 'accounts.id')
->leftJoin('accounts AS opposing_accounts', 'opposing.account_id', '=', 'opposing_accounts.id')
->leftJoin('transaction_types', 'transaction_journals.transaction_type_id', 'transaction_types.id')
->leftJoin('transaction_currencies', 'transactions.transaction_currency_id', '=', 'transaction_currencies.id')
->whereIn('transactions.account_id', $accountIds)
->where('transaction_journals.user_id', $this->job->user_id)
->where('transaction_journals.date', '>=', $this->start->format('Y-m-d'))
->where('transaction_journals.date', '<=', $this->end->format('Y-m-d'))
->where('transaction_journals.completed', 1)
->whereNull('transaction_journals.deleted_at')
->whereNull('transactions.deleted_at')
->whereNull('opposing.deleted_at')
->orderBy('transaction_journals.date', 'DESC')
->orderBy('transactions.identifier', 'ASC')
->get(
[
'transactions.id',
'transactions.amount',
'transactions.description',
'transactions.account_id',
'accounts.name as account_name',
'accounts.encrypted as account_name_encrypted',
'transactions.identifier',
'opposing.id as opposing_id',
'opposing.amount AS opposing_amount',
'opposing.description as opposing_description',
'opposing.account_id as opposing_account_id',
'opposing_accounts.name as opposing_account_name',
'opposing_accounts.encrypted as opposing_account_encrypted',
'opposing.identifier as opposing_identifier',
'transaction_journals.id as transaction_journal_id',
'transaction_journals.date',
'transaction_journals.description as journal_description',
'transaction_journals.encrypted as journal_encrypted',
'transaction_journals.transaction_type_id',
'transaction_types.type as transaction_type',
'transactions.transaction_currency_id',
'transaction_currencies.code AS transaction_currency_code',
]
);
}
}

2
app/Export/Collector/UploadCollector.php
View File

@@ -30,8 +30,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface
private $exportDisk;
/** @var \Illuminate\Contracts\Filesystem\Filesystem */
private $uploadDisk;
/** @var string */
private $vintageFormat;
/**
* AttachmentCollector constructor.

1
app/Export/ExpandedProcessor.php
View File

@@ -91,6 +91,7 @@ class ExpandedProcessor implements ProcessorInterface
// use journal collector thing.
/** @var JournalCollectorInterface $collector */
$collector = app(JournalCollectorInterface::class);
$collector->setUser($this->job->user);
$collector->setAccounts($this->accounts)->setRange($this->settings['startDate'], $this->settings['endDate'])
->withOpposingAccount()->withBudgetInformation()->withCategoryInformation()
->removeFilter(InternalTransferFilter::class);

203
app/Export/Processor.php
View File

@@ -1,203 +0,0 @@
<?php
/**
* Processor.php
* Copyright (C) 2016 thegrumpydictator@gmail.com
*
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Export;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Export\Collector\AttachmentCollector;
use FireflyIII\Export\Collector\JournalExportCollector;
use FireflyIII\Export\Collector\UploadCollector;
use FireflyIII\Export\Entry\Entry;
use FireflyIII\Models\ExportJob;
use Illuminate\Support\Collection;
use Log;
use Storage;
use ZipArchive;
/**
* Class Processor
*
* @package FireflyIII\Export
*/
class Processor implements ProcessorInterface
{
/** @var Collection */
public $accounts;
/** @var string */
public $exportFormat;
/** @var bool */
public $includeAttachments;
/** @var bool */
public $includeOldUploads;
/** @var ExportJob */
public $job;
/** @var array */
public $settings;
/** @var Collection */
private $exportEntries;
/** @var Collection */
private $files;
/** @var Collection */
private $journals;
/**
* Processor constructor.
*/
public function __construct()
{
$this->journals = new Collection;
$this->exportEntries = new Collection;
$this->files = new Collection;
}
/**
* @return bool
*/
public function collectAttachments(): bool
{
/** @var AttachmentCollector $attachmentCollector */
$attachmentCollector = app(AttachmentCollector::class);
$attachmentCollector->setJob($this->job);
$attachmentCollector->setDates($this->settings['startDate'], $this->settings['endDate']);
$attachmentCollector->run();
$this->files = $this->files->merge($attachmentCollector->getEntries());
return true;
}
/**
* @return bool
*/
public function collectJournals(): bool
{
/** @var JournalExportCollector $collector */
$collector = app(JournalExportCollector::class);
$collector->setJob($this->job);
$collector->setDates($this->settings['startDate'], $this->settings['endDate']);
$collector->setAccounts($this->settings['accounts']);
$collector->run();
$this->journals = $collector->getEntries();
Log::debug(sprintf('Count %d journals in collectJournals() ', $this->journals->count()));
return true;
}
/**
* @return bool
*/
public function collectOldUploads(): bool
{
/** @var UploadCollector $uploadCollector */
$uploadCollector = app(UploadCollector::class);
$uploadCollector->setJob($this->job);
$uploadCollector->run();
$this->files = $this->files->merge($uploadCollector->getEntries());
return true;
}
/**
* @return bool
*/
public function convertJournals(): bool
{
$count = 0;
foreach ($this->journals as $object) {
$this->exportEntries->push(Entry::fromObject($object));
$count++;
}
Log::debug(sprintf('Count %d entries in exportEntries (convertJournals)', $this->exportEntries->count()));
return true;
}
/**
* @return bool
* @throws FireflyException
*/
public function createZipFile(): bool
{
$zip = new ZipArchive;
$file = $this->job->key . '.zip';
$fullPath = storage_path('export') . '/' . $file;
if ($zip->open($fullPath, ZipArchive::CREATE) !== true) {
throw new FireflyException('Cannot store zip file.');
}
// for each file in the collection, add it to the zip file.
$disk = Storage::disk('export');
foreach ($this->getFiles() as $entry) {
// is part of this job?
$zipFileName = str_replace($this->job->key . '-', '', $entry);
$zip->addFromString($zipFileName, $disk->get($entry));
}
$zip->close();
// delete the files:
$this->deleteFiles();
return true;
}
/**
* @return bool
*/
public function exportJournals(): bool
{
$exporterClass = config('firefly.export_formats.' . $this->exportFormat);
$exporter = app($exporterClass);
$exporter->setJob($this->job);
$exporter->setEntries($this->exportEntries);
$exporter->run();
$this->files->push($exporter->getFileName());
return true;
}
/**
* @return Collection
*/
public function getFiles(): Collection
{
return $this->files;
}
/**
* @param array $settings
*/
public function setSettings(array $settings)
{
// save settings
$this->settings = $settings;
$this->accounts = $settings['accounts'];
$this->exportFormat = $settings['exportFormat'];
$this->includeAttachments = $settings['includeAttachments'];
$this->includeOldUploads = $settings['includeOldUploads'];
$this->job = $settings['job'];
}
/**
*
*/
private function deleteFiles()
{
$disk = Storage::disk('export');
foreach ($this->getFiles() as $file) {
$disk->delete($file);
}
}
}