kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-10-31 02:36:28 +00:00
Code Issues Projects Releases Wiki Activity

Updated export routine.

Browse Source
This commit is contained in:
James Cole
2017-08-18 14:45:42 +02:00
parent 6666d1a2f4
commit b955486f14
11 changed files with 482 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
app/Export/Collector/UploadCollector.php
View File

@@ -15,6 +15,7 @@ namespace FireflyIII\Export\Collector;
use Crypt;
use Illuminate\Contracts\Encryption\DecryptException;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Log;
use Storage;
@@ -50,14 +51,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface
public function run(): bool
{
Log::debug('Going to collect attachments', ['key' => $this->job->key]);
// file names associated with the old import routine.
$this->vintageFormat = sprintf('csv-upload-%d-', $this->job->user->id);
// collect old upload files (names beginning with "csv-upload".
$this->collectVintageUploads();
// then collect current upload files:
$this->collectModernUploads();
return true;
@@ -70,7 +63,8 @@ class UploadCollector extends BasicCollector implements CollectorInterface
*/
private function collectModernUploads(): bool
{
$set = $this->job->user->importJobs()->where('status', 'import_complete')->get(['import_jobs.*']);
$set = $this->job->user->importJobs()->whereIn('status', ['import_complete', 'finished'])->get(['import_jobs.*']);
Log::debug(sprintf('Found %d import jobs', $set->count()));
$keys = [];
if ($set->count() > 0) {
$keys = $set->pluck('key')->toArray();
@@ -83,59 +77,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface
return true;
}
/**
* This method collects all the uploads that are uploaded using the "old" importer. So from before the summer of 2016.
*
* @return bool
*/
private function collectVintageUploads(): bool
{
// grab upload directory.
$files = $this->uploadDisk->files();
foreach ($files as $entry) {
$this->processVintageUpload($entry);
}
return true;
}
/**
* This method tells you when the vintage upload file was actually uploaded.
*
* @param string $entry
*
* @return string
*/
private function getVintageUploadDate(string $entry): string
{
// this is an original upload.
$parts = explode('-', str_replace(['.csv.encrypted', $this->vintageFormat], '', $entry));
$originalUpload = intval($parts[1]);
$date = date('Y-m-d \a\t H-i-s', $originalUpload);
return $date;
}
/**
* Tells you if a file name is a vintage upload.
*
* @param string $entry
*
* @return bool
*/
private function isVintageImport(string $entry): bool
{
$len = strlen($this->vintageFormat);
// file is part of the old import routine:
if (substr($entry, 0, $len) === $this->vintageFormat) {
return true;
}
return false;
}
/**
* @param string $key
*
@@ -153,7 +94,7 @@ class UploadCollector extends BasicCollector implements CollectorInterface
$content = '';
try {
$content = Crypt::decrypt($this->uploadDisk->get(sprintf('%s.upload', $key)));
} catch (DecryptException $e) {
} catch (FileNotFoundException | DecryptException $e) {
Log::error(sprintf('Could not decrypt old import file "%s". Skipped because: %s', $key, $e->getMessage()));
}
@@ -168,47 +109,4 @@ class UploadCollector extends BasicCollector implements CollectorInterface
return true;
}
/**
* If the file is a vintage upload, process it.
*
* @param string $entry
*
* @return bool
*/
private function processVintageUpload(string $entry): bool
{
if ($this->isVintageImport($entry)) {
$this->saveVintageImportFile($entry);
return true;
}
return false;
}
/**
* This will store the content of the old vintage upload somewhere.
*
* @param string $entry
*/
private function saveVintageImportFile(string $entry)
{
$content = '';
try {
$content = Crypt::decrypt($this->uploadDisk->get($entry));
} catch (DecryptException $e) {
Log::error('Could not decrypt old CSV import file ' . $entry . '. Skipped because ' . $e->getMessage());
}
if (strlen($content) > 0) {
// add to export disk.
$date = $this->getVintageUploadDate($entry);
$file = $this->job->key . '-Old import dated ' . $date . '.csv';
$this->exportDisk->put($file, $content);
$this->getEntries()->push($file);
}
}
}

87
app/Export/Entry/Entry.php
View File

@@ -13,6 +13,7 @@ declare(strict_types=1);
namespace FireflyIII\Export\Entry;
use FireflyIII\Models\Transaction;
use Steam;
/**
@@ -37,24 +38,43 @@ final class Entry
{
// @formatter:off
public $journal_id;
public $transaction_id = 0;
public $date;
public $description;
public $currency_code;
public $amount;
public $foreign_currency_code = '';
public $foreign_amount = '0';
public $transaction_type;
public $asset_account_id;
public $asset_account_name;
public $asset_account_iban;
public $asset_account_bic;
public $asset_account_number;
public $asset_currency_code;
public $opposing_account_id;
public $opposing_account_name;
public $opposing_account_iban;
public $opposing_account_bic;
public $opposing_account_number;
public $opposing_currency_code;
public $budget_id;
public $budget_name;
public $category_id;
public $category_name;
public $bill_id;
public $bill_name;
public $notes;
public $tags;
// @formatter:on
/**
@@ -95,5 +115,72 @@ final class Entry
return $entry;
}
/**
* Converts a given transaction (as collected by the collector) into an export entry.
*
* @param Transaction $transaction
*
* @return Entry
*/
public static function fromTransaction(Transaction $transaction): Entry
{
$entry = new self;
$entry->journal_id = $transaction->journal_id;
$entry->transaction_id = $transaction->id;
$entry->date = $transaction->date->format('Ymd');
$entry->description = $transaction->description;
if (strlen(strval($transaction->transaction_description)) > 0) {
$entry->description = $transaction->transaction_description . '(' . $transaction->description . ')';
}
$entry->currency_code = $transaction->transactionCurrency->code;
$entry->amount = round($transaction->transaction_amount, $transaction->transactionCurrency->decimal_places);
$entry->foreign_currency_code = is_null($transaction->foreign_currency_id) ? null : $transaction->foreignCurrency->code;
$entry->foreign_amount = is_null($transaction->foreign_currency_id)
? null
: round(
$transaction->transaction_foreign_amount, $transaction->foreignCurrency->decimal_places
);
$entry->transaction_type = $transaction->transaction_type_type;
$entry->asset_account_id = $transaction->account_id;
$entry->asset_account_name = app('steam')->tryDecrypt($transaction->account_name);
$entry->asset_account_iban = $transaction->account_iban;
$entry->asset_account_number = $transaction->account_number;
$entry->asset_account_bic = $transaction->account_bic;
// asset_currency_code
$entry->opposing_account_id = $transaction->opposing_account_id;
$entry->opposing_account_name = app('steam')->tryDecrypt($transaction->opposing_account_name);
$entry->opposing_account_iban = $transaction->opposing_account_iban;
$entry->opposing_account_number = $transaction->opposing_account_number;
$entry->opposing_account_bic = $transaction->opposing_account_bic;
// opposing currency code
/** budget */
$entry->budget_id = $transaction->transaction_budget_id;
$entry->budget_name = app('steam')->tryDecrypt($transaction->transaction_budget_name);
if (is_null($transaction->transaction_budget_id)) {
$entry->budget_id = $transaction->transaction_journal_budget_id;
$entry->budget_name = app('steam')->tryDecrypt($transaction->transaction_journal_budget_name);
}
/** category */
$entry->category_id = $transaction->transaction_category_id;
$entry->category_name = app('steam')->tryDecrypt($transaction->transaction_category_name);
if (is_null($transaction->transaction_category_id)) {
$entry->category_id = $transaction->transaction_journal_category_id;
$entry->category_name = app('steam')->tryDecrypt($transaction->transaction_journal_category_name);
}
/** budget */
$entry->bill_id = $transaction->bill_id;
$entry->bill_name = app('steam')->tryDecrypt($transaction->bill_name);
$entry->tags = $transaction->tags;
$entry->notes = $transaction->notes;
return $entry;
}
}

308
app/Export/ExpandedProcessor.php Normal file
View File

@@ -0,0 +1,308 @@
<?php
/**
* ExpandedProcessor.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
* This software may be modified and distributed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International License.
*
* See the LICENSE file for details.
*/
declare(strict_types=1);
namespace FireflyIII\Export;
use Crypt;
use DB;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Export\Collector\AttachmentCollector;
use FireflyIII\Export\Collector\UploadCollector;
use FireflyIII\Export\Entry\Entry;
use FireflyIII\Helpers\Collector\JournalCollectorInterface;
use FireflyIII\Helpers\Filter\InternalTransferFilter;
use FireflyIII\Models\AccountMeta;
use FireflyIII\Models\ExportJob;
use FireflyIII\Models\Transaction;
use FireflyIII\Models\TransactionJournalMeta;
use Illuminate\Support\Collection;
use Log;
use Storage;
use ZipArchive;
/**
* Class ExpandedProcessor
*
* @package FireflyIII\Export
*/
class ExpandedProcessor implements ProcessorInterface
{
/** @var Collection */
public $accounts;
/** @var string */
public $exportFormat;
/** @var bool */
public $includeAttachments;
/** @var bool */
public $includeOldUploads;
/** @var ExportJob */
public $job;
/** @var array */
public $settings;
/** @var Collection */
private $exportEntries;
/** @var Collection */
private $files;
/** @var Collection */
private $journals;
/**
* Processor constructor.
*/
public function __construct()
{
$this->journals = new Collection;
$this->exportEntries = new Collection;
$this->files = new Collection;
}
/**
* @return bool
*/
public function collectAttachments(): bool
{
/** @var AttachmentCollector $attachmentCollector */
$attachmentCollector = app(AttachmentCollector::class);
$attachmentCollector->setJob($this->job);
$attachmentCollector->setDates($this->settings['startDate'], $this->settings['endDate']);
$attachmentCollector->run();
$this->files = $this->files->merge($attachmentCollector->getEntries());
return true;
}
/**
* @return bool
*/
public function collectJournals(): bool
{
// use journal collector thing.
/** @var JournalCollectorInterface $collector */
$collector = app(JournalCollectorInterface::class);
$collector->setAccounts($this->accounts)->setRange($this->settings['startDate'], $this->settings['endDate'])
->withOpposingAccount()->withBudgetInformation()->withCategoryInformation()
->removeFilter(InternalTransferFilter::class);
$transactions = $collector->getJournals();
// get some more meta data for each entry:
$ids = $transactions->pluck('journal_id')->toArray();
$assetIds = $transactions->pluck('account_id')->toArray();
$opposingIds = $transactions->pluck('opposing_account_id')->toArray();
$notes = $this->getNotes($ids);
$tags = $this->getTags($ids);
$ibans = $this->getIbans($assetIds) + $this->getIbans($opposingIds);
$transactions->each(
function (Transaction $transaction) use ($notes, $tags, $ibans) {
$journalId = intval($transaction->journal_id);
$accountId = intval($transaction->account_id);
$opposingId = intval($transaction->opposing_account_id);
$transaction->notes = $notes[$journalId] ?? '';
$transaction->tags = join(',', $tags[$journalId] ?? []);
$transaction->account_number = $ibans[$accountId]['accountNumber'] ?? '';
$transaction->account_bic = $ibans[$accountId]['BIC'] ?? '';
$transaction->opposing_account_number = $ibans[$opposingId]['accountNumber'] ?? '';
$transaction->opposing_account_bic = $ibans[$opposingId]['BIC'] ?? '';
}
);
$this->journals = $transactions;
return true;
}
/**
* @return bool
*/
public function collectOldUploads(): bool
{
/** @var UploadCollector $uploadCollector */
$uploadCollector = app(UploadCollector::class);
$uploadCollector->setJob($this->job);
$uploadCollector->run();
$this->files = $this->files->merge($uploadCollector->getEntries());
return true;
}
/**
* @return bool
*/
public function convertJournals(): bool
{
$this->journals->each(
function (Transaction $transaction) {
$this->exportEntries->push(Entry::fromTransaction($transaction));
}
);
Log::debug(sprintf('Count %d entries in exportEntries (convertJournals)', $this->exportEntries->count()));
return true;
}
/**
* @return bool
* @throws FireflyException
*/
public function createZipFile(): bool
{
$zip = new ZipArchive;
$file = $this->job->key . '.zip';
$fullPath = storage_path('export') . '/' . $file;
if ($zip->open($fullPath, ZipArchive::CREATE) !== true) {
throw new FireflyException('Cannot store zip file.');
}
// for each file in the collection, add it to the zip file.
$disk = Storage::disk('export');
foreach ($this->getFiles() as $entry) {
// is part of this job?
$zipFileName = str_replace($this->job->key . '-', '', $entry);
$zip->addFromString($zipFileName, $disk->get($entry));
}
$zip->close();
// delete the files:
$this->deleteFiles();
return true;
}
/**
* @return bool
*/
public function exportJournals(): bool
{
$exporterClass = config('firefly.export_formats.' . $this->exportFormat);
$exporter = app($exporterClass);
$exporter->setJob($this->job);
$exporter->setEntries($this->exportEntries);
$exporter->run();
$this->files->push($exporter->getFileName());
return true;
}
/**
* @return Collection
*/
public function getFiles(): Collection
{
return $this->files;
}
/**
* Save export job settings to class.
*
* @param array $settings
*/
public function setSettings(array $settings)
{
// save settings
$this->settings = $settings;
$this->accounts = $settings['accounts'];
$this->exportFormat = $settings['exportFormat'];
$this->includeAttachments = $settings['includeAttachments'];
$this->includeOldUploads = $settings['includeOldUploads'];
$this->job = $settings['job'];
}
/**
*
*/
private function deleteFiles()
{
$disk = Storage::disk('export');
foreach ($this->getFiles() as $file) {
$disk->delete($file);
}
}
/**
* Get all IBAN / SWIFT / account numbers
*
* @param array $array
*
* @return array
*/
private function getIbans(array $array): array
{
$array = array_unique($array);
$return = [];
$set = AccountMeta::whereIn('account_id', $array)
->leftJoin('accounts', 'accounts.id', 'account_meta.account_id')
->where('accounts.user_id', $this->job->user_id)
->whereIn('account_meta.name', ['accountNumber', 'BIC', 'currency_id'])
->get(['account_meta.id', 'account_meta.account_id', 'account_meta.name', 'account_meta.data']);
/** @var AccountMeta $meta */
foreach ($set as $meta) {
$id = intval($meta->account_id);
$return[$id][$meta->name] = $meta->data;
}
return $return;
}
/**
* Returns, if present, for the given journal ID's the notes.
*
* @param array $array
*
* @return array
*/
private function getNotes(array $array): array
{
$array = array_unique($array);
$set = TransactionJournalMeta::whereIn('journal_meta.transaction_journal_id', $array)
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
->where('transaction_journals.user_id', $this->job->user_id)
->where('journal_meta.name', 'notes')->get(
['journal_meta.transaction_journal_id', 'journal_meta.data', 'journal_meta.id']
);
$return = [];
/** @var TransactionJournalMeta $meta */
foreach ($set as $meta) {
$id = intval($meta->transaction_journal_id);
$return[$id] = $meta->data;
}
return $return;
}
/**
* Returns a comma joined list of all the users tags linked to these journals.
*
* @param array $array
*
* @return array
*/
private function getTags(array $array): array
{
$set = DB::table('tag_transaction_journal')
->whereIn('tag_transaction_journal.transaction_journal_id', $array)
->leftJoin('tags', 'tag_transaction_journal.tag_id', '=', 'tags.id')
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'tag_transaction_journal.transaction_journal_id')
->where('transaction_journals.user_id', $this->job->user_id)
->get(['tag_transaction_journal.transaction_journal_id', 'tags.tag']);
$result = [];
foreach ($set as $entry) {
$id = intval($entry->transaction_journal_id);
$result[$id] = isset($result[$id]) ? $result[$id] : [];
$result[$id][] = Crypt::decrypt($entry->tag);
}
return $result;
}
}

8
app/Export/Exporter/CsvExporter.php
View File

@@ -58,8 +58,12 @@ class CsvExporter extends BasicExporter implements ExporterInterface
// get field names for header row:
$first = $this->getEntries()->first();
$headers = array_keys(get_object_vars($first));
$rows[] = $headers;
$headers = [];
if (!is_null($first)) {
$headers = array_keys(get_object_vars($first));
}
$rows[] = $headers;
/** @var Entry $entry */
foreach ($this->getEntries() as $entry) {

21
app/Helpers/Collector/JournalCollector.php
View File

@@ -31,7 +31,6 @@ use FireflyIII\Models\Tag;
use FireflyIII\Models\Transaction;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
use FireflyIII\User;
use Illuminate\Contracts\Encryption\DecryptException;
use Illuminate\Database\Eloquent\Builder as EloquentBuilder;
use Illuminate\Database\Query\JoinClause;
use Illuminate\Pagination\LengthAwarePaginator;
@@ -86,7 +85,9 @@ class JournalCollector implements JournalCollectorInterface
'accounts.name as account_name',
'accounts.encrypted as account_encrypted',
'accounts.iban as account_iban',
'account_types.type as account_type',
];
/** @var bool */
private $filterTransfers = false;
@@ -175,12 +176,10 @@ class JournalCollector implements JournalCollectorInterface
if (!is_null($transaction->bill_name)) {
$transaction->bill_name = Steam::decrypt(intval($transaction->bill_name_encrypted), $transaction->bill_name);
}
$transaction->opposing_account_name = app('steam')->tryDecrypt($transaction->opposing_account_name);
$transaction->account_iban = app('steam')->tryDecrypt($transaction->account_iban);
$transaction->opposing_account_iban = app('steam')->tryDecrypt($transaction->opposing_account_iban);
try {
$transaction->opposing_account_name = Crypt::decrypt($transaction->opposing_account_name);
} catch (DecryptException $e) {
// if this fails its already decrypted.
}
}
);
@@ -677,10 +676,12 @@ class JournalCollector implements JournalCollectorInterface
$this->query->leftJoin('account_types as opposing_account_types', 'opposing_accounts.account_type_id', '=', 'opposing_account_types.id');
$this->query->whereNull('opposing.deleted_at');
$this->fields[] = 'opposing.id as opposing_id';
$this->fields[] = 'opposing.account_id as opposing_account_id';
$this->fields[] = 'opposing_accounts.name as opposing_account_name';
$this->fields[] = 'opposing_accounts.encrypted as opposing_account_encrypted';
$this->fields[] = 'opposing.id as opposing_id';
$this->fields[] = 'opposing.account_id as opposing_account_id';
$this->fields[] = 'opposing_accounts.name as opposing_account_name';
$this->fields[] = 'opposing_accounts.encrypted as opposing_account_encrypted';
$this->fields[] = 'opposing_accounts.iban as opposing_account_iban';
$this->fields[] = 'opposing_account_types.type as opposing_account_type';
$this->joinedOpposing = true;
Log::debug('joinedOpposing is now true!');

16
app/Http/Controllers/ExportController.php
View File

@@ -17,6 +17,7 @@ namespace FireflyIII\Http\Controllers;
use Carbon\Carbon;
use ExpandedForm;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Export\ExpandedProcessor;
use FireflyIII\Export\ProcessorInterface;
use FireflyIII\Http\Requests\ExportFormRequest;
use FireflyIII\Models\AccountType;
@@ -137,34 +138,40 @@ class ExportController extends Controller
public function postIndex(ExportFormRequest $request, AccountRepositoryInterface $repository, ExportJobRepositoryInterface $jobs)
{
$job = $jobs->findByKey($request->get('job'));
$accounts = $request->get('accounts') ?? [];
$settings = [
'accounts' => $repository->getAccountsById($request->get('accounts')),
'accounts' => $repository->getAccountsById($accounts),
'startDate' => new Carbon($request->get('export_start_range')),
'endDate' => new Carbon($request->get('export_end_range')),
'exportFormat' => $request->get('exportFormat'),
'includeAttachments' => intval($request->get('include_attachments')) === 1,
'includeOldUploads' => intval($request->get('include_old_uploads')) === 1,
'includeAttachments' => $request->boolean('include_attachments'),
'includeOldUploads' => $request->boolean('include_old_uploads'),
'job' => $job,
];
$jobs->changeStatus($job, 'export_status_make_exporter');
/** @var ProcessorInterface $processor */
$processor = app(ProcessorInterface::class);
$processor = app(ExpandedProcessor::class);
$processor->setSettings($settings);
/*
* Collect journals:
*/
$jobs->changeStatus($job, 'export_status_collecting_journals');
$processor->collectJournals();
$jobs->changeStatus($job, 'export_status_collected_journals');
/*
* Transform to exportable entries:
*/
$jobs->changeStatus($job, 'export_status_converting_to_export_format');
$processor->convertJournals();
$jobs->changeStatus($job, 'export_status_converted_to_export_format');
/*
* Transform to (temporary) file:
*/
@@ -180,6 +187,7 @@ class ExportController extends Controller
$jobs->changeStatus($job, 'export_status_collected_attachments');
}
/*
* Collect old uploads
*/

23
app/Http/Requests/ExportFormRequest.php
View File

@@ -38,20 +38,19 @@ class ExportFormRequest extends Request
public function rules()
{
$sessionFirst = clone session('first');
$first = $sessionFirst->subDay()->format('Y-m-d');
$today = Carbon::create()->addDay()->format('Y-m-d');
$formats = join(',', array_keys(config('firefly.export_formats')));
$first = $sessionFirst->subDay()->format('Y-m-d');
$today = Carbon::create()->addDay()->format('Y-m-d');
$formats = join(',', array_keys(config('firefly.export_formats')));
return [
'export_start_range' => 'required|date|after:' . $first,
'export_end_range' => 'required|date|before:' . $today,
'accounts' => 'required',
'job' => 'required|belongsToUser:export_jobs,key',
'accounts.*' => 'required|exists:accounts,id|belongsToUser:accounts',
'include_attachments' => 'in:0,1',
'include_config' => 'in:0,1',
'exportFormat' => 'in:' . $formats,
// 'export_start_range' => 'required|date|after:' . $first,
// 'export_end_range' => 'required|date|before:' . $today,
// 'accounts' => 'required',
// 'job' => 'required|belongsToUser:export_jobs,key',
// 'accounts.*' => 'required|exists:accounts,id|belongsToUser:accounts',
// 'include_attachments' => 'in:0,1',
// 'include_config' => 'in:0,1',
// 'exportFormat' => 'in:' . $formats,
];
}
}

2
app/Http/Requests/Request.php
View File

@@ -28,7 +28,7 @@ class Request extends FormRequest
*
* @return bool
*/
protected function boolean(string $field): bool
public function boolean(string $field): bool
{
return intval($this->input($field)) === 1;
}

36
app/Models/Transaction.php
View File

@@ -22,6 +22,42 @@ use Watson\Validating\ValidatingTrait;
/**
* Class Transaction
*
* @property-read int $journal_id
* @property-read Carbon $date
* @property-read string $transaction_description
* @property-read string $transaction_amount
* @property-read string $transaction_foreign_amount
* @property-read string $transaction_type_type
*
* @property-read int $account_id
* @property-read string $account_name
* @property string $account_iban
* @property string $account_number
* @property string $account_bic
*
* @property-read int $opposing_account_id
* @property string $opposing_account_name
* @property string $opposing_account_iban
* @property string $opposing_account_number
* @property string $opposing_account_bic
*
*
* @property-read int $transaction_budget_id
* @property-read string $transaction_budget_name
* @property-read int $transaction_journal_budget_id
* @property-read string $transaction_journal_budget_name
*
* @property-read int $transaction_category_id
* @property-read string $transaction_category_name
* @property-read int $transaction_journal_category_id
* @property-read string $transaction_journal_category_name
*
* @property-read int $bill_id
* @property string $bill_name
*
* @property string $notes
* @property string $tags
*
* @package FireflyIII\Models
*/
class Transaction extends Model