diff --git a/app/Console/Commands/CreateExport.php b/app/Console/Commands/CreateExport.php index 9db5a7290b..89c691a5ab 100644 --- a/app/Console/Commands/CreateExport.php +++ b/app/Console/Commands/CreateExport.php @@ -13,7 +13,15 @@ declare(strict_types=1); namespace FireflyIII\Console\Commands; +use Carbon\Carbon; +use FireflyIII\Export\ProcessorInterface; +use FireflyIII\Models\AccountType; +use FireflyIII\Repositories\Account\AccountRepositoryInterface; +use FireflyIII\Repositories\ExportJob\ExportJobRepositoryInterface; +use FireflyIII\Repositories\Journal\JournalRepositoryInterface; +use FireflyIII\Repositories\User\UserRepositoryInterface; use Illuminate\Console\Command; +use Storage; /** @@ -23,18 +31,25 @@ use Illuminate\Console\Command; */ class CreateExport extends Command { + use VerifiesAccessToken; /** * The console command description. * * @var string */ - protected $description = 'Used to create an export of your data. This will result in an UNENCRYPTED backup in your storage/export folder.'; + protected $description = 'Use this command to create a new import. Your user ID can be found on the /profile page.'; + /** * The name and signature of the console command. * * @var string */ - protected $signature = 'firefly:create-export {--with_attachments} {--with_uploads}'; + protected $signature + = 'firefly:create-export + {--user= : The user ID that the import should import for.} + {--token= : The user\'s access token.} + {--with_attachments : Include user\'s attachments?} + {--with_uploads : Include user\'s uploads?}'; /** * Create a new command instance. @@ -53,10 +68,71 @@ class CreateExport extends Command */ public function handle() { - //$user = User::find(1); // can ony - //$first = ''; - //$today = new Carbon; - // - $this->error('Export is under construction.'); + if (!$this->verifyAccessToken()) { + $this->error('Invalid access token.'); + + return; + } + $this->line('Full export is running...'); + // make repositories + /** @var UserRepositoryInterface $userRepository */ + $userRepository = app(UserRepositoryInterface::class); + /** @var ExportJobRepositoryInterface $jobRepository */ + $jobRepository = app(ExportJobRepositoryInterface::class); + /** @var AccountRepositoryInterface $accountRepository */ + $accountRepository = app(AccountRepositoryInterface::class); + /** @var JournalRepositoryInterface $journalRepository */ + $journalRepository = app(JournalRepositoryInterface::class); + + // set user + $user = $userRepository->find(intval($this->option('user'))); + $jobRepository->setUser($user); + $journalRepository->setUser($user); + $accountRepository->setUser($user); + + // first date + $firstJournal = $journalRepository->first(); + $first = new Carbon; + if (!is_null($firstJournal->id)) { + $first = $firstJournal->date; + } + + // create job and settings. + $job = $jobRepository->create(); + $settings = [ + 'accounts' => $accountRepository->getAccountsByType([AccountType::ASSET, AccountType::DEFAULT]), + 'startDate' => $first, + 'endDate' => new Carbon, + 'exportFormat' => 'csv', + 'includeAttachments' => $this->option('with_attachments'), + 'includeOldUploads' => $this->option('with_uploads'), + 'job' => $job, + ]; + + + /** @var ProcessorInterface $processor */ + $processor = app(ProcessorInterface::class); + $processor->setSettings($settings); + + $processor->collectJournals(); + $processor->convertJournals(); + $processor->exportJournals(); + if ($settings['includeAttachments']) { + $processor->collectAttachments(); + } + + if ($settings['includeOldUploads']) { + $processor->collectOldUploads(); + } + + $processor->createZipFile(); + $disk = Storage::disk('export'); + $fileName = sprintf('export-%s.zip', date('Y-m-d_H-i-s')); + $disk->move($job->key . '.zip', $fileName); + + $this->line('The export has finished! You can find the ZIP file in this location:'); + $this->line(storage_path(sprintf('export/%s', $fileName))); + + return; } } diff --git a/app/Console/Commands/CreateImport.php b/app/Console/Commands/CreateImport.php index 191f361582..d0be59aa1e 100644 --- a/app/Console/Commands/CreateImport.php +++ b/app/Console/Commands/CreateImport.php @@ -30,6 +30,7 @@ use Monolog\Formatter\LineFormatter; */ class CreateImport extends Command { + use VerifiesAccessToken; /** * The console command description. * @@ -42,7 +43,13 @@ class CreateImport extends Command * * @var string */ - protected $signature = 'firefly:create-import {file} {configuration} {--user=1} {--type=csv} {--start}'; + protected $signature = 'firefly:create-import + {file : The file to import.} + {configuration : The configuration file to use for the import/} + {--type=csv : The file type of the import.} + {--user= : The user ID that the import should import for.} + {--token= : The user\'s access token.} + {--start : Starts the job immediately.}'; /** * Create a new command instance. @@ -61,6 +68,11 @@ class CreateImport extends Command */ public function handle() { + if (!$this->verifyAccessToken()) { + $this->error('Invalid access token.'); + + return; + } /** @var UserRepositoryInterface $userRepository */ $userRepository = app(UserRepositoryInterface::class); $file = $this->argument('file'); @@ -150,12 +162,12 @@ class CreateImport extends Command $cwd = getcwd(); $validTypes = array_keys(config('firefly.import_formats')); $type = strtolower($this->option('type')); - if (is_null($user->id)) { $this->error(sprintf('There is no user with ID %d.', $this->option('user'))); return false; } + if (!in_array($type, $validTypes)) { $this->error(sprintf('Cannot import file of type "%s"', $type)); diff --git a/app/Console/Commands/VerifiesAccessToken.php b/app/Console/Commands/VerifiesAccessToken.php new file mode 100644 index 0000000000..c77afaf57d --- /dev/null +++ b/app/Console/Commands/VerifiesAccessToken.php @@ -0,0 +1,52 @@ +option('user')); + $token = strval($this->option('token')); + /** @var UserRepositoryInterface $repository */ + $repository = app(UserRepositoryInterface::class); + $user = $repository->find($userId); + + if (is_null($user->id)) { + Log::error(sprintf('verifyAccessToken(): no such user for input "%d"', $userId)); + + return false; + } + $accessToken = Preferences::getForUser($user, 'access_token', null); + if (is_null($accessToken)) { + Log::error(sprintf('User #%d has no access token, so cannot access command line options.', $userId)); + + return false; + } + if (!($accessToken->data === $token)) { + Log::error(sprintf('Invalid access token for user #%d.', $userId)); + + return false; + } + + return true; + } + +} \ No newline at end of file diff --git a/app/Console/Commands/VerifyDatabase.php b/app/Console/Commands/VerifyDatabase.php index f69a630f1a..60bda05c5d 100644 --- a/app/Console/Commands/VerifyDatabase.php +++ b/app/Console/Commands/VerifyDatabase.php @@ -108,6 +108,9 @@ class VerifyDatabase extends Command } + /** + * + */ private function createAccessTokens() { $users = User::get(); @@ -117,6 +120,7 @@ class VerifyDatabase extends Command if (is_null($pref)) { $token = $user->generateAccessToken(); Preferences::setForUser($user, 'access_token', $token); + $this->line(sprintf('Generated access token for user %s', $user->email)); } } } diff --git a/app/Export/Collector/AttachmentCollector.php b/app/Export/Collector/AttachmentCollector.php index e3b8f295e9..e87fb5ec8d 100644 --- a/app/Export/Collector/AttachmentCollector.php +++ b/app/Export/Collector/AttachmentCollector.php @@ -122,6 +122,7 @@ class AttachmentCollector extends BasicCollector implements CollectorInterface */ private function getAttachments(): Collection { + $this->repository->setUser($this->user); $attachments = $this->repository->getBetween($this->start, $this->end); return $attachments; diff --git a/app/Export/Collector/BasicCollector.php b/app/Export/Collector/BasicCollector.php index 1780f43246..41ebc2d57c 100644 --- a/app/Export/Collector/BasicCollector.php +++ b/app/Export/Collector/BasicCollector.php @@ -15,6 +15,7 @@ namespace FireflyIII\Export\Collector; use FireflyIII\Models\ExportJob; +use FireflyIII\User; use Illuminate\Support\Collection; /** @@ -26,6 +27,8 @@ class BasicCollector { /** @var ExportJob */ protected $job; + /** @var User */ + protected $user; /** @var Collection */ private $entries; @@ -58,7 +61,16 @@ class BasicCollector */ public function setJob(ExportJob $job) { - $this->job = $job; + $this->job = $job; + $this->user = $job->user; + } + + /** + * @param User $user + */ + public function setUser(User $user) + { + $this->user = $user; } diff --git a/app/Export/Collector/JournalExportCollector.php b/app/Export/Collector/JournalExportCollector.php deleted file mode 100644 index b0e04e115e..0000000000 --- a/app/Export/Collector/JournalExportCollector.php +++ /dev/null @@ -1,347 +0,0 @@ -getWorkSet(); - - /* - * Extract: - * possible budget ids for journals - * possible category ids journals - * possible budget ids for transactions - * possible category ids for transactions - * - * possible IBAN and account numbers? - * - */ - $journals = $this->extractJournalIds(); - $transactions = $this->extractTransactionIds(); - - - // extend work set with category data from journals: - $this->categoryDataForJournals($journals); - - // extend work set with category cate from transactions (overrules journals): - $this->categoryDataForTransactions($transactions); - - // same for budgets: - $this->budgetDataForJournals($journals); - $this->budgetDataForTransactions($transactions); - - $this->setEntries($this->workSet); - - return true; - } - - /** - * @param Collection $accounts - */ - public function setAccounts(Collection $accounts) - { - $this->accounts = $accounts; - } - - /** - * @param Carbon $start - * @param Carbon $end - */ - public function setDates(Carbon $start, Carbon $end) - { - $this->start = $start; - $this->end = $end; - } - - /** - * @param array $journals - * - * @return bool - */ - private function budgetDataForJournals(array $journals): bool - { - $set = DB::table('budget_transaction_journal') - ->leftJoin('budgets', 'budgets.id', '=', 'budget_transaction_journal.budget_id') - ->whereIn('budget_transaction_journal.transaction_journal_id', $journals) - ->get( - [ - 'budget_transaction_journal.budget_id', - 'budget_transaction_journal.transaction_journal_id', - 'budgets.name', - 'budgets.encrypted', - ] - ); - $set->each( - function ($obj) { - $obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name); - } - ); - $array = []; - foreach ($set as $obj) { - $array[$obj->transaction_journal_id] = ['id' => $obj->budget_id, 'name' => $obj->name]; - } - - $this->workSet->each( - function ($obj) use ($array) { - if (isset($array[$obj->transaction_journal_id])) { - $obj->budget_id = $array[$obj->transaction_journal_id]['id']; - $obj->budget_name = $array[$obj->transaction_journal_id]['name']; - } - } - ); - - return true; - - } - - /** - * @param array $transactions - * - * @return bool - */ - private function budgetDataForTransactions(array $transactions): bool - { - $set = DB::table('budget_transaction') - ->leftJoin('budgets', 'budgets.id', '=', 'budget_transaction.budget_id') - ->whereIn('budget_transaction.transaction_id', $transactions) - ->get( - [ - 'budget_transaction.budget_id', - 'budget_transaction.transaction_id', - 'budgets.name', - 'budgets.encrypted', - ] - ); - $set->each( - function ($obj) { - $obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name); - } - ); - $array = []; - foreach ($set as $obj) { - $array[$obj->transaction_id] = ['id' => $obj->budget_id, 'name' => $obj->name]; - } - - $this->workSet->each( - function ($obj) use ($array) { - - // first transaction - if (isset($array[$obj->id])) { - $obj->budget_id = $array[$obj->id]['id']; - $obj->budget_name = $array[$obj->id]['name']; - } - } - ); - - return true; - - } - - /** - * @param array $journals - * - * @return bool - */ - private function categoryDataForJournals(array $journals): bool - { - $set = DB::table('category_transaction_journal') - ->leftJoin('categories', 'categories.id', '=', 'category_transaction_journal.category_id') - ->whereIn('category_transaction_journal.transaction_journal_id', $journals) - ->get( - [ - 'category_transaction_journal.category_id', - 'category_transaction_journal.transaction_journal_id', - 'categories.name', - 'categories.encrypted', - ] - ); - $set->each( - function ($obj) { - $obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name); - } - ); - $array = []; - foreach ($set as $obj) { - $array[$obj->transaction_journal_id] = ['id' => $obj->category_id, 'name' => $obj->name]; - } - - $this->workSet->each( - function ($obj) use ($array) { - if (isset($array[$obj->transaction_journal_id])) { - $obj->category_id = $array[$obj->transaction_journal_id]['id']; - $obj->category_name = $array[$obj->transaction_journal_id]['name']; - } - } - ); - - return true; - - } - - /** - * @param array $transactions - * - * @return bool - */ - private function categoryDataForTransactions(array $transactions): bool - { - $set = DB::table('category_transaction') - ->leftJoin('categories', 'categories.id', '=', 'category_transaction.category_id') - ->whereIn('category_transaction.transaction_id', $transactions) - ->get( - [ - 'category_transaction.category_id', - 'category_transaction.transaction_id', - 'categories.name', - 'categories.encrypted', - ] - ); - $set->each( - function ($obj) { - $obj->name = Steam::decrypt(intval($obj->encrypted), $obj->name); - } - ); - $array = []; - foreach ($set as $obj) { - $array[$obj->transaction_id] = ['id' => $obj->category_id, 'name' => $obj->name]; - } - - $this->workSet->each( - function ($obj) use ($array) { - - // first transaction - if (isset($array[$obj->id])) { - $obj->category_id = $array[$obj->id]['id']; - $obj->category_name = $array[$obj->id]['name']; - } - } - ); - - return true; - - } - - /** - * @return array - */ - private function extractJournalIds(): array - { - return $this->workSet->pluck('transaction_journal_id')->toArray(); - } - - /** - * @return array - */ - private function extractTransactionIds() - { - $set = $this->workSet->pluck('id')->toArray(); - $opposing = $this->workSet->pluck('opposing_id')->toArray(); - $complete = $set + $opposing; - - return array_unique($complete); - } - - /** - * @SuppressWarnings(PHPMD.ExcessiveMethodLength) - */ - private function getWorkSet() - { - $accountIds = $this->accounts->pluck('id')->toArray(); - $this->workSet = Transaction::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id') - ->leftJoin( - 'transactions AS opposing', function (JoinClause $join) { - $join->on('opposing.transaction_journal_id', '=', 'transactions.transaction_journal_id') - ->where('opposing.amount', '=', DB::raw('transactions.amount * -1')) - ->where('transactions.identifier', '=', DB::raw('opposing.identifier')); - } - ) - ->leftJoin('accounts', 'transactions.account_id', '=', 'accounts.id') - ->leftJoin('accounts AS opposing_accounts', 'opposing.account_id', '=', 'opposing_accounts.id') - ->leftJoin('transaction_types', 'transaction_journals.transaction_type_id', 'transaction_types.id') - ->leftJoin('transaction_currencies', 'transactions.transaction_currency_id', '=', 'transaction_currencies.id') - ->whereIn('transactions.account_id', $accountIds) - ->where('transaction_journals.user_id', $this->job->user_id) - ->where('transaction_journals.date', '>=', $this->start->format('Y-m-d')) - ->where('transaction_journals.date', '<=', $this->end->format('Y-m-d')) - ->where('transaction_journals.completed', 1) - ->whereNull('transaction_journals.deleted_at') - ->whereNull('transactions.deleted_at') - ->whereNull('opposing.deleted_at') - ->orderBy('transaction_journals.date', 'DESC') - ->orderBy('transactions.identifier', 'ASC') - ->get( - [ - 'transactions.id', - 'transactions.amount', - 'transactions.description', - 'transactions.account_id', - 'accounts.name as account_name', - 'accounts.encrypted as account_name_encrypted', - 'transactions.identifier', - - 'opposing.id as opposing_id', - 'opposing.amount AS opposing_amount', - 'opposing.description as opposing_description', - 'opposing.account_id as opposing_account_id', - 'opposing_accounts.name as opposing_account_name', - 'opposing_accounts.encrypted as opposing_account_encrypted', - 'opposing.identifier as opposing_identifier', - - 'transaction_journals.id as transaction_journal_id', - 'transaction_journals.date', - 'transaction_journals.description as journal_description', - 'transaction_journals.encrypted as journal_encrypted', - 'transaction_journals.transaction_type_id', - 'transaction_types.type as transaction_type', - 'transactions.transaction_currency_id', - 'transaction_currencies.code AS transaction_currency_code', - - ] - ); - } -} diff --git a/app/Export/Collector/UploadCollector.php b/app/Export/Collector/UploadCollector.php index a27de79610..3647e478b8 100644 --- a/app/Export/Collector/UploadCollector.php +++ b/app/Export/Collector/UploadCollector.php @@ -30,8 +30,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface private $exportDisk; /** @var \Illuminate\Contracts\Filesystem\Filesystem */ private $uploadDisk; - /** @var string */ - private $vintageFormat; /** * AttachmentCollector constructor. diff --git a/app/Export/ExpandedProcessor.php b/app/Export/ExpandedProcessor.php index 8babc5b473..f8469246d6 100644 --- a/app/Export/ExpandedProcessor.php +++ b/app/Export/ExpandedProcessor.php @@ -91,6 +91,7 @@ class ExpandedProcessor implements ProcessorInterface // use journal collector thing. /** @var JournalCollectorInterface $collector */ $collector = app(JournalCollectorInterface::class); + $collector->setUser($this->job->user); $collector->setAccounts($this->accounts)->setRange($this->settings['startDate'], $this->settings['endDate']) ->withOpposingAccount()->withBudgetInformation()->withCategoryInformation() ->removeFilter(InternalTransferFilter::class); diff --git a/app/Export/Processor.php b/app/Export/Processor.php deleted file mode 100644 index 8df3a93dce..0000000000 --- a/app/Export/Processor.php +++ /dev/null @@ -1,203 +0,0 @@ -journals = new Collection; - $this->exportEntries = new Collection; - $this->files = new Collection; - - } - - /** - * @return bool - */ - public function collectAttachments(): bool - { - /** @var AttachmentCollector $attachmentCollector */ - $attachmentCollector = app(AttachmentCollector::class); - $attachmentCollector->setJob($this->job); - $attachmentCollector->setDates($this->settings['startDate'], $this->settings['endDate']); - $attachmentCollector->run(); - $this->files = $this->files->merge($attachmentCollector->getEntries()); - - return true; - } - - /** - * @return bool - */ - public function collectJournals(): bool - { - /** @var JournalExportCollector $collector */ - $collector = app(JournalExportCollector::class); - $collector->setJob($this->job); - $collector->setDates($this->settings['startDate'], $this->settings['endDate']); - $collector->setAccounts($this->settings['accounts']); - $collector->run(); - $this->journals = $collector->getEntries(); - Log::debug(sprintf('Count %d journals in collectJournals() ', $this->journals->count())); - - return true; - } - - /** - * @return bool - */ - public function collectOldUploads(): bool - { - /** @var UploadCollector $uploadCollector */ - $uploadCollector = app(UploadCollector::class); - $uploadCollector->setJob($this->job); - $uploadCollector->run(); - - $this->files = $this->files->merge($uploadCollector->getEntries()); - - return true; - } - - /** - * @return bool - */ - public function convertJournals(): bool - { - $count = 0; - foreach ($this->journals as $object) { - $this->exportEntries->push(Entry::fromObject($object)); - $count++; - } - Log::debug(sprintf('Count %d entries in exportEntries (convertJournals)', $this->exportEntries->count())); - - return true; - } - - /** - * @return bool - * @throws FireflyException - */ - public function createZipFile(): bool - { - $zip = new ZipArchive; - $file = $this->job->key . '.zip'; - $fullPath = storage_path('export') . '/' . $file; - - if ($zip->open($fullPath, ZipArchive::CREATE) !== true) { - throw new FireflyException('Cannot store zip file.'); - } - // for each file in the collection, add it to the zip file. - $disk = Storage::disk('export'); - foreach ($this->getFiles() as $entry) { - // is part of this job? - $zipFileName = str_replace($this->job->key . '-', '', $entry); - $zip->addFromString($zipFileName, $disk->get($entry)); - } - - $zip->close(); - - // delete the files: - $this->deleteFiles(); - - return true; - } - - /** - * @return bool - */ - public function exportJournals(): bool - { - $exporterClass = config('firefly.export_formats.' . $this->exportFormat); - $exporter = app($exporterClass); - $exporter->setJob($this->job); - $exporter->setEntries($this->exportEntries); - $exporter->run(); - $this->files->push($exporter->getFileName()); - - return true; - } - - /** - * @return Collection - */ - public function getFiles(): Collection - { - return $this->files; - } - - /** - * @param array $settings - */ - public function setSettings(array $settings) - { - // save settings - $this->settings = $settings; - $this->accounts = $settings['accounts']; - $this->exportFormat = $settings['exportFormat']; - $this->includeAttachments = $settings['includeAttachments']; - $this->includeOldUploads = $settings['includeOldUploads']; - $this->job = $settings['job']; - } - - /** - * - */ - private function deleteFiles() - { - $disk = Storage::disk('export'); - foreach ($this->getFiles() as $file) { - $disk->delete($file); - } - } -}