From beed44f065aa10d432af3620f1feae08cb7f425d Mon Sep 17 00:00:00 2001 From: James Cole Date: Thu, 9 Jun 2022 17:47:01 +0200 Subject: [PATCH] Add cron job to download transaction currencies from Azure (if enabled by user). --- .env.example | 6 + app/Console/Commands/Tools/Cron.php | 42 ++++ app/Jobs/DownloadExchangeRates.php | 182 ++++++++++++++++++ .../Currency/CurrencyRepository.php | 21 ++ .../Currency/CurrencyRepositoryInterface.php | 10 + app/Support/Cronjobs/ExchangeRatesCronjob.php | 95 +++++++++ config/cer.php | 7 +- .../seeders/TransactionCurrencySeeder.php | 2 +- 8 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 app/Jobs/DownloadExchangeRates.php create mode 100644 app/Support/Cronjobs/ExchangeRatesCronjob.php diff --git a/.env.example b/.env.example index f2d0531763..ee694fa0fb 100644 --- a/.env.example +++ b/.env.example @@ -163,6 +163,12 @@ SEND_REPORT_JOURNALS=true # and disabled by default. ENABLE_EXTERNAL_MAP=false +# Set this value to true if you want Firefly III to download currency exchange rates +# from the internet. These rates are hosted by the creator of Firefly III inside +# an Azure Storage Container. +# Not all currencies may be available. Rates may be wrong. +ENABLE_EXTERNAL_RATES=false + # The map will default to this location: MAP_DEFAULT_LAT=51.983333 MAP_DEFAULT_LONG=5.916667 diff --git a/app/Console/Commands/Tools/Cron.php b/app/Console/Commands/Tools/Cron.php index 81cd65fe54..bea0c2a2e8 100644 --- a/app/Console/Commands/Tools/Cron.php +++ b/app/Console/Commands/Tools/Cron.php @@ -28,6 +28,7 @@ use Carbon\Carbon; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Support\Cronjobs\AutoBudgetCronjob; use FireflyIII\Support\Cronjobs\BillWarningCronjob; +use FireflyIII\Support\Cronjobs\ExchangeRatesCronjob; use FireflyIII\Support\Cronjobs\RecurringCronjob; use Illuminate\Console\Command; use InvalidArgumentException; @@ -69,6 +70,19 @@ class Cron extends Command } $force = (bool) $this->option('force'); + /* + * Fire recurring transaction cron job. + */ + if (true === config('cer.enabled')) { + try { + $this->exchangeRatesCronJob($force, $date); + } catch (FireflyException $e) { + Log::error($e->getMessage()); + Log::error($e->getTraceAsString()); + $this->error($e->getMessage()); + } + } + /* * Fire recurring transaction cron job. */ @@ -190,4 +204,32 @@ class Cron extends Command } } + + /** + * @param bool $force + * @param Carbon|null $date + * @throws FireflyException + */ + private function exchangeRatesCronJob(bool $force, ?Carbon $date): void + { + $exchangeRates = new ExchangeRatesCronjob; + $exchangeRates->setForce($force); + // set date in cron job: + if (null !== $date) { + $exchangeRates->setDate($date); + } + + $exchangeRates->fire(); + + if ($exchangeRates->jobErrored) { + $this->error(sprintf('Error in "exchange rates" cron: %s', $exchangeRates->message)); + } + if ($exchangeRates->jobFired) { + $this->error(sprintf('"Exchange rates" cron fired: %s', $exchangeRates->message)); + } + if ($exchangeRates->jobSucceeded) { + $this->error(sprintf('"Exchange rates" cron ran with success: %s', $exchangeRates->message)); + } + + } } diff --git a/app/Jobs/DownloadExchangeRates.php b/app/Jobs/DownloadExchangeRates.php new file mode 100644 index 0000000000..a4c4b0cebf --- /dev/null +++ b/app/Jobs/DownloadExchangeRates.php @@ -0,0 +1,182 @@ +. + */ + +namespace FireflyIII\Jobs; + + +use Carbon\Carbon; +use FireflyIII\Models\TransactionCurrency; +use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface; +use FireflyIII\Repositories\User\UserRepositoryInterface; +use GuzzleHttp\Client; +use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Foundation\Bus\Dispatchable; +use Illuminate\Queue\InteractsWithQueue; +use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Collection; +use Log; + +/** + * Class DownloadExchangeRates + */ +class DownloadExchangeRates implements ShouldQueue +{ + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + + private Carbon $date; + private CurrencyRepositoryInterface $repository; + private array $active; + private Collection $users; + + /** + * Create a new job instance. + * + * @codeCoverageIgnore + * + * @param Carbon|null $date + */ + public function __construct(?Carbon $date) + { + $this->active = []; + $this->repository = app(CurrencyRepositoryInterface::class); + + // get all users: + /** @var UserRepositoryInterface $userRepository */ + $userRepository = app(UserRepositoryInterface::class); + $this->users = $userRepository->all(); + + if (null !== $date) { + $newDate = clone $date; + $newDate->startOfDay(); + $this->date = $newDate; + Log::debug(sprintf('Created new DownloadExchangeRates("%s")', $this->date->format('Y-m-d'))); + } + } + + /** + * Execute the job. + */ + public function handle(): void + { + $currencies = $this->repository->get(); + + /** @var TransactionCurrency $currency */ + foreach ($currencies as $currency) { + $this->downloadRates($currency); + } + } + + /** + * @param Carbon $date + */ + public function setDate(Carbon $date): void + { + $newDate = clone $date; + $newDate->startOfDay(); + $this->date = $newDate; + } + + /** + * @param TransactionCurrency $currency + * @return void + */ + private function downloadRates(TransactionCurrency $currency): void + { + Log::debug(sprintf('Now downloading new exchange rates for currency %s.', $currency->code)); + $base = sprintf('%s/%s/%s', (string) config('cer.url'), $this->date->year, $this->date->isoWeek); + $client = new Client; + $url = sprintf('%s/%s.json', $base, $currency->code); + $res = $client->get($url); + $statusCode = $res->getStatusCode(); + if (200 !== $statusCode) { + Log::warning(sprintf('Trying to grab "%s" resulted in %d.', $url, $statusCode)); + return; + } + $body = (string) $res->getBody(); + $json = json_decode($body, true); + if (false === $json || null === $json) { + Log::warning(sprintf('Trying to grab "%s" resulted in bad JSON.', $url)); + return; + } + $date = Carbon::createFromFormat('Y-m-d', $json['date']); + $this->saveRates($currency, $date, $json['rates']); + } + + /** + * @param TransactionCurrency $currency + * @param array $rates + * @return void + */ + private function saveRates(TransactionCurrency $currency, Carbon $date, array $rates): void + { + foreach ($rates as $code => $rate) { + $to = $this->getCurrency($code); + if (null === $to) { + Log::debug(sprintf('Currency %s is not in use, do not save rate.', $code)); + continue; + } + Log::debug(sprintf('Currency %s is in use.', $code)); + $this->saveRate($currency, $to, $date, $rate); + } + } + + /** + * @param string $code + * @return TransactionCurrency|null + */ + private function getCurrency(string $code): ?TransactionCurrency + { + // if we have it already, don't bother searching for it again. + if (array_key_exists($code, $this->active)) { + Log::debug(sprintf('Already know what the result is of searching for %s', $code)); + return $this->active[$code]; + } + // find it in the database. + $currency = $this->repository->findByCode($code); + if (null === $currency) { + Log::debug(sprintf('Did not find currency %s.', $code)); + $this->active[$code] = null; + return null; + } + if (false === $currency->enabled) { + Log::debug(sprintf('Currency %s is not enabled.', $code)); + $this->active[$code] = null; + return null; + } + Log::debug(sprintf('Currency %s is enabled.', $code)); + $this->active[$code] = $currency; + + return $currency; + } + + private function saveRate(TransactionCurrency $from, TransactionCurrency $to, Carbon $date, float $rate): void + { + foreach ($this->users as $user) { + $this->repository->setUser($user); + $existing = $this->repository->getExchangeRate($from, $to, $date); + if (null === $existing) { + Log::debug(sprintf('Saved rate from %s to %s for user #%d.', $from->code, $to->code, $user->id)); + $this->repository->setExchangeRate($from, $to, $date, $rate); + } + } + } +} diff --git a/app/Repositories/Currency/CurrencyRepository.php b/app/Repositories/Currency/CurrencyRepository.php index 4619703cfc..7bbe2b1f12 100644 --- a/app/Repositories/Currency/CurrencyRepository.php +++ b/app/Repositories/Currency/CurrencyRepository.php @@ -441,6 +441,27 @@ class CurrencyRepository implements CurrencyRepositoryInterface return null; } + /** + * TODO must be a factory + * @param TransactionCurrency $fromCurrency + * @param TransactionCurrency $toCurrency + * @param Carbon $date + * @param float $rate + * @return CurrencyExchangeRate + */ + public function setExchangeRate(TransactionCurrency $fromCurrency, TransactionCurrency $toCurrency, Carbon $date, float $rate): CurrencyExchangeRate + { + return CurrencyExchangeRate::create( + [ + 'user_id' => $this->user->id, + 'from_currency_id' => $fromCurrency->id, + 'to_currency_id' => $toCurrency->id, + 'date' => $date, + 'rate' => $rate, + ] + ); + } + /** * @inheritDoc */ diff --git a/app/Repositories/Currency/CurrencyRepositoryInterface.php b/app/Repositories/Currency/CurrencyRepositoryInterface.php index 35efcb3b76..e1c0e6c3ca 100644 --- a/app/Repositories/Currency/CurrencyRepositoryInterface.php +++ b/app/Repositories/Currency/CurrencyRepositoryInterface.php @@ -202,6 +202,16 @@ interface CurrencyRepositoryInterface */ public function getExchangeRate(TransactionCurrency $fromCurrency, TransactionCurrency $toCurrency, Carbon $date): ?CurrencyExchangeRate; + /** + * TODO must be a factory + * @param TransactionCurrency $fromCurrency + * @param TransactionCurrency $toCurrency + * @param Carbon $date + * @param float $rate + * @return CurrencyExchangeRate + */ + public function setExchangeRate(TransactionCurrency $fromCurrency, TransactionCurrency $toCurrency, Carbon $date, float $rate): CurrencyExchangeRate; + /** * @param TransactionCurrency $currency * diff --git a/app/Support/Cronjobs/ExchangeRatesCronjob.php b/app/Support/Cronjobs/ExchangeRatesCronjob.php new file mode 100644 index 0000000000..18d67b896f --- /dev/null +++ b/app/Support/Cronjobs/ExchangeRatesCronjob.php @@ -0,0 +1,95 @@ +. + */ + +declare(strict_types=1); + +namespace FireflyIII\Support\Cronjobs; + +use Carbon\Carbon; +use FireflyIII\Jobs\DownloadExchangeRates; +use FireflyIII\Models\Configuration; +use Log; + +/** + * Class ExchangeRatesCronjob + */ +class ExchangeRatesCronjob extends AbstractCronjob +{ + + /** + * @inheritDoc + */ + public function fire(): void + { + /** @var Configuration $config */ + $config = app('fireflyconfig')->get('last_cer_job', 0); + $lastTime = (int) $config->data; + $diff = time() - $lastTime; + $diffForHumans = Carbon::now()->diffForHumans(Carbon::createFromTimestamp($lastTime), null, true); + if (0 === $lastTime) { + Log::info('Exchange rates cron-job has never fired before.'); + } + // less than half a day ago: + if ($lastTime > 0 && $diff <= 43200) { + Log::info(sprintf('It has been %s since the exchange rates cron-job has fired.', $diffForHumans)); + if (false === $this->force) { + Log::info('The exchange rates cron-job will not fire now.'); + $this->message = sprintf('It has been %s since the exchange rates cron-job has fired. It will not fire now.', $diffForHumans); + + return; + } + + // fire job regardless. + if (true === $this->force) { + Log::info('Execution of the exchange rates cron-job has been FORCED.'); + } + } + + if ($lastTime > 0 && $diff > 43200) { + Log::info(sprintf('It has been %s since the exchange rates cron-job has fired. It will fire now!', $diffForHumans)); + } + + $this->fireAutoBudget(); + app('preferences')->mark(); + } + + /** + * + */ + private function fireAutoBudget(): void + { + Log::info(sprintf('Will now fire exchange rates cron job task for date "%s".', $this->date->format('Y-m-d'))); + /** @var DownloadExchangeRates $job */ + $job = app(DownloadExchangeRates::class); + $job->setDate($this->date); + $job->handle(); + + // get stuff from job: + $this->jobFired = true; + $this->jobErrored = false; + $this->jobSucceeded = true; + $this->message = 'Exchange rates cron job fired successfully.'; + + app('fireflyconfig')->set('last_cer_job', (int) $this->date->format('U')); + Log::info('Done with exchange rates job task.'); + } +} diff --git a/config/cer.php b/config/cer.php index 8e173a66f8..4614277cdb 100644 --- a/config/cer.php +++ b/config/cer.php @@ -21,11 +21,14 @@ return [ + + 'url' => 'https://ff3exchangerates.z6.web.core.windows.net', + 'enabled' => env('ENABLE_EXTERNAL_RATES', false), // if currencies are added, default rates must be added as well! // last exchange rate update: 6-6-2022 // source: https://www.xe.com/currencyconverter/ - 'date' => '2022-06-06', - 'rates' => [ + 'date' => '2022-06-06', + 'rates' => [ // europa ['EUR', 'HUF', 387.9629], ['EUR', 'GBP', 0.85420754], diff --git a/database/seeders/TransactionCurrencySeeder.php b/database/seeders/TransactionCurrencySeeder.php index e74248a0ed..37e4f2c36a 100644 --- a/database/seeders/TransactionCurrencySeeder.php +++ b/database/seeders/TransactionCurrencySeeder.php @@ -65,7 +65,7 @@ class TransactionCurrencySeeder extends Seeder // asian currencies $currencies[] = ['code' => 'JPY', 'name' => 'Japanese yen', 'symbol' => '¥', 'decimal_places' => 0]; - $currencies[] = ['code' => 'RMB', 'name' => 'Chinese yuan', 'symbol' => '¥', 'decimal_places' => 2]; + $currencies[] = ['code' => 'CNY', 'name' => 'Chinese yuan', 'symbol' => '¥', 'decimal_places' => 2]; $currencies[] = ['code' => 'RUB', 'name' => 'Russian ruble', 'symbol' => '₽', 'decimal_places' => 2]; $currencies[] = ['code' => 'INR', 'name' => 'Indian rupee', 'symbol' => '₹', 'decimal_places' => 2];