Various improvements for https://github.com/firefly-iii/firefly-iii/issues/8281

app/Api/V1/Controllers/Summary/BasicController.php

@@ -101,6 +101,10 @@ class BasicController extends Controller
         $billData     = $this->getBillInformation($start, $end);
         $spentData    = $this->getLeftToSpendInfo($start, $end);
         $netWorthData = $this->getNetWorthInfo($start, $end);
+        //        $balanceData  = [];
+        //        $billData     = [];
+        //        $spentData    = [];
+        //        $netWorthData = [];
         $total = array_merge($balanceData, $billData, $spentData, $netWorthData);
 
         // give new keys

app/Api/V2/Controllers/Chart/BudgetController.php

@@ -225,6 +225,7 @@ class BudgetController extends Controller
                 $return[$currencyId]['native_spent'] = bcadd($return[$currencyId]['native_spent'], $convertedAmount);
             }
         }
+        $converter->summarize();
 
         return $return;
     }
@@ -288,6 +289,7 @@ class BudgetController extends Controller
                 $result[$limitCurrencyId]['native_overspent'] = app('steam')->positive(bcadd($convertedLimitAmount, $result[$limitCurrencyId]['native_spent']));
             }
         }
+        $converter->summarize();
 
         return $result;
     }

app/Api/V2/Controllers/Chart/CategoryController.php

@@ -137,6 +137,7 @@ class CategoryController extends Controller
         usort($return, static function (array $a, array $b) {
             return (float)$a['native_amount'] < (float)$b['native_amount'] ? 1 : -1;
         });
+        $converter->summarize();
 
         return response()->json($this->clean($return));
     }

app/Api/V2/Controllers/Summary/BasicController.php

@@ -278,7 +278,6 @@ class BasicController extends Controller
          */
         foreach ($spent as $currencyId => $row) {
             app('log')->debug(sprintf('Processing spent array in currency #%d', $currencyId));
-            $currencyId  = $currencyId;
             $spent       = '0';
             $spentNative = '0';
 
@@ -351,6 +350,7 @@ class BasicController extends Controller
         }
         $return[] = $nativeLeft;
         $return[] = $nativePerDay;
+        $converter->summarize();
 
         return $return;
     }

app/Helpers/Collector/GroupCollector.php

@@ -927,7 +927,7 @@ class GroupCollector implements GroupCollectorInterface
     {
         $currentCollection = $collection;
 
-        app('log')->debug(sprintf('GroupCollector: postFilterCollection has %d filter(s) and %d transaction(s).', count($this->postFilters), count($currentCollection)));
+        // app('log')->debug(sprintf('GroupCollector: postFilterCollection has %d filter(s) and %d transaction(s).', count($this->postFilters), count($currentCollection)));
 
         /**
          * @var \Closure $function

app/Helpers/Report/NetWorth.php

@@ -138,6 +138,7 @@ class NetWorth implements NetWorthInterface
             $netWorth['native']['native_balance']    = bcadd($nativeBalance, $netWorth['native']['native_balance']);
         }
         $cache->store($netWorth);
+        $converter->summarize();
 
         return $netWorth;
     }

app/Repositories/UserGroups/Bill/BillRepository.php

@@ -118,6 +118,7 @@ class BillRepository implements BillRepositoryInterface
                 }
             }
         }
+        $converter->summarize();
 
         return $return;
     }
@@ -167,6 +168,7 @@ class BillRepository implements BillRepositoryInterface
                 $return[$currencyId]['native_sum'] = bcadd($return[$currencyId]['native_sum'], bcmul($nativeAverage, (string)$total));
             }
         }
+        $converter->summarize();
 
         return $return;
     }

app/Repositories/UserGroups/Budget/AvailableBudgetRepository.php

@@ -67,6 +67,7 @@ class AvailableBudgetRepository implements AvailableBudgetRepositoryInterface
             $return[$currencyId]['amount']        = bcadd($return[$currencyId]['amount'], $availableBudget->amount);
             $return[$currencyId]['native_amount'] = bcadd($return[$currencyId]['native_amount'], $nativeAmount);
         }
+        $converter->summarize();
 
         return $return;
     }

app/Support/Http/Api/AccountBalanceGrouped.php

@@ -193,6 +193,7 @@ class AccountBalanceGrouped
             $convertedKey                                    = sprintf('native_%s', $key);
             $this->data[$currencyId][$period][$convertedKey] = bcadd($this->data[$currencyId][$period][$convertedKey], $amountConverted);
         }
+        $converter->summarize();
     }
 
     public function setAccounts(Collection $accounts): void

app/Support/Http/Api/ExchangeRateConverter.php

@@ -37,6 +37,8 @@ use Illuminate\Support\Facades\Log;
 class ExchangeRateConverter
 {
     // use ConvertsExchangeRates;
+    private int   $queryCount = 0;
+    private array $prepared   = [];
 
     /**
      * @throws FireflyException
@@ -48,6 +50,56 @@ class ExchangeRateConverter
         return bcmul($amount, $rate);
     }
 
+    public function prepare(TransactionCurrency $from, TransactionCurrency $to, Carbon $start, Carbon $end): void
+    {
+        $start->startOfDay();
+        $end->endOfDay();
+        Log::debug(sprintf('Preparing for %s to %s between %s and %s', $from->code, $to->code, $start->format('Y-m-d'), $end->format('Y-m-d')));
+        $set      = auth()->user()
+            ->currencyExchangeRates()
+            ->where('from_currency_id', $from->id)
+            ->where('to_currency_id', $to->id)
+            ->where('date', '<=', $end->format('Y-m-d'))
+            ->where('date', '>=', $start->format('Y-m-d'))
+            ->orderBy('date', 'DESC')->get()
+        ;
+        $fallback = $this->getRate($from, $to, $start);
+        ++$this->queryCount;
+        if (0 === $set->count()) {
+            Log::debug('No rates found in this period.');
+
+            return;
+        }
+        /*
+         * Je moet een fallback rate hebben van voor de periode zodat je die altijd kan gebruiken.
+         * Dus de laatste met de meest recente datum voor je periode.
+         * Dan moet je gaan loopen per dag, en als er iets in $temp zit toevallig gebruik je die.
+         */
+        $temp = [];
+        $count = 0;
+        foreach ($set as $rate) {
+            $date        = $rate->date->format('Y-m-d');
+            $temp[$date] ??= [
+                $from->id => [
+                    $to->id => $rate->rate,
+                ],
+            ];
+            ++$count;
+        }
+        Log::debug(sprintf('Found %d rates in this period.', $count));
+        $currentStart = clone $start;
+        while ($currentStart->lte($end)) {
+            $currentDate                  = $currentStart->format('Y-m-d');
+            $this->prepared[$currentDate] ??= [];
+            $fallback                     = $temp[$currentDate][$from->id][$to->id] ?? $fallback;
+            if (0 === count($this->prepared[$currentDate])) {
+                // fill from temp or fallback or from temp (see before)
+                $this->prepared[$currentDate][$from->id][$to->id] = $fallback;
+            }
+            $currentStart->addDay();
+        }
+    }
+
     /**
      * @throws FireflyException
      */
@@ -58,6 +110,11 @@ class ExchangeRateConverter
         return '0' === $rate ? '1' : $rate;
     }
 
+    public function summarize(): void
+    {
+        Log::info(sprintf('ExchangeRateConverter ran %d queries.', $this->queryCount));
+    }
+
     /**
      * @throws FireflyException
      */
@@ -94,6 +151,12 @@ class ExchangeRateConverter
     {
         $key = sprintf('cer-%d-%d-%s', $from, $to, $date);
 
+        // perhaps the rate has been cached during this particular run
+        $preparedRate = $this->prepared[$date][$from][$to] ?? null;
+        if (null !== $preparedRate) {
+            return $preparedRate;
+        }
+
         $cache = new CacheProperties();
         $cache->addProperty($key);
         if ($cache->has()) {
@@ -115,8 +178,23 @@ class ExchangeRateConverter
             ->orderBy('date', 'DESC')
             ->first()
         ;
+        ++$this->queryCount;
         $rate = (string) $result?->rate;
         $cache->store($rate);
+
+        // if the rate has not been cached during this particular run, save it
+        $this->prepared[$date] ??= [
+            $from => [
+                $to => $rate,
+            ],
+        ];
+        // also save the exchange rate the other way around:
+        $this->prepared[$date] ??= [
+            $to => [
+                $from => bcdiv('1', $rate),
+            ],
+        ];
+
         if ('' === $rate) {
             return null;
         }
@@ -168,6 +246,7 @@ class ExchangeRateConverter
             return (int) $cache->get();
         }
         $euro = TransactionCurrency::whereCode('EUR')->first();
+        ++$this->queryCount;
         if (null === $euro) {
             throw new FireflyException('Cannot find EUR in system, cannot do currency conversion.');
         }

app/Support/Http/Api/SummaryBalanceGrouped.php

@@ -79,6 +79,7 @@ class SummaryBalanceGrouped
         }
         app('log')->debug(sprintf('this->amounts[%s][native] is now %s', $key, $this->amounts[$key]['native']));
         app('log')->debug(sprintf('this->amounts[%s][native] is now %s', self::SUM, $this->amounts[$key]['native']));
+        $converter->summarize();
     }
 
     public function groupData(): array

app/Support/Steam.php

@@ -24,7 +24,6 @@ declare(strict_types=1);
 namespace FireflyIII\Support;
 
 use Carbon\Carbon;
-use Exception;
 use FireflyIII\Exceptions\FireflyException;
 use FireflyIII\Models\Account;
 use FireflyIII\Models\Transaction;
@@ -157,7 +156,7 @@ class Steam
                 // use foreign amount:
                 $amount = $foreignModified;
             }
-            Log::debug(sprintf('Trying to add %s and %s.', var_export($currentBalance, true), var_export($amount, true)));
+            // Log::debug(sprintf('Trying to add %s and %s.', var_export($currentBalance, true), var_export($amount, true)));
             $currentBalance  = bcadd($currentBalance, $amount);
             $carbon          = new Carbon($entry->date, config('app.timezone'));
             $date            = $carbon->format('Y-m-d');
@@ -315,6 +314,7 @@ class Steam
         }
 
         $cache->store($balances);
+        $converter->summarize();
 
         return $balances;
     }
@@ -327,15 +327,15 @@ class Steam
      */
     public function balanceConverted(Account $account, Carbon $date, TransactionCurrency $native): string
     {
-        app('log')->debug(sprintf('Now in balanceConverted (%s) for account #%d, converting to %s', $date->format('Y-m-d'), $account->id, $native->code));
+        //        app('log')->debug(sprintf('Now in balanceConverted (%s) for account #%d, converting to %s', $date->format('Y-m-d'), $account->id, $native->code));
-        // abuse chart properties:
         $cache = new CacheProperties();
         $cache->addProperty($account->id);
         $cache->addProperty('balance');
         $cache->addProperty($date);
         $cache->addProperty($native->id);
         if ($cache->has()) {
-            return $cache->get();
+            //            Log::debug('Cached!');
+            //            return $cache->get();
         }
 
         /** @var AccountRepositoryInterface $repository */
@@ -343,6 +343,7 @@ class Steam
         $currency   = $repository->getAccountCurrency($account);
         $currency   = null === $currency ? app('amount')->getDefaultCurrencyByUserGroup($account->user->userGroup) : $currency;
         if ($native->id === $currency->id) {
+            //            Log::debug('No conversion necessary!');
             return $this->balance($account, $date);
         }
 
@@ -424,34 +425,43 @@ class Steam
         // need to convert the others. All sets use the "amount" value as their base (that's easy)
         // but we need to convert each transaction separately because the date difference may
         // incur huge currency changes.
+        $start     = clone $date;
+        $end       = clone $date;
         $converter = new ExchangeRateConverter();
         foreach ($new as $set) {
             foreach ($set as $transaction) {
-                $date = Carbon::createFromFormat('Y-m-d H:i:s', $transaction['date']);
+                $currentDate = Carbon::createFromFormat('Y-m-d H:i:s', $transaction['date']);
-                if (false === $date) {
+                if (false === $currentDate) {
-                    $date = today(config('app.timezone'));
+                    $currentDate = today(config('app.timezone'));
                 }
-                $rate            = $converter->getCurrencyRate($currency, $native, $date);
+                if ($currentDate->lte($start)) {
+                    $start = clone $currentDate;
+                }
+            }
+        }
+        unset($currentDate);
+        $converter->prepare($currency, $native, $start, $end);
+
+        foreach ($new as $set) {
+            foreach ($set as $transaction) {
+                $currentDate = Carbon::createFromFormat('Y-m-d H:i:s', $transaction['date']);
+                if (false === $currentDate) {
+                    $currentDate = today(config('app.timezone'));
+                }
+                $rate            = $converter->getCurrencyRate($currency, $native, $currentDate);
                 $convertedAmount = bcmul($transaction['amount'], $rate);
                 $balance         = bcadd($balance, $convertedAmount);
-                //                app('log')->debug(sprintf('Date: %s, rate: %s, amount: %s %s, new: %s %s',
-                //                                          $date->format('Y-m-d'),
-                //                                          $rate,
-                //                                          $currency->code,
-                //                                          $transaction['amount'],
-                //                                          $native->code,
-                //                                          $convertedAmount
-                //                                  ));
             }
-            // app('log')->debug(sprintf('Balance from new set #%d is %f', $index, $balance));
         }
 
         // add virtual balance (also needs conversion)
         $virtual = null === $account->virtual_balance ? '0' : $account->virtual_balance;
         $virtual = $converter->convert($currency, $native, $account->created_at, $virtual);
         $balance = bcadd($balance, $virtual);
+        $converter->summarize();
 
         $cache->store($balance);
+        $converter->summarize();
 
         return $balance;
     }

app/Transformers/V2/BillTransformer.php

@@ -192,6 +192,7 @@ class BillTransformer extends AbstractTransformer
         $date = null === $startParam ? today() : clone $startParam;
 
         $nextExpectedMatchDiff = $this->getNextExpectedMatchDiff($nextExpectedMatch, $payDates);
+        $this->converter->summarize();
 
         return [
             'id'                             => $bill->id,

app/Transformers/V2/PiggyBankTransformer.php

@@ -193,6 +193,7 @@ class PiggyBankTransformer extends AbstractTransformer
             $savePerMonth       = $this->getSuggestedMonthlyAmount($currentAmount, $targetAmount, $piggyBank->startdate, $piggyBank->targetdate);
             $nativeSavePerMonth = $this->converter->convert($this->default, $currency, today(), $savePerMonth);
         }
+        $this->converter->summarize();
 
         return [
             'id'                             => (string)$piggyBank->id,

app/Transformers/V2/TransactionGroupTransformer.php

@@ -161,6 +161,7 @@ class TransactionGroupTransformer extends AbstractTransformer
             $foreignAmount       = app('steam')->positive($transaction['foreign_amount']);
             $nativeForeignAmount = $this->converter->convert($this->default, $this->currencies[$foreignCurrencyId], $transaction['date'], $foreignAmount);
         }
+        $this->converter->summarize();
 
         return [
             'user'                            => (string)$transaction['user_id'],