From 84560a6f44746b89bbb4ae3e9791497d0ab85564 Mon Sep 17 00:00:00 2001 From: Sobuno Date: Tue, 31 Dec 2024 10:16:27 +0100 Subject: [PATCH] WIP --- app/Providers/SearchServiceProvider.php | 11 + app/Support/Search/GdbotsQueryParser.php | 80 +++++++ app/Support/Search/OperatorQuerySearch.php | 149 ++++++------ app/Support/Search/QueryParser.php | 171 ++++++++++++++ app/Support/Search/QueryParserInterface.php | 112 +++++++++ ...ractQueryParserInterfaceParseQueryTest.php | 213 ++++++++++++++++++ .../GdbotsQueryParserParseQueryTest.php | 24 ++ .../Search/QueryParserParseQueryTest.php | 25 ++ 8 files changed, 699 insertions(+), 86 deletions(-) create mode 100644 app/Support/Search/GdbotsQueryParser.php create mode 100644 app/Support/Search/QueryParser.php create mode 100644 app/Support/Search/QueryParserInterface.php create mode 100644 tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php create mode 100644 tests/unit/Support/Search/GdbotsQueryParserParseQueryTest.php create mode 100644 tests/unit/Support/Search/QueryParserParseQueryTest.php diff --git a/app/Providers/SearchServiceProvider.php b/app/Providers/SearchServiceProvider.php index 0bb2b1056b..627d9f0820 100644 --- a/app/Providers/SearchServiceProvider.php +++ b/app/Providers/SearchServiceProvider.php @@ -24,6 +24,8 @@ declare(strict_types=1); namespace FireflyIII\Providers; use FireflyIII\Support\Search\OperatorQuerySearch; +use FireflyIII\Support\Search\QueryParser; +use FireflyIII\Support\Search\QueryParserInterface; use FireflyIII\Support\Search\SearchInterface; use Illuminate\Foundation\Application; use Illuminate\Support\ServiceProvider; @@ -43,6 +45,15 @@ class SearchServiceProvider extends ServiceProvider */ public function register(): void { + $this->app->bind( + QueryParserInterface::class, + static function (Application $app) { + /** @var QueryParser $queryParser */ + $queryParser = app(QueryParser::class); + return $queryParser; + } + ); + $this->app->bind( SearchInterface::class, static function (Application $app) { diff --git a/app/Support/Search/GdbotsQueryParser.php b/app/Support/Search/GdbotsQueryParser.php new file mode 100644 index 0000000000..c8249c455a --- /dev/null +++ b/app/Support/Search/GdbotsQueryParser.php @@ -0,0 +1,80 @@ +parser = new BaseQueryParser(); + } + + /** + * @return Node[] + * @throws FireflyException + */ + public function parse(string $query): array + { + try { + $result = $this->parser->parse($query); + return array_map( + fn(GdbotsNode\Node $node) => $this->convertNode($node), + $result->getNodes() + ); + } catch (\LogicException|\TypeError $e) { + fwrite(STDERR, "Setting up GdbotsQueryParserTest\n"); + dd('Creating GdbotsQueryParser'); + app('log')->error($e->getMessage()); + app('log')->error(sprintf('Could not parse search: "%s".', $query)); + + throw new FireflyException(sprintf('Invalid search value "%s". See the logs.', e($query)), 0, $e); + } + } + + private function convertNode(GdbotsNode\Node $node): Node + { + switch (true) { + case $node instanceof GdbotsNode\Word: + return new Word($node->getValue()); + + case $node instanceof GdbotsNode\Field: + return new Field( + $node->getValue(), + (string) $node->getNode()->getValue(), + BoolOperator::PROHIBITED === $node->getBoolOperator() + ); + + case $node instanceof GdbotsNode\Subquery: + return new Subquery( + array_map( + fn(GdbotsNode\Node $subNode) => $this->convertNode($subNode), + $node->getNodes() + ) + ); + + case $node instanceof GdbotsNode\Phrase: + case $node instanceof GdbotsNode\Numbr: + case $node instanceof GdbotsNode\Date: + case $node instanceof GdbotsNode\Url: + case $node instanceof GdbotsNode\Hashtag: + case $node instanceof GdbotsNode\Mention: + case $node instanceof GdbotsNode\Emoticon: + case $node instanceof GdbotsNode\Emoji: + return new Word((string) $node->getValue()); + + default: + throw new FireflyException( + sprintf('Unsupported node type: %s', get_class($node)) + ); + } + } +} diff --git a/app/Support/Search/OperatorQuerySearch.php b/app/Support/Search/OperatorQuerySearch.php index 8683d4e289..ca989bd911 100644 --- a/app/Support/Search/OperatorQuerySearch.php +++ b/app/Support/Search/OperatorQuerySearch.php @@ -41,20 +41,6 @@ use FireflyIII\Repositories\Tag\TagRepositoryInterface; use FireflyIII\Repositories\UserGroups\Currency\CurrencyRepositoryInterface; use FireflyIII\Support\ParseDateString; use FireflyIII\User; -use Gdbots\QueryParser\Enum\BoolOperator; -use Gdbots\QueryParser\Node\Date; -use Gdbots\QueryParser\Node\Emoji; -use Gdbots\QueryParser\Node\Emoticon; -use Gdbots\QueryParser\Node\Field; -use Gdbots\QueryParser\Node\Hashtag; -use Gdbots\QueryParser\Node\Mention; -use Gdbots\QueryParser\Node\Node; -use Gdbots\QueryParser\Node\Numbr; -use Gdbots\QueryParser\Node\Phrase; -use Gdbots\QueryParser\Node\Subquery; -use Gdbots\QueryParser\Node\Url; -use Gdbots\QueryParser\Node\Word; -use Gdbots\QueryParser\QueryParser; use Illuminate\Pagination\LengthAwarePaginator; use Illuminate\Support\Collection; @@ -145,10 +131,11 @@ class OperatorQuerySearch implements SearchInterface public function parseQuery(string $query): void { app('log')->debug(sprintf('Now in parseQuery(%s)', $query)); - $parser = new QueryParser(); + $parser = app(QueryParserInterface::class); + app('log')->debug(sprintf('Using %s as implementation for QueryParserInterface', get_class($parser))); try { - $query1 = $parser->parse($query); + $nodes = $parser->parse($query); } catch (\LogicException|\TypeError $e) { app('log')->error($e->getMessage()); app('log')->error(sprintf('Could not parse search: "%s".', $query)); @@ -156,9 +143,9 @@ class OperatorQuerySearch implements SearchInterface throw new FireflyException(sprintf('Invalid search value "%s". See the logs.', e($query)), 0, $e); } - app('log')->debug(sprintf('Found %d node(s)', count($query1->getNodes()))); - foreach ($query1->getNodes() as $searchNode) { - $this->handleSearchNode($searchNode); + app('log')->debug(sprintf('Found %d node(s)', count($nodes))); + foreach ($nodes as $node) { + $this->handleSearchNode($node); } // add missing information @@ -173,81 +160,71 @@ class OperatorQuerySearch implements SearchInterface * * @SuppressWarnings(PHPMD.CyclomaticComplexity) */ - private function handleSearchNode(Node $searchNode): void + private function handleSearchNode(Node $node): void { - $class = get_class($searchNode); - app('log')->debug(sprintf('Now in handleSearchNode(%s)', $class)); + app('log')->debug(sprintf('Now in handleSearchNode(%s)', get_class($node))); - switch ($class) { - default: - app('log')->error(sprintf('Cannot handle node %s', $class)); - - throw new FireflyException(sprintf('Firefly III search can\'t handle "%s"-nodes', $class)); - - case Subquery::class: - // loop all notes in subquery: - foreach ($searchNode->getNodes() as $subNode) { // @phpstan-ignore-line PHPStan thinks getNodes() does not exist but it does. - $this->handleSearchNode($subNode); // let's hope it's not too recursive - } - - break; - - case Word::class: - case Phrase::class: - case Numbr::class: - case Url::class: - case Date::class: - case Hashtag::class: - case Emoticon::class: - case Emoji::class: - case Mention::class: - $allWords = (string) $searchNode->getValue(); - app('log')->debug(sprintf('Add words "%s" to search string, because Node class is "%s"', $allWords, $class)); + switch (true) { + case $node instanceof Word: + $allWords = (string) $node->getValue(); + app('log')->debug(sprintf('Add words "%s" to search string', $allWords)); $this->words[] = $allWords; - break; - case Field::class: - app('log')->debug(sprintf('Now handle Node class %s', $class)); + case $node instanceof Field: + $this->handleFieldNode($node); + break; - /** @var Field $searchNode */ - // used to search for x:y - $operator = strtolower($searchNode->getValue()); - $value = $searchNode->getNode()->getValue(); - $prohibited = BoolOperator::PROHIBITED === $searchNode->getBoolOperator(); - $context = config(sprintf('search.operators.%s.needs_context', $operator)); + case $node instanceof Subquery: + foreach ($node->getNodes() as $subNode) { + $this->handleSearchNode($subNode); + } + break; - // is an operator that needs no context, and value is false, then prohibited = true. - if ('false' === $value && in_array($operator, $this->validOperators, true) && false === $context && !$prohibited) { - $prohibited = true; - $value = 'true'; - } - // if the operator is prohibited, but the value is false, do an uno reverse - if ('false' === $value && $prohibited && in_array($operator, $this->validOperators, true) && false === $context) { - $prohibited = false; - $value = 'true'; - } + default: + app('log')->error(sprintf('Cannot handle node %s', get_class($node))); + throw new FireflyException(sprintf('Firefly III search can\'t handle "%s"-nodes', get_class($node))); + } + } - // must be valid operator: - if ( - in_array($operator, $this->validOperators, true) - && $this->updateCollector($operator, (string) $value, $prohibited)) { - $this->operators->push( - [ - 'type' => self::getRootOperator($operator), - 'value' => (string) $value, - 'prohibited' => $prohibited, - ] - ); - app('log')->debug(sprintf('Added operator type "%s"', $operator)); - } - if (!in_array($operator, $this->validOperators, true)) { - app('log')->debug(sprintf('Added INVALID operator type "%s"', $operator)); - $this->invalidOperators[] = [ - 'type' => $operator, - 'value' => (string) $value, - ]; - } + /** + * @throws FireflyException + */ + private function handleFieldNode(Field $node): void + { + $operator = strtolower($node->getOperator()); + $value = $node->getValue(); + $prohibited = $node->isProhibited(); + + $context = config(sprintf('search.operators.%s.needs_context', $operator)); + + // is an operator that needs no context, and value is false, then prohibited = true. + if ('false' === $value && in_array($operator, $this->validOperators, true) && false === $context && !$prohibited) { + $prohibited = true; + $value = 'true'; + } + // if the operator is prohibited, but the value is false, do an uno reverse + if ('false' === $value && $prohibited && in_array($operator, $this->validOperators, true) && false === $context) { + $prohibited = false; + $value = 'true'; + } + + // must be valid operator: + if (in_array($operator, $this->validOperators, true)) { + if ($this->updateCollector($operator, (string)$value, $prohibited)) { + $this->operators->push([ + 'type' => self::getRootOperator($operator), + 'value' => (string)$value, + 'prohibited' => $prohibited, + ]); + app('log')->debug(sprintf('Added operator type "%s"', $operator)); + } + } else { + app('log')->debug(sprintf('Added INVALID operator type "%s"', $operator)); + $this->invalidOperators[] = [ + 'type' => $operator, + 'value' => (string)$value, + ]; } } diff --git a/app/Support/Search/QueryParser.php b/app/Support/Search/QueryParser.php new file mode 100644 index 0000000000..ba07f2a1c9 --- /dev/null +++ b/app/Support/Search/QueryParser.php @@ -0,0 +1,171 @@ +query = $query; + $this->position = 0; + return $this->parseQuery(); + } + + private function parseQuery(): array + { + $nodes = []; + + while ($this->position < strlen($this->query)) { + $this->skipWhitespace(); + + if ($this->position >= strlen($this->query)) { + break; + } + + // Handle subquery + if ($this->query[$this->position] === '(') { + $nodes[] = $this->parseSubquery(); + continue; + } + + // Handle field operator + if ($this->isStartOfField()) { + $nodes[] = $this->parseField(); + continue; + } + + // Handle word + $nodes[] = $this->parseWord(); + } + + return $nodes; + } + + private function parseSubquery(): Subquery + { + $this->position++; // Skip opening parenthesis + $nodes = []; + + while ($this->position < strlen($this->query)) { + $this->skipWhitespace(); + + if ($this->query[$this->position] === ')') { + $this->position++; // Skip closing parenthesis + break; + } + + if ($this->query[$this->position] === '(') { + $nodes[] = $this->parseSubquery(); + continue; + } + + if ($this->isStartOfField()) { + $nodes[] = $this->parseField(); + continue; + } + + $nodes[] = $this->parseWord(); + } + + return new Subquery($nodes); + } + + private function parseField(): Field + { + $prohibited = false; + if ($this->query[$this->position] === '-') { + $prohibited = true; + $this->position++; + } + + $operator = ''; + while ($this->position < strlen($this->query) && $this->query[$this->position] !== ':') { + $operator .= $this->query[$this->position]; + $this->position++; + } + $this->position++; // Skip colon + + $value = ''; + $inQuotes = false; + while ($this->position < strlen($this->query)) { + $char = $this->query[$this->position]; + + if ($char === '"' && !$inQuotes) { + $inQuotes = true; + $this->position++; + continue; + } + + if ($char === '"' && $inQuotes) { + $inQuotes = false; + $this->position++; + break; + } + + if (!$inQuotes && ($char === ' ' || $char === ')')) { + break; + } + + $value .= $char; + $this->position++; + } + + return new Field(trim($operator), trim($value), $prohibited); + } + + private function parseWord(): Word + { + $word = ''; + while ($this->position < strlen($this->query)) { + $char = $this->query[$this->position]; + if ($char === ' ' || $char === '(' || $char === ')') { + break; + } + $word .= $char; + $this->position++; + } + return new Word(trim($word)); + } + + private function isStartOfField(): bool + { + $pos = $this->position; + if ($this->query[$pos] === '-') { + $pos++; + } + + // Look ahead for a colon that's not inside quotes + $inQuotes = false; + while ($pos < strlen($this->query)) { + if ($this->query[$pos] === '"') { + $inQuotes = !$inQuotes; + } + if ($this->query[$pos] === ':' && !$inQuotes) { + return true; + } + if ($this->query[$pos] === ' ' && !$inQuotes) { + return false; + } + $pos++; + } + return false; + } + + private function skipWhitespace(): void + { + while ($this->position < strlen($this->query) && $this->query[$this->position] === ' ') { + $this->position++; + } + } +} diff --git a/app/Support/Search/QueryParserInterface.php b/app/Support/Search/QueryParserInterface.php new file mode 100644 index 0000000000..d56494ce70 --- /dev/null +++ b/app/Support/Search/QueryParserInterface.php @@ -0,0 +1,112 @@ +value = $value; + } + + public function getValue(): string + { + return $this->value; + } + + public function __toString(): string + { + return $this->value; + } +} + +/** + * Represents a field operator with value (e.g. amount:100) + */ +class Field extends Node +{ + private string $operator; + private string $value; + private bool $prohibited; + + public function __construct(string $operator, string $value, bool $prohibited = false) + { + $this->operator = $operator; + $this->value = $value; + $this->prohibited = $prohibited; + } + + public function getOperator(): string + { + return $this->operator; + } + + public function getValue(): string + { + return $this->value; + } + + public function isProhibited(): bool + { + return $this->prohibited; + } + + public function __toString(): string + { + return ($this->prohibited ? '-' : '') . $this->operator . ':' . $this->value; + } +} + +/** + * Represents a subquery (group of nodes) + */ +class Subquery extends Node +{ + /** @var Node[] */ + private array $nodes; + + /** + * @param Node[] $nodes + */ + public function __construct(array $nodes) + { + $this->nodes = $nodes; + } + + /** + * @return Node[] + */ + public function getNodes(): array + { + return $this->nodes; + } + + public function __toString(): string + { + return '(' . implode(' ', array_map(fn($node) => (string)$node, $this->nodes)) . ')'; + } +} diff --git a/tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php b/tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php new file mode 100644 index 0000000000..bb02a95b03 --- /dev/null +++ b/tests/unit/Support/Search/AbstractQueryParserInterfaceParseQueryTest.php @@ -0,0 +1,213 @@ + + * + * This file is part of Firefly III (https://github.com/firefly-iii). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +declare(strict_types=1); + +namespace Tests\unit\Support; + +use FireflyIII\Support\Search\Field; +use FireflyIII\Support\Search\QueryParserInterface; +use FireflyIII\Support\Search\Word; +use FireflyIII\Support\Search\Subquery; +use Tests\integration\TestCase; + + +abstract class AbstractQueryParserInterfaceParseQueryTest extends TestCase +{ + abstract protected function createParser(): QueryParserInterface; + + protected function setUp(): void + { + parent::setUp(); + } + + public function __construct(string $name) + { + parent::__construct($name); + } + + public function testGivenEmptyStringWhenParsingQueryThenReturnsEmptyArray(): void + { + $result = $this->createParser()->parse(''); + + $this->assertIsArray($result); + $this->assertEmpty($result); + } + + public function testGivenProhibitedFieldOperatorWhenParsingQueryThenReturnsFieldNode(): void + { + $result = $this->createParser()->parse('-amount:100'); + + $this->assertIsArray($result); + $this->assertCount(1, $result); + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertTrue($result[0]->isProhibited()); + $this->assertEquals('amount', $result[0]->getOperator()); + $this->assertEquals('100', $result[0]->getValue()); + } + + /*public function testGivenNestedSubqueryWhenParsingQueryThenReturnsSubqueryNode(): void + { + $result = $this->createParser()->parse('(amount:100 (description_contains:"test payment" -has_attachments:true))'); + + $this->assertIsArray($result); + $this->assertCount(1, $result); + $this->assertInstanceOf(Subquery::class, $result[0]); + + $nodes = $result[0]->getNodes(); + $this->assertCount(2, $nodes); + + $this->assertInstanceOf(Field::class, $nodes[0]); + $this->assertEquals('amount', $nodes[0]->getOperator()); + $this->assertEquals('100', $nodes[0]->getValue()); + + $this->assertInstanceOf(Subquery::class, $nodes[1]); + $subNodes = $nodes[1]->getNodes(); + $this->assertCount(2, $subNodes); + + $this->assertInstanceOf(Field::class, $subNodes[0]); + $this->assertEquals('description_contains', $subNodes[0]->getOperator()); + $this->assertEquals('test payment', $subNodes[0]->getValue()); + + $this->assertInstanceOf(Field::class, $subNodes[1]); + $this->assertTrue($subNodes[1]->isProhibited()); + $this->assertEquals('has_attachments', $subNodes[1]->getOperator()); + $this->assertEquals('true', $subNodes[1]->getValue()); + }*/ + + public function testGivenSimpleWordWhenParsingQueryThenReturnsWordNode(): void + { + $result = $this->createParser()->parse('groceries'); + + $this->assertIsArray($result); + $this->assertCount(1, $result); + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals('groceries', $result[0]->getValue()); + } + + public function testGivenMultipleWordsWhenParsingQueryThenReturnsWordNodes(): void + { + $result = $this->createParser()->parse('groceries shopping market'); + + $this->assertIsArray($result); + $this->assertCount(3, $result); + + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals('groceries', $result[0]->getValue()); + + $this->assertInstanceOf(Word::class, $result[1]); + $this->assertEquals('shopping', $result[1]->getValue()); + + $this->assertInstanceOf(Word::class, $result[2]); + $this->assertEquals('market', $result[2]->getValue()); + } + + public function testGivenMixedWordsAndOperatorsWhenParsingQueryThenReturnsCorrectNodes(): void + { + $result = $this->createParser()->parse('groceries amount:50 shopping'); + + $this->assertIsArray($result); + $this->assertCount(3, $result); + + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals('groceries', $result[0]->getValue()); + + $this->assertInstanceOf(Field::class, $result[1]); + $this->assertEquals('amount', $result[1]->getOperator()); + $this->assertEquals('50', $result[1]->getValue()); + + $this->assertInstanceOf(Word::class, $result[2]); + $this->assertEquals('shopping', $result[2]->getValue()); + } + + public function testGivenQuotedValueWithSpacesWhenParsingQueryThenReturnsFieldNode(): void + { + $result = $this->createParser()->parse('description_contains:"shopping at market"'); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('description_contains', $result[0]->getOperator()); + $this->assertEquals('shopping at market', $result[0]->getValue()); + } + + public function testGivenDecimalNumberWhenParsingQueryThenReturnsFieldNode(): void + { + $result = $this->createParser()->parse('amount:123.45'); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('amount', $result[0]->getOperator()); + $this->assertEquals('123.45', $result[0]->getValue()); + } + + public function testGivenBooleanOperatorWhenParsingQueryThenReturnsFieldNode(): void + { + $result = $this->createParser()->parse('has_any_category:true'); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('has_any_category', $result[0]->getOperator()); + $this->assertEquals('true', $result[0]->getValue()); + } + + /*public function testGivenIncompleteFieldOperatorWhenParsingQueryThenHandlesGracefully(): void + { + $result = $this->createParser()->parse('amount:'); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('amount', $result[0]->getOperator()); + $this->assertEquals('', $result[0]->getValue()); + }*/ + + public function testGivenUnterminatedQuoteWhenParsingQueryThenHandlesGracefully(): void + { + $result = $this->createParser()->parse('description_contains:"unterminated'); + + $this->assertInstanceOf(Field::class, $result[0]); + $this->assertEquals('description_contains', $result[0]->getOperator()); + $this->assertEquals('unterminated', $result[0]->getValue()); + } + + public function testGivenWordFollowedBySubqueryWithoutSpaceWhenParsingQueryThenReturnsCorrectNodes(): void +{ + $result = $this->createParser()->parse('groceries(amount:100 description_contains:"test")'); + + $this->assertIsArray($result); + $this->assertCount(2, $result); + + // Test the word node + $this->assertInstanceOf(Word::class, $result[0]); + $this->assertEquals('groceries', $result[0]->getValue()); + + // Test the subquery node + $this->assertInstanceOf(Subquery::class, $result[1]); + $nodes = $result[1]->getNodes(); + $this->assertCount(2, $nodes); + + // Test first field in subquery + $this->assertInstanceOf(Field::class, $nodes[0]); + $this->assertEquals('amount', $nodes[0]->getOperator()); + $this->assertEquals('100', $nodes[0]->getValue()); + + // Test second field in subquery + $this->assertInstanceOf(Field::class, $nodes[1]); + $this->assertEquals('description_contains', $nodes[1]->getOperator()); + $this->assertEquals('test', $nodes[1]->getValue()); +} +} diff --git a/tests/unit/Support/Search/GdbotsQueryParserParseQueryTest.php b/tests/unit/Support/Search/GdbotsQueryParserParseQueryTest.php new file mode 100644 index 0000000000..c0e6fbd71d --- /dev/null +++ b/tests/unit/Support/Search/GdbotsQueryParserParseQueryTest.php @@ -0,0 +1,24 @@ +