From 9df09b8118ecbc065c690f0a31ed7c1f1f18b491 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 00:57:09 +0000 Subject: [PATCH 01/44] Feat: Usage library - Database adapter - ClickHouse adapter --- .github/workflows/tests.yml | 36 + CODE_OF_CONDUCT.md | 76 + CONTRIBUTING.md | 101 ++ Dockerfile | 26 + LICENSE.md | 20 + README.md | 261 ++++ composer.json | 17 +- composer.lock | 1806 +++++++++++++++++++++++- docker-compose.yml | 37 + phpunit.xml | 20 + src/Usage/Adapter.php | 90 ++ src/Usage/Adapter/ClickHouse.php | 593 ++++++++ src/Usage/Adapter/Database.php | 277 ++++ src/Usage/Usage.php | 247 ++++ tests/Usage/Adapter/ClickHouseTest.php | 40 + tests/Usage/Adapter/DatabaseTest.php | 42 + tests/Usage/UsageBase.php | 167 +++ 17 files changed, 3852 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 phpunit.xml create mode 100644 src/Usage/Adapter.php create mode 100644 src/Usage/Adapter/ClickHouse.php create mode 100644 src/Usage/Adapter/Database.php create mode 100644 src/Usage/Usage.php create mode 100644 tests/Usage/Adapter/ClickHouseTest.php create mode 100644 tests/Usage/Adapter/DatabaseTest.php create mode 100644 tests/Usage/UsageBase.php diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..4bd84db --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,36 @@ +name: "Tests" + +on: [pull_request, push] + +jobs: + tests: + name: Unit Tests + runs-on: ubuntu-latest + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-versions }} + extensions: pdo, pdo_mysql, mysqli + coverage: xdebug + + - name: Install dependencies + run: composer install + + - name: Start MariaDB + run: | + docker compose up -d mariadb + sleep 10 + + - name: Run PHPUnit + run: docker compose run --rm usage vendor/bin/phpunit --configuration phpunit.xml tests diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..5dba194 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity, expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at team@appwrite.io. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9d40d28 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,101 @@ +# Contributing + +We would ❤️ for you to contribute to Utopia-php and help make it better! We want contributing to Utopia-php to be fun, enjoyable, and educational for anyone and everyone. All contributions are welcome, including issues, new docs as well as updates and tweaks, blog posts, workshops, and more. + +## How to Start? + +If you are worried or don't know where to start, check out our next section explaining what kind of help we could use and where can you get involved. You can reach out with questions to [Eldad Fux (@eldadfux)](https://twitter.com/eldadfux) or anyone from the [Appwrite team on Discord](https://discord.gg/GSeTUeA). You can also submit an issue, and a maintainer can guide you! + +## Code of Conduct + +Help us keep Utopia-php open and inclusive. Please read and follow our [Code of Conduct](https://github.com/appwrite/appwrite/blob/master/CODE_OF_CONDUCT.md). + +## Submit a Pull Request 🚀 + +Branch naming convention is as following + +`TYPE-ISSUE_ID-DESCRIPTION` + +example: + +``` +doc-548-submit-a-pull-request-section-to-contribution-guide +``` + +When `TYPE` can be: + +- **feat** - is a new feature +- **doc** - documentation only changes +- **cicd** - changes related to CI/CD system +- **fix** - a bug fix +- **refactor** - code change that neither fixes a bug nor adds a feature + +**All PRs must include a commit message with the changes description!** + +For the initial start, fork the project and use git clone command to download the repository to your computer. A standard procedure for working on an issue would be to: + +1. `git pull`, before creating a new branch, pull the changes from upstream. Your master needs to be up to date. + +``` +$ git pull +``` + +2. Create new branch from `master` like: `doc-548-submit-a-pull-request-section-to-contribution-guide`
+ +``` +$ git checkout -b [name_of_your_new_branch] +``` + +3. Work - commit - repeat ( be sure to be in your branch ) + +4. Push changes to GitHub + +``` +$ git push origin [name_of_your_new_branch] +``` + +5. Submit your changes for review + If you go to your repository on GitHub, you'll see a `Compare & pull request` button. Click on that button. +6. Start a Pull Request + Now submit the pull request and click on `Create pull request`. +7. Get a code review approval/reject +8. After approval, merge your PR +9. GitHub will automatically delete the branch after the merge is done. (they can still be restored). + +## Introducing New Features + +We would 💖 you to contribute to Utopia-php, but we would also like to make sure Utopia-php is as great as possible and loyal to its vision and mission statement 🙏. + +For us to find the right balance, please open an issue explaining your ideas before introducing a new pull request. + +This will allow the Utopia-php community to have sufficient discussion about the new feature value and how it fits in the product roadmap and vision. + +This is also important for the Utopia-php lead developers to be able to give technical input and different emphasis regarding the feature design and architecture. Some bigger features might need to go through our [RFC process](https://github.com/appwrite/rfc). + +## Other Ways to Help + +Pull requests are great, but there are many other areas where you can help Utopia-php. + +### Blogging & Speaking + +Blogging, speaking about, or creating tutorials about one of Utopia-php's many features is great way to contribute and help our project grow. + +### Presenting at Meetups + +Presenting at meetups and conferences about your Utopia-php projects. Your unique challenges and successes in building things with Utopia-php can provide great speaking material. We'd love to review your talk abstract/CFP, so get in touch with us if you'd like some help! + +### Sending Feedbacks & Reporting Bugs + +Sending feedback is a great way for us to understand your different use cases of Utopia-php better. If you had any issues, bugs, or want to share about your experience, feel free to do so on our GitHub issues page or at our [Discord channel](https://discord.gg/GSeTUeA). + +### Submitting New Ideas + +If you think Utopia-php could use a new feature, please open an issue on our GitHub repository, stating as much information as you can think about your new idea and it's implications. We would also use this issue to gather more information, get more feedback from the community, and have a proper discussion about the new feature. + +### Improving Documentation + +Submitting documentation updates, enhancements, designs, or bug fixes. Spelling or grammar fixes will be very much appreciated. + +### Helping Someone + +Searching for Utopia-php, GitHub or StackOverflow and helping someone else who needs help. You can also help by teaching others how to contribute to Utopia-php's repo! diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2d6a28f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM composer:2.0 as step0 + +WORKDIR /src/ + +COPY composer.lock /src/ +COPY composer.json /src/ + +RUN composer install --ignore-platform-reqs --optimize-autoloader \ + --no-plugins --no-scripts --prefer-dist + +FROM php:8.3.3-cli-alpine3.19 as final + +LABEL maintainer="team@appwrite.io" + +RUN docker-php-ext-install pdo_mysql + +WORKDIR /code + +COPY --from=step0 /src/vendor /code/vendor + +# Add Source Code +COPY ./tests /code/tests +COPY ./src /code/src +COPY ./phpunit.xml /code/phpunit.xml + +CMD [ "tail", "-f", "/dev/null" ] diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..a2b469d --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2024 Appwrite Team + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..668fcce --- /dev/null +++ b/README.md @@ -0,0 +1,261 @@ +# Utopia Usage + +[![Build Status](https://travis-ci.org/utopia-php/usage.svg?branch=master)](https://travis-ci.com/utopia-php/usage) +![Total Downloads](https://img.shields.io/packagist/dt/utopia-php/usage.svg) +[![Discord](https://img.shields.io/discord/564160730845151244)](https://appwrite.io/discord) + +Utopia framework usage library is a simple and lite library for managing application usage statistics. This library is aiming to be as simple and easy to learn and use. This library is maintained by the [Appwrite team](https://appwrite.io). + +Although this library is part of the [Utopia Framework](https://github.com/utopia-php/framework) project it is dependency free and can be used as standalone with any other PHP project or framework. + +## Features + +- **Pluggable Adapters**: Use different storage backends (Database, ClickHouse) +- **Database Adapter**: Store metrics in any SQL database via utopia-php/database +- **ClickHouse Adapter**: High-performance analytics storage for massive scale +- **Flexible Periods**: Hourly (1h), Daily (1d), and Infinite (inf) periods +- **Batch Operations**: Log multiple metrics efficiently +- **Rich Queries**: Filter, limit, offset, and aggregate metrics +- **Tag Support**: Add custom tags for multi-dimensional analytics + +## Getting Started + +Install using composer: +```bash +composer require utopia-php/usage +``` + +### Using Database Adapter + +The Database adapter stores metrics using utopia-php/database, supporting MySQL, MariaDB, PostgreSQL, and more. + +```php + 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true, +]); + +$cache = new Cache(new NoCache()); +$database = new Database(new MySQL($pdo), $cache); +$database->setNamespace('namespace'); + +// Create Usage instance with Database adapter +$usage = Usage::withDatabase($database); +$usage->setup(); +``` + +### Using ClickHouse Adapter + +The ClickHouse adapter provides high-performance analytics storage for massive scale metrics. + +```php +setup(); +``` + +### Using Custom Adapter + +You can create custom adapters by extending the `Utopia\Usage\Adapter` abstract class. + +```php +setup(); +``` +**Log Usage** + +A simple example for logging a usage metric. + +```php +$metric = 'requests'; +$value = 100; +$period = '1h'; // Supported periods: '1h', '1d', 'inf' +$tags = ['region' => 'us-east', 'method' => 'GET']; + +$usage->log($metric, $value, $period, $tags); +``` + +**Log Batch Usage** + +Log multiple metrics in batch for better performance. + +```php +$metrics = [ + [ + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'tags' => ['region' => 'us-east'], + ], + [ + 'metric' => 'bandwidth', + 'value' => 50000, + 'period' => '1h', + 'tags' => ['region' => 'us-east'], + ], +]; + +$usage->logBatch($metrics); +``` + +**Get Usage By Period** + +Fetch all usage metrics by period. + +```php +$metrics = $usage->getByPeriod('requests', '1h'); +// Returns an array of all usage metrics for specific period +``` + +**Get Usage Between Dates** + +Fetch all usage metrics between two dates. + +```php +$start = '2024-01-01 00:00:00'; +$end = '2024-01-31 23:59:59'; + +$metrics = $usage->getBetweenDates('requests', $start, $end); +// Returns an array of usage metrics within the date range +``` + +**Count and Sum Usage** + +Get counts and sums of usage metrics. + +```php +// Count total records +$count = $usage->countByPeriod('requests', '1h'); + +// Sum all values +$sum = $usage->sumByPeriod('requests', '1h'); +``` + +**Purge Old Usage** + +Delete old usage metrics. + +```php +use Utopia\Database\DateTime; + +$datetime = DateTime::addSeconds(new \DateTime(), -86400); // Delete metrics older than 24 hours +$usage->purge($datetime); +``` + +## Periods + +The library supports three types of periods: + +- `1h` - Hourly periods (`Y-m-d H:00`) +- `1d` - Daily periods (`Y-m-d 00:00`) +- `inf` - Infinite/lifetime periods (`0000-00-00 00:00`) + +## Adapters + +### Database Adapter + +The Database adapter uses [utopia-php/database](https://github.com/utopia-php/database) to store metrics in SQL databases. + +**Features**: +- Works with MySQL, MariaDB, PostgreSQL, SQLite +- Full query support (filters, sorting, pagination) +- ACID compliance for data consistency +- Easy migration from existing databases + +**Example**: +```php +$usage = Usage::withDatabase($database); +``` + +### ClickHouse Adapter + +The ClickHouse adapter uses the HTTP interface to store metrics in ClickHouse for high-performance analytics. + +**Features**: +- Optimized for analytical queries +- Handles millions of metrics per second +- Automatic partitioning by month +- Efficient compression and storage +- Bloom filter indexes for fast lookups + +**Example**: +```php +$usage = Usage::withClickHouse( + host: 'clickhouse.example.com', + username: 'metrics_user', + password: 'secure_password', + port: 8123, + secure: true // Use HTTPS +); + +// Configure database and table (optional) +$adapter = $usage->getAdapter(); +$adapter->setDatabase('analytics'); +$adapter->setTable('metrics'); + +$usage->setup(); +``` + +### Creating Custom Adapters + +Extend the `Utopia\Usage\Adapter` abstract class and implement these methods: + +- `getName(): string` - Return adapter name +- `setup(): void` - Initialize storage structure +- `log(string $metric, int $value, string $period, array $tags): bool` - Log single metric +- `logBatch(array $metrics): bool` - Log multiple metrics +- `getByPeriod(string $metric, string $period, array $queries): array` - Get metrics by period +- `getBetweenDates(string $metric, string $startDate, string $endDate, array $queries): array` - Get metrics in date range +- `countByPeriod(string $metric, string $period, array $queries): int` - Count metrics +- `sumByPeriod(string $metric, string $period, array $queries): int` - Sum metric values +- `purge(string $datetime): bool` - Delete old metrics + +## System Requirements + +Utopia Framework requires PHP 8.0 or later. We recommend using the latest PHP version whenever possible. + +## Copyright and license + +The MIT License (MIT) [http://www.opensource.org/licenses/mit-license.php](http://www.opensource.org/licenses/mit-license.php) diff --git a/composer.json b/composer.json index 827e005..ad66768 100644 --- a/composer.json +++ b/composer.json @@ -11,13 +11,28 @@ ], "minimum-stability": "stable", "require": { + "php": ">=8.0", "utopia-php/fetch": "^0.4.2", "utopia-php/database": "^4.3" }, + "require-dev": { + "phpunit/phpunit": "^9.5", + "utopia-php/cache": "^0.13.0" + }, + "autoload": { + "psr-4": { + "Utopia\\Usage\\": "src/Usage" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests" + } + }, "config": { "allow-plugins": { "php-http/discovery": false, "tbachert/spi": false } } -} +} \ No newline at end of file diff --git a/composer.lock b/composer.lock index 4036504..de7d15b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fae1878621d4585a46e2dc9e5ce78d5f", + "content-hash": "8a5a1e92e201028a5f27180d0f7e2802", "packages": [ { "name": "brick/math", @@ -2476,13 +2476,1813 @@ "time": "2025-11-18T11:05:46+00:00" } ], - "packages-dev": [], + "packages-dev": [ + { + "name": "doctrine/instantiator", + "version": "2.0.0", + "source": { + "type": "git", + "url": "https://github.com/doctrine/instantiator.git", + "reference": "c6222283fa3f4ac679f8b9ced9a4e23f163e80d0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/c6222283fa3f4ac679f8b9ced9a4e23f163e80d0", + "reference": "c6222283fa3f4ac679f8b9ced9a4e23f163e80d0", + "shasum": "" + }, + "require": { + "php": "^8.1" + }, + "require-dev": { + "doctrine/coding-standard": "^11", + "ext-pdo": "*", + "ext-phar": "*", + "phpbench/phpbench": "^1.2", + "phpstan/phpstan": "^1.9.4", + "phpstan/phpstan-phpunit": "^1.3", + "phpunit/phpunit": "^9.5.27", + "vimeo/psalm": "^5.4" + }, + "type": "library", + "autoload": { + "psr-4": { + "Doctrine\\Instantiator\\": "src/Doctrine/Instantiator/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Marco Pivetta", + "email": "ocramius@gmail.com", + "homepage": "https://ocramius.github.io/" + } + ], + "description": "A small, lightweight utility to instantiate objects in PHP without invoking their constructors", + "homepage": "https://www.doctrine-project.org/projects/instantiator.html", + "keywords": [ + "constructor", + "instantiate" + ], + "support": { + "issues": "https://github.com/doctrine/instantiator/issues", + "source": "https://github.com/doctrine/instantiator/tree/2.0.0" + }, + "funding": [ + { + "url": "https://www.doctrine-project.org/sponsorship.html", + "type": "custom" + }, + { + "url": "https://www.patreon.com/phpdoctrine", + "type": "patreon" + }, + { + "url": "https://tidelift.com/funding/github/packagist/doctrine%2Finstantiator", + "type": "tidelift" + } + ], + "time": "2022-12-30T00:23:10+00:00" + }, + { + "name": "myclabs/deep-copy", + "version": "1.13.4", + "source": { + "type": "git", + "url": "https://github.com/myclabs/DeepCopy.git", + "reference": "07d290f0c47959fd5eed98c95ee5602db07e0b6a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/07d290f0c47959fd5eed98c95ee5602db07e0b6a", + "reference": "07d290f0c47959fd5eed98c95ee5602db07e0b6a", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0" + }, + "conflict": { + "doctrine/collections": "<1.6.8", + "doctrine/common": "<2.13.3 || >=3 <3.2.2" + }, + "require-dev": { + "doctrine/collections": "^1.6.8", + "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", + "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" + }, + "type": "library", + "autoload": { + "files": [ + "src/DeepCopy/deep_copy.php" + ], + "psr-4": { + "DeepCopy\\": "src/DeepCopy/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Create deep copies (clones) of your objects", + "keywords": [ + "clone", + "copy", + "duplicate", + "object", + "object graph" + ], + "support": { + "issues": "https://github.com/myclabs/DeepCopy/issues", + "source": "https://github.com/myclabs/DeepCopy/tree/1.13.4" + }, + "funding": [ + { + "url": "https://tidelift.com/funding/github/packagist/myclabs/deep-copy", + "type": "tidelift" + } + ], + "time": "2025-08-01T08:46:24+00:00" + }, + { + "name": "nikic/php-parser", + "version": "v5.7.0", + "source": { + "type": "git", + "url": "https://github.com/nikic/PHP-Parser.git", + "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/dca41cd15c2ac9d055ad70dbfd011130757d1f82", + "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82", + "shasum": "" + }, + "require": { + "ext-ctype": "*", + "ext-json": "*", + "ext-tokenizer": "*", + "php": ">=7.4" + }, + "require-dev": { + "ircmaxell/php-yacc": "^0.0.7", + "phpunit/phpunit": "^9.0" + }, + "bin": [ + "bin/php-parse" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.x-dev" + } + }, + "autoload": { + "psr-4": { + "PhpParser\\": "lib/PhpParser" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Nikita Popov" + } + ], + "description": "A PHP parser written in PHP", + "keywords": [ + "parser", + "php" + ], + "support": { + "issues": "https://github.com/nikic/PHP-Parser/issues", + "source": "https://github.com/nikic/PHP-Parser/tree/v5.7.0" + }, + "time": "2025-12-06T11:56:16+00:00" + }, + { + "name": "phar-io/manifest", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/phar-io/manifest.git", + "reference": "54750ef60c58e43759730615a392c31c80e23176" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/54750ef60c58e43759730615a392c31c80e23176", + "reference": "54750ef60c58e43759730615a392c31c80e23176", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-libxml": "*", + "ext-phar": "*", + "ext-xmlwriter": "*", + "phar-io/version": "^3.0.1", + "php": "^7.2 || ^8.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + }, + { + "name": "Sebastian Heuer", + "email": "sebastian@phpeople.de", + "role": "Developer" + }, + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "Developer" + } + ], + "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", + "support": { + "issues": "https://github.com/phar-io/manifest/issues", + "source": "https://github.com/phar-io/manifest/tree/2.0.4" + }, + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2024-03-03T12:33:53+00:00" + }, + { + "name": "phar-io/version", + "version": "3.2.1", + "source": { + "type": "git", + "url": "https://github.com/phar-io/version.git", + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phar-io/version/zipball/4f7fd7836c6f332bb2933569e566a0d6c4cbed74", + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + }, + { + "name": "Sebastian Heuer", + "email": "sebastian@phpeople.de", + "role": "Developer" + }, + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "Developer" + } + ], + "description": "Library for handling version information and constraints", + "support": { + "issues": "https://github.com/phar-io/version/issues", + "source": "https://github.com/phar-io/version/tree/3.2.1" + }, + "time": "2022-02-21T01:04:05+00:00" + }, + { + "name": "phpunit/php-code-coverage", + "version": "9.2.32", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-code-coverage.git", + "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/85402a822d1ecf1db1096959413d35e1c37cf1a5", + "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-libxml": "*", + "ext-xmlwriter": "*", + "nikic/php-parser": "^4.19.1 || ^5.1.0", + "php": ">=7.3", + "phpunit/php-file-iterator": "^3.0.6", + "phpunit/php-text-template": "^2.0.4", + "sebastian/code-unit-reverse-lookup": "^2.0.3", + "sebastian/complexity": "^2.0.3", + "sebastian/environment": "^5.1.5", + "sebastian/lines-of-code": "^1.0.4", + "sebastian/version": "^3.0.2", + "theseer/tokenizer": "^1.2.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.6" + }, + "suggest": { + "ext-pcov": "PHP extension that provides line coverage", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "9.2.x-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", + "homepage": "https://github.com/sebastianbergmann/php-code-coverage", + "keywords": [ + "coverage", + "testing", + "xunit" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", + "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.32" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-08-22T04:23:01+00:00" + }, + { + "name": "phpunit/php-file-iterator", + "version": "3.0.6", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-file-iterator.git", + "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", + "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "FilterIterator implementation that filters files based on a list of suffixes.", + "homepage": "https://github.com/sebastianbergmann/php-file-iterator/", + "keywords": [ + "filesystem", + "iterator" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-file-iterator/issues", + "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/3.0.6" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2021-12-02T12:48:52+00:00" + }, + { + "name": "phpunit/php-invoker", + "version": "3.1.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-invoker.git", + "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/5a10147d0aaf65b58940a0b72f71c9ac0423cc67", + "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "ext-pcntl": "*", + "phpunit/phpunit": "^9.3" + }, + "suggest": { + "ext-pcntl": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.1-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Invoke callables with a timeout", + "homepage": "https://github.com/sebastianbergmann/php-invoker/", + "keywords": [ + "process" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-invoker/issues", + "source": "https://github.com/sebastianbergmann/php-invoker/tree/3.1.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-09-28T05:58:55+00:00" + }, + { + "name": "phpunit/php-text-template", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-text-template.git", + "reference": "5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28", + "reference": "5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Simple template engine.", + "homepage": "https://github.com/sebastianbergmann/php-text-template/", + "keywords": [ + "template" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-text-template/issues", + "source": "https://github.com/sebastianbergmann/php-text-template/tree/2.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T05:33:50+00:00" + }, + { + "name": "phpunit/php-timer", + "version": "5.0.3", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-timer.git", + "reference": "5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2", + "reference": "5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Utility class for timing", + "homepage": "https://github.com/sebastianbergmann/php-timer/", + "keywords": [ + "timer" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-timer/issues", + "source": "https://github.com/sebastianbergmann/php-timer/tree/5.0.3" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:16:10+00:00" + }, + { + "name": "phpunit/phpunit", + "version": "9.6.31", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/phpunit.git", + "reference": "945d0b7f346a084ce5549e95289962972c4272e5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/945d0b7f346a084ce5549e95289962972c4272e5", + "reference": "945d0b7f346a084ce5549e95289962972c4272e5", + "shasum": "" + }, + "require": { + "doctrine/instantiator": "^1.5.0 || ^2", + "ext-dom": "*", + "ext-json": "*", + "ext-libxml": "*", + "ext-mbstring": "*", + "ext-xml": "*", + "ext-xmlwriter": "*", + "myclabs/deep-copy": "^1.13.4", + "phar-io/manifest": "^2.0.4", + "phar-io/version": "^3.2.1", + "php": ">=7.3", + "phpunit/php-code-coverage": "^9.2.32", + "phpunit/php-file-iterator": "^3.0.6", + "phpunit/php-invoker": "^3.1.1", + "phpunit/php-text-template": "^2.0.4", + "phpunit/php-timer": "^5.0.3", + "sebastian/cli-parser": "^1.0.2", + "sebastian/code-unit": "^1.0.8", + "sebastian/comparator": "^4.0.9", + "sebastian/diff": "^4.0.6", + "sebastian/environment": "^5.1.5", + "sebastian/exporter": "^4.0.8", + "sebastian/global-state": "^5.0.8", + "sebastian/object-enumerator": "^4.0.4", + "sebastian/resource-operations": "^3.0.4", + "sebastian/type": "^3.2.1", + "sebastian/version": "^3.0.2" + }, + "suggest": { + "ext-soap": "To be able to generate mocks based on WSDL files", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" + }, + "bin": [ + "phpunit" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.6-dev" + } + }, + "autoload": { + "files": [ + "src/Framework/Assert/Functions.php" + ], + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "The PHP Unit Testing framework.", + "homepage": "https://phpunit.de/", + "keywords": [ + "phpunit", + "testing", + "xunit" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/phpunit/issues", + "security": "https://github.com/sebastianbergmann/phpunit/security/policy", + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.31" + }, + "funding": [ + { + "url": "https://phpunit.de/sponsors.html", + "type": "custom" + }, + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/phpunit", + "type": "tidelift" + } + ], + "time": "2025-12-06T07:45:52+00:00" + }, + { + "name": "sebastian/cli-parser", + "version": "1.0.2", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/cli-parser.git", + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/2b56bea83a09de3ac06bb18b92f068e60cc6f50b", + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for parsing CLI options", + "homepage": "https://github.com/sebastianbergmann/cli-parser", + "support": { + "issues": "https://github.com/sebastianbergmann/cli-parser/issues", + "source": "https://github.com/sebastianbergmann/cli-parser/tree/1.0.2" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-02T06:27:43+00:00" + }, + { + "name": "sebastian/code-unit", + "version": "1.0.8", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/code-unit.git", + "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/1fc9f64c0927627ef78ba436c9b17d967e68e120", + "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Collection of value objects that represent the PHP code units", + "homepage": "https://github.com/sebastianbergmann/code-unit", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit/issues", + "source": "https://github.com/sebastianbergmann/code-unit/tree/1.0.8" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:08:54+00:00" + }, + { + "name": "sebastian/code-unit-reverse-lookup", + "version": "2.0.3", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", + "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", + "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Looks up which function or method a line of code belongs to", + "homepage": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/issues", + "source": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/tree/2.0.3" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-09-28T05:30:19+00:00" + }, + { + "name": "sebastian/comparator", + "version": "4.0.9", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/comparator.git", + "reference": "67a2df3a62639eab2cc5906065e9805d4fd5dfc5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/67a2df3a62639eab2cc5906065e9805d4fd5dfc5", + "reference": "67a2df3a62639eab2cc5906065e9805d4fd5dfc5", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/diff": "^4.0", + "sebastian/exporter": "^4.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Volker Dusch", + "email": "github@wallbash.com" + }, + { + "name": "Bernhard Schussek", + "email": "bschussek@2bepublished.at" + } + ], + "description": "Provides the functionality to compare PHP values for equality", + "homepage": "https://github.com/sebastianbergmann/comparator", + "keywords": [ + "comparator", + "compare", + "equality" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/comparator/issues", + "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.9" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/comparator", + "type": "tidelift" + } + ], + "time": "2025-08-10T06:51:50+00:00" + }, + { + "name": "sebastian/complexity", + "version": "2.0.3", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/complexity.git", + "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/25f207c40d62b8b7aa32f5ab026c53561964053a", + "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a", + "shasum": "" + }, + "require": { + "nikic/php-parser": "^4.18 || ^5.0", + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for calculating the complexity of PHP code units", + "homepage": "https://github.com/sebastianbergmann/complexity", + "support": { + "issues": "https://github.com/sebastianbergmann/complexity/issues", + "source": "https://github.com/sebastianbergmann/complexity/tree/2.0.3" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-12-22T06:19:30+00:00" + }, + { + "name": "sebastian/diff", + "version": "4.0.6", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/diff.git", + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/ba01945089c3a293b01ba9badc29ad55b106b0bc", + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "symfony/process": "^4.2 || ^5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Kore Nordmann", + "email": "mail@kore-nordmann.de" + } + ], + "description": "Diff implementation", + "homepage": "https://github.com/sebastianbergmann/diff", + "keywords": [ + "diff", + "udiff", + "unidiff", + "unified diff" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/diff/issues", + "source": "https://github.com/sebastianbergmann/diff/tree/4.0.6" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-02T06:30:58+00:00" + }, + { + "name": "sebastian/environment", + "version": "5.1.5", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/environment.git", + "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", + "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "suggest": { + "ext-posix": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.1-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Provides functionality to handle HHVM/PHP environments", + "homepage": "http://www.github.com/sebastianbergmann/environment", + "keywords": [ + "Xdebug", + "environment", + "hhvm" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/environment/issues", + "source": "https://github.com/sebastianbergmann/environment/tree/5.1.5" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:03:51+00:00" + }, + { + "name": "sebastian/exporter", + "version": "4.0.8", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/exporter.git", + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/14c6ba52f95a36c3d27c835d65efc7123c446e8c", + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/recursion-context": "^4.0" + }, + "require-dev": { + "ext-mbstring": "*", + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Volker Dusch", + "email": "github@wallbash.com" + }, + { + "name": "Adam Harvey", + "email": "aharvey@php.net" + }, + { + "name": "Bernhard Schussek", + "email": "bschussek@gmail.com" + } + ], + "description": "Provides the functionality to export PHP variables for visualization", + "homepage": "https://www.github.com/sebastianbergmann/exporter", + "keywords": [ + "export", + "exporter" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/exporter/issues", + "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.8" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", + "type": "tidelift" + } + ], + "time": "2025-09-24T06:03:27+00:00" + }, + { + "name": "sebastian/global-state", + "version": "5.0.8", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/global-state.git", + "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/b6781316bdcd28260904e7cc18ec983d0d2ef4f6", + "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/object-reflector": "^2.0", + "sebastian/recursion-context": "^4.0" + }, + "require-dev": { + "ext-dom": "*", + "phpunit/phpunit": "^9.3" + }, + "suggest": { + "ext-uopz": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Snapshotting of global state", + "homepage": "http://www.github.com/sebastianbergmann/global-state", + "keywords": [ + "global state" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/global-state/issues", + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.8" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/global-state", + "type": "tidelift" + } + ], + "time": "2025-08-10T07:10:35+00:00" + }, + { + "name": "sebastian/lines-of-code", + "version": "1.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/lines-of-code.git", + "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/e1e4a170560925c26d424b6a03aed157e7dcc5c5", + "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5", + "shasum": "" + }, + "require": { + "nikic/php-parser": "^4.18 || ^5.0", + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for counting the lines of code in PHP source code", + "homepage": "https://github.com/sebastianbergmann/lines-of-code", + "support": { + "issues": "https://github.com/sebastianbergmann/lines-of-code/issues", + "source": "https://github.com/sebastianbergmann/lines-of-code/tree/1.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-12-22T06:20:34+00:00" + }, + { + "name": "sebastian/object-enumerator", + "version": "4.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/object-enumerator.git", + "reference": "5c9eeac41b290a3712d88851518825ad78f45c71" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/5c9eeac41b290a3712d88851518825ad78f45c71", + "reference": "5c9eeac41b290a3712d88851518825ad78f45c71", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/object-reflector": "^2.0", + "sebastian/recursion-context": "^4.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Traverses array structures and object graphs to enumerate all referenced objects", + "homepage": "https://github.com/sebastianbergmann/object-enumerator/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-enumerator/issues", + "source": "https://github.com/sebastianbergmann/object-enumerator/tree/4.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:12:34+00:00" + }, + { + "name": "sebastian/object-reflector", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/object-reflector.git", + "reference": "b4f479ebdbf63ac605d183ece17d8d7fe49c15c7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/b4f479ebdbf63ac605d183ece17d8d7fe49c15c7", + "reference": "b4f479ebdbf63ac605d183ece17d8d7fe49c15c7", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Allows reflection of object attributes, including inherited and non-public ones", + "homepage": "https://github.com/sebastianbergmann/object-reflector/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-reflector/issues", + "source": "https://github.com/sebastianbergmann/object-reflector/tree/2.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:14:26+00:00" + }, + { + "name": "sebastian/recursion-context", + "version": "4.0.6", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/recursion-context.git", + "reference": "539c6691e0623af6dc6f9c20384c120f963465a0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/539c6691e0623af6dc6f9c20384c120f963465a0", + "reference": "539c6691e0623af6dc6f9c20384c120f963465a0", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Adam Harvey", + "email": "aharvey@php.net" + } + ], + "description": "Provides functionality to recursively process PHP variables", + "homepage": "https://github.com/sebastianbergmann/recursion-context", + "support": { + "issues": "https://github.com/sebastianbergmann/recursion-context/issues", + "source": "https://github.com/sebastianbergmann/recursion-context/tree/4.0.6" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/recursion-context", + "type": "tidelift" + } + ], + "time": "2025-08-10T06:57:39+00:00" + }, + { + "name": "sebastian/resource-operations", + "version": "3.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/resource-operations.git", + "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/05d5692a7993ecccd56a03e40cd7e5b09b1d404e", + "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Provides a list of PHP built-in functions that operate on resources", + "homepage": "https://www.github.com/sebastianbergmann/resource-operations", + "support": { + "source": "https://github.com/sebastianbergmann/resource-operations/tree/3.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-14T16:00:52+00:00" + }, + { + "name": "sebastian/type", + "version": "3.2.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/type.git", + "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", + "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.2-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Collection of value objects that represent the types of the PHP type system", + "homepage": "https://github.com/sebastianbergmann/type", + "support": { + "issues": "https://github.com/sebastianbergmann/type/issues", + "source": "https://github.com/sebastianbergmann/type/tree/3.2.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:13:03+00:00" + }, + { + "name": "sebastian/version", + "version": "3.0.2", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/version.git", + "reference": "c6c1022351a901512170118436c764e473f6de8c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/version/zipball/c6c1022351a901512170118436c764e473f6de8c", + "reference": "c6c1022351a901512170118436c764e473f6de8c", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library that helps with managing the version number of Git-hosted PHP projects", + "homepage": "https://github.com/sebastianbergmann/version", + "support": { + "issues": "https://github.com/sebastianbergmann/version/issues", + "source": "https://github.com/sebastianbergmann/version/tree/3.0.2" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-09-28T06:39:44+00:00" + }, + { + "name": "theseer/tokenizer", + "version": "1.3.1", + "source": { + "type": "git", + "url": "https://github.com/theseer/tokenizer.git", + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b7489ce515e168639d17feec34b8847c326b0b3c", + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-tokenizer": "*", + "ext-xmlwriter": "*", + "php": "^7.2 || ^8.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + } + ], + "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", + "support": { + "issues": "https://github.com/theseer/tokenizer/issues", + "source": "https://github.com/theseer/tokenizer/tree/1.3.1" + }, + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2025-11-17T20:03:58+00:00" + } + ], "aliases": [], "minimum-stability": "stable", "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, - "platform": {}, + "platform": { + "php": ">=8.0" + }, "platform-dev": {}, "plugin-api-version": "2.6.0" } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..659dd28 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' + +services: + mariadb: + image: mariadb:10.7 + container_name: utopia-usage-mariadb + restart: unless-stopped + networks: + - usage + ports: + - "3307:3306" + volumes: + - mariadb:/var/lib/mysql:rw + environment: + - MYSQL_ROOT_PASSWORD=password + - MYSQL_DATABASE=utopiaTests + - MYSQL_USER=user + - MYSQL_PASSWORD=password + + usage: + container_name: utopia-usage + build: + context: . + dockerfile: Dockerfile + networks: + - usage + volumes: + - ./tests:/code/tests + - ./src:/code/src + depends_on: + - mariadb + +networks: + usage: + +volumes: + mariadb: diff --git a/phpunit.xml b/phpunit.xml new file mode 100644 index 0000000..85a1d76 --- /dev/null +++ b/phpunit.xml @@ -0,0 +1,20 @@ + + + + + tests + + + + + src + + + + + + + diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php new file mode 100644 index 0000000..32c8a58 --- /dev/null +++ b/src/Usage/Adapter.php @@ -0,0 +1,90 @@ + $tags + * @return bool + */ + abstract public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool; + + /** + * Log multiple metrics in batch + * + * @param array}> $metrics + * @return bool + */ + abstract public function logBatch(array $metrics): bool; + + /** + * Get usage metrics by period + * + * @param string $metric + * @param string $period + * @param array<\Utopia\Database\Query> $queries + * @return array + */ + abstract public function getByPeriod(string $metric, string $period, array $queries = []): array; + + /** + * Get usage metrics between dates + * + * @param string $metric + * @param string $startDate + * @param string $endDate + * @param array<\Utopia\Database\Query> $queries + * @return array + */ + abstract public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array; + + /** + * Count usage metrics by period + * + * @param string $metric + * @param string $period + * @param array<\Utopia\Database\Query> $queries + * @return int + */ + abstract public function countByPeriod(string $metric, string $period, array $queries = []): int; + + /** + * Sum usage metrics by period + * + * @param string $metric + * @param string $period + * @param array<\Utopia\Database\Query> $queries + * @return int + */ + abstract public function sumByPeriod(string $metric, string $period, array $queries = []): int; + + /** + * Purge old usage metrics + * + * @param string $datetime + * @return bool + */ + abstract public function purge(string $datetime): bool; +} diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php new file mode 100644 index 0000000..59f13f9 --- /dev/null +++ b/src/Usage/Adapter/ClickHouse.php @@ -0,0 +1,593 @@ + */ + public const PERIODS = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00', + ]; + + private string $host; + private int $port; + private string $database = self::DEFAULT_DATABASE; + private string $table = self::DEFAULT_TABLE; + private string $username; + private string $password; + + /** @var bool Whether to use HTTPS for ClickHouse HTTP interface */ + private bool $secure = false; + + private Client $client; + + /** + * @param string $host ClickHouse host + * @param string $username ClickHouse username (default: 'default') + * @param string $password ClickHouse password (default: '') + * @param int $port ClickHouse HTTP port (default: 8123) + * @param bool $secure Whether to use HTTPS (default: false) + */ + public function __construct( + string $host, + string $username = 'default', + string $password = '', + int $port = self::DEFAULT_PORT, + bool $secure = false + ) { + $this->validateHost($host); + $this->validatePort($port); + + $this->host = $host; + $this->port = $port; + $this->username = $username; + $this->password = $password; + $this->secure = $secure; + + $this->client = new Client(); + } + + /** + * Validate host parameter. + * + * @param string $host + * @throws Exception + */ + private function validateHost(string $host): void + { + if (empty($host)) { + throw new Exception('ClickHouse host cannot be empty'); + } + + // Allow hostnames, IP addresses, and localhost + if (! preg_match('/^[a-zA-Z0-9._\-]+$/', $host)) { + throw new Exception('ClickHouse host must be a valid hostname or IP address'); + } + } + + /** + * Validate port parameter. + * + * @param int $port + * @throws Exception + */ + private function validatePort(int $port): void + { + if ($port < 1 || $port > 65535) { + throw new Exception('ClickHouse port must be between 1 and 65535'); + } + } + + /** + * Validate identifier (database, table). + * + * @param string $identifier + * @param string $type Name of the identifier type for error messages + * @throws Exception + */ + private function validateIdentifier(string $identifier, string $type = 'Identifier'): void + { + if (empty($identifier)) { + throw new Exception("{$type} cannot be empty"); + } + + if (strlen($identifier) > 255) { + throw new Exception("{$type} cannot exceed 255 characters"); + } + + // ClickHouse identifiers: alphanumeric, underscores, cannot start with number + if (! preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $identifier)) { + throw new Exception("{$type} must start with a letter or underscore and contain only alphanumeric characters and underscores"); + } + + // Check against SQL keywords + $keywords = ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'DROP', 'CREATE', 'ALTER', 'TABLE', 'DATABASE']; + if (in_array(strtoupper($identifier), $keywords, true)) { + throw new Exception("{$type} cannot be a reserved SQL keyword"); + } + } + + /** + * Escape an identifier for safe use in SQL. + * + * @param string $identifier + * @return string + */ + private function escapeIdentifier(string $identifier): string + { + return '`' . str_replace('`', '``', $identifier) . '`'; + } + + /** + * Escape a string value for safe use in ClickHouse SQL queries. + * + * @param string $value + * @return string The escaped value without surrounding quotes + */ + private function escapeString(string $value): string + { + return str_replace( + ["\\", "'"], + ["\\\\", "''"], + $value + ); + } + + /** + * Set the database name for subsequent operations. + * + * @param string $database + * @return self + * @throws Exception + */ + public function setDatabase(string $database): self + { + $this->validateIdentifier($database, 'Database'); + $this->database = $database; + + return $this; + } + + /** + * Set the table name for subsequent operations. + * + * @param string $table + * @return self + * @throws Exception + */ + public function setTable(string $table): self + { + $this->validateIdentifier($table, 'Table'); + $this->table = $table; + + return $this; + } + + /** + * Execute a ClickHouse query via HTTP interface. + * + * @param string $sql SQL query to execute + * @param array $params Query parameters for prepared statements + * @return string Query result as string + * @throws Exception + */ + private function query(string $sql, array $params = []): string + { + $protocol = $this->secure ? 'https' : 'http'; + $url = "{$protocol}://{$this->host}:{$this->port}/"; + + // Replace parameters in SQL + foreach ($params as $key => $value) { + $placeholder = ":{$key}"; + if (is_string($value)) { + $escapedValue = "'" . $this->escapeString($value) . "'"; + } elseif (is_null($value)) { + $escapedValue = 'NULL'; + } else { + $escapedValue = (string) $value; + } + $sql = str_replace($placeholder, $escapedValue, $sql); + } + + // Set authentication headers + $this->client->addHeader('X-ClickHouse-User', $this->username); + $this->client->addHeader('X-ClickHouse-Key', $this->password); + $this->client->addHeader('X-ClickHouse-Database', $this->database); + + try { + $response = $this->client->fetch( + url: $url, + method: Client::METHOD_POST, + body: ['query' => $sql] + ); + + if ($response->getStatusCode() !== 200) { + $body = $response->getBody(); + $bodyStr = is_string($body) ? $body : ''; + throw new Exception("ClickHouse query failed with HTTP {$response->getStatusCode()}: {$bodyStr}"); + } + + $body = $response->getBody(); + + return is_string($body) ? $body : ''; + } catch (Exception $e) { + throw new Exception( + "ClickHouse query execution failed: {$e->getMessage()}", + 0, + $e + ); + } + } + + public function getName(): string + { + return 'ClickHouse'; + } + + /** + * Setup ClickHouse table structure. + * + * Creates the database and table if they don't exist. + * + * @throws Exception + */ + public function setup(): void + { + // Create database if not exists + $escapedDatabase = $this->escapeIdentifier($this->database); + $createDbSql = "CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"; + $this->query($createDbSql); + + // Build column definitions + $columns = [ + 'id String', + 'metric String', + 'value Int64', + 'period String', + 'time DateTime64(3)', + 'tags String', // JSON string + ]; + + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + // Create table with MergeTree engine for optimal performance + $createTableSql = " + CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( + " . implode(",\n ", $columns) . ", + INDEX idx_metric metric TYPE bloom_filter GRANULARITY 1, + INDEX idx_period period TYPE bloom_filter GRANULARITY 1 + ) + ENGINE = MergeTree() + ORDER BY (metric, period, time) + PARTITION BY toYYYYMM(time) + SETTINGS index_granularity = 8192 + "; + + $this->query($createTableSql); + } + + /** + * Log a usage metric. + * + * @param string $metric + * @param int $value + * @param string $period + * @param array $tags + * @return bool + * @throws Exception + */ + public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool + { + if (! isset(self::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + } + + $id = uniqid('', true); + $now = new \DateTime(); + $time = $now->format(self::PERIODS[$period]); + + // Format timestamp for ClickHouse DateTime64(3) + $microtime = microtime(true); + $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); + + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + $sql = " + INSERT INTO {$escapedDatabaseAndTable} + (id, metric, value, period, time, tags) + VALUES ( + :id, + :metric, + :value, + :period, + :time, + :tags + ) + "; + + $this->query($sql, [ + 'id' => $id, + 'metric' => $metric, + 'value' => $value, + 'period' => $period, + 'time' => $timestamp, + 'tags' => json_encode($tags), + ]); + + return true; + } + + /** + * Log multiple usage metrics in batch. + * + * @param array> $metrics + * @return bool + * @throws Exception + */ + public function logBatch(array $metrics): bool + { + if (empty($metrics)) { + return true; + } + + $values = []; + foreach ($metrics as $metricData) { + $period = $metricData['period'] ?? '1h'; + + if (! isset(self::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + } + + $id = uniqid('', true); + $microtime = microtime(true); + $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); + + $values[] = sprintf( + "('%s', '%s', %d, '%s', '%s', '%s')", + $id, + $this->escapeString((string) $metricData['metric']), + (int) $metricData['value'], + $this->escapeString($period), + $timestamp, + $this->escapeString((string) json_encode($metricData['tags'] ?? [])) + ); + } + + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + $insertSql = " + INSERT INTO {$escapedDatabaseAndTable} + (id, metric, value, period, time, tags) + VALUES " . implode(', ', $values); + + $this->query($insertSql); + + return true; + } + + /** + * Parse ClickHouse TabSeparated results into Document array. + * + * @param string $result + * @return array + */ + private function parseResults(string $result): array + { + if (empty(trim($result))) { + return []; + } + + $lines = explode("\n", trim($result)); + $documents = []; + + foreach ($lines as $line) { + if (empty(trim($line))) { + continue; + } + + $columns = explode("\t", $line); + if (count($columns) < 6) { + continue; + } + + $documents[] = new Document([ + '$id' => $columns[0], + 'metric' => $columns[1], + 'value' => (int) $columns[2], + 'period' => $columns[3], + 'time' => $columns[4], + 'tags' => json_decode($columns[5], true) ?? [], + ]); + } + + return $documents; + } + + /** + * Get usage metrics by period. + * + * @param string $metric + * @param string $period + * @param array $queries + * @return array + * @throws Exception + */ + public function getByPeriod(string $metric, string $period, array $queries = []): array + { + $limit = 25; + $offset = 0; + + foreach ($queries as $query) { + if (is_object($query) && method_exists($query, 'getMethod') && method_exists($query, 'getValue')) { + if ($query->getMethod() === 'limit') { + $limit = (int) $query->getValue(); + } elseif ($query->getMethod() === 'offset') { + $offset = (int) $query->getValue(); + } + } + } + + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + $sql = " + SELECT id, metric, value, period, time, tags + FROM {$escapedDatabaseAndTable} + WHERE metric = :metric AND period = :period + ORDER BY time DESC + LIMIT :limit OFFSET :offset + FORMAT TabSeparated + "; + + $result = $this->query($sql, [ + 'metric' => $metric, + 'period' => $period, + 'limit' => $limit, + 'offset' => $offset, + ]); + + return $this->parseResults($result); + } + + /** + * Get usage metrics between dates. + * + * @param string $metric + * @param string $startDate + * @param string $endDate + * @param array $queries + * @return array + * @throws Exception + */ + public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array + { + $limit = 25; + $offset = 0; + + foreach ($queries as $query) { + if (is_object($query) && method_exists($query, 'getMethod') && method_exists($query, 'getValue')) { + if ($query->getMethod() === 'limit') { + $limit = (int) $query->getValue(); + } elseif ($query->getMethod() === 'offset') { + $offset = (int) $query->getValue(); + } + } + } + + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + $sql = " + SELECT id, metric, value, period, time, tags + FROM {$escapedDatabaseAndTable} + WHERE metric = :metric AND time >= :startDate AND time <= :endDate + ORDER BY time DESC + LIMIT :limit OFFSET :offset + FORMAT TabSeparated + "; + + $result = $this->query($sql, [ + 'metric' => $metric, + 'startDate' => $startDate, + 'endDate' => $endDate, + 'limit' => $limit, + 'offset' => $offset, + ]); + + return $this->parseResults($result); + } + + /** + * Count usage metrics by period. + * + * @param string $metric + * @param string $period + * @param array $queries + * @return int + * @throws Exception + */ + public function countByPeriod(string $metric, string $period, array $queries = []): int + { + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + $sql = " + SELECT count() as count + FROM {$escapedDatabaseAndTable} + WHERE metric = :metric AND period = :period + FORMAT TabSeparated + "; + + $result = $this->query($sql, [ + 'metric' => $metric, + 'period' => $period, + ]); + + return (int) trim($result); + } + + /** + * Sum usage metric values by period. + * + * @param string $metric + * @param string $period + * @param array $queries + * @return int + * @throws Exception + */ + public function sumByPeriod(string $metric, string $period, array $queries = []): int + { + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + $sql = " + SELECT sum(value) as total + FROM {$escapedDatabaseAndTable} + WHERE metric = :metric AND period = :period + FORMAT TabSeparated + "; + + $result = $this->query($sql, [ + 'metric' => $metric, + 'period' => $period, + ]); + + $total = trim($result); + + return empty($total) ? 0 : (int) $total; + } + + /** + * Purge usage metrics older than the specified datetime. + * + * @param string $datetime + * @return bool + * @throws Exception + */ + public function purge(string $datetime): bool + { + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + + $sql = " + DELETE FROM {$escapedDatabaseAndTable} + WHERE time < :datetime + "; + + $this->query($sql, ['datetime' => $datetime]); + + return true; + } +} diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php new file mode 100644 index 0000000..20644b0 --- /dev/null +++ b/src/Usage/Adapter/Database.php @@ -0,0 +1,277 @@ + */ + public const PERIODS = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00', + ]; + + public const ATTRIBUTES = [ + [ + '$id' => 'metric', + 'type' => UtopiaDatabase::VAR_STRING, + 'size' => 255, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => UtopiaDatabase::VAR_INTEGER, + 'size' => 0, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => UtopiaDatabase::VAR_STRING, + 'size' => 16, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => 'tags', + 'type' => UtopiaDatabase::VAR_STRING, + 'size' => 16777216, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['json'], + ], + ]; + + public const INDEXES = [ + [ + '$id' => 'index-metric', + 'type' => UtopiaDatabase::INDEX_KEY, + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-period', + 'type' => UtopiaDatabase::INDEX_KEY, + 'attributes' => ['period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-metric-period', + 'type' => UtopiaDatabase::INDEX_KEY, + 'attributes' => ['metric', 'period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-time', + 'type' => UtopiaDatabase::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => [UtopiaDatabase::ORDER_DESC], + ], + ]; + + private UtopiaDatabase $db; + + public function __construct(UtopiaDatabase $db) + { + $this->db = $db; + } + + public function getName(): string + { + return 'Database'; + } + + public function setup(): void + { + if (! $this->db->exists($this->db->getDatabase())) { + throw new Exception('You need to create the database before running Usage setup'); + } + + $attributes = \array_map(function ($attribute) { + return new Document($attribute); + }, self::ATTRIBUTES); + + $indexes = \array_map(function ($index) { + return new Document($index); + }, self::INDEXES); + + try { + $this->db->createCollection( + self::COLLECTION, + $attributes, + $indexes + ); + } catch (DuplicateException) { + // Collection already exists + } + } + + public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool + { + if (! isset(self::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + } + + $now = new \DateTime(); + $time = $now->format(self::PERIODS[$period]); + + $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags) { + $this->db->createDocument(self::COLLECTION, new Document([ + '$permissions' => [], + 'metric' => $metric, + 'value' => $value, + 'period' => $period, + 'time' => $time, + 'tags' => $tags, + ])); + }); + + return true; + } + + public function logBatch(array $metrics): bool + { + $this->db->getAuthorization()->skip(function () use ($metrics) { + $documents = \array_map(function ($metric) { + $period = $metric['period'] ?? '1h'; + + if (! isset(self::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + } + + $now = new \DateTime(); + $time = $now->format(self::PERIODS[$period]); + + return new Document([ + '$permissions' => [], + 'metric' => $metric['metric'], + 'value' => $metric['value'], + 'period' => $period, + 'time' => $time, + 'tags' => $metric['tags'] ?? [], + ]); + }, $metrics); + + $this->db->createDocuments(self::COLLECTION, $documents); + }); + + return true; + } + + public function getByPeriod(string $metric, string $period, array $queries = []): array + { + /** @var array $result */ + $result = $this->db->getAuthorization()->skip(function () use ($queries, $metric, $period) { + $queries[] = Query::equal('metric', [$metric]); + $queries[] = Query::equal('period', [$period]); + $queries[] = Query::orderDesc(); + + return $this->db->find( + collection: self::COLLECTION, + queries: $queries, + ); + }); + + return $result; + } + + public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array + { + /** @var array $result */ + $result = $this->db->getAuthorization()->skip(function () use ($queries, $metric, $startDate, $endDate) { + $queries[] = Query::equal('metric', [$metric]); + $queries[] = Query::greaterThanEqual('time', $startDate); + $queries[] = Query::lessThanEqual('time', $endDate); + $queries[] = Query::orderDesc(); + + return $this->db->find( + collection: self::COLLECTION, + queries: $queries, + ); + }); + + return $result; + } + + public function countByPeriod(string $metric, string $period, array $queries = []): int + { + /** @var int $count */ + $count = $this->db->getAuthorization()->skip(function () use ($queries, $metric, $period) { + return $this->db->count( + collection: self::COLLECTION, + queries: [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + ...$queries, + ] + ); + }); + + return $count; + } + + public function sumByPeriod(string $metric, string $period, array $queries = []): int + { + /** @var array $results */ + $results = $this->getByPeriod($metric, $period, $queries); + + $sum = 0; + foreach ($results as $result) { + $sum += $result->getAttribute('value', 0); + } + + return $sum; + } + + public function purge(string $datetime): bool + { + $this->db->getAuthorization()->skip(function () use ($datetime) { + do { + $documents = $this->db->find( + collection: self::COLLECTION, + queries: [ + Query::lessThan('time', $datetime), + Query::limit(100), + ] + ); + + foreach ($documents as $document) { + $this->db->deleteDocument(self::COLLECTION, $document->getId()); + } + } while (! empty($documents)); + }); + + return true; + } +} diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php new file mode 100644 index 0000000..f0b36f2 --- /dev/null +++ b/src/Usage/Usage.php @@ -0,0 +1,247 @@ +adapter = $adapter; + } + + /** + * Get the current adapter. + * + * @return Adapter + */ + public function getAdapter(): Adapter + { + return $this->adapter; + } + + /** + * Setup the usage metrics storage. + * + * @return void + * @throws \Exception + */ + public function setup(): void + { + $this->adapter->setup(); + } + + /** + * Log a usage metric. + * + * @param string $metric + * @param int $value + * @param string $period + * @param array $tags + * @return bool + * @throws \Exception + */ + public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool + { + return $this->adapter->log($metric, $value, $period, $tags); + } + + /** + * Log multiple usage metrics in batch. + * + * @param array> $metrics + * @return bool + * @throws \Exception + */ + public function logBatch(array $metrics): bool + { + return $this->adapter->logBatch($metrics); + } + + /** + * Get usage metrics by period. + * + * @param string $metric + * @param string $period + * @param array $queries + * @return array + * @throws \Exception + */ + public function getByPeriod(string $metric, string $period, array $queries = []): array + { + return $this->adapter->getByPeriod($metric, $period, $queries); + } + + /** + * Get usage metrics between dates. + * + * @param string $metric + * @param string $startDate + * @param string $endDate + * @param array $queries + * @return array + * @throws \Exception + */ + public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array + { + return $this->adapter->getBetweenDates($metric, $startDate, $endDate, $queries); + } + + /** + * Count usage metrics by period. + * + * @param string $metric + * @param string $period + * @param array $queries + * @return int + * @throws \Exception + */ + public function countByPeriod(string $metric, string $period, array $queries = []): int + { + return $this->adapter->countByPeriod($metric, $period, $queries); + } + + /** + * Sum usage metric values by period. + * + * @param string $metric + * @param string $period + * @param array $queries + * @return int + * @throws \Exception + */ + public function sumByPeriod(string $metric, string $period, array $queries = []): int + { + return $this->adapter->sumByPeriod($metric, $period, $queries); + } + + /** + * Purge usage metrics older than the specified datetime. + * + * @param string $datetime + * @return bool + * @throws \Exception + */ + public function purge(string $datetime): bool + { + return $this->adapter->purge($datetime); + } + + /** + * @deprecated Use constructor with adapter instead + * @internal Legacy support - will be removed in future version + */ + public const COLLECTION = 'usage'; + + /** + * @deprecated Use Adapter\Database::PERIODS instead + * @var array + */ + public const PERIODS = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00', + ]; + + /** + * @deprecated Use Adapter\Database::ATTRIBUTES instead + */ + public const ATTRIBUTES = [ + [ + '$id' => 'metric', + 'type' => \Utopia\Database\Database::VAR_STRING, + 'size' => 255, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => \Utopia\Database\Database::VAR_INTEGER, + 'size' => 0, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => \Utopia\Database\Database::VAR_STRING, + 'size' => 16, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => \Utopia\Database\Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => 'tags', + 'type' => \Utopia\Database\Database::VAR_STRING, + 'size' => 16777216, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['json'], + ], + ]; + + /** + * @deprecated Use Adapter\Database::INDEXES instead + */ + public const INDEXES = [ + [ + '$id' => 'index-metric', + 'type' => \Utopia\Database\Database::INDEX_KEY, + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-period', + 'type' => \Utopia\Database\Database::INDEX_KEY, + 'attributes' => ['period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-metric-period', + 'type' => \Utopia\Database\Database::INDEX_KEY, + 'attributes' => ['metric', 'period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-time', + 'type' => \Utopia\Database\Database::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => [\Utopia\Database\Database::ORDER_DESC], + ], + ]; +} diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php new file mode 100644 index 0000000..ff8e21f --- /dev/null +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -0,0 +1,40 @@ +markTestSkipped('CLICKHOUSE_HOST not set; skipping ClickHouse adapter tests.'); + } + + $adapter = new ClickHouseAdapter($host, $username, $password, $port, $secure); + + // Optional customization via env vars + if ($database = getenv('CLICKHOUSE_DATABASE')) { + $adapter->setDatabase($database); + } + + if ($table = getenv('CLICKHOUSE_TABLE')) { + $adapter->setTable($table); + } + + $this->usage = new Usage($adapter); + $this->usage->setup(); + } +} diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php new file mode 100644 index 0000000..fdba02e --- /dev/null +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -0,0 +1,42 @@ +database = new Database(new MariaDB($pdo), $cache); + $this->database->setDatabase('utopiaTests'); + $this->database->setNamespace('utopia_usage'); + + $this->usage = new Usage(new AdapterDatabase($this->database)); + + // Create database and collection if needed + $this->database->create(); + // Always run setup to ensure collection exists + $this->usage->setup(); + } +} diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php new file mode 100644 index 0000000..fce189e --- /dev/null +++ b/tests/Usage/UsageBase.php @@ -0,0 +1,167 @@ +initializeUsage(); + $this->createUsageMetrics(); + } + + + public function tearDown(): void + { + $this->usage->purge(DateTime::now()); + } + + public function createUsageMetrics(): void + { + $this->assertTrue($this->usage->log('requests', 100, '1h', ['region' => 'us-east'])); + $this->assertTrue($this->usage->log('requests', 150, '1h', ['region' => 'us-west'])); + $this->assertTrue($this->usage->log('requests', 200, '1d', ['region' => 'us-east'])); + $this->assertTrue($this->usage->log('bandwidth', 5000, '1h', ['region' => 'us-east'])); + $this->assertTrue($this->usage->log('storage', 10000, 'inf', ['region' => 'us-east'])); + } + + public function testLogUsage(): void + { + $result = $this->usage->log('test-metric', 42, '1h', ['foo' => 'bar']); + $this->assertTrue($result); + } + + public function testLogBatch(): void + { + // First cleanup existing logs + $this->usage->purge(DateTime::now()); + + $metrics = [ + [ + 'metric' => 'batch-requests', + 'value' => 100, + 'period' => '1h', + 'tags' => ['region' => 'eu-west'], + ], + [ + 'metric' => 'batch-requests', + 'value' => 150, + 'period' => '1h', + 'tags' => ['region' => 'eu-east'], + ], + [ + 'metric' => 'batch-bandwidth', + 'value' => 3000, + 'period' => '1d', + 'tags' => ['region' => 'eu-west'], + ], + ]; + + $this->assertTrue($this->usage->logBatch($metrics)); + + $results = $this->usage->getByPeriod('batch-requests', '1h'); + $this->assertEquals(2, count($results)); + } + + public function testGetByPeriod(): void + { + $results1h = $this->usage->getByPeriod('requests', '1h'); + $results1d = $this->usage->getByPeriod('requests', '1d'); + $resultsInf = $this->usage->getByPeriod('storage', 'inf'); + + $this->assertEquals(2, count($results1h)); + $this->assertEquals(1, count($results1d)); + $this->assertEquals(1, count($resultsInf)); + } + + public function testGetBetweenDates(): void + { + $start = DateTime::addSeconds(new \DateTime(), -3600); // 1 hour ago + $end = DateTime::now(); + + $results = $this->usage->getBetweenDates('requests', $start, $end); + $this->assertGreaterThanOrEqual(0, count($results)); + } + + public function testCountByPeriod(): void + { + $count1h = $this->usage->countByPeriod('requests', '1h'); + $count1d = $this->usage->countByPeriod('requests', '1d'); + $countBandwidth = $this->usage->countByPeriod('bandwidth', '1h'); + + $this->assertEquals(2, $count1h); + $this->assertEquals(1, $count1d); + $this->assertEquals(1, $countBandwidth); + } + + public function testSumByPeriod(): void + { + $sum = $this->usage->sumByPeriod('requests', '1h'); + $this->assertEquals(250, $sum); // 100 + 150 + + $sumBandwidth = $this->usage->sumByPeriod('bandwidth', '1h'); + $this->assertEquals(5000, $sumBandwidth); + } + + public function testWithQueries(): void + { + $results = $this->usage->getByPeriod('requests', '1h', [ + Query::limit(1), + ]); + + $this->assertEquals(1, count($results)); + + $results2 = $this->usage->getByPeriod('requests', '1h', [ + Query::limit(1), + Query::offset(1), + ]); + + $this->assertEquals(1, count($results2)); + } + + public function testPurge(): void + { + sleep(2); + + // Add a metric + $this->usage->log('purge-test', 999, '1h'); + + // Wait a bit + sleep(2); + + // Purge all metrics + $status = $this->usage->purge(DateTime::now()); + $this->assertTrue($status); + + // Verify metrics were purged + $results = $this->usage->getByPeriod('purge-test', '1h'); + $this->assertEquals(0, count($results)); + } + + public function testInvalidPeriod(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->usage->log('test', 100, 'invalid-period'); + } + + public function testPeriodFormats(): void + { + $periods = Usage::PERIODS; + + $this->assertArrayHasKey('1h', $periods); + $this->assertArrayHasKey('1d', $periods); + $this->assertArrayHasKey('inf', $periods); + + $this->assertEquals('Y-m-d H:00', $periods['1h']); + $this->assertEquals('Y-m-d 00:00', $periods['1d']); + $this->assertEquals('0000-00-00 00:00', $periods['inf']); + } +} From cb6c26eea58db8e3802f0caaee6a0a8855a31ed7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:01:07 +0000 Subject: [PATCH 02/44] Update and fix tests --- docker-compose.yml | 20 ++++++++++++++++++-- src/Usage/Adapter/Database.php | 8 ++++++-- tests/Usage/Adapter/ClickHouseTest.php | 4 ++-- tests/Usage/Adapter/DatabaseTest.php | 15 ++++++++++++--- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 659dd28..2bee203 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: mariadb: image: mariadb:10.7 @@ -16,6 +14,24 @@ services: - MYSQL_DATABASE=utopiaTests - MYSQL_USER=user - MYSQL_PASSWORD=password + clickhouse: + image: clickhouse/clickhouse-server:25.11-alpine + environment: + - CLICKHOUSE_DB=default + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD=clickhouse + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + networks: + - usage + ports: + - "8123:8123" + - "9000:9000" + healthcheck: + test: ["CMD", "clickhouse-client", "--host=localhost", "--port=9000", "-q", "SELECT 1"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 15s usage: container_name: utopia-usage diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 20644b0..7b1efd5 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -144,7 +144,9 @@ public function log(string $metric, int $value, string $period = '1h', array $ta } $now = new \DateTime(); - $time = $now->format(self::PERIODS[$period]); + $time = $period === 'inf' + ? '1000-01-01 00:00:00' + : $now->format(self::PERIODS[$period]); $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags) { $this->db->createDocument(self::COLLECTION, new Document([ @@ -171,7 +173,9 @@ public function logBatch(array $metrics): bool } $now = new \DateTime(); - $time = $now->format(self::PERIODS[$period]); + $time = $period === 'inf' + ? '1000-01-01 00:00:00' + : $now->format(self::PERIODS[$period]); return new Document([ '$permissions' => [], diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index ff8e21f..8482a2e 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -15,11 +15,11 @@ protected function initializeUsage(): void { $host = getenv('CLICKHOUSE_HOST') ?: 'clickhouse'; $username = getenv('CLICKHOUSE_USER') ?: 'default'; - $password = getenv('CLICKHOUSE_PASSWORD') ?: ''; + $password = getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse'; $port = (int) (getenv('CLICKHOUSE_PORT') ?: 8123); $secure = (bool) (getenv('CLICKHOUSE_SECURE') ?: false); - if ($host === null) { + if ($host === null || $host === '') { $this->markTestSkipped('CLICKHOUSE_HOST not set; skipping ClickHouse adapter tests.'); } diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index fdba02e..ff117cf 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -9,6 +9,7 @@ use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\DateTime; +use Utopia\Database\Exception\Duplicate; use Utopia\Database\Query; use Utopia\Tests\Usage\UsageBase; use Utopia\Usage\Adapter\Database as AdapterDatabase; @@ -34,9 +35,17 @@ protected function initializeUsage(): void $this->usage = new Usage(new AdapterDatabase($this->database)); - // Create database and collection if needed - $this->database->create(); + // Create database if missing + if (! $this->database->exists($this->database->getDatabase())) { + $this->database->create(); + } + // Always run setup to ensure collection exists - $this->usage->setup(); + try { + + $this->usage->setup(); + } catch (Duplicate $ex) { + // ignore duplicate exception + } } } From 9c385149dc6313d592997bd234c47eb71f6993dd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:05:18 +0000 Subject: [PATCH 03/44] Update tests --- .github/workflows/codeql-analysis.yml | 20 +++++ .github/workflows/linter.yml | 20 +++++ .github/workflows/tests.yml | 33 +++---- composer.json | 9 +- composer.lock | 122 +++++++++++++++++++++++++- 5 files changed, 179 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/linter.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..e5100db --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,20 @@ +name: "CodeQL" + +on: [ pull_request ] +jobs: + lint: + name: CodeQL + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + + - name: Run CodeQL + run: | + docker run --rm -v $PWD:/app composer sh -c \ + "composer install --profile --ignore-platform-reqs && composer check" \ No newline at end of file diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..15853db --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,20 @@ +name: "Linter" + +on: [ pull_request ] +jobs: + lint: + name: Linter + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + + - name: Run Linter + run: | + docker run --rm -v $PWD:/app composer sh -c \ + "composer install --profile --ignore-platform-reqs && composer lint" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4bd84db..f0ad936 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,36 +1,23 @@ name: "Tests" -on: [pull_request, push] - +on: [ pull_request ] jobs: - tests: - name: Unit Tests + lint: + name: Tests runs-on: ubuntu-latest - strategy: - matrix: - php-versions: ['8.0', '8.1', '8.2', '8.3'] - steps: - name: Checkout repository uses: actions/checkout@v3 with: - submodules: recursive - - - name: Setup PHP - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-versions }} - extensions: pdo, pdo_mysql, mysqli - coverage: xdebug + fetch-depth: 2 - - name: Install dependencies - run: composer install + - run: git checkout HEAD^2 - - name: Start MariaDB + - name: Build run: | - docker compose up -d mariadb - sleep 10 + docker compose build + docker compose up -d --wait - - name: Run PHPUnit - run: docker compose run --rm usage vendor/bin/phpunit --configuration phpunit.xml tests + - name: Run Tests + run: docker compose exec usage vendor/bin/phpunit --configuration phpunit.xml tests \ No newline at end of file diff --git a/composer.json b/composer.json index ad66768..dc68ede 100644 --- a/composer.json +++ b/composer.json @@ -9,6 +9,11 @@ "email": "team@appwrite.io" } ], + "scripts": { + "lint": "./vendor/bin/pint --test", + "format": "./vendor/bin/pint", + "check": "./vendor/bin/phpstan analyse --level max src tests" + }, "minimum-stability": "stable", "require": { "php": ">=8.0", @@ -17,7 +22,9 @@ }, "require-dev": { "phpunit/phpunit": "^9.5", - "utopia-php/cache": "^0.13.0" + "utopia-php/cache": "^0.13.0", + "phpstan/phpstan": "1.*", + "laravel/pint": "1.*" }, "autoload": { "psr-4": { diff --git a/composer.lock b/composer.lock index de7d15b..d64c0ef 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8a5a1e92e201028a5f27180d0f7e2802", + "content-hash": "ea595e5dda2475807e9de0f50c141a57", "packages": [ { "name": "brick/math", @@ -2547,6 +2547,73 @@ ], "time": "2022-12-30T00:23:10+00:00" }, + { + "name": "laravel/pint", + "version": "v1.26.0", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "69dcca060ecb15e4b564af63d1f642c81a241d6f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/69dcca060ecb15e4b564af63d1f642c81a241d6f", + "reference": "69dcca060ecb15e4b564af63d1f642c81a241d6f", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.2.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.90.0", + "illuminate/view": "^12.40.1", + "larastan/larastan": "^3.8.0", + "laravel-zero/framework": "^12.0.4", + "mockery/mockery": "^1.6.12", + "nunomaduro/termwind": "^2.3.3", + "pestphp/pest": "^3.8.4" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "dev", + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2025-11-25T21:15:52+00:00" + }, { "name": "myclabs/deep-copy", "version": "1.13.4", @@ -2783,6 +2850,59 @@ }, "time": "2022-02-21T01:04:05+00:00" }, + { + "name": "phpstan/phpstan", + "version": "1.12.32", + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", + "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", + "shasum": "" + }, + "require": { + "php": "^7.2|^8.0" + }, + "conflict": { + "phpstan/phpstan-shim": "*" + }, + "bin": [ + "phpstan", + "phpstan.phar" + ], + "type": "library", + "autoload": { + "files": [ + "bootstrap.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHPStan - PHP Static Analysis Tool", + "keywords": [ + "dev", + "static analysis" + ], + "support": { + "docs": "https://phpstan.org/user-guide/getting-started", + "forum": "https://github.com/phpstan/phpstan/discussions", + "issues": "https://github.com/phpstan/phpstan/issues", + "security": "https://github.com/phpstan/phpstan/security/policy", + "source": "https://github.com/phpstan/phpstan-src" + }, + "funding": [ + { + "url": "https://github.com/ondrejmirtes", + "type": "github" + }, + { + "url": "https://github.com/phpstan", + "type": "github" + } + ], + "time": "2025-09-30T10:16:31+00:00" + }, { "name": "phpunit/php-code-coverage", "version": "9.2.32", From 360120e53f04fc25ba5f894bb34fe0b978ce42e2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:05:24 +0000 Subject: [PATCH 04/44] format --- src/Usage/Adapter.php | 35 ++------ src/Usage/Adapter/ClickHouse.php | 115 ++++++++++++--------------- src/Usage/Adapter/Database.php | 8 +- src/Usage/Usage.php | 47 ++++------- tests/Usage/Adapter/DatabaseTest.php | 5 +- tests/Usage/UsageBase.php | 3 +- 6 files changed, 80 insertions(+), 133 deletions(-) diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 32c8a58..da9461d 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -8,43 +8,32 @@ abstract class Adapter { /** * Get adapter name - * - * @return string */ abstract public function getName(): string; /** * Setup database structure - * - * @return void */ abstract public function setup(): void; /** * Log usage metric * - * @param string $metric - * @param int $value - * @param string $period - * @param array $tags - * @return bool + * @param array $tags */ abstract public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool; /** * Log multiple metrics in batch * - * @param array}> $metrics - * @return bool + * @param array}> $metrics */ abstract public function logBatch(array $metrics): bool; /** * Get usage metrics by period * - * @param string $metric - * @param string $period - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Database\Query> $queries * @return array */ abstract public function getByPeriod(string $metric, string $period, array $queries = []): array; @@ -52,10 +41,7 @@ abstract public function getByPeriod(string $metric, string $period, array $quer /** * Get usage metrics between dates * - * @param string $metric - * @param string $startDate - * @param string $endDate - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Database\Query> $queries * @return array */ abstract public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array; @@ -63,28 +49,19 @@ abstract public function getBetweenDates(string $metric, string $startDate, stri /** * Count usage metrics by period * - * @param string $metric - * @param string $period - * @param array<\Utopia\Database\Query> $queries - * @return int + * @param array<\Utopia\Database\Query> $queries */ abstract public function countByPeriod(string $metric, string $period, array $queries = []): int; /** * Sum usage metrics by period * - * @param string $metric - * @param string $period - * @param array<\Utopia\Database\Query> $queries - * @return int + * @param array<\Utopia\Database\Query> $queries */ abstract public function sumByPeriod(string $metric, string $period, array $queries = []): int; /** * Purge old usage metrics - * - * @param string $datetime - * @return bool */ abstract public function purge(string $datetime): bool; } diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 59f13f9..ba48872 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -16,7 +16,9 @@ class ClickHouse extends Adapter { private const DEFAULT_PORT = 8123; + private const DEFAULT_TABLE = 'usage'; + private const DEFAULT_DATABASE = 'default'; /** @var array */ @@ -27,10 +29,15 @@ class ClickHouse extends Adapter ]; private string $host; + private int $port; + private string $database = self::DEFAULT_DATABASE; + private string $table = self::DEFAULT_TABLE; + private string $username; + private string $password; /** @var bool Whether to use HTTPS for ClickHouse HTTP interface */ @@ -39,11 +46,11 @@ class ClickHouse extends Adapter private Client $client; /** - * @param string $host ClickHouse host - * @param string $username ClickHouse username (default: 'default') - * @param string $password ClickHouse password (default: '') - * @param int $port ClickHouse HTTP port (default: 8123) - * @param bool $secure Whether to use HTTPS (default: false) + * @param string $host ClickHouse host + * @param string $username ClickHouse username (default: 'default') + * @param string $password ClickHouse password (default: '') + * @param int $port ClickHouse HTTP port (default: 8123) + * @param bool $secure Whether to use HTTPS (default: false) */ public function __construct( string $host, @@ -61,13 +68,12 @@ public function __construct( $this->password = $password; $this->secure = $secure; - $this->client = new Client(); + $this->client = new Client; } /** * Validate host parameter. * - * @param string $host * @throws Exception */ private function validateHost(string $host): void @@ -85,7 +91,6 @@ private function validateHost(string $host): void /** * Validate port parameter. * - * @param int $port * @throws Exception */ private function validatePort(int $port): void @@ -98,8 +103,8 @@ private function validatePort(int $port): void /** * Validate identifier (database, table). * - * @param string $identifier - * @param string $type Name of the identifier type for error messages + * @param string $type Name of the identifier type for error messages + * * @throws Exception */ private function validateIdentifier(string $identifier, string $type = 'Identifier'): void @@ -126,26 +131,22 @@ private function validateIdentifier(string $identifier, string $type = 'Identifi /** * Escape an identifier for safe use in SQL. - * - * @param string $identifier - * @return string */ private function escapeIdentifier(string $identifier): string { - return '`' . str_replace('`', '``', $identifier) . '`'; + return '`'.str_replace('`', '``', $identifier).'`'; } /** * Escape a string value for safe use in ClickHouse SQL queries. * - * @param string $value * @return string The escaped value without surrounding quotes */ private function escapeString(string $value): string { return str_replace( - ["\\", "'"], - ["\\\\", "''"], + ['\\', "'"], + ['\\\\', "''"], $value ); } @@ -153,8 +154,6 @@ private function escapeString(string $value): string /** * Set the database name for subsequent operations. * - * @param string $database - * @return self * @throws Exception */ public function setDatabase(string $database): self @@ -168,8 +167,6 @@ public function setDatabase(string $database): self /** * Set the table name for subsequent operations. * - * @param string $table - * @return self * @throws Exception */ public function setTable(string $table): self @@ -183,9 +180,10 @@ public function setTable(string $table): self /** * Execute a ClickHouse query via HTTP interface. * - * @param string $sql SQL query to execute - * @param array $params Query parameters for prepared statements + * @param string $sql SQL query to execute + * @param array $params Query parameters for prepared statements * @return string Query result as string + * * @throws Exception */ private function query(string $sql, array $params = []): string @@ -197,7 +195,7 @@ private function query(string $sql, array $params = []): string foreach ($params as $key => $value) { $placeholder = ":{$key}"; if (is_string($value)) { - $escapedValue = "'" . $this->escapeString($value) . "'"; + $escapedValue = "'".$this->escapeString($value)."'"; } elseif (is_null($value)) { $escapedValue = 'NULL'; } else { @@ -265,12 +263,12 @@ public function setup(): void 'tags String', // JSON string ]; - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); // Create table with MergeTree engine for optimal performance $createTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( - " . implode(",\n ", $columns) . ", + ".implode(",\n ", $columns).', INDEX idx_metric metric TYPE bloom_filter GRANULARITY 1, INDEX idx_period period TYPE bloom_filter GRANULARITY 1 ) @@ -278,7 +276,7 @@ public function setup(): void ORDER BY (metric, period, time) PARTITION BY toYYYYMM(time) SETTINGS index_granularity = 8192 - "; + '; $this->query($createTableSql); } @@ -286,28 +284,25 @@ public function setup(): void /** * Log a usage metric. * - * @param string $metric - * @param int $value - * @param string $period - * @param array $tags - * @return bool + * @param array $tags + * * @throws Exception */ public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool { if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); } $id = uniqid('', true); - $now = new \DateTime(); + $now = new \DateTime; $time = $now->format(self::PERIODS[$period]); // Format timestamp for ClickHouse DateTime64(3) $microtime = microtime(true); - $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); + $timestamp = date('Y-m-d H:i:s', (int) $microtime).'.'.sprintf('%03d', ($microtime - floor($microtime)) * 1000); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); $sql = " INSERT INTO {$escapedDatabaseAndTable} @@ -337,8 +332,8 @@ public function log(string $metric, int $value, string $period = '1h', array $ta /** * Log multiple usage metrics in batch. * - * @param array> $metrics - * @return bool + * @param array> $metrics + * * @throws Exception */ public function logBatch(array $metrics): bool @@ -352,12 +347,12 @@ public function logBatch(array $metrics): bool $period = $metricData['period'] ?? '1h'; if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); } $id = uniqid('', true); $microtime = microtime(true); - $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); + $timestamp = date('Y-m-d H:i:s', (int) $microtime).'.'.sprintf('%03d', ($microtime - floor($microtime)) * 1000); $values[] = sprintf( "('%s', '%s', %d, '%s', '%s', '%s')", @@ -370,12 +365,12 @@ public function logBatch(array $metrics): bool ); } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); $insertSql = " INSERT INTO {$escapedDatabaseAndTable} (id, metric, value, period, time, tags) - VALUES " . implode(', ', $values); + VALUES ".implode(', ', $values); $this->query($insertSql); @@ -385,7 +380,6 @@ public function logBatch(array $metrics): bool /** * Parse ClickHouse TabSeparated results into Document array. * - * @param string $result * @return array */ private function parseResults(string $result): array @@ -423,10 +417,9 @@ private function parseResults(string $result): array /** * Get usage metrics by period. * - * @param string $metric - * @param string $period - * @param array $queries + * @param array $queries * @return array + * * @throws Exception */ public function getByPeriod(string $metric, string $period, array $queries = []): array @@ -444,7 +437,7 @@ public function getByPeriod(string $metric, string $period, array $queries = []) } } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); $sql = " SELECT id, metric, value, period, time, tags @@ -468,11 +461,9 @@ public function getByPeriod(string $metric, string $period, array $queries = []) /** * Get usage metrics between dates. * - * @param string $metric - * @param string $startDate - * @param string $endDate - * @param array $queries + * @param array $queries * @return array + * * @throws Exception */ public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array @@ -490,7 +481,7 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa } } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); $sql = " SELECT id, metric, value, period, time, tags @@ -515,15 +506,13 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa /** * Count usage metrics by period. * - * @param string $metric - * @param string $period - * @param array $queries - * @return int + * @param array $queries + * * @throws Exception */ public function countByPeriod(string $metric, string $period, array $queries = []): int { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); $sql = " SELECT count() as count @@ -543,15 +532,13 @@ public function countByPeriod(string $metric, string $period, array $queries = [ /** * Sum usage metric values by period. * - * @param string $metric - * @param string $period - * @param array $queries - * @return int + * @param array $queries + * * @throws Exception */ public function sumByPeriod(string $metric, string $period, array $queries = []): int { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); $sql = " SELECT sum(value) as total @@ -573,13 +560,11 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) /** * Purge usage metrics older than the specified datetime. * - * @param string $datetime - * @return bool * @throws Exception */ public function purge(string $datetime): bool { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); $sql = " DELETE FROM {$escapedDatabaseAndTable} diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 7b1efd5..1bdda85 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -140,10 +140,10 @@ public function setup(): void public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool { if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); } - $now = new \DateTime(); + $now = new \DateTime; $time = $period === 'inf' ? '1000-01-01 00:00:00' : $now->format(self::PERIODS[$period]); @@ -169,10 +169,10 @@ public function logBatch(array $metrics): bool $period = $metric['period'] ?? '1h'; if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); } - $now = new \DateTime(); + $now = new \DateTime; $time = $period === 'inf' ? '1000-01-01 00:00:00' : $now->format(self::PERIODS[$period]); diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index f0b36f2..1bdcd51 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -19,7 +19,7 @@ class Usage /** * Constructor. * - * @param Adapter $adapter The adapter to use for storing usage metrics + * @param Adapter $adapter The adapter to use for storing usage metrics */ public function __construct(Adapter $adapter) { @@ -28,8 +28,6 @@ public function __construct(Adapter $adapter) /** * Get the current adapter. - * - * @return Adapter */ public function getAdapter(): Adapter { @@ -39,7 +37,6 @@ public function getAdapter(): Adapter /** * Setup the usage metrics storage. * - * @return void * @throws \Exception */ public function setup(): void @@ -50,11 +47,8 @@ public function setup(): void /** * Log a usage metric. * - * @param string $metric - * @param int $value - * @param string $period - * @param array $tags - * @return bool + * @param array $tags + * * @throws \Exception */ public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool @@ -65,8 +59,8 @@ public function log(string $metric, int $value, string $period = '1h', array $ta /** * Log multiple usage metrics in batch. * - * @param array> $metrics - * @return bool + * @param array> $metrics + * * @throws \Exception */ public function logBatch(array $metrics): bool @@ -77,10 +71,9 @@ public function logBatch(array $metrics): bool /** * Get usage metrics by period. * - * @param string $metric - * @param string $period - * @param array $queries + * @param array $queries * @return array + * * @throws \Exception */ public function getByPeriod(string $metric, string $period, array $queries = []): array @@ -91,11 +84,9 @@ public function getByPeriod(string $metric, string $period, array $queries = []) /** * Get usage metrics between dates. * - * @param string $metric - * @param string $startDate - * @param string $endDate - * @param array $queries + * @param array $queries * @return array + * * @throws \Exception */ public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array @@ -106,10 +97,8 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa /** * Count usage metrics by period. * - * @param string $metric - * @param string $period - * @param array $queries - * @return int + * @param array $queries + * * @throws \Exception */ public function countByPeriod(string $metric, string $period, array $queries = []): int @@ -120,10 +109,8 @@ public function countByPeriod(string $metric, string $period, array $queries = [ /** * Sum usage metric values by period. * - * @param string $metric - * @param string $period - * @param array $queries - * @return int + * @param array $queries + * * @throws \Exception */ public function sumByPeriod(string $metric, string $period, array $queries = []): int @@ -134,8 +121,6 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) /** * Purge usage metrics older than the specified datetime. * - * @param string $datetime - * @return bool * @throws \Exception */ public function purge(string $datetime): bool @@ -145,13 +130,15 @@ public function purge(string $datetime): bool /** * @deprecated Use constructor with adapter instead + * * @internal Legacy support - will be removed in future version */ public const COLLECTION = 'usage'; - /** + /** * @deprecated Use Adapter\Database::PERIODS instead - * @var array + * + * @var array */ public const PERIODS = [ '1h' => 'Y-m-d H:00', diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index ff117cf..f5ebf45 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -8,9 +8,7 @@ use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; -use Utopia\Database\DateTime; use Utopia\Database\Exception\Duplicate; -use Utopia\Database\Query; use Utopia\Tests\Usage\UsageBase; use Utopia\Usage\Adapter\Database as AdapterDatabase; use Utopia\Usage\Usage; @@ -18,6 +16,7 @@ class DatabaseTest extends TestCase { use UsageBase; + protected Database $database; protected function initializeUsage(): void @@ -28,7 +27,7 @@ protected function initializeUsage(): void $dbPass = 'password'; $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPdoAttributes()); - $cache = new Cache(new NoCache()); + $cache = new Cache(new NoCache); $this->database = new Database(new MariaDB($pdo), $cache); $this->database->setDatabase('utopiaTests'); $this->database->setNamespace('utopia_usage'); diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index fce189e..0d25ad4 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -18,7 +18,6 @@ public function setUp(): void $this->createUsageMetrics(); } - public function tearDown(): void { $this->usage->purge(DateTime::now()); @@ -84,7 +83,7 @@ public function testGetByPeriod(): void public function testGetBetweenDates(): void { - $start = DateTime::addSeconds(new \DateTime(), -3600); // 1 hour ago + $start = DateTime::addSeconds(new \DateTime, -3600); // 1 hour ago $end = DateTime::now(); $results = $this->usage->getBetweenDates('requests', $start, $end); From f650422db522dbe6a3a3f8222cdca5fe613f35ed Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:11:06 +0000 Subject: [PATCH 05/44] improve tests --- docker-compose.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2bee203..0253e7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,13 @@ services: - MYSQL_DATABASE=utopiaTests - MYSQL_USER=user - MYSQL_PASSWORD=password + healthcheck: + test: ["CMD", "sh", "-c", "mysqladmin ping -h localhost -u root -p$$MYSQL_ROOT_PASSWORD"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 30s + clickhouse: image: clickhouse/clickhouse-server:25.11-alpine environment: @@ -44,7 +51,16 @@ services: - ./tests:/code/tests - ./src:/code/src depends_on: - - mariadb + mariadb: + condition: service_healthy + clickhouse: + condition: service_healthy + healthcheck: + test: ["CMD", "php", "--version"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s networks: usage: From e03ba506c1e7905423754a46e4b093a38139ec60 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:17:18 +0000 Subject: [PATCH 06/44] format and fix codeql analysis --- pint.json | 3 + src/Usage/Adapter/ClickHouse.php | 77 ++++++++++++++++---------- src/Usage/Adapter/Database.php | 4 +- src/Usage/Usage.php | 12 ++-- tests/Usage/Adapter/ClickHouseTest.php | 8 ++- tests/Usage/Adapter/DatabaseTest.php | 2 +- tests/Usage/UsageBase.php | 2 +- 7 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 pint.json diff --git a/pint.json b/pint.json new file mode 100644 index 0000000..c781933 --- /dev/null +++ b/pint.json @@ -0,0 +1,3 @@ +{ + "preset": "psr12" +} \ No newline at end of file diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index ba48872..22a8176 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -68,7 +68,7 @@ public function __construct( $this->password = $password; $this->secure = $secure; - $this->client = new Client; + $this->client = new Client(); } /** @@ -134,7 +134,7 @@ private function validateIdentifier(string $identifier, string $type = 'Identifi */ private function escapeIdentifier(string $identifier): string { - return '`'.str_replace('`', '``', $identifier).'`'; + return '`' . str_replace('`', '``', $identifier) . '`'; } /** @@ -193,15 +193,27 @@ private function query(string $sql, array $params = []): string // Replace parameters in SQL foreach ($params as $key => $value) { - $placeholder = ":{$key}"; - if (is_string($value)) { - $escapedValue = "'".$this->escapeString($value)."'"; + if (is_int($value) || is_float($value)) { + // Numeric values should not be quoted + $strValue = (string) $value; + } elseif (is_string($value)) { + $strValue = "'" . $this->escapeString($value) . "'"; } elseif (is_null($value)) { - $escapedValue = 'NULL'; + $strValue = 'NULL'; + } elseif (is_bool($value)) { + $strValue = $value ? '1' : '0'; + } elseif (is_array($value)) { + $encoded = json_encode($value); + if (is_string($encoded)) { + $strValue = "'" . $this->escapeString($encoded) . "'"; + } else { + $strValue = 'NULL'; + } } else { - $escapedValue = (string) $value; + /** @var scalar $value */ + $strValue = "'" . $this->escapeString((string) $value) . "'"; } - $sql = str_replace($placeholder, $escapedValue, $sql); + $sql = str_replace(":{$key}", $strValue, $sql); } // Set authentication headers @@ -263,12 +275,12 @@ public function setup(): void 'tags String', // JSON string ]; - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); // Create table with MergeTree engine for optimal performance $createTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( - ".implode(",\n ", $columns).', + " . implode(",\n ", $columns) . ', INDEX idx_metric metric TYPE bloom_filter GRANULARITY 1, INDEX idx_period period TYPE bloom_filter GRANULARITY 1 ) @@ -291,18 +303,18 @@ public function setup(): void public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool { if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); } $id = uniqid('', true); - $now = new \DateTime; + $now = new \DateTime(); $time = $now->format(self::PERIODS[$period]); // Format timestamp for ClickHouse DateTime64(3) $microtime = microtime(true); - $timestamp = date('Y-m-d H:i:s', (int) $microtime).'.'.sprintf('%03d', ($microtime - floor($microtime)) * 1000); + $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); $sql = " INSERT INTO {$escapedDatabaseAndTable} @@ -347,30 +359,35 @@ public function logBatch(array $metrics): bool $period = $metricData['period'] ?? '1h'; if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); } $id = uniqid('', true); $microtime = microtime(true); - $timestamp = date('Y-m-d H:i:s', (int) $microtime).'.'.sprintf('%03d', ($microtime - floor($microtime)) * 1000); + $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); + + $metric = $metricData['metric']; + $value = $metricData['value']; + assert(is_string($metric)); + assert(is_int($value)); $values[] = sprintf( "('%s', '%s', %d, '%s', '%s', '%s')", $id, - $this->escapeString((string) $metricData['metric']), - (int) $metricData['value'], + $this->escapeString($metric), + $value, $this->escapeString($period), $timestamp, $this->escapeString((string) json_encode($metricData['tags'] ?? [])) ); } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); $insertSql = " INSERT INTO {$escapedDatabaseAndTable} (id, metric, value, period, time, tags) - VALUES ".implode(', ', $values); + VALUES " . implode(', ', $values); $this->query($insertSql); @@ -402,12 +419,12 @@ private function parseResults(string $result): array } $documents[] = new Document([ - '$id' => $columns[0], - 'metric' => $columns[1], + '$id' => (string) $columns[0], + 'metric' => (string) $columns[1], 'value' => (int) $columns[2], - 'period' => $columns[3], - 'time' => $columns[4], - 'tags' => json_decode($columns[5], true) ?? [], + 'period' => (string) $columns[3], + 'time' => (string) $columns[4], + 'tags' => json_decode((string) $columns[5], true) ?? [], ]); } @@ -437,7 +454,7 @@ public function getByPeriod(string $metric, string $period, array $queries = []) } } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); $sql = " SELECT id, metric, value, period, time, tags @@ -481,7 +498,7 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa } } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); $sql = " SELECT id, metric, value, period, time, tags @@ -512,7 +529,7 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa */ public function countByPeriod(string $metric, string $period, array $queries = []): int { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); $sql = " SELECT count() as count @@ -538,7 +555,7 @@ public function countByPeriod(string $metric, string $period, array $queries = [ */ public function sumByPeriod(string $metric, string $period, array $queries = []): int { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); $sql = " SELECT sum(value) as total @@ -564,7 +581,7 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) */ public function purge(string $datetime): bool { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database).'.'.$this->escapeIdentifier($this->table); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); $sql = " DELETE FROM {$escapedDatabaseAndTable} diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 1bdda85..3297a35 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -143,7 +143,7 @@ public function log(string $metric, int $value, string $period = '1h', array $ta throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); } - $now = new \DateTime; + $now = new \DateTime(); $time = $period === 'inf' ? '1000-01-01 00:00:00' : $now->format(self::PERIODS[$period]); @@ -172,7 +172,7 @@ public function logBatch(array $metrics): bool throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); } - $now = new \DateTime; + $now = new \DateTime(); $time = $period === 'inf' ? '1000-01-01 00:00:00' : $now->format(self::PERIODS[$period]); diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 1bdcd51..3dccf43 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -59,8 +59,8 @@ public function log(string $metric, int $value, string $period = '1h', array $ta /** * Log multiple usage metrics in batch. * - * @param array> $metrics - * + * @param array}> $metrics + * @return bool * @throws \Exception */ public function logBatch(array $metrics): bool @@ -71,7 +71,7 @@ public function logBatch(array $metrics): bool /** * Get usage metrics by period. * - * @param array $queries + * @param array<\Utopia\Database\Query> $queries * @return array * * @throws \Exception @@ -84,7 +84,7 @@ public function getByPeriod(string $metric, string $period, array $queries = []) /** * Get usage metrics between dates. * - * @param array $queries + * @param array<\Utopia\Database\Query> $queries * @return array * * @throws \Exception @@ -97,7 +97,7 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa /** * Count usage metrics by period. * - * @param array $queries + * @param array<\Utopia\Database\Query> $queries * * @throws \Exception */ @@ -109,7 +109,7 @@ public function countByPeriod(string $metric, string $period, array $queries = [ /** * Sum usage metric values by period. * - * @param array $queries + * @param array<\Utopia\Database\Query> $queries * * @throws \Exception */ diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 8482a2e..ee4309f 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -13,14 +13,16 @@ class ClickHouseTest extends TestCase protected function initializeUsage(): void { - $host = getenv('CLICKHOUSE_HOST') ?: 'clickhouse'; + $host = getenv('CLICKHOUSE_HOST'); $username = getenv('CLICKHOUSE_USER') ?: 'default'; $password = getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse'; $port = (int) (getenv('CLICKHOUSE_PORT') ?: 8123); $secure = (bool) (getenv('CLICKHOUSE_SECURE') ?: false); - if ($host === null || $host === '') { - $this->markTestSkipped('CLICKHOUSE_HOST not set; skipping ClickHouse adapter tests.'); + $enable = getenv('CLICKHOUSE_ENABLE_TESTS'); + + if ($enable !== '1' || $host === false || $host === '') { + $this->markTestSkipped('ClickHouse tests disabled (set CLICKHOUSE_ENABLE_TESTS=1 and CLICKHOUSE_HOST to run).'); } $adapter = new ClickHouseAdapter($host, $username, $password, $port, $secure); diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index f5ebf45..d6f3cae 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -27,7 +27,7 @@ protected function initializeUsage(): void $dbPass = 'password'; $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPdoAttributes()); - $cache = new Cache(new NoCache); + $cache = new Cache(new NoCache()); $this->database = new Database(new MariaDB($pdo), $cache); $this->database->setDatabase('utopiaTests'); $this->database->setNamespace('utopia_usage'); diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index 0d25ad4..03ac29e 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -83,7 +83,7 @@ public function testGetByPeriod(): void public function testGetBetweenDates(): void { - $start = DateTime::addSeconds(new \DateTime, -3600); // 1 hour ago + $start = DateTime::addSeconds(new \DateTime(), -3600); // 1 hour ago $end = DateTime::now(); $results = $this->usage->getBetweenDates('requests', $start, $end); From 9de887839c2a2cac67a405781722dbaee51ac12f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:18:32 +0000 Subject: [PATCH 07/44] update headers --- src/Usage/Adapter/ClickHouse.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 22a8176..9e158fd 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -69,6 +69,9 @@ public function __construct( $this->secure = $secure; $this->client = new Client(); + $this->client->addHeader('X-ClickHouse-User', $this->username); + $this->client->addHeader('X-ClickHouse-Key', $this->password); + $this->client->addHeader('X-ClickHouse-Database', $this->database); } /** @@ -160,6 +163,7 @@ public function setDatabase(string $database): self { $this->validateIdentifier($database, 'Database'); $this->database = $database; + $this->client->addHeader('X-ClickHouse-Database', $this->database); return $this; } @@ -216,11 +220,6 @@ private function query(string $sql, array $params = []): string $sql = str_replace(":{$key}", $strValue, $sql); } - // Set authentication headers - $this->client->addHeader('X-ClickHouse-User', $this->username); - $this->client->addHeader('X-ClickHouse-Key', $this->password); - $this->client->addHeader('X-ClickHouse-Database', $this->database); - try { $response = $this->client->fetch( url: $url, From 5303bf32d62780d3b74cde4e1bcb5b049563e00e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:23:54 +0000 Subject: [PATCH 08/44] support tenant --- src/Usage/Adapter/ClickHouse.php | 191 +++++++++++++++++++++++++------ 1 file changed, 157 insertions(+), 34 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 9e158fd..11daad0 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -45,6 +45,10 @@ class ClickHouse extends Adapter private Client $client; + protected ?int $tenant = null; + + protected bool $sharedTables = false; + /** * @param string $host ClickHouse host * @param string $username ClickHouse username (default: 'default') @@ -181,6 +185,52 @@ public function setTable(string $table): self return $this; } + /** + * Set the tenant ID for multi-tenant support. + * Tenant is used to isolate usage metrics by tenant. + * + * @param int|null $tenant + * @return self + */ + public function setTenant(?int $tenant): self + { + $this->tenant = $tenant; + return $this; + } + + /** + * Get the tenant ID. + * + * @return int|null + */ + public function getTenant(): ?int + { + return $this->tenant; + } + + /** + * Set whether tables are shared across tenants. + * When enabled, a tenant column is added to the table for data isolation. + * + * @param bool $sharedTables + * @return self + */ + public function setSharedTables(bool $sharedTables): self + { + $this->sharedTables = $sharedTables; + return $this; + } + + /** + * Get whether tables are shared across tenants. + * + * @return bool + */ + public function isSharedTables(): bool + { + return $this->sharedTables; + } + /** * Execute a ClickHouse query via HTTP interface. * @@ -274,6 +324,11 @@ public function setup(): void 'tags String', // JSON string ]; + // Add tenant column only if tables are shared across tenants + if ($this->sharedTables) { + $columns[] = 'tenant Nullable(UInt64)'; + } + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); // Create table with MergeTree engine for optimal performance @@ -315,27 +370,34 @@ public function log(string $metric, int $value, string $period = '1h', array $ta $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); - $sql = " - INSERT INTO {$escapedDatabaseAndTable} - (id, metric, value, period, time, tags) - VALUES ( - :id, - :metric, - :value, - :period, - :time, - :tags - ) - "; + // Build column list and values based on sharedTables setting + $columns = ['id', 'metric', 'value', 'period', 'time', 'tags']; + $placeholders = [':id', ':metric', ':value', ':period', ':time', ':tags']; - $this->query($sql, [ + $params = [ 'id' => $id, 'metric' => $metric, 'value' => $value, 'period' => $period, 'time' => $timestamp, 'tags' => json_encode($tags), - ]); + ]; + + if ($this->sharedTables) { + $columns[] = 'tenant'; + $placeholders[] = ':tenant'; + $params['tenant'] = $this->tenant; + } + + $sql = " + INSERT INTO {$escapedDatabaseAndTable} + (" . implode(', ', $columns) . ") + VALUES ( + " . implode(", ", $placeholders) . " + ) + "; + + $this->query($sql, $params); return true; } @@ -370,22 +432,42 @@ public function logBatch(array $metrics): bool assert(is_string($metric)); assert(is_int($value)); - $values[] = sprintf( - "('%s', '%s', %d, '%s', '%s', '%s')", - $id, - $this->escapeString($metric), - $value, - $this->escapeString($period), - $timestamp, - $this->escapeString((string) json_encode($metricData['tags'] ?? [])) - ); + if ($this->sharedTables) { + $tenant = $this->tenant !== null ? (int) $this->tenant : 'NULL'; + $values[] = sprintf( + "('%s', '%s', %d, '%s', '%s', '%s', %s)", + $id, + $this->escapeString($metric), + $value, + $this->escapeString($period), + $timestamp, + $this->escapeString((string) json_encode($metricData['tags'] ?? [])), + $tenant + ); + } else { + $values[] = sprintf( + "('%s', '%s', %d, '%s', '%s', '%s')", + $id, + $this->escapeString($metric), + $value, + $this->escapeString($period), + $timestamp, + $this->escapeString((string) json_encode($metricData['tags'] ?? [])) + ); + } } $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + // Build column list based on sharedTables setting + $columns = 'id, metric, value, period, time, tags'; + if ($this->sharedTables) { + $columns .= ', tenant'; + } + $insertSql = " INSERT INTO {$escapedDatabaseAndTable} - (id, metric, value, period, time, tags) + ({$columns}) VALUES " . implode(', ', $values); $this->query($insertSql); @@ -413,23 +495,59 @@ private function parseResults(string $result): array } $columns = explode("\t", $line); - if (count($columns) < 6) { + $expectedColumns = $this->sharedTables ? 7 : 6; + if (count($columns) < $expectedColumns) { continue; } - $documents[] = new Document([ + $document = [ '$id' => (string) $columns[0], 'metric' => (string) $columns[1], 'value' => (int) $columns[2], 'period' => (string) $columns[3], 'time' => (string) $columns[4], 'tags' => json_decode((string) $columns[5], true) ?? [], - ]); + ]; + + // Add tenant only if sharedTables is enabled + if ($this->sharedTables && isset($columns[6])) { + $document['tenant'] = $columns[6] === '\\N' ? null : (int) $columns[6]; + } + + $documents[] = new Document($document); } return $documents; } + /** + * Get the SELECT column list for queries. + * Returns 6 columns if not using shared tables, 7 if using shared tables. + * + * @return string + */ + private function getSelectColumns(): string + { + if ($this->sharedTables) { + return 'id, metric, value, period, time, tags, tenant'; + } + return 'id, metric, value, period, time, tags'; + } + + /** + * Build tenant filter clause based on current tenant context. + * + * @return string + */ + private function getTenantFilter(): string + { + if (!$this->sharedTables || $this->tenant === null) { + return ''; + } + + return " AND tenant = {$this->tenant}"; + } + /** * Get usage metrics by period. * @@ -454,11 +572,12 @@ public function getByPeriod(string $metric, string $period, array $queries = []) } $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tenantFilter = $this->getTenantFilter(); $sql = " - SELECT id, metric, value, period, time, tags + SELECT " . $this->getSelectColumns() . " FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND period = :period + WHERE metric = :metric AND period = :period{$tenantFilter} ORDER BY time DESC LIMIT :limit OFFSET :offset FORMAT TabSeparated @@ -498,11 +617,12 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa } $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tenantFilter = $this->getTenantFilter(); $sql = " - SELECT id, metric, value, period, time, tags + SELECT " . $this->getSelectColumns() . " FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND time >= :startDate AND time <= :endDate + WHERE metric = :metric AND time >= :startDate AND time <= :endDate{$tenantFilter} ORDER BY time DESC LIMIT :limit OFFSET :offset FORMAT TabSeparated @@ -529,11 +649,12 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa public function countByPeriod(string $metric, string $period, array $queries = []): int { $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tenantFilter = $this->getTenantFilter(); $sql = " SELECT count() as count FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND period = :period + WHERE metric = :metric AND period = :period{$tenantFilter} FORMAT TabSeparated "; @@ -555,11 +676,12 @@ public function countByPeriod(string $metric, string $period, array $queries = [ public function sumByPeriod(string $metric, string $period, array $queries = []): int { $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tenantFilter = $this->getTenantFilter(); $sql = " SELECT sum(value) as total FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND period = :period + WHERE metric = :metric AND period = :period{$tenantFilter} FORMAT TabSeparated "; @@ -581,10 +703,11 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) public function purge(string $datetime): bool { $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tenantFilter = $this->getTenantFilter(); $sql = " DELETE FROM {$escapedDatabaseAndTable} - WHERE time < :datetime + WHERE time < :datetime{$tenantFilter} "; $this->query($sql, ['datetime' => $datetime]); From 6b0bed7d8c2def84dae0c4164eb123ac29a71c47 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:41:14 +0000 Subject: [PATCH 09/44] Refactor and new return object --- src/Usage/Adapter.php | 12 +- src/Usage/Adapter/ClickHouse.php | 33 ++-- src/Usage/Adapter/Database.php | 114 ++---------- src/Usage/Metric.php | 300 +++++++++++++++++++++++++++++++ src/Usage/Usage.php | 43 +++-- 5 files changed, 369 insertions(+), 133 deletions(-) create mode 100644 src/Usage/Metric.php diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index da9461d..84f9249 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -2,8 +2,6 @@ namespace Utopia\Usage; -use Utopia\Database\Document; - abstract class Adapter { /** @@ -13,8 +11,12 @@ abstract public function getName(): string; /** * Setup database structure + * + * @param string $table Table name + * @param array> $columns Column definitions + * @param array> $indexes Index definitions */ - abstract public function setup(): void; + abstract public function setup(string $table, array $columns, array $indexes): void; /** * Log usage metric @@ -34,7 +36,7 @@ abstract public function logBatch(array $metrics): bool; * Get usage metrics by period * * @param array<\Utopia\Database\Query> $queries - * @return array + * @return array */ abstract public function getByPeriod(string $metric, string $period, array $queries = []): array; @@ -42,7 +44,7 @@ abstract public function getByPeriod(string $metric, string $period, array $quer * Get usage metrics between dates * * @param array<\Utopia\Database\Query> $queries - * @return array + * @return array */ abstract public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array; diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 11daad0..0c83d62 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -3,9 +3,9 @@ namespace Utopia\Usage\Adapter; use Exception; -use Utopia\Database\Document; use Utopia\Fetch\Client; use Utopia\Usage\Adapter; +use Utopia\Usage\Metric; /** * ClickHouse Adapter for Usage @@ -305,17 +305,22 @@ public function getName(): string * * Creates the database and table if they don't exist. * + * @param string $table Table name + * @param array> $columns Column definitions (not used - ClickHouse uses hardcoded schema) + * @param array> $indexes Index definitions (not used - ClickHouse uses hardcoded indexes) * @throws Exception */ - public function setup(): void + public function setup(string $table, array $columns, array $indexes): void { + $this->setTable($table); + // Create database if not exists $escapedDatabase = $this->escapeIdentifier($this->database); $createDbSql = "CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"; $this->query($createDbSql); // Build column definitions - $columns = [ + $columnDefs = [ 'id String', 'metric String', 'value Int64', @@ -326,7 +331,7 @@ public function setup(): void // Add tenant column only if tables are shared across tenants if ($this->sharedTables) { - $columns[] = 'tenant Nullable(UInt64)'; + $columnDefs[] = 'tenant Nullable(UInt64)'; } $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); @@ -334,7 +339,7 @@ public function setup(): void // Create table with MergeTree engine for optimal performance $createTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( - " . implode(",\n ", $columns) . ', + " . implode(",\n ", $columnDefs) . ', INDEX idx_metric metric TYPE bloom_filter GRANULARITY 1, INDEX idx_period period TYPE bloom_filter GRANULARITY 1 ) @@ -476,9 +481,9 @@ public function logBatch(array $metrics): bool } /** - * Parse ClickHouse TabSeparated results into Document array. + * Parse ClickHouse TabSeparated results into Metric array. * - * @return array + * @return array */ private function parseResults(string $result): array { @@ -487,7 +492,7 @@ private function parseResults(string $result): array } $lines = explode("\n", trim($result)); - $documents = []; + $metrics = []; foreach ($lines as $line) { if (empty(trim($line))) { @@ -500,7 +505,7 @@ private function parseResults(string $result): array continue; } - $document = [ + $data = [ '$id' => (string) $columns[0], 'metric' => (string) $columns[1], 'value' => (int) $columns[2], @@ -511,13 +516,13 @@ private function parseResults(string $result): array // Add tenant only if sharedTables is enabled if ($this->sharedTables && isset($columns[6])) { - $document['tenant'] = $columns[6] === '\\N' ? null : (int) $columns[6]; + $data['tenant'] = $columns[6] === '\\\\N' ? null : (int) $columns[6]; } - $documents[] = new Document($document); + $metrics[] = new Metric($data); } - return $documents; + return $metrics; } /** @@ -552,7 +557,7 @@ private function getTenantFilter(): string * Get usage metrics by period. * * @param array $queries - * @return array + * @return array * * @throws Exception */ @@ -597,7 +602,7 @@ public function getByPeriod(string $metric, string $period, array $queries = []) * Get usage metrics between dates. * * @param array $queries - * @return array + * @return array * * @throws Exception */ diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 3297a35..9c5f205 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -8,10 +8,11 @@ use Utopia\Database\Query; use Utopia\Exception; use Utopia\Usage\Adapter; +use Utopia\Usage\Metric; class Database extends Adapter { - public const COLLECTION = 'usage'; + protected string $collection = 'usage_metrics'; /** @var array */ public const PERIODS = [ @@ -20,86 +21,6 @@ class Database extends Adapter 'inf' => '0000-00-00 00:00', ]; - public const ATTRIBUTES = [ - [ - '$id' => 'metric', - 'type' => UtopiaDatabase::VAR_STRING, - 'size' => 255, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'value', - 'type' => UtopiaDatabase::VAR_INTEGER, - 'size' => 0, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'period', - 'type' => UtopiaDatabase::VAR_STRING, - 'size' => 16, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'time', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => 'tags', - 'type' => UtopiaDatabase::VAR_STRING, - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], - ]; - - public const INDEXES = [ - [ - '$id' => 'index-metric', - 'type' => UtopiaDatabase::INDEX_KEY, - 'attributes' => ['metric'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-period', - 'type' => UtopiaDatabase::INDEX_KEY, - 'attributes' => ['period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-metric-period', - 'type' => UtopiaDatabase::INDEX_KEY, - 'attributes' => ['metric', 'period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-time', - 'type' => UtopiaDatabase::INDEX_KEY, - 'attributes' => ['time'], - 'lengths' => [], - 'orders' => [UtopiaDatabase::ORDER_DESC], - ], - ]; - private UtopiaDatabase $db; public function __construct(UtopiaDatabase $db) @@ -112,23 +33,24 @@ public function getName(): string return 'Database'; } - public function setup(): void + public function setup(string $table, array $columns, array $indexes): void { + $this->collection = $table; if (! $this->db->exists($this->db->getDatabase())) { throw new Exception('You need to create the database before running Usage setup'); } $attributes = \array_map(function ($attribute) { return new Document($attribute); - }, self::ATTRIBUTES); + }, $columns); $indexes = \array_map(function ($index) { return new Document($index); - }, self::INDEXES); + }, $indexes); try { $this->db->createCollection( - self::COLLECTION, + $table, $attributes, $indexes ); @@ -140,7 +62,7 @@ public function setup(): void public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool { if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); } $now = new \DateTime(); @@ -149,7 +71,7 @@ public function log(string $metric, int $value, string $period = '1h', array $ta : $now->format(self::PERIODS[$period]); $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags) { - $this->db->createDocument(self::COLLECTION, new Document([ + $this->db->createDocument($this->collection, new Document([ '$permissions' => [], 'metric' => $metric, 'value' => $value, @@ -169,7 +91,7 @@ public function logBatch(array $metrics): bool $period = $metric['period'] ?? '1h'; if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: '.implode(', ', array_keys(self::PERIODS))); + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); } $now = new \DateTime(); @@ -187,7 +109,7 @@ public function logBatch(array $metrics): bool ]); }, $metrics); - $this->db->createDocuments(self::COLLECTION, $documents); + $this->db->createDocuments($this->collection, $documents); }); return true; @@ -202,12 +124,12 @@ public function getByPeriod(string $metric, string $period, array $queries = []) $queries[] = Query::orderDesc(); return $this->db->find( - collection: self::COLLECTION, + collection: $this->collection, queries: $queries, ); }); - return $result; + return \array_map(fn ($doc) => new Metric($doc->getArrayCopy()), $result); } public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array @@ -220,12 +142,12 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa $queries[] = Query::orderDesc(); return $this->db->find( - collection: self::COLLECTION, + collection: $this->collection, queries: $queries, ); }); - return $result; + return \array_map(fn ($doc) => new Metric($doc->getArrayCopy()), $result); } public function countByPeriod(string $metric, string $period, array $queries = []): int @@ -233,7 +155,7 @@ public function countByPeriod(string $metric, string $period, array $queries = [ /** @var int $count */ $count = $this->db->getAuthorization()->skip(function () use ($queries, $metric, $period) { return $this->db->count( - collection: self::COLLECTION, + collection: $this->collection, queries: [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), @@ -263,7 +185,7 @@ public function purge(string $datetime): bool $this->db->getAuthorization()->skip(function () use ($datetime) { do { $documents = $this->db->find( - collection: self::COLLECTION, + collection: $this->collection, queries: [ Query::lessThan('time', $datetime), Query::limit(100), @@ -271,7 +193,7 @@ public function purge(string $datetime): bool ); foreach ($documents as $document) { - $this->db->deleteDocument(self::COLLECTION, $document->getId()); + $this->db->deleteDocument($this->collection, $document->getId()); } } while (! empty($documents)); }); diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php new file mode 100644 index 0000000..c04e6a2 --- /dev/null +++ b/src/Usage/Metric.php @@ -0,0 +1,300 @@ + 'unique-id', + * 'metric' => 'bandwidth', + * 'value' => 1024, + * 'period' => '1h', + * 'time' => '2025-12-09 10:00:00', + * 'tags' => ['region' => 'us-east', 'project' => 'my-app'] + * ]); + * + * echo $metric->getMetric(); // 'bandwidth' + * echo $metric->getValue(); // 1024 + * ``` + * + * @extends ArrayObject + */ +class Metric extends ArrayObject +{ + /** + * Construct a new metric object. + * + * Initializes the metric with the provided data array. + * The array can contain any attributes, but common ones include: + * - $id: Unique identifier for the metric + * - metric: Name/type of the metric being tracked + * - value: Numeric value of the metric + * - period: Time period (1h, 1d, inf) + * - time: Timestamp when the metric was recorded + * - tags: Additional metadata as key-value pairs + * - tenant: Tenant ID for multi-tenant environments + * + * @param array $input Metric data + */ + public function __construct(array $input = []) + { + parent::__construct($input); + } + + /** + * Get metric ID. + * + * Returns the unique identifier for this metric entry. + * This is typically a UUID or auto-generated ID from the storage backend. + * + * @return string The metric ID, or empty string if not set + */ + public function getId(): string + { + $id = $this->getAttribute('$id', ''); + return is_string($id) ? $id : ''; + } + + /** + * Get metric name. + * + * Returns the name or type of metric being tracked. + * Examples: 'bandwidth', 'requests', 'storage', 'executions' + * + * @return string The metric name, or empty string if not set + */ + public function getMetric(): string + { + $metric = $this->getAttribute('metric', ''); + return is_string($metric) ? $metric : ''; + } + + /** + * Get metric value. + * + * Returns the numeric value associated with this metric. + * For example, number of requests, bytes transferred, or execution count. + * + * @param int|null $default Default value to return if not set + * @return int|null The metric value, or the default if not set or invalid + */ + public function getValue(?int $default = null): ?int + { + $value = $this->getAttribute('value', $default ?? 0); + return is_int($value) ? $value : $default; + } + + /** + * Get time period. + * + * Returns the aggregation period for this metric. + * Common values: + * - '1h': Hourly aggregation + * - '1d': Daily aggregation + * - 'inf': Infinite/lifetime aggregation + * + * @return string The period identifier, defaults to '1h' + */ + public function getPeriod(): string + { + $period = $this->getAttribute('period', '1h'); + return is_string($period) ? $period : '1h'; + } + + /** + * Get timestamp. + * + * Returns the timestamp when this metric was recorded or the + * aggregation period start time. Format depends on the storage backend, + * typically ISO 8601 or database datetime format. + * + * @return string|null The timestamp string, or null if not set + */ + public function getTime(): ?string + { + $time = $this->getAttribute('time', null); + return is_string($time) ? $time : null; + } + + /** + * Get tags. + * + * Returns additional metadata associated with this metric as key-value pairs. + * Tags are useful for filtering, grouping, and contextualizing metrics. + * + * Common tag examples: + * - region: Geographic region (us-east, eu-west) + * - project: Project or application identifier + * - environment: dev, staging, production + * - resource: Specific resource being measured + * + * @return array Associative array of tags + */ + public function getTags(): array + { + $tags = $this->getAttribute('tags', []); + return is_array($tags) ? $tags : []; + } + + /** + * Get tenant ID. + * + * Returns the tenant identifier when using shared tables in multi-tenant + * architectures. This allows data isolation at the application level while + * sharing the same database tables. + * + * @return int|null The tenant ID, or null if not set or not using multi-tenancy + */ + public function getTenant(): ?int + { + $tenant = $this->getAttribute('tenant'); + + if ($tenant === null) { + return null; + } + + if (is_int($tenant)) { + return $tenant; + } + + if (is_numeric($tenant)) { + return (int) $tenant; + } + + return null; + } + + /** + * Get all attributes. + * + * Returns all metric data as an associative array. + * This includes both standard fields (id, metric, value, etc.) and + * any custom attributes that were set on the metric. + * + * @return array All metric attributes + */ + public function getAttributes(): array + { + $attributes = []; + + foreach ($this as $key => $value) { + $attributes[$key] = $value; + } + + return $attributes; + } + + /** + * Get a specific attribute. + * + * Retrieves the value of a named attribute. If the attribute doesn't exist, + * returns the provided default value. + * + * This is a generic accessor - prefer using the type-safe getters + * (getId(), getMetric(), etc.) for standard attributes. + * + * @param string $name The attribute name to retrieve + * @param mixed $default Default value if attribute is not set + * @return mixed The attribute value or default + */ + public function getAttribute(string $name, mixed $default = null): mixed + { + if (isset($this[$name])) { + return $this[$name]; + } + + return $default; + } + + /** + * Set a specific attribute. + * + * Sets or updates the value of a named attribute. + * Returns the metric instance for method chaining. + * + * Example: + * ```php + * $metric->setAttribute('custom', 'value') + * ->setAttribute('another', 123); + * ``` + * + * @param string $key The attribute name + * @param mixed $value The attribute value + * @return static This metric instance for chaining + */ + public function setAttribute(string $key, mixed $value): static + { + $this[$key] = $value; + + return $this; + } + + /** + * Check if an attribute exists. + * + * Determines whether a named attribute is set on this metric, + * regardless of its value (including null). + * + * @param string $name The attribute name to check + * @return bool True if the attribute exists, false otherwise + */ + public function hasAttribute(string $name): bool + { + return isset($this[$name]); + } + + /** + * Remove an attribute. + * + * Removes a named attribute from the metric. + * Returns the metric instance for method chaining. + * + * @param string $name The attribute name to remove + * @return static This metric instance for chaining + */ + public function removeAttribute(string $name): static + { + unset($this[$name]); + + /** @var static */ + return $this; + } + + /** + * Check if the metric is empty. + * + * A metric is considered empty if it has no ID set. + * This is useful for checking if a query returned valid results. + * + * @return bool True if the metric has no ID, false otherwise + */ + public function isEmpty(): bool + { + return empty($this->getId()); + } + + /** + * Convert to array. + * + * Returns a plain PHP array representation of the metric. + * This is useful for serialization, JSON encoding, or passing + * to functions that expect arrays. + * + * @return array Array representation of the metric + */ + public function toArray(): array + { + return $this->getArrayCopy(); + } +} diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 3dccf43..9a59961 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -2,10 +2,6 @@ namespace Utopia\Usage; -use Utopia\Database\Database; -use Utopia\Database\Document; -use Utopia\Usage\Adapter\ClickHouse; - /** * Usage Metrics Manager * @@ -37,11 +33,22 @@ public function getAdapter(): Adapter /** * Setup the usage metrics storage. * + * @param string $table Table name for storing usage metrics + * @param array> $columns Column definitions + * @param array> $indexes Index definitions * @throws \Exception */ - public function setup(): void + public function setup(string $table = 'usage', array $columns = [], array $indexes = []): void { - $this->adapter->setup(); + // Use legacy constants if no columns/indexes provided (for backward compatibility) + if (empty($columns)) { + $columns = self::ATTRIBUTES; + } + if (empty($indexes)) { + $indexes = self::INDEXES; + } + + $this->adapter->setup($table, $columns, $indexes); } /** @@ -72,7 +79,7 @@ public function logBatch(array $metrics): bool * Get usage metrics by period. * * @param array<\Utopia\Database\Query> $queries - * @return array + * @return array * * @throws \Exception */ @@ -85,7 +92,7 @@ public function getByPeriod(string $metric, string $period, array $queries = []) * Get usage metrics between dates. * * @param array<\Utopia\Database\Query> $queries - * @return array + * @return array * * @throws \Exception */ @@ -152,7 +159,7 @@ public function purge(string $datetime): bool public const ATTRIBUTES = [ [ '$id' => 'metric', - 'type' => \Utopia\Database\Database::VAR_STRING, + 'type' => 'string', 'size' => 255, 'required' => true, 'signed' => true, @@ -161,7 +168,7 @@ public function purge(string $datetime): bool ], [ '$id' => 'value', - 'type' => \Utopia\Database\Database::VAR_INTEGER, + 'type' => 'integer', 'size' => 0, 'required' => true, 'signed' => true, @@ -170,7 +177,7 @@ public function purge(string $datetime): bool ], [ '$id' => 'period', - 'type' => \Utopia\Database\Database::VAR_STRING, + 'type' => 'string', 'size' => 16, 'required' => true, 'signed' => true, @@ -179,7 +186,7 @@ public function purge(string $datetime): bool ], [ '$id' => 'time', - 'type' => \Utopia\Database\Database::VAR_DATETIME, + 'type' => 'datetime', 'format' => '', 'size' => 0, 'signed' => true, @@ -189,7 +196,7 @@ public function purge(string $datetime): bool ], [ '$id' => 'tags', - 'type' => \Utopia\Database\Database::VAR_STRING, + 'type' => 'string', 'size' => 16777216, 'required' => false, 'signed' => true, @@ -204,31 +211,31 @@ public function purge(string $datetime): bool public const INDEXES = [ [ '$id' => 'index-metric', - 'type' => \Utopia\Database\Database::INDEX_KEY, + 'type' => 'key', 'attributes' => ['metric'], 'lengths' => [], 'orders' => [], ], [ '$id' => 'index-period', - 'type' => \Utopia\Database\Database::INDEX_KEY, + 'type' => 'key', 'attributes' => ['period'], 'lengths' => [], 'orders' => [], ], [ '$id' => 'index-metric-period', - 'type' => \Utopia\Database\Database::INDEX_KEY, + 'type' => 'key', 'attributes' => ['metric', 'period'], 'lengths' => [], 'orders' => [], ], [ '$id' => 'index-time', - 'type' => \Utopia\Database\Database::INDEX_KEY, + 'type' => 'key', 'attributes' => ['time'], 'lengths' => [], - 'orders' => [\Utopia\Database\Database::ORDER_DESC], + 'orders' => ['desc'], ], ]; } From 364d6273eef5cca8bb31bea061aedf253899fd5e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 01:51:07 +0000 Subject: [PATCH 10/44] namespace and tenant index --- src/Usage/Adapter/ClickHouse.php | 193 ++++++++++++++++++++++++++----- 1 file changed, 165 insertions(+), 28 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 0c83d62..05a5248 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -49,6 +49,8 @@ class ClickHouse extends Adapter protected bool $sharedTables = false; + protected string $namespace = ''; + /** * @param string $host ClickHouse host * @param string $username ClickHouse username (default: 'default') @@ -231,6 +233,50 @@ public function isSharedTables(): bool return $this->sharedTables; } + /** + * Set the namespace for multi-project support. + * Namespace is used as a prefix for table names. + * + * @param string $namespace + * @return self + * @throws Exception + */ + public function setNamespace(string $namespace): self + { + if (!empty($namespace)) { + $this->validateIdentifier($namespace, 'Namespace'); + } + $this->namespace = $namespace; + return $this; + } + + /** + * Get the namespace. + * + * @return string + */ + public function getNamespace(): string + { + return $this->namespace; + } + + /** + * Get the table name with namespace prefix. + * Namespace is used to isolate tables for different projects/applications. + * + * @return string + */ + private function getTableName(): string + { + $tableName = $this->table; + + if (!empty($this->namespace)) { + $tableName = $this->namespace . '_' . $tableName; + } + + return $tableName; + } + /** * Execute a ClickHouse query via HTTP interface. * @@ -304,10 +350,11 @@ public function getName(): string * Setup ClickHouse table structure. * * Creates the database and table if they don't exist. + * Uses the provided column definitions and adds internal fields (_id, _createdAt, _updatedAt, tenant). * * @param string $table Table name - * @param array> $columns Column definitions (not used - ClickHouse uses hardcoded schema) - * @param array> $indexes Index definitions (not used - ClickHouse uses hardcoded indexes) + * @param array> $columns Column definitions from the application + * @param array> $indexes Index definitions from the application * @throws Exception */ public function setup(string $table, array $columns, array $indexes): void @@ -319,39 +366,122 @@ public function setup(string $table, array $columns, array $indexes): void $createDbSql = "CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"; $this->query($createDbSql); - // Build column definitions - $columnDefs = [ - 'id String', - 'metric String', - 'value Int64', - 'period String', - 'time DateTime64(3)', - 'tags String', // JSON string - ]; + // Track which internal fields are already present + $hasId = false; + $hasCreatedAt = false; + $hasUpdatedAt = false; + $hasTenant = false; + + // Build column definitions from provided columns + $columnDefs = []; + foreach ($columns as $column) { + $columnId = $column['$id'] ?? ''; + + if ($columnId === '_id' || $columnId === '$id') { + $hasId = true; + } elseif ($columnId === '_createdAt' || $columnId === '$createdAt') { + $hasCreatedAt = true; + } elseif ($columnId === '_updatedAt' || $columnId === '$updatedAt') { + $hasUpdatedAt = true; + } elseif ($columnId === 'tenant' || $columnId === '$tenant') { + $hasTenant = true; + } - // Add tenant column only if tables are shared across tenants - if ($this->sharedTables) { + $columnDefs[] = $this->getClickHouseColumnDefinition($column); + } + + // Add internal fields if not present + if (! $hasId) { + array_unshift($columnDefs, '_id String'); + } + if (! $hasCreatedAt) { + $columnDefs[] = '_createdAt DateTime64(3) DEFAULT now64(3)'; + } + if (! $hasUpdatedAt) { + $columnDefs[] = '_updatedAt DateTime64(3) DEFAULT now64(3)'; + } + + // Add tenant column only if tables are shared across tenants and not already present + if ($this->sharedTables && ! $hasTenant) { $columnDefs[] = 'tenant Nullable(UInt64)'; } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + // Build indexes from provided index definitions + $indexDefs = []; + foreach ($indexes as $index) { + $indexId = $index['$id'] ?? ''; + $attributes = $index['attributes'] ?? []; + + if (! empty($indexId) && is_string($indexId) && is_array($attributes) && ! empty($attributes)) { + /** @var array $attributes */ + $attributeList = implode(', ', $attributes); + $indexDefs[] = 'INDEX ' . $indexId . ' (' . $attributeList . ') TYPE bloom_filter GRANULARITY 1'; + } + } + + // Add tenant index if tables are shared across tenants + if ($this->sharedTables) { + $indexDefs[] = 'INDEX idx_tenant tenant TYPE bloom_filter GRANULARITY 1'; + } + + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + // Determine ORDER BY clause - use first index or default + $orderBy = '_createdAt'; + if (! empty($indexes) && isset($indexes[0]['attributes']) && is_array($indexes[0]['attributes'])) { + /** @var array $orderAttributes */ + $orderAttributes = $indexes[0]['attributes']; + $orderBy = implode(', ', $orderAttributes); + } // Create table with MergeTree engine for optimal performance + $indexClause = ! empty($indexDefs) ? ',\n ' . implode(",\n ", $indexDefs) : ''; $createTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( - " . implode(",\n ", $columnDefs) . ', - INDEX idx_metric metric TYPE bloom_filter GRANULARITY 1, - INDEX idx_period period TYPE bloom_filter GRANULARITY 1 + " . implode(",\n ", $columnDefs) . $indexClause . " ) ENGINE = MergeTree() - ORDER BY (metric, period, time) - PARTITION BY toYYYYMM(time) + ORDER BY ({$orderBy}) + PARTITION BY toYYYYMM(_createdAt) SETTINGS index_granularity = 8192 - '; + "; $this->query($createTableSql); } + /** + * Convert a column definition to ClickHouse column syntax. + * + * @param array $column Column definition + * @return string ClickHouse column definition + */ + private function getClickHouseColumnDefinition(array $column): string + { + $columnId = $column['$id'] ?? ''; + $type = $column['type'] ?? 'string'; + $required = $column['required'] ?? false; + $size = $column['size'] ?? 0; + + // Map Utopia Database types to ClickHouse types + $clickHouseType = match ($type) { + 'string' => $size > 0 && $size <= 255 ? 'String' : 'String', + 'integer' => 'Int64', + 'float' => 'Float64', + 'boolean' => 'UInt8', + 'datetime' => 'DateTime64(3)', + 'json' => 'String', // Store JSON as string + default => 'String', + }; + + // Add Nullable wrapper if not required + if (! $required && $type !== 'boolean') { + $clickHouseType = 'Nullable(' . $clickHouseType . ')'; + } + + return $columnId . ' ' . $clickHouseType; + } + /** * Log a usage metric. * @@ -373,8 +503,6 @@ public function log(string $metric, int $value, string $period = '1h', array $ta $microtime = microtime(true); $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); - // Build column list and values based on sharedTables setting $columns = ['id', 'metric', 'value', 'period', 'time', 'tags']; $placeholders = [':id', ':metric', ':value', ':period', ':time', ':tags']; @@ -394,6 +522,9 @@ public function log(string $metric, int $value, string $period = '1h', array $ta $params['tenant'] = $this->tenant; } + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $sql = " INSERT INTO {$escapedDatabaseAndTable} (" . implode(', ', $columns) . ") @@ -462,7 +593,8 @@ public function logBatch(array $metrics): bool } } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); // Build column list based on sharedTables setting $columns = 'id, metric, value, period, time, tags'; @@ -576,7 +708,8 @@ public function getByPeriod(string $metric, string $period, array $queries = []) } } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); $tenantFilter = $this->getTenantFilter(); $sql = " @@ -621,7 +754,8 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa } } - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); $tenantFilter = $this->getTenantFilter(); $sql = " @@ -653,7 +787,8 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa */ public function countByPeriod(string $metric, string $period, array $queries = []): int { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); $tenantFilter = $this->getTenantFilter(); $sql = " @@ -680,7 +815,8 @@ public function countByPeriod(string $metric, string $period, array $queries = [ */ public function sumByPeriod(string $metric, string $period, array $queries = []): int { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); $tenantFilter = $this->getTenantFilter(); $sql = " @@ -707,7 +843,8 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) */ public function purge(string $datetime): bool { - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($this->table); + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); $tenantFilter = $this->getTenantFilter(); $sql = " From fad62e03d7a45927fb3fa8c03c03a8b3ae24b43f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 02:09:30 +0000 Subject: [PATCH 11/44] support namespace --- src/Usage/Adapter/ClickHouse.php | 7 +++++-- tests/Usage/Adapter/ClickHouseTest.php | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 05a5248..f02576d 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -415,7 +415,9 @@ public function setup(string $table, array $columns, array $indexes): void if (! empty($indexId) && is_string($indexId) && is_array($attributes) && ! empty($attributes)) { /** @var array $attributes */ $attributeList = implode(', ', $attributes); - $indexDefs[] = 'INDEX ' . $indexId . ' (' . $attributeList . ') TYPE bloom_filter GRANULARITY 1'; + // ClickHouse doesn't allow hyphens in index names, replace with underscores + $safeIndexId = str_replace('-', '_', $indexId); + $indexDefs[] = 'INDEX ' . $safeIndexId . ' (' . $attributeList . ') TYPE bloom_filter GRANULARITY 1'; } } @@ -436,7 +438,8 @@ public function setup(string $table, array $columns, array $indexes): void } // Create table with MergeTree engine for optimal performance - $indexClause = ! empty($indexDefs) ? ',\n ' . implode(",\n ", $indexDefs) : ''; + // ClickHouse indexes must be defined inside the column list + $indexClause = ! empty($indexDefs) ? ",\n " . implode(",\n ", $indexDefs) : ''; $createTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( " . implode(",\n ", $columnDefs) . $indexClause . " diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index ee4309f..f0ade35 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -13,15 +13,14 @@ class ClickHouseTest extends TestCase protected function initializeUsage(): void { - $host = getenv('CLICKHOUSE_HOST'); + $host = getenv('CLICKHOUSE_HOST') ?: 'clickhouse'; $username = getenv('CLICKHOUSE_USER') ?: 'default'; $password = getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse'; $port = (int) (getenv('CLICKHOUSE_PORT') ?: 8123); $secure = (bool) (getenv('CLICKHOUSE_SECURE') ?: false); - $enable = getenv('CLICKHOUSE_ENABLE_TESTS'); - if ($enable !== '1' || $host === false || $host === '') { + if ($host === false || $host === '') { $this->markTestSkipped('ClickHouse tests disabled (set CLICKHOUSE_ENABLE_TESTS=1 and CLICKHOUSE_HOST to run).'); } From a599c07e46fccda06702e6fc8bc0d3c968b59452 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 02:13:56 +0000 Subject: [PATCH 12/44] fix clickhouse test and column definitions --- src/Usage/Adapter/ClickHouse.php | 8 ++++---- tests/Usage/Adapter/ClickHouseTest.php | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index f02576d..f01d334 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -507,7 +507,7 @@ public function log(string $metric, int $value, string $period = '1h', array $ta $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); // Build column list and values based on sharedTables setting - $columns = ['id', 'metric', 'value', 'period', 'time', 'tags']; + $columns = ['_id', 'metric', 'value', 'period', 'time', 'tags']; $placeholders = [':id', ':metric', ':value', ':period', ':time', ':tags']; $params = [ @@ -600,7 +600,7 @@ public function logBatch(array $metrics): bool $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); // Build column list based on sharedTables setting - $columns = 'id, metric, value, period, time, tags'; + $columns = '_id, metric, value, period, time, tags'; if ($this->sharedTables) { $columns .= ', tenant'; } @@ -669,9 +669,9 @@ private function parseResults(string $result): array private function getSelectColumns(): string { if ($this->sharedTables) { - return 'id, metric, value, period, time, tags, tenant'; + return '_id, metric, value, period, time, tags, tenant'; } - return 'id, metric, value, period, time, tags'; + return '_id, metric, value, period, time, tags'; } /** diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index f0ade35..98ab8fe 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -20,11 +20,9 @@ protected function initializeUsage(): void $secure = (bool) (getenv('CLICKHOUSE_SECURE') ?: false); - if ($host === false || $host === '') { - $this->markTestSkipped('ClickHouse tests disabled (set CLICKHOUSE_ENABLE_TESTS=1 and CLICKHOUSE_HOST to run).'); - } - $adapter = new ClickHouseAdapter($host, $username, $password, $port, $secure); + $adapter->setNamespace('utopia_usage'); + $adapter->setTenant(1); // Optional customization via env vars if ($database = getenv('CLICKHOUSE_DATABASE')) { From 803353207c803193cbc84a8bb1d7583da1b25bf6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 02:21:26 +0000 Subject: [PATCH 13/44] fix database test --- tests/Usage/Adapter/DatabaseTest.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index d6f3cae..124338f 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -35,9 +35,7 @@ protected function initializeUsage(): void $this->usage = new Usage(new AdapterDatabase($this->database)); // Create database if missing - if (! $this->database->exists($this->database->getDatabase())) { - $this->database->create(); - } + $this->database->create(); // Always run setup to ensure collection exists try { From 1e2519547d1c20ced1e7f2fe994ceaaf751a3629 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 03:06:07 +0000 Subject: [PATCH 14/44] fix duplicate setup --- tests/Usage/Adapter/DatabaseTest.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index 124338f..3300a0d 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -35,7 +35,11 @@ protected function initializeUsage(): void $this->usage = new Usage(new AdapterDatabase($this->database)); // Create database if missing - $this->database->create(); + try { + $this->database->create(); + } catch (Duplicate $ex) { + // ignore duplicate exception + } // Always run setup to ensure collection exists try { From ddbcbe17bfe26a17782f2db8174c7b190801625f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 03:16:10 +0000 Subject: [PATCH 15/44] Refactor duplicates --- src/Usage/Adapter.php | 16 ++++++++++++++++ src/Usage/Adapter/ClickHouse.php | 24 ------------------------ src/Usage/Adapter/Database.php | 9 +-------- src/Usage/Usage.php | 20 +++++--------------- 4 files changed, 22 insertions(+), 47 deletions(-) diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 84f9249..986aeee 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -4,6 +4,22 @@ abstract class Adapter { + /** + * Default table name for usage metrics + */ + public const DEFAULT_TABLE = 'usage'; + + /** + * Period format mappings + * + * @var array + */ + public const PERIODS = [ + '1h' => 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00', + ]; + /** * Get adapter name */ diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index f01d334..b2d7548 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -17,17 +17,8 @@ class ClickHouse extends Adapter { private const DEFAULT_PORT = 8123; - private const DEFAULT_TABLE = 'usage'; - private const DEFAULT_DATABASE = 'default'; - /** @var array */ - public const PERIODS = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00', - ]; - private string $host; private int $port; @@ -174,19 +165,6 @@ public function setDatabase(string $database): self return $this; } - /** - * Set the table name for subsequent operations. - * - * @throws Exception - */ - public function setTable(string $table): self - { - $this->validateIdentifier($table, 'Table'); - $this->table = $table; - - return $this; - } - /** * Set the tenant ID for multi-tenant support. * Tenant is used to isolate usage metrics by tenant. @@ -359,8 +337,6 @@ public function getName(): string */ public function setup(string $table, array $columns, array $indexes): void { - $this->setTable($table); - // Create database if not exists $escapedDatabase = $this->escapeIdentifier($this->database); $createDbSql = "CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"; diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 9c5f205..7a0ab70 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -12,14 +12,7 @@ class Database extends Adapter { - protected string $collection = 'usage_metrics'; - - /** @var array */ - public const PERIODS = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00', - ]; + protected string $collection = self::DEFAULT_TABLE; private UtopiaDatabase $db; diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 9a59961..00ac143 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -38,7 +38,7 @@ public function getAdapter(): Adapter * @param array> $indexes Index definitions * @throws \Exception */ - public function setup(string $table = 'usage', array $columns = [], array $indexes = []): void + public function setup(string $table = Adapter::DEFAULT_TABLE, array $columns = [], array $indexes = []): void { // Use legacy constants if no columns/indexes provided (for backward compatibility) if (empty($columns)) { @@ -136,26 +136,19 @@ public function purge(string $datetime): bool } /** - * @deprecated Use constructor with adapter instead + * @deprecated Use Adapter::DEFAULT_TABLE instead * * @internal Legacy support - will be removed in future version */ - public const COLLECTION = 'usage'; + public const COLLECTION = Adapter::DEFAULT_TABLE; /** - * @deprecated Use Adapter\Database::PERIODS instead + * @deprecated Use Adapter::PERIODS instead * * @var array */ - public const PERIODS = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00', - ]; + public const PERIODS = Adapter::PERIODS; - /** - * @deprecated Use Adapter\Database::ATTRIBUTES instead - */ public const ATTRIBUTES = [ [ '$id' => 'metric', @@ -205,9 +198,6 @@ public function purge(string $datetime): bool ], ]; - /** - * @deprecated Use Adapter\Database::INDEXES instead - */ public const INDEXES = [ [ '$id' => 'index-metric', From dae64eef86f02a896484450c4eddab18783030a3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 03:20:14 +0000 Subject: [PATCH 16/44] cleanup --- src/Usage/Adapter.php | 5 - src/Usage/Adapter/ClickHouse.php | 4 +- src/Usage/Adapter/Database.php | 2 +- src/Usage/Usage.php | 185 +++++++++++++++---------------- 4 files changed, 94 insertions(+), 102 deletions(-) diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 986aeee..62e4e84 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -4,11 +4,6 @@ abstract class Adapter { - /** - * Default table name for usage metrics - */ - public const DEFAULT_TABLE = 'usage'; - /** * Period format mappings * diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index b2d7548..9f11767 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -6,6 +6,7 @@ use Utopia\Fetch\Client; use Utopia\Usage\Adapter; use Utopia\Usage\Metric; +use Utopia\Usage\Usage; /** * ClickHouse Adapter for Usage @@ -25,7 +26,7 @@ class ClickHouse extends Adapter private string $database = self::DEFAULT_DATABASE; - private string $table = self::DEFAULT_TABLE; + private string $table; private string $username; @@ -337,6 +338,7 @@ public function getName(): string */ public function setup(string $table, array $columns, array $indexes): void { + $this->table = $table; // Create database if not exists $escapedDatabase = $this->escapeIdentifier($this->database); $createDbSql = "CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"; diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 7a0ab70..c2a54c0 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -12,7 +12,7 @@ class Database extends Adapter { - protected string $collection = self::DEFAULT_TABLE; + protected string $collection; private UtopiaDatabase $db; diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 00ac143..793737b 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -10,6 +10,95 @@ */ class Usage { + public const COLLECTION = 'usage'; + + /** + * @deprecated Use Adapter::PERIODS instead + * + * @var array + */ + public const PERIODS = Adapter::PERIODS; + + public const ATTRIBUTES = [ + [ + '$id' => 'metric', + 'type' => 'string', + 'size' => 255, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => 'integer', + 'size' => 0, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => 'string', + 'size' => 16, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => 'datetime', + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => 'tags', + 'type' => 'string', + 'size' => 16777216, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['json'], + ], + ]; + + public const INDEXES = [ + [ + '$id' => 'index-metric', + 'type' => 'key', + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-period', + 'type' => 'key', + 'attributes' => ['period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-metric-period', + 'type' => 'key', + 'attributes' => ['metric', 'period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-time', + 'type' => 'key', + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => ['desc'], + ], + ]; + private Adapter $adapter; /** @@ -38,7 +127,7 @@ public function getAdapter(): Adapter * @param array> $indexes Index definitions * @throws \Exception */ - public function setup(string $table = Adapter::DEFAULT_TABLE, array $columns = [], array $indexes = []): void + public function setup(string $table = self::COLLECTION, array $columns = [], array $indexes = []): void { // Use legacy constants if no columns/indexes provided (for backward compatibility) if (empty($columns)) { @@ -134,98 +223,4 @@ public function purge(string $datetime): bool { return $this->adapter->purge($datetime); } - - /** - * @deprecated Use Adapter::DEFAULT_TABLE instead - * - * @internal Legacy support - will be removed in future version - */ - public const COLLECTION = Adapter::DEFAULT_TABLE; - - /** - * @deprecated Use Adapter::PERIODS instead - * - * @var array - */ - public const PERIODS = Adapter::PERIODS; - - public const ATTRIBUTES = [ - [ - '$id' => 'metric', - 'type' => 'string', - 'size' => 255, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'value', - 'type' => 'integer', - 'size' => 0, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'period', - 'type' => 'string', - 'size' => 16, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'time', - 'type' => 'datetime', - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => 'tags', - 'type' => 'string', - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], - ]; - - public const INDEXES = [ - [ - '$id' => 'index-metric', - 'type' => 'key', - 'attributes' => ['metric'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-period', - 'type' => 'key', - 'attributes' => ['period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-metric-period', - 'type' => 'key', - 'attributes' => ['metric', 'period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-time', - 'type' => 'key', - 'attributes' => ['time'], - 'lengths' => [], - 'orders' => ['desc'], - ], - ]; } From 244aa000907c5cf3cf507e72e0050c41470d7e7e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Dec 2025 03:21:17 +0000 Subject: [PATCH 17/44] fix codeql --- tests/Usage/Adapter/ClickHouseTest.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 98ab8fe..78d2957 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -29,10 +29,6 @@ protected function initializeUsage(): void $adapter->setDatabase($database); } - if ($table = getenv('CLICKHOUSE_TABLE')) { - $adapter->setTable($table); - } - $this->usage = new Usage($adapter); $this->usage->setup(); } From a84af65bada31ce5b61c75365e41e05add034a8f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 01:30:33 +0000 Subject: [PATCH 18/44] feat: Refactor Usage class and introduce ClickHouse adapter - Removed hardcoded column definitions in Usage class, replacing with dynamic schema derived from SQL adapter. - Introduced new Query class for building ClickHouse queries with fluent interface. - Added support for advanced query operations including find and count methods. - Enhanced error handling and SQL injection prevention mechanisms. - Created comprehensive usage guide for ClickHouse adapter. - Added unit tests for Query class to ensure functionality and robustness. - Maintained backward compatibility with existing methods while improving overall architecture. --- src/Usage/Adapter.php | 15 +- src/Usage/Adapter/ClickHouse.php | 1114 +++++++++++++++++++++--------- src/Usage/Adapter/SQL.php | 202 ++++++ src/Usage/Query.php | 288 ++++++++ src/Usage/Usage.php | 105 +-- tests/Usage/QueryTest.php | 225 ++++++ 6 files changed, 1500 insertions(+), 449 deletions(-) create mode 100644 src/Usage/Adapter/SQL.php create mode 100644 src/Usage/Query.php create mode 100644 tests/Usage/QueryTest.php diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 62e4e84..99b3511 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -4,17 +4,6 @@ abstract class Adapter { - /** - * Period format mappings - * - * @var array - */ - public const PERIODS = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00', - ]; - /** * Get adapter name */ @@ -27,14 +16,14 @@ abstract public function getName(): string; * @param array> $columns Column definitions * @param array> $indexes Index definitions */ - abstract public function setup(string $table, array $columns, array $indexes): void; + abstract public function setup(): void; /** * Log usage metric * * @param array $tags */ - abstract public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool; + abstract public function log(string $metric, int $value, string $period = Usage::PERIOD_1H, array $tags = []): bool; /** * Log multiple metrics in batch diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 9f11767..9947737 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -3,30 +3,45 @@ namespace Utopia\Usage\Adapter; use Exception; +use Utopia\Database\Database; +use Utopia\Database\Query; use Utopia\Fetch\Client; use Utopia\Usage\Adapter; use Utopia\Usage\Metric; use Utopia\Usage\Usage; +use Utopia\Validator\Hostname; /** * ClickHouse Adapter for Usage * * This adapter stores usage metrics in ClickHouse using HTTP interface. * ClickHouse is optimized for analytical queries and can handle massive amounts of metrics data. + * + * Features: + * - Dynamic schema based on SQL adapter attributes (no hardcoded columns) + * - Safe SQL injection prevention using ClickHouse parameter binding + * - Support for find() and count() operations with Query objects + * - Multi-tenant support with optional shared tables + * - Namespace support for table name prefixes + * - Proper index creation for optimized analytical queries + * - Bloom filter indexes for efficient filtering + * - MergeTree engine with monthly partitioning by time */ -class ClickHouse extends Adapter +class ClickHouse extends SQL { private const DEFAULT_PORT = 8123; private const DEFAULT_DATABASE = 'default'; + private const DEFAULT_TABLE = self::COLLECTION; + private string $host; private int $port; private string $database = self::DEFAULT_DATABASE; - private string $table; + private string $table = self::DEFAULT_TABLE; private string $username; @@ -66,32 +81,39 @@ public function __construct( $this->password = $password; $this->secure = $secure; + // Initialize the HTTP client for connection reuse $this->client = new Client(); $this->client->addHeader('X-ClickHouse-User', $this->username); $this->client->addHeader('X-ClickHouse-Key', $this->password); - $this->client->addHeader('X-ClickHouse-Database', $this->database); + $this->client->setTimeout(30_000); // 30 seconds + } + + /** + * Get adapter name. + */ + public function getName(): string + { + return 'ClickHouse'; } /** * Validate host parameter. * + * @param string $host * @throws Exception */ private function validateHost(string $host): void { - if (empty($host)) { - throw new Exception('ClickHouse host cannot be empty'); - } - - // Allow hostnames, IP addresses, and localhost - if (! preg_match('/^[a-zA-Z0-9._\-]+$/', $host)) { - throw new Exception('ClickHouse host must be a valid hostname or IP address'); + $validator = new Hostname(); + if (!$validator->isValid($host)) { + throw new Exception('ClickHouse host is not a valid hostname or IP address'); } } /** * Validate port parameter. * + * @param int $port * @throws Exception */ private function validatePort(int $port): void @@ -102,10 +124,11 @@ private function validatePort(int $port): void } /** - * Validate identifier (database, table). - * - * @param string $type Name of the identifier type for error messages + * Validate identifier (database, table, namespace). + * ClickHouse identifiers follow SQL standard rules. * + * @param string $identifier + * @param string $type Name of the identifier type for error messages * @throws Exception */ private function validateIdentifier(string $identifier, string $type = 'Identifier'): void @@ -119,11 +142,11 @@ private function validateIdentifier(string $identifier, string $type = 'Identifi } // ClickHouse identifiers: alphanumeric, underscores, cannot start with number - if (! preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $identifier)) { + if (!preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $identifier)) { throw new Exception("{$type} must start with a letter or underscore and contain only alphanumeric characters and underscores"); } - // Check against SQL keywords + // Check against SQL keywords (common ones) $keywords = ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'DROP', 'CREATE', 'ALTER', 'TABLE', 'DATABASE']; if (in_array(strtoupper($identifier), $keywords, true)) { throw new Exception("{$type} cannot be a reserved SQL keyword"); @@ -131,46 +154,73 @@ private function validateIdentifier(string $identifier, string $type = 'Identifi } /** - * Escape an identifier for safe use in SQL. + * Escape an identifier (database name, table name, column name) for safe use in SQL. + * Uses backticks as per SQL standard for identifier quoting. + * + * @param string $identifier + * @return string */ private function escapeIdentifier(string $identifier): string { + // Backtick escaping: replace any backticks in the identifier with double backticks return '`' . str_replace('`', '``', $identifier) . '`'; } /** - * Escape a string value for safe use in ClickHouse SQL queries. + * Set the namespace for multi-project support. + * Namespace is used as a prefix for table names. * - * @return string The escaped value without surrounding quotes + * @param string $namespace + * @return self + * @throws Exception */ - private function escapeString(string $value): string + public function setNamespace(string $namespace): self { - return str_replace( - ['\\', "'"], - ['\\\\', "''"], - $value - ); + if (!empty($namespace)) { + $this->validateIdentifier($namespace, 'Namespace'); + } + $this->namespace = $namespace; + return $this; } /** * Set the database name for subsequent operations. * + * @param string $database + * @return self * @throws Exception */ public function setDatabase(string $database): self { $this->validateIdentifier($database, 'Database'); $this->database = $database; - $this->client->addHeader('X-ClickHouse-Database', $this->database); + return $this; + } + /** + * Enable or disable HTTPS for ClickHouse HTTP interface. + */ + public function setSecure(bool $secure): self + { + $this->secure = $secure; return $this; } + /** + * Get the namespace. + * + * @return string + */ + public function getNamespace(): string + { + return $this->namespace; + } + /** * Set the tenant ID for multi-tenant support. - * Tenant is used to isolate usage metrics by tenant. + * Tenant is used to isolate metrics by tenant. * - * @param int|null $tenant + * @param int|null $tenant * @return self */ public function setTenant(?int $tenant): self @@ -193,7 +243,7 @@ public function getTenant(): ?int * Set whether tables are shared across tenants. * When enabled, a tenant column is added to the table for data isolation. * - * @param bool $sharedTables + * @param bool $sharedTables * @return self */ public function setSharedTables(bool $sharedTables): self @@ -212,33 +262,6 @@ public function isSharedTables(): bool return $this->sharedTables; } - /** - * Set the namespace for multi-project support. - * Namespace is used as a prefix for table names. - * - * @param string $namespace - * @return self - * @throws Exception - */ - public function setNamespace(string $namespace): self - { - if (!empty($namespace)) { - $this->validateIdentifier($namespace, 'Namespace'); - } - $this->namespace = $namespace; - return $this; - } - - /** - * Get the namespace. - * - * @return string - */ - public function getNamespace(): string - { - return $this->namespace; - } - /** * Get the table name with namespace prefix. * Namespace is used to isolate tables for different projects/applications. @@ -257,61 +280,56 @@ private function getTableName(): string } /** - * Execute a ClickHouse query via HTTP interface. + * Execute a ClickHouse query via HTTP interface using Fetch Client. + * + * Uses ClickHouse query parameters (sent as POST multipart form data) to prevent SQL injection. + * This is ClickHouse's native parameter mechanism - parameters are safely + * transmitted separately from the query structure. + * + * Parameters are referenced in the SQL using the syntax: {paramName:Type}. + * For example: SELECT * WHERE id = {id:String} * - * @param string $sql SQL query to execute - * @param array $params Query parameters for prepared statements - * @return string Query result as string + * ClickHouse handles all parameter escaping and type conversion internally, + * making this approach fully injection-safe without needing manual escaping. * + * Using POST body avoids URL length limits for batch operations with many parameters. + * Equivalent to: curl -X POST -F 'query=...' -F 'param_key=value' http://host/ + * + * @param array $params Key-value pairs for query parameters * @throws Exception */ private function query(string $sql, array $params = []): string { - $protocol = $this->secure ? 'https' : 'http'; - $url = "{$protocol}://{$this->host}:{$this->port}/"; + $scheme = $this->secure ? 'https' : 'http'; + $url = "{$scheme}://{$this->host}:{$this->port}/"; + + // Update the database header for each query (in case setDatabase was called) + $this->client->addHeader('X-ClickHouse-Database', $this->database); - // Replace parameters in SQL + // Build multipart form data body with query and parameters + // The Fetch client will automatically encode arrays as multipart/form-data + $body = ['query' => $sql]; foreach ($params as $key => $value) { - if (is_int($value) || is_float($value)) { - // Numeric values should not be quoted - $strValue = (string) $value; - } elseif (is_string($value)) { - $strValue = "'" . $this->escapeString($value) . "'"; - } elseif (is_null($value)) { - $strValue = 'NULL'; - } elseif (is_bool($value)) { - $strValue = $value ? '1' : '0'; - } elseif (is_array($value)) { - $encoded = json_encode($value); - if (is_string($encoded)) { - $strValue = "'" . $this->escapeString($encoded) . "'"; - } else { - $strValue = 'NULL'; - } - } else { - /** @var scalar $value */ - $strValue = "'" . $this->escapeString((string) $value) . "'"; - } - $sql = str_replace(":{$key}", $strValue, $sql); + $body['param_' . $key] = $this->formatParamValue($value); } try { $response = $this->client->fetch( url: $url, method: Client::METHOD_POST, - body: ['query' => $sql] + body: $body ); - if ($response->getStatusCode() !== 200) { - $body = $response->getBody(); - $bodyStr = is_string($body) ? $body : ''; + $bodyStr = $response->getBody(); + $bodyStr = is_string($bodyStr) ? $bodyStr : ''; throw new Exception("ClickHouse query failed with HTTP {$response->getStatusCode()}: {$bodyStr}"); } $body = $response->getBody(); - return is_string($body) ? $body : ''; } catch (Exception $e) { + // Preserve the original exception context for better debugging + // Re-throw with additional context while maintaining the original exception chain throw new Exception( "ClickHouse query execution failed: {$e->getMessage()}", 0, @@ -320,111 +338,112 @@ private function query(string $sql, array $params = []): string } } - public function getName(): string + /** + * Format a parameter value for safe transmission to ClickHouse. + * + * Converts PHP values to their string representation without SQL quoting. + * ClickHouse's query parameter mechanism handles type conversion and escaping. + * + * @param mixed $value + * @return string + */ + private function formatParamValue(mixed $value): string { - return 'ClickHouse'; + if (is_int($value) || is_float($value)) { + return (string) $value; + } + + if ($value === null) { + return ''; + } + + if (is_bool($value)) { + return $value ? '1' : '0'; + } + + if (is_array($value)) { + $encoded = json_encode($value); + return is_string($encoded) ? $encoded : ''; + } + + if (is_string($value)) { + return $value; + } + + // For objects or other types, attempt to convert to string + if (is_object($value) && method_exists($value, '__toString')) { + return (string) $value; + } + + return ''; } /** * Setup ClickHouse table structure. * * Creates the database and table if they don't exist. - * Uses the provided column definitions and adds internal fields (_id, _createdAt, _updatedAt, tenant). + * Uses schema definitions from the base SQL adapter. * - * @param string $table Table name - * @param array> $columns Column definitions from the application - * @param array> $indexes Index definitions from the application * @throws Exception */ - public function setup(string $table, array $columns, array $indexes): void + public function setup(): void { - $this->table = $table; // Create database if not exists $escapedDatabase = $this->escapeIdentifier($this->database); $createDbSql = "CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"; $this->query($createDbSql); - // Track which internal fields are already present - $hasId = false; - $hasCreatedAt = false; - $hasUpdatedAt = false; - $hasTenant = false; - - // Build column definitions from provided columns - $columnDefs = []; - foreach ($columns as $column) { - $columnId = $column['$id'] ?? ''; - - if ($columnId === '_id' || $columnId === '$id') { - $hasId = true; - } elseif ($columnId === '_createdAt' || $columnId === '$createdAt') { - $hasCreatedAt = true; - } elseif ($columnId === '_updatedAt' || $columnId === '$updatedAt') { - $hasUpdatedAt = true; - } elseif ($columnId === 'tenant' || $columnId === '$tenant') { - $hasTenant = true; - } - - $columnDefs[] = $this->getClickHouseColumnDefinition($column); - } - - // Add internal fields if not present - if (! $hasId) { - array_unshift($columnDefs, '_id String'); - } - if (! $hasCreatedAt) { - $columnDefs[] = '_createdAt DateTime64(3) DEFAULT now64(3)'; - } - if (! $hasUpdatedAt) { - $columnDefs[] = '_updatedAt DateTime64(3) DEFAULT now64(3)'; - } - - // Add tenant column only if tables are shared across tenants and not already present - if ($this->sharedTables && ! $hasTenant) { - $columnDefs[] = 'tenant Nullable(UInt64)'; - } + // Build column definitions from base adapter schema + $columns = [ + 'id String', + ]; - // Build indexes from provided index definitions - $indexDefs = []; - foreach ($indexes as $index) { - $indexId = $index['$id'] ?? ''; - $attributes = $index['attributes'] ?? []; + foreach ($this->getAttributes() as $attribute) { + /** @var string $id */ + $id = $attribute['$id']; - if (! empty($indexId) && is_string($indexId) && is_array($attributes) && ! empty($attributes)) { - /** @var array $attributes */ - $attributeList = implode(', ', $attributes); - // ClickHouse doesn't allow hyphens in index names, replace with underscores - $safeIndexId = str_replace('-', '_', $indexId); - $indexDefs[] = 'INDEX ' . $safeIndexId . ' (' . $attributeList . ') TYPE bloom_filter GRANULARITY 1'; + // Special handling for time column - must be NOT NULL for partition key + if ($id === 'time') { + // Use DateTime64(3) without Nullable wrapper for time since it's used as partition key + $columns[] = 'time DateTime64(3)'; + } else { + $columns[] = $this->getColumnDefinition($id); } } - // Add tenant index if tables are shared across tenants + // Add tenant column only if tables are shared across tenants if ($this->sharedTables) { - $indexDefs[] = 'INDEX idx_tenant tenant TYPE bloom_filter GRANULARITY 1'; + $columns[] = 'tenant Nullable(UInt64)'; // Supports 11-digit MySQL auto-increment IDs + } + + // Build indexes from base adapter schema + $indexes = []; + foreach ($this->getIndexes() as $index) { + /** @var string $indexName */ + $indexName = $index['$id']; + /** @var array $attributes */ + $attributes = $index['attributes']; + // Escape index name and attribute names to prevent SQL injection + $escapedIndexName = $this->escapeIdentifier($indexName); + $escapedAttributes = array_map(fn ($attr) => $this->escapeIdentifier($attr), $attributes); + $attributeList = implode(', ', $escapedAttributes); + $indexes[] = "INDEX {$escapedIndexName} ({$attributeList}) TYPE bloom_filter GRANULARITY 1"; } $tableName = $this->getTableName(); $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - // Determine ORDER BY clause - use first index or default - $orderBy = '_createdAt'; - if (! empty($indexes) && isset($indexes[0]['attributes']) && is_array($indexes[0]['attributes'])) { - /** @var array $orderAttributes */ - $orderAttributes = $indexes[0]['attributes']; - $orderBy = implode(', ', $orderAttributes); - } - // Create table with MergeTree engine for optimal performance - // ClickHouse indexes must be defined inside the column list - $indexClause = ! empty($indexDefs) ? ",\n " . implode(",\n ", $indexDefs) : ''; + $columnDefs = implode(",\n ", $columns); + $indexDefs = !empty($indexes) ? ",\n " . implode(",\n ", $indexes) : ''; + $createTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( - " . implode(",\n ", $columnDefs) . $indexClause . " + {$columnDefs}{$indexDefs} ) ENGINE = MergeTree() - ORDER BY ({$orderBy}) - PARTITION BY toYYYYMM(_createdAt) + ORDER BY (time, id) + PARTITION BY toYYYYMM(time) SETTINGS index_granularity = 8192 "; @@ -432,35 +451,116 @@ public function setup(string $table, array $columns, array $indexes): void } /** - * Convert a column definition to ClickHouse column syntax. + * Validate that an attribute name exists in the schema. + * Prevents SQL injection by ensuring only valid column names are used. + * + * @param string $attributeName The attribute name to validate + * @return bool True if valid + * @throws Exception If attribute name is invalid + */ + private function validateAttributeName(string $attributeName): bool + { + + // Special case: 'id' is always valid + if ($attributeName === 'id') { + return true; + } + + // Check if tenant is valid (only when sharedTables is enabled) + if ($attributeName === 'tenant' && $this->sharedTables) { + return true; + } + + // Check against defined attributes + foreach ($this->getAttributes() as $attribute) { + if ($attribute['$id'] === $attributeName) { + return true; + } + } + + throw new Exception("Invalid attribute name: {$attributeName}"); + } + + /** + * Format datetime for ClickHouse compatibility. + * Converts datetime to 'YYYY-MM-DD HH:MM:SS.mmm' format without timezone suffix. + * ClickHouse DateTime64(3) type expects this format as timezone is handled by column metadata. + * Works with DateTime objects, strings, and other datetime representations. + * + * @param \DateTime|string|null $dateTime The datetime value to format + * @return string The formatted datetime string in ClickHouse compatible format + * @throws Exception If the datetime string cannot be parsed + */ + private function formatDateTime($dateTime): string + { + if ($dateTime === null) { + return (new \DateTime())->format('Y-m-d H:i:s.v'); + } + + if ($dateTime instanceof \DateTime) { + return $dateTime->format('Y-m-d H:i:s.v'); + } + + if (is_string($dateTime)) { + try { + // Parse the datetime string, handling ISO 8601 format with timezone + $dt = new \DateTime($dateTime); + return $dt->format('Y-m-d H:i:s.v'); + } catch (\Exception $e) { + throw new Exception("Invalid datetime string: {$dateTime}"); + } + } + + // This is unreachable code but kept for completeness - all valid types are handled above + // @phpstan-ignore-next-line + throw new Exception('DateTime must be a DateTime object or string'); + } + + /** + * Get ClickHouse-specific SQL column definition for a given attribute ID. * - * @param array $column Column definition + * Dynamically determines the ClickHouse type based on attribute metadata and nullability + * + * @param string $id The attribute ID * @return string ClickHouse column definition + * @throws Exception + */ + /** + * Get ClickHouse type for an attribute. + * + * Maps PHP attribute types to ClickHouse types and applies Nullable wrapper. + * + * @param string $id Attribute identifier + * @return string ClickHouse type (e.g., "String", "Nullable(Int64)", "DateTime64(3)") + * @throws Exception */ - private function getClickHouseColumnDefinition(array $column): string + private function getColumnType(string $id): string { - $columnId = $column['$id'] ?? ''; - $type = $column['type'] ?? 'string'; - $required = $column['required'] ?? false; - $size = $column['size'] ?? 0; + $attribute = $this->getAttribute($id); + if (!$attribute) { + throw new Exception("Attribute {$id} not found"); + } - // Map Utopia Database types to ClickHouse types - $clickHouseType = match ($type) { - 'string' => $size > 0 && $size <= 255 ? 'String' : 'String', + // Map attribute type to ClickHouse type + $attributeType = $attribute['type'] ?? 'string'; + $baseType = match ($attributeType) { 'integer' => 'Int64', 'float' => 'Float64', 'boolean' => 'UInt8', 'datetime' => 'DateTime64(3)', - 'json' => 'String', // Store JSON as string + Database::VAR_DATETIME => 'DateTime64(3)', default => 'String', }; // Add Nullable wrapper if not required - if (! $required && $type !== 'boolean') { - $clickHouseType = 'Nullable(' . $clickHouseType . ')'; - } + return !$attribute['required'] ? 'Nullable(' . $baseType . ')' : $baseType; + } - return $columnId . ' ' . $clickHouseType; + protected function getColumnDefinition(string $id): string + { + $type = $this->getColumnType($id); + $escapedId = $this->escapeIdentifier($id); + return "{$escapedId} {$type}"; } /** @@ -470,26 +570,23 @@ private function getClickHouseColumnDefinition(array $column): string * * @throws Exception */ - public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool + public function log(string $metric, int $value, string $period = Usage::PERIOD_1H, array $tags = []): bool { - if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + if (!isset(Usage::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); } $id = uniqid('', true); $now = new \DateTime(); - $time = $now->format(self::PERIODS[$period]); - - // Format timestamp for ClickHouse DateTime64(3) - $microtime = microtime(true); - $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); + $timestamp = $this->formatDateTime($now); - // Build column list and values based on sharedTables setting - $columns = ['_id', 'metric', 'value', 'period', 'time', 'tags']; - $placeholders = [':id', ':metric', ':value', ':period', ':time', ':tags']; + // Build insert columns dynamically from attributes + $insertColumns = ['id']; + $queryParams = ['id' => $id]; + $valuePlaceholders = ['{id:String}']; - $params = [ - 'id' => $id, + // Map attribute values to their positions + $attributeMap = [ 'metric' => $metric, 'value' => $value, 'period' => $period, @@ -497,10 +594,26 @@ public function log(string $metric, int $value, string $period = '1h', array $ta 'tags' => json_encode($tags), ]; + // Add columns from attributes in order + foreach ($this->getAttributes() as $attribute) { + $attrId = $attribute['$id']; + if (!isset($attributeMap[$attrId])) { + continue; // Skip attributes not in our data + } + + $insertColumns[] = $attrId; + $queryParams[$attrId] = $attributeMap[$attrId]; + + // Determine ClickHouse type hint + $type = $this->getColumnType($attrId); + $valuePlaceholders[] = '{' . $attrId . ':' . $type . '}'; + } + + // Add tenant column if using shared tables if ($this->sharedTables) { - $columns[] = 'tenant'; - $placeholders[] = ':tenant'; - $params['tenant'] = $this->tenant; + $insertColumns[] = 'tenant'; + $valuePlaceholders[] = '{tenant:Nullable(UInt64)}'; + $queryParams['tenant'] = $this->tenant; } $tableName = $this->getTableName(); @@ -508,13 +621,13 @@ public function log(string $metric, int $value, string $period = '1h', array $ta $sql = " INSERT INTO {$escapedDatabaseAndTable} - (" . implode(', ', $columns) . ") + (" . implode(', ', $insertColumns) . ") VALUES ( - " . implode(", ", $placeholders) . " + " . implode(", ", $valuePlaceholders) . " ) "; - $this->query($sql, $params); + $this->query($sql, $queryParams); return true; } @@ -532,67 +645,350 @@ public function logBatch(array $metrics): bool return true; } - $values = []; + $tableName = $this->getTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + // Build column list dynamically from attributes + $insertColumns = ['id']; + foreach ($this->getAttributes() as $attribute) { + $insertColumns[] = $attribute['$id']; + } + if ($this->sharedTables) { + $insertColumns[] = 'tenant'; + } + + $paramCounter = 0; + $queryParams = []; + $valueClauses = []; + foreach ($metrics as $metricData) { - $period = $metricData['period'] ?? '1h'; + $period = $metricData['period'] ?? Usage::PERIOD_1H; - if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + if (!isset(Usage::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); } $id = uniqid('', true); - $microtime = microtime(true); - $timestamp = date('Y-m-d H:i:s', (int) $microtime) . '.' . sprintf('%03d', ($microtime - floor($microtime)) * 1000); + $now = new \DateTime(); + $timestamp = $this->formatDateTime($now); $metric = $metricData['metric']; $value = $metricData['value']; assert(is_string($metric)); assert(is_int($value)); - if ($this->sharedTables) { - $tenant = $this->tenant !== null ? (int) $this->tenant : 'NULL'; - $values[] = sprintf( - "('%s', '%s', %d, '%s', '%s', '%s', %s)", - $id, - $this->escapeString($metric), - $value, - $this->escapeString($period), - $timestamp, - $this->escapeString((string) json_encode($metricData['tags'] ?? [])), - $tenant - ); - } else { - $values[] = sprintf( - "('%s', '%s', %d, '%s', '%s', '%s')", - $id, - $this->escapeString($metric), - $value, - $this->escapeString($period), - $timestamp, - $this->escapeString((string) json_encode($metricData['tags'] ?? [])) - ); + $valuePlaceholders = []; + + // Add id + $idKey = 'id_' . $paramCounter; + $queryParams[$idKey] = $id; + $valuePlaceholders[] = '{' . $idKey . ':String}'; + + // Add attributes dynamically + $attributeMap = [ + 'metric' => $metric, + 'value' => $value, + 'period' => $period, + 'time' => $timestamp, + 'tags' => json_encode($metricData['tags'] ?? []), + ]; + + foreach ($this->getAttributes() as $attribute) { + $attrId = $attribute['$id']; + if (!isset($attributeMap[$attrId])) { + continue; + } + + $attrKey = $attrId . '_' . $paramCounter; + $queryParams[$attrKey] = $attributeMap[$attrId]; + + // Determine ClickHouse type hint + $type = $this->getColumnType($attrId); + $valuePlaceholders[] = '{' . $attrKey . ':' . $type . '}'; } - } - $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + if ($this->sharedTables) { + $tenantKey = 'tenant_' . $paramCounter; + $queryParams[$tenantKey] = $this->tenant; + $valuePlaceholders[] = '{' . $tenantKey . ':Nullable(UInt64)}'; + } - // Build column list based on sharedTables setting - $columns = '_id, metric, value, period, time, tags'; - if ($this->sharedTables) { - $columns .= ', tenant'; + $valueClauses[] = '(' . implode(', ', $valuePlaceholders) . ')'; + $paramCounter++; } $insertSql = " INSERT INTO {$escapedDatabaseAndTable} - ({$columns}) - VALUES " . implode(', ', $values); + (" . implode(', ', $insertColumns) . ") + VALUES " . implode(', ', $valueClauses); - $this->query($insertSql); + $this->query($insertSql, $queryParams); return true; } + /** + * Find metrics using Query objects. + * + * @param array $queries + * @return array + * @throws Exception + */ + public function find(array $queries = []): array + { + $tableName = $this->getTableName(); + $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + // Parse queries + $parsed = $this->parseQueries($queries); + + // Build SELECT clause + $selectColumns = $this->getSelectColumns(); + + // Build WHERE clause + $whereClause = ''; + $tenantFilter = $this->getTenantFilter(); + if (!empty($parsed['filters']) || $tenantFilter) { + $conditions = $parsed['filters']; + if ($tenantFilter) { + $conditions[] = ltrim($tenantFilter, ' AND'); + } + $whereClause = ' WHERE ' . implode(' AND ', $conditions); + } + + // Build ORDER BY clause + $orderClause = ''; + if (!empty($parsed['orderBy'])) { + $orderClause = ' ORDER BY ' . implode(', ', $parsed['orderBy']); + } + + // Build LIMIT and OFFSET + $limitClause = isset($parsed['limit']) ? ' LIMIT {limit:UInt64}' : ''; + $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; + + $sql = " + SELECT {$selectColumns} + FROM {$escapedTable}{$whereClause}{$orderClause}{$limitClause}{$offsetClause} + FORMAT TabSeparated + "; + + $result = $this->query($sql, $parsed['params']); + return $this->parseResults($result); + } + + /** + * Count metrics using Query objects. + * + * @param array $queries + * @return int + * @throws Exception + */ + public function count(array $queries = []): int + { + $tableName = $this->getTableName(); + $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + // Parse queries - we only need filters and params + $parsed = $this->parseQueries($queries); + + // Build WHERE clause + $whereClause = ''; + $tenantFilter = $this->getTenantFilter(); + if (!empty($parsed['filters']) || $tenantFilter) { + $conditions = $parsed['filters']; + if ($tenantFilter) { + $conditions[] = ltrim($tenantFilter, ' AND'); + } + $whereClause = ' WHERE ' . implode(' AND ', $conditions); + } + + // Remove limit and offset from params + $params = $parsed['params']; + unset($params['limit'], $params['offset']); + + $sql = " + SELECT COUNT(*) as count + FROM {$escapedTable}{$whereClause} + FORMAT TabSeparated + "; + + $result = $this->query($sql, $params); + $trimmed = trim($result); + return $trimmed !== '' ? (int) $trimmed : 0; + } + + /** + * Parse Query objects into SQL clauses. + * + * @param array $queries + * @return array{filters: array, params: array, orderBy?: array, limit?: int, offset?: int} + * @throws Exception + */ + private function parseQueries(array $queries): array + { + $filters = []; + $params = []; + $orderBy = []; + $limit = null; + $offset = null; + $paramCounter = 0; + + foreach ($queries as $query) { + if (!$query instanceof Query) { + /** @phpstan-ignore-next-line ternary.alwaysTrue - runtime validation despite type hint */ + $type = is_object($query) ? get_class($query) : gettype($query); + throw new \InvalidArgumentException("Invalid query item: expected instance of Query, got {$type}"); + } + + $method = $query->getMethod(); + $attribute = $query->getAttribute(); + $values = $query->getValues(); + + switch ($method) { + case Query::TYPE_EQUAL: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $paramName = 'param_' . $paramCounter++; + // Query values are arrays, use first element + $value = is_array($values) && !empty($values) ? $values[0] : $values; + $filters[] = "{$escapedAttr} = {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + break; + + case Query::TYPE_LESSER: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $paramName = 'param_' . $paramCounter++; + $value = is_array($values) && !empty($values) ? $values[0] : $values; + if ($attribute === 'time') { + $filters[] = "{$escapedAttr} < {{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime($value); + } else { + $filters[] = "{$escapedAttr} < {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + } + break; + + case Query::TYPE_GREATER: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $paramName = 'param_' . $paramCounter++; + $value = is_array($values) && !empty($values) ? $values[0] : $values; + if ($attribute === 'time') { + $filters[] = "{$escapedAttr} > {{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime($value); + } else { + $filters[] = "{$escapedAttr} > {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + } + break; + + case Query::TYPE_BETWEEN: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $paramName1 = 'param_' . $paramCounter++; + $paramName2 = 'param_' . $paramCounter++; + // Between has two values + $value1 = is_array($values) && isset($values[0]) ? $values[0] : $values; + $value2 = is_array($values) && isset($values[1]) ? $values[1] : $values; + if ($attribute === 'time') { + $paramType = 'DateTime64(3)'; + $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:{$paramType}} AND {{$paramName2}:{$paramType}}"; + $params[$paramName1] = $this->formatDateTime($value1); + $params[$paramName2] = $this->formatDateTime($value2); + } else { + $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:String} AND {{$paramName2}:String}"; + $params[$paramName1] = $this->formatParamValue($value1); + $params[$paramName2] = $this->formatParamValue($value2); + } + break; + + case Query::TYPE_SEARCH: + // SEARCH is like LIKE + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $paramName = 'param_' . $paramCounter++; + $value = is_array($values) && !empty($values) ? $values[0] : $values; + $filters[] = "{$escapedAttr} LIKE {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + break; + + case Query::TYPE_SELECT: + // SELECT allows selecting multiple columns/values + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $inParams = []; + $valuesToUse = is_array($values) ? $values : [$values]; + foreach ($valuesToUse as $value) { + $paramName = 'param_' . $paramCounter++; + $inParams[] = "{{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + } + if (!empty($inParams)) { + $filters[] = "{$escapedAttr} IN (" . implode(', ', $inParams) . ")"; + } + break; + + case Query::TYPE_ORDER_DESC: + // Skip special Query attributes (like $sequence) that aren't real columns + if (str_starts_with($attribute, '$')) { + break; + } + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $orderBy[] = "{$escapedAttr} DESC"; + break; + + case Query::TYPE_ORDER_ASC: + // Skip special Query attributes (like $sequence) that aren't real columns + if (str_starts_with($attribute, '$')) { + break; + } + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $orderBy[] = "{$escapedAttr} ASC"; + break; + + case Query::TYPE_LIMIT: + $limitVal = is_array($values) && !empty($values) ? $values[0] : $values; + if (!\is_int($limitVal)) { + throw new \Exception('Invalid limit value. Expected int'); + } + $limit = $limitVal; + $params['limit'] = $limit; + break; + + case Query::TYPE_OFFSET: + $offsetVal = is_array($values) && !empty($values) ? $values[0] : $values; + if (!\is_int($offsetVal)) { + throw new \Exception('Invalid offset value. Expected int'); + } + $offset = $offsetVal; + $params['offset'] = $offset; + break; + } + } + + $result = [ + 'filters' => $filters, + 'params' => $params, + ]; + + if (!empty($orderBy)) { + $result['orderBy'] = $orderBy; + } + + if ($limit !== null) { + $result['limit'] = $limit; + } + + if ($offset !== null) { + $result['offset'] = $offset; + } + + return $result; + } + /** * Parse ClickHouse TabSeparated results into Metric array. * @@ -607,32 +1003,79 @@ private function parseResults(string $result): array $lines = explode("\n", trim($result)); $metrics = []; + // Build select columns list matching getSelectColumns() + $selectColumns = ['id']; + foreach ($this->getAttributes() as $attribute) { + $selectColumns[] = $attribute['$id']; + } + + if ($this->sharedTables) { + $selectColumns[] = 'tenant'; + } + + $expectedColumns = count($selectColumns); + foreach ($lines as $line) { if (empty(trim($line))) { continue; } $columns = explode("\t", $line); - $expectedColumns = $this->sharedTables ? 7 : 6; + if (count($columns) < $expectedColumns) { continue; } - $data = [ - '$id' => (string) $columns[0], - 'metric' => (string) $columns[1], - 'value' => (int) $columns[2], - 'period' => (string) $columns[3], - 'time' => (string) $columns[4], - 'tags' => json_decode((string) $columns[5], true) ?? [], - ]; + // Helper function to parse nullable string fields + $parseNullableString = static function ($value): ?string { + if ($value === '\\N' || $value === '') { + return null; + } + return $value; + }; + + // Build document dynamically by mapping columns to values + $document = []; + foreach ($selectColumns as $index => $columnName) { + if (!isset($columns[$index])) { + continue; + } + + $value = $columns[$index]; + + if ($columnName === 'tenant') { + // Parse tenant as integer or null + $document[$columnName] = ($value === '\\N' || $value === '') ? null : (int) $value; + } elseif ($columnName === 'time') { + // Convert ClickHouse timestamp format back to ISO 8601 + $parsedTime = $value; + if (strpos($parsedTime, 'T') === false) { + $parsedTime = str_replace(' ', 'T', $parsedTime) . '+00:00'; + } + $document[$columnName] = $parsedTime; + } elseif ($columnName === 'tags') { + // Decode JSON tags column + $document[$columnName] = json_decode($value, true) ?? []; + } else { + // Get attribute metadata to check if nullable + $attribute = $this->getAttribute($columnName); + if ($attribute && !$attribute['required']) { + // Nullable field - parse null values + $document[$columnName] = $parseNullableString($value); + } else { + // Required field - use value as-is + $document[$columnName] = $value; + } + } + } - // Add tenant only if sharedTables is enabled - if ($this->sharedTables && isset($columns[6])) { - $data['tenant'] = $columns[6] === '\\\\N' ? null : (int) $columns[6]; + // Add special $id field if present + if (isset($document['id'])) { + $document['$id'] = $document['id']; + unset($document['id']); } - $metrics[] = new Metric($data); + $metrics[] = new Metric($document); } return $metrics; @@ -640,16 +1083,29 @@ private function parseResults(string $result): array /** * Get the SELECT column list for queries. - * Returns 6 columns if not using shared tables, 7 if using shared tables. + * Dynamically builds the column list from attributes. * * @return string */ private function getSelectColumns(): string { + $columns = []; + + // Add id column first + $columns[] = $this->escapeIdentifier('id'); + + // Dynamically add all attribute columns + foreach ($this->getAttributes() as $attribute) { + $id = $attribute['$id']; + $columns[] = $this->escapeIdentifier($id); + } + + // Add tenant column if shared tables are enabled if ($this->sharedTables) { - return '_id, metric, value, period, time, tags, tenant'; + $columns[] = $this->escapeIdentifier('tenant'); } - return '_id, metric, value, period, time, tags'; + + return implode(', ', $columns); } /** @@ -663,155 +1119,128 @@ private function getTenantFilter(): string return ''; } - return " AND tenant = {$this->tenant}"; + return " AND tenant = {tenant:Nullable(UInt64)}"; } /** * Get usage metrics by period. * - * @param array $queries + * @param array $queries * @return array * * @throws Exception */ public function getByPeriod(string $metric, string $period, array $queries = []): array { - $limit = 25; - $offset = 0; + $allQueries = [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + ]; + // Add custom queries foreach ($queries as $query) { - if (is_object($query) && method_exists($query, 'getMethod') && method_exists($query, 'getValue')) { - if ($query->getMethod() === 'limit') { - $limit = (int) $query->getValue(); - } elseif ($query->getMethod() === 'offset') { - $offset = (int) $query->getValue(); - } - } + $allQueries[] = $query; } - $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $tenantFilter = $this->getTenantFilter(); - - $sql = " - SELECT " . $this->getSelectColumns() . " - FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND period = :period{$tenantFilter} - ORDER BY time DESC - LIMIT :limit OFFSET :offset - FORMAT TabSeparated - "; - - $result = $this->query($sql, [ - 'metric' => $metric, - 'period' => $period, - 'limit' => $limit, - 'offset' => $offset, - ]); + // Add default ordering + $allQueries[] = Query::orderDesc(); - return $this->parseResults($result); + return $this->find($allQueries); } /** * Get usage metrics between dates. * - * @param array $queries + * @param array $queries * @return array * * @throws Exception */ public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array { - $limit = 25; - $offset = 0; + $allQueries = [ + Query::equal('metric', [$metric]), + Query::greaterThanEqual('time', $startDate), + Query::lessThanEqual('time', $endDate), + ]; + // Add custom queries foreach ($queries as $query) { - if (is_object($query) && method_exists($query, 'getMethod') && method_exists($query, 'getValue')) { - if ($query->getMethod() === 'limit') { - $limit = (int) $query->getValue(); - } elseif ($query->getMethod() === 'offset') { - $offset = (int) $query->getValue(); - } - } + $allQueries[] = $query; } - $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $tenantFilter = $this->getTenantFilter(); + // Add default ordering + $allQueries[] = Query::orderDesc(); - $sql = " - SELECT " . $this->getSelectColumns() . " - FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND time >= :startDate AND time <= :endDate{$tenantFilter} - ORDER BY time DESC - LIMIT :limit OFFSET :offset - FORMAT TabSeparated - "; - - $result = $this->query($sql, [ - 'metric' => $metric, - 'startDate' => $startDate, - 'endDate' => $endDate, - 'limit' => $limit, - 'offset' => $offset, - ]); - - return $this->parseResults($result); + return $this->find($allQueries); } /** * Count usage metrics by period. * - * @param array $queries + * @param array $queries * * @throws Exception */ public function countByPeriod(string $metric, string $period, array $queries = []): int { - $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $tenantFilter = $this->getTenantFilter(); - - $sql = " - SELECT count() as count - FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND period = :period{$tenantFilter} - FORMAT TabSeparated - "; + $allQueries = [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + ]; - $result = $this->query($sql, [ - 'metric' => $metric, - 'period' => $period, - ]); + // Add custom queries + foreach ($queries as $query) { + $allQueries[] = $query; + } - return (int) trim($result); + return $this->count($allQueries); } /** * Sum usage metric values by period. * - * @param array $queries + * @param array $queries * * @throws Exception */ public function sumByPeriod(string $metric, string $period, array $queries = []): int { $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + // Build query constraints + $allQueries = [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + ]; + + foreach ($queries as $query) { + $allQueries[] = $query; + } + + $parsed = $this->parseQueries($allQueries); + + // Build WHERE clause + $whereClause = ''; $tenantFilter = $this->getTenantFilter(); + if (!empty($parsed['filters']) || $tenantFilter) { + $conditions = $parsed['filters']; + if ($tenantFilter) { + $conditions[] = ltrim($tenantFilter, ' AND'); + // Add tenant param + $parsed['params']['tenant'] = $this->tenant; + } + $whereClause = ' WHERE ' . implode(' AND ', $conditions); + } $sql = " SELECT sum(value) as total - FROM {$escapedDatabaseAndTable} - WHERE metric = :metric AND period = :period{$tenantFilter} + FROM {$escapedTable}{$whereClause} FORMAT TabSeparated "; - $result = $this->query($sql, [ - 'metric' => $metric, - 'period' => $period, - ]); - + $result = $this->query($sql, $parsed['params']); $total = trim($result); return empty($total) ? 0 : (int) $total; @@ -825,15 +1254,20 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) public function purge(string $datetime): bool { $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); $tenantFilter = $this->getTenantFilter(); + $params = ['datetime' => $datetime]; + if ($this->sharedTables) { + $params['tenant'] = $this->tenant; + } + $sql = " - DELETE FROM {$escapedDatabaseAndTable} - WHERE time < :datetime{$tenantFilter} + DELETE FROM {$escapedTable} + WHERE time < {datetime:DateTime64(3)}{$tenantFilter} "; - $this->query($sql, ['datetime' => $datetime]); + $this->query($sql, $params); return true; } diff --git a/src/Usage/Adapter/SQL.php b/src/Usage/Adapter/SQL.php new file mode 100644 index 0000000..7dd26ad --- /dev/null +++ b/src/Usage/Adapter/SQL.php @@ -0,0 +1,202 @@ + + * + * @return array> + */ + public function getAttributes(): array + { + return [ + [ + '$id' => 'metric', + 'type' => 'string', + 'size' => 255, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => 'integer', + 'size' => 0, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => 'string', + 'size' => 16, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => 'datetime', + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => 'tags', + 'type' => 'string', + 'size' => 16777216, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['json'], + ], + ]; + } + + /** + * Get attribute documents for audit logs. + * + * @return array + */ + public function getAttributeDocuments(): array + { + return array_map(static fn (array $attribute) => new Document($attribute), $this->getAttributes()); + } + + /** + * Get index definitions for audit logs. + * + * Each index is an array with the following string keys: + * - $id: string (index identifier) + * - type: string + * - attributes: array + * + * @return array> + */ + public function getIndexes(): array + { + return [ + [ + '$id' => 'index-metric', + 'type' => 'key', + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-period', + 'type' => 'key', + 'attributes' => ['period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-metric-period', + 'type' => 'key', + 'attributes' => ['metric', 'period'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => 'index-time', + 'type' => 'key', + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => ['desc'], + ], + ]; + } + + /** + * Get index documents for audit logs. + * + * @return array + */ + public function getIndexDocuments(): array + { + return array_map(static fn (array $index) => new Document($index), $this->getIndexes()); + } + + /** + * Get a single attribute by ID. + * + * @param string $id + * @return array|null + */ + protected function getAttribute(string $id) + { + foreach ($this->getAttributes() as $attribute) { + if ($attribute['$id'] === $id) { + return $attribute; + } + } + + return null; + } + + /** + * Get SQL column definition for a given attribute ID. + * This method is database-specific and must be implemented by each concrete adapter. + * + * @param string $id Attribute identifier + * @return string Database-specific column definition + */ + abstract protected function getColumnDefinition(string $id): string; + + /** + * Get all SQL column definitions. + * Uses the concrete adapter's implementation of getColumnDefinition. + * + * @return array + */ + protected function getAllColumnDefinitions(): array + { + $definitions = []; + foreach ($this->getAttributes() as $attribute) { + /** @var string $id */ + $id = $attribute['$id']; + $definitions[] = $this->getColumnDefinition($id); + } + + return $definitions; + } +} diff --git a/src/Usage/Query.php b/src/Usage/Query.php new file mode 100644 index 0000000..3cf7fe0 --- /dev/null +++ b/src/Usage/Query.php @@ -0,0 +1,288 @@ + + */ + protected array $values = []; + + /** + * Construct a new query object + * + * @param string $method + * @param string $attribute + * @param array $values + */ + public function __construct(string $method, string $attribute = '', array $values = []) + { + $this->method = $method; + $this->attribute = $attribute; + $this->values = $values; + } + + /** + * @return string + */ + public function getMethod(): string + { + return $this->method; + } + + /** + * @return string + */ + public function getAttribute(): string + { + return $this->attribute; + } + + /** + * @return array + */ + public function getValues(): array + { + return $this->values; + } + + /** + * @param mixed $default + * @return mixed + */ + public function getValue(mixed $default = null): mixed + { + return $this->values[0] ?? $default; + } + + /** + * Filter by equal condition + * + * @param string $attribute + * @param mixed $value + * @return self + */ + public static function equal(string $attribute, mixed $value): self + { + return new self(self::TYPE_EQUAL, $attribute, [$value]); + } + + /** + * Filter by less than condition + * + * @param string $attribute + * @param mixed $value + * @return self + */ + public static function lessThan(string $attribute, mixed $value): self + { + return new self(self::TYPE_LESSER, $attribute, [$value]); + } + + /** + * Filter by greater than condition + * + * @param string $attribute + * @param mixed $value + * @return self + */ + public static function greaterThan(string $attribute, mixed $value): self + { + return new self(self::TYPE_GREATER, $attribute, [$value]); + } + + /** + * Filter by BETWEEN condition + * + * @param string $attribute + * @param mixed $start + * @param mixed $end + * @return self + */ + public static function between(string $attribute, mixed $start, mixed $end): self + { + return new self(self::TYPE_BETWEEN, $attribute, [$start, $end]); + } + + /** + * Filter by IN condition + * + * @param string $attribute + * @param array $values + * @return self + */ + public static function in(string $attribute, array $values): self + { + return new self(self::TYPE_IN, $attribute, $values); + } + + /** + * Order by descending + * + * @param string $attribute + * @return self + */ + public static function orderDesc(string $attribute = 'time'): self + { + return new self(self::TYPE_ORDER_DESC, $attribute); + } + + /** + * Order by ascending + * + * @param string $attribute + * @return self + */ + public static function orderAsc(string $attribute = 'time'): self + { + return new self(self::TYPE_ORDER_ASC, $attribute); + } + + /** + * Limit number of results + * + * @param int $limit + * @return self + */ + public static function limit(int $limit): self + { + return new self(self::TYPE_LIMIT, '', [$limit]); + } + + /** + * Offset results + * + * @param int $offset + * @return self + */ + public static function offset(int $offset): self + { + return new self(self::TYPE_OFFSET, '', [$offset]); + } + + /** + * Parse query from JSON string + * + * @param string $query + * @return self + * @throws \Exception + */ + public static function parse(string $query): self + { + try { + $query = \json_decode($query, true, flags: JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new \Exception('Invalid query: ' . $e->getMessage()); + } + + if (!\is_array($query)) { + throw new \Exception('Invalid query. Must be an array, got ' . \gettype($query)); + } + + return self::parseQuery($query); + } + + /** + * Parse an array of queries + * + * @param array $queries + * @return array + * @throws \Exception + */ + public static function parseQueries(array $queries): array + { + $parsed = []; + + foreach ($queries as $query) { + $parsed[] = self::parse($query); + } + + return $parsed; + } + + /** + * Parse query from array + * + * @param array $query + * @return self + * @throws \Exception + */ + protected static function parseQuery(array $query): self + { + $method = $query['method'] ?? ''; + $attribute = $query['attribute'] ?? ''; + $values = $query['values'] ?? []; + + if (!\is_string($method)) { + throw new \Exception('Invalid query method. Must be a string, got ' . \gettype($method)); + } + + if (!\is_string($attribute)) { + throw new \Exception('Invalid query attribute. Must be a string, got ' . \gettype($attribute)); + } + + if (!\is_array($values)) { + throw new \Exception('Invalid query values. Must be an array, got ' . \gettype($values)); + } + + return new self($method, $attribute, $values); + } + + /** + * Convert query to array + * + * @return array + */ + public function toArray(): array + { + $array = ['method' => $this->method]; + + if (!empty($this->attribute)) { + $array['attribute'] = $this->attribute; + } + + $array['values'] = $this->values; + + return $array; + } + + /** + * Convert query to JSON string + * + * @return string + * @throws \Exception + */ + public function toString(): string + { + try { + return \json_encode($this->toArray(), flags: JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new \Exception('Invalid Json: ' . $e->getMessage()); + } + } +} diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 793737b..d5d4775 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -10,94 +10,15 @@ */ class Usage { - public const COLLECTION = 'usage'; - - /** - * @deprecated Use Adapter::PERIODS instead - * - * @var array - */ - public const PERIODS = Adapter::PERIODS; - - public const ATTRIBUTES = [ - [ - '$id' => 'metric', - 'type' => 'string', - 'size' => 255, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'value', - 'type' => 'integer', - 'size' => 0, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'period', - 'type' => 'string', - 'size' => 16, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'time', - 'type' => 'datetime', - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => 'tags', - 'type' => 'string', - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], + public const PERIOD_1H = '1h'; + public const PERIOD_1D = '1d'; + public const PERIOD_INF = 'inf'; + public const PERIODS = [ + self::PERIOD_1H => 'Y-m-d H:00', + self::PERIOD_1D => 'Y-m-d 00:00', + self::PERIOD_INF => '0000-00-00 00:00', ]; - public const INDEXES = [ - [ - '$id' => 'index-metric', - 'type' => 'key', - 'attributes' => ['metric'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-period', - 'type' => 'key', - 'attributes' => ['period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-metric-period', - 'type' => 'key', - 'attributes' => ['metric', 'period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-time', - 'type' => 'key', - 'attributes' => ['time'], - 'lengths' => [], - 'orders' => ['desc'], - ], - ]; private Adapter $adapter; @@ -127,17 +48,9 @@ public function getAdapter(): Adapter * @param array> $indexes Index definitions * @throws \Exception */ - public function setup(string $table = self::COLLECTION, array $columns = [], array $indexes = []): void + public function setup(): void { - // Use legacy constants if no columns/indexes provided (for backward compatibility) - if (empty($columns)) { - $columns = self::ATTRIBUTES; - } - if (empty($indexes)) { - $indexes = self::INDEXES; - } - - $this->adapter->setup($table, $columns, $indexes); + $this->adapter->setup(); } /** diff --git a/tests/Usage/QueryTest.php b/tests/Usage/QueryTest.php new file mode 100644 index 0000000..aeed447 --- /dev/null +++ b/tests/Usage/QueryTest.php @@ -0,0 +1,225 @@ +assertEquals(Query::TYPE_EQUAL, $query->getMethod()); + $this->assertEquals('userId', $query->getAttribute()); + $this->assertEquals(['123'], $query->getValues()); + + // Test lessThan + $query = Query::lessThan('time', '2024-01-01'); + $this->assertEquals(Query::TYPE_LESSER, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2024-01-01'], $query->getValues()); + + // Test greaterThan + $query = Query::greaterThan('time', '2023-01-01'); + $this->assertEquals(Query::TYPE_GREATER, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2023-01-01'], $query->getValues()); + + // Test between + $query = Query::between('time', '2023-01-01', '2024-01-01'); + $this->assertEquals(Query::TYPE_BETWEEN, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2023-01-01', '2024-01-01'], $query->getValues()); + + // Test in + $query = Query::in('event', ['create', 'update', 'delete']); + $this->assertEquals(Query::TYPE_IN, $query->getMethod()); + $this->assertEquals('event', $query->getAttribute()); + $this->assertEquals(['create', 'update', 'delete'], $query->getValues()); + + // Test orderDesc + $query = Query::orderDesc('time'); + $this->assertEquals(Query::TYPE_ORDER_DESC, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals([], $query->getValues()); + + // Test orderAsc + $query = Query::orderAsc('userId'); + $this->assertEquals(Query::TYPE_ORDER_ASC, $query->getMethod()); + $this->assertEquals('userId', $query->getAttribute()); + $this->assertEquals([], $query->getValues()); + + // Test limit + $query = Query::limit(10); + $this->assertEquals(Query::TYPE_LIMIT, $query->getMethod()); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([10], $query->getValues()); + + // Test offset + $query = Query::offset(5); + $this->assertEquals(Query::TYPE_OFFSET, $query->getMethod()); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([5], $query->getValues()); + } + + /** + * Test Query parse and toString methods + */ + public function testQueryParseAndToString(): void + { + // Test parsing equal query + $json = '{"method":"equal","attribute":"userId","values":["123"]}'; + $query = Query::parse($json); + $this->assertEquals(Query::TYPE_EQUAL, $query->getMethod()); + $this->assertEquals('userId', $query->getAttribute()); + $this->assertEquals(['123'], $query->getValues()); + + // Test toString + $query = Query::equal('event', 'create'); + $json = $query->toString(); + $this->assertJson($json); + + $parsed = Query::parse($json); + $this->assertEquals(Query::TYPE_EQUAL, $parsed->getMethod()); + $this->assertEquals('event', $parsed->getAttribute()); + $this->assertEquals(['create'], $parsed->getValues()); + + // Test toArray + $array = $query->toArray(); + $this->assertArrayHasKey('method', $array); + $this->assertArrayHasKey('attribute', $array); + $this->assertArrayHasKey('values', $array); + $this->assertEquals(Query::TYPE_EQUAL, $array['method']); + $this->assertEquals('event', $array['attribute']); + $this->assertEquals(['create'], $array['values']); + } + + /** + * Test Query parseQueries method + */ + public function testQueryParseQueries(): void + { + $queries = [ + '{"method":"equal","attribute":"userId","values":["123"]}', + '{"method":"greaterThan","attribute":"time","values":["2023-01-01"]}', + '{"method":"limit","values":[10]}' + ]; + + $parsed = Query::parseQueries($queries); + + $this->assertCount(3, $parsed); + $this->assertInstanceOf(Query::class, $parsed[0]); + $this->assertInstanceOf(Query::class, $parsed[1]); + $this->assertInstanceOf(Query::class, $parsed[2]); + + $this->assertEquals(Query::TYPE_EQUAL, $parsed[0]->getMethod()); + $this->assertEquals(Query::TYPE_GREATER, $parsed[1]->getMethod()); + $this->assertEquals(Query::TYPE_LIMIT, $parsed[2]->getMethod()); + } + + /** + * Test Query getValue method + */ + public function testGetValue(): void + { + $query = Query::equal('userId', '123'); + $this->assertEquals('123', $query->getValue()); + + $query = Query::limit(10); + $this->assertEquals(10, $query->getValue()); + + // Test with default value + $query = Query::orderAsc('time'); + $this->assertNull($query->getValue()); + $this->assertEquals('default', $query->getValue('default')); + } + + /** + * Test Query with empty attribute + */ + public function testQueryWithEmptyAttribute(): void + { + $query = Query::limit(25); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([25], $query->getValues()); + + $query = Query::offset(10); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([10], $query->getValues()); + } + + /** + * Test Query parse with invalid JSON + */ + public function testQueryParseInvalidJson(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query'); + + Query::parse('{"method":"equal","attribute":"userId"'); // Invalid JSON + } + + /** + * Test Query parse with non-array value + */ + public function testQueryParseNonArray(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query. Must be an array'); + + Query::parse('"string"'); + } + + /** + * Test Query parse with invalid method type + */ + public function testQueryParseInvalidMethodType(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query method. Must be a string'); + + Query::parse('{"method":["array"],"attribute":"test","values":[]}'); + } + + /** + * Test Query parse with invalid attribute type + */ + public function testQueryParseInvalidAttributeType(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query attribute. Must be a string'); + + Query::parse('{"method":"equal","attribute":123,"values":[]}'); + } + + /** + * Test Query parse with invalid values type + */ + public function testQueryParseInvalidValuesType(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query values. Must be an array'); + + Query::parse('{"method":"equal","attribute":"test","values":"string"}'); + } + + /** + * Test Query toString with complex values + */ + public function testQueryToStringWithComplexValues(): void + { + $query = Query::between('time', '2023-01-01', '2024-12-31'); + $json = $query->toString(); + $this->assertJson($json); + + $parsed = Query::parse($json); + $this->assertEquals(Query::TYPE_BETWEEN, $parsed->getMethod()); + $this->assertEquals('time', $parsed->getAttribute()); + $this->assertEquals(['2023-01-01', '2024-12-31'], $parsed->getValues()); + } +} From 5ca8ac140f9b5c604abbd85f5d08bb1cbe9b03f0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 01:38:11 +0000 Subject: [PATCH 19/44] feat: Enhance Adapter with find and count methods, update setup logic --- src/Usage/Adapter.php | 20 +++++++-- src/Usage/Adapter/ClickHouse.php | 12 ++--- src/Usage/Adapter/Database.php | 76 +++++++++++++++++++++++++++----- src/Usage/Usage.php | 27 ++++++++++-- 4 files changed, 111 insertions(+), 24 deletions(-) diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 99b3511..03b67a3 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -11,10 +11,6 @@ abstract public function getName(): string; /** * Setup database structure - * - * @param string $table Table name - * @param array> $columns Column definitions - * @param array> $indexes Index definitions */ abstract public function setup(): void; @@ -66,4 +62,20 @@ abstract public function sumByPeriod(string $metric, string $period, array $quer * Purge old usage metrics */ abstract public function purge(string $datetime): bool; + + /** + * Find metrics using Query objects. + * + * @param array<\Utopia\Database\Query> $queries + * @return array + */ + abstract public function find(array $queries = []): array; + + /** + * Count metrics using Query objects. + * + * @param array<\Utopia\Database\Query> $queries + * @return int + */ + abstract public function count(array $queries = []): int; } diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 9947737..9569dda 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -542,13 +542,12 @@ private function getColumnType(string $id): string } // Map attribute type to ClickHouse type - $attributeType = $attribute['type'] ?? 'string'; + $attributeType = is_string($attribute['type'] ?? null) ? $attribute['type'] : 'string'; $baseType = match ($attributeType) { 'integer' => 'Int64', 'float' => 'Float64', 'boolean' => 'UInt8', 'datetime' => 'DateTime64(3)', - Database::VAR_DATETIME => 'DateTime64(3)', default => 'String', }; @@ -918,8 +917,7 @@ private function parseQueries(array $queries): array $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); $inParams = []; - $valuesToUse = is_array($values) ? $values : [$values]; - foreach ($valuesToUse as $value) { + foreach ($values as $value) { $paramName = 'param_' . $paramCounter++; $inParams[] = "{{$paramName}:String}"; $params[$paramName] = $this->formatParamValue($value); @@ -1058,7 +1056,7 @@ private function parseResults(string $result): array $document[$columnName] = json_decode($value, true) ?? []; } else { // Get attribute metadata to check if nullable - $attribute = $this->getAttribute($columnName); + $attribute = is_string($columnName) ? $this->getAttribute($columnName) : null; if ($attribute && !$attribute['required']) { // Nullable field - parse null values $document[$columnName] = $parseNullableString($value); @@ -1097,7 +1095,9 @@ private function getSelectColumns(): string // Dynamically add all attribute columns foreach ($this->getAttributes() as $attribute) { $id = $attribute['$id']; - $columns[] = $this->escapeIdentifier($id); + if (is_string($id)) { + $columns[] = $this->escapeIdentifier($id); + } } // Add tenant column if shared tables are enabled diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index c2a54c0..e53c062 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -9,6 +9,7 @@ use Utopia\Exception; use Utopia\Usage\Adapter; use Utopia\Usage\Metric; +use Utopia\Usage\Usage; class Database extends Adapter { @@ -26,26 +27,41 @@ public function getName(): string return 'Database'; } - public function setup(string $table, array $columns, array $indexes): void + public function setup(): void { - $this->collection = $table; + $this->collection = 'usage'; if (! $this->db->exists($this->db->getDatabase())) { throw new Exception('You need to create the database before running Usage setup'); } + // Define columns based on the metric structure + $columns = [ + ['$id' => 'metric', 'type' => 'string', 'size' => 255, 'required' => true], + ['$id' => 'value', 'type' => 'integer', 'required' => true], + ['$id' => 'period', 'type' => 'string', 'size' => 10, 'required' => true], + ['$id' => 'time', 'type' => 'datetime', 'required' => true], + ['$id' => 'tags', 'type' => 'string', 'size' => 16777216, 'required' => false], // JSON text + ]; + + $indexes = [ + ['$id' => 'index-metric', 'type' => 'key', 'attributes' => ['metric']], + ['$id' => 'index-period', 'type' => 'key', 'attributes' => ['period']], + ['$id' => 'index-time', 'type' => 'key', 'attributes' => ['time']], + ]; + $attributes = \array_map(function ($attribute) { return new Document($attribute); }, $columns); - $indexes = \array_map(function ($index) { + $indexDocs = \array_map(function ($index) { return new Document($index); }, $indexes); try { $this->db->createCollection( - $table, + $this->collection, $attributes, - $indexes + $indexDocs ); } catch (DuplicateException) { // Collection already exists @@ -54,14 +70,14 @@ public function setup(string $table, array $columns, array $indexes): void public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool { - if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + if (! isset(Usage::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); } $now = new \DateTime(); $time = $period === 'inf' ? '1000-01-01 00:00:00' - : $now->format(self::PERIODS[$period]); + : $now->format(Usage::PERIODS[$period]); $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags) { $this->db->createDocument($this->collection, new Document([ @@ -83,14 +99,14 @@ public function logBatch(array $metrics): bool $documents = \array_map(function ($metric) { $period = $metric['period'] ?? '1h'; - if (! isset(self::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(self::PERIODS))); + if (! isset(Usage::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); } $now = new \DateTime(); $time = $period === 'inf' ? '1000-01-01 00:00:00' - : $now->format(self::PERIODS[$period]); + : $now->format(Usage::PERIODS[$period]); return new Document([ '$permissions' => [], @@ -193,4 +209,42 @@ public function purge(string $datetime): bool return true; } + + /** + * Find metrics using Query objects. + * + * @param array $queries + * @return array + */ + public function find(array $queries = []): array + { + /** @var array $result */ + $result = $this->db->getAuthorization()->skip(function () use ($queries) { + return $this->db->find( + collection: $this->collection, + queries: $queries, + ); + }); + + return \array_map(fn ($doc) => new Metric($doc->getArrayCopy()), $result); + } + + /** + * Count metrics using Query objects. + * + * @param array $queries + * @return int + */ + public function count(array $queries = []): int + { + /** @var int $count */ + $count = $this->db->getAuthorization()->skip(function () use ($queries) { + return $this->db->count( + collection: $this->collection, + queries: $queries + ); + }); + + return $count; + } } diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index d5d4775..2e31951 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -43,9 +43,6 @@ public function getAdapter(): Adapter /** * Setup the usage metrics storage. * - * @param string $table Table name for storing usage metrics - * @param array> $columns Column definitions - * @param array> $indexes Index definitions * @throws \Exception */ public function setup(): void @@ -136,4 +133,28 @@ public function purge(string $datetime): bool { return $this->adapter->purge($datetime); } + + /** + * Find metrics using Query objects. + * + * @param array<\Utopia\Database\Query> $queries + * @return array + * @throws \Exception + */ + public function find(array $queries = []): array + { + return $this->adapter->find($queries); + } + + /** + * Count metrics using Query objects. + * + * @param array<\Utopia\Database\Query> $queries + * @return int + * @throws \Exception + */ + public function count(array $queries = []): int + { + return $this->adapter->count($queries); + } } From 90c804b234a8c185a0a199a62699fc3f59db33fc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 01:44:37 +0000 Subject: [PATCH 20/44] feat: Refactor Database adapter to extend SQL and streamline setup logic --- src/Usage/Adapter/Database.php | 36 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index e53c062..cb639c9 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -11,7 +11,7 @@ use Utopia\Usage\Metric; use Utopia\Usage\Usage; -class Database extends Adapter +class Database extends SQL { protected string $collection; @@ -34,28 +34,9 @@ public function setup(): void throw new Exception('You need to create the database before running Usage setup'); } - // Define columns based on the metric structure - $columns = [ - ['$id' => 'metric', 'type' => 'string', 'size' => 255, 'required' => true], - ['$id' => 'value', 'type' => 'integer', 'required' => true], - ['$id' => 'period', 'type' => 'string', 'size' => 10, 'required' => true], - ['$id' => 'time', 'type' => 'datetime', 'required' => true], - ['$id' => 'tags', 'type' => 'string', 'size' => 16777216, 'required' => false], // JSON text - ]; - - $indexes = [ - ['$id' => 'index-metric', 'type' => 'key', 'attributes' => ['metric']], - ['$id' => 'index-period', 'type' => 'key', 'attributes' => ['period']], - ['$id' => 'index-time', 'type' => 'key', 'attributes' => ['time']], - ]; - - $attributes = \array_map(function ($attribute) { - return new Document($attribute); - }, $columns); - - $indexDocs = \array_map(function ($index) { - return new Document($index); - }, $indexes); + // Use column and index definitions from parent SQL adapter + $attributes = $this->getAttributeDocuments(); + $indexDocs = $this->getIndexDocuments(); try { $this->db->createCollection( @@ -68,6 +49,15 @@ public function setup(): void } } + /** + * Get column definition for Database adapter (not used, but required by SQL parent) + */ + protected function getColumnDefinition(string $id): string + { + // Not used in Database adapter, but required by SQL abstract class + return ''; + } + public function log(string $metric, int $value, string $period = '1h', array $tags = []): bool { if (! isset(Usage::PERIODS[$period])) { From da10c563b97cd1395f911154cdd48038318b7765 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 01:46:11 +0000 Subject: [PATCH 21/44] cleanup: Remove unused imports in ClickHouse and Database adapters --- src/Usage/Adapter/ClickHouse.php | 2 -- src/Usage/Adapter/Database.php | 1 - 2 files changed, 3 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 9569dda..83af521 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -3,10 +3,8 @@ namespace Utopia\Usage\Adapter; use Exception; -use Utopia\Database\Database; use Utopia\Database\Query; use Utopia\Fetch\Client; -use Utopia\Usage\Adapter; use Utopia\Usage\Metric; use Utopia\Usage\Usage; use Utopia\Validator\Hostname; diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index cb639c9..b500ff3 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -7,7 +7,6 @@ use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Query; use Utopia\Exception; -use Utopia\Usage\Adapter; use Utopia\Usage\Metric; use Utopia\Usage\Usage; From 78ea8d7a19961b98b755322a0b8143d0364d9718 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 02:16:36 +0000 Subject: [PATCH 22/44] feat: Add data validation methods for metrics in ClickHouse adapter --- src/Usage/Adapter/ClickHouse.php | 192 ++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 6 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 83af521..3757ac5 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -560,6 +560,104 @@ protected function getColumnDefinition(string $id): string return "{$escapedId} {$type}"; } + /** + * Validate data format against attribute metadata. + * + * @param array $data + * @throws Exception + */ + private function validateDataFormat(array $data): void + { + $attributes = $this->getAttributes(); + + foreach ($attributes as $attribute) { + /** @var string $attrId */ + $attrId = $attribute['$id']; + $required = $attribute['required'] ?? false; + $type = $attribute['type'] ?? 'string'; + /** @var int $size */ + $size = $attribute['size'] ?? 0; + + // Check if required attribute is present + if ($required && !isset($data[$attrId])) { + throw new Exception("Required attribute '{$attrId}' is missing"); + } + + // Skip validation if not present and not required + if (!isset($data[$attrId])) { + continue; + } + + $value = $data[$attrId]; + + // Special handling for tags: accept array (will be JSON-encoded) + if ($attrId === 'tags') { + if (!is_array($value)) { + throw new Exception("Attribute '{$attrId}' must be an array, got " . gettype($value)); + } + continue; + } + + // Validate based on attribute type + match ($type) { + 'string' => $this->validateStringAttribute($attrId, $value, $size), + 'integer' => $this->validateIntegerAttribute($attrId, $value), + 'datetime' => $this->validateDatetimeAttribute($attrId, $value), + default => null, + }; + } + } + + /** + * Validate string attribute value. + * + * @throws Exception + */ + private function validateStringAttribute(string $attrId, mixed $value, int $size): void + { + if (!is_string($value)) { + throw new Exception("Attribute '{$attrId}' must be a string, got " . gettype($value)); + } + + if ($size > 0 && strlen($value) > $size) { + throw new Exception("Attribute '{$attrId}' exceeds maximum size of {$size} characters"); + } + } + + /** + * Validate integer attribute value. + * + * @throws Exception + */ + private function validateIntegerAttribute(string $attrId, mixed $value): void + { + if (!is_int($value)) { + throw new Exception("Attribute '{$attrId}' must be an integer, got " . gettype($value)); + } + } + + /** + * Validate datetime attribute value. + * + * @throws Exception + */ + private function validateDatetimeAttribute(string $attrId, mixed $value): void + { + if ($value instanceof \DateTime) { + return; // Valid DateTime object + } + + if (!is_string($value)) { + throw new Exception("Attribute '{$attrId}' must be a DateTime object or string, got " . gettype($value)); + } + + try { + new \DateTime($value); + } catch (\Exception $e) { + throw new Exception("Attribute '{$attrId}' is not a valid datetime string: {$e->getMessage()}"); + } + } + /** * Log a usage metric. * @@ -569,10 +667,38 @@ protected function getColumnDefinition(string $id): string */ public function log(string $metric, int $value, string $period = Usage::PERIOD_1H, array $tags = []): bool { + // Validate period if (!isset(Usage::PERIODS[$period])) { throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); } + // Validate metric and value + if (empty($metric)) { + throw new Exception('Metric cannot be empty'); + } + + if (strlen($metric) > 255) { + throw new Exception('Metric exceeds maximum size of 255 characters'); + } + + if ($value < 0) { + throw new Exception('Value cannot be negative'); + } + + // Validate tags format + if (!is_array($tags)) { + throw new Exception('Tags must be an array'); + } + + // Validate complete data structure + $data = [ + 'metric' => $metric, + 'value' => $value, + 'period' => $period, + 'tags' => $tags, + ]; + $this->validateDataFormat($data); + $id = uniqid('', true); $now = new \DateTime(); $timestamp = $this->formatDateTime($now); @@ -642,6 +768,66 @@ public function logBatch(array $metrics): bool return true; } + // Validate all metrics before processing + foreach ($metrics as $index => $metricData) { + try { + // Validate required fields exist + if (!isset($metricData['metric'])) { + throw new Exception("Metric #{$index}: 'metric' is required"); + } + if (!isset($metricData['value'])) { + throw new Exception("Metric #{$index}: 'value' is required"); + } + + $metric = $metricData['metric']; + $value = $metricData['value']; + $period = $metricData['period'] ?? Usage::PERIOD_1H; + + // Validate types + if (!is_string($metric)) { + throw new Exception("Metric #{$index}: 'metric' must be a string, got " . gettype($metric)); + } + if (!is_int($value)) { + throw new Exception("Metric #{$index}: 'value' must be an integer, got " . gettype($value)); + } + if (!is_string($period)) { + throw new Exception("Metric #{$index}: 'period' must be a string, got " . gettype($period)); + } + + // Validate metric and value constraints + if (empty($metric)) { + throw new Exception("Metric #{$index}: 'metric' cannot be empty"); + } + if (strlen($metric) > 255) { + throw new Exception("Metric #{$index}: 'metric' exceeds maximum size of 255 characters"); + } + if ($value < 0) { + throw new Exception("Metric #{$index}: 'value' cannot be negative"); + } + + // Validate period + if (!isset(Usage::PERIODS[$period])) { + throw new Exception("Metric #{$index}: Invalid period '{$period}'. Allowed: " . implode(', ', array_keys(Usage::PERIODS))); + } + + // Validate tags if provided + if (isset($metricData['tags']) && !is_array($metricData['tags'])) { + throw new Exception("Metric #{$index}: 'tags' must be an array, got " . gettype($metricData['tags'])); + } + + // Validate complete data structure against attributes + $data = [ + 'metric' => $metric, + 'value' => $value, + 'period' => $period, + 'tags' => $metricData['tags'] ?? [], + ]; + $this->validateDataFormat($data); + } catch (Exception $e) { + throw new Exception($e->getMessage()); + } + } + $tableName = $this->getTableName(); $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); @@ -661,18 +847,12 @@ public function logBatch(array $metrics): bool foreach ($metrics as $metricData) { $period = $metricData['period'] ?? Usage::PERIOD_1H; - if (!isset(Usage::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); - } - $id = uniqid('', true); $now = new \DateTime(); $timestamp = $this->formatDateTime($now); $metric = $metricData['metric']; $value = $metricData['value']; - assert(is_string($metric)); - assert(is_int($value)); $valuePlaceholders = []; From abf8c14e1534238403c3d9aae61f9142a3166241 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 02:29:03 +0000 Subject: [PATCH 23/44] feat: Introduce Metric class for schema definition and validation in adapters --- src/Usage/Adapter/ClickHouse.php | 106 +----- src/Usage/Adapter/SQL.php | 97 +----- src/Usage/Metric.php | 204 +++++++++++ tests/Usage/MetricTest.php | 576 +++++++++++++++++++++++++++++++ 4 files changed, 790 insertions(+), 193 deletions(-) create mode 100644 tests/Usage/MetricTest.php diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 3757ac5..091e5ad 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -560,104 +560,6 @@ protected function getColumnDefinition(string $id): string return "{$escapedId} {$type}"; } - /** - * Validate data format against attribute metadata. - * - * @param array $data - * @throws Exception - */ - private function validateDataFormat(array $data): void - { - $attributes = $this->getAttributes(); - - foreach ($attributes as $attribute) { - /** @var string $attrId */ - $attrId = $attribute['$id']; - $required = $attribute['required'] ?? false; - $type = $attribute['type'] ?? 'string'; - /** @var int $size */ - $size = $attribute['size'] ?? 0; - - // Check if required attribute is present - if ($required && !isset($data[$attrId])) { - throw new Exception("Required attribute '{$attrId}' is missing"); - } - - // Skip validation if not present and not required - if (!isset($data[$attrId])) { - continue; - } - - $value = $data[$attrId]; - - // Special handling for tags: accept array (will be JSON-encoded) - if ($attrId === 'tags') { - if (!is_array($value)) { - throw new Exception("Attribute '{$attrId}' must be an array, got " . gettype($value)); - } - continue; - } - - // Validate based on attribute type - match ($type) { - 'string' => $this->validateStringAttribute($attrId, $value, $size), - 'integer' => $this->validateIntegerAttribute($attrId, $value), - 'datetime' => $this->validateDatetimeAttribute($attrId, $value), - default => null, - }; - } - } - - /** - * Validate string attribute value. - * - * @throws Exception - */ - private function validateStringAttribute(string $attrId, mixed $value, int $size): void - { - if (!is_string($value)) { - throw new Exception("Attribute '{$attrId}' must be a string, got " . gettype($value)); - } - - if ($size > 0 && strlen($value) > $size) { - throw new Exception("Attribute '{$attrId}' exceeds maximum size of {$size} characters"); - } - } - - /** - * Validate integer attribute value. - * - * @throws Exception - */ - private function validateIntegerAttribute(string $attrId, mixed $value): void - { - if (!is_int($value)) { - throw new Exception("Attribute '{$attrId}' must be an integer, got " . gettype($value)); - } - } - - /** - * Validate datetime attribute value. - * - * @throws Exception - */ - private function validateDatetimeAttribute(string $attrId, mixed $value): void - { - if ($value instanceof \DateTime) { - return; // Valid DateTime object - } - - if (!is_string($value)) { - throw new Exception("Attribute '{$attrId}' must be a DateTime object or string, got " . gettype($value)); - } - - try { - new \DateTime($value); - } catch (\Exception $e) { - throw new Exception("Attribute '{$attrId}' is not a valid datetime string: {$e->getMessage()}"); - } - } - /** * Log a usage metric. * @@ -690,14 +592,14 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 throw new Exception('Tags must be an array'); } - // Validate complete data structure + // Validate complete data structure using Metric class $data = [ 'metric' => $metric, 'value' => $value, 'period' => $period, 'tags' => $tags, ]; - $this->validateDataFormat($data); + Metric::validate($data); $id = uniqid('', true); $now = new \DateTime(); @@ -815,14 +717,14 @@ public function logBatch(array $metrics): bool throw new Exception("Metric #{$index}: 'tags' must be an array, got " . gettype($metricData['tags'])); } - // Validate complete data structure against attributes + // Validate complete data structure using Metric class $data = [ 'metric' => $metric, 'value' => $value, 'period' => $period, 'tags' => $metricData['tags'] ?? [], ]; - $this->validateDataFormat($data); + Metric::validate($data); } catch (Exception $e) { throw new Exception($e->getMessage()); } diff --git a/src/Usage/Adapter/SQL.php b/src/Usage/Adapter/SQL.php index 7dd26ad..c56d2d9 100644 --- a/src/Usage/Adapter/SQL.php +++ b/src/Usage/Adapter/SQL.php @@ -3,6 +3,7 @@ namespace Utopia\Usage\Adapter; use Utopia\Usage\Adapter; +use Utopia\Usage\Metric; use Utopia\Database\Database; use Utopia\Database\Document; @@ -10,7 +11,7 @@ * Base SQL Adapter for Audit * * This is an abstract base class for SQL-based adapters (Database, ClickHouse, etc.) - * It provides common functionality and schema definitions for all SQL adapters. + * It provides common functionality and references schema definitions from the Metric class. */ abstract class SQL extends Adapter { @@ -29,67 +30,13 @@ public function getCollectionName(): string /** * Get attribute definitions for audit logs. * - * Each attribute is an array with the following string keys: - * - $id: string (attribute identifier) - * - type: string - * - size: int - * - required: bool - * - signed: bool - * - array: bool - * - filters: array + * Delegates to Metric class which defines the metric schema. * * @return array> */ public function getAttributes(): array { - return [ - [ - '$id' => 'metric', - 'type' => 'string', - 'size' => 255, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'value', - 'type' => 'integer', - 'size' => 0, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'period', - 'type' => 'string', - 'size' => 16, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'time', - 'type' => 'datetime', - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => 'tags', - 'type' => 'string', - 'size' => 16777216, - 'required' => false, - 'signed' => true, - 'array' => false, - 'filters' => ['json'], - ], - ]; + return Metric::getSchema(); } /** @@ -105,45 +52,13 @@ public function getAttributeDocuments(): array /** * Get index definitions for audit logs. * - * Each index is an array with the following string keys: - * - $id: string (index identifier) - * - type: string - * - attributes: array + * Delegates to Metric class which defines the metric indexes. * * @return array> */ public function getIndexes(): array { - return [ - [ - '$id' => 'index-metric', - 'type' => 'key', - 'attributes' => ['metric'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-period', - 'type' => 'key', - 'attributes' => ['period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-metric-period', - 'type' => 'key', - 'attributes' => ['metric', 'period'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => 'index-time', - 'type' => 'key', - 'attributes' => ['time'], - 'lengths' => [], - 'orders' => ['desc'], - ], - ]; + return Metric::getIndexes(); } /** diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index c04e6a2..2eaea1a 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -297,4 +297,208 @@ public function toArray(): array { return $this->getArrayCopy(); } + + /** + * Get metric schema definition. + * + * Returns the attribute schema that defines the structure of metric data. + * This is used by adapters to understand the metric structure and create + * appropriate database tables/collections. + * + * Each attribute definition includes: + * - $id: string (attribute identifier) + * - type: string (attribute data type: string, integer, datetime) + * - size: int (max size for strings, 0 for others) + * - required: bool (whether the attribute is required) + * - signed: bool (for numeric types) + * - array: bool (whether value is an array) + * - filters: array (data filters/validation rules) + * + * @return array> + */ + public static function getSchema(): array + { + return [ + [ + '$id' => 'metric', + 'type' => 'string', + 'size' => 255, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => 'integer', + 'size' => 0, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => 'string', + 'size' => 16, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => 'datetime', + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => 'tags', + 'type' => 'string', + 'size' => 16777216, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['json'], + ], + ]; + } + + /** + * Get metric indexes definition. + * + * Returns the index definitions that should be created on the metric table. + * Indexes are used to optimize query performance for common filter operations. + * + * @return array> + */ + public static function getIndexes(): array + { + return [ + [ + '$id' => 'index-metric', + 'type' => 'key', + 'attributes' => ['metric'], + ], + [ + '$id' => 'index-period', + 'type' => 'key', + 'attributes' => ['period'], + ], + [ + '$id' => 'index-time', + 'type' => 'key', + 'attributes' => ['time'], + ], + ]; + } + + /** + * Validate metric data against schema. + * + * Validates that metric data conforms to the schema definition. + * Checks for: + * - Required attributes are present + * - Data types match schema types + * - String sizes don't exceed limits + * - Values are in valid ranges + * + * @param array $data The metric data to validate + * @throws \Exception If validation fails + */ + public static function validate(array $data): void + { + $schema = self::getSchema(); + + foreach ($schema as $attribute) { + /** @var string $attrId */ + $attrId = $attribute['$id']; + $required = $attribute['required'] ?? false; + $type = $attribute['type'] ?? 'string'; + /** @var int $size */ + $size = $attribute['size'] ?? 0; + + // Check if required attribute is present + if ($required && !isset($data[$attrId])) { + throw new \Exception("Required attribute '{$attrId}' is missing"); + } + + // Skip validation if not present and not required + if (!isset($data[$attrId])) { + continue; + } + + $value = $data[$attrId]; + + // Special handling for tags: accept array (will be JSON-encoded) + if ($attrId === 'tags') { + if (!is_array($value)) { + throw new \Exception("Attribute '{$attrId}' must be an array, got " . gettype($value)); + } + continue; + } + + // Validate based on attribute type + match ($type) { + 'string' => self::validateStringAttribute($attrId, $value, $size), + 'integer' => self::validateIntegerAttribute($attrId, $value), + 'datetime' => self::validateDatetimeAttribute($attrId, $value), + default => null, + }; + } + } + + /** + * Validate string attribute value. + * + * @throws \Exception + */ + private static function validateStringAttribute(string $attrId, mixed $value, int $size): void + { + if (!is_string($value)) { + throw new \Exception("Attribute '{$attrId}' must be a string, got " . gettype($value)); + } + + if ($size > 0 && strlen($value) > $size) { + throw new \Exception("Attribute '{$attrId}' exceeds maximum size of {$size} characters"); + } + } + + /** + * Validate integer attribute value. + * + * @throws \Exception + */ + private static function validateIntegerAttribute(string $attrId, mixed $value): void + { + if (!is_int($value)) { + throw new \Exception("Attribute '{$attrId}' must be an integer, got " . gettype($value)); + } + } + + /** + * Validate datetime attribute value. + * + * @throws \Exception + */ + private static function validateDatetimeAttribute(string $attrId, mixed $value): void + { + if ($value instanceof \DateTime) { + return; // Valid DateTime object + } + + if (!is_string($value)) { + throw new \Exception("Attribute '{$attrId}' must be a DateTime object or string, got " . gettype($value)); + } + + try { + new \DateTime($value); + } catch (\Exception $e) { + throw new \Exception("Attribute '{$attrId}' is not a valid datetime string: {$e->getMessage()}"); + } + } } diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php new file mode 100644 index 0000000..df9e44b --- /dev/null +++ b/tests/Usage/MetricTest.php @@ -0,0 +1,576 @@ +assertIsArray($schema); + $this->assertCount(5, $schema); + + // Test metric attribute + $metricAttr = $schema[0]; + $this->assertEquals('metric', $metricAttr['$id']); + $this->assertEquals('string', $metricAttr['type']); + $this->assertEquals(255, $metricAttr['size']); + $this->assertTrue($metricAttr['required']); + + // Test value attribute + $valueAttr = $schema[1]; + $this->assertEquals('value', $valueAttr['$id']); + $this->assertEquals('integer', $valueAttr['type']); + $this->assertTrue($valueAttr['required']); + + // Test period attribute + $periodAttr = $schema[2]; + $this->assertEquals('period', $periodAttr['$id']); + $this->assertEquals('string', $periodAttr['type']); + $this->assertEquals(16, $periodAttr['size']); + $this->assertTrue($periodAttr['required']); + + // Test time attribute (optional) + $timeAttr = $schema[3]; + $this->assertEquals('time', $timeAttr['$id']); + $this->assertEquals('datetime', $timeAttr['type']); + $this->assertFalse($timeAttr['required']); + + // Test tags attribute (optional) + $tagsAttr = $schema[4]; + $this->assertEquals('tags', $tagsAttr['$id']); + $this->assertEquals('string', $tagsAttr['type']); + $this->assertFalse($tagsAttr['required']); + } + + /** + * Test Metric::getIndexes() returns correct index definitions + */ + public function testGetIndexesReturnsIndexDefinitions(): void + { + $indexes = Metric::getIndexes(); + + $this->assertIsArray($indexes); + $this->assertCount(3, $indexes); + + // Test metric index + $metricIndex = $indexes[0]; + $this->assertEquals('index-metric', $metricIndex['$id']); + $this->assertEquals('key', $metricIndex['type']); + $this->assertEquals(['metric'], $metricIndex['attributes']); + + // Test period index + $periodIndex = $indexes[1]; + $this->assertEquals('index-period', $periodIndex['$id']); + $this->assertEquals(['period'], $periodIndex['attributes']); + + // Test time index + $timeIndex = $indexes[2]; + $this->assertEquals('index-time', $timeIndex['$id']); + $this->assertEquals(['time'], $timeIndex['attributes']); + } + + /** + * Test Metric::validate() accepts valid data + */ + public function testValidateAcceptsValidData(): void + { + $validData = [ + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'time' => '2024-01-01 12:00:00', + 'tags' => ['region' => 'us-east', 'env' => 'prod'], + ]; + + // Should not throw exception + Metric::validate($validData); + $this->assertTrue(true); + } + + /** + * Test Metric::validate() accepts minimal required data + */ + public function testValidateAcceptsMinimalData(): void + { + $minimalData = [ + 'metric' => 'requests', + 'value' => 50, + 'period' => '1h', + ]; + + Metric::validate($minimalData); + $this->assertTrue(true); + } + + /** + * Test Metric::validate() rejects missing required metric + */ + public function testValidateRejectsMissingMetric(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Required attribute 'metric' is missing"); + + Metric::validate([ + 'value' => 100, + 'period' => '1h', + ]); + } + + /** + * Test Metric::validate() rejects missing required value + */ + public function testValidateRejectsMissingValue(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Required attribute 'value' is missing"); + + Metric::validate([ + 'metric' => 'requests', + 'period' => '1h', + ]); + } + + /** + * Test Metric::validate() rejects missing required period + */ + public function testValidateRejectsMissingPeriod(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Required attribute 'period' is missing"); + + Metric::validate([ + 'metric' => 'requests', + 'value' => 100, + ]); + } + + /** + * Test Metric::validate() rejects non-string metric + */ + public function testValidateRejectsNonStringMetric(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Attribute 'metric' must be a string"); + + Metric::validate([ + 'metric' => 123, + 'value' => 100, + 'period' => '1h', + ]); + } + + /** + * Test Metric::validate() rejects oversized metric string + */ + public function testValidateRejectsOversizedMetric(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("exceeds maximum size of 255 characters"); + + Metric::validate([ + 'metric' => str_repeat('a', 256), + 'value' => 100, + 'period' => '1h', + ]); + } + + /** + * Test Metric::validate() rejects non-integer value + */ + public function testValidateRejectsNonIntegerValue(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Attribute 'value' must be an integer"); + + Metric::validate([ + 'metric' => 'requests', + 'value' => '100', + 'period' => '1h', + ]); + } + + /** + * Test Metric::validate() rejects non-string period + */ + public function testValidateRejectsNonStringPeriod(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Attribute 'period' must be a string"); + + Metric::validate([ + 'metric' => 'requests', + 'value' => 100, + 'period' => 123, + ]); + } + + /** + * Test Metric::validate() accepts DateTime object for time + */ + public function testValidateAcceptsDateTimeForTime(): void + { + $data = [ + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'time' => new \DateTime('2024-01-01 12:00:00'), + ]; + + Metric::validate($data); + $this->assertTrue(true); + } + + /** + * Test Metric::validate() accepts datetime string for time + */ + public function testValidateAcceptsDatetimeStringForTime(): void + { + $data = [ + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'time' => '2024-01-01 12:00:00', + ]; + + Metric::validate($data); + $this->assertTrue(true); + } + + /** + * Test Metric::validate() rejects invalid datetime string + */ + public function testValidateRejectsInvalidDatetimeString(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("not a valid datetime string"); + + Metric::validate([ + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'time' => 'invalid-date', + ]); + } + + /** + * Test Metric::validate() rejects non-array tags + */ + public function testValidateRejectsNonArrayTags(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Attribute 'tags' must be an array"); + + Metric::validate([ + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'tags' => 'not-an-array', + ]); + } + + /** + * Test Metric::validate() accepts empty tags array + */ + public function testValidateAcceptsEmptyTags(): void + { + $data = [ + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'tags' => [], + ]; + + Metric::validate($data); + $this->assertTrue(true); + } + + /** + * Test Metric constructor initializes with data + */ + public function testConstructorInitializesWithData(): void + { + $data = [ + '$id' => 'metric-1', + 'metric' => 'requests', + 'value' => 100, + 'period' => '1h', + 'tags' => ['env' => 'prod'], + ]; + + $metric = new Metric($data); + + $this->assertEquals('metric-1', $metric->getId()); + $this->assertEquals('requests', $metric->getMetric()); + $this->assertEquals(100, $metric->getValue()); + $this->assertEquals('1h', $metric->getPeriod()); + $this->assertEquals(['env' => 'prod'], $metric->getTags()); + } + + /** + * Test Metric::getId() returns metric ID + */ + public function testGetIdReturnsMetricId(): void + { + $metric = new Metric(['$id' => 'metric-123']); + $this->assertEquals('metric-123', $metric->getId()); + } + + /** + * Test Metric::getId() returns empty string when ID not set + */ + public function testGetIdReturnsEmptyStringWhenNotSet(): void + { + $metric = new Metric([]); + $this->assertEquals('', $metric->getId()); + } + + /** + * Test Metric::getMetric() returns metric name + */ + public function testGetMetricReturnsMetricName(): void + { + $metric = new Metric(['metric' => 'bandwidth']); + $this->assertEquals('bandwidth', $metric->getMetric()); + } + + /** + * Test Metric::getValue() returns metric value + */ + public function testGetValueReturnsValue(): void + { + $metric = new Metric(['value' => 1024]); + $this->assertEquals(1024, $metric->getValue()); + } + + /** + * Test Metric::getValue() returns default when not set + */ + public function testGetValueReturnsDefaultWhenNotSet(): void + { + $metric = new Metric([]); + $this->assertEquals(0, $metric->getValue()); + } + + /** + * Test Metric::getPeriod() returns period + */ + public function testGetPeriodReturnsPeriod(): void + { + $metric = new Metric(['period' => '1d']); + $this->assertEquals('1d', $metric->getPeriod()); + } + + /** + * Test Metric::getPeriod() returns default period + */ + public function testGetPeriodReturnsDefaultPeriod(): void + { + $metric = new Metric([]); + $this->assertEquals('1h', $metric->getPeriod()); + } + + /** + * Test Metric::getTime() returns timestamp + */ + public function testGetTimeReturnsTimestamp(): void + { + $time = '2024-01-01 12:00:00'; + $metric = new Metric(['time' => $time]); + $this->assertEquals($time, $metric->getTime()); + } + + /** + * Test Metric::getTime() returns null when not set + */ + public function testGetTimeReturnsNullWhenNotSet(): void + { + $metric = new Metric([]); + $this->assertNull($metric->getTime()); + } + + /** + * Test Metric::getTags() returns tags + */ + public function testGetTagsReturnsTags(): void + { + $tags = ['region' => 'us-east', 'env' => 'prod']; + $metric = new Metric(['tags' => $tags]); + $this->assertEquals($tags, $metric->getTags()); + } + + /** + * Test Metric::getTags() returns empty array when not set + */ + public function testGetTagsReturnsEmptyArrayWhenNotSet(): void + { + $metric = new Metric([]); + $this->assertEquals([], $metric->getTags()); + } + + /** + * Test Metric::getTenant() returns tenant ID + */ + public function testGetTenantReturnsTenantId(): void + { + $metric = new Metric(['tenant' => 123]); + $this->assertEquals(123, $metric->getTenant()); + } + + /** + * Test Metric::getTenant() returns null when not set + */ + public function testGetTenantReturnsNullWhenNotSet(): void + { + $metric = new Metric([]); + $this->assertNull($metric->getTenant()); + } + + /** + * Test Metric::getTenant() converts numeric tenant to int + */ + public function testGetTenantConvertsNumericToInt(): void + { + $metric = new Metric(['tenant' => '456']); + $this->assertEquals(456, $metric->getTenant()); + $this->assertIsInt($metric->getTenant()); + } + + /** + * Test Metric::getAttributes() returns all attributes + */ + public function testGetAttributesReturnsAllAttributes(): void + { + $data = [ + '$id' => 'metric-1', + 'metric' => 'requests', + 'value' => 100, + ]; + + $metric = new Metric($data); + $attributes = $metric->getAttributes(); + + $this->assertIsArray($attributes); + $this->assertEquals('metric-1', $attributes['$id']); + $this->assertEquals('requests', $attributes['metric']); + $this->assertEquals(100, $attributes['value']); + } + + /** + * Test Metric::getAttribute() returns attribute value + */ + public function testGetAttributeReturnsValue(): void + { + $metric = new Metric(['custom' => 'custom-value']); + $this->assertEquals('custom-value', $metric->getAttribute('custom')); + } + + /** + * Test Metric::getAttribute() returns default when not set + */ + public function testGetAttributeReturnsDefaultWhenNotSet(): void + { + $metric = new Metric([]); + $this->assertEquals('default', $metric->getAttribute('missing', 'default')); + } + + /** + * Test Metric::setAttribute() sets attribute and returns self + */ + public function testSetAttributeSetsAndReturnsSelf(): void + { + $metric = new Metric([]); + $result = $metric->setAttribute('custom', 'value'); + + $this->assertSame($metric, $result); + $this->assertEquals('value', $metric->getAttribute('custom')); + } + + /** + * Test Metric::setAttribute() supports method chaining + */ + public function testSetAttributeSupportsChaining(): void + { + $metric = (new Metric([])) + ->setAttribute('attr1', 'value1') + ->setAttribute('attr2', 'value2'); + + $this->assertEquals('value1', $metric->getAttribute('attr1')); + $this->assertEquals('value2', $metric->getAttribute('attr2')); + } + + /** + * Test Metric::hasAttribute() returns true when attribute exists + */ + public function testHasAttributeReturnsTrueWhenExists(): void + { + $metric = new Metric(['key' => 'value']); + $this->assertTrue($metric->hasAttribute('key')); + } + + /** + * Test Metric::hasAttribute() returns false when attribute doesn't exist + */ + public function testHasAttributeReturnsFalseWhenNotExists(): void + { + $metric = new Metric([]); + $this->assertFalse($metric->hasAttribute('missing')); + } + + /** + * Test Metric::removeAttribute() removes attribute and returns self + */ + public function testRemoveAttributeRemovesAndReturnsSelf(): void + { + $metric = new Metric(['key' => 'value']); + $result = $metric->removeAttribute('key'); + + $this->assertSame($metric, $result); + $this->assertFalse($metric->hasAttribute('key')); + } + + /** + * Test Metric::isEmpty() returns false when ID is set + */ + public function testIsEmptyReturnsFalseWhenIdSet(): void + { + $metric = new Metric(['$id' => 'metric-1']); + $this->assertFalse($metric->isEmpty()); + } + + /** + * Test Metric::isEmpty() returns true when ID is not set + */ + public function testIsEmptyReturnsTrueWhenNoId(): void + { + $metric = new Metric([]); + $this->assertTrue($metric->isEmpty()); + } + + /** + * Test Metric::toArray() returns array representation + */ + public function testToArrayReturnsArray(): void + { + $data = [ + '$id' => 'metric-1', + 'metric' => 'requests', + 'value' => 100, + ]; + + $metric = new Metric($data); + $array = $metric->toArray(); + + $this->assertIsArray($array); + $this->assertEquals('metric-1', $array['$id']); + $this->assertEquals('requests', $array['metric']); + $this->assertEquals(100, $array['value']); + } +} From 86113b592323ea75f5980efee60016f79807279c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 26 Jan 2026 06:20:42 +0000 Subject: [PATCH 24/44] upgrade fetch --- composer.json | 2 +- composer.lock | 235 +++++++++++++++++++++++++------------------------- 2 files changed, 118 insertions(+), 119 deletions(-) diff --git a/composer.json b/composer.json index dc68ede..3f4c5b1 100644 --- a/composer.json +++ b/composer.json @@ -17,7 +17,7 @@ "minimum-stability": "stable", "require": { "php": ">=8.0", - "utopia-php/fetch": "^0.4.2", + "utopia-php/fetch": "0.5.*", "utopia-php/database": "^4.3" }, "require-dev": { diff --git a/composer.lock b/composer.lock index d64c0ef..ef28994 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ea595e5dda2475807e9de0f50c141a57", + "content-hash": "4ba30891f6fa26facbf57fc7d902ec92", "packages": [ { "name": "brick/math", @@ -145,16 +145,16 @@ }, { "name": "google/protobuf", - "version": "v4.33.2", + "version": "v4.33.4", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "fbd96b7bf1343f4b0d8fb358526c7ba4d72f1318" + "reference": "22d28025cda0d223a2e48c2e16c5284ecc9f5402" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/fbd96b7bf1343f4b0d8fb358526c7ba4d72f1318", - "reference": "fbd96b7bf1343f4b0d8fb358526c7ba4d72f1318", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/22d28025cda0d223a2e48c2e16c5284ecc9f5402", + "reference": "22d28025cda0d223a2e48c2e16c5284ecc9f5402", "shasum": "" }, "require": { @@ -183,9 +183,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.2" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.4" }, - "time": "2025-12-05T22:12:22+00:00" + "time": "2026-01-12T17:58:43+00:00" }, { "name": "mongodb/mongodb", @@ -410,16 +410,16 @@ }, { "name": "open-telemetry/api", - "version": "1.7.1", + "version": "1.8.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4" + "reference": "df5197c6fd0ddd8e9883b87de042d9341300e2ad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4", - "reference": "45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/df5197c6fd0ddd8e9883b87de042d9341300e2ad", + "reference": "df5197c6fd0ddd8e9883b87de042d9341300e2ad", "shasum": "" }, "require": { @@ -429,7 +429,7 @@ "symfony/polyfill-php82": "^1.26" }, "conflict": { - "open-telemetry/sdk": "<=1.0.8" + "open-telemetry/sdk": "<=1.11" }, "type": "library", "extra": { @@ -476,7 +476,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-10-19T10:49:48+00:00" + "time": "2026-01-21T04:14:03+00:00" }, { "name": "open-telemetry/context", @@ -539,16 +539,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.3.3", + "version": "1.3.4", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "07b02bc71838463f6edcc78d3485c04b48fb263d" + "reference": "62e680d587beb42e5247aa6ecd89ad1ca406e8ca" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/07b02bc71838463f6edcc78d3485c04b48fb263d", - "reference": "07b02bc71838463f6edcc78d3485c04b48fb263d", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/62e680d587beb42e5247aa6ecd89ad1ca406e8ca", + "reference": "62e680d587beb42e5247aa6ecd89ad1ca406e8ca", "shasum": "" }, "require": { @@ -599,7 +599,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-13T08:04:37+00:00" + "time": "2026-01-15T09:31:34+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -666,16 +666,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.10.0", + "version": "1.12.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99" + "reference": "7f1bd524465c1ca42755a9ef1143ba09913f5be0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/7f1bd524465c1ca42755a9ef1143ba09913f5be0", + "reference": "7f1bd524465c1ca42755a9ef1143ba09913f5be0", "shasum": "" }, "require": { @@ -716,7 +716,7 @@ ] }, "branch-alias": { - "dev-main": "1.9.x-dev" + "dev-main": "1.12.x-dev" } }, "autoload": { @@ -759,20 +759,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-25T10:59:15+00:00" + "time": "2026-01-21T04:14:03+00:00" }, { "name": "open-telemetry/sem-conv", - "version": "1.37.0", + "version": "1.38.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sem-conv.git", - "reference": "8da7ec497c881e39afa6657d72586e27efbd29a1" + "reference": "e613bc640a407def4991b8a936a9b27edd9a3240" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/8da7ec497c881e39afa6657d72586e27efbd29a1", - "reference": "8da7ec497c881e39afa6657d72586e27efbd29a1", + "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/e613bc640a407def4991b8a936a9b27edd9a3240", + "reference": "e613bc640a407def4991b8a936a9b27edd9a3240", "shasum": "" }, "require": { @@ -812,11 +812,11 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/php", + "docs": "https://opentelemetry.io/docs/languages/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-09-03T12:08:10+00:00" + "time": "2026-01-21T04:14:03+00:00" }, { "name": "php-http/discovery", @@ -1238,20 +1238,20 @@ }, { "name": "ramsey/uuid", - "version": "4.9.1", + "version": "4.9.2", "source": { "type": "git", "url": "https://github.com/ramsey/uuid.git", - "reference": "81f941f6f729b1e3ceea61d9d014f8b6c6800440" + "reference": "8429c78ca35a09f27565311b98101e2826affde0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/uuid/zipball/81f941f6f729b1e3ceea61d9d014f8b6c6800440", - "reference": "81f941f6f729b1e3ceea61d9d014f8b6c6800440", + "url": "https://api.github.com/repos/ramsey/uuid/zipball/8429c78ca35a09f27565311b98101e2826affde0", + "reference": "8429c78ca35a09f27565311b98101e2826affde0", "shasum": "" }, "require": { - "brick/math": "^0.8.8 || ^0.9 || ^0.10 || ^0.11 || ^0.12 || ^0.13 || ^0.14", + "brick/math": "^0.8.16 || ^0.9 || ^0.10 || ^0.11 || ^0.12 || ^0.13 || ^0.14", "php": "^8.0", "ramsey/collection": "^1.2 || ^2.0" }, @@ -1310,9 +1310,9 @@ ], "support": { "issues": "https://github.com/ramsey/uuid/issues", - "source": "https://github.com/ramsey/uuid/tree/4.9.1" + "source": "https://github.com/ramsey/uuid/tree/4.9.2" }, - "time": "2025-09-04T20:59:21+00:00" + "time": "2025-12-14T04:43:48+00:00" }, { "name": "symfony/deprecation-contracts", @@ -1383,16 +1383,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.1", + "version": "v7.4.4", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "26cc224ea7103dda90e9694d9e139a389092d007" + "reference": "d63c23357d74715a589454c141c843f0172bec6c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/26cc224ea7103dda90e9694d9e139a389092d007", - "reference": "26cc224ea7103dda90e9694d9e139a389092d007", + "url": "https://api.github.com/repos/symfony/http-client/zipball/d63c23357d74715a589454c141c843f0172bec6c", + "reference": "d63c23357d74715a589454c141c843f0172bec6c", "shasum": "" }, "require": { @@ -1460,7 +1460,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.1" + "source": "https://github.com/symfony/http-client/tree/v7.4.4" }, "funding": [ { @@ -1480,7 +1480,7 @@ "type": "tidelift" } ], - "time": "2025-12-04T21:12:57+00:00" + "time": "2026-01-23T16:34:22+00:00" }, { "name": "symfony/http-client-contracts", @@ -2026,16 +2026,16 @@ }, { "name": "utopia-php/cache", - "version": "0.13.1", + "version": "0.13.2", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "97220cb3b3822b166ee016d1646e2ae2815dc540" + "reference": "5768498c9f451482f0bf3eede4d6452ddcd4a0f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/97220cb3b3822b166ee016d1646e2ae2815dc540", - "reference": "97220cb3b3822b166ee016d1646e2ae2815dc540", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/5768498c9f451482f0bf3eede4d6452ddcd4a0f6", + "reference": "5768498c9f451482f0bf3eede4d6452ddcd4a0f6", "shasum": "" }, "require": { @@ -2044,7 +2044,7 @@ "ext-redis": "*", "php": ">=8.0", "utopia-php/pools": "0.8.*", - "utopia-php/telemetry": "0.1.*" + "utopia-php/telemetry": "*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -2072,9 +2072,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.13.1" + "source": "https://github.com/utopia-php/cache/tree/0.13.2" }, - "time": "2025-05-09T14:43:52+00:00" + "time": "2025-12-17T08:55:43+00:00" }, { "name": "utopia-php/compression", @@ -2124,16 +2124,16 @@ }, { "name": "utopia-php/database", - "version": "4.3.0", + "version": "4.6.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "fe7a1326ad623609e65587fe8c01a630a7075fee" + "reference": "53394759c44067e9db4660635765e2056f83788c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/fe7a1326ad623609e65587fe8c01a630a7075fee", - "reference": "fe7a1326ad623609e65587fe8c01a630a7075fee", + "url": "https://api.github.com/repos/utopia-php/database/zipball/53394759c44067e9db4660635765e2056f83788c", + "reference": "53394759c44067e9db4660635765e2056f83788c", "shasum": "" }, "require": { @@ -2176,26 +2176,26 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.3.0" + "source": "https://github.com/utopia-php/database/tree/4.6.2" }, - "time": "2025-11-14T03:43:10+00:00" + "time": "2026-01-22T07:14:12+00:00" }, { "name": "utopia-php/fetch", - "version": "0.4.2", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "83986d1be75a2fae4e684107fe70dd78a8e19b77" + "reference": "a96a010e1c273f3888765449687baf58cbc61fcd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/83986d1be75a2fae4e684107fe70dd78a8e19b77", - "reference": "83986d1be75a2fae4e684107fe70dd78a8e19b77", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/a96a010e1c273f3888765449687baf58cbc61fcd", + "reference": "a96a010e1c273f3888765449687baf58cbc61fcd", "shasum": "" }, "require": { - "php": ">=8.0" + "php": ">=8.1" }, "require-dev": { "laravel/pint": "^1.5.0", @@ -2215,29 +2215,29 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.4.2" + "source": "https://github.com/utopia-php/fetch/tree/0.5.1" }, - "time": "2025-04-25T13:48:02+00:00" + "time": "2025-12-18T16:25:10+00:00" }, { "name": "utopia-php/framework", - "version": "0.33.34", + "version": "0.33.37", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "76def92594c32504ec80eaacdb60ff8fad73c856" + "reference": "30a119d76531d89da9240496940c84fcd9e1758b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/76def92594c32504ec80eaacdb60ff8fad73c856", - "reference": "76def92594c32504ec80eaacdb60ff8fad73c856", + "url": "https://api.github.com/repos/utopia-php/http/zipball/30a119d76531d89da9240496940c84fcd9e1758b", + "reference": "30a119d76531d89da9240496940c84fcd9e1758b", "shasum": "" }, "require": { "php": ">=8.3", "utopia-php/compression": "0.1.*", "utopia-php/telemetry": "0.1.*", - "utopia-php/validators": "0.1.*" + "utopia-php/validators": "0.2.*" }, "require-dev": { "laravel/pint": "1.*", @@ -2263,9 +2263,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.34" + "source": "https://github.com/utopia-php/http/tree/0.33.37" }, - "time": "2025-12-08T07:55:31+00:00" + "time": "2026-01-13T10:10:21+00:00" }, { "name": "utopia-php/mongo", @@ -2330,21 +2330,21 @@ }, { "name": "utopia-php/pools", - "version": "0.8.2", + "version": "0.8.3", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d" + "reference": "ad7d6ba946376e81c603204285ce9a674b6502b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", - "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/ad7d6ba946376e81c603204285ce9a674b6502b8", + "reference": "ad7d6ba946376e81c603204285ce9a674b6502b8", "shasum": "" }, "require": { - "php": ">=8.3", - "utopia-php/telemetry": "0.1.*" + "php": ">=8.4", + "utopia-php/telemetry": "*" }, "require-dev": { "laravel/pint": "1.*", @@ -2376,9 +2376,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.8.2" + "source": "https://github.com/utopia-php/pools/tree/0.8.3" }, - "time": "2025-04-17T02:04:54+00:00" + "time": "2025-12-17T09:35:18+00:00" }, { "name": "utopia-php/telemetry", @@ -2432,16 +2432,16 @@ }, { "name": "utopia-php/validators", - "version": "0.1.0", + "version": "0.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "5c57d5b6cf964f8981807c1d3ea8df620c869080" + "reference": "30b6030a5b100fc1dff34506e5053759594b2a20" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/5c57d5b6cf964f8981807c1d3ea8df620c869080", - "reference": "5c57d5b6cf964f8981807c1d3ea8df620c869080", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/30b6030a5b100fc1dff34506e5053759594b2a20", + "reference": "30b6030a5b100fc1dff34506e5053759594b2a20", "shasum": "" }, "require": { @@ -2449,7 +2449,7 @@ }, "require-dev": { "laravel/pint": "1.*", - "phpstan/phpstan": "1.*", + "phpstan/phpstan": "2.*", "phpunit/phpunit": "11.*" }, "type": "library", @@ -2471,38 +2471,37 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.1.0" + "source": "https://github.com/utopia-php/validators/tree/0.2.0" }, - "time": "2025-11-18T11:05:46+00:00" + "time": "2026-01-13T09:16:51+00:00" } ], "packages-dev": [ { "name": "doctrine/instantiator", - "version": "2.0.0", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "c6222283fa3f4ac679f8b9ced9a4e23f163e80d0" + "reference": "23da848e1a2308728fe5fdddabf4be17ff9720c7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/c6222283fa3f4ac679f8b9ced9a4e23f163e80d0", - "reference": "c6222283fa3f4ac679f8b9ced9a4e23f163e80d0", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/23da848e1a2308728fe5fdddabf4be17ff9720c7", + "reference": "23da848e1a2308728fe5fdddabf4be17ff9720c7", "shasum": "" }, "require": { - "php": "^8.1" + "php": "^8.4" }, "require-dev": { - "doctrine/coding-standard": "^11", + "doctrine/coding-standard": "^14", "ext-pdo": "*", "ext-phar": "*", "phpbench/phpbench": "^1.2", - "phpstan/phpstan": "^1.9.4", - "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.5.27", - "vimeo/psalm": "^5.4" + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^10.5.58" }, "type": "library", "autoload": { @@ -2529,7 +2528,7 @@ ], "support": { "issues": "https://github.com/doctrine/instantiator/issues", - "source": "https://github.com/doctrine/instantiator/tree/2.0.0" + "source": "https://github.com/doctrine/instantiator/tree/2.1.0" }, "funding": [ { @@ -2545,20 +2544,20 @@ "type": "tidelift" } ], - "time": "2022-12-30T00:23:10+00:00" + "time": "2026-01-05T06:47:08+00:00" }, { "name": "laravel/pint", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "69dcca060ecb15e4b564af63d1f642c81a241d6f" + "reference": "c67b4195b75491e4dfc6b00b1c78b68d86f54c90" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/69dcca060ecb15e4b564af63d1f642c81a241d6f", - "reference": "69dcca060ecb15e4b564af63d1f642c81a241d6f", + "url": "https://api.github.com/repos/laravel/pint/zipball/c67b4195b75491e4dfc6b00b1c78b68d86f54c90", + "reference": "c67b4195b75491e4dfc6b00b1c78b68d86f54c90", "shasum": "" }, "require": { @@ -2569,9 +2568,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.90.0", - "illuminate/view": "^12.40.1", - "larastan/larastan": "^3.8.0", + "friendsofphp/php-cs-fixer": "^3.92.4", + "illuminate/view": "^12.44.0", + "larastan/larastan": "^3.8.1", "laravel-zero/framework": "^12.0.4", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3.3", @@ -2612,7 +2611,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-11-25T21:15:52+00:00" + "time": "2026-01-05T16:49:17+00:00" }, { "name": "myclabs/deep-copy", @@ -3224,16 +3223,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.31", + "version": "9.6.32", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "945d0b7f346a084ce5549e95289962972c4272e5" + "reference": "492ee10a8369a1c1ac390a3b46e0c846e384c5a4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/945d0b7f346a084ce5549e95289962972c4272e5", - "reference": "945d0b7f346a084ce5549e95289962972c4272e5", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/492ee10a8369a1c1ac390a3b46e0c846e384c5a4", + "reference": "492ee10a8369a1c1ac390a3b46e0c846e384c5a4", "shasum": "" }, "require": { @@ -3255,7 +3254,7 @@ "phpunit/php-timer": "^5.0.3", "sebastian/cli-parser": "^1.0.2", "sebastian/code-unit": "^1.0.8", - "sebastian/comparator": "^4.0.9", + "sebastian/comparator": "^4.0.10", "sebastian/diff": "^4.0.6", "sebastian/environment": "^5.1.5", "sebastian/exporter": "^4.0.8", @@ -3307,7 +3306,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.31" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.32" }, "funding": [ { @@ -3331,7 +3330,7 @@ "type": "tidelift" } ], - "time": "2025-12-06T07:45:52+00:00" + "time": "2026-01-24T16:04:20+00:00" }, { "name": "sebastian/cli-parser", @@ -3502,16 +3501,16 @@ }, { "name": "sebastian/comparator", - "version": "4.0.9", + "version": "4.0.10", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "67a2df3a62639eab2cc5906065e9805d4fd5dfc5" + "reference": "e4df00b9b3571187db2831ae9aada2c6efbd715d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/67a2df3a62639eab2cc5906065e9805d4fd5dfc5", - "reference": "67a2df3a62639eab2cc5906065e9805d4fd5dfc5", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/e4df00b9b3571187db2831ae9aada2c6efbd715d", + "reference": "e4df00b9b3571187db2831ae9aada2c6efbd715d", "shasum": "" }, "require": { @@ -3564,7 +3563,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", - "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.9" + "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.10" }, "funding": [ { @@ -3584,7 +3583,7 @@ "type": "tidelift" } ], - "time": "2025-08-10T06:51:50+00:00" + "time": "2026-01-24T09:22:56+00:00" }, { "name": "sebastian/complexity", From aa60dd115c83be608c83e9692c3d0694cbacfd6e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 26 Jan 2026 13:14:28 +0000 Subject: [PATCH 25/44] feat: Update ClickHouse and Database adapters to support incremental metric logging with deterministic IDs --- src/Usage/Adapter/ClickHouse.php | 37 ++++++++++++++++++++++++++------ src/Usage/Adapter/Database.php | 21 ++++++++++++------ tests/Usage/UsageBase.php | 18 ++++++++++++++++ 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 091e5ad..94114f2 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -431,16 +431,18 @@ public function setup(): void $tableName = $this->getTableName(); $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - // Create table with MergeTree engine for optimal performance + // Create table with SummingMergeTree engine so inserts act as increments for matching keys $columnDefs = implode(",\n ", $columns); $indexDefs = !empty($indexes) ? ",\n " . implode(",\n ", $indexes) : ''; + $orderByExpr = $this->sharedTables ? '(tenant, id)' : '(id)'; + $createTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( {$columnDefs}{$indexDefs} ) - ENGINE = MergeTree() - ORDER BY (time, id) + ENGINE = SummingMergeTree() + ORDER BY {$orderByExpr} PARTITION BY toYYYYMM(time) SETTINGS index_granularity = 8192 "; @@ -601,9 +603,19 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 ]; Metric::validate($data); - $id = uniqid('', true); + // Period-aligned time so increments fall into the correct bucket $now = new \DateTime(); - $timestamp = $this->formatDateTime($now); + $time = $period === Usage::PERIOD_INF + ? '1000-01-01 00:00:00' + : $now->format(Usage::PERIODS[$period]); + $timestamp = $this->formatDateTime($time); + + // Deterministic id so SummingMergeTree will aggregate increments for the same group + $idComponents = [$timestamp, $period, $metric]; + if ($this->sharedTables) { + $idComponents[] = (string)$this->tenant; + } + $id = md5(implode('_', $idComponents)); // Build insert columns dynamically from attributes $insertColumns = ['id']; @@ -749,9 +761,18 @@ public function logBatch(array $metrics): bool foreach ($metrics as $metricData) { $period = $metricData['period'] ?? Usage::PERIOD_1H; - $id = uniqid('', true); + // Period-aligned time so increments fall into the correct bucket $now = new \DateTime(); - $timestamp = $this->formatDateTime($now); + $time = $period === Usage::PERIOD_INF + ? '1000-01-01 00:00:00' + : $now->format(Usage::PERIODS[$period]); + $timestamp = $this->formatDateTime($time); + + $idComponents = [$timestamp, $period, $metric]; + if ($this->sharedTables) { + $idComponents[] = (string)$this->tenant; + } + $id = md5(implode('_', $idComponents)); $metric = $metricData['metric']; $value = $metricData['value']; @@ -806,6 +827,8 @@ public function logBatch(array $metrics): bool return true; } + + /** * Find metrics using Query objects. * diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index b500ff3..d8ff191 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -69,14 +69,19 @@ public function log(string $metric, int $value, string $period = '1h', array $ta : $now->format(Usage::PERIODS[$period]); $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags) { - $this->db->createDocument($this->collection, new Document([ + $id = \md5("{$time}_{$period}_{$metric}"); + + $doc = new Document([ + '$id' => $id, '$permissions' => [], 'metric' => $metric, 'value' => $value, 'period' => $period, 'time' => $time, 'tags' => $tags, - ])); + ]); + + $this->db->upsertDocumentsWithIncrease($this->collection, 'value', [$doc]); }); return true; @@ -85,7 +90,8 @@ public function log(string $metric, int $value, string $period = '1h', array $ta public function logBatch(array $metrics): bool { $this->db->getAuthorization()->skip(function () use ($metrics) { - $documents = \array_map(function ($metric) { + $documents = []; + foreach ($metrics as $metric) { $period = $metric['period'] ?? '1h'; if (! isset(Usage::PERIODS[$period])) { @@ -97,7 +103,10 @@ public function logBatch(array $metrics): bool ? '1000-01-01 00:00:00' : $now->format(Usage::PERIODS[$period]); - return new Document([ + $id = \md5("{$time}_{$period}_{$metric['metric']}"); + + $documents[] = new Document([ + '$id' => $id, '$permissions' => [], 'metric' => $metric['metric'], 'value' => $metric['value'], @@ -105,9 +114,9 @@ public function logBatch(array $metrics): bool 'time' => $time, 'tags' => $metric['tags'] ?? [], ]); - }, $metrics); + } - $this->db->createDocuments($this->collection, $documents); + $this->db->upsertDocumentsWithIncrease($this->collection, 'value', $documents); }); return true; diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index 03ac29e..db2b630 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -110,6 +110,24 @@ public function testSumByPeriod(): void $this->assertEquals(5000, $sumBandwidth); } + public function testIncrementingDefaultBehavior(): void + { + // Ensure clean state + $this->usage->purge(\Utopia\Database\DateTime::now()); + + // Log the same metric twice with identical period and tags + $this->assertTrue($this->usage->log('increment-test', 5, '1h', [])); + $this->assertTrue($this->usage->log('increment-test', 7, '1h', [])); + + // Because adapters now aggregate by deterministic id/time/period (and tenant where applicable), + // there should be a single record and the summed value should be 12. + $results = $this->usage->getByPeriod('increment-test', '1h'); + $this->assertEquals(1, count($results)); + + $sum = $this->usage->sumByPeriod('increment-test', '1h'); + $this->assertEquals(12, $sum); + } + public function testWithQueries(): void { $results = $this->usage->getByPeriod('requests', '1h', [ From e06f0362f583434500a4373ee725deb799a903ad Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 27 Jan 2026 04:41:33 +0000 Subject: [PATCH 26/44] feat: Implement deterministic ID generation for metrics with normalized tags in ClickHouse and Database adapters --- src/Usage/Adapter/ClickHouse.php | 26 ++++++++++++-------------- src/Usage/Adapter/Database.php | 13 +++++++++---- src/Usage/Adapter/SQL.php | 12 ++++++++++++ 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 94114f2..4611237 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -603,6 +603,9 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 ]; Metric::validate($data); + // Normalize tags for deterministic hashing + ksort($tags); + // Period-aligned time so increments fall into the correct bucket $now = new \DateTime(); $time = $period === Usage::PERIOD_INF @@ -611,11 +614,8 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 $timestamp = $this->formatDateTime($time); // Deterministic id so SummingMergeTree will aggregate increments for the same group - $idComponents = [$timestamp, $period, $metric]; - if ($this->sharedTables) { - $idComponents[] = (string)$this->tenant; - } - $id = md5(implode('_', $idComponents)); + $tenant = $this->sharedTables ? $this->tenant : null; + $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); // Build insert columns dynamically from attributes $insertColumns = ['id']; @@ -760,6 +760,10 @@ public function logBatch(array $metrics): bool foreach ($metrics as $metricData) { $period = $metricData['period'] ?? Usage::PERIOD_1H; + $metric = $metricData['metric']; + $value = $metricData['value']; + $tags = $metricData['tags'] ?? []; + ksort($tags); // Period-aligned time so increments fall into the correct bucket $now = new \DateTime(); @@ -768,14 +772,9 @@ public function logBatch(array $metrics): bool : $now->format(Usage::PERIODS[$period]); $timestamp = $this->formatDateTime($time); - $idComponents = [$timestamp, $period, $metric]; - if ($this->sharedTables) { - $idComponents[] = (string)$this->tenant; - } - $id = md5(implode('_', $idComponents)); - - $metric = $metricData['metric']; - $value = $metricData['value']; + // Deterministic id for aggregation + $tenant = $this->sharedTables ? $this->tenant : null; + $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); $valuePlaceholders = []; @@ -867,7 +866,6 @@ public function find(array $queries = []): array // Build LIMIT and OFFSET $limitClause = isset($parsed['limit']) ? ' LIMIT {limit:UInt64}' : ''; $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; - $sql = " SELECT {$selectColumns} FROM {$escapedTable}{$whereClause}{$orderClause}{$limitClause}{$offsetClause} diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index d8ff191..2436027 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -68,9 +68,11 @@ public function log(string $metric, int $value, string $period = '1h', array $ta ? '1000-01-01 00:00:00' : $now->format(Usage::PERIODS[$period]); - $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags) { - $id = \md5("{$time}_{$period}_{$metric}"); + // Sort tags for consistent storage + ksort($tags); + $id = $this->buildDeterministicId($metric, $period, $time); + $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags, $id) { $doc = new Document([ '$id' => $id, '$permissions' => [], @@ -103,7 +105,10 @@ public function logBatch(array $metrics): bool ? '1000-01-01 00:00:00' : $now->format(Usage::PERIODS[$period]); - $id = \md5("{$time}_{$period}_{$metric['metric']}"); + $tags = $metric['tags'] ?? []; + ksort($tags); + + $id = $this->buildDeterministicId($metric['metric'], $period, $time); $documents[] = new Document([ '$id' => $id, @@ -112,7 +117,7 @@ public function logBatch(array $metrics): bool 'value' => $metric['value'], 'period' => $period, 'time' => $time, - 'tags' => $metric['tags'] ?? [], + 'tags' => $tags, ]); } diff --git a/src/Usage/Adapter/SQL.php b/src/Usage/Adapter/SQL.php index c56d2d9..4e6e7fd 100644 --- a/src/Usage/Adapter/SQL.php +++ b/src/Usage/Adapter/SQL.php @@ -114,4 +114,16 @@ protected function getAllColumnDefinitions(): array return $definitions; } + + /** + * Build deterministic document ID based on time bucket, period, metric, and tenant (when applicable). + * Tags are intentionally excluded to ensure aggregation regardless of tag differences. + */ + protected function buildDeterministicId(string $metric, string $period, string $timeBucket, ?int $tenant = null): string + { + $tenantPart = $tenant !== null ? ('|' . $tenant) : ''; + $hashInput = $timeBucket . '|' . $period . '|' . $metric . $tenantPart; + + return md5($hashInput); + } } From 760a66e7c9caf9a20c75d36eaa2c7b9003abfe81 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 27 Jan 2026 06:57:56 +0000 Subject: [PATCH 27/44] fix codeql --- src/Usage/Adapter/ClickHouse.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 4611237..335f429 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -604,6 +604,7 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 Metric::validate($data); // Normalize tags for deterministic hashing + /** @var array $tags */ ksort($tags); // Period-aligned time so increments fall into the correct bucket @@ -615,6 +616,9 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 // Deterministic id so SummingMergeTree will aggregate increments for the same group $tenant = $this->sharedTables ? $this->tenant : null; + /** @var string $metric */ + /** @var string $period */ + /** @var string $timestamp */ $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); // Build insert columns dynamically from attributes @@ -762,7 +766,7 @@ public function logBatch(array $metrics): bool $period = $metricData['period'] ?? Usage::PERIOD_1H; $metric = $metricData['metric']; $value = $metricData['value']; - $tags = $metricData['tags'] ?? []; + $tags = (array) ($metricData['tags'] ?? []); ksort($tags); // Period-aligned time so increments fall into the correct bucket @@ -774,6 +778,9 @@ public function logBatch(array $metrics): bool // Deterministic id for aggregation $tenant = $this->sharedTables ? $this->tenant : null; + /** @var string $metric */ + /** @var string $period */ + /** @var string $timestamp */ $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); $valuePlaceholders = []; From c74eea7a43775ebf9af25360955cb8cf05c64dc4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 27 Jan 2026 07:45:56 +0000 Subject: [PATCH 28/44] feat: Enhance logBatch to aggregate metrics by deterministic ID and update tests for new behavior --- src/Usage/Adapter/Database.php | 33 ++++++++---- src/Usage/Adapter/SQL.php | 4 +- tests/Usage/UsageBase.php | 91 ++++++++++++++++++++++++---------- 3 files changed, 89 insertions(+), 39 deletions(-) diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 2436027..734ca00 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -92,7 +92,7 @@ public function log(string $metric, int $value, string $period = '1h', array $ta public function logBatch(array $metrics): bool { $this->db->getAuthorization()->skip(function () use ($metrics) { - $documents = []; + $documentsById = []; foreach ($metrics as $metric) { $period = $metric['period'] ?? '1h'; @@ -110,18 +110,29 @@ public function logBatch(array $metrics): bool $id = $this->buildDeterministicId($metric['metric'], $period, $time); - $documents[] = new Document([ - '$id' => $id, - '$permissions' => [], - 'metric' => $metric['metric'], - 'value' => $metric['value'], - 'period' => $period, - 'time' => $time, - 'tags' => $tags, - ]); + if (isset($documentsById[$id])) { + $documentsById[$id]['value'] += $metric['value']; + } else { + $documentsById[$id] = [ + '$id' => $id, + '$permissions' => [], + 'metric' => $metric['metric'], + 'value' => $metric['value'], + 'period' => $period, + 'time' => $time, + 'tags' => $tags, + ]; + } + } + + $documents = []; + foreach ($documentsById as $doc) { + $documents[] = new Document($doc); } - $this->db->upsertDocumentsWithIncrease($this->collection, 'value', $documents); + if (!empty($documents)) { + $this->db->upsertDocumentsWithIncrease($this->collection, 'value', $documents); + } }); return true; diff --git a/src/Usage/Adapter/SQL.php b/src/Usage/Adapter/SQL.php index 4e6e7fd..3632083 100644 --- a/src/Usage/Adapter/SQL.php +++ b/src/Usage/Adapter/SQL.php @@ -121,8 +121,8 @@ protected function getAllColumnDefinitions(): array */ protected function buildDeterministicId(string $metric, string $period, string $timeBucket, ?int $tenant = null): string { - $tenantPart = $tenant !== null ? ('|' . $tenant) : ''; - $hashInput = $timeBucket . '|' . $period . '|' . $metric . $tenantPart; + $tenantPart = $tenant !== null ? ('_' . $tenant) : ''; + $hashInput = $timeBucket . '_' . $period . '_' . $metric . $tenantPart; return md5($hashInput); } diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index db2b630..3cec1bb 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -10,6 +10,33 @@ trait UsageBase { protected Usage $usage; + /** + * Retry the provided assertions until they pass or timeout (seconds). + * + * @param callable $fn Assertions to run + * @param int $timeout Seconds to wait before failing + * @param float $interval Seconds between retries + */ + protected function assertEventually(callable $fn, int $timeout = 5, float $interval = 0.5): void + { + $start = microtime(true); + $lastException = null; + + while (microtime(true) - $start < $timeout) { + try { + $fn(); + return; + } catch (\Throwable $e) { + $lastException = $e; + usleep((int) ($interval * 1_000_000)); + } + } + + if ($lastException) { + throw $lastException; + } + } + abstract protected function initializeUsage(): void; public function setUp(): void @@ -67,18 +94,22 @@ public function testLogBatch(): void $this->assertTrue($this->usage->logBatch($metrics)); $results = $this->usage->getByPeriod('batch-requests', '1h'); - $this->assertEquals(2, count($results)); + // Aggregated by deterministic id/hash, entries with same metric/period/time merge + $this->assertEquals(1, count($results)); } public function testGetByPeriod(): void { - $results1h = $this->usage->getByPeriod('requests', '1h'); - $results1d = $this->usage->getByPeriod('requests', '1d'); - $resultsInf = $this->usage->getByPeriod('storage', 'inf'); - - $this->assertEquals(2, count($results1h)); - $this->assertEquals(1, count($results1d)); - $this->assertEquals(1, count($resultsInf)); + $this->assertEventually(function () { + $results1h = $this->usage->getByPeriod('requests', '1h'); + $results1d = $this->usage->getByPeriod('requests', '1d'); + $resultsInf = $this->usage->getByPeriod('storage', 'inf'); + + // SummingMergeTree / upsert-with-increase aggregates by deterministic id + $this->assertEquals(1, count($results1h)); + $this->assertEquals(1, count($results1d)); + $this->assertEquals(1, count($resultsInf)); + }); } public function testGetBetweenDates(): void @@ -92,22 +123,27 @@ public function testGetBetweenDates(): void public function testCountByPeriod(): void { - $count1h = $this->usage->countByPeriod('requests', '1h'); - $count1d = $this->usage->countByPeriod('requests', '1d'); - $countBandwidth = $this->usage->countByPeriod('bandwidth', '1h'); - - $this->assertEquals(2, $count1h); - $this->assertEquals(1, $count1d); - $this->assertEquals(1, $countBandwidth); + $this->assertEventually(function () { + $count1h = $this->usage->countByPeriod('requests', '1h'); + $count1d = $this->usage->countByPeriod('requests', '1d'); + $countBandwidth = $this->usage->countByPeriod('bandwidth', '1h'); + + // Aggregated by deterministic id: multiple logs in same period/time collapse + $this->assertEquals(1, $count1h); + $this->assertEquals(1, $count1d); + $this->assertEquals(1, $countBandwidth); + }); } public function testSumByPeriod(): void { - $sum = $this->usage->sumByPeriod('requests', '1h'); - $this->assertEquals(250, $sum); // 100 + 150 + $this->assertEventually(function () { + $sum = $this->usage->sumByPeriod('requests', '1h'); + $this->assertEquals(250, $sum); // 100 + 150 - $sumBandwidth = $this->usage->sumByPeriod('bandwidth', '1h'); - $this->assertEquals(5000, $sumBandwidth); + $sumBandwidth = $this->usage->sumByPeriod('bandwidth', '1h'); + $this->assertEquals(5000, $sumBandwidth); + }); } public function testIncrementingDefaultBehavior(): void @@ -118,14 +154,16 @@ public function testIncrementingDefaultBehavior(): void // Log the same metric twice with identical period and tags $this->assertTrue($this->usage->log('increment-test', 5, '1h', [])); $this->assertTrue($this->usage->log('increment-test', 7, '1h', [])); + $this->assertEventually(function () { + // Because adapters now aggregate by deterministic id/time/period (and tenant where applicable), + // there should be a single record and the summed value should be 12. + $results = $this->usage->getByPeriod('increment-test', '1h'); + $this->assertEquals(1, count($results)); - // Because adapters now aggregate by deterministic id/time/period (and tenant where applicable), - // there should be a single record and the summed value should be 12. - $results = $this->usage->getByPeriod('increment-test', '1h'); - $this->assertEquals(1, count($results)); + $sum = $this->usage->sumByPeriod('increment-test', '1h'); + $this->assertEquals(12, $sum); - $sum = $this->usage->sumByPeriod('increment-test', '1h'); - $this->assertEquals(12, $sum); + }, 2); } public function testWithQueries(): void @@ -141,7 +179,8 @@ public function testWithQueries(): void Query::offset(1), ]); - $this->assertEquals(1, count($results2)); + // After aggregation there may be only a single row; offset 1 yields zero rows + $this->assertEquals(0, count($results2)); } public function testPurge(): void From f04d279735a1ec24e3643335ebcf658c5fdbde81 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 27 Jan 2026 08:20:29 +0000 Subject: [PATCH 29/44] feat: Add support for FINAL in SELECT queries in ClickHouse adapter and update tests --- src/Usage/Adapter/ClickHouse.php | 21 ++++++++++++++++++--- tests/Usage/UsageBase.php | 1 - 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 335f429..6b2271d 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -50,6 +50,9 @@ class ClickHouse extends SQL private Client $client; + /** @var bool Whether to use FINAL in SELECT queries to force merge-on-read (tests) */ + private bool $useFinal = true; + protected ?int $tenant = null; protected bool $sharedTables = false; @@ -86,6 +89,15 @@ public function __construct( $this->client->setTimeout(30_000); // 30 seconds } + /** + * Enable or disable using FINAL in SELECT queries. + */ + public function setUseFinal(bool $useFinal): self + { + $this->useFinal = $useFinal; + return $this; + } + /** * Get adapter name. */ @@ -846,6 +858,7 @@ public function find(array $queries = []): array { $tableName = $this->getTableName(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $fromTable = $escapedTable . ($this->useFinal ? ' FINAL' : ''); // Parse queries $parsed = $this->parseQueries($queries); @@ -875,7 +888,7 @@ public function find(array $queries = []): array $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; $sql = " SELECT {$selectColumns} - FROM {$escapedTable}{$whereClause}{$orderClause}{$limitClause}{$offsetClause} + FROM {$fromTable}{$whereClause}{$orderClause}{$limitClause}{$offsetClause} FORMAT TabSeparated "; @@ -894,6 +907,7 @@ public function count(array $queries = []): int { $tableName = $this->getTableName(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $fromTable = $escapedTable . ($this->useFinal ? ' FINAL' : ''); // Parse queries - we only need filters and params $parsed = $this->parseQueries($queries); @@ -915,7 +929,7 @@ public function count(array $queries = []): int $sql = " SELECT COUNT(*) as count - FROM {$escapedTable}{$whereClause} + FROM {$fromTable}{$whereClause} FORMAT TabSeparated "; @@ -1316,6 +1330,7 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) { $tableName = $this->getTableName(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $fromTable = $escapedTable . ($this->useFinal ? ' FINAL' : ''); // Build query constraints $allQueries = [ @@ -1344,7 +1359,7 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) $sql = " SELECT sum(value) as total - FROM {$escapedTable}{$whereClause} + FROM {$fromTable}{$whereClause} FORMAT TabSeparated "; diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index 3cec1bb..c0e6b5e 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -162,7 +162,6 @@ public function testIncrementingDefaultBehavior(): void $sum = $this->usage->sumByPeriod('increment-test', '1h'); $this->assertEquals(12, $sum); - }, 2); } From 116880ba2b34b37f7c04cbf90e46e6aa3897a00e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 27 Jan 2026 08:34:20 +0000 Subject: [PATCH 30/44] remove assert eventually --- tests/Usage/UsageBase.php | 89 ++++++++++++--------------------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index c0e6b5e..778b205 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -10,33 +10,6 @@ trait UsageBase { protected Usage $usage; - /** - * Retry the provided assertions until they pass or timeout (seconds). - * - * @param callable $fn Assertions to run - * @param int $timeout Seconds to wait before failing - * @param float $interval Seconds between retries - */ - protected function assertEventually(callable $fn, int $timeout = 5, float $interval = 0.5): void - { - $start = microtime(true); - $lastException = null; - - while (microtime(true) - $start < $timeout) { - try { - $fn(); - return; - } catch (\Throwable $e) { - $lastException = $e; - usleep((int) ($interval * 1_000_000)); - } - } - - if ($lastException) { - throw $lastException; - } - } - abstract protected function initializeUsage(): void; public function setUp(): void @@ -100,16 +73,14 @@ public function testLogBatch(): void public function testGetByPeriod(): void { - $this->assertEventually(function () { - $results1h = $this->usage->getByPeriod('requests', '1h'); - $results1d = $this->usage->getByPeriod('requests', '1d'); - $resultsInf = $this->usage->getByPeriod('storage', 'inf'); - - // SummingMergeTree / upsert-with-increase aggregates by deterministic id - $this->assertEquals(1, count($results1h)); - $this->assertEquals(1, count($results1d)); - $this->assertEquals(1, count($resultsInf)); - }); + $results1h = $this->usage->getByPeriod('requests', '1h'); + $results1d = $this->usage->getByPeriod('requests', '1d'); + $resultsInf = $this->usage->getByPeriod('storage', 'inf'); + + // SummingMergeTree / upsert-with-increase aggregates by deterministic id + $this->assertEquals(1, count($results1h)); + $this->assertEquals(1, count($results1d)); + $this->assertEquals(1, count($resultsInf)); } public function testGetBetweenDates(): void @@ -123,27 +94,23 @@ public function testGetBetweenDates(): void public function testCountByPeriod(): void { - $this->assertEventually(function () { - $count1h = $this->usage->countByPeriod('requests', '1h'); - $count1d = $this->usage->countByPeriod('requests', '1d'); - $countBandwidth = $this->usage->countByPeriod('bandwidth', '1h'); - - // Aggregated by deterministic id: multiple logs in same period/time collapse - $this->assertEquals(1, $count1h); - $this->assertEquals(1, $count1d); - $this->assertEquals(1, $countBandwidth); - }); + $count1h = $this->usage->countByPeriod('requests', '1h'); + $count1d = $this->usage->countByPeriod('requests', '1d'); + $countBandwidth = $this->usage->countByPeriod('bandwidth', '1h'); + + // Aggregated by deterministic id: multiple logs in same period/time collapse + $this->assertEquals(1, $count1h); + $this->assertEquals(1, $count1d); + $this->assertEquals(1, $countBandwidth); } public function testSumByPeriod(): void { - $this->assertEventually(function () { - $sum = $this->usage->sumByPeriod('requests', '1h'); - $this->assertEquals(250, $sum); // 100 + 150 + $sum = $this->usage->sumByPeriod('requests', '1h'); + $this->assertEquals(250, $sum); // 100 + 150 - $sumBandwidth = $this->usage->sumByPeriod('bandwidth', '1h'); - $this->assertEquals(5000, $sumBandwidth); - }); + $sumBandwidth = $this->usage->sumByPeriod('bandwidth', '1h'); + $this->assertEquals(5000, $sumBandwidth); } public function testIncrementingDefaultBehavior(): void @@ -154,15 +121,13 @@ public function testIncrementingDefaultBehavior(): void // Log the same metric twice with identical period and tags $this->assertTrue($this->usage->log('increment-test', 5, '1h', [])); $this->assertTrue($this->usage->log('increment-test', 7, '1h', [])); - $this->assertEventually(function () { - // Because adapters now aggregate by deterministic id/time/period (and tenant where applicable), - // there should be a single record and the summed value should be 12. - $results = $this->usage->getByPeriod('increment-test', '1h'); - $this->assertEquals(1, count($results)); - - $sum = $this->usage->sumByPeriod('increment-test', '1h'); - $this->assertEquals(12, $sum); - }, 2); + // Because adapters now aggregate by deterministic id/time/period (and tenant where applicable), + // there should be a single record and the summed value should be 12. + $results = $this->usage->getByPeriod('increment-test', '1h'); + $this->assertEquals(1, count($results)); + + $sum = $this->usage->sumByPeriod('increment-test', '1h'); + $this->assertEquals(12, $sum); } public function testWithQueries(): void From 5643c4d5ace37aef41ff221e336dff185dc531b2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 27 Jan 2026 10:18:25 +0000 Subject: [PATCH 31/44] feat: Add tenant validation and override logic in ClickHouse adapter and corresponding tests --- src/Usage/Adapter/ClickHouse.php | 46 ++++++++++++++++++++++++-- tests/Usage/Adapter/ClickHouseTest.php | 45 +++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 6b2271d..5639252 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -745,6 +745,23 @@ public function logBatch(array $metrics): bool throw new Exception("Metric #{$index}: 'tags' must be an array, got " . gettype($metricData['tags'])); } + // Validate tenant when provided (metric-level tenant overrides adapter tenant) + if (array_key_exists('tenant', $metricData)) { + $tenantValue = $metricData['tenant']; + + if ($tenantValue !== null) { + if (is_int($tenantValue)) { + if ($tenantValue < 0) { + throw new Exception("Metric #{$index}: 'tenant' cannot be negative"); + } + } elseif (is_string($tenantValue) && ctype_digit($tenantValue)) { + // ok numeric string + } else { + throw new Exception("Metric #{$index}: 'tenant' must be a non-negative integer, got " . gettype($tenantValue)); + } + } + } + // Validate complete data structure using Metric class $data = [ 'metric' => $metric, @@ -789,7 +806,7 @@ public function logBatch(array $metrics): bool $timestamp = $this->formatDateTime($time); // Deterministic id for aggregation - $tenant = $this->sharedTables ? $this->tenant : null; + $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; /** @var string $metric */ /** @var string $period */ /** @var string $timestamp */ @@ -827,7 +844,7 @@ public function logBatch(array $metrics): bool if ($this->sharedTables) { $tenantKey = 'tenant_' . $paramCounter; - $queryParams[$tenantKey] = $this->tenant; + $queryParams[$tenantKey] = $tenant; $valuePlaceholders[] = '{' . $tenantKey . ':Nullable(UInt64)}'; } @@ -845,6 +862,31 @@ public function logBatch(array $metrics): bool return true; } + /** + * Resolve tenant for a single metric entry, giving precedence to metric-level tenant. + * + * @param array $metricData + */ + private function resolveTenantFromMetric(array $metricData): ?int + { + $tenant = array_key_exists('tenant', $metricData) ? $metricData['tenant'] : $this->tenant; + + if ($tenant === null) { + return null; + } + + if (is_int($tenant)) { + return $tenant; + } + + if (is_string($tenant) && ctype_digit($tenant)) { + return (int) $tenant; + } + + // Validation should prevent reaching here, but return null defensively + return null; + } + /** diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 78d2957..6dcf3b2 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -3,6 +3,7 @@ namespace Utopia\Tests\Adapter; use PHPUnit\Framework\TestCase; +use Utopia\Database\DateTime; use Utopia\Tests\Usage\UsageBase; use Utopia\Usage\Adapter\ClickHouse as ClickHouseAdapter; use Utopia\Usage\Usage; @@ -32,4 +33,48 @@ protected function initializeUsage(): void $this->usage = new Usage($adapter); $this->usage->setup(); } + + public function testMetricTenantOverridesAdapterTenantInBatch(): void + { + $host = getenv('CLICKHOUSE_HOST') ?: 'clickhouse'; + $username = getenv('CLICKHOUSE_USER') ?: 'default'; + $password = getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse'; + $port = (int) (getenv('CLICKHOUSE_PORT') ?: 8123); + $secure = (bool) (getenv('CLICKHOUSE_SECURE') ?: false); + + $adapter = new ClickHouseAdapter($host, $username, $password, $port, $secure); + $adapter->setNamespace('utopia_usage_shared'); + $adapter->setSharedTables(true); + $adapter->setTenant(1); + + if ($database = getenv('CLICKHOUSE_DATABASE')) { + $adapter->setDatabase($database); + } + + $usage = new Usage($adapter); + $usage->setup(); + $usage->purge(DateTime::now()); + + $metrics = [ + [ + 'metric' => 'tenant-override', + 'value' => 5, + 'period' => '1h', + 'tenant' => 2, + 'tags' => [], + ], + ]; + + $this->assertTrue($usage->logBatch($metrics)); + + // Switch adapter scope to the metric tenant to verify the row was stored under the override + $adapter->setTenant(2); + + $results = $usage->getByPeriod('tenant-override', '1h'); + + $this->assertCount(1, $results); + $this->assertEquals(2, $results[0]->getTenant()); + + $usage->purge(DateTime::now()); + } } From 13e637b5afa5ddd9e99b14c5d77b2cc078f6eca0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 27 Jan 2026 10:20:48 +0000 Subject: [PATCH 32/44] fix: Correct tenant key usage in ClickHouse adapter and tests --- src/Usage/Adapter/ClickHouse.php | 4 ++-- tests/Usage/Adapter/ClickHouseTest.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 5639252..c73d85d 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -747,7 +747,7 @@ public function logBatch(array $metrics): bool // Validate tenant when provided (metric-level tenant overrides adapter tenant) if (array_key_exists('tenant', $metricData)) { - $tenantValue = $metricData['tenant']; + $tenantValue = $metricData['$tenant']; if ($tenantValue !== null) { if (is_int($tenantValue)) { @@ -869,7 +869,7 @@ public function logBatch(array $metrics): bool */ private function resolveTenantFromMetric(array $metricData): ?int { - $tenant = array_key_exists('tenant', $metricData) ? $metricData['tenant'] : $this->tenant; + $tenant = array_key_exists('$tenant', $metricData) ? $metricData['$tenant'] : $this->tenant; if ($tenant === null) { return null; diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 6dcf3b2..8eec9d4 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -60,7 +60,7 @@ public function testMetricTenantOverridesAdapterTenantInBatch(): void 'metric' => 'tenant-override', 'value' => 5, 'period' => '1h', - 'tenant' => 2, + '$tenant' => 2, 'tags' => [], ], ]; From e7997109d7e10fe1ebaa1a5dfc1c69b39a503ea1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jan 2026 00:19:38 +0000 Subject: [PATCH 33/44] fix: Update ClickHouse and Database adapters to handle nullable timestamps and improve deterministic ID generation --- src/Usage/Adapter/ClickHouse.php | 20 +++++++++++++------- src/Usage/Adapter/Database.php | 2 +- src/Usage/Adapter/SQL.php | 5 +++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index c73d85d..b991448 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -456,7 +456,7 @@ public function setup(): void ENGINE = SummingMergeTree() ORDER BY {$orderByExpr} PARTITION BY toYYYYMM(time) - SETTINGS index_granularity = 8192 + SETTINGS index_granularity = 8192, allow_nullable_key = 1 "; $this->query($createTableSql); @@ -622,15 +622,15 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 // Period-aligned time so increments fall into the correct bucket $now = new \DateTime(); $time = $period === Usage::PERIOD_INF - ? '1000-01-01 00:00:00' + ? null : $now->format(Usage::PERIODS[$period]); - $timestamp = $this->formatDateTime($time); + $timestamp = $time !== null ? $this->formatDateTime($time) : null; // Deterministic id so SummingMergeTree will aggregate increments for the same group $tenant = $this->sharedTables ? $this->tenant : null; /** @var string $metric */ /** @var string $period */ - /** @var string $timestamp */ + /** @var string|null $timestamp */ $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); // Build insert columns dynamically from attributes @@ -801,15 +801,15 @@ public function logBatch(array $metrics): bool // Period-aligned time so increments fall into the correct bucket $now = new \DateTime(); $time = $period === Usage::PERIOD_INF - ? '1000-01-01 00:00:00' + ? null : $now->format(Usage::PERIODS[$period]); - $timestamp = $this->formatDateTime($time); + $timestamp = $time !== null ? $this->formatDateTime($time) : null; // Deterministic id for aggregation $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; /** @var string $metric */ /** @var string $period */ - /** @var string $timestamp */ + /** @var string|null $timestamp */ $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); $valuePlaceholders = []; @@ -915,6 +915,7 @@ public function find(array $queries = []): array $conditions = $parsed['filters']; if ($tenantFilter) { $conditions[] = ltrim($tenantFilter, ' AND'); + $parsed['params']['tenant'] = $this->tenant; } $whereClause = ' WHERE ' . implode(' AND ', $conditions); } @@ -969,6 +970,11 @@ public function count(array $queries = []): int $params = $parsed['params']; unset($params['limit'], $params['offset']); + // Add tenant param if filter is active + if ($tenantFilter) { + $params['tenant'] = $this->tenant; + } + $sql = " SELECT COUNT(*) as count FROM {$fromTable}{$whereClause} diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 734ca00..a191069 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -102,7 +102,7 @@ public function logBatch(array $metrics): bool $now = new \DateTime(); $time = $period === 'inf' - ? '1000-01-01 00:00:00' + ? null : $now->format(Usage::PERIODS[$period]); $tags = $metric['tags'] ?? []; diff --git a/src/Usage/Adapter/SQL.php b/src/Usage/Adapter/SQL.php index 3632083..11b83f0 100644 --- a/src/Usage/Adapter/SQL.php +++ b/src/Usage/Adapter/SQL.php @@ -119,10 +119,11 @@ protected function getAllColumnDefinitions(): array * Build deterministic document ID based on time bucket, period, metric, and tenant (when applicable). * Tags are intentionally excluded to ensure aggregation regardless of tag differences. */ - protected function buildDeterministicId(string $metric, string $period, string $timeBucket, ?int $tenant = null): string + protected function buildDeterministicId(string $metric, string $period, ?string $timeBucket, ?int $tenant = null): string { $tenantPart = $tenant !== null ? ('_' . $tenant) : ''; - $hashInput = $timeBucket . '_' . $period . '_' . $metric . $tenantPart; + $timePart = $timeBucket ?? ''; + $hashInput = $timePart . '_' . $period . '_' . $metric . $tenantPart; return md5($hashInput); } From 3661fd131eda833a9432f1d362da4c142bb25128 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jan 2026 00:28:28 +0000 Subject: [PATCH 34/44] Fix timestamp for infinity --- src/Usage/Adapter/Database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index a191069..a70de92 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -65,7 +65,7 @@ public function log(string $metric, int $value, string $period = '1h', array $ta $now = new \DateTime(); $time = $period === 'inf' - ? '1000-01-01 00:00:00' + ? null : $now->format(Usage::PERIODS[$period]); // Sort tags for consistent storage From 1689ed9f36ae925e62ff67f188ba7123c4e641f8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jan 2026 02:11:08 +0000 Subject: [PATCH 35/44] fix: Update usage of Query class in Adapter and Usage files --- src/Usage/Adapter.php | 12 ++-- src/Usage/Adapter/ClickHouse.php | 29 ++------ src/Usage/Adapter/Database.php | 117 ++++++++++++++++++++++++++----- src/Usage/Query.php | 10 +-- src/Usage/Usage.php | 12 ++-- tests/Usage/UsageBase.php | 2 +- 6 files changed, 121 insertions(+), 61 deletions(-) diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 03b67a3..9c5fb99 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -31,7 +31,7 @@ abstract public function logBatch(array $metrics): bool; /** * Get usage metrics by period * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return array */ abstract public function getByPeriod(string $metric, string $period, array $queries = []): array; @@ -39,7 +39,7 @@ abstract public function getByPeriod(string $metric, string $period, array $quer /** * Get usage metrics between dates * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return array */ abstract public function getBetweenDates(string $metric, string $startDate, string $endDate, array $queries = []): array; @@ -47,14 +47,14 @@ abstract public function getBetweenDates(string $metric, string $startDate, stri /** * Count usage metrics by period * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries */ abstract public function countByPeriod(string $metric, string $period, array $queries = []): int; /** * Sum usage metrics by period * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries */ abstract public function sumByPeriod(string $metric, string $period, array $queries = []): int; @@ -66,7 +66,7 @@ abstract public function purge(string $datetime): bool; /** * Find metrics using Query objects. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return array */ abstract public function find(array $queries = []): array; @@ -74,7 +74,7 @@ abstract public function find(array $queries = []): array; /** * Count metrics using Query objects. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return int */ abstract public function count(array $queries = []): int; diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index b991448..39e2529 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -3,7 +3,7 @@ namespace Utopia\Usage\Adapter; use Exception; -use Utopia\Database\Query; +use Utopia\Usage\Query; use Utopia\Fetch\Client; use Utopia\Usage\Metric; use Utopia\Usage\Usage; @@ -1072,17 +1072,7 @@ private function parseQueries(array $queries): array } break; - case Query::TYPE_SEARCH: - // SEARCH is like LIKE - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $paramName = 'param_' . $paramCounter++; - $value = is_array($values) && !empty($values) ? $values[0] : $values; - $filters[] = "{$escapedAttr} LIKE {{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($value); - break; - - case Query::TYPE_SELECT: + case Query::TYPE_IN: // SELECT allows selecting multiple columns/values $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); @@ -1098,20 +1088,12 @@ private function parseQueries(array $queries): array break; case Query::TYPE_ORDER_DESC: - // Skip special Query attributes (like $sequence) that aren't real columns - if (str_starts_with($attribute, '$')) { - break; - } $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); $orderBy[] = "{$escapedAttr} DESC"; break; case Query::TYPE_ORDER_ASC: - // Skip special Query attributes (like $sequence) that aren't real columns - if (str_starts_with($attribute, '$')) { - break; - } $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); $orderBy[] = "{$escapedAttr} ASC"; @@ -1313,7 +1295,7 @@ public function getByPeriod(string $metric, string $period, array $queries = []) } // Add default ordering - $allQueries[] = Query::orderDesc(); + $allQueries[] = Query::orderDesc('time'); return $this->find($allQueries); } @@ -1330,8 +1312,7 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa { $allQueries = [ Query::equal('metric', [$metric]), - Query::greaterThanEqual('time', $startDate), - Query::lessThanEqual('time', $endDate), + Query::between('time', $startDate, $endDate) ]; // Add custom queries @@ -1340,7 +1321,7 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa } // Add default ordering - $allQueries[] = Query::orderDesc(); + $allQueries[] = Query::orderDesc('time'); return $this->find($allQueries); } diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index a70de92..d5074d4 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -5,9 +5,10 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate as DuplicateException; -use Utopia\Database\Query; +use Utopia\Database\Query as DatabaseQuery; use Utopia\Exception; use Utopia\Usage\Metric; +use Utopia\Usage\Query; use Utopia\Usage\Usage; class Database extends SQL @@ -138,17 +139,92 @@ public function logBatch(array $metrics): bool return true; } + /** + * Convert Utopia\Usage\Query to Utopia\Database\Query for use with the Database class. + * + * @param array $queries + * @return array + */ + private function convertQueriesToDatabase(array $queries): array + { + $dbQueries = []; + foreach ($queries as $query) { + $method = $query->getMethod(); + $attribute = $query->getAttribute(); + $values = $query->getValues(); + + switch ($method) { + case Query::TYPE_EQUAL: + /** @var array|bool|float|int|string> $values */ + $dbQueries[] = DatabaseQuery::equal($attribute, $values); + break; + case Query::TYPE_GREATER: + if (!empty($values)) { + /** @var bool|float|int|string $value */ + $value = $values[0]; + $dbQueries[] = DatabaseQuery::greaterThan($attribute, $value); + } + break; + case Query::TYPE_LESSER: + if (!empty($values)) { + /** @var bool|float|int|string $value */ + $value = $values[0]; + $dbQueries[] = DatabaseQuery::lessThan($attribute, $value); + } + break; + case Query::TYPE_BETWEEN: + if (count($values) >= 2) { + /** @var bool|float|int|string $start */ + $start = $values[0]; + /** @var bool|float|int|string $end */ + $end = $values[1]; + $dbQueries[] = DatabaseQuery::between($attribute, $start, $end); + } + break; + case Query::TYPE_IN: + // For IN queries, the values are the items to match + // Create using equal with array values for compatibility + /** @var array|bool|float|int|string> $values */ + $dbQueries[] = DatabaseQuery::contains($attribute, $values); + break; + case Query::TYPE_ORDER_DESC: + $dbQueries[] = DatabaseQuery::orderDesc($attribute); + break; + case Query::TYPE_ORDER_ASC: + $dbQueries[] = DatabaseQuery::orderAsc($attribute); + break; + case Query::TYPE_LIMIT: + if (!empty($values)) { + /** @var int|string $val */ + $val = $values[0] ?? 0; + $dbQueries[] = DatabaseQuery::limit((int) $val); + } + break; + case Query::TYPE_OFFSET: + if (!empty($values)) { + /** @var int|string $val */ + $val = $values[0] ?? 0; + $dbQueries[] = DatabaseQuery::offset((int) $val); + } + break; + } + } + + return $dbQueries; + } + public function getByPeriod(string $metric, string $period, array $queries = []): array { /** @var array $result */ $result = $this->db->getAuthorization()->skip(function () use ($queries, $metric, $period) { - $queries[] = Query::equal('metric', [$metric]); - $queries[] = Query::equal('period', [$period]); - $queries[] = Query::orderDesc(); + $dbQueries = $this->convertQueriesToDatabase($queries); + $dbQueries[] = DatabaseQuery::equal('metric', [$metric]); + $dbQueries[] = DatabaseQuery::equal('period', [$period]); + $dbQueries[] = DatabaseQuery::orderDesc(); return $this->db->find( collection: $this->collection, - queries: $queries, + queries: $dbQueries, ); }); @@ -159,14 +235,15 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa { /** @var array $result */ $result = $this->db->getAuthorization()->skip(function () use ($queries, $metric, $startDate, $endDate) { - $queries[] = Query::equal('metric', [$metric]); - $queries[] = Query::greaterThanEqual('time', $startDate); - $queries[] = Query::lessThanEqual('time', $endDate); - $queries[] = Query::orderDesc(); + $dbQueries = $this->convertQueriesToDatabase($queries); + $dbQueries[] = DatabaseQuery::equal('metric', [$metric]); + $dbQueries[] = DatabaseQuery::greaterThanEqual('time', $startDate); + $dbQueries[] = DatabaseQuery::lessThanEqual('time', $endDate); + $dbQueries[] = DatabaseQuery::orderDesc(); return $this->db->find( collection: $this->collection, - queries: $queries, + queries: $dbQueries, ); }); @@ -177,13 +254,13 @@ public function countByPeriod(string $metric, string $period, array $queries = [ { /** @var int $count */ $count = $this->db->getAuthorization()->skip(function () use ($queries, $metric, $period) { + $dbQueries = $this->convertQueriesToDatabase($queries); + $dbQueries[] = DatabaseQuery::equal('metric', [$metric]); + $dbQueries[] = DatabaseQuery::equal('period', [$period]); + return $this->db->count( collection: $this->collection, - queries: [ - Query::equal('metric', [$metric]), - Query::equal('period', [$period]), - ...$queries, - ] + queries: $dbQueries ); }); @@ -210,8 +287,8 @@ public function purge(string $datetime): bool $documents = $this->db->find( collection: $this->collection, queries: [ - Query::lessThan('time', $datetime), - Query::limit(100), + DatabaseQuery::lessThan('time', $datetime), + DatabaseQuery::limit(100), ] ); @@ -234,9 +311,10 @@ public function find(array $queries = []): array { /** @var array $result */ $result = $this->db->getAuthorization()->skip(function () use ($queries) { + $dbQueries = $this->convertQueriesToDatabase($queries); return $this->db->find( collection: $this->collection, - queries: $queries, + queries: $dbQueries, ); }); @@ -253,9 +331,10 @@ public function count(array $queries = []): int { /** @var int $count */ $count = $this->db->getAuthorization()->skip(function () use ($queries) { + $dbQueries = $this->convertQueriesToDatabase($queries); return $this->db->count( collection: $this->collection, - queries: $queries + queries: $dbQueries ); }); diff --git a/src/Usage/Query.php b/src/Usage/Query.php index 3cf7fe0..a2d356e 100644 --- a/src/Usage/Query.php +++ b/src/Usage/Query.php @@ -84,12 +84,12 @@ public function getValue(mixed $default = null): mixed * Filter by equal condition * * @param string $attribute - * @param mixed $value + * @param array $value * @return self */ - public static function equal(string $attribute, mixed $value): self + public static function equal(string $attribute, array $value): self { - return new self(self::TYPE_EQUAL, $attribute, [$value]); + return new self(self::TYPE_EQUAL, $attribute, $value); } /** @@ -147,7 +147,7 @@ public static function in(string $attribute, array $values): self * @param string $attribute * @return self */ - public static function orderDesc(string $attribute = 'time'): self + public static function orderDesc(string $attribute): self { return new self(self::TYPE_ORDER_DESC, $attribute); } @@ -158,7 +158,7 @@ public static function orderDesc(string $attribute = 'time'): self * @param string $attribute * @return self */ - public static function orderAsc(string $attribute = 'time'): self + public static function orderAsc(string $attribute): self { return new self(self::TYPE_ORDER_ASC, $attribute); } diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 2e31951..58be2ff 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -77,7 +77,7 @@ public function logBatch(array $metrics): bool /** * Get usage metrics by period. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return array * * @throws \Exception @@ -90,7 +90,7 @@ public function getByPeriod(string $metric, string $period, array $queries = []) /** * Get usage metrics between dates. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return array * * @throws \Exception @@ -103,7 +103,7 @@ public function getBetweenDates(string $metric, string $startDate, string $endDa /** * Count usage metrics by period. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * * @throws \Exception */ @@ -115,7 +115,7 @@ public function countByPeriod(string $metric, string $period, array $queries = [ /** * Sum usage metric values by period. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * * @throws \Exception */ @@ -137,7 +137,7 @@ public function purge(string $datetime): bool /** * Find metrics using Query objects. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return array * @throws \Exception */ @@ -149,7 +149,7 @@ public function find(array $queries = []): array /** * Count metrics using Query objects. * - * @param array<\Utopia\Database\Query> $queries + * @param array<\Utopia\Usage\Query> $queries * @return int * @throws \Exception */ diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index 778b205..eb45022 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -3,7 +3,7 @@ namespace Utopia\Tests\Usage; use Utopia\Database\DateTime; -use Utopia\Database\Query; +use Utopia\Usage\Query; use Utopia\Usage\Usage; trait UsageBase From 3d8adb2fcaf8cdde190afcece49ebe7add9035eb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jan 2026 06:28:02 +0000 Subject: [PATCH 36/44] fix check --- src/Usage/Query.php | 6 +++--- tests/Usage/QueryTest.php | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Usage/Query.php b/src/Usage/Query.php index a2d356e..06f9990 100644 --- a/src/Usage/Query.php +++ b/src/Usage/Query.php @@ -84,12 +84,12 @@ public function getValue(mixed $default = null): mixed * Filter by equal condition * * @param string $attribute - * @param array $value + * @param array> $values * @return self */ - public static function equal(string $attribute, array $value): self + public static function equal(string $attribute, array $values): self { - return new self(self::TYPE_EQUAL, $attribute, $value); + return new self(self::TYPE_EQUAL, $attribute, $values); } /** diff --git a/tests/Usage/QueryTest.php b/tests/Usage/QueryTest.php index aeed447..b77dac3 100644 --- a/tests/Usage/QueryTest.php +++ b/tests/Usage/QueryTest.php @@ -13,7 +13,7 @@ class QueryTest extends TestCase public function testQueryStaticFactoryMethods(): void { // Test equal - $query = Query::equal('userId', '123'); + $query = Query::equal('userId', ['123']); $this->assertEquals(Query::TYPE_EQUAL, $query->getMethod()); $this->assertEquals('userId', $query->getAttribute()); $this->assertEquals(['123'], $query->getValues()); @@ -80,7 +80,7 @@ public function testQueryParseAndToString(): void $this->assertEquals(['123'], $query->getValues()); // Test toString - $query = Query::equal('event', 'create'); + $query = Query::equal('event', ['create']); $json = $query->toString(); $this->assertJson($json); @@ -127,7 +127,7 @@ public function testQueryParseQueries(): void */ public function testGetValue(): void { - $query = Query::equal('userId', '123'); + $query = Query::equal('userId', ['123']); $this->assertEquals('123', $query->getValue()); $query = Query::limit(10); From a89db31eede4a205428a1f98242172bfbf2cf9a3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jan 2026 06:51:33 +0000 Subject: [PATCH 37/44] fix: Enhance Query class with lessThanEqual and greaterThanEqual methods; update ClickHouse and Database adapters to support new query types and improve handling of array values --- src/Usage/Adapter/ClickHouse.php | 83 +++++++++++++++++++++++++++----- src/Usage/Adapter/Database.php | 17 +++++-- src/Usage/Query.php | 24 +++++++-- tests/Usage/QueryTest.php | 18 +++++-- tests/Usage/UsageBase.php | 46 ++++++++++++++++++ 5 files changed, 166 insertions(+), 22 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 39e2529..d8f3d1c 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1017,11 +1017,36 @@ private function parseQueries(array $queries): array case Query::TYPE_EQUAL: $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); - $paramName = 'param_' . $paramCounter++; - // Query values are arrays, use first element - $value = is_array($values) && !empty($values) ? $values[0] : $values; - $filters[] = "{$escapedAttr} = {{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($value); + + // Support arrays of values (produce IN (...) ) or single value equality + if (is_array($values)) { + $inParams = []; + foreach ($values as $value) { + $paramName = 'param_' . $paramCounter++; + if ($attribute === 'time') { + $inParams[] = "{{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime($value); + } else { + $inParams[] = "{{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + } + } + + if (count($inParams) === 1) { + $filters[] = "{$escapedAttr} = " . $inParams[0]; + } else { + $filters[] = "{$escapedAttr} IN (" . implode(', ', $inParams) . ")"; + } + } else { + $paramName = 'param_' . $paramCounter++; + if ($attribute === 'time') { + $filters[] = "{$escapedAttr} = {{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime($values); + } else { + $filters[] = "{$escapedAttr} = {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($values); + } + } break; case Query::TYPE_LESSER: @@ -1072,31 +1097,63 @@ private function parseQueries(array $queries): array } break; - case Query::TYPE_IN: - // SELECT allows selecting multiple columns/values + + + case Query::TYPE_ORDER_DESC: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $orderBy[] = "{$escapedAttr} DESC"; + break; + + case Query::TYPE_ORDER_ASC: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $orderBy[] = "{$escapedAttr} ASC"; + break; + + case Query::TYPE_CONTAINS: $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); $inParams = []; foreach ($values as $value) { $paramName = 'param_' . $paramCounter++; - $inParams[] = "{{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($value); + if ($attribute === 'time') { + $inParams[] = "{{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime($value); + } else { + $inParams[] = "{{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + } } if (!empty($inParams)) { $filters[] = "{$escapedAttr} IN (" . implode(', ', $inParams) . ")"; } break; - case Query::TYPE_ORDER_DESC: + case Query::TYPE_LESSER_EQUAL: $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); - $orderBy[] = "{$escapedAttr} DESC"; + $paramName = 'param_' . $paramCounter++; + if ($attribute === 'time') { + $filters[] = "{$escapedAttr} <= {{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime(is_array($values) ? ($values[0] ?? null) : $values); + } else { + $filters[] = "{$escapedAttr} <= {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue(is_array($values) ? ($values[0] ?? null) : $values); + } break; - case Query::TYPE_ORDER_ASC: + case Query::TYPE_GREATER_EQUAL: $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); - $orderBy[] = "{$escapedAttr} ASC"; + $paramName = 'param_' . $paramCounter++; + if ($attribute === 'time') { + $filters[] = "{$escapedAttr} >= {{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime(is_array($values) ? ($values[0] ?? null) : $values); + } else { + $filters[] = "{$escapedAttr} >= {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue(is_array($values) ? ($values[0] ?? null) : $values); + } break; case Query::TYPE_LIMIT: diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index d5074d4..5cff6cf 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -181,12 +181,23 @@ private function convertQueriesToDatabase(array $queries): array $dbQueries[] = DatabaseQuery::between($attribute, $start, $end); } break; - case Query::TYPE_IN: - // For IN queries, the values are the items to match - // Create using equal with array values for compatibility + case Query::TYPE_CONTAINS: + // For contains queries, the values are the items to match /** @var array|bool|float|int|string> $values */ $dbQueries[] = DatabaseQuery::contains($attribute, $values); break; + case Query::TYPE_LESSER_EQUAL: + if (!empty($values)) { + $value = $values[0]; + $dbQueries[] = DatabaseQuery::lessThanEqual($attribute, $value); + } + break; + case Query::TYPE_GREATER_EQUAL: + if (!empty($values)) { + $value = $values[0]; + $dbQueries[] = DatabaseQuery::greaterThanEqual($attribute, $value); + } + break; case Query::TYPE_ORDER_DESC: $dbQueries[] = DatabaseQuery::orderDesc($attribute); break; diff --git a/src/Usage/Query.php b/src/Usage/Query.php index 06f9990..5de8443 100644 --- a/src/Usage/Query.php +++ b/src/Usage/Query.php @@ -15,7 +15,9 @@ class Query public const TYPE_GREATER = 'greaterThan'; public const TYPE_LESSER = 'lessThan'; public const TYPE_BETWEEN = 'between'; - public const TYPE_IN = 'contains'; + public const TYPE_LESSER_EQUAL = 'lessThanEqual'; + public const TYPE_GREATER_EQUAL = 'greaterThanEqual'; + public const TYPE_CONTAINS = 'contains'; // Order methods public const TYPE_ORDER_DESC = 'orderDesc'; @@ -104,6 +106,14 @@ public static function lessThan(string $attribute, mixed $value): self return new self(self::TYPE_LESSER, $attribute, [$value]); } + /** + * Filter by less than or equal condition + */ + public static function lessThanEqual(string $attribute, mixed $value): self + { + return new self(self::TYPE_LESSER_EQUAL, $attribute, [$value]); + } + /** * Filter by greater than condition * @@ -116,6 +126,14 @@ public static function greaterThan(string $attribute, mixed $value): self return new self(self::TYPE_GREATER, $attribute, [$value]); } + /** + * Filter by greater than or equal condition + */ + public static function greaterThanEqual(string $attribute, mixed $value): self + { + return new self(self::TYPE_GREATER_EQUAL, $attribute, [$value]); + } + /** * Filter by BETWEEN condition * @@ -136,9 +154,9 @@ public static function between(string $attribute, mixed $start, mixed $end): sel * @param array $values * @return self */ - public static function in(string $attribute, array $values): self + public static function contains(string $attribute, array $values): self { - return new self(self::TYPE_IN, $attribute, $values); + return new self(self::TYPE_CONTAINS, $attribute, $values); } /** diff --git a/tests/Usage/QueryTest.php b/tests/Usage/QueryTest.php index b77dac3..13e1af2 100644 --- a/tests/Usage/QueryTest.php +++ b/tests/Usage/QueryTest.php @@ -30,15 +30,27 @@ public function testQueryStaticFactoryMethods(): void $this->assertEquals('time', $query->getAttribute()); $this->assertEquals(['2023-01-01'], $query->getValues()); + // Test greaterThanEqual + $query = Query::greaterThanEqual('time', '2023-01-01'); + $this->assertEquals(Query::TYPE_GREATER_EQUAL, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2023-01-01'], $query->getValues()); + + // Test lessThanEqual + $query = Query::lessThanEqual('time', '2024-01-01'); + $this->assertEquals(Query::TYPE_LESSER_EQUAL, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2024-01-01'], $query->getValues()); + // Test between $query = Query::between('time', '2023-01-01', '2024-01-01'); $this->assertEquals(Query::TYPE_BETWEEN, $query->getMethod()); $this->assertEquals('time', $query->getAttribute()); $this->assertEquals(['2023-01-01', '2024-01-01'], $query->getValues()); - // Test in - $query = Query::in('event', ['create', 'update', 'delete']); - $this->assertEquals(Query::TYPE_IN, $query->getMethod()); + // Test contains + $query = Query::contains('event', ['create', 'update', 'delete']); + $this->assertEquals(Query::TYPE_CONTAINS, $query->getMethod()); $this->assertEquals('event', $query->getAttribute()); $this->assertEquals(['create', 'update', 'delete'], $query->getValues()); diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index eb45022..f8f2362 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -147,6 +147,52 @@ public function testWithQueries(): void $this->assertEquals(0, count($results2)); } + public function testEqualWithArrayValues(): void + { + // Test equal query with array of values (IN clause) + $results = $this->usage->find([ + Query::equal('metric', ['requests', 'bandwidth']), + ]); + + // Should find all metrics matching either 'requests' or 'bandwidth' + $this->assertGreaterThanOrEqual(2, count($results)); + } + + public function testContainsQuery(): void + { + // Test contains query with multiple values + $results = $this->usage->find([ + Query::contains('metric', ['requests', 'storage']), + ]); + + // Should find all metrics matching either 'requests' or 'storage' + $this->assertGreaterThanOrEqual(2, count($results)); + } + + public function testLessThanEqualQuery(): void + { + // Get current time and subtract some time to test lessThanEqual + $now = (new \DateTime())->format('Y-m-d\TH:i:s'); + $results = $this->usage->find([ + Query::lessThanEqual('time', $now), + ]); + + // Should find all metrics with time <= now + $this->assertGreaterThanOrEqual(0, count($results)); + } + + public function testGreaterThanEqualQuery(): void + { + // Get a time in the past (formatted as ISO 8601 string) + $past = (new \DateTime())->modify('-24 hours')->format('Y-m-d\TH:i:s'); + $results = $this->usage->find([ + Query::greaterThanEqual('time', $past), + ]); + + // Should find all metrics with time >= past (most recent metrics) + $this->assertGreaterThanOrEqual(0, count($results)); + } + public function testPurge(): void { sleep(2); From b6c260d42050d364d11e24b07cddf839009a6446 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jan 2026 10:04:32 +0000 Subject: [PATCH 38/44] fix: Update Database adapter to support lessThanEqual and greaterThanEqual query types --- src/Usage/Adapter/ClickHouse.php | 66 ++++++++++++++++++++++---------- src/Usage/Adapter/Database.php | 2 + 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index d8f3d1c..a67d0d7 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -523,9 +523,6 @@ private function formatDateTime($dateTime): string } } - // This is unreachable code but kept for completeness - all valid types are handled above - // @phpstan-ignore-next-line - throw new Exception('DateTime must be a DateTime object or string'); } /** @@ -1003,11 +1000,7 @@ private function parseQueries(array $queries): array $paramCounter = 0; foreach ($queries as $query) { - if (!$query instanceof Query) { - /** @phpstan-ignore-next-line ternary.alwaysTrue - runtime validation despite type hint */ - $type = is_object($query) ? get_class($query) : gettype($query); - throw new \InvalidArgumentException("Invalid query item: expected instance of Query, got {$type}"); - } + $method = $query->getMethod(); $attribute = $query->getAttribute(); @@ -1019,20 +1012,28 @@ private function parseQueries(array $queries): array $escapedAttr = $this->escapeIdentifier($attribute); // Support arrays of values (produce IN (...) ) or single value equality - if (is_array($values)) { + if (count($values) > 1) { + /** @var array $arrayValues */ + $arrayValues = $values; $inParams = []; - foreach ($values as $value) { + foreach ($arrayValues as $value) { $paramName = 'param_' . $paramCounter++; if ($attribute === 'time') { $inParams[] = "{{$paramName}:DateTime64(3)}"; - $params[$paramName] = $this->formatDateTime($value); + /** @var \DateTime|string|null $timeValue */ + $timeValue = $value; + $params[$paramName] = $this->formatDateTime($timeValue); } else { $inParams[] = "{{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($value); + /** @var bool|float|int|string $scalarValue */ + $scalarValue = $value; + $params[$paramName] = $this->formatParamValue($scalarValue); } } - if (count($inParams) === 1) { + /** @var int $inParamCount */ + $inParamCount = count($inParams); + if ($inParamCount === 1) { $filters[] = "{$escapedAttr} = " . $inParams[0]; } else { $filters[] = "{$escapedAttr} IN (" . implode(', ', $inParams) . ")"; @@ -1040,12 +1041,15 @@ private function parseQueries(array $queries): array } else { $paramName = 'param_' . $paramCounter++; if ($attribute === 'time') { + /** @var array<\DateTime|string|null> $values */ + $formattedValue = $this->formatDateTime($values[0]); $filters[] = "{$escapedAttr} = {{$paramName}:DateTime64(3)}"; - $params[$paramName] = $this->formatDateTime($values); } else { + /** @var bool|float|int|string $formattedValue */ + $formattedValue = $this->formatParamValue($values[0]); $filters[] = "{$escapedAttr} = {{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($values); } + $params[$paramName] = $formattedValue; } break; @@ -1119,10 +1123,14 @@ private function parseQueries(array $queries): array $paramName = 'param_' . $paramCounter++; if ($attribute === 'time') { $inParams[] = "{{$paramName}:DateTime64(3)}"; - $params[$paramName] = $this->formatDateTime($value); + /** @var \DateTime|string|null $singleValue */ + $singleValue = $value; + $params[$paramName] = $this->formatDateTime($singleValue); } else { $inParams[] = "{{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($value); + /** @var bool|float|int|string $singleValue */ + $singleValue = $value; + $params[$paramName] = $this->formatParamValue($singleValue); } } if (!empty($inParams)) { @@ -1135,11 +1143,19 @@ private function parseQueries(array $queries): array $escapedAttr = $this->escapeIdentifier($attribute); $paramName = 'param_' . $paramCounter++; if ($attribute === 'time') { + if (is_array($values)) { + /** @var \DateTime|string|null $singleValue */ + $singleValue = $values[0] ?? null; + } $filters[] = "{$escapedAttr} <= {{$paramName}:DateTime64(3)}"; - $params[$paramName] = $this->formatDateTime(is_array($values) ? ($values[0] ?? null) : $values); + $params[$paramName] = $this->formatDateTime($singleValue); } else { + if (is_array($values)) { + /** @var bool|float|int|string $singleValue */ + $singleValue = $values[0] ?? null; + } $filters[] = "{$escapedAttr} <= {{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue(is_array($values) ? ($values[0] ?? null) : $values); + $params[$paramName] = $this->formatParamValue($singleValue); } break; @@ -1148,11 +1164,19 @@ private function parseQueries(array $queries): array $escapedAttr = $this->escapeIdentifier($attribute); $paramName = 'param_' . $paramCounter++; if ($attribute === 'time') { + if (is_array($values)) { + /** @var \DateTime|string|null $singleValue */ + $singleValue = $values[0] ?? null; + } $filters[] = "{$escapedAttr} >= {{$paramName}:DateTime64(3)}"; - $params[$paramName] = $this->formatDateTime(is_array($values) ? ($values[0] ?? null) : $values); + $params[$paramName] = $this->formatDateTime($singleValue); } else { + if (is_array($values)) { + /** @var bool|float|int|string $singleValue */ + $singleValue = $values[0] ?? null; + } $filters[] = "{$escapedAttr} >= {{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue(is_array($values) ? ($values[0] ?? null) : $values); + $params[$paramName] = $this->formatParamValue($singleValue); } break; diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 5cff6cf..0d979b7 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -188,12 +188,14 @@ private function convertQueriesToDatabase(array $queries): array break; case Query::TYPE_LESSER_EQUAL: if (!empty($values)) { + /** @var bool|float|int|string $value */ $value = $values[0]; $dbQueries[] = DatabaseQuery::lessThanEqual($attribute, $value); } break; case Query::TYPE_GREATER_EQUAL: if (!empty($values)) { + /** @var bool|float|int|string $value */ $value = $values[0]; $dbQueries[] = DatabaseQuery::greaterThanEqual($attribute, $value); } From 864d0cb2832b7a8ecc1a4e7552dcbe7e76adbcfe Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 29 Jan 2026 05:54:49 +0000 Subject: [PATCH 39/44] feat: add logCounter and logBatchCounter methods for usage metrics - Introduced logCounter method to log individual usage counter metrics with upsert behavior. - Added logBatchCounter method to log multiple usage counter metrics in batch, allowing for individual entries without aggregation. - Updated logBatch method to accept a batch size parameter for better control over batch processing. - Enhanced Usage class to support new logging methods and batch size functionality. - Implemented tests for new methods, including scenarios for batch sizes, counter behavior, and metrics with tags. --- src/Usage/Adapter.php | 18 +- src/Usage/Adapter/ClickHouse.php | 475 +++++++++++++++++++--------- src/Usage/Adapter/Database.php | 94 +++++- src/Usage/Usage.php | 30 +- tests/Usage/ClickHouseBatchTest.php | 254 +++++++++++++++ tests/Usage/UsageBase.php | 162 +++++++++- 6 files changed, 874 insertions(+), 159 deletions(-) create mode 100644 tests/Usage/ClickHouseBatchTest.php diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 9c5fb99..928c9af 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -25,8 +25,24 @@ abstract public function log(string $metric, int $value, string $period = Usage: * Log multiple metrics in batch * * @param array}> $metrics + * @param int $batchSize Maximum number of metrics per INSERT statement */ - abstract public function logBatch(array $metrics): bool; + abstract public function logBatch(array $metrics, int $batchSize = 1000): bool; + + /** + * Log usage counter metric (individual entry without aggregation) + * + * @param array $tags + */ + abstract public function logCounter(string $metric, int $value, string $period = Usage::PERIOD_1H, array $tags = []): bool; + + /** + * Log multiple counter metrics in batch (individual entries without aggregation) + * + * @param array}> $metrics + * @param int $batchSize Maximum number of metrics per INSERT statement + */ + abstract public function logBatchCounter(array $metrics, int $batchSize = 1000): bool; /** * Get usage metrics by period diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index a67d0d7..728c205 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -33,6 +33,10 @@ class ClickHouse extends SQL private const DEFAULT_TABLE = self::COLLECTION; + private const DEFAULT_COUNTER_TABLE = self::COLLECTION . '_counter'; + + private const INSERT_BATCH_SIZE = 1_000; + private string $host; private int $port; @@ -289,6 +293,23 @@ private function getTableName(): string return $tableName; } + /** + * Get the counter table name with namespace prefix. + * Counter table stores logs as individual entries without aggregation. + * + * @return string + */ + private function getCounterTableName(): string + { + $tableName = self::DEFAULT_COUNTER_TABLE; + + if (!empty($this->namespace)) { + $tableName = $this->namespace . '_' . $tableName; + } + + return $tableName; + } + /** * Execute a ClickHouse query via HTTP interface using Fetch Client. * @@ -443,7 +464,7 @@ public function setup(): void $tableName = $this->getTableName(); $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - // Create table with SummingMergeTree engine so inserts act as increments for matching keys + // Create aggregated table with SummingMergeTree engine so inserts act as increments for matching keys $columnDefs = implode(",\n ", $columns); $indexDefs = !empty($indexes) ? ",\n " . implode(",\n ", $indexes) : ''; @@ -460,6 +481,22 @@ public function setup(): void "; $this->query($createTableSql); + + // Create counter table with ReplacingMergeTree engine (replaces on duplicate ORDER BY key) + $counterTableName = $this->getCounterTableName(); + $escapedCounterDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($counterTableName); + + $createCounterTableSql = " + CREATE TABLE IF NOT EXISTS {$escapedCounterDatabaseAndTable} ( + {$columnDefs}{$indexDefs} + ) + ENGINE = ReplacingMergeTree() + ORDER BY {$orderByExpr} + PARTITION BY toYYYYMM(time) + SETTINGS index_granularity = 8192, allow_nullable_key = 1 + "; + + $this->query($createCounterTableSql); } /** @@ -523,6 +560,8 @@ private function formatDateTime($dateTime): string } } + // For any other type, try to convert to DateTime + throw new Exception("Invalid datetime value type: " . gettype($dateTime)); } /** @@ -572,35 +611,37 @@ protected function getColumnDefinition(string $id): string } /** - * Log a usage metric. - * - * @param array $tags + * Validate a metric's basic structure and constraints. * + * @param string $metric Metric name + * @param int $value Metric value + * @param string $period Period identifier + * @param array $tags Tags + * @param int|null $metricIndex Index for batch error messages * @throws Exception */ - public function log(string $metric, int $value, string $period = Usage::PERIOD_1H, array $tags = []): bool + private function validateMetricData(string $metric, int $value, string $period, array $tags, ?int $metricIndex = null): void { - // Validate period - if (!isset(Usage::PERIODS[$period])) { - throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); - } + $prefix = $metricIndex !== null ? "Metric #{$metricIndex}: " : ''; - // Validate metric and value if (empty($metric)) { - throw new Exception('Metric cannot be empty'); + throw new Exception($prefix . 'Metric cannot be empty'); } if (strlen($metric) > 255) { - throw new Exception('Metric exceeds maximum size of 255 characters'); + throw new Exception($prefix . 'Metric exceeds maximum size of 255 characters'); } if ($value < 0) { - throw new Exception('Value cannot be negative'); + throw new Exception($prefix . 'Value cannot be negative'); + } + + if (!isset(Usage::PERIODS[$period])) { + throw new \InvalidArgumentException($prefix . 'Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); } - // Validate tags format if (!is_array($tags)) { - throw new Exception('Tags must be an array'); + throw new Exception($prefix . 'Tags must be an array'); } // Validate complete data structure using Metric class @@ -611,31 +652,50 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 'tags' => $tags, ]; Metric::validate($data); + } - // Normalize tags for deterministic hashing - /** @var array $tags */ + /** + * Build insert value placeholders and query parameters for a metric. + * + * @param string $metric Metric name + * @param int $value Metric value + * @param string $period Period identifier + * @param array $tags Tags + * @param int|null $tenant Tenant ID + * @param int $paramCounter Parameter counter for batch operations + * @return array{queryParams: array, valuePlaceholders: array} + * @throws Exception + */ + private function buildInsertValuesForMetric( + string $metric, + int $value, + string $period, + array $tags, + ?int $tenant, + int $paramCounter = 0 + ): array { + $queryParams = []; + $valuePlaceholders = []; + + // Normalize tags ksort($tags); - // Period-aligned time so increments fall into the correct bucket + // Period-aligned time $now = new \DateTime(); $time = $period === Usage::PERIOD_INF ? null : $now->format(Usage::PERIODS[$period]); $timestamp = $time !== null ? $this->formatDateTime($time) : null; - // Deterministic id so SummingMergeTree will aggregate increments for the same group - $tenant = $this->sharedTables ? $this->tenant : null; - /** @var string $metric */ - /** @var string $period */ - /** @var string|null $timestamp */ + // Deterministic id $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); - // Build insert columns dynamically from attributes - $insertColumns = ['id']; - $queryParams = ['id' => $id]; - $valuePlaceholders = ['{id:String}']; + // Build id + $idKey = 'id' . ($paramCounter > 0 ? '_' . $paramCounter : ''); + $queryParams[$idKey] = $id; + $valuePlaceholders[] = '{' . $idKey . ':String}'; - // Map attribute values to their positions + // Map attribute values $attributeMap = [ 'metric' => $metric, 'value' => $value, @@ -644,28 +704,68 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 'tags' => json_encode($tags), ]; - // Add columns from attributes in order + // Add attributes dynamically - must include ALL attributes in schema order foreach ($this->getAttributes() as $attribute) { $attrId = $attribute['$id']; - if (!isset($attributeMap[$attrId])) { - continue; // Skip attributes not in our data - } - $insertColumns[] = $attrId; - $queryParams[$attrId] = $attributeMap[$attrId]; - - // Determine ClickHouse type hint + $attrKey = $attrId . ($paramCounter > 0 ? '_' . $paramCounter : ''); $type = $this->getColumnType($attrId); - $valuePlaceholders[] = '{' . $attrId . ':' . $type . '}'; + + // Use the value from map, or null if not present + $queryParams[$attrKey] = $attributeMap[$attrId] ?? null; + $valuePlaceholders[] = '{' . $attrKey . ':' . $type . '}'; + } + + // Add tenant if shared tables + if ($this->sharedTables) { + $tenantKey = 'tenant' . ($paramCounter > 0 ? '_' . $paramCounter : ''); + $queryParams[$tenantKey] = $tenant; + $valuePlaceholders[] = '{' . $tenantKey . ':Nullable(UInt64)}'; + } + + return [ + 'queryParams' => $queryParams, + 'valuePlaceholders' => $valuePlaceholders, + ]; + } + + /** + * Build the INSERT column list (same for all rows). + * + * @return array + */ + private function buildInsertColumns(): array + { + $insertColumns = ['id']; + + foreach ($this->getAttributes() as $attribute) { + $insertColumns[] = $attribute['$id']; } - // Add tenant column if using shared tables if ($this->sharedTables) { $insertColumns[] = 'tenant'; - $valuePlaceholders[] = '{tenant:Nullable(UInt64)}'; - $queryParams['tenant'] = $this->tenant; } + return $insertColumns; + } + + /** + * Log a usage metric. + * + * @param array $tags + * + * @throws Exception + */ + public function log(string $metric, int $value, string $period = Usage::PERIOD_1H, array $tags = []): bool + { + // Validate + $this->validateMetricData($metric, $value, $period, $tags); + + // Build query + $tenant = $this->sharedTables ? $this->tenant : null; + $result = $this->buildInsertValuesForMetric($metric, $value, $period, $tags, $tenant); + + $insertColumns = $this->buildInsertColumns(); $tableName = $this->getTableName(); $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); @@ -673,29 +773,114 @@ public function log(string $metric, int $value, string $period = Usage::PERIOD_1 INSERT INTO {$escapedDatabaseAndTable} (" . implode(', ', $insertColumns) . ") VALUES ( - " . implode(", ", $valuePlaceholders) . " + " . implode(", ", $result['valuePlaceholders']) . " ) "; - $this->query($sql, $queryParams); + $this->query($sql, $result['queryParams']); return true; } /** - * Log multiple usage metrics in batch. + * Log a usage counter metric (uses deterministic ID, replaces if ID matches). + * + * @param array $tags + * + * @throws Exception + */ + public function logCounter(string $metric, int $value, string $period = Usage::PERIOD_1H, array $tags = []): bool + { + // Validate + $this->validateMetricData($metric, $value, $period, $tags); + + // Build query + $tenant = $this->sharedTables ? $this->tenant : null; + $result = $this->buildInsertValuesForMetric($metric, $value, $period, $tags, $tenant); + + $insertColumns = $this->buildInsertColumns(); + $counterTableName = $this->getCounterTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($counterTableName); + + $sql = " + INSERT INTO {$escapedDatabaseAndTable} + (" . implode(', ', $insertColumns) . ") + VALUES ( + " . implode(", ", $result['valuePlaceholders']) . " + ) + "; + + $this->query($sql, $result['queryParams']); + + return true; + } + + /** + * Log multiple usage counter metrics in batch (individual entries without aggregation). * * @param array> $metrics + * @param int $batchSize Maximum number of metrics per INSERT statement * * @throws Exception */ - public function logBatch(array $metrics): bool + public function logBatchCounter(array $metrics, int $batchSize = self::INSERT_BATCH_SIZE): bool { if (empty($metrics)) { return true; } // Validate all metrics before processing + $this->validateMetricsBatch($metrics); + + // Ensure batch size is within acceptable range + $batchSize = \min(self::INSERT_BATCH_SIZE, \max(1, $batchSize)); + + $counterTableName = $this->getCounterTableName(); + $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($counterTableName); + + // Build column list (same for all rows) + $insertColumns = $this->buildInsertColumns(); + + // Process metrics in batches + foreach (\array_chunk($metrics, $batchSize) as $metricsBatch) { + $paramCounter = 0; + $queryParams = []; + $valueClauses = []; + + foreach ($metricsBatch as $metricData) { + $period = $metricData['period'] ?? Usage::PERIOD_1H; + $metric = $metricData['metric']; + $value = $metricData['value']; + $tags = (array) ($metricData['tags'] ?? []); + + // Build values for this metric + $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; + $result = $this->buildInsertValuesForMetric($metric, $value, $period, $tags, $tenant, $paramCounter); + + $queryParams = array_merge($queryParams, $result['queryParams']); + $valueClauses[] = '(' . implode(', ', $result['valuePlaceholders']) . ')'; + $paramCounter++; + } + + $insertSql = " + INSERT INTO {$escapedDatabaseAndTable} + (" . implode(', ', $insertColumns) . ") + VALUES " . implode(', ', $valueClauses); + + $this->query($insertSql, $queryParams); + } + + return true; + } + + /** + * Validate all metrics in a batch before processing. + * + * @param array> $metrics + * @throws Exception + */ + private function validateMetricsBatch(array $metrics): void + { foreach ($metrics as $index => $metricData) { try { // Validate required fields exist @@ -721,26 +906,8 @@ public function logBatch(array $metrics): bool throw new Exception("Metric #{$index}: 'period' must be a string, got " . gettype($period)); } - // Validate metric and value constraints - if (empty($metric)) { - throw new Exception("Metric #{$index}: 'metric' cannot be empty"); - } - if (strlen($metric) > 255) { - throw new Exception("Metric #{$index}: 'metric' exceeds maximum size of 255 characters"); - } - if ($value < 0) { - throw new Exception("Metric #{$index}: 'value' cannot be negative"); - } - - // Validate period - if (!isset(Usage::PERIODS[$period])) { - throw new Exception("Metric #{$index}: Invalid period '{$period}'. Allowed: " . implode(', ', array_keys(Usage::PERIODS))); - } - - // Validate tags if provided - if (isset($metricData['tags']) && !is_array($metricData['tags'])) { - throw new Exception("Metric #{$index}: 'tags' must be an array, got " . gettype($metricData['tags'])); - } + $tags = $metricData['tags'] ?? []; + $this->validateMetricData($metric, $value, $period, $tags, $index); // Validate tenant when provided (metric-level tenant overrides adapter tenant) if (array_key_exists('tenant', $metricData)) { @@ -758,103 +925,66 @@ public function logBatch(array $metrics): bool } } } - - // Validate complete data structure using Metric class - $data = [ - 'metric' => $metric, - 'value' => $value, - 'period' => $period, - 'tags' => $metricData['tags'] ?? [], - ]; - Metric::validate($data); } catch (Exception $e) { throw new Exception($e->getMessage()); } } + } + + /** + * Log multiple usage metrics in batch. + * + * @param array> $metrics + * @param int $batchSize Maximum number of metrics per INSERT statement + * + * @throws Exception + */ + public function logBatch(array $metrics, int $batchSize = self::INSERT_BATCH_SIZE): bool + { + if (empty($metrics)) { + return true; + } + + // Validate all metrics before processing + $this->validateMetricsBatch($metrics); + + // Ensure batch size is within acceptable range + $batchSize = \min(self::INSERT_BATCH_SIZE, \max(1, $batchSize)); $tableName = $this->getTableName(); $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - // Build column list dynamically from attributes - $insertColumns = ['id']; - foreach ($this->getAttributes() as $attribute) { - $insertColumns[] = $attribute['$id']; - } - if ($this->sharedTables) { - $insertColumns[] = 'tenant'; - } + // Build column list (same for all rows) + $insertColumns = $this->buildInsertColumns(); - $paramCounter = 0; - $queryParams = []; - $valueClauses = []; - - foreach ($metrics as $metricData) { - $period = $metricData['period'] ?? Usage::PERIOD_1H; - $metric = $metricData['metric']; - $value = $metricData['value']; - $tags = (array) ($metricData['tags'] ?? []); - ksort($tags); - - // Period-aligned time so increments fall into the correct bucket - $now = new \DateTime(); - $time = $period === Usage::PERIOD_INF - ? null - : $now->format(Usage::PERIODS[$period]); - $timestamp = $time !== null ? $this->formatDateTime($time) : null; - - // Deterministic id for aggregation - $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; - /** @var string $metric */ - /** @var string $period */ - /** @var string|null $timestamp */ - $id = $this->buildDeterministicId($metric, $period, $timestamp, $tenant); - - $valuePlaceholders = []; - - // Add id - $idKey = 'id_' . $paramCounter; - $queryParams[$idKey] = $id; - $valuePlaceholders[] = '{' . $idKey . ':String}'; - - // Add attributes dynamically - $attributeMap = [ - 'metric' => $metric, - 'value' => $value, - 'period' => $period, - 'time' => $timestamp, - 'tags' => json_encode($metricData['tags'] ?? []), - ]; - - foreach ($this->getAttributes() as $attribute) { - $attrId = $attribute['$id']; - if (!isset($attributeMap[$attrId])) { - continue; - } + // Process metrics in batches + foreach (\array_chunk($metrics, $batchSize) as $metricsBatch) { + $paramCounter = 0; + $queryParams = []; + $valueClauses = []; - $attrKey = $attrId . '_' . $paramCounter; - $queryParams[$attrKey] = $attributeMap[$attrId]; + foreach ($metricsBatch as $metricData) { + $period = $metricData['period'] ?? Usage::PERIOD_1H; + $metric = $metricData['metric']; + $value = $metricData['value']; + $tags = (array) ($metricData['tags'] ?? []); - // Determine ClickHouse type hint - $type = $this->getColumnType($attrId); - $valuePlaceholders[] = '{' . $attrKey . ':' . $type . '}'; - } + // Build values for this metric + $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; + $result = $this->buildInsertValuesForMetric($metric, $value, $period, $tags, $tenant, $paramCounter); - if ($this->sharedTables) { - $tenantKey = 'tenant_' . $paramCounter; - $queryParams[$tenantKey] = $tenant; - $valuePlaceholders[] = '{' . $tenantKey . ':Nullable(UInt64)}'; + $queryParams = array_merge($queryParams, $result['queryParams']); + $valueClauses[] = '(' . implode(', ', $result['valuePlaceholders']) . ')'; + $paramCounter++; } - $valueClauses[] = '(' . implode(', ', $valuePlaceholders) . ')'; - $paramCounter++; - } + $insertSql = " + INSERT INTO {$escapedDatabaseAndTable} + (" . implode(', ', $insertColumns) . ") + VALUES " . implode(', ', $valueClauses); - $insertSql = " - INSERT INTO {$escapedDatabaseAndTable} - (" . implode(', ', $insertColumns) . ") - VALUES " . implode(', ', $valueClauses); - - $this->query($insertSql, $queryParams); + $this->query($insertSql, $queryParams); + } return true; } @@ -884,10 +1014,9 @@ private function resolveTenantFromMetric(array $metricData): ?int return null; } - - /** * Find metrics using Query objects. + * Queries both aggregated and counter tables and combines results. * * @param array $queries * @return array @@ -896,8 +1025,12 @@ private function resolveTenantFromMetric(array $metricData): ?int public function find(array $queries = []): array { $tableName = $this->getTableName(); + $counterTableName = $this->getCounterTableName(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedCounterTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($counterTableName); + // FINAL on both tables (SummingMergeTree and ReplacingMergeTree) $fromTable = $escapedTable . ($this->useFinal ? ' FINAL' : ''); + $fromCounterTable = $escapedCounterTable . ($this->useFinal ? ' FINAL' : ''); // Parse queries $parsed = $this->parseQueries($queries); @@ -926,9 +1059,15 @@ public function find(array $queries = []): array // Build LIMIT and OFFSET $limitClause = isset($parsed['limit']) ? ' LIMIT {limit:UInt64}' : ''; $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; + + // Query both tables with UNION ALL $sql = " SELECT {$selectColumns} - FROM {$fromTable}{$whereClause}{$orderClause}{$limitClause}{$offsetClause} + FROM {$fromTable}{$whereClause} + UNION ALL + SELECT {$selectColumns} + FROM {$fromCounterTable}{$whereClause} + {$orderClause}{$limitClause}{$offsetClause} FORMAT TabSeparated "; @@ -938,6 +1077,7 @@ public function find(array $queries = []): array /** * Count metrics using Query objects. + * Counts from both aggregated and counter tables. * * @param array $queries * @return int @@ -946,8 +1086,12 @@ public function find(array $queries = []): array public function count(array $queries = []): int { $tableName = $this->getTableName(); + $counterTableName = $this->getCounterTableName(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedCounterTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($counterTableName); + // FINAL on both tables (SummingMergeTree and ReplacingMergeTree) $fromTable = $escapedTable . ($this->useFinal ? ' FINAL' : ''); + $fromCounterTable = $escapedCounterTable . ($this->useFinal ? ' FINAL' : ''); // Parse queries - we only need filters and params $parsed = $this->parseQueries($queries); @@ -972,9 +1116,14 @@ public function count(array $queries = []): int $params['tenant'] = $this->tenant; } + // Count from both tables $sql = " - SELECT COUNT(*) as count - FROM {$fromTable}{$whereClause} + SELECT SUM(cnt) as total + FROM ( + SELECT COUNT(*) as cnt FROM {$fromTable}{$whereClause} + UNION ALL + SELECT COUNT(*) as cnt FROM {$fromCounterTable}{$whereClause} + ) FORMAT TabSeparated "; @@ -1431,6 +1580,7 @@ public function countByPeriod(string $metric, string $period, array $queries = [ /** * Sum usage metric values by period. + * Sums from both aggregated and counter tables. * * @param array $queries * @@ -1439,8 +1589,12 @@ public function countByPeriod(string $metric, string $period, array $queries = [ public function sumByPeriod(string $metric, string $period, array $queries = []): int { $tableName = $this->getTableName(); + $counterTableName = $this->getCounterTableName(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedCounterTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($counterTableName); + // FINAL on both tables (SummingMergeTree and ReplacingMergeTree) $fromTable = $escapedTable . ($this->useFinal ? ' FINAL' : ''); + $fromCounterTable = $escapedCounterTable . ($this->useFinal ? ' FINAL' : ''); // Build query constraints $allQueries = [ @@ -1467,9 +1621,14 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) $whereClause = ' WHERE ' . implode(' AND ', $conditions); } + // Sum from both tables $sql = " - SELECT sum(value) as total - FROM {$fromTable}{$whereClause} + SELECT SUM(total) as grand_total + FROM ( + SELECT sum(value) as total FROM {$fromTable}{$whereClause} + UNION ALL + SELECT sum(value) as total FROM {$fromCounterTable}{$whereClause} + ) FORMAT TabSeparated "; @@ -1481,13 +1640,16 @@ public function sumByPeriod(string $metric, string $period, array $queries = []) /** * Purge usage metrics older than the specified datetime. + * Purges from both aggregated and counter tables. * * @throws Exception */ public function purge(string $datetime): bool { $tableName = $this->getTableName(); + $counterTableName = $this->getCounterTableName(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedCounterTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($counterTableName); $tenantFilter = $this->getTenantFilter(); $params = ['datetime' => $datetime]; @@ -1495,11 +1657,18 @@ public function purge(string $datetime): bool $params['tenant'] = $this->tenant; } + // Purge from aggregated table $sql = " DELETE FROM {$escapedTable} WHERE time < {datetime:DateTime64(3)}{$tenantFilter} "; + $this->query($sql, $params); + // Purge from counter table + $sql = " + DELETE FROM {$escapedCounterTable} + WHERE time < {datetime:DateTime64(3)}{$tenantFilter} + "; $this->query($sql, $params); return true; diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 0d979b7..9232946 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -90,7 +90,7 @@ public function log(string $metric, int $value, string $period = '1h', array $ta return true; } - public function logBatch(array $metrics): bool + public function logBatch(array $metrics, int $batchSize = 1000): bool { $this->db->getAuthorization()->skip(function () use ($metrics) { $documentsById = []; @@ -139,6 +139,98 @@ public function logBatch(array $metrics): bool return true; } + /** + * Log usage counter metric (upserts document, replaces if ID matches). + * + * @param array $tags + * @return bool + * @throws Exception + */ + public function logCounter(string $metric, int $value, string $period = '1h', array $tags = []): bool + { + if (! isset(Usage::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); + } + + $now = new \DateTime(); + $time = $period === 'inf' + ? null + : $now->format(Usage::PERIODS[$period]); + + // Sort tags for consistent storage + ksort($tags); + $id = $this->buildDeterministicId($metric, $period, $time); + + $this->db->getAuthorization()->skip(function () use ($metric, $value, $period, $time, $tags, $id) { + $doc = new Document([ + '$id' => $id, + '$permissions' => [], + 'metric' => $metric, + 'value' => $value, + 'period' => $period, + 'time' => $time, + 'tags' => $tags, + ]); + + $this->db->upsertDocument($this->collection, $doc); + }); + + return true; + } + + /** + * Log multiple usage counter metrics in batch (upserts documents, replaces if ID matches). + * + * @param array}> $metrics + * @return bool + * @throws Exception + */ + public function logBatchCounter(array $metrics, int $batchSize = 1000): bool + { + $this->db->getAuthorization()->skip(function () use ($metrics) { + $documentsById = []; + foreach ($metrics as $metric) { + $period = $metric['period'] ?? '1h'; + + if (! isset(Usage::PERIODS[$period])) { + throw new \InvalidArgumentException('Invalid period. Allowed: ' . implode(', ', array_keys(Usage::PERIODS))); + } + + $now = new \DateTime(); + $time = $period === 'inf' + ? null + : $now->format(Usage::PERIODS[$period]); + + $tags = $metric['tags'] ?? []; + ksort($tags); + + $id = $this->buildDeterministicId($metric['metric'], $period, $time); + + // Last one wins for the same ID (counter behavior, not aggregating) + $documentsById[$id] = [ + '$id' => $id, + '$permissions' => [], + 'metric' => $metric['metric'], + 'value' => $metric['value'], + 'period' => $period, + 'time' => $time, + 'tags' => $tags, + ]; + } + + $documents = []; + foreach ($documentsById as $doc) { + $documents[] = new Document($doc); + } + + if (!empty($documents)) { + $this->db->upsertDocuments($this->collection, $documents); + } + }); + + return true; + } + /** * Convert Utopia\Usage\Query to Utopia\Database\Query for use with the Database class. * diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 58be2ff..dea57c4 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -66,12 +66,38 @@ public function log(string $metric, int $value, string $period = '1h', array $ta * Log multiple usage metrics in batch. * * @param array}> $metrics + * @param int $batchSize Maximum number of metrics per INSERT statement * @return bool * @throws \Exception */ - public function logBatch(array $metrics): bool + public function logBatch(array $metrics, int $batchSize = 1000): bool { - return $this->adapter->logBatch($metrics); + return $this->adapter->logBatch($metrics, $batchSize); + } + + /** + * Log a usage counter metric (individual entry without aggregation). + * + * @param array $tags + * + * @throws \Exception + */ + public function logCounter(string $metric, int $value, string $period = '1h', array $tags = []): bool + { + return $this->adapter->logCounter($metric, $value, $period, $tags); + } + + /** + * Log multiple usage counter metrics in batch (individual entries without aggregation). + * + * @param array}> $metrics + * @param int $batchSize Maximum number of metrics per INSERT statement + * @return bool + * @throws \Exception + */ + public function logBatchCounter(array $metrics, int $batchSize = 1000): bool + { + return $this->adapter->logBatchCounter($metrics, $batchSize); } /** diff --git a/tests/Usage/ClickHouseBatchTest.php b/tests/Usage/ClickHouseBatchTest.php new file mode 100644 index 0000000..32d1274 --- /dev/null +++ b/tests/Usage/ClickHouseBatchTest.php @@ -0,0 +1,254 @@ +adapter = new ClickHouse( + host: 'localhost', + username: 'default', + password: '', + port: 8123 + ); + + $this->adapter->setUseFinal(true); + $this->usage = new Usage($this->adapter); + + try { + $this->usage->setup(); + } catch (\Exception $e) { + $this->markTestSkipped('ClickHouse not available: ' . $e->getMessage()); + } + } + + protected function tearDown(): void + { + try { + $this->usage->purge(DateTime::now()); + } catch (\Exception $e) { + // Ignore cleanup errors + } + } + + /** + * Test logBatch with explicit batch size parameter + */ + public function testLogBatchWithBatchSize(): void + { + $metrics = [ + ['metric' => 'metric-1', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'metric-2', 'value' => 20, 'period' => '1h', 'tags' => []], + ['metric' => 'metric-3', 'value' => 30, 'period' => '1h', 'tags' => []], + ['metric' => 'metric-4', 'value' => 40, 'period' => '1h', 'tags' => []], + ]; + + // Process with batch size of 2 + $this->assertTrue($this->usage->logBatch($metrics, 2)); + + // Verify all metrics were inserted + $results = $this->usage->find(); + $this->assertGreaterThanOrEqual(4, count($results)); + } + + /** + * Test logBatchCounter with explicit batch size parameter + */ + public function testLogBatchCounterWithBatchSize(): void + { + $metrics = [ + ['metric' => 'counter-1', 'value' => 100, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-2', 'value' => 200, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-3', 'value' => 300, 'period' => '1h', 'tags' => []], + ]; + + // Process with batch size of 2 + $this->assertTrue($this->usage->logBatchCounter($metrics, 2)); + + // Verify counter metrics were inserted (they don't aggregate) + $results = $this->usage->find(); + $this->assertGreaterThanOrEqual(3, count($results)); + } + + /** + * Test large batch with small batch size + */ + public function testLargeBatchWithSmallBatchSize(): void + { + $metrics = []; + for ($i = 0; $i < 100; $i++) { + $metrics[] = [ + 'metric' => 'large-batch-metric', + 'value' => $i, + 'period' => '1h', + 'tags' => ['index' => (string) $i], + ]; + } + + $this->assertTrue($this->usage->logBatch($metrics, 10)); + + // Verify metrics were processed (will be aggregated due to SummingMergeTree) + $results = $this->usage->getByPeriod('large-batch-metric', '1h'); + $this->assertGreaterThanOrEqual(1, count($results)); + } + + /** + * Test counter metrics don't aggregate + */ + public function testCounterMetricsNoAggregation(): void + { + $metrics = [ + ['metric' => 'counter-test', 'value' => 5, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-test', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-test', 'value' => 15, 'period' => '1h', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatchCounter($metrics)); + + // Counter metrics should replace, not aggregate + $results = $this->usage->find([]); + $this->assertGreaterThanOrEqual(1, count($results)); + + // Get the sum - should be just the last value (15) since counter replaces + $sum = $this->usage->sumByPeriod('counter-test', '1h'); + $this->assertEquals(15, $sum); + } + + /** + * Test aggregated metrics do aggregate + */ + public function testAggregatedMetricsAggregate(): void + { + $metrics = [ + ['metric' => 'agg-test', 'value' => 5, 'period' => '1h', 'tags' => []], + ['metric' => 'agg-test', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'agg-test', 'value' => 15, 'period' => '1h', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatch($metrics)); + + // Aggregated metrics should sum: 5 + 10 + 15 = 30 + $sum = $this->usage->sumByPeriod('agg-test', '1h'); + $this->assertEquals(30, $sum); + } + + /** + * Test empty batch + */ + public function testEmptyBatch(): void + { + $this->assertTrue($this->usage->logBatch([])); + $this->assertTrue($this->usage->logBatchCounter([])); + } + + /** + * Test batch with different periods + */ + public function testBatchWithMultiplePeriods(): void + { + $metrics = [ + ['metric' => 'multi-period', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'multi-period', 'value' => 20, 'period' => '1d', 'tags' => []], + ['metric' => 'multi-period', 'value' => 30, 'period' => 'inf', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatch($metrics)); + + // Verify each period has its own aggregated value + $sum1h = $this->usage->sumByPeriod('multi-period', '1h'); + $sum1d = $this->usage->sumByPeriod('multi-period', '1d'); + $sumInf = $this->usage->sumByPeriod('multi-period', 'inf'); + + $this->assertEquals(10, $sum1h); + $this->assertEquals(20, $sum1d); + $this->assertEquals(30, $sumInf); + } + + /** + * Test batch with tags + */ + public function testBatchWithTags(): void + { + $metrics = [ + ['metric' => 'tagged', 'value' => 10, 'period' => '1h', 'tags' => ['region' => 'us-east']], + ['metric' => 'tagged', 'value' => 20, 'period' => '1h', 'tags' => ['region' => 'us-west']], + ['metric' => 'tagged', 'value' => 15, 'period' => '1h', 'tags' => ['region' => 'eu-west']], + ]; + + $this->assertTrue($this->usage->logBatch($metrics)); + + // Verify metrics with different tags are separate entries + $results = $this->usage->getByPeriod('tagged', '1h'); + $this->assertGreaterThanOrEqual(1, count($results)); + } + + /** + * Test batch size at maximum (1000) + */ + public function testBatchSizeAtMaximum(): void + { + $metrics = []; + for ($i = 0; $i < 500; $i++) { + $metrics[] = [ + 'metric' => 'boundary-test', + 'value' => 1, + 'period' => '1h', + 'tags' => [], + ]; + } + + $this->assertTrue($this->usage->logBatch($metrics, 1000)); + + $sum = $this->usage->sumByPeriod('boundary-test', '1h'); + $this->assertEquals(500, $sum); + } + + /** + * Test batch size of 1 + */ + public function testBatchSizeOfOne(): void + { + $metrics = [ + ['metric' => 'size-one-1', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'size-one-2', 'value' => 20, 'period' => '1h', 'tags' => []], + ['metric' => 'size-one-3', 'value' => 30, 'period' => '1h', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatch($metrics, 1)); + + // All metrics should be inserted + $results = $this->usage->find(); + $this->assertGreaterThanOrEqual(3, count($results)); + } + + /** + * Test default batch size (1000) + */ + public function testDefaultBatchSize(): void + { + $metrics = []; + for ($i = 0; $i < 50; $i++) { + $metrics[] = [ + 'metric' => 'default-batch-test', + 'value' => 1, + 'period' => '1h', + 'tags' => [], + ]; + } + + // Use default batch size + $this->assertTrue($this->usage->logBatch($metrics)); + + $sum = $this->usage->sumByPeriod('default-batch-test', '1h'); + $this->assertEquals(50, $sum); + } +} diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index f8f2362..02540bf 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -143,8 +143,9 @@ public function testWithQueries(): void Query::offset(1), ]); - // After aggregation there may be only a single row; offset 1 yields zero rows - $this->assertEquals(0, count($results2)); + // With UNION ALL querying both tables, and SummingMergeTree eventual consistency, + // offset 1 may yield 0 or more rows depending on merge timing + $this->assertLessThanOrEqual(1, count($results2)); } public function testEqualWithArrayValues(): void @@ -230,4 +231,161 @@ public function testPeriodFormats(): void $this->assertEquals('Y-m-d 00:00', $periods['1d']); $this->assertEquals('0000-00-00 00:00', $periods['inf']); } + + public function testLogCounter(): void + { + // Clear existing data + $this->usage->purge(DateTime::now()); + + $result = $this->usage->logCounter('counter-metric', 42, '1h', ['foo' => 'bar']); + $this->assertTrue($result); + + $results = $this->usage->getByPeriod('counter-metric', '1h'); + $this->assertEquals(1, count($results)); + } + + public function testCounterMetricsReplaceOnDuplicate(): void + { + // Clear existing data + $this->usage->purge(DateTime::now()); + + // Log the same counter metric twice + $this->assertTrue($this->usage->logCounter('counter-replace-test', 10, '1h', [])); + $this->assertTrue($this->usage->logCounter('counter-replace-test', 20, '1h', [])); + + // Counter should have the last value (20), not aggregated (30) + $results = $this->usage->getByPeriod('counter-replace-test', '1h'); + $this->assertEquals(1, count($results)); + + $sum = $this->usage->sumByPeriod('counter-replace-test', '1h'); + $this->assertEquals(20, $sum); + } + + public function testLogBatchCounter(): void + { + // Clear existing data + $this->usage->purge(DateTime::now()); + + $metrics = [ + [ + 'metric' => 'batch-counter-1', + 'value' => 100, + 'period' => '1h', + 'tags' => ['region' => 'eu-west'], + ], + [ + 'metric' => 'batch-counter-2', + 'value' => 200, + 'period' => '1d', + 'tags' => ['region' => 'eu-east'], + ], + [ + 'metric' => 'batch-counter-3', + 'value' => 300, + 'period' => 'inf', + 'tags' => ['region' => 'us-west'], + ], + ]; + + $this->assertTrue($this->usage->logBatchCounter($metrics)); + + // Each metric should be stored as individual entry (counter, no aggregation) + $results1h = $this->usage->getByPeriod('batch-counter-1', '1h'); + $results1d = $this->usage->getByPeriod('batch-counter-2', '1d'); + $resultsInf = $this->usage->getByPeriod('batch-counter-3', 'inf'); + + $this->assertEquals(1, count($results1h)); + $this->assertEquals(1, count($results1d)); + $this->assertEquals(1, count($resultsInf)); + } + + public function testDifferenceBetweenAggregatedAndCounter(): void + { + // Clear existing data + $this->usage->purge(DateTime::now()); + + // Log same metric 3 times using aggregated (logBatch) + $this->assertTrue($this->usage->log('agg-vs-counter', 10, '1h', [])); + $this->assertTrue($this->usage->log('agg-vs-counter', 20, '1h', [])); + $this->assertTrue($this->usage->log('agg-vs-counter', 30, '1h', [])); + + $aggSum = $this->usage->sumByPeriod('agg-vs-counter', '1h'); + $aggCount = $this->usage->countByPeriod('agg-vs-counter', '1h'); + + // Clear for counter test + $this->usage->purge(DateTime::now()); + + // Log same counter metric 3 times (last one wins) + $this->assertTrue($this->usage->logCounter('counter-vs-agg', 10, '1h', [])); + $this->assertTrue($this->usage->logCounter('counter-vs-agg', 20, '1h', [])); + $this->assertTrue($this->usage->logCounter('counter-vs-agg', 30, '1h', [])); + + $counterSum = $this->usage->sumByPeriod('counter-vs-agg', '1h'); + $counterCount = $this->usage->countByPeriod('counter-vs-agg', '1h'); + + // Aggregated: sums to 60 (10 + 20 + 30) + $this->assertEquals(60, $aggSum); + // Counter: only has last value (30) + $this->assertEquals(30, $counterSum); + } + + public function testBatchCounterWithMultiplePeriods(): void + { + // Clear existing data + $this->usage->purge(DateTime::now()); + + $metrics = [ + ['metric' => 'multi-period-counter', 'value' => 100, 'period' => '1h', 'tags' => []], + ['metric' => 'multi-period-counter', 'value' => 200, 'period' => '1d', 'tags' => []], + ['metric' => 'multi-period-counter', 'value' => 300, 'period' => 'inf', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatchCounter($metrics)); + + // Each period should have independent counter value + $sum1h = $this->usage->sumByPeriod('multi-period-counter', '1h'); + $sum1d = $this->usage->sumByPeriod('multi-period-counter', '1d'); + $sumInf = $this->usage->sumByPeriod('multi-period-counter', 'inf'); + + $this->assertEquals(100, $sum1h); + $this->assertEquals(200, $sum1d); + $this->assertEquals(300, $sumInf); + } + + public function testBatchCounterWithDuplicateMetricsInBatch(): void + { + // Clear existing data + $this->usage->purge(DateTime::now()); + + // Multiple entries of the same metric in batch (last one wins) + $metrics = [ + ['metric' => 'dup-counter', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'dup-counter', 'value' => 20, 'period' => '1h', 'tags' => []], + ['metric' => 'dup-counter', 'value' => 30, 'period' => '1h', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatchCounter($metrics)); + + // Should only have the last value (30) + $sum = $this->usage->sumByPeriod('dup-counter', '1h'); + $this->assertEquals(30, $sum); + } + + public function testLogBatchCounterWithTags(): void + { + // Clear existing data + $this->usage->purge(DateTime::now()); + + $metrics = [ + ['metric' => 'tagged-counter', 'value' => 50, 'period' => '1h', 'tags' => ['region' => 'us-east']], + ['metric' => 'tagged-counter', 'value' => 75, 'period' => '1h', 'tags' => ['region' => 'us-west']], + ['metric' => 'tagged-counter', 'value' => 100, 'period' => '1h', 'tags' => ['region' => 'eu-west']], + ]; + + $this->assertTrue($this->usage->logBatchCounter($metrics)); + + $results = $this->usage->getByPeriod('tagged-counter', '1h'); + // Each tag variant should be separate entry (deterministic id differs) + $this->assertGreaterThanOrEqual(1, count($results)); + } } From b14345a107c7a3e4d96429063bae0e4a43823f1c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 29 Jan 2026 07:03:34 +0000 Subject: [PATCH 40/44] refactor: consolidate ClickHouse batch tests into ClickHouseTest and remove ClickHouseBatchTest --- tests/Usage/Adapter/ClickHouseTest.php | 212 +++++++++++++++++++++ tests/Usage/ClickHouseBatchTest.php | 254 ------------------------- 2 files changed, 212 insertions(+), 254 deletions(-) delete mode 100644 tests/Usage/ClickHouseBatchTest.php diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index 8eec9d4..cd55e68 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -77,4 +77,216 @@ public function testMetricTenantOverridesAdapterTenantInBatch(): void $usage->purge(DateTime::now()); } + + /** + * Test logBatch with explicit batch size parameter + */ + public function testLogBatchWithBatchSize(): void + { + $metrics = [ + ['metric' => 'metric-1', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'metric-2', 'value' => 20, 'period' => '1h', 'tags' => []], + ['metric' => 'metric-3', 'value' => 30, 'period' => '1h', 'tags' => []], + ['metric' => 'metric-4', 'value' => 40, 'period' => '1h', 'tags' => []], + ]; + + // Process with batch size of 2 + $this->assertTrue($this->usage->logBatch($metrics, 2)); + + // Verify all metrics were inserted + $results = $this->usage->find(); + $this->assertGreaterThanOrEqual(4, count($results)); + } + + /** + * Test logBatchCounter with explicit batch size parameter + */ + public function testLogBatchCounterWithBatchSize(): void + { + $metrics = [ + ['metric' => 'counter-1', 'value' => 100, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-2', 'value' => 200, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-3', 'value' => 300, 'period' => '1h', 'tags' => []], + ]; + + // Process with batch size of 2 + $this->assertTrue($this->usage->logBatchCounter($metrics, 2)); + + // Verify counter metrics were inserted (they don't aggregate) + $results = $this->usage->find(); + $this->assertGreaterThanOrEqual(3, count($results)); + } + + /** + * Test large batch with small batch size + */ + public function testLargeBatchWithSmallBatchSize(): void + { + $metrics = []; + for ($i = 0; $i < 100; $i++) { + $metrics[] = [ + 'metric' => 'large-batch-metric', + 'value' => $i, + 'period' => '1h', + 'tags' => ['index' => (string) $i], + ]; + } + + $this->assertTrue($this->usage->logBatch($metrics, 10)); + + // Verify metrics were processed (will be aggregated due to SummingMergeTree) + $results = $this->usage->getByPeriod('large-batch-metric', '1h'); + $this->assertGreaterThanOrEqual(1, count($results)); + } + + /** + * Test counter metrics don't aggregate + */ + public function testCounterMetricsNoAggregation(): void + { + $metrics = [ + ['metric' => 'counter-test', 'value' => 5, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-test', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'counter-test', 'value' => 15, 'period' => '1h', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatchCounter($metrics)); + + // Counter metrics should replace, not aggregate + $results = $this->usage->find([]); + $this->assertGreaterThanOrEqual(1, count($results)); + + // Get the sum - should be just the last value (15) since counter replaces + $sum = $this->usage->sumByPeriod('counter-test', '1h'); + $this->assertEquals(15, $sum); + } + + /** + * Test aggregated metrics do aggregate + */ + public function testAggregatedMetricsAggregate(): void + { + $metrics = [ + ['metric' => 'agg-test', 'value' => 5, 'period' => '1h', 'tags' => []], + ['metric' => 'agg-test', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'agg-test', 'value' => 15, 'period' => '1h', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatch($metrics)); + + // Aggregated metrics should sum: 5 + 10 + 15 = 30 + $sum = $this->usage->sumByPeriod('agg-test', '1h'); + $this->assertEquals(30, $sum); + } + + /** + * Test empty batch + */ + public function testEmptyBatch(): void + { + $this->assertTrue($this->usage->logBatch([])); + $this->assertTrue($this->usage->logBatchCounter([])); + } + + /** + * Test batch with different periods + */ + public function testBatchWithMultiplePeriods(): void + { + $metrics = [ + ['metric' => 'multi-period', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'multi-period', 'value' => 20, 'period' => '1d', 'tags' => []], + ['metric' => 'multi-period', 'value' => 30, 'period' => 'inf', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatch($metrics)); + + // Verify each period has its own aggregated value + $sum1h = $this->usage->sumByPeriod('multi-period', '1h'); + $sum1d = $this->usage->sumByPeriod('multi-period', '1d'); + $sumInf = $this->usage->sumByPeriod('multi-period', 'inf'); + + $this->assertEquals(10, $sum1h); + $this->assertEquals(20, $sum1d); + $this->assertEquals(30, $sumInf); + } + + /** + * Test batch with tags + */ + public function testBatchWithTags(): void + { + $metrics = [ + ['metric' => 'tagged', 'value' => 10, 'period' => '1h', 'tags' => ['region' => 'us-east']], + ['metric' => 'tagged', 'value' => 20, 'period' => '1h', 'tags' => ['region' => 'us-west']], + ['metric' => 'tagged', 'value' => 15, 'period' => '1h', 'tags' => ['region' => 'eu-west']], + ]; + + $this->assertTrue($this->usage->logBatch($metrics)); + + // Verify metrics with different tags are separate entries + $results = $this->usage->getByPeriod('tagged', '1h'); + $this->assertGreaterThanOrEqual(1, count($results)); + } + + /** + * Test batch size at maximum (1000) + */ + public function testBatchSizeAtMaximum(): void + { + $metrics = []; + for ($i = 0; $i < 500; $i++) { + $metrics[] = [ + 'metric' => 'boundary-test', + 'value' => 1, + 'period' => '1h', + 'tags' => [], + ]; + } + + $this->assertTrue($this->usage->logBatch($metrics, 1000)); + + $sum = $this->usage->sumByPeriod('boundary-test', '1h'); + $this->assertEquals(500, $sum); + } + + /** + * Test batch size of 1 + */ + public function testBatchSizeOfOne(): void + { + $metrics = [ + ['metric' => 'size-one-1', 'value' => 10, 'period' => '1h', 'tags' => []], + ['metric' => 'size-one-2', 'value' => 20, 'period' => '1h', 'tags' => []], + ['metric' => 'size-one-3', 'value' => 30, 'period' => '1h', 'tags' => []], + ]; + + $this->assertTrue($this->usage->logBatch($metrics, 1)); + + // All metrics should be inserted + $results = $this->usage->find(); + $this->assertGreaterThanOrEqual(3, count($results)); + } + + /** + * Test default batch size (1000) + */ + public function testDefaultBatchSize(): void + { + $metrics = []; + for ($i = 0; $i < 50; $i++) { + $metrics[] = [ + 'metric' => 'default-batch-test', + 'value' => 1, + 'period' => '1h', + 'tags' => [], + ]; + } + + // Use default batch size + $this->assertTrue($this->usage->logBatch($metrics)); + + $sum = $this->usage->sumByPeriod('default-batch-test', '1h'); + $this->assertEquals(50, $sum); + } } diff --git a/tests/Usage/ClickHouseBatchTest.php b/tests/Usage/ClickHouseBatchTest.php deleted file mode 100644 index 32d1274..0000000 --- a/tests/Usage/ClickHouseBatchTest.php +++ /dev/null @@ -1,254 +0,0 @@ -adapter = new ClickHouse( - host: 'localhost', - username: 'default', - password: '', - port: 8123 - ); - - $this->adapter->setUseFinal(true); - $this->usage = new Usage($this->adapter); - - try { - $this->usage->setup(); - } catch (\Exception $e) { - $this->markTestSkipped('ClickHouse not available: ' . $e->getMessage()); - } - } - - protected function tearDown(): void - { - try { - $this->usage->purge(DateTime::now()); - } catch (\Exception $e) { - // Ignore cleanup errors - } - } - - /** - * Test logBatch with explicit batch size parameter - */ - public function testLogBatchWithBatchSize(): void - { - $metrics = [ - ['metric' => 'metric-1', 'value' => 10, 'period' => '1h', 'tags' => []], - ['metric' => 'metric-2', 'value' => 20, 'period' => '1h', 'tags' => []], - ['metric' => 'metric-3', 'value' => 30, 'period' => '1h', 'tags' => []], - ['metric' => 'metric-4', 'value' => 40, 'period' => '1h', 'tags' => []], - ]; - - // Process with batch size of 2 - $this->assertTrue($this->usage->logBatch($metrics, 2)); - - // Verify all metrics were inserted - $results = $this->usage->find(); - $this->assertGreaterThanOrEqual(4, count($results)); - } - - /** - * Test logBatchCounter with explicit batch size parameter - */ - public function testLogBatchCounterWithBatchSize(): void - { - $metrics = [ - ['metric' => 'counter-1', 'value' => 100, 'period' => '1h', 'tags' => []], - ['metric' => 'counter-2', 'value' => 200, 'period' => '1h', 'tags' => []], - ['metric' => 'counter-3', 'value' => 300, 'period' => '1h', 'tags' => []], - ]; - - // Process with batch size of 2 - $this->assertTrue($this->usage->logBatchCounter($metrics, 2)); - - // Verify counter metrics were inserted (they don't aggregate) - $results = $this->usage->find(); - $this->assertGreaterThanOrEqual(3, count($results)); - } - - /** - * Test large batch with small batch size - */ - public function testLargeBatchWithSmallBatchSize(): void - { - $metrics = []; - for ($i = 0; $i < 100; $i++) { - $metrics[] = [ - 'metric' => 'large-batch-metric', - 'value' => $i, - 'period' => '1h', - 'tags' => ['index' => (string) $i], - ]; - } - - $this->assertTrue($this->usage->logBatch($metrics, 10)); - - // Verify metrics were processed (will be aggregated due to SummingMergeTree) - $results = $this->usage->getByPeriod('large-batch-metric', '1h'); - $this->assertGreaterThanOrEqual(1, count($results)); - } - - /** - * Test counter metrics don't aggregate - */ - public function testCounterMetricsNoAggregation(): void - { - $metrics = [ - ['metric' => 'counter-test', 'value' => 5, 'period' => '1h', 'tags' => []], - ['metric' => 'counter-test', 'value' => 10, 'period' => '1h', 'tags' => []], - ['metric' => 'counter-test', 'value' => 15, 'period' => '1h', 'tags' => []], - ]; - - $this->assertTrue($this->usage->logBatchCounter($metrics)); - - // Counter metrics should replace, not aggregate - $results = $this->usage->find([]); - $this->assertGreaterThanOrEqual(1, count($results)); - - // Get the sum - should be just the last value (15) since counter replaces - $sum = $this->usage->sumByPeriod('counter-test', '1h'); - $this->assertEquals(15, $sum); - } - - /** - * Test aggregated metrics do aggregate - */ - public function testAggregatedMetricsAggregate(): void - { - $metrics = [ - ['metric' => 'agg-test', 'value' => 5, 'period' => '1h', 'tags' => []], - ['metric' => 'agg-test', 'value' => 10, 'period' => '1h', 'tags' => []], - ['metric' => 'agg-test', 'value' => 15, 'period' => '1h', 'tags' => []], - ]; - - $this->assertTrue($this->usage->logBatch($metrics)); - - // Aggregated metrics should sum: 5 + 10 + 15 = 30 - $sum = $this->usage->sumByPeriod('agg-test', '1h'); - $this->assertEquals(30, $sum); - } - - /** - * Test empty batch - */ - public function testEmptyBatch(): void - { - $this->assertTrue($this->usage->logBatch([])); - $this->assertTrue($this->usage->logBatchCounter([])); - } - - /** - * Test batch with different periods - */ - public function testBatchWithMultiplePeriods(): void - { - $metrics = [ - ['metric' => 'multi-period', 'value' => 10, 'period' => '1h', 'tags' => []], - ['metric' => 'multi-period', 'value' => 20, 'period' => '1d', 'tags' => []], - ['metric' => 'multi-period', 'value' => 30, 'period' => 'inf', 'tags' => []], - ]; - - $this->assertTrue($this->usage->logBatch($metrics)); - - // Verify each period has its own aggregated value - $sum1h = $this->usage->sumByPeriod('multi-period', '1h'); - $sum1d = $this->usage->sumByPeriod('multi-period', '1d'); - $sumInf = $this->usage->sumByPeriod('multi-period', 'inf'); - - $this->assertEquals(10, $sum1h); - $this->assertEquals(20, $sum1d); - $this->assertEquals(30, $sumInf); - } - - /** - * Test batch with tags - */ - public function testBatchWithTags(): void - { - $metrics = [ - ['metric' => 'tagged', 'value' => 10, 'period' => '1h', 'tags' => ['region' => 'us-east']], - ['metric' => 'tagged', 'value' => 20, 'period' => '1h', 'tags' => ['region' => 'us-west']], - ['metric' => 'tagged', 'value' => 15, 'period' => '1h', 'tags' => ['region' => 'eu-west']], - ]; - - $this->assertTrue($this->usage->logBatch($metrics)); - - // Verify metrics with different tags are separate entries - $results = $this->usage->getByPeriod('tagged', '1h'); - $this->assertGreaterThanOrEqual(1, count($results)); - } - - /** - * Test batch size at maximum (1000) - */ - public function testBatchSizeAtMaximum(): void - { - $metrics = []; - for ($i = 0; $i < 500; $i++) { - $metrics[] = [ - 'metric' => 'boundary-test', - 'value' => 1, - 'period' => '1h', - 'tags' => [], - ]; - } - - $this->assertTrue($this->usage->logBatch($metrics, 1000)); - - $sum = $this->usage->sumByPeriod('boundary-test', '1h'); - $this->assertEquals(500, $sum); - } - - /** - * Test batch size of 1 - */ - public function testBatchSizeOfOne(): void - { - $metrics = [ - ['metric' => 'size-one-1', 'value' => 10, 'period' => '1h', 'tags' => []], - ['metric' => 'size-one-2', 'value' => 20, 'period' => '1h', 'tags' => []], - ['metric' => 'size-one-3', 'value' => 30, 'period' => '1h', 'tags' => []], - ]; - - $this->assertTrue($this->usage->logBatch($metrics, 1)); - - // All metrics should be inserted - $results = $this->usage->find(); - $this->assertGreaterThanOrEqual(3, count($results)); - } - - /** - * Test default batch size (1000) - */ - public function testDefaultBatchSize(): void - { - $metrics = []; - for ($i = 0; $i < 50; $i++) { - $metrics[] = [ - 'metric' => 'default-batch-test', - 'value' => 1, - 'period' => '1h', - 'tags' => [], - ]; - } - - // Use default batch size - $this->assertTrue($this->usage->logBatch($metrics)); - - $sum = $this->usage->sumByPeriod('default-batch-test', '1h'); - $this->assertEquals(50, $sum); - } -} From b5bf5f23bb4676588752b5aa50c17cf417b50643 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 29 Jan 2026 07:10:55 +0000 Subject: [PATCH 41/44] fix: improve type hinting and documentation for ClickHouse metrics handling --- src/Usage/Adapter/ClickHouse.php | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 728c205..fbbc85e 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -559,11 +559,7 @@ private function formatDateTime($dateTime): string throw new Exception("Invalid datetime string: {$dateTime}"); } } - - // For any other type, try to convert to DateTime - throw new Exception("Invalid datetime value type: " . gettype($dateTime)); } - /** * Get ClickHouse-specific SQL column definition for a given attribute ID. * @@ -706,6 +702,7 @@ private function buildInsertValuesForMetric( // Add attributes dynamically - must include ALL attributes in schema order foreach ($this->getAttributes() as $attribute) { + /** @var string $attrId */ $attrId = $attribute['$id']; $attrKey = $attrId . ($paramCounter > 0 ? '_' . $paramCounter : ''); @@ -746,6 +743,7 @@ private function buildInsertColumns(): array $insertColumns[] = 'tenant'; } + /** @var array */ return $insertColumns; } @@ -848,10 +846,14 @@ public function logBatchCounter(array $metrics, int $batchSize = self::INSERT_BA $valueClauses = []; foreach ($metricsBatch as $metricData) { + /** @var string $period */ $period = $metricData['period'] ?? Usage::PERIOD_1H; + /** @var string $metric */ $metric = $metricData['metric']; + /** @var int $value */ $value = $metricData['value']; - $tags = (array) ($metricData['tags'] ?? []); + /** @var array $tags */ + $tags = $metricData['tags'] ?? []; // Build values for this metric $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; @@ -906,6 +908,7 @@ private function validateMetricsBatch(array $metrics): void throw new Exception("Metric #{$index}: 'period' must be a string, got " . gettype($period)); } + /** @var array */ $tags = $metricData['tags'] ?? []; $this->validateMetricData($metric, $value, $period, $tags, $index); @@ -964,10 +967,14 @@ public function logBatch(array $metrics, int $batchSize = self::INSERT_BATCH_SIZ $valueClauses = []; foreach ($metricsBatch as $metricData) { + /** @var string $period */ $period = $metricData['period'] ?? Usage::PERIOD_1H; + /** @var string $metric */ $metric = $metricData['metric']; + /** @var int $value */ $value = $metricData['value']; - $tags = (array) ($metricData['tags'] ?? []); + /** @var array $tags */ + $tags = $metricData['tags'] ?? []; // Build values for this metric $tenant = $this->sharedTables ? $this->resolveTenantFromMetric($metricData) : null; From 20c71563d84ee9e6168625c71cdb3a3e5d231edf Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 29 Jan 2026 08:28:28 +0000 Subject: [PATCH 42/44] fix codeql with runtime safety --- src/Usage/Adapter/ClickHouse.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index fbbc85e..f676c03 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -456,7 +456,7 @@ public function setup(): void $attributes = $index['attributes']; // Escape index name and attribute names to prevent SQL injection $escapedIndexName = $this->escapeIdentifier($indexName); - $escapedAttributes = array_map(fn ($attr) => $this->escapeIdentifier($attr), $attributes); + $escapedAttributes = array_map(fn($attr) => $this->escapeIdentifier($attr), $attributes); $attributeList = implode(', ', $escapedAttributes); $indexes[] = "INDEX {$escapedIndexName} ({$attributeList}) TYPE bloom_filter GRANULARITY 1"; } @@ -559,6 +559,9 @@ private function formatDateTime($dateTime): string throw new Exception("Invalid datetime string: {$dateTime}"); } } + + /** @phpstan-ignore-next-line */ + throw new Exception("Invalid datetime value type: " . gettype($dateTime)); } /** * Get ClickHouse-specific SQL column definition for a given attribute ID. From 0eb17a13c64914627608af3467df352ebb2726c0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 29 Jan 2026 08:28:38 +0000 Subject: [PATCH 43/44] format --- src/Usage/Adapter/ClickHouse.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index f676c03..1cde350 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -456,7 +456,7 @@ public function setup(): void $attributes = $index['attributes']; // Escape index name and attribute names to prevent SQL injection $escapedIndexName = $this->escapeIdentifier($indexName); - $escapedAttributes = array_map(fn($attr) => $this->escapeIdentifier($attr), $attributes); + $escapedAttributes = array_map(fn ($attr) => $this->escapeIdentifier($attr), $attributes); $attributeList = implode(', ', $escapedAttributes); $indexes[] = "INDEX {$escapedIndexName} ({$attributeList}) TYPE bloom_filter GRANULARITY 1"; } From 888ada557701b69b67b48f217455483608ed9043 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 29 Jan 2026 08:37:23 +0000 Subject: [PATCH 44/44] feat: add methods for namespace, tenant, and shared tables support in Adapter and Database classes --- src/Usage/Adapter.php | 24 +++++++++++++++++++++ src/Usage/Adapter/Database.php | 39 ++++++++++++++++++++++++++++++++++ src/Usage/Usage.php | 37 ++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index 928c9af..7a4e312 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -94,4 +94,28 @@ abstract public function find(array $queries = []): array; * @return int */ abstract public function count(array $queries = []): int; + + /** + * Set the namespace prefix for table names. + * + * @param string $namespace + * @return self + */ + abstract public function setNamespace(string $namespace): self; + + /** + * Set the tenant ID for multi-tenant support. + * + * @param int|null $tenant + * @return self + */ + abstract public function setTenant(?int $tenant): self; + + /** + * Enable or disable shared tables mode (multi-tenant with tenant column). + * + * @param bool $sharedTables + * @return self + */ + abstract public function setSharedTables(bool $sharedTables): self; } diff --git a/src/Usage/Adapter/Database.php b/src/Usage/Adapter/Database.php index 9232946..7f19d2d 100644 --- a/src/Usage/Adapter/Database.php +++ b/src/Usage/Adapter/Database.php @@ -445,4 +445,43 @@ public function count(array $queries = []): int return $count; } + + /** + * Set the namespace prefix for table names. + * (Not supported in Database adapter) + * + * @param string $namespace + * @return self + */ + public function setNamespace(string $namespace): self + { + $this->db->setNamespace($namespace); + return $this; + } + + /** + * Set the tenant ID for multi-tenant support. + * (Not supported in Database adapter) + * + * @param int|null $tenant + * @return self + */ + public function setTenant(?int $tenant): self + { + $this->db->setTenant($tenant); + return $this; + } + + /** + * Enable or disable shared tables mode (multi-tenant with tenant column). + * (Not supported in Database adapter) + * + * @param bool $sharedTables + * @return self + */ + public function setSharedTables(bool $sharedTables): self + { + $this->setSharedTables($sharedTables); + return $this; + } } diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index dea57c4..7551637 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -183,4 +183,41 @@ public function count(array $queries = []): int { return $this->adapter->count($queries); } + + /** + * Set the namespace prefix for table names. + * + * @param string $namespace + * @return $this + * @throws \Exception + */ + public function setNamespace(string $namespace): self + { + $this->adapter->setNamespace($namespace); + return $this; + } + + /** + * Set the tenant ID for multi-tenant support. + * + * @param int|null $tenant + * @return $this + */ + public function setTenant(?int $tenant): self + { + $this->adapter->setTenant($tenant); + return $this; + } + + /** + * Enable or disable shared tables mode (multi-tenant with tenant column). + * + * @param bool $sharedTables + * @return $this + */ + public function setSharedTables(bool $sharedTables): self + { + $this->adapter->setSharedTables($sharedTables); + return $this; + } }