diff --git a/CHANGELOG.md b/CHANGELOG.md index e446d49c2..b5f026489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Change Log +## [0.9.9](https://github.com/php-enqueue/enqueue-dev/tree/0.9.9) (2019-04-04) +[Full Changelog](https://github.com/php-enqueue/enqueue-dev/compare/0.9.8...0.9.9) + +**Merged pull requests:** + +- \[amqp-bunny\] Fix bunny producer to properly map headers to expected by bunny headers [\#816](https://github.com/php-enqueue/enqueue-dev/pull/816) ([nightlinus](https://github.com/nightlinus)) +- \[amqp-bunny\]\[doc\] Update amqp\_bunny.md [\#797](https://github.com/php-enqueue/enqueue-dev/pull/797) ([enumag](https://github.com/enumag)) +- \[dbal\] Fix DBAL Consumer duplicating messages when rejecting with requeue [\#815](https://github.com/php-enqueue/enqueue-dev/pull/815) ([Steveb-p](https://github.com/Steveb-p)) +- \[rdkafka\] Set `commit\_async` as true by default for Kafka, update docs [\#810](https://github.com/php-enqueue/enqueue-dev/pull/810) ([Steveb-p](https://github.com/Steveb-p)) +- \[rdkafka\] stats\_cb support [\#798](https://github.com/php-enqueue/enqueue-dev/pull/798) ([fkulakov](https://github.com/fkulakov)) +- \[Monitoring\]\[InfluxDB\] Allow passing Client as configuration option. [\#809](https://github.com/php-enqueue/enqueue-dev/pull/809) ([Steveb-p](https://github.com/Steveb-p)) +- \[doc\] better doc for traceable message producer [\#813](https://github.com/php-enqueue/enqueue-dev/pull/813) ([sylfabre](https://github.com/sylfabre)) +- \[doc\] Minor typo fix in docblock [\#805](https://github.com/php-enqueue/enqueue-dev/pull/805) ([gpenverne](https://github.com/gpenverne)) +- fix comment on QueueConsumer constructor [\#796](https://github.com/php-enqueue/enqueue-dev/pull/796) ([kaznovac](https://github.com/kaznovac)) + ## [0.9.8](https://github.com/php-enqueue/enqueue-dev/tree/0.9.8) (2019-02-27) [Full Changelog](https://github.com/php-enqueue/enqueue-dev/compare/0.9.7...0.9.8) diff --git a/README.md b/README.md index df5b154aa..0c8bbbc46 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ Enqueue is an MIT-licensed open source project with its ongoing development made possible entirely by the support of community and our customers. If you'd like to join them, please consider: -- [Become a sponsor](https://www.patreon.com/makasim) - [Become our client](http://forma-pro.com/) --- diff --git a/bin/release b/bin/release index 41735ff64..c4179d307 100755 --- a/bin/release +++ b/bin/release @@ -22,7 +22,7 @@ git add CHANGELOG.md && git commit -m "Release $1" -S && git push origin "$CURRE ./bin/subtree-split -for REMOTE in origin stomp amqp-ext amqp-lib amqp-bunny amqp-tools pheanstalk gearman sqs gps fs redis dbal null rdkafka enqueue simple-client enqueue-bundle job-queue test async-event-dispatcher async-command mongodb wamp monitoring dsn +for REMOTE in origin stomp amqp-ext amqp-lib amqp-bunny amqp-tools pheanstalk gearman sqs sns snsqs gps fs redis dbal null rdkafka enqueue simple-client enqueue-bundle job-queue test async-event-dispatcher async-command mongodb wamp monitoring dsn do echo "" echo "" diff --git a/docs/bundle/functional_testing.md b/docs/bundle/functional_testing.md index 6afa4f4c9..ae25293a9 100644 --- a/docs/bundle/functional_testing.md +++ b/docs/bundle/functional_testing.md @@ -34,7 +34,7 @@ enqueue: ## Traceable message producer -Imagine you have a service that internally sends a message and you have to find out was the message sent or not. +Imagine you have a service `my_service` with a method `someMethod()` that internally sends a message and you have to find out was the message sent or not. There is a solution for that. You have to enable traceable message producer in test environment. ```yaml @@ -65,10 +65,11 @@ class FooTest extends WebTestCase public function testMessageSentToFooTopic() { - $service = $this->client->getContainer()->get('a_service'); + // Use your own business logic here: + $service = $this->client->getContainer()->get('my_service'); - // the method calls inside $producer->send('fooTopic', 'messageBody'); - $service->do(); + // someMethod() is part of your business logic and is calling somewhere $producer->send('fooTopic', 'messageBody'); + $service->someMethod(); $traces = $this->getProducer()->getTopicTraces('fooTopic'); @@ -86,4 +87,4 @@ class FooTest extends WebTestCase } ``` -[back to index](../index.md) \ No newline at end of file +[back to index](../index.md) diff --git a/docs/monitoring.md b/docs/monitoring.md index 12fa85fd1..93ea8876e 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -9,19 +9,19 @@ Enqueue is an MIT-licensed open source project with its ongoing development made # Monitoring. -Enqueue provides a tool for monitoring message queues. +Enqueue provides a tool for monitoring message queues. With it, you can control how many messages were sent, how many processed successfuly or failed. -How many consumers are working, their up time, processed messages stats, memory usage and system load. +How many consumers are working, their up time, processed messages stats, memory usage and system load. The tool could be integrated with virtually any analytics and monitoring platform. -There are several integration: +There are several integration: * [Datadog StatsD](https://datadoghq.com) * [InfluxDB](https://www.influxdata.com/) and [Grafana](https://grafana.com/) - * [WAMP (Web Application Messaging Protocol)](https://wamp-proto.org/) + * [WAMP (Web Application Messaging Protocol)](https://wamp-proto.org/) We are working on a JS\WAMP based real-time UI tool, for more information please [contact us](opensource@forma-pro.com). ![Grafana Monitoring](images/grafana_monitoring.jpg) -[contact us](opensource@forma-pro.com) if need a Grafana template such as on the picture. +[contact us](opensource@forma-pro.com) if need a Grafana template such as on the picture. * [Installation](#installation) * [Track sent messages](#track-sent-messages) @@ -40,7 +40,7 @@ We are working on a JS\WAMP based real-time UI tool, for more information please composer req enqueue/monitoring:0.9.x-dev ``` -## Track sent messages +## Track sent messages ```php create('influxdb://127.0.0.1:8086?db=foo'); $statsStorage->pushSentMessageStats(new SentMessageStats( (int) (microtime(true) * 1000), // timestamp - 'queue_name', // queue + 'queue_name', // queue 'aMessageId', 'aCorrelationId', [], // headers @@ -76,7 +76,7 @@ $context->createProducer()->send($queue, $message); $statsStorage = (new GenericStatsStorageFactory())->create('influxdb://127.0.0.1:8086?db=foo'); $statsStorage->pushSentMessageStats(new SentMessageStats( (int) (microtime(true) * 1000), - $queue->getQueueName(), + $queue->getQueueName(), $message->getMessageId(), $message->getCorrelationId(), $message->getHeaders()[], @@ -99,7 +99,7 @@ $statsStorage = (new GenericStatsStorageFactory())->create('influxdb://127.0.0.1 $statsStorage->pushConsumedMessageStats(new ConsumedMessageStats( 'consumerId', (int) (microtime(true) * 1000), // now - $receivedAt, + $receivedAt, 'aQueue', 'aMessageId', 'aCorrelationId', @@ -127,16 +127,16 @@ $consumer = $context->createConsumer($queue); $consumerId = uniqid('consumer-id', true); // we suggest using UUID here if ($message = $consumer->receiveNoWait()) { $receivedAt = (int) (microtime(true) * 1000); - + // heavy processing here. - + $consumer->acknowledge($message); - + $statsStorage = (new GenericStatsStorageFactory())->create('influxdb://127.0.0.1:8086?db=foo'); $statsStorage->pushConsumedMessageStats(new ConsumedMessageStats( $consumerId, (int) (microtime(true) * 1000), // now - $receivedAt, + $receivedAt, $queue->getQueueName(), $message->getMessageId(), $message->getCorrelationId(), @@ -151,7 +151,7 @@ if ($message = $consumer->receiveNoWait()) { ## Track consumer metrics Consumers are long running processes. It vital to know how many of them are running right now, how they perform, how much memory do they use and so. -This example shows how you can send such metrics. +This example shows how you can send such metrics. Call this code from time to time between processing messages. ```php @@ -165,13 +165,13 @@ $statsStorage = (new GenericStatsStorageFactory())->create('influxdb://127.0.0.1 $statsStorage->pushConsumerStats(new ConsumerStats( 'consumerId', (int) (microtime(true) * 1000), // now - $startedAt, + $startedAt, null, // finished at - true, // is started? + true, // is started? false, // is finished? false, // is failed ['foo'], // consume from queues - 123, // received messages + 123, // received messages 120, // acknowledged messages 1, // rejected messages 1, // requeued messages @@ -182,7 +182,7 @@ $statsStorage->pushConsumerStats(new ConsumerStats( ## Consumption extension -There is an extension `ConsumerMonitoringExtension` for Enqueue [QueueConsumer](quick_tour.md#consumption). +There is an extension `ConsumerMonitoringExtension` for Enqueue [QueueConsumer](quick_tour.md#consumption). It could collect consumed messages and consumer stats for you. ```php @@ -236,8 +236,16 @@ There are available options: * 'measurementSentMessages' => 'sent-messages', * 'measurementConsumedMessages' => 'consumed-messages', * 'measurementConsumers' => 'consumers', +* 'client' => null, +* 'retentionPolicy' => null, ``` +You can pass InfluxDB\Client instance in `client` option. Otherwise, it will be created on first use according to other +options. + +If your InfluxDB\Client uses driver that implements InfluxDB\Driver\QueryDriverInterface, then database will be +automatically created for you if it doesn't exist. Default InfluxDB\Client will also do that. + ## Datadog storage Install additional packages: @@ -256,7 +264,7 @@ $statsStorage = (new GenericStatsStorageFactory())->create('datadog://127.0.0.1: For best experience please adjust units and types in metric summary. Example dashboard: - + ![Datadog monitoring](images/datadog_monitoring.png) @@ -311,7 +319,7 @@ There are available options: ## Symfony App -You have to register some services in order to incorporate monitoring facilities into your Symfony application. +You have to register some services in order to incorporate monitoring facilities into your Symfony application. ```yaml # config/packages/enqueue.yaml @@ -325,11 +333,11 @@ enqueue: transport: 'amqp://guest:guest@foo:5672/%2f' monitoring: 'wamp://127.0.0.1:9090?topic=stats' client: ~ - + datadog: transport: 'amqp://guest:guest@foo:5672/%2f' monitoring: 'datadog://127.0.0.1:8125?batched=false' client: ~ ``` -[back to index](index.md) +[back to index](index.md) diff --git a/docs/transport/amqp_bunny.md b/docs/transport/amqp_bunny.md index a87245b36..cf0361f80 100644 --- a/docs/transport/amqp_bunny.md +++ b/docs/transport/amqp_bunny.md @@ -15,7 +15,7 @@ Build on top of [bunny lib](https://github.com/jakubkulhan/bunny). * [Installation](#installation) * [Create context](#create-context) * [Declare topic](#declare-topic) -* [Declare queue](#decalre-queue) +* [Declare queue](#declare-queue) * [Bind queue to topic](#bind-queue-to-topic) * [Send message to topic](#send-message-to-topic) * [Send message to queue](#send-message-to-queue) @@ -267,4 +267,4 @@ $queue = $context->createQueue('aQueue'); $context->purgeQueue($queue); ``` -[back to index](../index.md) \ No newline at end of file +[back to index](../index.md) diff --git a/docs/transport/kafka.md b/docs/transport/kafka.md index 78ba998dc..4fefb6914 100644 --- a/docs/transport/kafka.md +++ b/docs/transport/kafka.md @@ -41,7 +41,7 @@ $connectionFactory = new RdKafkaConnectionFactory('kafka:'); $connectionFactory = new RdKafkaConnectionFactory([]); // connect to Kafka broker at example.com:1000 plus custom options -$connectionFactory = new RdKafkaConnectionFactory([ +$connectionFactory = new RdKafkaConnectionFactory([ 'global' => [ 'group.id' => uniqid('', true), 'metadata.broker.list' => 'example.com:1000', @@ -54,11 +54,11 @@ $connectionFactory = new RdKafkaConnectionFactory([ $context = $connectionFactory->createContext(); -// if you have enqueue/enqueue library installed you can use a factory to build context from DSN +// if you have enqueue/enqueue library installed you can use a factory to build context from DSN $context = (new \Enqueue\ConnectionFactoryFactory())->create('kafka:')->createContext(); ``` -## Send message to topic +## Send message to topic ```php createTopic('foo'); $context->createProducer()->send($fooTopic, $message); ``` -## Send message to queue +## Send message to queue ```php createQueue('foo'); $consumer = $context->createConsumer($fooQueue); -// Enable async commit to gain better performance. +// Enable async commit to gain better performance (true by default since version 0.9.9). //$consumer->setCommitAsync(true); $message = $consumer->receive(); @@ -108,7 +108,7 @@ $consumer->acknowledge($message); ## Serialize message By default the transport serializes messages to json format but you might want to use another format such as [Apache Avro](https://avro.apache.org/docs/1.2.0/). -For that you have to implement Serializer interface and set it to the context, producer or consumer. +For that you have to implement Serializer interface and set it to the context, producer or consumer. If a serializer set to context it will be injected to all consumers and producers created by the context. ```php @@ -119,7 +119,7 @@ use Enqueue\RdKafka\RdKafkaMessage; class FooSerializer implements Serializer { public function toMessage($string) {} - + public function toString(RdKafkaMessage $message) {} } @@ -145,4 +145,44 @@ $consumer->setOffset(123); $message = $consumer->receive(2000); ``` -[back to index](index.md) \ No newline at end of file +## Usage with Symfony bundle + +Set your enqueue to use rdkafka as your transport + +```yaml +# app/config/config.yml + +enqueue: + default: + transport: "rdkafka:" + client: ~ +``` + +You can also you extended configuration to pass additional options, if you don't want to pass them via DSN string or +need to pass specific options. Since rdkafka uses librdkafka (being basically a wrapper around it) most configuration +options are identical to those found at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md. + +```yaml +# app/config/config.yml + +enqueue: + default: + transport: + dsn: "rdkafka://" + global: + ### Make sure this is unique for each application / consumer group and does not change + ### Otherwise, Kafka won't be able to track your last offset and will always start according to + ### `auto.offset.reset` setting. + ### See Kafka documentation regarding `group.id` property if you want to know more + group.id: 'foo-app' + metadata.broker.list: 'example.com:1000' + topic: + auto.offset.reset: beginning + ### Commit async is true by default since version 0.9.9. + ### It is suggested to set it to true in earlier versions since otherwise consumers become extremely slow, + ### waiting for offset to be stored on Kafka before continuing. + commit_async: true + client: ~ +``` + +[back to index](index.md) diff --git a/pkg/amqp-bunny/AmqpContext.php b/pkg/amqp-bunny/AmqpContext.php index 9338e12b0..f57554335 100644 --- a/pkg/amqp-bunny/AmqpContext.php +++ b/pkg/amqp-bunny/AmqpContext.php @@ -312,6 +312,7 @@ public function getBunnyChannel(): Channel public function convertMessage(BunnyMessage $bunnyMessage): InteropAmqpMessage { $headers = $bunnyMessage->headers; + $headers = $this->convertHeadersFromBunnyNotation($headers); $properties = []; if (isset($headers['application_headers'])) { @@ -333,4 +334,106 @@ public function convertMessage(BunnyMessage $bunnyMessage): InteropAmqpMessage return $message; } + + /** @internal It must be used here and in the producer only */ + public function convertHeadersToBunnyNotation(array $headers): array + { + if (isset($headers['content_type'])) { + $headers['content-type'] = $headers['content_type']; + unset($headers['content_type']); + } + + if (isset($headers['content_encoding'])) { + $headers['content-encoding'] = $headers['content_encoding']; + unset($headers['content_encoding']); + } + + if (isset($headers['delivery_mode'])) { + $headers['delivery-mode'] = $headers['delivery_mode']; + unset($headers['delivery_mode']); + } + + if (isset($headers['correlation_id'])) { + $headers['correlation-id'] = $headers['correlation_id']; + unset($headers['correlation_id']); + } + + if (isset($headers['reply_to'])) { + $headers['reply-to'] = $headers['reply_to']; + unset($headers['reply_to']); + } + + if (isset($headers['message_id'])) { + $headers['message-id'] = $headers['message_id']; + unset($headers['message_id']); + } + + if (isset($headers['user_id'])) { + $headers['user-id'] = $headers['user_id']; + unset($headers['user_id']); + } + + if (isset($headers['app_id'])) { + $headers['app-id'] = $headers['app_id']; + unset($headers['app_id']); + } + + if (isset($headers['cluster_id'])) { + $headers['cluster-id'] = $headers['cluster_id']; + unset($headers['cluster_id']); + } + + return $headers; + } + + /** @internal It must be used here and in the consumer only */ + public function convertHeadersFromBunnyNotation(array $bunnyHeaders): array + { + if (isset($bunnyHeaders['content-type'])) { + $bunnyHeaders['content_type'] = $bunnyHeaders['content-type']; + unset($bunnyHeaders['content-type']); + } + + if (isset($bunnyHeaders['content-encoding'])) { + $bunnyHeaders['content_encoding'] = $bunnyHeaders['content-encoding']; + unset($bunnyHeaders['content-encoding']); + } + + if (isset($bunnyHeaders['delivery-mode'])) { + $bunnyHeaders['delivery_mode'] = $bunnyHeaders['delivery-mode']; + unset($bunnyHeaders['delivery-mode']); + } + + if (isset($bunnyHeaders['correlation-id'])) { + $bunnyHeaders['correlation_id'] = $bunnyHeaders['correlation-id']; + unset($bunnyHeaders['correlation-id']); + } + + if (isset($bunnyHeaders['reply-to'])) { + $bunnyHeaders['reply_to'] = $bunnyHeaders['reply-to']; + unset($bunnyHeaders['reply-to']); + } + + if (isset($bunnyHeaders['message-id'])) { + $bunnyHeaders['message_id'] = $bunnyHeaders['message-id']; + unset($bunnyHeaders['message-id']); + } + + if (isset($bunnyHeaders['user-id'])) { + $bunnyHeaders['user_id'] = $bunnyHeaders['user-id']; + unset($bunnyHeaders['user-id']); + } + + if (isset($bunnyHeaders['app-id'])) { + $bunnyHeaders['app_id'] = $bunnyHeaders['app-id']; + unset($bunnyHeaders['app-id']); + } + + if (isset($bunnyHeaders['cluster-id'])) { + $bunnyHeaders['cluster_id'] = $bunnyHeaders['cluster-id']; + unset($bunnyHeaders['cluster-id']); + } + + return $bunnyHeaders; + } } diff --git a/pkg/amqp-bunny/AmqpProducer.php b/pkg/amqp-bunny/AmqpProducer.php index e052cd312..76892ebeb 100644 --- a/pkg/amqp-bunny/AmqpProducer.php +++ b/pkg/amqp-bunny/AmqpProducer.php @@ -136,6 +136,7 @@ private function doSend(InteropAmqpDestination $destination, InteropAmqpMessage } $amqpProperties = $message->getHeaders(); + $amqpProperties = $this->context->convertHeadersToBunnyNotation($amqpProperties); if (array_key_exists('timestamp', $amqpProperties) && null !== $amqpProperties['timestamp']) { $amqpProperties['timestamp'] = \DateTime::createFromFormat('U', (string) $amqpProperties['timestamp']); diff --git a/pkg/amqp-bunny/Tests/AmqpProducerTest.php b/pkg/amqp-bunny/Tests/AmqpProducerTest.php index 6a476790b..df33598b2 100644 --- a/pkg/amqp-bunny/Tests/AmqpProducerTest.php +++ b/pkg/amqp-bunny/Tests/AmqpProducerTest.php @@ -134,11 +134,39 @@ public function testShouldSetMessageHeaders() $channel ->expects($this->once()) ->method('publish') - ->with($this->anything(), ['content_type' => 'text/plain']) + ->with($this->anything(), ['misc' => 'text/plain']) ; $producer = new AmqpProducer($channel, $this->createContextMock()); - $producer->send(new AmqpTopic('name'), new AmqpMessage('body', [], ['content_type' => 'text/plain'])); + $producer->send(new AmqpTopic('name'), new AmqpMessage('body', [], ['misc' => 'text/plain'])); + } + + public function testShouldConvertStandartHeadersToBunnyFormat() + { + $channel = $this->createBunnyChannelMock(); + $expectedHeaders = [ + 'content-encoding' => 'utf8', + 'content-type' => 'text/plain', + 'message-id' => 'id', + 'correlation-id' => 'correlation', + 'reply-to' => 'reply', + 'delivery-mode' => 2, + ]; + $channel + ->expects($this->once()) + ->method('publish') + ->with($this->anything(), $expectedHeaders); + + $producer = new AmqpProducer($channel, $this->createContextMock()); + $message = new AmqpMessage('body', []); + $message->setMessageId('id'); + $message->setReplyTo('reply'); + $message->setDeliveryMode(2); + $message->setContentType('text/plain'); + $message->setContentEncoding('utf8'); + $message->setCorrelationId('correlation'); + + $producer->send(new AmqpTopic('name'), $message); } public function testShouldSetMessageProperties() @@ -200,7 +228,7 @@ private function createBunnyChannelMock() */ private function createContextMock() { - return $this->createMock(AmqpContext::class); + return $this->createPartialMock(AmqpContext::class, []); } /** diff --git a/pkg/dbal/DbalConsumer.php b/pkg/dbal/DbalConsumer.php index e95a8f513..9d99d5a58 100644 --- a/pkg/dbal/DbalConsumer.php +++ b/pkg/dbal/DbalConsumer.php @@ -100,16 +100,14 @@ public function reject(Message $message, bool $requeue = false): void { InvalidMessageException::assertMessageInstanceOf($message, DbalMessage::class); + $this->acknowledge($message); + if ($requeue) { $message = clone $message; $message->setRedelivered(false); $this->getContext()->createProducer()->send($this->queue, $message); - - return; } - - $this->deleteMessage($message->getDeliveryId()); } protected function getContext(): DbalContext diff --git a/pkg/dbal/Tests/DbalConsumerTest.php b/pkg/dbal/Tests/DbalConsumerTest.php index 85130893d..c042c5c86 100644 --- a/pkg/dbal/Tests/DbalConsumerTest.php +++ b/pkg/dbal/Tests/DbalConsumerTest.php @@ -169,6 +169,7 @@ public function testRejectShouldReSendMessageToSameQueueOnRequeue() $message = new DbalMessage(); $message->setBody('theBody'); + $message->setDeliveryId(__METHOD__); $producerMock = $this->createProducerMock(); $producerMock diff --git a/pkg/dbal/Tests/Functional/DbalConsumerTest.php b/pkg/dbal/Tests/Functional/DbalConsumerTest.php index 1f8757e15..686f87d16 100644 --- a/pkg/dbal/Tests/Functional/DbalConsumerTest.php +++ b/pkg/dbal/Tests/Functional/DbalConsumerTest.php @@ -144,6 +144,31 @@ public function testShouldDeleteExpiredMessage() $this->assertSame(0, $this->getQuerySize()); } + public function testShouldRemoveOriginalMessageThatHaveBeenRejectedWithRequeue() + { + $context = $this->context; + $queue = $context->createQueue(__METHOD__); + + $consumer = $context->createConsumer($queue); + + // guard + $this->assertSame(0, $this->getQuerySize()); + + $producer = $context->createProducer(); + + /** @var DbalMessage $message */ + $message = $context->createMessage(__CLASS__); + $producer->send($queue, $message); + + $this->assertSame(1, $this->getQuerySize()); + + $message = $consumer->receive(100); // 100ms + + $this->assertInstanceOf(DbalMessage::class, $message); + $consumer->reject($message, true); + $this->assertSame(1, $this->getQuerySize()); + } + private function getQuerySize(): int { return (int) $this->context->getDbalConnection() diff --git a/pkg/enqueue/Consumption/QueueConsumer.php b/pkg/enqueue/Consumption/QueueConsumer.php index 30d438dcd..6cb90290e 100644 --- a/pkg/enqueue/Consumption/QueueConsumer.php +++ b/pkg/enqueue/Consumption/QueueConsumer.php @@ -43,7 +43,7 @@ final class QueueConsumer implements QueueConsumerInterface private $boundProcessors; /** - * @var int|float in milliseconds + * @var int in milliseconds */ private $receiveTimeout; @@ -59,7 +59,7 @@ final class QueueConsumer implements QueueConsumerInterface /** * @param BoundProcessor[] $boundProcessors - * @param int|float $receiveTimeout the time in milliseconds queue consumer waits for a message (10 ms by default) + * @param int $receiveTimeout the time in milliseconds queue consumer waits for a message (10000 ms by default) */ public function __construct( InteropContext $interopContext, diff --git a/pkg/enqueue/Consumption/Result.php b/pkg/enqueue/Consumption/Result.php index b94b7447c..c5659bc92 100644 --- a/pkg/enqueue/Consumption/Result.php +++ b/pkg/enqueue/Consumption/Result.php @@ -13,12 +13,12 @@ class Result const ACK = Processor::ACK; /** - * @see Processor::ACK for more details + * @see Processor::REJECT for more details */ const REJECT = Processor::REJECT; /** - * @see Processor::ACK for more details + * @see Processor::REQUEUE for more details */ const REQUEUE = Processor::REQUEUE; diff --git a/pkg/monitoring/InfluxDbStorage.php b/pkg/monitoring/InfluxDbStorage.php index d1e8e7c00..401d79ddb 100644 --- a/pkg/monitoring/InfluxDbStorage.php +++ b/pkg/monitoring/InfluxDbStorage.php @@ -6,6 +6,8 @@ use Enqueue\Dsn\Dsn; use InfluxDB\Client; use InfluxDB\Database; +use InfluxDB\Driver\QueryDriverInterface; +use InfluxDB\Exception as InfluxDBException; use InfluxDB\Point; class InfluxDbStorage implements StatsStorage @@ -38,6 +40,8 @@ class InfluxDbStorage implements StatsStorage * 'measurementSentMessages' => 'sent-messages', * 'measurementConsumedMessages' => 'consumed-messages', * 'measurementConsumers' => 'consumers', + * 'client' => null, # Client instance. Null by default. + * 'retentionPolicy' => null, * ] * * or @@ -55,10 +59,17 @@ public function __construct($config = 'influxdb:') if (empty($config)) { $config = []; } elseif (is_string($config)) { - $config = $this->parseDsn($config); + $config = self::parseDsn($config); } elseif (is_array($config)) { - $config = empty($config['dsn']) ? $config : $this->parseDsn($config['dsn']); + $config = empty($config['dsn']) ? $config : self::parseDsn($config['dsn']); } elseif ($config instanceof Client) { + // Passing Client instead of array config is deprecated because it prevents setting any configuration values + // and causes library to use defaults. + @trigger_error( + sprintf('Passing %s as %s argument is deprecated. Pass it as "client" array property or use createWithClient instead', + Client::class, + __METHOD__ + ), E_USER_DEPRECATED); $this->client = $config; $config = []; } else { @@ -74,11 +85,41 @@ public function __construct($config = 'influxdb:') 'measurementSentMessages' => 'sent-messages', 'measurementConsumedMessages' => 'consumed-messages', 'measurementConsumers' => 'consumers', + 'client' => null, + 'retentionPolicy' => null, ], $config); + if (null !== $config['client']) { + if (!$config['client'] instanceof Client) { + throw new \InvalidArgumentException(sprintf( + '%s configuration property is expected to be an instance of %s class. %s was passed instead.', + 'client', + Client::class, + gettype($config['client']) + )); + } + $this->client = $config['client']; + } + $this->config = $config; } + /** + * @param Client $client + * @param string $config + * + * @return InfluxDbStorage + */ + public static function createWithClient(Client $client, $config = 'influxdb:'): self + { + if (is_string($config)) { + $config = self::parseDsn($config); + } + $config['client'] = $client; + + return new static($config); + } + public function pushConsumerStats(ConsumerStats $stats): void { $points = []; @@ -109,7 +150,7 @@ public function pushConsumerStats(ConsumerStats $stats): void $points[] = new Point($this->config['measurementConsumers'], null, $tags, $values, $stats->getTimestampMs()); } - $this->getDb()->writePoints($points, Database::PRECISION_MILLISECONDS); + $this->doWrite($points); } public function pushConsumedMessageStats(ConsumedMessageStats $stats): void @@ -135,7 +176,7 @@ public function pushConsumedMessageStats(ConsumedMessageStats $stats): void new Point($this->config['measurementConsumedMessages'], $runtime, $tags, $values, $stats->getTimestampMs()), ]; - $this->getDb()->writePoints($points, Database::PRECISION_MILLISECONDS); + $this->doWrite($points); } public function pushSentMessageStats(SentMessageStats $stats): void @@ -158,29 +199,47 @@ public function pushSentMessageStats(SentMessageStats $stats): void new Point($this->config['measurementSentMessages'], 1, $tags, [], $stats->getTimestampMs()), ]; - $this->getDb()->writePoints($points, Database::PRECISION_MILLISECONDS); + $this->doWrite($points); } - private function getDb(): Database + private function doWrite(array $points): void { - if (null === $this->database) { - if (null === $this->client) { - $this->client = new Client( - $this->config['host'], - $this->config['port'], - $this->config['user'], - $this->config['password'] - ); + if (null === $this->client) { + $this->client = new Client( + $this->config['host'], + $this->config['port'], + $this->config['user'], + $this->config['password'] + ); + } + + if ($this->client->getDriver() instanceof QueryDriverInterface) { + if (null === $this->database) { + $this->database = $this->client->selectDB($this->config['db']); + $this->database->create(); } - $this->database = $this->client->selectDB($this->config['db']); - $this->database->create(); + $this->database->writePoints($points, Database::PRECISION_MILLISECONDS, $this->config['retentionPolicy']); + } else { + // Code below mirrors what `writePoints` method of Database does. + try { + $parameters = [ + 'url' => sprintf('write?db=%s&precision=%s', $this->config['db'], Database::PRECISION_MILLISECONDS), + 'database' => $this->config['db'], + 'method' => 'post', + ]; + if (null !== $this->config['retentionPolicy']) { + $parameters['url'] .= sprintf('&rp=%s', $this->config['retentionPolicy']); + } + + $this->client->write($parameters, $points); + } catch (\Exception $e) { + throw new InfluxDBException($e->getMessage(), $e->getCode()); + } } - - return $this->database; } - private function parseDsn(string $dsn): array + private static function parseDsn(string $dsn): array { $dsn = Dsn::parseFirst($dsn); @@ -200,6 +259,7 @@ private function parseDsn(string $dsn): array 'measurementSentMessages' => $dsn->getString('measurementSentMessages'), 'measurementConsumedMessages' => $dsn->getString('measurementConsumedMessages'), 'measurementConsumers' => $dsn->getString('measurementConsumers'), + 'retentionPolicy' => $dsn->getString('retentionPolicy'), ]), function ($value) { return null !== $value; }); } } diff --git a/pkg/rdkafka/RdKafkaConsumer.php b/pkg/rdkafka/RdKafkaConsumer.php index 5e89f3d34..9522b32ee 100644 --- a/pkg/rdkafka/RdKafkaConsumer.php +++ b/pkg/rdkafka/RdKafkaConsumer.php @@ -51,7 +51,7 @@ public function __construct(KafkaConsumer $consumer, RdKafkaContext $context, Rd $this->context = $context; $this->topic = $topic; $this->subscribed = false; - $this->commitAsync = false; + $this->commitAsync = true; $this->setSerializer($serializer); } diff --git a/pkg/rdkafka/RdKafkaContext.php b/pkg/rdkafka/RdKafkaContext.php index cfe8e0eb1..22d4a8e47 100644 --- a/pkg/rdkafka/RdKafkaContext.php +++ b/pkg/rdkafka/RdKafkaContext.php @@ -186,6 +186,10 @@ private function getConf(): Conf $this->conf->setRebalanceCb($this->config['rebalance_cb']); } + if (isset($this->config['stats_cb'])) { + $this->conf->setStatsCb($this->config['stats_cb']); + } + $this->conf->setDefaultTopicConf($topicConf); }