amazon sqs(1)php producer
1. some basic information
price: 1 million 2$
send throughput:
receive throughput:
message latency: 500 ms
message size: 256 kb
batch size: 10 message
2. env set up
install composer
> curl -ss https://getcomposer.org/installer | php
install the latest package
> php composer.phar require aws/aws-sdk-php
this will install all the dependencies
> php composer.phar install
here is my composer.json which will identify the dependencies.
{
require: {
aws/aws-sdk-php: 3.0.6,
predis/predis: 1.0.1
}
}
and command below will do the magic, something maven or ivy
>php composer.phar install
some import classes will be as follow,
sqsconnector.inc
client = sqsclient::factory(array(
'region' => 'us-east-1',
'version' => 'latest'
));
echo successfully connected to sqs\n;
} catch (exception $e) {
echo couldn't connected to sqs;
echo $e->getmessage();
}
}
}
a time track class which can provide us more detail about the time consuming during the process, timetracker.inc
name = $name;
$this->distribution = array();
$this->distributionprecision = $distributionprecision;
$this->start();
}
public function count() {
return $this->count;
}
public function start($time = null) {
return $this->starttime = ($time === null) ? microtime(true) : $time;
}
public function stop($count = 1, $time = null) {
$interval = (($time === null) ? microtime(true) : $time) - $this->starttime;
$this->totaltime += $interval;
$key = (string)round($interval, $this->distributionprecision);
if (!array_key_exists($key, $this->distribution)) $this->distribution[$key] = 0;
$this->distribution[$key]++;
$this->count += $count;
return $interval;
}
public function __tostring() {
ksort($this->distribution);
//return {$this->name}: . print_r($this->distribution, true) .
return {$this->name}: {$this->count}m . round($this->totaltime,2) . 's ' .
round($this->count/$this->totaltime,1) . m/s\n;
}
}
async send the message via php in sqssender.php
#!/usr/bin/php
= maxsize_request)
{
echo the size of message is larger than 256kb\n;
var_dump($job);
return;
}
$promise = $this->client->sendmessageasync(array(
'queueurl' => self::queueurl,
'messagebody' => $job,
));
$promise->wait();
$timer->stop();
print $timer;
}
function sendmessagebatch($jobs, $timer) {
$promise_array = array();
$id=1;
$message = '';
$msg_num = 0;
foreach($jobs as $job)
{
$maxsize_msg = (int)(maxsize_request/msgnum_request);
$tmp_msg = $message.$job;
if(strlen($job) >= $maxsize_msg)
{
$this->sendmessage($job);
}
else if(strlen($tmp_msg) >= $maxsize_msg)
{
$entries[] = array(
'id' => $id,
'messagebody' => $message
);
$id++;
$message = $job;
if($id > 10){
$promise = $this->client->sendmessagebatchasync(array(
'queueurl' => self::queueurl,
'entries' => $entries,
));
$entries = array();
$id = 1;
$promise_array['key' . count($timer)] = $promise;
/*if(count($promise_array) % 50 == 0){
promise\unwrap($promise_array); //unwrap the promise list and wait for all promise complete
$promise_array = array();
}*/
$timer->stop($msg_num);
$msg_num = 0;
$timer->start();
}
$msg_num++;
}
else
{
$message = $tmp_msg;
$msg_num++;
}
}
$entries[] = array(
'id' => $id,
'messagebody' => $message
);
$promise = $this->client->sendmessagebatchasync(array(
'queueurl' => self::queueurl,
'entries' => $entries,
));
$promise_array['key' . count($timer)] = $promise;
promise\unwrap($promise_array); //unwrap the promise list and wait for all promise complete
$promise_array = array();
$timer->stop($msg_num);
print $timer;
$timer->start();
}
}
/*$sender = new sqssender;
$promise = $sender->send(500);
if(count($promise) > 0){
promise\unwrap($promises);
}*/
?>
before we run the test, the amazon require us to configure the key here>
carl-mac:.aws carl$ pwd
/users/carl/.aws
carl-mac:.aws carl$ ls -l
total 8
-rw-r--r-- 1 carl staff 116 jul 6 17:00 credentials
carl-mac:.aws carl$
the content will be as follow:
[default]
aws_access_key_id = xxxxxxxx
aws_secret_access_key = xxxxxxxxxxxxx
references:
http://colby.id.au/benchmarking-sqs/
http://www.warski.org/blog/2014/06/benchmarking-sqs/
http://nsono.net/amazon-sqs-vs-rabbitmq/
kafka
http://engineering.linkedin.com/kafka/benchmarking-apache-kafka-2-million-writes-second-three-cheap-machines
sqs
http://docs.aws.amazon.com/aws-sdk-php/guide/latest/service-sqs.html
http://aws.amazon.com/documentation/sqs/
http://docs.aws.amazon.com/aws-sdk-php/v3/guide/guide/promises.html
http://docs.aws.amazon.com/aws-sdk-php/v3/api/index.html
scala queue
https://github.com/adamw/mqperf/tree/master/src/main/scala/com/softwaremill/mqperf/mq
sqs auto scaling
https://aws.amazon.com/articles/amazon-sqs/1464