module app;
import vibe.core.concurrency : async, Future;
import vibe.core.core;
import vibe.core.log;
import vibe.db.redis.redis;
import std.exception : assertThrown;
import std.format;
import std.getopt;
import std.range; //chunks, iota;
int main (string[] args)
{
auto num_records = 1_000_000UL;
getopt(args, "records|n", &num_records);
RedisClient redis;
try
redis = new RedisClient();
catch (Exception)
{
logInfo("Failed to connect to local Redis server.");
return 1;
}
auto processed_records = 0UL;
int ret = 0;
runTask({
auto num_chunks = 32;
auto max_items_x_chunk = num_records / num_chunks;
auto chunk_records = chunks(iota(0, num_records), max_items_x_chunk);
logInfo("Number of chunks: %d", chunk_records.length);
logInfo("Maximum number of items per chunk: %d",
chunk_records[0].length);
Future!(ulong)[] workers;
foreach (chunk; chunk_records)
{
workers ~= async({
try
{
foreach (item; chunk)
{
redis.getDatabase(0).set(
"Key%d".format(item), "Value%d".format(item));
}
}
catch (Throwable th)
{
logError("Set operation failed: %s", th.msg);
logDiagnostic("Full error: %s", th);
ret = 1;
}
return chunk.length;
});
}
foreach (worker; workers)
processed_records += worker.getResult();
exitEventLoop();
});
runEventLoop();
logInfo("Processed records: %d", processed_records);
return ret;
}
Running that version, the elapsed time and the context switches were considerable reduced
$ /usr/bin/time -v ./bulktest
[main(k1sG) INF] Number of chunks: 32
[main(k1sG) INF] Maximum number of items per chunk: 31250
[main(----) INF] Processed records: 1000000
Command being timed: "./bulktest"
User time (seconds): 8.42
System time (seconds): 6.23
Percent of CPU this job got: 99%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:14.67
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 12400
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 832
Voluntary context switches: 37
Involuntary context switches: 1190
Swaps: 0
File system inputs: 0
File system outputs: 0
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0