From be2ed08d6712a241c48dd619d2151dd0bfaea7f5 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 22 May 2014 21:26:08 -0700 Subject: [PATCH 1/4] Adds map_shm to open and mmap a region of POSIX shared memory. The map_shm function takes a name variable instead of an fd or file. First, map_shm creates and/or attaches to a region of shared memory, which returns an fd. It then calls the usual mmap function with the fd. Finally, it adds an unlink function to the returned buffer. The unlink function marks the shared memory region for deletion, once all processes have detached using unmap (which can be called explicitly or implicitly during destruction as usual). All other functionality of the returned buffer remains the same. --- index.js | 1 + mmap.cpp | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/index.js b/index.js index a5fc002..09da417 100644 --- a/index.js +++ b/index.js @@ -21,5 +21,6 @@ function mmap_wrapper(size, protection, flags, fd, offset) { }; for(var k in mmap) mmap_wrapper[k] = mmap[k]; mmap_wrapper.map = mmap_wrapper; +mmap_wrapper.map_shm = mmap.map_shm; module.exports = mmap_wrapper; diff --git a/mmap.cpp b/mmap.cpp index 94f3994..e911478 100644 --- a/mmap.cpp +++ b/mmap.cpp @@ -8,8 +8,12 @@ static v8::Persistent length_symbol; static v8::Persistent unmap_symbol; static v8::Persistent sync_symbol; +static v8::Persistent unlink_symbol; +static v8::Persistent name_symbol; static v8::Persistent buffer_symbol; +static v8::Persistent map_symbol; + static void Map_finalise(char *data, void*hint) { munmap(data, (size_t)hint); @@ -112,6 +116,52 @@ v8::Handle Map(const v8::Arguments& args) return scope.Close(actualBuffer); } +v8::Handle Unlink(const v8::Arguments& args) +{ + v8::HandleScope scope; + v8::String::Utf8Value name(args.This()->GetHiddenValue(name_symbol)->ToString()); + if (shm_unlink(*name) < 0) + { + return v8::ThrowException(node::ErrnoException(errno, "unlink", "")); + } + return v8::True(); +} + +v8::Handle MapShm(const v8::Arguments& args) +{ + v8::HandleScope scope; + + if (args.Length() <= 3) + { + return v8::ThrowException( + v8::Exception::Error( + v8::String::New("map_shm() takes 4 arguments: size, protection, flags, name and offset."))); + } + + const size_t size = args[0]->ToInteger()->Value(); + const v8::String::Utf8Value name(args[3]->ToString()); + const int fd = shm_open(*name, O_CREAT | O_RDWR | O_SYNC, 0666); + if (fd < 0) + { + return v8::ThrowException(node::ErrnoException(errno, "map_shm", "")); + } + ftruncate(fd, size); + + v8::Handle *mapArgs = (v8::Handle *)calloc(sizeof(v8::Handle), args.Length()); + for (int i = 0; i < args.Length(); i++) + mapArgs[i] = args[i]; + mapArgs[3] = v8::Integer::New(fd); + + v8::Handle global = v8::Context::GetCurrent()->Global(); + v8::Handle actualBuffer = map_symbol->Call(global, args.Length(), mapArgs)->ToObject(); + free(mapArgs); + close(fd); + + actualBuffer->ToObject()->Set(unlink_symbol, v8::FunctionTemplate::New(Unlink)->GetFunction()); + actualBuffer->SetHiddenValue(name_symbol, v8::String::New(*name)); + + return scope.Close(actualBuffer); +} static void RegisterModule(v8::Handle target) { @@ -120,8 +170,12 @@ static void RegisterModule(v8::Handle target) length_symbol = NODE_PSYMBOL("length"); sync_symbol = NODE_PSYMBOL("sync"); unmap_symbol = NODE_PSYMBOL("unmap"); + unlink_symbol = NODE_PSYMBOL("unlink"); + name_symbol = NODE_PSYMBOL("name"); buffer_symbol = NODE_PSYMBOL("Buffer"); + map_symbol = v8::Persistent::New(v8::FunctionTemplate::New(Map)->GetFunction()); + const v8::PropertyAttribute attribs = (v8::PropertyAttribute) (v8::ReadOnly | v8::DontDelete); target->Set(v8::String::New("PROT_READ"), v8::Integer::New(PROT_READ), attribs); @@ -136,6 +190,7 @@ static void RegisterModule(v8::Handle target) target->Set(v8::String::New("MS_INVALIDATE"), v8::Integer::New(MS_INVALIDATE), attribs); target->Set(v8::String::NewSymbol("map"), v8::FunctionTemplate::New(Map)->GetFunction(), attribs); + target->Set(v8::String::NewSymbol("map_shm"), v8::FunctionTemplate::New(MapShm)->GetFunction(), attribs); } NODE_MODULE(mmap, RegisterModule); From ed86adb16032faa7923fcdf06a8dfa71251380b5 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 22 May 2014 21:46:55 -0700 Subject: [PATCH 2/4] Create a map_shm example using the existing bloom example. --- example/bloomwithshm.js | 82 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 example/bloomwithshm.js diff --git a/example/bloomwithshm.js b/example/bloomwithshm.js new file mode 100644 index 0000000..fe33447 --- /dev/null +++ b/example/bloomwithshm.js @@ -0,0 +1,82 @@ +fs = require("fs"), +mmap = require("../index.js"); // obv. use require("mmap") if you lift +HASH_FUNCTIONS=7; // how many hashes do you want? + +size = 1024*1024; // 1mb +buffer = mmap.map_shm(size, mmap.PROT_READ|mmap.PROT_WRITE, mmap.MAP_SHARED, "/tmp/test_bloom"); +add(buffer, get_offsets("testing", buffer.length)); +console.log("check: ", check(buffer, get_offsets("testing", buffer.length))); + + +/* basic bloom filter tricks */ +function get_offsets(string, buffer_size) { + var a = fnv_1a(string); + var b = fnv_1a_b(a); + var r = []; + var i = -1; + var x = a % buffer_size; + while (++i < HASH_FUNCTIONS) { + r.push(x < 0 ? (x + buffer_size) : x); + x = (x + b) % buffer_size; + } + return r; +} +function add(buffer, offsets) { + var i = -1; + var w = 0; + while (++i < HASH_FUNCTIONS) w += !!(buffer[ offsets[i] ]++); + if (HASH_FUNCTIONS == w) { + while (i-->0) --buffer[offsets[i]]; + return true; + } + return false; +} +function check(buffer, offsets) { + var i = -1; + while (++i < HASH_FUNCTIONS) + if (!buffer[offsets[i]]) + return false; + return true; +} + +/* fowler-noll-vo (fnv) hashing */ +function fnv_1a(v) { + var n = v.length, a = 2166136261, c, d, i = -1; + while (++i < n) { + c = v.charCodeAt(i); + if (d = c & 0xff000000) { + a ^= d >> 24; + a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); + } + if (d = c & 0xff0000) { + a ^= d >> 16; + a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); + } + if (d = c & 0xff00) { + a ^= d >> 8; + a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); + } + a ^= c & 0xff; + a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); + } + /* from http://home.comcast.net/~bretm/hash/6.html */ + a += a << 13; + a ^= a >> 7; + a += a << 3; + a ^= a >> 17; + a += a << 5; + return a & 0xffffffff; +} + +/* one additional iteration of fnv, given a hash */ +function fnv_1a_b(a) { + a += (a << 1) + (a << 4) + (a << 7) + (a << 8) + (a << 24); + a += a << 13; + a ^= a >> 7; + a += a << 3; + a ^= a >> 17; + a += a << 5; + return a & 0xffffffff; +} + +buffer.unlink(); From c123ee379536053c13070cddb16cd0300bd94ce2 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 22 May 2014 21:45:28 -0700 Subject: [PATCH 3/4] Update README for map_shm. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index c9a9835..e825176 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,15 @@ You can also [msync()](http://pubs.opengroup.org/onlinepubs/9699919799/functions For compatibility, mmap.map() is an alias for mmap() +If you wish to use [shared memory](http://en.wikipedia.org/wiki/Shared_memory#In_software) instead of a file, you can do so by calling: `mmap.map_shm(n_bytes, protection, flags, name, offset)` method. All arguments are the same as mmap() except: + + + + + + +
nameName of the POSIX shared memory region. mmap.map_shm() will create and/or attach to the region with read and write permissions, and truncate or expand the region to size bytes.
+ ## See Also * POSIX 1003.1 [mmap](http://pubs.opengroup.org/onlinepubs/9699919799/functions/mmap.html) and [msync](http://pubs.opengroup.org/onlinepubs/9699919799/functions/msync.html) From a97f43c440a29e76a8c6dab417517015ab338195 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 22 May 2014 21:47:01 -0700 Subject: [PATCH 4/4] Bump version number to represent the addition of a new feature. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9eb5579..f27a50d 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "mmap", "description": "mmap() fds into buffers", "license": "BSD", - "version": "2.0.2", + "version": "2.1.0", "main": "./index.js", "repository": { "type": "git", "url": "https://github.com/geocar/mmap.git" }, "scripts": {