ref: aee91c4ed0277e343a62cd124ee7175f54ffefb3
author: Linux User <[email protected]>
date: Sat Jul 22 12:17:55 EDT 2023
wip - main, config, ctlfs
--- /dev/null
+++ b/.gitignore
@@ -1,0 +1,2 @@
+*.dis
+*.sbl
--- /dev/null
+++ b/LICENSE
@@ -1,0 +1,27 @@
+Copyright © 2023, kitzman @ disroot.org
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of {{ project }} nor the names of its contributors+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
--- /dev/null
+++ b/README
@@ -1,0 +1,32 @@
+dddb (Distributed DisVM Database) is a WIP RDBMS, aimed
+to run on DisVM, hosted or native.
+
+TODO Components and functionalities
+
+* dddbctl fs - authenticated 9p fs to control, request, and query
+ operations related to the database instance; it
+ is the main driver for grid database
+* dddbclient fs - the tcp port 'dddb' is used such that clients
+ can have a fs in which queries can be opened;
+ it will support ANSI SQL
+* db registry - each instance will, on startup, connect to
+ other instances defined in an ndb configuration
+ file, register itself in that instance, and
+ will be able to use it for: duplicating data,
+ sharding data, and mount further instances;
+ (i.e if a cluster is behind a firewall, and
+ only one instance exposed, the hidden ones
+ can be mounted outside the fw)
+* data structures (1) - primitives, pages, sections, indices,
+ tables, rows
+* data structures (2) - B-trees, Bloom filters
+* storage operations - a mounted directory tree will be used
+ for RDRW operations; memfs will be used
+ for caching the data in RAM (performance?)
+* venti/vac support - venti + vac + memfs should suffice for
+ storing and persisting data (ramfossil-like);
+ this means scripts will clean and archive it
+* SQL parser - obviously
+* statement optimizer - obviously
+* wm/dddbmon - a nice admin monitor would be nice
+* dddbjdbc/dddbodbc - JDBC and ODBC drivers are mandatory
--- /dev/null
+++ b/appl/cmd/config.b
@@ -1,0 +1,118 @@
+include "bufio.m";
+include "attrdb.m";
+ attrdb: Attrdb;
+ Attr, Db, Dbptr, Dbentry: import attrdb;
+
+# Default configuration values
+DCFGPATH: con "/lib/ndb/dddbcfg";
+
+DADDR: con "tcp!*!dddbctl";
+DFSWRKS: con 10;
+
+# Attrdb constants
+KNAME: con "nodename";
+KSYSNAME: con "nodesysn";
+KADDR: con "addr";
+KSTORAGE: con "storage";
+KFSWRKS: con "readworkers";
+
+Config.open(nodename: string, path: string): Config
+{+ sys = load Sys Sys->PATH;
+
+ n: int;
+ sysname: string;
+ buf := array[Sys->NAMEMAX] of byte;
+
+ attrdb = load Attrdb Attrdb->PATH;
+ if(attrdb == nil)
+ error("Attrb not found");+ attrdb->init();
+
+ if(len path == 0)
+ path = DCFGPATH;
+
+ if(debug)
+ sys->fprint(stderr, "config: opening %s\n", path);
+ db := Db.open(path);
+
+ if(db == nil)
+ error(sys->sprint("ndb file %s could not be opened", path));+
+ sysname_fd := sys->open("#c/sysname", Sys->OREAD);+ if(sysname_fd == nil)
+ error(sys->sprint("#c/sysname could not be read"));+ n = sys->read(sysname_fd, buf, len buf);
+ if(n <= 0)
+ error(sys->sprint("could not read sysname"));+ sysname = string buf[0:n];
+
+ if(len nodename == 0) {+ nodename = sysname;
+ }
+
+ thiscfg: ref NodeConfig;
+ nodecfgs: list of NodeConfig;
+ entry: ref Dbentry;
+ dbptr: ref Dbptr;
+
+ (entry, dbptr) = db.find(dbptr, KNAME);
+ while(entry != nil) {+ nodecfg := NodeConfig.new(entry);
+
+ if(debug)
+ sys->fprint(stderr, "config: found node %s\n", nodecfg.name);
+ if(nodecfg.name == nodename) {+ thiscfg = ref nodecfg;
+ }
+ else
+ nodecfgs = nodecfg :: nodecfgs;
+
+ (entry, dbptr) = db.find(dbptr, KNAME);
+ }
+
+ if(thiscfg == nil)
+ error("could not find config for nodename");+
+ if(thiscfg.addr == "")
+ error("node lacks an address");+
+ if(thiscfg.storage == "")
+ error("node lacks a storage path");+
+ if(thiscfg.sysn != sysname)
+ error("node and system sysname do not match");+
+ return Config(
+ thiscfg.name, thiscfg.sysn, thiscfg.addr, # own configuration
+ thiscfg.storage, thiscfg.fswrks,
+ nodecfgs); # configured nodes
+}
+
+NodeConfig.new(entry: ref Dbentry): NodeConfig
+{+ name := entry.findfirst(KNAME);
+ sysname := entry.findfirst(KSYSNAME);
+ addr := entry.findfirst(KADDR);
+ storage := entry.findfirst(KSTORAGE);
+ fswrks_s := entry.findfirst(KFSWRKS);
+
+ fswrks := DFSWRKS;
+
+ if(len name == 0)
+ error("entry has no name");+ if(len sysname == 0)
+ error("entry has no sysname");+ if(len addr == 0)
+ addr = DADDR;
+ if(len fswrks_s != 0) {+ (fswrks_i, rm) := strm->toint(fswrks_s, 10);
+ if(rm != "")
+ error(sys->sprint("malformed fs workers count: %s", fswrks_s));+ fswrks = fswrks_i;
+ }
+
+ return NodeConfig(
+ name, sysname, addr, storage, # basic information
+ fswrks); # tunable options
+}
--- /dev/null
+++ b/appl/cmd/ctlfs.b
@@ -1,0 +1,222 @@
+include "dial.m";
+ dial: Dial;
+
+include "security.m";
+ auth: Auth;
+
+include "styx.m";
+ styx: Styx;
+ Tmsg, Rmsg: import Styx;
+
+include "styxservers.m";
+ styxservers: Styxservers;
+ Styxserver, Fid, Navigator,
+ Navop, Enotfound, Enotdir: import styxservers;
+
+# FS file index
+Qroot, Qctl, Qstats, Qmax: con iota;
+tab := array[] of {+ (Qroot, ".", Sys->DMDIR|8r555),
+ (Qctl, "ctl", 8r222),
+ (Qstats, "stats", 8r111),
+};
+
+# create ctlfs and the appropriate listeners
+init_ctlfs(cfg: Config, keyfile: string, algs: list of string)
+{+ dial = load Dial Dial->PATH;
+ auth = load Auth Auth->PATH;
+ styx = load Styx Styx->PATH;
+
+ if(dial == nil)
+ error("ctlfs: dial module not found");+ if(auth == nil)
+ error("ctlfs: auth module not found");+ if(styx == nil)
+ error("ctlfs: styx module not found");+
+ auth->init();
+
+ styx->init();
+ styxservers->init(styx);
+ styxservers->traceset(chatty);
+
+ # authinfo init
+ if(debug)
+ sys->fprint(stderr, "ctlfs: reading authinfo");
+ authinfo: ref Keyring->Authinfo;
+ if (doauth) {+ if (keyfile == nil)
+ keyfile = "/usr/" + user() + "/keyring/default";
+ authinfo = keyring->readauthinfo(keyfile);
+ if (authinfo == nil)
+ error(sys->sprint("ctlfs: cannot read %s: %r", keyfile));+ }
+
+ # announcing
+ if(debug)
+ sys->fprint(stderr, "ctlfs: announcing dddbctl");
+ addr := dial->netmkaddr(cfg.addr, "tcp", "dddbctl");
+ c := dial->announce(addr);
+ if(c == nil)
+ error(sys->sprint("ctlfs: cannot listen on %s\n", addr));+
+ # bootstrapping
+ if(debug)
+ sys->fprint
+ sys->unmount(nil, "/mnt/keys");
+
+ navch := chan of ref Navop;
+ spawn ctlfs_navigator(navch);
+
+ nav := Navigator.new(navch);
+ (tc, srv) := Styxserver.new(fildes(0), nav, big Qroot);
+
+ # listener entrypoint
+ listener(c, authinfo, algs);
+}
+
+# dddbctl listener loop
+ctlfs_listener(c: ref Dial->Connection, authinfo: ref Keyring->Authinfo, algs: list of string)
+{+ for (;;) {+ nc := dial->listen(c);
+ if (nc == nil)
+ error(sys->sprint("listen failed: %r"));+ if (debug)
+ sys->fprint(stderr, "ctlfs: got connection from %s\n",
+ readfile(nc.dir + "/remote"));
+ dfd := dial->accept(nc);
+ if (dfd != nil) {+ if(nc.cfd != nil)
+ sys->fprint(nc.cfd, "keepalive");
+ hostname: string;
+ if(passhostnames){+ hostname = readfile(nc.dir + "/remote");
+ if(hostname != nil)
+ hostname = hostname[0:len hostname - 1];
+ }
+
+ spawn ctlfs_authenticator(dfd, authinfo, algs, hostname);
+ }
+ }
+}
+
+# authenticate a connection and set the user id.
+ctlfs_authenticator(dfd: ref Sys->FD, authinfo: ref Keyring->Authinfo,
+ algs: list of string, hostname: string)
+{+ # authenticate and change user id appropriately
+ (fd, err) := auth->server(algs, authinfo, dfd, 1);
+ if (fd == nil) {+ if (debug)
+ sys->fprint(stderr(), "ctlfs: authentication failed: %s\n", err);
+ return;
+ }
+ if (debug)
+ sys->fprint(stderr(), "ctlfs: client authenticated as %s\n", err);
+
+ spawn exportproc(sync, mfd, err, hostname, fd);
+}
+
+ctlfs_loop()
+{+ # Primary server loop
+ loop:
+ while((tmsg := <-tc) != nil) {+ # Switch on operations being performed on a given Fid
+ pick msg := tmsg {+ Open =>
+ srv.default(msg);
+ Read =>
+ fid := srv.getfid(msg.fid);
+
+ if(fid.qtype & Sys->QTDIR) {+ # This is a directory read
+ srv.default(msg);
+ continue loop;
+ }
+
+ case int fid.path {+ Qlog =>
+ # A read on our log file, tell them what they've already said ?
+ s := "";
+
+ for(l := log; l != nil; l = tl l)
+ s = hd l + s;
+
+ srv.reply(styxservers->readstr(msg, s));
+
+ * =>
+ srv.default(msg);
+ }
+
+ Write =>
+ fid := srv.getfid(msg.fid);
+
+ case int fid.path {+ Qctl =>
+ # Don't care about offset
+ cmd := string msg.data;
+
+ reply: ref Rmsg = ref Rmsg.Write(msg.tag, len msg.data);
+
+ case cmd {+ * =>
+ # Ignore empty writes
+ if(cmd != nil)
+ log = cmd :: log;
+ else
+ reply = ref Rmsg.Error(msg.tag, "empty write!");
+ }
+ srv.reply(reply);
+
+ * =>
+ srv.default(msg);
+ }
+
+ * =>
+ srv.default(msg);
+ }
+ }
+
+ exit;
+}
+
+# Navigator function for moving around under /
+ctlfs_navigator(c: chan of ref Navop) {+ loop:
+ for(;;) {+ navop := <-c;
+ pick op := navop {+ Stat =>
+ op.reply <-= (dir(int op.path), nil);
+
+ Walk =>
+ if(op.name == "..") {+ op.reply <-= (dir(Qroot), nil);
+ continue loop;
+ }
+
+ case int op.path&16rff {+
+ Qroot =>
+ for(i := 1; i < Qmax; i++)
+ if(tab[i].t1 == op.name) {+ op.reply <-= (dir(i), nil);
+ continue loop;
+ }
+
+ op.reply <-= (nil, Enotfound);
+ * =>
+ op.reply <-= (nil, Enotdir);
+ }
+
+ Readdir =>
+ for(i := 0; i < op.count && i + op.offset < (len tab) - 1; i++)
+ op.reply <-= (dir(Qroot+1+i+op.offset), nil);
+
+ op.reply <-= (nil, nil);
+ }
+ }
+}
--- /dev/null
+++ b/appl/cmd/dddb.b
@@ -1,0 +1,94 @@
+implement Dddb;
+
+include "sys.m";
+ sys: Sys;
+include "arg.m";
+include "draw.m";
+include "string.m";
+ strm: String;
+
+include "config.b";
+include "ctlfs.b";
+
+stderr: ref Sys->FD;
+debug: int;
+
+error(s: string)
+{+ sys->fprint(stderr, "dddb: %s\n", s);
+ raise "dddb:error";
+}
+
+Dddb: module {+ init: fn(nil: ref Draw->Context, args: list of string);
+ run_fs: fn(cfg: Config);
+
+ Config: adt {+ name: string;
+ sysn: string;
+ addr: string;
+ storage: string;
+ fswrks: int;
+ nodes: list of NodeConfig;
+
+ open: fn(nodename: string, path: string): Config;
+ };
+
+ NodeConfig: adt {+ name: string;
+ sysn: string;
+ addr: string;
+ storage: string;
+ fswrks: int;
+
+ new: fn(entry: ref Dbentry): NodeConfig;
+ };
+
+};
+
+init(nil: ref Draw->Context, args: list of string)
+{+ sys = load Sys Sys->PATH;
+ arg := load Arg Arg->PATH;
+ strm = load String String->PATH;
+
+ stderr = sys->fildes(2);
+ cfgpath: string = "";
+
+ arg->init(args);
+ arg->setusage(arg->progname()+ " [-d] [-c config] nodename");
+ while((c := arg->opt()) != 0)
+ case c {+ 'd' => debug++;
+ 'c' =>
+ cfgpath = arg->earg();
+ * =>
+ sys->fprint(sys->fildes(2), "bad option: -%c\n", c);
+ arg->usage();
+ }
+
+ args = arg->argv();
+
+ nodename := hd args;
+
+ if(nodename == nil) {+ sys->fprint(stderr, "dddb: no nodename supplied\n");
+ arg->usage();
+ }
+
+ if(debug)
+ sys->fprint(stderr, "dddb: opening config file\n");
+ cfg := Config.open(nodename, cfgpath);
+
+ if(debug) {+ sys->fprint(stderr, "dddb: database parms:\n");
+ sys->fprint(stderr, "cfg.name: %s\n", cfg.name);
+ sys->fprint(stderr, "cfg.sysn: %s\n", cfg.sysn);
+ sys->fprint(stderr, "cfg.storage: %s\n", cfg.storage);
+ sys->fprint(stderr, "cfg.fswrks: %d\n", cfg.fswrks);
+ }
+
+ run_fs(cfg);
+}
+
+
--- /dev/null
+++ b/appl/cmd/mkfile
@@ -1,0 +1,18 @@
+<../../mkconfig
+
+TARG=\
+ dddb.dis\
+
+MODULES=\
+
+SYSMODULES=\
+ arg.m\
+ dial.m\
+ draw.m\
+ sys.m\
+ styx.m\
+ styxservers.m\
+
+DISBIN=$home/dis
+
+<$ROOT/mkfiles/mkdis
--- /dev/null
+++ b/appl/lib/mkfile
@@ -1,0 +1,11 @@
+<../../mkconfig
+
+TARG=\
+
+MODULES=\
+
+SYSMODULES=\
+
+DISBIN=$ROOT/dis/lib
+
+<$ROOT/mkfiles/mkdis
--- /dev/null
+++ b/appl/mkfile
@@ -1,0 +1,6 @@
+<../mkconfig
+
+DIRS=\
+ cmd\
+
+<$ROOT/mkfiles/mksubdirs
--- /dev/null
+++ b/mkconfig
@@ -1,0 +1,6 @@
+<$ROOT/mkconfig
+
+# Comment the below lines for hosted builds
+SYSHOST=Inferno
+SYSTARG=Inferno
+ROOT=/
--- /dev/null
+++ b/mkfile
@@ -1,0 +1,6 @@
+<mkconfig
+
+DIRS=\
+ appl\
+
+<$ROOT/mkfiles/mksubdirs