Project

General

Profile

Statistics
| Branch: | Revision:

gdp-if / tensorflow / gdpfs.cc @ master

History | View | Annotate | Download (33.6 KB)

1 13a1f6b0 Eric Allman
/* vim: set ai sw=4 sts=4 ts=4 :*/
2
3 3c340135 Nitesh Mor
#include "gdpfs.h"
4
5
#include <assert.h>
6
#include <time.h>
7 a531ea78 Nitesh Mor
#include <iostream>
8
#include <cstdint>
9
#include <map>
10
#include <vector>
11
#include <string>
12
13
#include "GDPfs.pb.h"
14
15 33678164 Nitesh Mor
static EP_DBG Dbg = EP_DBG_INIT("gdpfs.main", "GDP FileSystem");
16 c5a0c26c Nitesh Mor
static gdp_iomode_t GLOBAL_IOMODE;
17 13a1f6b0 Eric Allman
static const char *LogPrefixDef = "tensorflow";
18 c5a0c26c Nitesh Mor
19 13a1f6b0 Eric Allman
static gdp_recno_t RawAppend(
20
                                                gdp_gin_t *gin,
21
                                                const std::string& s,
22
                                                bool async = false);
23 c4c264f3 Nitesh Mor
24
// get time since epoch in ns
25 418bfd3f Nitesh Mor
uint64_t TimeNS()
26 c4c264f3 Nitesh Mor
{
27
        struct timeval t;
28
        gettimeofday(&t, NULL);
29
        return (uint64_t) 1000*(t.tv_sec*1000000 + t.tv_usec);
30
}
31
32 c5a0c26c Nitesh Mor
// returns whether a given i/o mode is good enough for write ops
33
bool __writable_mode(gdp_iomode_t m)
34
{
35
        return (((uint32_t) m & (uint32_t) GDP_MODE_AO) != 0);
36
}
37 a531ea78 Nitesh Mor
38 8185b9b5 Nitesh Mor
// Initializes the library. "mode" provides a general global I/O mode,
39
// invidual usage can be more restrictive. i.e. memory-mapped files are
40
// always in RO mode, even if the initialization mode allows writing.
41 a231577b Nitesh Mor
GdpfsStatus GDPfsInit(gdp_iomode_t mode, const char* debug_setting)
42 7d5c5334 Nitesh Mor
{
43 beec3457 Nitesh Mor
        srand (time(NULL));
44 0277d6eb Nitesh Mor
45 c5a0c26c Nitesh Mor
        ep_dbg_cprintf(Dbg, 2, "Setting global I/O mode to: %d\n", mode);
46
        GLOBAL_IOMODE = mode;
47 0277d6eb Nitesh Mor
        ep_dbg_cprintf(Dbg, 2, "Done reading the configuration\n");
48 2813f1d2 Nitesh Mor
49
        return kSuccess;
50 7d5c5334 Nitesh Mor
}
51
52 a531ea78 Nitesh Mor
53 13a1f6b0 Eric Allman
// Initialize a GDPfs-based filesystem
54
// Since there is no root directory to used as an anchor object this
55
//   has to stick to low level primitives, hence the code duplication
56
//   (from GDPDir::NewEntry and GDPDir::AddEntry).
57
//   XXX This is a prime target for refactoring. XXX
58
// NOTE WELL: This does not update the cache, so you cannot combine it
59
//   with other operations, that is, the process must exit without using
60
//   the filesystem.
61
static GdpfsStatus RawCreateLog(std::string logname, gdp_gin_t **ginp);
62
63
GdpfsStatus GDPfsMkfs(const std::string& rootdir)
64
{
65
        GdpfsStatus gstat;
66
        gdp_gin_t *gin;
67
68
        ep_dbg_cprintf(Dbg, 10, "Initializing filesystem %s\n", rootdir.c_str());
69
        GDPfsInit(GDP_MODE_RA, NULL);
70
71
        // actually create the physical log
72
        gstat = RawCreateLog(rootdir, &gin);
73
        if (gstat != kSuccess)
74
                return gstat;
75
76
        // create a GDPfsMsg that we'll append to the log
77
        ep_dbg_cprintf(Dbg, 15, "Adding [%s] to the log\n", rootdir.c_str());
78
79
        GDPfs::GDPfsMsg m;
80
        GDPfs::GDPfsMeta *meta;
81
82
        meta = m.mutable_meta();
83
        meta->set_type(GDPfs::DIR);
84
85
        m.set_time_ns(TimeNS());
86
87
        // actually append it to the log
88
        std::string s;
89
        m.SerializeToString(&s);
90
        RawAppend(gin, s);
91
        gdp_gin_close(gin);
92
}
93
94
95 8185b9b5 Nitesh Mor
// Create a null terminated string of length "len" (including the null)
96
// and store it at provided memory location. If "cheat" is true, then
97
// a deterministic string is generated instead of truly random string,
98
// usually resulting in much faster creation.
99
// Assumes that "s" can hold "len" number of bytes.
100 418bfd3f Nitesh Mor
void RandData(char *s, const int len, bool cheat)
101 a531ea78 Nitesh Mor
{
102
        static const char alphanum[] =
103
            "0123456789"
104
                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
105
        "abcdefghijklmnopqrstuvwxyz";
106
107
        if (cheat)
108
        {
109
                memset(s, 88, len-1);
110
        }
111
        else
112
        {
113 6740856c Nitesh Mor
                for (int i = 0; i < len; ++i)
114 a531ea78 Nitesh Mor
                        s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];
115
        }
116
    s[len] = 0;
117
}
118
119 59709b8c Nitesh Mor
120 8185b9b5 Nitesh Mor
// Returns the full path (fname) split into individual components.
121
// Uses "://" as a delimiter for scheme, and "/" as a delimiter for
122
// path components. An example:
123
// "gdp://x/y/z" => "scheme" = "gdp", "parts"={"x", "y", "z"}
124 418bfd3f Nitesh Mor
void SplitPath(const std::string& fname, std::string* scheme,
125 2b567fbd Nitesh Mor
                                std::vector<std::string> *p)
126 59709b8c Nitesh Mor
{
127 418bfd3f Nitesh Mor
        ep_dbg_cprintf(Dbg, 53, "SplitPath(%s)\n", fname.c_str());
128 59709b8c Nitesh Mor
129 2b567fbd Nitesh Mor
        std::size_t s;
130
        s = fname.find("://");
131 59709b8c Nitesh Mor
132 2b567fbd Nitesh Mor
        if (s != std::string::npos)
133
        {
134
                scheme->assign(fname.substr(0, s));
135
                // to account for the length of '://'
136
                s = s+3;
137
        }
138
        else
139
                s = 0;
140 59709b8c Nitesh Mor
141 2b567fbd Nitesh Mor
        char *tmp = new char[fname.length()-s+1];
142
        strncpy(tmp, fname.c_str()+s, (fname.length()-s+1));
143
144
        p->clear();
145
        char *tok;
146
        std::string str;
147
148
        tok = strtok(tmp, "/");
149 59709b8c Nitesh Mor
        while (tok != NULL)
150
        {
151 2b567fbd Nitesh Mor
                str.clear();
152
                str.assign(tok);
153
                p->push_back(str);
154 59709b8c Nitesh Mor
                tok = strtok(NULL, "/");
155
        }
156
157 2b567fbd Nitesh Mor
        // just debugging output below.
158 2505d471 Nitesh Mor
        if (ep_dbg_test(Dbg, 53))
159 2b567fbd Nitesh Mor
        {
160
                std::vector<std::string>::iterator it;
161
162 418bfd3f Nitesh Mor
                ep_dbg_printf("SplitPath(%s) => scheme: [%s]\n",
163 2b567fbd Nitesh Mor
                                                fname.c_str(), scheme->c_str());
164
                for (it=p->begin(); it!=p->end(); it++)
165
                        ep_dbg_printf("=> '%s' ", it->c_str());
166
167
                ep_dbg_printf("\n");
168
        }
169 59709b8c Nitesh Mor
}
170
171 2b567fbd Nitesh Mor
172 8185b9b5 Nitesh Mor
// Takes in a full path (including "gdp://") and returns the
173
// name of the directory log in "topdir" and everything else
174
// in "remaining". As a side effect, performs sanitization of
175
// the path (duplicated '//', etc).
176
// e.g. "gdp://x/y/z" => "topdir" = "x", "remaining" = "y/z"
177 6c1b37e9 Nitesh Mor
// Returns error when the path can not be parsed as a gdp path.
178
GdpfsStatus ParsePath(const std::string& fname, std::string* topdir,
179 59709b8c Nitesh Mor
                                                std::string* remaining)
180 a531ea78 Nitesh Mor
{
181 940e8410 Nitesh Mor
        uint32_t i=0;
182 2b567fbd Nitesh Mor
        std::string scheme;
183
        std::vector<std::string> parts;
184 633a8941 Nitesh Mor
185 2b567fbd Nitesh Mor
        topdir->clear();
186
        remaining->clear();
187 3c5b891d Nitesh Mor
188 418bfd3f Nitesh Mor
        SplitPath(fname, &scheme, &parts);
189 6c1b37e9 Nitesh Mor
190
        if (scheme != "gdp" || parts.size() == 0)
191
        {
192
                ep_dbg_cprintf(Dbg, 2, "Invalid GDP path: %s\n", fname.c_str());
193
                return kInvalidArg;
194
        }
195 2b567fbd Nitesh Mor
        
196 940e8410 Nitesh Mor
        topdir->assign(parts[0]);
197
        for (i=1; i<parts.size(); i++)
198
        {
199
                remaining->append(parts[i]);
200
                if (i!=parts.size()-1)
201
                        remaining->append("/");
202
        }
203 2561f2b3 Nitesh Mor
204 418bfd3f Nitesh Mor
        ep_dbg_cprintf(Dbg, 43, "ParsePath(%s) => %s, %s\n",
205 3c5b891d Nitesh Mor
                                                        fname.c_str(), topdir->c_str(),
206
                                                        remaining->c_str());
207 6c1b37e9 Nitesh Mor
        return kSuccess;
208 a531ea78 Nitesh Mor
}
209
210 8185b9b5 Nitesh Mor
// Splits the name into dirname and basename (the last part
211
// of a name. Input name may or may not have the protocol
212
// (gdp://) included. However, dirname never includes this
213
// protocol name.
214 13a1f6b0 Eric Allman
// e.g., "gdp://x/y/z" => dirname = "x/y", basename = "z".
215 70e2bd67 Nitesh Mor
void BaseDirName(const std::string& fname, std::string* dirname,
216 41631f5e Nitesh Mor
                                        std::string* basename)
217
{
218 2b567fbd Nitesh Mor
        uint32_t i;
219
        std::string scheme;
220
        std::vector<std::string> parts;
221 41631f5e Nitesh Mor
222
        dirname->clear();
223
        basename->clear();
224
225 418bfd3f Nitesh Mor
        SplitPath(fname, &scheme, &parts);
226 2b567fbd Nitesh Mor
227
        for (i=0; i<parts.size()-1; i++){
228
                dirname->append(parts[i]);
229
                if (i!=parts.size()-2)
230
                        dirname->append("/");
231 41631f5e Nitesh Mor
        }
232 2b567fbd Nitesh Mor
        basename->assign(parts[parts.size()-1]);
233 41631f5e Nitesh Mor
234 418bfd3f Nitesh Mor
        ep_dbg_cprintf(Dbg, 42, "BaseDirName(%s) => %s, %s\n",
235 41631f5e Nitesh Mor
                                                        fname.c_str(), dirname->c_str(),
236
                                                        basename->c_str());
237
}
238
239 8185b9b5 Nitesh Mor
// Returns whether the path is a valid file/directory name.
240
// At the moment, simply checks for presence of a '/' in the name.
241
// XXX the checking should be more extensive and thorough
242 418bfd3f Nitesh Mor
bool NameValid(const std::string& path)
243 7ca4823f Nitesh Mor
{
244
        std::size_t found = path.find("/");
245
        return (found == std::string::npos);
246
}
247
248 13a1f6b0 Eric Allman
// Do a low (GDP) level log creation.  The (human-oriented) log name
249
// must already be created.
250
static GdpfsStatus RawCreateLog(std::string logname, gdp_gin_t **ginp)
251 a531ea78 Nitesh Mor
{
252
        EP_STAT estat;
253
        gdp_md_t *gmd;
254 d769e0e0 Eric Allman
        gdp_name_t log_internal;
255 ec84b225 Nitesh Mor
        // sig-key management. We don't create keys, however.
256
        bool usesig;
257
        const char *keyfile = NULL;
258
        EP_CRYPTO_KEY *key = NULL;
259 d769e0e0 Eric Allman
        gdp_create_info_t *gci = gdp_create_info_new();
260 13a1f6b0 Eric Allman
        const char *logname_c = NULL;
261 6740856c Nitesh Mor
262 13a1f6b0 Eric Allman
        if (logname.length() > 0)
263
        {
264
                logname_c = logname.c_str();
265
                estat = gdp_parse_name(logname_c, log_internal);
266
                if (!EP_STAT_ISOK(estat))
267
                        return kGdpNameError;
268
        }
269 2813f1d2 Nitesh Mor
270 13a1f6b0 Eric Allman
        ep_dbg_cprintf(Dbg, 4, "Creating log: %s\n", logname_c);
271 a531ea78 Nitesh Mor
272
        gmd = gdp_md_new(0);
273 ec84b225 Nitesh Mor
274
        // read a signature key if we are supposed to use signatures
275
        usesig = ep_adm_getboolparam("swarm.gdpfs.usesig", false);
276
277
        if (usesig)
278
        {
279
280
                // read the key in "key"
281
                keyfile = ep_adm_getstrparam("swarm.gdpfs.keyfile", NULL);
282
                if (keyfile == NULL)
283
                {
284
                        ep_app_error("No keyfile provided.");
285
                        exit(70);        // EX_SOFTWARE XXX fix this
286
                }
287
288 d769e0e0 Eric Allman
                // read the provided keyfile and provide to create info
289
                key = ep_crypto_key_read_file(keyfile, EP_CRYPTO_KEYFORM_PEM,
290
                                                        EP_CRYPTO_F_SECRET);
291
                if (key == NULL)
292 ec84b225 Nitesh Mor
                {
293 d769e0e0 Eric Allman
                        ep_app_error("Cannot read keyfile %s", keyfile);
294 ec84b225 Nitesh Mor
                        exit(70);        // EX_SOFTWARE XXX fix this
295
                }
296 d769e0e0 Eric Allman
                estat = gdp_create_info_set_owner_key(gci, key, "sha256");
297 ec84b225 Nitesh Mor
                if (!EP_STAT_ISOK(estat))
298
                {
299 d769e0e0 Eric Allman
                        ep_app_error("Could not set owner key");
300 ec84b225 Nitesh Mor
                        exit(70);        // EX_SOFTWARE XXX fix this
301
                }
302
        }
303 5b00eecd Eric Allman
        else
304
        {
305
                estat = gdp_create_info_new_owner_key(gci, NULL, "none",
306
                                0, NULL, "none");
307
                if (!EP_STAT_ISOK(estat))
308
                {
309
                        ep_app_error("Could not set null owner key for %s",
310 13a1f6b0 Eric Allman
                                        logname_c);
311 5b00eecd Eric Allman
                        exit(70);        // EX_SOFTWARE XXX fix this
312
                }
313
        }
314 ec84b225 Nitesh Mor
315 13a1f6b0 Eric Allman
        estat = gdp_gin_create(gci, logname_c, ginp);
316 d769e0e0 Eric Allman
        gdp_create_info_free(&gci);
317 a531ea78 Nitesh Mor
318
        if (EP_STAT_ISOK(estat))
319
        {
320 2505d471 Nitesh Mor
                ep_dbg_cprintf(Dbg, 4, "Successfully created log\n");
321 2813f1d2 Nitesh Mor
                return kSuccess;
322 a531ea78 Nitesh Mor
        }
323
        else
324
        {
325 13a1f6b0 Eric Allman
                ep_app_message(estat, "Error creating log %s", logname_c);
326 2813f1d2 Nitesh Mor
                return kGdpErrorCreate;
327 a531ea78 Nitesh Mor
        }
328 13a1f6b0 Eric Allman
}
329
330
// Create a new log with a random name and return the full name in
331
// `logname` and the open GIN in `ginp`.
332
GdpfsStatus CreateLog(std::string *logname, gdp_gin_t **ginp)
333
{
334
335
        assert (__writable_mode(GLOBAL_IOMODE));
336
337
#if 0
338
        const char *prefix;
339
        char suffix[12];
340

341
        prefix = ep_adm_getstrparam("swarm.gdpfs.logprefix", NULL);
342
        if (prefix == NULL || prefix[0] == '\0')
343
                prefix = getenv("GDP_NAME_ROOT");
344
        if (prefix == NULL || prefix[0] == '\0')
345
                prefix = LogPrefixDef;
346
        RandData(suffix, sizeof suffix);
347

348
        char generated_name[strlen(prefix) + sizeof suffix + 2];
349
        snprintf(generated_name, sizeof generated_name, "%s.%s", prefix, suffix);
350
#endif
351 a531ea78 Nitesh Mor
352 13a1f6b0 Eric Allman
        // passing a null string creates an anonymous log
353
        std::string nullname;
354
        gdp_gin_t *gin;
355
        GdpfsStatus kstat = RawCreateLog(nullname, &gin);
356
        if (kstat == kSuccess)
357
        {
358
                gdp_pname_t pname;
359
                gdp_printable_name(*gdp_gin_getname(gin), pname);
360
                logname->assign(pname);
361
                if (ginp != NULL)
362
                        *ginp = gin;
363
                else
364
                        gdp_gin_close(gin);
365
        }
366
        return kstat;
367 a531ea78 Nitesh Mor
}
368
369 8185b9b5 Nitesh Mor
// Parsing utility function. Given a string s, parse this as
370
// an GDPfsMsg set with GDPfsFchunk. Returns offset,
371
// length and actual data (as a string).
372 2813f1d2 Nitesh Mor
GdpfsStatus ParseMsgFchunk(const std::string& s, size_t *offset,
373 f7954a6e Nitesh Mor
                                                        size_t *len, std::string *d)
374
{
375
        GDPfs::GDPfsMsg m;
376 2813f1d2 Nitesh Mor
        bool stat;
377
378
        stat = m.ParseFromString(s);
379
        if (!stat) return kProtoParsingError;
380
381 f7954a6e Nitesh Mor
        ep_dbg_cprintf(Dbg, 60, "Parsed message: %s\n",
382 2813f1d2 Nitesh Mor
                                                                m.DebugString().c_str());
383
384
        // FIXME this shouldn't be an assertion; we are reading
385
        // data potentially generated by a different program with
386
        // a different MAX_RECSIZE
387 f7954a6e Nitesh Mor
        assert (m.fchunk().length() <= MAX_RECSIZE);
388
389
        *offset = (size_t) m.fchunk().offset();
390
        *len = (size_t) m.fchunk().length();
391
        d->assign(m.fchunk().data());
392 2813f1d2 Nitesh Mor
393
        return kSuccess;
394 f7954a6e Nitesh Mor
}
395
396
397 3c5b891d Nitesh Mor
/***********************************************************/
398
/***********************************************************/
399
/***********************************************************/
400 a531ea78 Nitesh Mor
401 13a1f6b0 Eric Allman
GDPFileLowLevel::GDPFileLowLevel(const std::string& rootlog, gdp_iomode_t mode)
402 a531ea78 Nitesh Mor
{
403
404
        EP_STAT estat;
405 ec84b225 Nitesh Mor
        bool usesig = false;
406
        gdp_open_info_t* info;
407 a531ea78 Nitesh Mor
408
        // Let's just initialize things manually instead of
409
        // initialization lists, at least for now.
410 13a1f6b0 Eric Allman
        logname_ = rootlog;
411 ebd8f911 Nitesh Mor
        mode_ = (gdp_iomode_t) ((uint32_t) mode & (uint32_t) GLOBAL_IOMODE);
412 a531ea78 Nitesh Mor
413 6b980dad Nitesh Mor
        ep_dbg_cprintf(Dbg, 10, "Opening %s [mode=%d]\n", logname_.c_str(), mode_);
414
        estat = gdp_parse_name(logname_.c_str(), gobname_);
415 39672fc0 Eric Allman
        if (!EP_STAT_ISOK(estat))
416
        {
417
                char ebuf[100];
418
                ep_app_fatal("cannot parse %s: %s",
419
                                logname_.c_str(),
420
                                ep_stat_tostr(estat, ebuf, sizeof ebuf));
421
        }
422 ec84b225 Nitesh Mor
        info = gdp_open_info_new();
423
424
        // read a signature key if we are supposed to use signatures
425
        usesig = ep_adm_getboolparam("swarm.gdpfs.usesig", false);
426
        if (usesig && __writable_mode(mode_))
427
        {
428
                FILE *fp;
429
                EP_CRYPTO_KEY *skey = NULL;
430
                const char *keyfile = NULL;
431
432
                keyfile = ep_adm_getstrparam("swarm.gdpfs.keyfile", NULL);
433
                fp = fopen(keyfile, "r");
434
                if (fp == NULL)
435
                {
436
                        ep_app_error("cannot open signing key file %s", keyfile);
437
                        exit(70);        // EX_SOFTWARE XXX fix this
438
                }
439
440
                skey = ep_crypto_key_read_fp(fp, keyfile,
441
                                EP_CRYPTO_KEYFORM_PEM, EP_CRYPTO_F_SECRET);
442
                if (skey == NULL)
443
                {
444
                        ep_app_error("cannot read signing key file %s", keyfile);
445
                        exit(70);        // EX_SOFTWARE XXX fix this
446
                }
447
448
                estat = gdp_open_info_set_signing_key(info, skey);
449
        }
450
451
        if (usesig && !__writable_mode(mode_))
452
        {
453
                gdp_open_info_set_vrfy(info, true);
454
        }
455
456
        estat = gdp_gin_open(gobname_, mode_, info, &handle_);
457 5b00eecd Eric Allman
        if (EP_STAT_ISFAIL(estat))
458 39672fc0 Eric Allman
        {
459
                char ebuf[100];
460
                ep_app_fatal("cannot open %s: %s",
461
                                logname_.c_str(),
462
                                ep_stat_tostr(estat, ebuf, sizeof ebuf));
463
        }
464 a531ea78 Nitesh Mor
465
        // initialize some useful state variables
466 41ecf852 Nitesh Mor
        SyncLog();
467 a531ea78 Nitesh Mor
}
468
469 69da5427 Nitesh Mor
470 a531ea78 Nitesh Mor
GDPFileLowLevel::~GDPFileLowLevel()
471
{
472 6b980dad Nitesh Mor
        ep_dbg_cprintf(Dbg, 10, "Closing log %s\n", logname_.c_str());
473 41ecf852 Nitesh Mor
474 a531ea78 Nitesh Mor
        EP_STAT estat;
475
        std::map<gdp_recno_t, std::string*>::iterator it;
476
477 ebd8f911 Nitesh Mor
        estat = gdp_gin_close(handle_);
478 a531ea78 Nitesh Mor
479
        // also need to free memory allocated for rec_cache
480 ebd8f911 Nitesh Mor
        for (it=rec_cache_.begin(); it!=rec_cache_.end(); it++)
481 a531ea78 Nitesh Mor
        {
482 41ecf852 Nitesh Mor
                ep_dbg_cprintf(Dbg, 40, "Freeing memory for rec: %ld\n", it->first);
483 a531ea78 Nitesh Mor
                delete it->second;
484
        }
485
        ep_dbg_cprintf(Dbg, 10, "Done cleaning up memory\n");
486
}
487
488 69da5427 Nitesh Mor
489 41ecf852 Nitesh Mor
// read contents of a record in string, returns read recno which may
490
// be different
491 70e2bd67 Nitesh Mor
gdp_recno_t GDPFileLowLevel::ReadRecord(gdp_recno_t recno, std::string *s)
492 a531ea78 Nitesh Mor
{
493 41ecf852 Nitesh Mor
        ep_dbg_cprintf(Dbg, 25, "Reading record %ld\n", recno);
494
495 a531ea78 Nitesh Mor
        EP_STAT estat;
496
        gdp_buf_t *buf;
497
        size_t buflen, readbytes;
498
        gdp_recno_t _recno;
499
        gdp_datum_t *datum;
500
        char localbuf[MAX_RECSIZE];
501
502 da9fafe5 Nitesh Mor
        if (GetCache(recno, s)==kSuccess)
503 a531ea78 Nitesh Mor
        {
504
                ep_dbg_cprintf(Dbg, 40, "Cache hit %ld\n", recno);
505 418bfd3f Nitesh Mor
                // GetCache already filled "s" with data
506 a531ea78 Nitesh Mor
                _recno = recno;
507
        }
508
        else
509
        {
510
                ep_dbg_cprintf(Dbg, 40, "Cache miss %ld\n", recno);
511
                datum = gdp_datum_new();
512 ebd8f911 Nitesh Mor
                estat = gdp_gin_read_by_recno(handle_, recno, datum);
513 a531ea78 Nitesh Mor
514
                if (EP_STAT_ISOK(estat))
515
                {
516
                        buf = gdp_datum_getbuf(datum);
517
                        _recno = gdp_datum_getrecno(datum);
518
                        buflen = gdp_datum_getdlen(datum);
519
520
                        ep_dbg_cprintf(Dbg, 30, "Record size: %ld\n", buflen);
521
522
                        assert (buflen<=MAX_RECSIZE);
523
                        readbytes = gdp_buf_peek(buf, &localbuf, buflen);
524
                        assert (readbytes==buflen);
525
526
                        s->assign(localbuf, readbytes);
527 418bfd3f Nitesh Mor
                        SetCache(_recno, s);
528 a531ea78 Nitesh Mor
                }
529
                else
530
                {
531 13a1f6b0 Eric Allman
                        ep_app_message(estat, "Error reading %ld", recno);
532 a531ea78 Nitesh Mor
                        _recno = 0;
533
                }
534
                gdp_datum_free(datum);
535
        }
536
537
        // just a sanity check.
538
        if (recno > 0)
539
                assert (_recno == recno);
540
541
        return _recno;
542
}
543
544
// s should have space for at least numrec string pointers
545
// returns the number of records read.
546 70e2bd67 Nitesh Mor
int32_t GDPFileLowLevel::ReadRecordAsync(gdp_recno_t startrec,
547
                                                                                 int32_t numrec, std::string **s)
548 a531ea78 Nitesh Mor
{
549 41ecf852 Nitesh Mor
        ep_dbg_cprintf(Dbg, 25, "Reading record %ld[+%d]\n", startrec, numrec);
550
551 a531ea78 Nitesh Mor
        EP_STAT estat;
552
        gdp_buf_t *buf;
553
        size_t buflen, readbytes;
554
        gdp_recno_t _recno;
555
        gdp_datum_t *datum;
556
        gdp_event_t *ev;
557
        int32_t ctr = 0;
558
        char localbuf[MAX_RECSIZE];
559
560 beec3457 Nitesh Mor
        // early exit if numrec == 0
561
        if (numrec <= 0)
562
                return 0;
563 a531ea78 Nitesh Mor
564 41ecf852 Nitesh Mor
        estat = gdp_gin_read_by_recno_async(handle_, startrec, numrec, NULL, NULL);
565 a531ea78 Nitesh Mor
566
        if (EP_STAT_ISOK(estat))
567
        {
568
                while (true)
569
                {
570
                        // this blocks
571 ebd8f911 Nitesh Mor
                        ev = gdp_event_next(handle_, NULL);
572 a531ea78 Nitesh Mor
573
                        if (gdp_event_gettype(ev) == GDP_EVENT_DONE)
574
                        {
575
                                gdp_event_free(ev);
576
                                break;
577
                        }
578
579
                        datum = gdp_event_getdatum(ev);
580
581
                        buf = gdp_datum_getbuf(datum);
582
                        _recno = gdp_datum_getrecno(datum);
583
                        buflen = gdp_datum_getdlen(datum);
584
585
                        ep_dbg_cprintf(Dbg, 30, "Record size: %ld\n", buflen);
586
587
                        assert (buflen<=MAX_RECSIZE);
588
                        readbytes = gdp_buf_peek(buf, &localbuf, buflen);
589
                        assert (readbytes==buflen);
590
591
                        s[ctr]->assign(localbuf, readbytes);
592 418bfd3f Nitesh Mor
                        SetCache(_recno, s[ctr]);
593 a531ea78 Nitesh Mor
                        ctr++;
594
595
                        gdp_event_free(ev);
596
                }
597
        }
598
        else
599
        {
600 13a1f6b0 Eric Allman
                ep_app_message(estat, "Error reading %ld[+%d]", startrec, numrec);
601 a531ea78 Nitesh Mor
        }
602
603 2505d471 Nitesh Mor
        ep_dbg_cprintf(Dbg, 25, "Done reading records\n");
604 a531ea78 Nitesh Mor
        return ctr;
605
}
606
607
608 13a1f6b0 Eric Allman
// Append data from the string.  This is a function, not a method, so it
609
// can be used when initializing a filesystem, taking the GIN as a parameter
610
// rather than from `GDPFileLowLevel::handle_`.
611
static gdp_recno_t RawAppend(
612
                                                gdp_gin_t *gin,
613
                                                const std::string& s,
614
                                                bool async /*=false*/)
615 a531ea78 Nitesh Mor
{
616 13a1f6b0 Eric Allman
        if (ep_dbg_test(Dbg, 25))
617
        {
618
                gdp_pname_t pname;
619 a531ea78 Nitesh Mor
620 13a1f6b0 Eric Allman
                ep_dbg_printf("RawAppend: %ld bytes (%s) to %s\n",
621
                        s.length(), async ? "async" : "sync",
622
                        gdp_printable_name(*gdp_gin_getname(gin), pname));
623
        }
624 a531ea78 Nitesh Mor
625
        gdp_recno_t _recno;
626
        gdp_datum_t *datum;
627
        gdp_buf_t *buf;
628
        std::string *c;
629
        EP_STAT estat;
630
631
        datum = gdp_datum_new();
632
        buf = gdp_datum_getbuf(datum);
633 e7441468 Nitesh Mor
        gdp_buf_write(buf, s.data(), s.length());
634 a531ea78 Nitesh Mor
635
        if (async)
636 13a1f6b0 Eric Allman
                estat = gdp_gin_append_async(gin, 1, &datum, NULL, NULL, NULL);
637 a531ea78 Nitesh Mor
        else
638 13a1f6b0 Eric Allman
                estat = gdp_gin_append(gin, datum, NULL);
639 a531ea78 Nitesh Mor
640
        if (!EP_STAT_ISOK(estat))
641
        {
642 13a1f6b0 Eric Allman
                ep_app_message(estat, "Error appending data");
643
                _recno = -1;
644
        }
645
        else if (!async)
646
        {
647
                _recno = gdp_datum_getrecno(datum);
648 a531ea78 Nitesh Mor
        }
649
        else
650
        {
651 13a1f6b0 Eric Allman
                _recno = 0;
652 a531ea78 Nitesh Mor
        }
653
        gdp_datum_free(datum);
654 89b1655d Nitesh Mor
655 13a1f6b0 Eric Allman
        return _recno;
656
}
657
658
// Method version of the same thing.  Keeps track of the current
659
// record number.
660
GdpfsStatus GDPFileLowLevel::AppendRecord(
661
                                                                const std::string& s,
662
                                                                bool async /*=false*/)
663
{
664
        if (ep_dbg_test(Dbg, 22))
665
        {
666
                gdp_pname_t pname;
667
                ep_dbg_printf("GDPFileLowLevel::AppendRecord: %ld %s\n",
668
                                s.length(), logname_.c_str());
669
        }
670
671
        assert (__writable_mode(mode_));
672
673
        gdp_recno_t _recno = RawAppend(handle_, s, async);
674
        if (_recno == 0)
675
                _recno = maxrecs_ + 1;
676
        if (_recno < 0)
677
                return kFailure;
678
        SetCache(_recno, &s);
679
        maxrecs_ = _recno;
680
        return kSuccess;
681 a531ea78 Nitesh Mor
}
682
683 418bfd3f Nitesh Mor
void GDPFileLowLevel::SetType(GDPfs::FileType type)
684 a531ea78 Nitesh Mor
{
685 418bfd3f Nitesh Mor
        ep_dbg_cprintf(Dbg, 10, "SetType(%d)\n", (int)type);
686 ebd8f911 Nitesh Mor
        assert (__writable_mode(mode_));
687 c5a0c26c Nitesh Mor
688 a531ea78 Nitesh Mor
        std::string s;
689 15ba9a3b Nitesh Mor
        GDPfs::GDPfsMsg m;
690
        GDPfs::GDPfsMeta *meta;
691
692
        meta = m.mutable_meta();
693
        meta->set_type(type);
694 c5a0c26c Nitesh Mor
695 418bfd3f Nitesh Mor
        m.set_time_ns(TimeNS());
696 15ba9a3b Nitesh Mor
697 a531ea78 Nitesh Mor
        m.SerializeToString(&s);
698 e7441468 Nitesh Mor
        AppendRecord(s);
699 c5a0c26c Nitesh Mor
700
        // also update the local variable type
701 ebd8f911 Nitesh Mor
        type_ = type;
702 a531ea78 Nitesh Mor
}
703
704
705 41ecf852 Nitesh Mor
GDPfs::FileType GDPFileLowLevel::GetType(bool sync /*=false*/)
706 a531ea78 Nitesh Mor
{
707 2505d471 Nitesh Mor
        ep_dbg_cprintf(Dbg, 15, "Querying for type\n");
708 9a538e6d Nitesh Mor
        if (sync==true)
709 418bfd3f Nitesh Mor
                SyncLog();
710 41ecf852 Nitesh Mor
        return type_;
711
}
712 a531ea78 Nitesh Mor
713
714 41ecf852 Nitesh Mor
gdp_recno_t GDPFileLowLevel::GetNumRecs(bool sync /*=false*/)
715
{
716
        if (sync == true)
717
                SyncLog();
718
        ep_dbg_cprintf(Dbg, 15, "# records: %ld\n", maxrecs_);
719
        return maxrecs_;
720
}
721
722
uint64_t GDPFileLowLevel::GetMTime()
723
{
724
        ep_dbg_cprintf(Dbg, 15, "GetMTime()\n");
725
        SyncLog(); // we always sync.
726
        return mtime_ns_;
727 a531ea78 Nitesh Mor
}
728
729
730 418bfd3f Nitesh Mor
void GDPFileLowLevel::SyncLog()
731 625d5ff1 Nitesh Mor
{
732 2505d471 Nitesh Mor
        ep_dbg_cprintf(Dbg, 15, "sync: checking for new data\n");
733 625d5ff1 Nitesh Mor
734
        gdp_recno_t r, maxrecs;
735
        std::string s;
736
        GDPfs::GDPfsMsg m;
737
738 ebd8f911 Nitesh Mor
        maxrecs = gdp_gin_getnrecs(handle_);
739 625d5ff1 Nitesh Mor
740 ebd8f911 Nitesh Mor
        if (maxrecs > maxrecs_)                // something new
741 625d5ff1 Nitesh Mor
        {
742 2505d471 Nitesh Mor
                ep_dbg_cprintf(Dbg, 15, "New records\n");
743 625d5ff1 Nitesh Mor
744
                // do we need to check for type?
745 41ecf852 Nitesh Mor
                if (maxrecs_ == 0)        // yes; we didn't have type info already
746 625d5ff1 Nitesh Mor
                {
747 ebd8f911 Nitesh Mor
                        assert (type_ == GDPfs::UNKNOWN_TYPE);
748 625d5ff1 Nitesh Mor
749 418bfd3f Nitesh Mor
                        r = ReadRecord(1, &s);
750 625d5ff1 Nitesh Mor
                        assert (r==1);
751 41ecf852 Nitesh Mor
752 15ba9a3b Nitesh Mor
                        m.ParseFromString(s);
753
                        assert (m.has_meta());
754 41ecf852 Nitesh Mor
755 ebd8f911 Nitesh Mor
                        type_ = m.meta().type();
756 625d5ff1 Nitesh Mor
                }
757
758 41ecf852 Nitesh Mor
                {
759
                        // may as well cause cache hit, so need to worry
760
                        // about extra network overhead.
761 418bfd3f Nitesh Mor
                        r = ReadRecord(maxrecs, &s);
762 625d5ff1 Nitesh Mor
                        m.ParseFromString(s);
763 ebd8f911 Nitesh Mor
                        mtime_ns_ = m.time_ns();
764 625d5ff1 Nitesh Mor
                }
765
766 ebd8f911 Nitesh Mor
                maxrecs_ = maxrecs;
767 625d5ff1 Nitesh Mor
        }
768
769 41ecf852 Nitesh Mor
        // logging... nothing fancy.
770
        if ep_dbg_test(Dbg, 15)
771
        {
772
                switch (type_)
773
                {
774
                        case GDPfs::UNKNOWN_TYPE:
775
                                ep_dbg_printf("Type: UNKNOWN_TYPE\n");
776
                                break;
777
                        case GDPfs::FILE:
778
                                ep_dbg_printf("Type: FILE\n");
779
                                break;
780
                        case GDPfs::DIR:
781
                                ep_dbg_printf("Type: DIR\n");
782
                                break;
783
                        default:
784
                                ep_dbg_printf("Can't figure out type\n");
785
                }
786 625d5ff1 Nitesh Mor
787 41ecf852 Nitesh Mor
                ep_dbg_printf("Last update time: %ld\n", mtime_ns_);
788
                ep_dbg_printf("Number of records: %ld\n", maxrecs_);
789
        }
790 625d5ff1 Nitesh Mor
791 41ecf852 Nitesh Mor
        return;
792 625d5ff1 Nitesh Mor
}
793
794 89b1655d Nitesh Mor
// copies data from string s, and sets the cache
795
GdpfsStatus GDPFileLowLevel::SetCache(const gdp_recno_t recno,
796
                                                                                        const std::string *s)
797
{
798
        ep_dbg_cprintf(Dbg, 40, "Setting cache for %ld\n", recno);
799
        if (recno <= 0)
800
                return kFailure;
801
802
        std::string *c;
803
        c = new std::string;
804
        c->assign(s->data(), s->length());
805
        rec_cache_[recno] = c;
806
807
        return kSuccess;
808
}
809
810
// attempts to find recno in cache and set string with the appropriate
811
// value. Returns false if recno can not be found in the cache.
812
GdpfsStatus GDPFileLowLevel::GetCache(const gdp_recno_t recno, std::string *s)
813
{
814
        ep_dbg_cprintf(Dbg, 40, "Retrieving %ld from cache\n", recno);
815
        std::string *sptr;
816
817
        if (recno <= 0)
818
                return kFailure;
819
820
        if (rec_cache_.find(recno) != rec_cache_.end())
821
        {
822
                sptr = rec_cache_.find(recno)->second;
823
                s->assign(*sptr);
824
                return kSuccess;
825
        }
826
        return kFailure;
827
}
828
829
830 b6438f0c Nitesh Mor
/***********************************************************/
831
/***********************************************************/
832
/***********************************************************/
833
834 13a1f6b0 Eric Allman
GDPDir::GDPDir(const std::string& rootlog, gdp_iomode_t mode)
835
                                : GDPFileLowLevel(rootlog, mode)
836 a531ea78 Nitesh Mor
{
837
        int i;
838 6b980dad Nitesh Mor
        std::string s, name, _logname_;
839 a531ea78 Nitesh Mor
        GDPfs::GDPfsMsg m;
840
        GDPfs::Operation op;
841
        std::map<std::string, std::string>::iterator it;
842
843 ebd8f911 Nitesh Mor
        if (type_ != GDPfs::DIR)
844 78fd96a8 Nitesh Mor
        {
845 ebd8f911 Nitesh Mor
                if (__writable_mode(mode_))
846 418bfd3f Nitesh Mor
                        SetType(GDPfs::DIR);
847 78fd96a8 Nitesh Mor
                else
848
                        ep_dbg_cprintf(Dbg, 1, "FileType isn't setup properly, "
849
                                                                        "but we are in R/O mode\n");
850
        }
851 a531ea78 Nitesh Mor
852 27ef568b Nitesh Mor
        // just to populate our local cache with ReadRecordAsync
853 5b00eecd Eric Allman
        if (maxrecs_ > 0)
854 27ef568b Nitesh Mor
        {
855
                int i, readrecs, numrec;
856
                std::string **sarray;
857
858
                numrec = maxrecs_-1;
859
                sarray = new std::string* [numrec];
860
861
                for (i=0; i<numrec; i++)
862
                        sarray[i] = new std::string;
863
864
                readrecs = ReadRecordAsync(2, numrec, sarray);
865
                assert (readrecs == numrec);
866
867
                for (i=0; i<numrec; i++)
868
                        delete sarray[i];
869
                delete sarray;
870
        }
871
872 ebd8f911 Nitesh Mor
        // populate dentries_
873
        for (i=2; i<=maxrecs_; i++)
874 a531ea78 Nitesh Mor
        {
875
                s.clear();
876
                name.clear();
877 6b980dad Nitesh Mor
                _logname_.clear();
878 a531ea78 Nitesh Mor
879 418bfd3f Nitesh Mor
                ReadRecord(i, &s);
880 a531ea78 Nitesh Mor
                m.ParseFromString(s);
881
882
                op = m.dentry().op();
883
884
                if (op == GDPfs::UNKNOWN_OP)
885
                {
886
                        ep_dbg_cprintf(Dbg, 1, "unknown op in dentry\n");
887
                }
888
                else
889
                {
890
                        name.assign(m.dentry().name());
891 ebd8f911 Nitesh Mor
                        it = dentries_.find(name);
892 a531ea78 Nitesh Mor
                        if (op == GDPfs::DELETE)
893
                        {
894 ebd8f911 Nitesh Mor
                                if (it != dentries_.end())
895 a531ea78 Nitesh Mor
                                {
896
                                        ep_dbg_cprintf(Dbg, 10, "Deleting entry: %s\n",
897
                                                                                                        name.c_str());
898 ebd8f911 Nitesh Mor
                                        dentries_.erase(name);
899 a531ea78 Nitesh Mor
                                }
900
                        }
901
                        else if (op == GDPfs::ADD)
902
                        {
903 6b980dad Nitesh Mor
                                _logname_.assign(m.dentry().logname());
904 9f162415 Nitesh Mor
                                ep_dbg_cprintf(Dbg, 13, "Entry: %s => %s\n",
905 6b980dad Nitesh Mor
                                                                                name.c_str(), _logname_.c_str());
906 ebd8f911 Nitesh Mor
                                if (it != dentries_.end())
907 a531ea78 Nitesh Mor
                                {
908
                                        ep_dbg_cprintf(Dbg, 15, "First deleting %s\n",
909
                                                                                                        name.c_str());
910 ebd8f911 Nitesh Mor
                                        dentries_.erase(name);
911 a531ea78 Nitesh Mor
                                }
912 6b980dad Nitesh Mor
                                dentries_[name] = _logname_;
913 a531ea78 Nitesh Mor
                        }
914
                        else
915
                                assert (false);
916
                }
917
        }
918
919 ebd8f911 Nitesh Mor
        ep_dbg_cprintf(Dbg, 12, "loaded %ld entries\n", dentries_.size());
920 a531ea78 Nitesh Mor
}
921
922
923 13a1f6b0 Eric Allman
// create a new file+log; stores the logname. If recursive==true,
924
// any non-existent paths on the way are created.
925 70e2bd67 Nitesh Mor
void GDPDir::NewFile(const std::string name, std::string *logname,
926 9a538e6d Nitesh Mor
                                                bool recursive)
927 21201b53 Nitesh Mor
{
928
        ep_dbg_cprintf(Dbg, 5, "Creating new file: %s\n", name.c_str());
929 9a538e6d Nitesh Mor
        NewEntry(name, GDPfs::FILE, logname, recursive);
930 21201b53 Nitesh Mor
        ep_dbg_cprintf(Dbg, 5, "Done creating new file: %s => %s\n",
931
                                                                        name.c_str(), logname->c_str());
932
}
933
934 13a1f6b0 Eric Allman
// create a directory. Any non-existent parent directories
935
// are created if recursive==true
936 2651f137 Nitesh Mor
void GDPDir::CreateDir(const std::string name, bool recursive)
937 21201b53 Nitesh Mor
{
938
        ep_dbg_cprintf(Dbg, 5, "Creating new dir: %s\n", name.c_str());
939 9a538e6d Nitesh Mor
        NewEntry(name, GDPfs::DIR, NULL, recursive);
940 21201b53 Nitesh Mor
        ep_dbg_cprintf(Dbg, 5, "Done creating new dir: %s\n", name.c_str());
941
}
942
943 13a1f6b0 Eric Allman
944
// Add a new entry to the meta-log describing the item with "file
945
// system" name `name` and type `t`.  If `recursive` is set, any
946
// intermediate "directories" will be created.  If `t` is
947
// `GDPfs::FILE` it will also create a new log to hold the contents
948
// and store the name of that log in `logname`.
949 70e2bd67 Nitesh Mor
void GDPDir::NewEntry(std::string name, GDPfs::FileType t,
950 6b980dad Nitesh Mor
                                                std::string *logname, bool recursive)
951 a531ea78 Nitesh Mor
{
952 13a1f6b0 Eric Allman
        gdp_pname_t pname;
953
        ep_dbg_cprintf(Dbg, 10, "Adding entry %s to log %s\n",
954
                        name.c_str(),
955
                        gdp_printable_name(*gdp_gin_getname(handle_), pname));
956 21201b53 Nitesh Mor
957 418bfd3f Nitesh Mor
        if (NameExists(name))
958 21201b53 Nitesh Mor
        {
959
                ep_dbg_cprintf(Dbg, 5, "Name [%s] exists\n", name.c_str());
960
                if (t==GDPfs::FILE)
961 6b980dad Nitesh Mor
                        GetEntryLogname(name, logname);
962 21201b53 Nitesh Mor
                return;
963
        }
964
965
        // first, some assertions
966 ebd8f911 Nitesh Mor
        assert (__writable_mode(mode_));
967 418bfd3f Nitesh Mor
        assert (!NameExists(name));
968 21201b53 Nitesh Mor
        if (t==GDPfs::DIR)
969 6b980dad Nitesh Mor
                assert (logname == NULL);
970 05e107f7 Nitesh Mor
971 21201b53 Nitesh Mor
        // declare variables
972 13a1f6b0 Eric Allman
        std::string dirname, basename;
973 a531ea78 Nitesh Mor
974 21201b53 Nitesh Mor
        // make sure the parent directory exists
975 418bfd3f Nitesh Mor
        BaseDirName(name, &dirname, &basename);
976 9a538e6d Nitesh Mor
        if ((dirname.length()!=0) && recursive)
977 21201b53 Nitesh Mor
        {
978
                // recursively create the path.
979
                ep_dbg_cprintf(Dbg, 10, "Creating new dir [%s]\n",
980
                                                                                        dirname.c_str());
981 9a538e6d Nitesh Mor
                NewEntry(dirname, GDPfs::DIR, NULL, recursive);
982 21201b53 Nitesh Mor
        }
983
984
        // this should be fine now, unless we were asked
985
        // explicitly to not create the dirname
986 418bfd3f Nitesh Mor
        assert (dirname.length()==0 || NameExists(dirname));
987 21201b53 Nitesh Mor
988
        // create a log if we are creating a new file.
989
        if (t == GDPfs::FILE)
990
        {
991
                ep_dbg_cprintf(Dbg, 10, "Creating new log\n");
992 13a1f6b0 Eric Allman
                CreateLog(logname, NULL);
993 21201b53 Nitesh Mor
                // the following just initializes the file type
994 13a1f6b0 Eric Allman
                //GDPFile f(logname, GDP_MODE_RA);
995 21201b53 Nitesh Mor
        }
996
997 13a1f6b0 Eric Allman
        // actually add the entry to the metainfo log
998
        AddEntry(name, *logname, t);
999 beec3457 Nitesh Mor
}
1000
1001 13a1f6b0 Eric Allman
1002
// Add an entry to the meta log for this filesystem that describes a new
1003
// file or directory.  The `name` is the human-oriented "file name",
1004
// `logname` is the internal log name (relevent for files only),
1005
// and `t` is the type (GDPfs::DIR or GDPfs::FILE).
1006 70e2bd67 Nitesh Mor
void GDPDir::AddEntry(std::string name, std::string logname, GDPfs::FileType t)
1007 beec3457 Nitesh Mor
{
1008
1009 13a1f6b0 Eric Allman
        // create a GDPfsMsg that we'll append to the metadata log
1010
        if (ep_dbg_test(Dbg, 15))
1011
        {
1012
                gdp_pname_t pname;
1013
                ep_dbg_printf("Adding [%s => %s] to metalog %s\n",
1014
                                                name.c_str(), logname.c_str(),
1015
                                                gdp_printable_name(*gdp_gin_getname(handle_), pname));
1016
        }
1017 beec3457 Nitesh Mor
1018 ebd8f911 Nitesh Mor
        assert (__writable_mode(mode_));
1019 beec3457 Nitesh Mor
1020
        std::string s;
1021
        GDPfs::GDPfsMsg m;
1022
        GDPfs::GDPfsDentry *d;
1023
        std::map<std::string, std::string>::iterator it;
1024
1025 a531ea78 Nitesh Mor
        d = m.mutable_dentry();
1026
        d->set_op(GDPfs::ADD);
1027
        d->set_name(name);
1028 beec3457 Nitesh Mor
        if (!logname.empty())
1029
                d->set_logname(logname);
1030 a531ea78 Nitesh Mor
        d->set_type(t);
1031
1032 418bfd3f Nitesh Mor
        m.set_time_ns(TimeNS());
1033 21201b53 Nitesh Mor
        // actually append it to the log
1034 a531ea78 Nitesh Mor
        m.SerializeToString(&s);
1035 e7441468 Nitesh Mor
        AppendRecord(s);
1036 a531ea78 Nitesh Mor
1037
        // also update our local state
1038 ebd8f911 Nitesh Mor
        it = dentries_.find(name);
1039
        if (it != dentries_.end())
1040
                dentries_.erase(name);
1041
        dentries_[name] = logname;
1042 beec3457 Nitesh Mor
1043 a531ea78 Nitesh Mor
}
1044
1045 13a1f6b0 Eric Allman
1046
// Add a "delete" entry to the master meta file.  This undoes the effect
1047
// of the corresponding AddEntry.
1048 418bfd3f Nitesh Mor
void GDPDir::DelEntry(std::string name)
1049 a531ea78 Nitesh Mor
{
1050
1051
        ep_dbg_cprintf(Dbg, 10, "Deleting entry %s\n", name.c_str());
1052 ebd8f911 Nitesh Mor
        assert (__writable_mode(mode_));
1053 a531ea78 Nitesh Mor
1054
        std::string s;
1055
        GDPfs::GDPfsMsg m;
1056
        GDPfs::GDPfsDentry *d;
1057
        std::map<std::string, std::string>::iterator it;
1058
1059
        d = m.mutable_dentry();
1060
        d->set_op(GDPfs::DELETE);
1061
        d->set_name(name);
1062
1063 418bfd3f Nitesh Mor
        m.set_time_ns(TimeNS());
1064 a531ea78 Nitesh Mor
        m.SerializeToString(&s);
1065 e7441468 Nitesh Mor
        AppendRecord(s);
1066 a531ea78 Nitesh Mor
1067
        // also update our local state
1068 ebd8f911 Nitesh Mor
        it = dentries_.find(name);
1069
        if (it != dentries_.end())
1070
                dentries_.erase(name);
1071 a531ea78 Nitesh Mor
}
1072
1073 70e2bd67 Nitesh Mor
void GDPDir::RenameEntry(const std::string& oldname, const std::string& newname)
1074 beec3457 Nitesh Mor
{
1075
1076
        ep_dbg_cprintf(Dbg, 10, "Renaming entry %s to %s\n",
1077
                                                                oldname.c_str(), newname.c_str());
1078 ebd8f911 Nitesh Mor
        assert (__writable_mode(mode_));
1079 418bfd3f Nitesh Mor
        assert (NameExists(oldname));
1080
        if (NameExists(newname))
1081 beec3457 Nitesh Mor
                ep_dbg_cprintf(Dbg, 2, "Overwriting existing name\n");
1082
1083
        // collect information about our source
1084
        std::string logname;
1085
        GDPfs::FileType t;
1086 418bfd3f Nitesh Mor
        t = GetEntryType(oldname);
1087 beec3457 Nitesh Mor
        if (t == GDPfs::FILE)
1088 418bfd3f Nitesh Mor
                GetEntryLogname(oldname, &logname);
1089 beec3457 Nitesh Mor
1090
        // execute the delete + add
1091 418bfd3f Nitesh Mor
        DelEntry(oldname);
1092
        AddEntry(newname, logname, t);
1093 beec3457 Nitesh Mor
}
1094 a531ea78 Nitesh Mor
1095 adceff07 Nitesh Mor
void GDPDir::GetChildren(const std::string& dirname,
1096
                                                        std::vector<std::string>* children)
1097
{
1098
1099
        std::string dir, base;
1100
        std::map<std::string, std::string>::iterator it;
1101
1102
        for (it= dentries_.begin(); it != dentries_.end(); it++)
1103
        {
1104
                BaseDirName(it->first, &dir, &base);
1105
                ep_dbg_cprintf(Dbg, 15, "comparing %s with %s\n",
1106
                                                        dir.c_str(), dirname.c_str());
1107
                if (dir.compare(dirname) == 0)
1108
                {
1109
                        ep_dbg_cprintf(Dbg, 12, "Child => %s\n", base.c_str());
1110
                        children->push_back(base);
1111
                }
1112
        }
1113
}
1114
1115
1116 8dce4a9a Nitesh Mor
void GDPDir::GetMatchingPaths(const std::string& pattern,
1117
                                                                std::vector<std::string>* results)
1118
{
1119
1120
        ep_dbg_cprintf(Dbg, 4, "GetMatchingPaths(%s)\n", pattern.c_str());
1121 a48a4a87 Nitesh Mor
        ep_dbg_cprintf(Dbg, 1, "Note that this isn't fully implemented yet.\n");
1122 8dce4a9a Nitesh Mor
1123
        if (pattern.find('*')!=std::string::npos)
1124
        {
1125
                ep_dbg_cprintf(Dbg, 1, " '*' Not yet implemented\n");
1126
                return;
1127
        }
1128
        if (pattern.find('?')!=std::string::npos)
1129
        {
1130
                ep_dbg_cprintf(Dbg, 1, " '?' Not yet implemented\n");
1131
                return;
1132
        }
1133
        if (pattern.find('[')!=std::string::npos)
1134
        {
1135
                ep_dbg_cprintf(Dbg, 1, " '[' Not yet implemented\n");
1136
                return;
1137
        }
1138
        if (pattern.find(']')!=std::string::npos)
1139
        {
1140
                ep_dbg_cprintf(Dbg, 1, " ']' Not yet implemented\n");
1141
                return;
1142
        }
1143
        if (pattern.find("\\\\")!=std::string::npos)
1144
        {
1145
                ep_dbg_cprintf(Dbg, 1, " ds Not yet implemented\n");
1146
                return;
1147
        }
1148
1149
        size_t pos;
1150
        std::map<std::string, std::string>::iterator it;
1151
1152
        for (it=dentries_.begin(); it!=dentries_.end(); it++)
1153
        {
1154
                ep_dbg_cprintf(Dbg, 30, "Matching %s against: %s\n",
1155
                                                                        pattern.c_str(), it->first.c_str());
1156
                pos = it->first.find(pattern);
1157
                if (pos == 0)
1158
                {
1159
                        ep_dbg_cprintf(Dbg, 22, "Found match %s\n", it->first.c_str());
1160
                        results->push_back(it->first);
1161
                }
1162
        }
1163
1164
        return;
1165
1166
}
1167
1168
1169 418bfd3f Nitesh Mor
bool GDPDir::NameExists(const std::string& name)
1170 a531ea78 Nitesh Mor
{
1171 418bfd3f Nitesh Mor
        ep_dbg_cprintf(Dbg, 20, "NameExists(%s)\n", name.c_str());
1172 a531ea78 Nitesh Mor
        std::map<std::string, std::string>::iterator it;
1173 ebd8f911 Nitesh Mor
        it = dentries_.find(name);
1174
        return (it != dentries_.end());
1175 a531ea78 Nitesh Mor
}
1176
1177 ca58f4e8 Nitesh Mor
void GDPDir::Stat(const std::string& name, GdpStat* stat)
1178
{
1179
        ep_dbg_cprintf(Dbg, 4, "Stat(%s)\n", name.c_str());
1180
1181
        stat->is_directory = (GetEntryType(name) == GDPfs::DIR);
1182
        if (!stat->is_directory)
1183
        {
1184
                // first get the logname
1185 13a1f6b0 Eric Allman
                std::string logname;
1186 ca58f4e8 Nitesh Mor
                GetEntryLogname(name, &logname);
1187
                GDPFile f(logname, GDP_MODE_RO);
1188
1189
                stat->length = (uint64_t) f.GetFileSize();
1190
                stat->mtime_nsec = f.GetMTime();
1191
        }
1192
        else
1193
        {
1194
                stat->length = 0;
1195
                stat->mtime_nsec = TimeNS();        // XXX: fix this
1196
        }
1197
1198
        ep_dbg_cprintf(Dbg, 4, "Stat(%s)=%ld, %ld\n", name.c_str(),
1199
                                                                        stat->length, stat->mtime_nsec);
1200
}
1201
1202
1203 13a1f6b0 Eric Allman
// Given a gdpfs file name, return the corresponding log name.
1204 418bfd3f Nitesh Mor
void GDPDir::GetEntryLogname(const std::string& name,
1205 a531ea78 Nitesh Mor
                                                        std::string *logname)
1206
{
1207 418bfd3f Nitesh Mor
        assert (NameExists(name));
1208 41631f5e Nitesh Mor
1209 a531ea78 Nitesh Mor
        std::map<std::string, std::string>::iterator it;
1210 ebd8f911 Nitesh Mor
        it = dentries_.find(name);
1211
        assert (it != dentries_.end());
1212 a531ea78 Nitesh Mor
        logname->assign(it->second.c_str(), it->second.length());
1213
}
1214
1215 418bfd3f Nitesh Mor
GDPfs::FileType GDPDir::GetEntryType(const std::string& name)
1216 2e1342ef Nitesh Mor
{
1217 418bfd3f Nitesh Mor
        assert (NameExists(name));
1218 41631f5e Nitesh Mor
1219 2e1342ef Nitesh Mor
        std::map<std::string, std::string>::iterator it;
1220 ebd8f911 Nitesh Mor
        it = dentries_.find(name);
1221
        assert (it != dentries_.end());
1222 2e1342ef Nitesh Mor
        if (it->second.empty())
1223
                return GDPfs::DIR;        // dirs don't have a logname associated
1224
        else
1225
                return GDPfs::FILE;
1226
}
1227
1228 05e107f7 Nitesh Mor
/***********************************************************/
1229
/***********************************************************/
1230
/***********************************************************/
1231
1232 6b980dad Nitesh Mor
GDPFile::GDPFile(const std::string& logname, gdp_iomode_t mode)
1233 05e107f7 Nitesh Mor
                                : GDPFileLowLevel(logname, mode)
1234
{
1235 ebd8f911 Nitesh Mor
        if (type_ != GDPfs::FILE)
1236 05e107f7 Nitesh Mor
        {
1237
                if (__writable_mode(mode))
1238 418bfd3f Nitesh Mor
                        SetType(GDPfs::FILE);
1239 05e107f7 Nitesh Mor
                else
1240
                        ep_dbg_cprintf(Dbg, 1, "FileType isn't set properly, "
1241
                                                                                "but we are in RO mode.\n");
1242
        }
1243 8263d948 Nitesh Mor
        filesize_ = GetFileSize(true);
1244 05e107f7 Nitesh Mor
}
1245
1246 8263d948 Nitesh Mor
size_t GDPFile::GetFileSize(bool sync /*=true*/)
1247 05e107f7 Nitesh Mor
{
1248 2505d471 Nitesh Mor
        ep_dbg_cprintf(Dbg, 15, "Querying for filesize\n");
1249 8263d948 Nitesh Mor
1250
        size_t offset, len;
1251 05e107f7 Nitesh Mor
        std::string msg, parsed;
1252 ebd8f911 Nitesh Mor
        assert (type_ == GDPfs::FILE);
1253 05e107f7 Nitesh Mor
1254 8263d948 Nitesh Mor
        if (sync)
1255
        {
1256
                maxrecs_ = GetNumRecs(true);
1257
                ReadRecord(maxrecs_, &msg);
1258
                ParseMsgFchunk(msg, &offset, &len, &parsed);
1259
                filesize_ = offset + len;
1260
        }
1261 05e107f7 Nitesh Mor
1262 8263d948 Nitesh Mor
        ep_dbg_cprintf(Dbg, 15, "Filesize is %ld\n", filesize_);
1263
        return filesize_;
1264 05e107f7 Nitesh Mor
}
1265
/***********************************************************/
1266
/***********************************************************/
1267
/***********************************************************/
1268 a531ea78 Nitesh Mor
1269 6b980dad Nitesh Mor
GDPFileROMemMap::GDPFileROMemMap(const std::string& logname,
1270
                                                                        bool async/*=false*/)
1271 70e2bd67 Nitesh Mor
                                : GDPFile(logname, GDP_MODE_RO)
1272 a531ea78 Nitesh Mor
{
1273
        ep_dbg_cprintf(Dbg, 4, "initializing memmap\n");
1274 ebd8f911 Nitesh Mor
        filedata_.clear();
1275
        FetchRecords(2, maxrecs_);
1276 abd157eb Nitesh Mor
}
1277
1278
// fetches records [startrec...endrec] (both ends including)
1279 70e2bd67 Nitesh Mor
void GDPFileROMemMap::FetchRecords(gdp_recno_t startrec,
1280
                                                                   gdp_recno_t endrec, bool async)
1281 abd157eb Nitesh Mor
{
1282
1283 e7441468 Nitesh Mor
        if (endrec < startrec)        // early exit
1284
                return;
1285 abd157eb Nitesh Mor
1286
        int i, j=0, readrecs, numrec;
1287 a531ea78 Nitesh Mor
        size_t offset, len;
1288
        std::string s, parsed;
1289
        std::string **sarray;
1290
1291
        if (!async)
1292
        {
1293 abd157eb Nitesh Mor
                for (i=startrec; i<=endrec; i++)
1294 a531ea78 Nitesh Mor
                {
1295
                        s.clear();
1296
                        parsed.clear();
1297 418bfd3f Nitesh Mor
                        ReadRecord(i, &s);
1298 8263d948 Nitesh Mor
                        ParseMsgFchunk(s, &offset, &len, &parsed);
1299 a531ea78 Nitesh Mor
1300 ebd8f911 Nitesh Mor
                        filedata_.append(parsed.c_str(), parsed.length());
1301 a531ea78 Nitesh Mor
                }
1302
        }
1303
        else
1304 6740856c Nitesh Mor
        {
1305 a531ea78 Nitesh Mor
                // this might have problems with 0 records
1306 abd157eb Nitesh Mor
                numrec = endrec-startrec+1;
1307
                sarray = new std::string* [numrec];
1308 e7441468 Nitesh Mor
1309 abd157eb Nitesh Mor
                for (i=0; i<numrec; i++)
1310 a531ea78 Nitesh Mor
                        sarray[i] = new std::string;
1311 e7441468 Nitesh Mor
1312 418bfd3f Nitesh Mor
                readrecs = ReadRecordAsync(startrec, numrec, sarray);
1313 abd157eb Nitesh Mor
                assert (readrecs == numrec);
1314 a531ea78 Nitesh Mor
1315
                for (i=0; i<readrecs; i++)
1316
                {
1317 8263d948 Nitesh Mor
                        ParseMsgFchunk(*sarray[i], &offset, &len, &parsed);
1318 ebd8f911 Nitesh Mor
                        filedata_.append(parsed.c_str(), parsed.length());
1319 a531ea78 Nitesh Mor
                }
1320 e7441468 Nitesh Mor
1321 abd157eb Nitesh Mor
                for (i=0; i<numrec; i++)
1322 a531ea78 Nitesh Mor
                        delete sarray[i];
1323
                delete sarray;
1324
        }
1325
}
1326
1327
1328 05e107f7 Nitesh Mor
/***********************************************************/
1329
/***********************************************************/
1330
/***********************************************************/
1331
1332 3c340135 Nitesh Mor
size_t GDPFileRO::Read(size_t offset, size_t len, char *buf)
1333 a531ea78 Nitesh Mor
{
1334 abd157eb Nitesh Mor
1335
        // XXX A lame attempt at tolerating situations where
1336
        // someone updates the file elsewhere.
1337 ebd8f911 Nitesh Mor
        gdp_recno_t cur_recs = maxrecs_;
1338 418bfd3f Nitesh Mor
        SyncLog();
1339 ebd8f911 Nitesh Mor
        if (maxrecs_ > cur_recs)
1340 abd157eb Nitesh Mor
        {
1341
                ep_dbg_cprintf(Dbg, 5, "The file seems to have updated\n");
1342
                // need to update things?
1343 ebd8f911 Nitesh Mor
                filesize_ = GetFileSize();
1344
                FetchRecords(cur_recs+1, maxrecs_);
1345 abd157eb Nitesh Mor
        }
1346
        else
1347
                ep_dbg_cprintf(Dbg, 9, "Reading data; file hasn't been updated\n");
1348
1349 ebd8f911 Nitesh Mor
        size_t read = offset + len > filesize_ ? filesize_ - offset : len;
1350
        memcpy(buf, filedata_.c_str()+offset, read);
1351 a531ea78 Nitesh Mor
        return read;
1352
}
1353
1354 05e107f7 Nitesh Mor
/***********************************************************/
1355
/***********************************************************/
1356
/***********************************************************/
1357
1358 a531ea78 Nitesh Mor
1359 70b43258 Nitesh Mor
void GDPFileWO::Append(const std::string& s, bool async/*=false*/)
1360 a531ea78 Nitesh Mor
{
1361
        // append the data in the string
1362 70b43258 Nitesh Mor
        ep_dbg_cprintf(Dbg, 25, "Appending %ld bytes\n", s.length());
1363 a531ea78 Nitesh Mor
1364
        std::string tmp, a;
1365
        GDPfs::GDPfsMsg m;
1366
        GDPfs::GDPfsFchunk* f;
1367
        gdp_event_t *ev;
1368
        int i, _l, num_appends=0, acks=0;
1369
1370 70b43258 Nitesh Mor
        // Split the string into smaller chunks, and call appropriate
1371
        // append (sync vs async)
1372
        for (i=0; i<s.length(); i=i+MAX_WRITESIZE)
1373 a531ea78 Nitesh Mor
        {
1374 70b43258 Nitesh Mor
                tmp.assign(s, i, MAX_WRITESIZE);
1375 a531ea78 Nitesh Mor
1376
                _l = tmp.length();
1377
                f = m.mutable_fchunk();
1378
1379 ebd8f911 Nitesh Mor
                f->set_offset((uint64_t)filesize_);
1380 a531ea78 Nitesh Mor
                f->set_length((uint32_t)_l);
1381
                f->set_data(tmp);
1382
1383 418bfd3f Nitesh Mor
                m.set_time_ns(TimeNS());
1384 a531ea78 Nitesh Mor
                m.SerializeToString(&a);
1385
                ep_dbg_cprintf(Dbg, 60, "Appending %s\n",
1386
                                                                m.DebugString().c_str());
1387
1388 70b43258 Nitesh Mor
                // actual append call.
1389 e7441468 Nitesh Mor
                AppendRecord(a, async);
1390 a531ea78 Nitesh Mor
1391
                num_appends++;
1392
                if (!async) acks++;
1393
1394 ebd8f911 Nitesh Mor
                filesize_ += _l;
1395 a531ea78 Nitesh Mor
        }
1396
1397 70b43258 Nitesh Mor
        // if we used asynchronous version, fetch the status codes.
1398 a531ea78 Nitesh Mor
        if (async)
1399
        {
1400
                while (acks<num_appends)
1401
                {
1402 ebd8f911 Nitesh Mor
                        ev = gdp_event_next(handle_, NULL);
1403 a531ea78 Nitesh Mor
                        if (gdp_event_gettype(ev) == GDP_EVENT_CREATED ||
1404
                                        gdp_event_gettype(ev) == GDP_EVENT_SUCCESS)
1405
                                acks++;
1406
                        else
1407
                        {
1408
                                ep_dbg_cprintf(Dbg, 2, "Unknown event type\n");
1409 be7fdc95 Nitesh Mor
                                gdp_event_print(ev, stderr);
1410 a531ea78 Nitesh Mor
                                break;
1411
                        }
1412
                }
1413
                assert (acks == num_appends);
1414
        }
1415
}