gdp-if / tensorflow / gdpfs_cmd.cc @ master
History | View | Annotate | Download (20.5 KB)
1 | 13a1f6b0 | Eric Allman | /* vim: set ai sw=4 sts=4 ts=4 :*/
|
---|---|---|---|
2 | |||
3 | a531ea78 | Nitesh Mor | #include <iostream> |
4 | #include <cstdint> |
||
5 | #include <map> |
||
6 | #include <vector> |
||
7 | #include <string> |
||
8 | #include <assert.h> |
||
9 | #include <fstream> |
||
10 | |||
11 | a48a4a87 | Nitesh Mor | #include <unistd.h> |
12 | 71a551f6 | Nitesh Mor | #include <dirent.h> |
13 | #include <libgen.h> |
||
14 | b830903f | Nitesh Mor | #include <limits.h> |
15 | #include <stdlib.h> |
||
16 | 71a551f6 | Nitesh Mor | #include <sys/types.h> |
17 | #include <sys/stat.h> |
||
18 | ec84b225 | Nitesh Mor | #include <fcntl.h> |
19 | 298bdf41 | Nitesh Mor | |
20 | a531ea78 | Nitesh Mor | #include "GDPfs.pb.h" |
21 | aa8aefa0 | Nitesh Mor | #include "gdpfs.h" |
22 | a531ea78 | Nitesh Mor | |
23 | extern "C" { |
||
24 | #include <gdp/gdp.h> |
||
25 | #include <ep/ep_dbg.h> |
||
26 | } |
||
27 | |||
28 | 15ba9a3b | Nitesh Mor | static EP_DBG Dbg = EP_DBG_INIT("gdpfs.util", |
29 | 33678164 | Nitesh Mor | "GDP FileSystem: tests/utilities");
|
30 | a531ea78 | Nitesh Mor | |
31 | // read len bytes to an append-only file, and read
|
||
32 | // them back using a read-only file. Returns if the
|
||
33 | // read data is equal to the data written.
|
||
34 | ff1b1d31 | Nitesh Mor | GdpfsStatus rw_test(const char *logname, size_t len) |
35 | a531ea78 | Nitesh Mor | { |
36 | char *buf;
|
||
37 | std::string sw, sr;
|
||
38 | bool result;
|
||
39 | c4c264f3 | Nitesh Mor | uint64_t t1, t2, t3; |
40 | d96212ef | Nitesh Mor | float t_write, t_read;
|
41 | a531ea78 | Nitesh Mor | |
42 | ep_dbg_cprintf(Dbg, 2, "performing R/W test with " |
||
43 | "logname %s, and length %zu\n",
|
||
44 | logname, len); |
||
45 | |||
46 | buf = new char[len]; |
||
47 | 418bfd3f | Nitesh Mor | RandData(buf, len, true);
|
48 | a531ea78 | Nitesh Mor | sw.assign(buf, len); |
49 | |||
50 | 418bfd3f | Nitesh Mor | t1 = TimeNS(); |
51 | d96212ef | Nitesh Mor | |
52 | a531ea78 | Nitesh Mor | // write data
|
53 | d96212ef | Nitesh Mor | ep_dbg_cprintf(Dbg, 2, "Starting to write data.\n"); |
54 | a531ea78 | Nitesh Mor | GDPFileWO fw(logname); |
55 | 70b43258 | Nitesh Mor | fw.Append(sw, true);
|
56 | a531ea78 | Nitesh Mor | |
57 | 418bfd3f | Nitesh Mor | t2 = TimeNS(); |
58 | d96212ef | Nitesh Mor | |
59 | a531ea78 | Nitesh Mor | // read data
|
60 | d96212ef | Nitesh Mor | ep_dbg_cprintf(Dbg, 2, "Done writing data. Reading now.\n"); |
61 | a531ea78 | Nitesh Mor | GDPFileROMemMap fr(logname, true);
|
62 | sr.assign(fr.data(), fr.length()); |
||
63 | |||
64 | 418bfd3f | Nitesh Mor | t3 = TimeNS(); |
65 | d96212ef | Nitesh Mor | |
66 | // compare data
|
||
67 | ep_dbg_cprintf(Dbg, 2, "Done reading data. Comparing now.\n"); |
||
68 | a531ea78 | Nitesh Mor | result = sw.compare(sr) == 0;
|
69 | ep_dbg_cprintf(Dbg, 2, "R/W test result: %d\n", result); |
||
70 | d96212ef | Nitesh Mor | |
71 | // calculate timing statistics
|
||
72 | c4c264f3 | Nitesh Mor | t_write = (t2-t1)/(1000*1000*1000.0); |
73 | t_read = (t3-t2)/(1000*1000*1000.0); |
||
74 | d96212ef | Nitesh Mor | ep_dbg_cprintf(Dbg, 2, "Writing time: %f; Reading time: %f\n", |
75 | t_write, t_read); |
||
76 | ff1b1d31 | Nitesh Mor | if (result)
|
77 | return kSuccess;
|
||
78 | else
|
||
79 | return kFailure;
|
||
80 | a531ea78 | Nitesh Mor | } |
81 | |||
82 | |||
83 | b6438f0c | Nitesh Mor | /*
|
84 | a531ea78 | Nitesh Mor | |
85 | void write_dir(const char *logname, size_t num_entries,
|
||
86 | char **names, char **logs)
|
||
87 | {
|
||
88 | int i;
|
||
89 | 05e107f7 | Nitesh Mor | GDPDir d(logname, GDP_MODE_RA);
|
90 | a531ea78 | Nitesh Mor | std::string s1, s2;
|
91 | for (i=0; i<num_entries; i++)
|
||
92 | {
|
||
93 | s1.clear();
|
||
94 | s2.clear();
|
||
95 | s1.assign(names[i]);
|
||
96 | s2.assign(logs[i]);
|
||
97 | d.add_file(s1, s2);
|
||
98 | }
|
||
99 | }
|
||
100 | |||
101 | bool dir_test(const char *logname)
|
||
102 | {
|
||
103 | |||
104 | int entries = 100;
|
||
105 | char *namebufs[entries], *logs[entries];
|
||
106 | int i=0;
|
||
107 | |||
108 | for (i=0; i<entries; i++) {
|
||
109 | namebufs[i] = new char[9];
|
||
110 | logs[i] = new char[33];
|
||
111 | 418bfd3f | Nitesh Mor | RandData(namebufs[i], 9);
|
112 | RandData(logs[i], 33);
|
||
113 | a531ea78 | Nitesh Mor | }
|
114 | |||
115 | write_dir(logname, entries, namebufs, logs);
|
||
116 | |||
117 | 05e107f7 | Nitesh Mor | GDPDir d(logname, GDP_MODE_RO);
|
118 | a531ea78 | Nitesh Mor | std::map<std::string, std::string>::iterator it;
|
119 | |||
120 | ebd8f911 | Nitesh Mor | for (it=d.dentries_.begin(); it!= d.dentries_.end(); it++)
|
121 | a531ea78 | Nitesh Mor | {
|
122 | std::cout<< it->first << ": " << it->second << "\n";
|
||
123 | }
|
||
124 | |||
125 | }
|
||
126 | |||
127 | b6438f0c | Nitesh Mor | */
|
128 | a531ea78 | Nitesh Mor | |
129 | 71a551f6 | Nitesh Mor | |
130 | // Simply copy a file from local disk to a given logname.
|
||
131 | // This assumes that the logname already exists (and is setup
|
||
132 | // to be a file) [Initialized with NewFile]
|
||
133 | c3f40cc0 | Nitesh Mor | GdpfsStatus copy_file_to_gdp(const std::string localpath, |
134 | 71a551f6 | Nitesh Mor | const std::string logname) |
135 | { |
||
136 | |||
137 | std::ifstream f(localpath.c_str(), std::ios::binary|std::ios::ate); |
||
138 | d25807f1 | Nitesh Mor | if (f.fail())
|
139 | b830903f | Nitesh Mor | { |
140 | std::cerr<<"Can't open file "<<localpath<<std::endl;
|
||
141 | d25807f1 | Nitesh Mor | return kFailure;
|
142 | b830903f | Nitesh Mor | } |
143 | 71a551f6 | Nitesh Mor | |
144 | d25807f1 | Nitesh Mor | GDPFileWO g(logname.c_str()); |
145 | |||
146 | 5cee3d9f | Eric Allman | f.seekg(0, std::ios::end);
|
147 | d25807f1 | Nitesh Mor | std::streampos size = f.tellg(); |
148 | 71a551f6 | Nitesh Mor | std::vector<char> buffer(size);
|
149 | d25807f1 | Nitesh Mor | |
150 | f.seekg(0, std::ios::beg);
|
||
151 | 71a551f6 | Nitesh Mor | f.read(buffer.data(), size); |
152 | |||
153 | std::string alldata;
|
||
154 | alldata.assign(buffer.data(), size); |
||
155 | g.Append(alldata, true);
|
||
156 | |||
157 | f.close(); |
||
158 | return kSuccess;
|
||
159 | } |
||
160 | |||
161 | c3f40cc0 | Nitesh Mor | // Simply copy a file from a remote log to a local file name
|
162 | GdpfsStatus copy_file_from_gdp(const std::string logname, |
||
163 | const std::string localpath) |
||
164 | { |
||
165 | std::ofstream f(localpath.c_str(), std::ios::binary); |
||
166 | d25807f1 | Nitesh Mor | if (f.fail())
|
167 | b830903f | Nitesh Mor | { |
168 | std::cerr<<"Can't open file "<<localpath<<std::endl;
|
||
169 | d25807f1 | Nitesh Mor | return kFailure;
|
170 | b830903f | Nitesh Mor | } |
171 | c3f40cc0 | Nitesh Mor | |
172 | GDPFileROMemMap g(logname.c_str()); |
||
173 | |||
174 | d25807f1 | Nitesh Mor | f.write(g.data(), g.length()); |
175 | c3f40cc0 | Nitesh Mor | f.close(); |
176 | return kSuccess;
|
||
177 | } |
||
178 | |||
179 | 71a551f6 | Nitesh Mor | // Returns a list of all directories in the vectors "dirs" and
|
180 | // list of all files in "files"
|
||
181 | GdpfsStatus ftw_list_dirs(const std::string path, |
||
182 | std::vector<std::string>*dirs,
|
||
183 | std::vector<std::string>*files)
|
||
184 | { |
||
185 | struct stat dir_stat;
|
||
186 | DIR *dp; |
||
187 | struct dirent *ep;
|
||
188 | |||
189 | stat(path.c_str(), &dir_stat); |
||
190 | if (!S_ISDIR(dir_stat.st_mode))
|
||
191 | { |
||
192 | files->push_back(path); |
||
193 | // early return if this is not a directory.
|
||
194 | return kSuccess;
|
||
195 | } |
||
196 | else
|
||
197 | { |
||
198 | dirs->push_back(path); |
||
199 | } |
||
200 | |||
201 | dp = opendir(path.c_str()); |
||
202 | if (dp == NULL) |
||
203 | return kFailure;
|
||
204 | |||
205 | while (ep = readdir(dp))
|
||
206 | { |
||
207 | if (strcmp(ep->d_name, ".")==0 || strcmp(ep->d_name, "..")==0) |
||
208 | continue;
|
||
209 | |||
210 | std::string newname;
|
||
211 | newname.assign(path); |
||
212 | newname.append("/");
|
||
213 | newname.append(ep->d_name); |
||
214 | |||
215 | // XXX: error checking.
|
||
216 | ftw_list_dirs(newname, dirs, files); |
||
217 | } |
||
218 | |||
219 | // XXX: error checking.
|
||
220 | return kSuccess;
|
||
221 | } |
||
222 | |||
223 | c205187f | Nitesh Mor | // XXX: This needs to be cleaned up. Working with band-aids right now
|
224 | ff1b1d31 | Nitesh Mor | GdpfsStatus copy_to_gdp(const std::string localpath, |
225 | 71a551f6 | Nitesh Mor | const std::string gdppath, bool recursive=false) |
226 | a531ea78 | Nitesh Mor | { |
227 | std::string rootlog, path, logname;
|
||
228 | b830903f | Nitesh Mor | std::string remote, full_remote;
|
229 | std::string full_localpath;
|
||
230 | |||
231 | 6c1b37e9 | Nitesh Mor | assert (ParsePath(gdppath, &rootlog, &path) == kSuccess); |
232 | 05e107f7 | Nitesh Mor | GDPDir root(rootlog.c_str(), GDP_MODE_RA); |
233 | a531ea78 | Nitesh Mor | |
234 | b830903f | Nitesh Mor | char *_tmp = realpath(localpath.c_str(), NULL); |
235 | full_localpath.assign(_tmp); |
||
236 | |||
237 | char *localpath_copy = strdup(full_localpath.c_str());
|
||
238 | char *localpath_basename = basename(localpath_copy);
|
||
239 | char *localpath_dirname = dirname(localpath_copy);
|
||
240 | |||
241 | // make a copy of path into remote
|
||
242 | remote.assign(path); |
||
243 | if (gdppath[gdppath.length()-1] == '/') |
||
244 | { |
||
245 | // remote destination is a directory, since it ends with a '/'.
|
||
246 | // We need to take this into account
|
||
247 | remote.append("/");
|
||
248 | remote.append(localpath_basename); |
||
249 | } |
||
250 | |||
251 | if (root.NameExists(remote))
|
||
252 | 21201b53 | Nitesh Mor | { |
253 | ep_dbg_cprintf(Dbg, 2, "Path exists!!!\n"); |
||
254 | ff1b1d31 | Nitesh Mor | return kFailure;
|
255 | 21201b53 | Nitesh Mor | } |
256 | a531ea78 | Nitesh Mor | |
257 | 71a551f6 | Nitesh Mor | // check if the source is a file or directory
|
258 | struct stat localpath_stat;
|
||
259 | b830903f | Nitesh Mor | stat(full_localpath.c_str(), &localpath_stat); |
260 | a531ea78 | Nitesh Mor | |
261 | 71a551f6 | Nitesh Mor | if (S_ISDIR(localpath_stat.st_mode))
|
262 | { |
||
263 | if (!recursive)
|
||
264 | { |
||
265 | std::cerr<<"Need recursive option for directories"<<std::endl;
|
||
266 | return kFailure;
|
||
267 | } |
||
268 | a531ea78 | Nitesh Mor | |
269 | b830903f | Nitesh Mor | // First, create the directory
|
270 | root.CreateDir(remote); |
||
271 | |||
272 | 71a551f6 | Nitesh Mor | // We can't use the available FTW because it does not
|
273 | // support passing user-supplied data to the callback
|
||
274 | // function. We really need to pass the GDPDir root to
|
||
275 | // such a callback :(
|
||
276 | // Thus, we are stuck with implementing it on our own.
|
||
277 | a531ea78 | Nitesh Mor | |
278 | 71a551f6 | Nitesh Mor | std::vector <std::string> dirlist;
|
279 | std::vector <std::string> filelist;
|
||
280 | std::vector <std::string>::iterator it;
|
||
281 | a531ea78 | Nitesh Mor | |
282 | b830903f | Nitesh Mor | ftw_list_dirs(full_localpath, &dirlist, &filelist); |
283 | 71a551f6 | Nitesh Mor | |
284 | // some debugging output
|
||
285 | if (ep_dbg_test(Dbg, 23)) |
||
286 | { |
||
287 | for (it=dirlist.begin(); it!=dirlist.end(); it++)
|
||
288 | ep_dbg_printf("Directory: %s\n", it->c_str());
|
||
289 | for (it=filelist.begin(); it!=filelist.end(); it++)
|
||
290 | ep_dbg_printf("File: %s\n", it->c_str());
|
||
291 | } |
||
292 | |||
293 | // Create directories
|
||
294 | for (it=dirlist.begin(); it!=dirlist.end(); it++)
|
||
295 | { |
||
296 | b830903f | Nitesh Mor | full_remote.assign(remote); |
297 | // full_remote.append("/");
|
||
298 | full_remote.append(it->substr(full_localpath.length())); |
||
299 | 71a551f6 | Nitesh Mor | ep_dbg_cprintf(Dbg, 11, "Creating directory: %s\n", |
300 | b830903f | Nitesh Mor | full_remote.c_str()); |
301 | root.CreateDir(full_remote); |
||
302 | 71a551f6 | Nitesh Mor | } |
303 | // copy files
|
||
304 | for (it=filelist.begin(); it!=filelist.end(); it++)
|
||
305 | { |
||
306 | b830903f | Nitesh Mor | full_remote.assign(remote); |
307 | // full_remote.append("/");
|
||
308 | full_remote.append(it->substr(full_localpath.length())); |
||
309 | 71a551f6 | Nitesh Mor | ep_dbg_cprintf(Dbg, 11, "Copying file %s to %s\n", |
310 | b830903f | Nitesh Mor | it->c_str(), full_remote.c_str()); |
311 | root.NewFile(full_remote, &logname); |
||
312 | c3f40cc0 | Nitesh Mor | copy_file_to_gdp(*it, logname); |
313 | 71a551f6 | Nitesh Mor | } |
314 | } |
||
315 | else
|
||
316 | { |
||
317 | b830903f | Nitesh Mor | root.NewFile(remote, &logname); |
318 | return copy_file_to_gdp(full_localpath, logname);
|
||
319 | 71a551f6 | Nitesh Mor | } |
320 | a531ea78 | Nitesh Mor | } |
321 | |||
322 | 64bfe00c | Nitesh Mor | |
323 | c205187f | Nitesh Mor | // XXX: This needs to be cleaned up. Working with band-aids right now
|
324 | 64bfe00c | Nitesh Mor | GdpfsStatus copy_from_gdp(const std::string gdppath, |
325 | const std::string localpath, bool recursive=false) |
||
326 | { |
||
327 | std::string rootlog, path, logname;
|
||
328 | a4e78ca7 | Nitesh Mor | std::string basename_, dirname_;
|
329 | b830903f | Nitesh Mor | std::string local, full_local;
|
330 | 64bfe00c | Nitesh Mor | assert (ParsePath(gdppath, &rootlog, &path) == kSuccess); |
331 | GDPDir root(rootlog.c_str(), GDP_MODE_RO); |
||
332 | |||
333 | a4e78ca7 | Nitesh Mor | // we will need this later to figure out the localpath
|
334 | b830903f | Nitesh Mor | BaseDirName(path, &dirname_, &basename_); |
335 | |||
336 | // make a copy of localpath into local
|
||
337 | local.assign(localpath); |
||
338 | if (localpath[localpath.length()-1] == '/') |
||
339 | { |
||
340 | // local destination is a directory, since it ends with a '/'.
|
||
341 | // We need to take this into account
|
||
342 | local.append(basename_); |
||
343 | } |
||
344 | a4e78ca7 | Nitesh Mor | |
345 | 64bfe00c | Nitesh Mor | // check if name exists
|
346 | if (!root.NameExists(path))
|
||
347 | { |
||
348 | std::cerr<<"Doesn't exist"<<std::endl;
|
||
349 | return kFailure;
|
||
350 | } |
||
351 | |||
352 | if (root.GetEntryType(path) == GDPfs::DIR)
|
||
353 | { |
||
354 | |||
355 | if (!recursive)
|
||
356 | { |
||
357 | std::cerr<<"Need recursive option for directories"<<std::endl;
|
||
358 | return kFailure;
|
||
359 | } |
||
360 | |||
361 | b830903f | Nitesh Mor | // Make the local directory first.
|
362 | mkdir(local.c_str(), 0777);
|
||
363 | |||
364 | 64bfe00c | Nitesh Mor | // need to list down all the children. We use GetMatchingPaths
|
365 | // instead of GetChildren as a shortcut.
|
||
366 | std::vector<std::string> children;
|
||
367 | std::vector<std::string>::iterator it;
|
||
368 | |||
369 | root.GetMatchingPaths(path, &children); |
||
370 | for (it=children.begin(); it!=children.end(); it++)
|
||
371 | { |
||
372 | b830903f | Nitesh Mor | full_local.assign(local); |
373 | //full_local.append("/");
|
||
374 | full_local.append(it->substr(path.length())); |
||
375 | ep_dbg_cprintf(Dbg, 10, "Creating %s\n", full_local.c_str()); |
||
376 | 64bfe00c | Nitesh Mor | if (root.GetEntryType(*it) == GDPfs::DIR)
|
377 | { |
||
378 | b830903f | Nitesh Mor | mkdir(full_local.c_str(), 0777);
|
379 | 64bfe00c | Nitesh Mor | } |
380 | else
|
||
381 | { |
||
382 | root.GetEntryLogname(*it, &logname); |
||
383 | b830903f | Nitesh Mor | copy_file_from_gdp(logname, full_local); |
384 | 64bfe00c | Nitesh Mor | } |
385 | } |
||
386 | } |
||
387 | else
|
||
388 | { |
||
389 | root.GetEntryLogname(path, &logname); |
||
390 | b830903f | Nitesh Mor | return copy_file_from_gdp(logname, local);
|
391 | 64bfe00c | Nitesh Mor | } |
392 | |||
393 | a531ea78 | Nitesh Mor | } |
394 | |||
395 | c205187f | Nitesh Mor | // XXX: This needs to be cleaned up. Working with band-aids right now
|
396 | d63aa760 | Nitesh Mor | GdpfsStatus ls(const std::string gdpdir, bool fancy=false) |
397 | a531ea78 | Nitesh Mor | { |
398 | std::string rootlog, path;
|
||
399 | 6c1b37e9 | Nitesh Mor | assert (ParsePath(gdpdir, &rootlog, &path) == kSuccess); |
400 | 0277d6eb | Nitesh Mor | |
401 | 05e107f7 | Nitesh Mor | GDPDir root(rootlog.c_str(), GDP_MODE_RO); |
402 | 0277d6eb | Nitesh Mor | |
403 | 7a3ab78f | Nitesh Mor | std::vector<std::string> children;
|
404 | std::vector<std::string>::iterator it;
|
||
405 | 596cfe85 | Nitesh Mor | std::string logname, tmp;
|
406 | |||
407 | if (path.size()>0 && !root.NameExists(path)) |
||
408 | { |
||
409 | std::cout<<"No such file"<<std::endl;
|
||
410 | return kInvalidArg;
|
||
411 | } |
||
412 | 7a3ab78f | Nitesh Mor | |
413 | root.GetMatchingPaths(path, &children); |
||
414 | for (it = children.begin(); it != children.end(); it++)
|
||
415 | a531ea78 | Nitesh Mor | { |
416 | 7a3ab78f | Nitesh Mor | root.GetEntryLogname(*it, &logname); |
417 | 596cfe85 | Nitesh Mor | tmp = *it; |
418 | |||
419 | // make sure we really have the correct stuff to show
|
||
420 | assert (tmp.find(path) == 0);
|
||
421 | |||
422 | if (children.size()>1) |
||
423 | { |
||
424 | // now remove the initial parts of our result. This is
|
||
425 | // to make sure if the user asks `ls gdp://X/1/2`, then
|
||
426 | // we don't show them 1/2/3/4, but 3/4 instead.
|
||
427 | // This behavior is somewhat similar to unix `ls`.
|
||
428 | tmp.erase(0, path.size());
|
||
429 | } |
||
430 | |||
431 | 7a3ab78f | Nitesh Mor | if (logname.empty())
|
432 | d63aa760 | Nitesh Mor | { |
433 | 7a3ab78f | Nitesh Mor | logname.assign("[DIRECTORY]");
|
434 | 596cfe85 | Nitesh Mor | tmp.append("/");
|
435 | d63aa760 | Nitesh Mor | } |
436 | |||
437 | if (fancy)
|
||
438 | 596cfe85 | Nitesh Mor | std::cout<< "[*] " << tmp << " => " << logname << std::endl; |
439 | d63aa760 | Nitesh Mor | else
|
440 | 596cfe85 | Nitesh Mor | std::cout<< tmp << std::endl; |
441 | 7a3ab78f | Nitesh Mor | } |
442 | ff1b1d31 | Nitesh Mor | return kSuccess;
|
443 | a531ea78 | Nitesh Mor | } |
444 | |||
445 | |||
446 | c205187f | Nitesh Mor | // XXX: This needs to be cleaned up. Working with band-aids right now
|
447 | 71a551f6 | Nitesh Mor | GdpfsStatus mkdir_(const std::string gdpdir) |
448 | 272b17c9 | Nitesh Mor | { |
449 | |||
450 | std::string rootlog, path;
|
||
451 | 6c1b37e9 | Nitesh Mor | assert (ParsePath(gdpdir, &rootlog, &path) == kSuccess); |
452 | 272b17c9 | Nitesh Mor | |
453 | assert (path.length() != 0);
|
||
454 | GDPDir root(rootlog.c_str(), GDP_MODE_RA); |
||
455 | |||
456 | 2651f137 | Nitesh Mor | root.CreateDir(path); |
457 | ff1b1d31 | Nitesh Mor | return kSuccess;
|
458 | 272b17c9 | Nitesh Mor | } |
459 | |||
460 | c205187f | Nitesh Mor | // XXX: This needs to be cleaned up. Working with band-aids right now
|
461 | 298bdf41 | Nitesh Mor | GdpfsStatus rm(const std::string gdppath, bool recursive=false) |
462 | b7971154 | Nitesh Mor | { |
463 | |||
464 | std::string rootlog, path;
|
||
465 | assert (ParsePath(gdppath, &rootlog, &path) == kSuccess); |
||
466 | |||
467 | assert (path.length() != 0); // can't delete topdirectory |
||
468 | GDPDir root(rootlog.c_str(), GDP_MODE_RA); |
||
469 | |||
470 | // check if name exists
|
||
471 | if (!root.NameExists(path))
|
||
472 | { |
||
473 | std::cerr<<"Doesn't exist"<<std::endl;
|
||
474 | ff1b1d31 | Nitesh Mor | return kFailure;
|
475 | b7971154 | Nitesh Mor | } |
476 | |||
477 | // make sure we aren't deleting non-empty directories, even
|
||
478 | // though the concept of a directory is fuzzy.
|
||
479 | if (root.GetEntryType(path) == GDPfs::DIR)
|
||
480 | { |
||
481 | std::vector<std::string> children;
|
||
482 | root.GetChildren(path, &children); |
||
483 | 298bdf41 | Nitesh Mor | if (children.size() != 0 && !recursive) |
484 | b7971154 | Nitesh Mor | { |
485 | std::cerr<<"Can't delete non-empty directory"<<std::endl;
|
||
486 | ff1b1d31 | Nitesh Mor | return kFailure;
|
487 | b7971154 | Nitesh Mor | } |
488 | |||
489 | 298bdf41 | Nitesh Mor | if (!recursive)
|
490 | { |
||
491 | ep_dbg_cprintf(Dbg, 10, "Deleting %s\n", path.c_str()); |
||
492 | root.DeleteDir(path); |
||
493 | } |
||
494 | else
|
||
495 | { |
||
496 | // need to list down all the children. We use GetMatchingPaths
|
||
497 | // instead of GetChildren as a shortcut.
|
||
498 | std::vector<std::string> children;
|
||
499 | std::vector<std::string>::iterator it;
|
||
500 | |||
501 | root.GetMatchingPaths(path, &children); |
||
502 | for (it=children.begin(); it!=children.end(); it++)
|
||
503 | { |
||
504 | ep_dbg_cprintf(Dbg, 10, "Deleting %s\n", it->c_str()); |
||
505 | if (root.GetEntryType(*it) == GDPfs::DIR)
|
||
506 | root.DeleteDir(*it); |
||
507 | else
|
||
508 | root.DeleteFile(*it); |
||
509 | } |
||
510 | } |
||
511 | b7971154 | Nitesh Mor | } |
512 | else
|
||
513 | { |
||
514 | root.DeleteFile(path); |
||
515 | } |
||
516 | ff1b1d31 | Nitesh Mor | |
517 | // FIXME check return status of Delete*
|
||
518 | return kSuccess;
|
||
519 | b7971154 | Nitesh Mor | } |
520 | |||
521 | ec84b225 | Nitesh Mor | // creates a signature key and stores the private part in the
|
522 | // given filename. Note that the key is not protected at all.
|
||
523 | // Thus, one must be careful in making sure that the appropriate
|
||
524 | // file system permissions are set
|
||
525 | GdpfsStatus keygen(const std::string filename) |
||
526 | { |
||
527 | |||
528 | EP_CRYPTO_KEY *key; |
||
529 | int keytype;
|
||
530 | const char *curve; |
||
531 | EP_STAT estat; |
||
532 | |||
533 | keytype = ep_crypto_keytype_byname("ec");
|
||
534 | curve = ep_adm_getstrparam("swarm.gdp.crypto.ec.curve", NULL); |
||
535 | |||
536 | key = ep_crypto_key_create(keytype, 0, 0, curve); |
||
537 | if (key == NULL) |
||
538 | { |
||
539 | ep_app_error("Could not create new key");
|
||
540 | return kFailure;
|
||
541 | } |
||
542 | |||
543 | |||
544 | int fd;
|
||
545 | FILE *fp; |
||
546 | |||
547 | if ((fd = open(filename.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0400)) < 0 || |
||
548 | (fp = fdopen(fd, "w")) == NULL) |
||
549 | { |
||
550 | ep_app_error("Cannot create %s", filename.c_str());
|
||
551 | return kFailure;
|
||
552 | } |
||
553 | |||
554 | estat = ep_crypto_key_write_fp(key, fp, EP_CRYPTO_KEYFORM_PEM, |
||
555 | 0, NULL, EP_CRYPTO_F_SECRET); |
||
556 | |||
557 | // TODO: should really clear the fp buffer memory here to
|
||
558 | // avoid exposing any secret key information after free
|
||
559 | fclose(fp); |
||
560 | } |
||
561 | 298bdf41 | Nitesh Mor | |
562 | 13a1f6b0 | Eric Allman | // initializes the "root directory" in the file system implemented
|
563 | // in the GDP.
|
||
564 | GdpfsStatus gdpfs_mkfs(const std::string rootdir) |
||
565 | { |
||
566 | std::string rootlog, path;
|
||
567 | GdpfsStatus gstat; |
||
568 | |||
569 | assert (ParsePath(rootdir, &rootlog, &path) == kSuccess); |
||
570 | assert (path.length() == 0);
|
||
571 | |||
572 | gstat = GDPfsMkfs(rootlog); |
||
573 | return gstat;
|
||
574 | } |
||
575 | |||
576 | 298bdf41 | Nitesh Mor | |
577 | |||
578 | 03752acd | Nitesh Mor | /////////////////// argument processing and handling below ///////////////
|
579 | |||
580 | b7971154 | Nitesh Mor | |
581 | 3d594591 | Nitesh Mor | // returns the usage string
|
582 | std::string usage_string()
|
||
583 | { |
||
584 | cab16857 | Nitesh Mor | std::string s = "Basic Usage: gdpfs_cmd [-D dbgspec] <cmd> [<args>]\n" |
585 | "\n"
|
||
586 | "Available commands are:\n"
|
||
587 | d63aa760 | Nitesh Mor | " ls List file/dirs with extended info. Use: `ls <GDP-path>`\n"
|
588 | cab16857 | Nitesh Mor | " mkdir Make dirs. Use: `mkdir <GDP-directory>`\n"
|
589 | " to_gdp Copy files to GDP. Use: `to_gdp <local-path> <GDP-path>`\n"
|
||
590 | " from_gdp Copy file from GDP. Use: `from_gdp <GDP-path> <local-path>`\n"
|
||
591 | " rm Delete file/dir. Use: `rm <GDP-path>\n"
|
||
592 | ec84b225 | Nitesh Mor | " keygen Create a signature key. Use: `keygen <output-file>\n"
|
593 | 3d594591 | Nitesh Mor | "\n"
|
594 | 7a3ab78f | Nitesh Mor | "GDP-path: a full filepath that starts with 'gdp://'";
|
595 | 3d594591 | Nitesh Mor | |
596 | return s;
|
||
597 | } |
||
598 | |||
599 | a531ea78 | Nitesh Mor | |
600 | 03752acd | Nitesh Mor | std::string usage_string_ls()
|
601 | { |
||
602 | std::string s = "Basic Usage: gdpfs_cmd [-D dbgspec] ls [OPTION]... PATH\n" |
||
603 | "\n"
|
||
604 | "Available options are:\n"
|
||
605 | " -1 List files for programmatic use and supress extended info.\n"
|
||
606 | " -h Show this message.\n"
|
||
607 | "\n"
|
||
608 | "PATH: a full filepath that starts with 'gdp://'";
|
||
609 | |||
610 | return s;
|
||
611 | } |
||
612 | |||
613 | 298bdf41 | Nitesh Mor | std::string usage_string_rm()
|
614 | { |
||
615 | std::string s = "Basic Usage: gdpfs_cmd [-D dbgspec] rm [OPTION]... PATH\n" |
||
616 | "\n"
|
||
617 | "Available options are:\n"
|
||
618 | " -r Delete recursively (only makes sense for directories).\n"
|
||
619 | " -h Show this message.\n"
|
||
620 | "\n"
|
||
621 | "PATH: a full filepath that starts with 'gdp://'";
|
||
622 | |||
623 | return s;
|
||
624 | } |
||
625 | |||
626 | 64bfe00c | Nitesh Mor | std::string usage_string_from_gdp()
|
627 | { |
||
628 | std::string s = "Basic Usage: gdpfs_cmd [-D dbgspec] " |
||
629 | "from_gdp [OPTION]... PATH LOCALPATH\n"
|
||
630 | "\n"
|
||
631 | "Available options are:\n"
|
||
632 | " -r Copy recursively (only makes sense for directories).\n"
|
||
633 | " -h Show this message.\n"
|
||
634 | "\n"
|
||
635 | "PATH: a full filepath that starts with 'gdp://'\n"
|
||
636 | "LOCALPATH: a local filesystem path.";
|
||
637 | |||
638 | return s;
|
||
639 | } |
||
640 | 298bdf41 | Nitesh Mor | |
641 | 71a551f6 | Nitesh Mor | std::string usage_string_to_gdp()
|
642 | { |
||
643 | std::string s = "Basic Usage: gdpfs_cmd [-D dbgspec] " |
||
644 | "to_gdp [OPTION]... LOCALPATH PATH\n"
|
||
645 | "\n"
|
||
646 | "Available options are:\n"
|
||
647 | " -r Copy recursively (only makes sense for directories).\n"
|
||
648 | " -h Show this message.\n"
|
||
649 | "\n"
|
||
650 | "PATH: a full filepath that starts with 'gdp://'\n"
|
||
651 | "LOCALPATH: a local filesystem path.";
|
||
652 | |||
653 | return s;
|
||
654 | } |
||
655 | |||
656 | ec84b225 | Nitesh Mor | std::string usage_string_keygen()
|
657 | { |
||
658 | std::string s = "Basic Usage: gdpfs_cmd [-D dbgspec] keygen LOCALPATH\n" |
||
659 | "\n"
|
||
660 | "Available options are:\n"
|
||
661 | " -h Show this message.\n"
|
||
662 | "\n"
|
||
663 | "LOCALPATH: a local filename where the *private* key should be stored";
|
||
664 | |||
665 | return s;
|
||
666 | } |
||
667 | |||
668 | 13a1f6b0 | Eric Allman | std::string usage_string_mkfs()
|
669 | { |
||
670 | std::string s = "Basic Usage: gdpfs_cmd [-D dbgspec] mkfs ROOTDIR\n" |
||
671 | "\n"
|
||
672 | "ROOTDIR: the name of the GDP-based root directory.";
|
||
673 | |||
674 | return s;
|
||
675 | } |
||
676 | |||
677 | 71a551f6 | Nitesh Mor | |
678 | ////////////////// main function below /////////////////////////
|
||
679 | |||
680 | a531ea78 | Nitesh Mor | int main(int argc, char **argv) |
681 | { |
||
682 | |||
683 | a231577b | Nitesh Mor | int opt;
|
684 | bool show_usage = false; |
||
685 | 7a3ab78f | Nitesh Mor | char *debug_setting = (char*) "*=0"; |
686 | a231577b | Nitesh Mor | |
687 | 03752acd | Nitesh Mor | while ((opt = getopt(argc, argv, "+D:")) > 0) |
688 | a231577b | Nitesh Mor | { |
689 | switch (opt)
|
||
690 | { |
||
691 | case 'D': |
||
692 | debug_setting = optarg; |
||
693 | break;
|
||
694 | |||
695 | default:
|
||
696 | show_usage = true;
|
||
697 | break;
|
||
698 | } |
||
699 | } |
||
700 | |||
701 | argc -= optind; |
||
702 | argv += optind; |
||
703 | |||
704 | if (show_usage || argc == 0) |
||
705 | be7fdc95 | Nitesh Mor | { |
706 | 3d594591 | Nitesh Mor | std::cerr<<usage_string()<<std::endl; |
707 | ff1b1d31 | Nitesh Mor | return (int) kInvalidArg; |
708 | be7fdc95 | Nitesh Mor | } |
709 | |||
710 | 39672fc0 | Eric Allman | EP_STAT estat = gdp_init(NULL);
|
711 | if (!EP_STAT_ISOK(estat))
|
||
712 | { |
||
713 | char ebuf[100]; |
||
714 | ep_app_fatal("gdp_init failed: %s",
|
||
715 | ep_stat_tostr(estat, ebuf, sizeof ebuf));
|
||
716 | } |
||
717 | |||
718 | // read the per-app configuration files.
|
||
719 | ep_adm_readparams("gdpfs");
|
||
720 | |||
721 | // set the debugging parameters for the GDP library
|
||
722 | // as well as for the gdpfs library. Config file settings
|
||
723 | // can be overridden by debug_setting.
|
||
724 | if (debug_setting == NULL) |
||
725 | { |
||
726 | ep_dbg_set(ep_adm_getstrparam("swarm.gdp.debug", NULL)); |
||
727 | ep_dbg_set(ep_adm_getstrparam("swarm.gdpfs.debug", NULL)); |
||
728 | } |
||
729 | else
|
||
730 | { |
||
731 | ep_dbg_set(debug_setting); |
||
732 | } |
||
733 | |||
734 | a231577b | Nitesh Mor | if (strcmp(argv[0], "to_gdp")==0) |
735 | e04c0ee8 | Nitesh Mor | { |
736 | 71a551f6 | Nitesh Mor | bool recursive = false; |
737 | bool show_usage_local = false; |
||
738 | optind = 0;
|
||
739 | |||
740 | while ((opt = getopt(argc, argv, "+rh")) > 0) |
||
741 | { |
||
742 | switch (opt)
|
||
743 | { |
||
744 | case 'r': |
||
745 | recursive = true;
|
||
746 | break;
|
||
747 | case 'h': |
||
748 | default:
|
||
749 | show_usage_local = true;
|
||
750 | } |
||
751 | } |
||
752 | |||
753 | argc -= optind; |
||
754 | argv += optind; |
||
755 | |||
756 | if (show_usage_local || argc == 0) |
||
757 | { |
||
758 | std::cerr<<usage_string_to_gdp()<<std::endl; |
||
759 | return (int) kInvalidArg; |
||
760 | } |
||
761 | |||
762 | a231577b | Nitesh Mor | GDPfsInit(GDP_MODE_RA, debug_setting); |
763 | 71a551f6 | Nitesh Mor | return (int) copy_to_gdp(argv[0], argv[1], recursive); |
764 | e04c0ee8 | Nitesh Mor | } |
765 | |||
766 | a231577b | Nitesh Mor | if (strcmp(argv[0], "from_gdp")==0) |
767 | e04c0ee8 | Nitesh Mor | { |
768 | 64bfe00c | Nitesh Mor | bool recursive = false; |
769 | bool show_usage_local = false; |
||
770 | optind = 0;
|
||
771 | |||
772 | while ((opt = getopt(argc, argv, "+rh")) > 0) |
||
773 | { |
||
774 | switch (opt)
|
||
775 | { |
||
776 | case 'r': |
||
777 | recursive = true;
|
||
778 | break;
|
||
779 | case 'h': |
||
780 | default:
|
||
781 | show_usage_local = true;
|
||
782 | } |
||
783 | } |
||
784 | |||
785 | argc -= optind; |
||
786 | argv += optind; |
||
787 | |||
788 | if (show_usage_local || argc == 0) |
||
789 | { |
||
790 | std::cerr<<usage_string_from_gdp()<<std::endl; |
||
791 | return (int) kInvalidArg; |
||
792 | } |
||
793 | |||
794 | a231577b | Nitesh Mor | GDPfsInit(GDP_MODE_RO, debug_setting); |
795 | 64bfe00c | Nitesh Mor | return (int) copy_from_gdp(argv[0], argv[1], recursive); |
796 | e04c0ee8 | Nitesh Mor | } |
797 | a531ea78 | Nitesh Mor | |
798 | a231577b | Nitesh Mor | if (strcmp(argv[0], "ls")==0) |
799 | e04c0ee8 | Nitesh Mor | { |
800 | 03752acd | Nitesh Mor | bool fancy = true; |
801 | bool show_usage_local = false; |
||
802 | a48a4a87 | Nitesh Mor | optind = 0;
|
803 | 03752acd | Nitesh Mor | |
804 | a48a4a87 | Nitesh Mor | while ((opt = getopt(argc, argv, "+1h")) > 0) |
805 | { |
||
806 | 03752acd | Nitesh Mor | switch (opt)
|
807 | { |
||
808 | case '1': |
||
809 | fancy = false;
|
||
810 | break;
|
||
811 | case 'h': |
||
812 | default:
|
||
813 | show_usage_local = true;
|
||
814 | } |
||
815 | } |
||
816 | |||
817 | argc -= optind; |
||
818 | argv += optind; |
||
819 | |||
820 | if (show_usage_local || argc == 0) |
||
821 | { |
||
822 | std::cerr<<usage_string_ls()<<std::endl; |
||
823 | return (int) kInvalidArg; |
||
824 | } |
||
825 | d63aa760 | Nitesh Mor | |
826 | GDPfsInit(GDP_MODE_RO, debug_setting); |
||
827 | 03752acd | Nitesh Mor | return (int) ls(argv[0], fancy); |
828 | e04c0ee8 | Nitesh Mor | } |
829 | a531ea78 | Nitesh Mor | |
830 | a231577b | Nitesh Mor | if (strcmp(argv[0], "mkdir")==0) |
831 | 272b17c9 | Nitesh Mor | { |
832 | a231577b | Nitesh Mor | GDPfsInit(GDP_MODE_RA, debug_setting); |
833 | 71a551f6 | Nitesh Mor | return (int) mkdir_(argv[1]); |
834 | 272b17c9 | Nitesh Mor | } |
835 | |||
836 | b7971154 | Nitesh Mor | if (strcmp(argv[0], "rm") == 0) |
837 | { |
||
838 | 298bdf41 | Nitesh Mor | bool recursive = false; |
839 | bool show_usage_local = false; |
||
840 | a48a4a87 | Nitesh Mor | optind = 0;
|
841 | |||
842 | while ((opt = getopt(argc, argv, "+rh")) > 0) |
||
843 | 298bdf41 | Nitesh Mor | { |
844 | |||
845 | switch (opt)
|
||
846 | { |
||
847 | case 'r': |
||
848 | recursive = true;
|
||
849 | break;
|
||
850 | case 'h': |
||
851 | default:
|
||
852 | show_usage_local = true;
|
||
853 | } |
||
854 | } |
||
855 | |||
856 | argc -= optind; |
||
857 | argv += optind; |
||
858 | |||
859 | if (show_usage_local || argc == 0) |
||
860 | { |
||
861 | std::cerr<<usage_string_rm()<<std::endl; |
||
862 | return (int) kInvalidArg; |
||
863 | } |
||
864 | |||
865 | b7971154 | Nitesh Mor | GDPfsInit(GDP_MODE_RA, debug_setting); |
866 | 298bdf41 | Nitesh Mor | return (int) rm(argv[0], recursive); |
867 | b7971154 | Nitesh Mor | } |
868 | |||
869 | ec84b225 | Nitesh Mor | if (strcmp(argv[0], "keygen")==0) |
870 | { |
||
871 | bool show_usage_local = false; |
||
872 | optind = 0;
|
||
873 | |||
874 | while ((opt = getopt(argc, argv, "+h")) > 0) |
||
875 | { |
||
876 | switch (opt)
|
||
877 | { |
||
878 | case 'h': |
||
879 | default:
|
||
880 | show_usage_local = true;
|
||
881 | } |
||
882 | } |
||
883 | |||
884 | argc -= optind; |
||
885 | argv += optind; |
||
886 | |||
887 | if (show_usage_local || argc == 0) |
||
888 | { |
||
889 | std::cerr<<usage_string_keygen()<<std::endl; |
||
890 | return (int) kInvalidArg; |
||
891 | } |
||
892 | |||
893 | GDPfsInit(GDP_MODE_RA, debug_setting); |
||
894 | return (int) keygen(argv[0]); |
||
895 | } |
||
896 | |||
897 | a231577b | Nitesh Mor | if (strcmp(argv[0], "io_test")==0) |
898 | d96212ef | Nitesh Mor | { |
899 | a231577b | Nitesh Mor | // this is intentionally undocumented
|
900 | GDPfsInit(GDP_MODE_RA, debug_setting); |
||
901 | int len = atoi(argv[1]); |
||
902 | ff1b1d31 | Nitesh Mor | GdpfsStatus res = rw_test(argv[2], len*1024*1024); |
903 | return (int) res; |
||
904 | d96212ef | Nitesh Mor | } |
905 | |||
906 | 13a1f6b0 | Eric Allman | if (strcmp(argv[0], "mkfs")==0) |
907 | { |
||
908 | bool show_usage_local = false; |
||
909 | optind = 0;
|
||
910 | |||
911 | GDPfsInit(GDP_MODE_RA, debug_setting); |
||
912 | while ((opt = getopt(argc, argv, "+h")) > 0) |
||
913 | { |
||
914 | switch (opt)
|
||
915 | { |
||
916 | case 'h': |
||
917 | default:
|
||
918 | show_usage_local = true;
|
||
919 | } |
||
920 | } |
||
921 | |||
922 | argc -= optind; |
||
923 | argv += optind; |
||
924 | |||
925 | if (show_usage_local || argc != 1) |
||
926 | { |
||
927 | std::cerr<<usage_string_mkfs()<<std::endl; |
||
928 | return (int) kInvalidArg; |
||
929 | } |
||
930 | |||
931 | ep_dbg_cprintf(Dbg, 10, "mkfs(%s)\n", argv[0]); |
||
932 | return (int) gdpfs_mkfs(argv[0]); |
||
933 | } |
||
934 | |||
935 | d63aa760 | Nitesh Mor | // if we reach here, that's a bad sign
|
936 | std::cerr<<usage_string()<<std::endl; |
||
937 | return (int) kInvalidArg; |
||
938 | a531ea78 | Nitesh Mor | } |