Project

General

Profile

Statistics
| Branch: | Revision:

gdp-router-click / src / gdp_v4_router.cc @ master

History | View | Annotate | Download (210 KB)

1
/*
2
**        GDPv4Router --- Click-based GDP v4 router implementation
3
**
4
**        ----- BEGIN LICENSE BLOCK -----
5
**        GDPv4Router --- Click-based GDP v4 router implementation
6
**  From the Ubiquitous Swarm Lab, 490 Cory Hall, U.C. Berkeley.
7
**
8
**        Copyright (c) 2017-2019, Regents of the University of California.
9
**        All rights reserved.
10
**
11
**        Permission is hereby granted, without written agreement and without
12
**        license or royalty fees, to use, copy, modify, and distribute this
13
**        software and its documentation for any purpose, provided that the above
14
**        copyright notice and the following two paragraphs appear in all copies
15
**        of this software.
16
**
17
**        IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
18
**        SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST
19
**        PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION,
20
**        EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
21
**
22
**        REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
23
**        LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24
**        FOR A PARTICULAR PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION,
25
**        IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO
26
**        OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS,
27
**        OR MODIFICATIONS.
28
**        ----- END LICENSE BLOCK -----
29
*/
30
#include <click/config.h>
31
#include <click/error.hh>
32
#include <click/args.hh>
33
#include <click/glue.hh>
34
#include <click/standard/scheduleinfo.hh>
35
#include <click/packet_anno.hh>
36
#include <click/packet.hh>
37
#include <click/router.hh>
38
#include <unistd.h>
39
#include <sys/ioctl.h>
40
#include <sys/socket.h>
41
#include <sys/un.h>
42
#include <arpa/inet.h>
43
#include <netinet/tcp.h>
44
#include <fcntl.h>
45
#include <functional>
46
#include <chrono>
47
#include <thread>
48
#include "version_gdp_v4_router.hh"
49
#include "gdp_v4_router.hh"
50

    
51
extern "C" {
52
# include <openssl/err.h>
53
# include <netinet/in.h>
54
# include <arpa/nameser.h>
55
# include <resolv.h>
56
# include <systemd/sd-daemon.h>
57

    
58
// ============================================================================
59
// library reduction by duplicating a little libgdp and libep C code
60
// ============================================================================
61

    
62
# define EP_UT_BITSET(bit, word)        (((bit) & (word)) != 0)
63
# define EP_B64_PAD 0x04 // pad with '='
64
# define EP_B64_ENC_URL "-_@" // NOWRAP -PAD -IGNCRUD
65

    
66
        static const char *EncChars =
67
                "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
68

    
69
        static char
70
        getenc (int b, const char *encoding)
71
        {
72
                assert(b >= 0 && b < 64);
73

    
74
                if (b == 62)
75
                        return encoding[0];
76
                if (b == 63)
77
                        return encoding[1];
78
                return EncChars[b];
79
        }
80

    
81
        // based on ep_b64_encode with router specific simplifications
82
        char *
83
        gdp_printable_name (const void *bbin, char *txt)
84
        {
85
                size_t bsize = sizeof(gdp_name_t);
86
                const char *encoding = EP_B64_ENC_URL;
87
                const uint8_t *bin = (uint8_t *) bbin;
88
                size_t bx, tx;                        // indexes into binary & text
89
                size_t lx;                        // index into current output line
90
                int nextc = 0;
91

    
92
                for (bx = tx = lx = 0; bx < bsize;)
93
                {
94
                        switch (bx % 3)
95
                        {
96
                        case 0:
97
                                txt[tx++] = getenc(bin[bx] >> 2, encoding);
98
                                lx++;
99
                                nextc = (bin[bx++] & 0x03) << 4;
100
                                break;
101

    
102
                        case 1:
103
                                nextc |= (bin[bx] & 0xf0) >> 4;
104
                                txt[tx++] = getenc(nextc, encoding);
105
                                lx++;
106
                                nextc = (bin[bx++] & 0x0f) << 2;
107
                                break;
108

    
109
                        case 2:
110
                                nextc |= (bin[bx] & 0xc0) >> 6;
111
                                txt[tx++] = getenc(nextc, encoding);
112
                                txt[tx++] = getenc(bin[bx++] & 0x3f, encoding);
113
                                lx += 2;
114
                                break;
115
                        }
116
                }
117

    
118
                // insert final output character
119
                switch (bx % 3)
120
                {
121
                case 0:
122
                        // no additional data to push
123
                        break;
124

    
125
                case 1:
126
                        txt[tx++] = getenc(nextc, encoding);
127
                        if (EP_UT_BITSET(EP_B64_PAD, encoding[2]))
128
                        {
129
                                txt[tx++] = '=';
130
                                txt[tx++] = '=';
131
                        }
132
                        break;
133

    
134
                case 2:
135
                        txt[tx++] = getenc(nextc, encoding);
136
                        if (EP_UT_BITSET(EP_B64_PAD, encoding[2]))
137
                                txt[tx++] = '=';
138
                        break;
139
                }
140

    
141
                // success!
142
                txt[tx] = '\0';
143
                return txt;
144
        }
145

    
146
// ============================================================================
147

    
148
} // extern "C"
149

    
150
CLICK_DECLS
151

    
152
// UNIT TESTS
153
//#define TEST_ADDR_FIND_RETRIES
154
//#define TEST_DTLS_RX_DROP
155
//#define TEST_ADVERT_LOAD
156

    
157
// TEST_ADDR_FIND_RETRIES variations
158
#define TEST_ADDR_FIND_RETRIES_TX (ADDR_FIND_RETRIES / 2)
159

    
160
// TEST_DTLS_RX_DROP default
161
#define TEST_DTLS_RX_DROP_COUNT 1
162

    
163
// TEST_DTLS_RX_DROP variations (uncomment exactly one TEST_DTLS_RX_DROP_NN)
164
#ifdef TEST_DTLS_RX_DROP
165
//#define TEST_DTLS_RX_DROP_01
166
//#define TEST_DTLS_RX_DROP_02
167
//#define TEST_DTLS_RX_DROP_03
168
//#define TEST_DTLS_RX_DROP_04
169
//#define TEST_DTLS_RX_DROP_05
170
//#define TEST_DTLS_RX_DROP_06
171
//#define TEST_DTLS_RX_DROP_07
172
//#define TEST_DTLS_RX_DROP_08
173
//#define TEST_DTLS_RX_DROP_09
174
//#define TEST_DTLS_RX_DROP_10
175
//#define TEST_DTLS_RX_DROP_11
176
#define TEST_DTLS_RX_DROP_12
177
#endif
178

    
179
#ifdef TEST_DTLS_RX_DROP_01
180
// drops 1st p (test start sequence nack)
181
#define TEST_DTLS_RX_DROP_ITEM 0
182
#define TEST_DTLS_RX_DROP_WRAP 1024
183
#endif
184
#ifdef TEST_DTLS_RX_DROP_02
185
// drops 2nd p (test not start sequence nack)
186
#define TEST_DTLS_RX_DROP_ITEM 1
187
#define TEST_DTLS_RX_DROP_WRAP 1024
188
#endif
189
#ifdef TEST_DTLS_RX_DROP_03
190
// drop 1/2 from 1st p
191
#define TEST_DTLS_RX_DROP_ITEM 0
192
#define TEST_DTLS_RX_DROP_WRAP 2
193
#endif
194
#ifdef TEST_DTLS_RX_DROP_04
195
// drop 1/2 from 2nd p
196
#define TEST_DTLS_RX_DROP_ITEM 1
197
#define TEST_DTLS_RX_DROP_WRAP 2
198
#endif
199
#ifdef TEST_DTLS_RX_DROP_05
200
// drop 1/3 from 1st p
201
#define TEST_DTLS_RX_DROP_ITEM 0
202
#define TEST_DTLS_RX_DROP_WRAP 3
203
#endif
204
#ifdef TEST_DTLS_RX_DROP_06
205
// drop 1/4 from 1st p
206
#define TEST_DTLS_RX_DROP_ITEM 0
207
#define TEST_DTLS_RX_DROP_WRAP 4
208
#endif
209
#ifdef TEST_DTLS_RX_DROP_07
210
// drop 1/5 from 1st p
211
#define TEST_DTLS_RX_DROP_ITEM 0
212
#define TEST_DTLS_RX_DROP_WRAP 5
213
#endif
214
#ifdef TEST_DTLS_RX_DROP_08
215
// drop 1/10 from 1st p
216
#define TEST_DTLS_RX_DROP_ITEM 0
217
#define TEST_DTLS_RX_DROP_WRAP 10
218
#endif
219
#ifdef TEST_DTLS_RX_DROP_09
220
// drop 1/20 from 1st p
221
#define TEST_DTLS_RX_DROP_ITEM 0
222
#define TEST_DTLS_RX_DROP_WRAP 20
223
#endif
224
#ifdef TEST_DTLS_RX_DROP_10
225
// drop 1/50 from 1st p
226
#define TEST_DTLS_RX_DROP_ITEM 0
227
#define TEST_DTLS_RX_DROP_WRAP 50
228
#endif
229
#ifdef TEST_DTLS_RX_DROP_11
230
// drop 1/100 from 1st p
231
#define TEST_DTLS_RX_DROP_ITEM 0
232
#define TEST_DTLS_RX_DROP_WRAP 100
233
#endif
234
#ifdef TEST_DTLS_RX_DROP_12
235
// drop count/100 from item p
236
//#define TEST_DTLS_RX_DROP_ITEM 8
237
//#define TEST_DTLS_RX_DROP_WRAP 24
238
#define TEST_DTLS_RX_DROP_ITEM 10
239
#define TEST_DTLS_RX_DROP_WRAP 1000
240
// TEST_DTLS_RX_DROP default override
241
#undef TEST_DTLS_RX_DROP_COUNT
242
//#define TEST_DTLS_RX_DROP_COUNT 3
243
#define TEST_DTLS_RX_DROP_COUNT 4
244
//#define TEST_DTLS_RX_DROP_COUNT 32
245
#endif
246

    
247
// =============================================================================
248

    
249
// interim location to ease ssl_info_callback access
250
uint8_t _debug_level;
251

    
252
// socket setup for mandatory fds, stored in unused fd slots (stdin, ...)
253
#define _SAV_INDEX_UDP 0
254
#define _SAV_INDEX_TCP 1
255
#define _SAV_INDEX_DIR 2
256
#define _SAV_MIN_SIZE  3
257

    
258
// FIXME: some interim net3 overlay header handling code will be
259
// apparent in these source files, because the first phases of
260
// development involved forwarding net3 pdus. While the overlay header
261
// is no longer used, as of the switchover to net4 pdus, some of the
262
// overlay header code remains intact in these source files for the
263
// time being...
264

    
265
// track unprocessed bytes held in headroom
266
#define SET_REVEAL_ANNO(_p_, _v_)  SET_AGGREGATE_ANNO(_p_, (_v_))
267
#define REVEAL_ANNO(_p_)           AGGREGATE_ANNO(_p_)
268

    
269
// track pdu mlen
270
#define SET_MLEN_ANNO(_p_, _v_)    SET_EXTRA_LENGTH_ANNO(_p_, (_v_))
271
#define MLEN_ANNO(_p_)             EXTRA_LENGTH_ANNO(_p_)
272

    
273
// track sequence assignment
274
#define SET_SEQ_ANNO(_p_, _v_)     SET_SEQUENCE_NUMBER_ANNO(_p_, (_v_))
275
#define SEQ_ANNO(_p_)              SEQUENCE_NUMBER_ANNO(_p_)
276

    
277
// track p type
278
#define SET_PTYPE_ANNO(_p_, _v_)   SET_SEND_ERR_ANNO(_p_, (_v_))
279
#define PTYPE_ANNO(_p_)            SEND_ERR_ANNO(_p_)
280

    
281
// track p retx retries
282
#define SET_RETRIES_ANNO(_p_, _v_) SET_PAINT_ANNO(_p_, (_v_))
283
#define RETRIES_ANNO(_p_)          PAINT_ANNO(_p_)
284

    
285
// track p rx fd
286
static_assert(sizeof(int) <= 4, "IPSEC_SPI_ANNO cannot store int on this host");
287
#define SET_RXFD_ANNO(_p_, _v_)    SET_IPSEC_SPI_ANNO(_p_, (_v_))
288
#define RXFD_ANNO(_p_)             IPSEC_SPI_ANNO(_p_)
289

    
290
// track p retx aka do not assign p a new seq_mf_off within dtls_tx()
291
#define SET_SEQ_PRESET_ANNO(_p_, _v_)    SET_ICMP_PARAMPROB_ANNO(_p_, (_v_))
292
#define SEQ_PRESET_ANNO(_p_)             ICMP_PARAMPROB_ANNO(_p_)
293

    
294
// track rx halt (rx fd throttled) threshold
295
#define SET_RXHALT_ANNO(_p_, _v_)  SET_FIX_IP_SRC_ANNO(_p_, (_v_))
296
#define RXHALT_ANNO(_p_)           FIX_IP_SRC_ANNO(_p_)
297

    
298
#define PTYPE_DIR          1
299
#define PTYPE_GDP_PDU      2
300
#define PTYPE_GDP_PDU_NEXT 3
301
#define PTYPE_GDP_PKT      4
302
#define PTYPE_GDP_PKT_FREE 5
303

    
304
GDPv4Router::GDPv4Router()
305
        : _headroom(Packet::default_headroom),
306
          _fd_dev_urandom(-1),
307
          _fd_directory(-1),
308
          _fd_directory_status(0),
309
          _fd_udp_listen(-1),
310
          _fd_tcp_listen(-1)
311
{
312
}
313

    
314
GDPv4Router::~GDPv4Router()
315
{
316
}
317

    
318
int
319
GDPv4Router::configure(Vector<String> &conf, ErrorHandler * errh)
320
{
321
        Args args = Args(this, errh).bind(conf);
322
        String arg_hostname;
323
        String arg_ribname;
324
        char hostname_buf[ARG_BUF_SIZE];
325
        char *h;
326
        bool fqdn;
327
        int index = 0;
328
        IPAddress arg_ip;
329
        bool ip_bind = false;
330
        uint16_t arg_udp_port;
331
        uint16_t arg_tcp_port;
332
        IPAddress arg_ip_rib;
333
        uint16_t arg_udp_port_rib;
334
        String arg_peer_str;
335
        bool arg_peer_str_unused;
336
        char *end_str;
337
        dst_map_res_t gguid_dmr;
338

    
339
    if (ninputs() != 0)
340
                return general_error("invalid input connectivity");                
341
    if (noutputs() != 0)
342
                return general_error("invalid output connectivity");                
343

    
344
        // must init sequence source before first generate_sequence call
345
        _fd_dev_urandom = open("/dev/urandom", O_RDONLY);
346
        if (_fd_dev_urandom == -1)
347
                return general_error("failed to open /dev/urandom");
348
        
349
        // shared id field init
350
        _dr_id = generate_sequence();
351

    
352
        // no config ? default peer : no default peer
353
        if (conf.size() == 0)
354
        {
355
                _no_config_mode = true;
356
        }
357
        else
358
        {
359
                _no_config_mode = false;
360
        }
361

    
362
        // UDP_PORT ? user def : default
363
        args.read_or_set("UDP_PORT", arg_udp_port, DEFAULT_UDP_PORT);
364
        args.consume();
365

    
366
        // TCP_PORT ? user def : default
367
        args.read_or_set("TCP_PORT", arg_tcp_port, DEFAULT_TCP_PORT);
368
        args.consume();
369
        
370
        // GDPNAME ? user def : { reversed(HOSTNAME) | user def } + .udp_port
371
        args.read("GDPNAME", _gdpr_xname);
372
        if (args.read_status() == 0)
373
        {
374
                // HOSTNAME ? user def : tolower(gethostname())
375
                args.read("HOSTNAME", arg_hostname);
376
                if (args.read_status() == 0)
377
                {
378
                        if (gethostname(hostname_buf, ARG_BUF_MAX) < 0)
379
                        {
380
                                return general_error("gethostname error");
381
                        }
382
                        hostname_buf[ARG_BUF_MAX] = '\0';
383
                        for (h = hostname_buf; *h != '\0'; ++h)
384
                        {
385
                                *h = tolower(*h);
386
                        }
387
                        arg_hostname.append(hostname_buf);
388
                }
389
                else
390
                {
391
                        args.consume();
392
                }
393

    
394
                // reversed(HOSTNAME)
395
                strcpy(hostname_buf, arg_hostname.c_str());
396
                fqdn = false;
397
                for (h = hostname_buf; *h != '\0'; ++h)
398
                {
399
                        if (*h == '.')
400
                        {
401
                                fqdn = true;
402
                        }
403
                }
404
                if (fqdn)
405
                {
406
                        // work from back to front, reversing dot-delimited tokens
407
                        for (h = h - 1; h > hostname_buf; --h)
408
                        {
409
                                if (*h == '.')
410
                                {
411
                                        _gdpr_xname += &h[1];
412
                                        _gdpr_xname += ".";
413
                                        *h = '\0';
414
                                }
415
                        }
416
                }
417
                // if fqdn, append remaining (first) token, else append entire string
418
                _gdpr_xname += hostname_buf;
419

    
420
                // always append .udp_port
421
                _gdpr_xname += ".";
422
                _gdpr_xname += std::to_string(arg_udp_port).c_str();
423
        }
424
        else
425
        {
426
                args.consume();
427
        }
428

    
429
        // IP ? used def (e.g., bind to specific ip to isolate testbed) : INADDR_ANY
430
        args.read("IP", arg_ip);
431
        if (args.read_status() == 1)
432
        {
433
                args.consume();
434
                ip_bind = true;
435
        }
436

    
437
        // SINGLE_ROUTER ? true : default false
438
        args.read_or_set("SINGLE_ROUTER", _single_router_mode, false);
439
        args.consume();
440

    
441
        if (_single_router_mode == false)
442
        {
443
                // RIBNAME ? user_supplied : default("gdp-03.eecs.berkeley.edu")
444
                args.read_or_set("RIBNAME", arg_ribname, "gdp-03.eecs.berkeley.edu");
445
                args.consume();
446
        
447
                // IP_RIB ? user def (e.g., no dns or testbed dir) : resolve(RIBNAME)
448
                args.read("IP_RIB", arg_ip_rib);
449
                if (args.read_status() == 0)
450
                {
451
                        if (resolve(errh, arg_ribname, arg_ip_rib) < 0)
452
                        {
453
                                return -1;
454
                        }
455
                }
456
                args.consume();
457
        
458
                // UDP_PORT_RIB ? user def : 9001 default
459
                args.read_or_set("UDP_PORT_RIB", arg_udp_port_rib, 9001);
460
                args.consume();
461

    
462
                // ADVERT_TIMER ? user def : 128000 usec (128 msec) default
463
                args.read_or_set("ADVERT_TIMER", _advert_timer, 128000);
464
                args.consume();
465
        }
466
        
467
        // DEBUG ? user def : DOFF default
468
        args.read_or_set("DEBUG", _debug_level, DOFF);
469
        args.consume();
470

    
471
        // insert a default peer, if no config
472
        if (_no_config_mode)
473
        {
474
                if (_dr_id % 2 == 0)
475
                {
476
                        args.push_back("PEER gdp-01.eecs.berkeley.edu");
477
                }
478
                else
479
                {
480
                        args.push_back("PEER gdp-03.eecs.berkeley.edu");                        
481
                }
482
        }
483
        
484
        // conf.size() covers worst (and most likely) case (100% default ports)
485
        if ((_sav = new _sav_t(_SAV_MIN_SIZE + conf.size())) == nullptr)
486
        {
487
                return general_error("new sockaddr vector");
488
        }
489

    
490
        // udp port listen
491
        (*_sav)[_SAV_INDEX_UDP].sin_family = AF_INET;
492
        (*_sav)[_SAV_INDEX_UDP].sin_port = htons(arg_udp_port);
493
        if (ip_bind)
494
        {
495
                // on an explicit ip address only (development testbed feature)
496
                (*_sav)[_SAV_INDEX_UDP].sin_addr = arg_ip.in_addr();
497
        }
498
        else
499
        {
500
                // on INADDR_ANY (production mode)
501
                (*_sav)[_SAV_INDEX_UDP].sin_addr.s_addr = INADDR_ANY;
502
        }
503

    
504
        // tcp port listen on INADDR_ANY
505
        (*_sav)[_SAV_INDEX_TCP].sin_family = AF_INET;
506
        (*_sav)[_SAV_INDEX_TCP].sin_port = htons(arg_tcp_port);
507
        (*_sav)[_SAV_INDEX_TCP].sin_addr.s_addr = INADDR_ANY;
508

    
509
        // directory ip + port (may be dormant if _single_router_mode)
510
        (*_sav)[_SAV_INDEX_DIR].sin_family = AF_INET;
511
        (*_sav)[_SAV_INDEX_DIR].sin_addr = arg_ip_rib.in_addr();
512
        (*_sav)[_SAV_INDEX_DIR].sin_port = htons(arg_udp_port_rib);
513

    
514
        // this router's gdpname
515
        SHA256((const unsigned char*)_gdpr_xname.c_str(), _gdpr_xname.length(),
516
                   _gdpr);
517

    
518
        click_chatter("Starting gdp-router-click (v%s %s %s)\n"
519
                                  "\tas %s\n"
520
                                  "\taka %s\n"
521
                                  "\twith debug %d\n"
522
                                  "\tlistening on %s udp %d and tcp %d",
523
                                  ver_major, git_branch, git_hash,
524
                                  _gdpr_xname.c_str(), gdp_printable_name(_gdpr, _gdpr_pname),
525
                                  _debug_level,
526
                                  (ip_bind ? IPAddress((*_sav)[_SAV_INDEX_UDP]
527
                                                                           .sin_addr).unparse().c_str() : "any"),
528
                                  ntohs((*_sav)[_SAV_INDEX_UDP].sin_port),
529
                                  ntohs((*_sav)[_SAV_INDEX_TCP].sin_port));
530
        if (_single_router_mode)
531
        {
532
                click_chatter("\tsingle router mode (no directory service)");
533
        }
534
        else
535
        {
536
                click_chatter("\tgdp directory on %s udp %d, advert timer %d usecs",
537
                                          IPAddress((*_sav)[_SAV_INDEX_DIR]
538
                                                                .sin_addr).unparse().c_str(),
539
                                          ntohs((*_sav)[_SAV_INDEX_DIR].sin_port),
540
                                          _advert_timer);
541
        }
542

    
543
        // statically configured peer sa entries follow listen ports and dir port
544
        index = _SAV_MIN_SIZE;
545

    
546
        // parse peer config format: "{ resolve(dns) | ip }, [ port, ] ..."
547
        while (conf.size() > 0)
548
        {
549
                args.read_p("PEER", arg_peer_str);
550
                if (args.read_status() == 1)
551
                {
552
                        args.consume();
553

    
554
                preloaded_arg_peer_str:
555

    
556
                        arg_udp_port = -1;
557

    
558
                        // { resolve(dns) | ip } 
559
                        if (inet_pton(AF_INET, arg_peer_str.c_str(), arg_ip.data()) == 0)
560
                        {
561
                                if (resolve(errh, arg_peer_str, arg_ip) < 0)
562
                                {
563
                                        return -1;
564
                                }
565
                        }
566

    
567
                        // [ port, ]
568
                        args.read_p("PORT", arg_peer_str);
569
                        if (args.read_status() == 1)
570
                        {
571
                                args.consume();
572
                                arg_udp_port = strtol(arg_peer_str.c_str(), &end_str, 0);
573
                                if (*arg_peer_str.c_str() != '\0' && *end_str == '\0')
574
                                {
575
                                        arg_peer_str_unused = false;
576
                                }
577
                                else
578
                                {
579
                                        arg_udp_port = DEFAULT_UDP_PORT;
580
                                        arg_peer_str_unused = true;
581
                                }
582
                        }
583
                        else
584
                        {
585
                                arg_udp_port = DEFAULT_UDP_PORT;
586
                                arg_peer_str_unused = false;
587
                        }
588

    
589
                        // set arp_ip + arg_udp_port
590
                        (*_sav)[index].sin_family = AF_INET;
591
                        (*_sav)[index].sin_addr = arg_ip.in_addr();
592
                        (*_sav)[index].sin_port = htons(arg_udp_port);
593
                        click_chatter("\t%s peer %s:%d\n",
594
                                                  (_no_config_mode ? "default" : "static"),
595
                                                  IPAddress((*_sav)[index].sin_addr).unparse().c_str(),
596
                                                  ntohs((*_sav)[index].sin_port));
597
                        ++index; // after click_chatter has displayed index
598
                        
599
                        if (arg_peer_str_unused)
600
                        {
601
                                goto preloaded_arg_peer_str;
602
                        }
603
                }
604
                else
605
                {
606
                        return general_error("unrecognized configuration parameter");
607
                }
608
        }
609

    
610
        if (conf.size() > 0)
611
        {
612
                return general_error("extraneous parameters");
613
        }
614

    
615
        // trim unused sa vector slots (due to non-default ports) to skip in startup
616
        (*_sav).resize(index);
617
        (*_sav).shrink_to_fit();
618

    
619
        // fd vector table
620
        _fdv_size = 512; // FIXME: no dynamic resize support at the moment, big init
621
        if ((_fdv = new _fdv_t(_fdv_size, _fdv_init)) == nullptr)
622
        {
623
                return general_error("fd vector new");
624
        }
625

    
626
        // insert _gdpr_ca in map to block spoof attempts
627
        _gdpr_ca = (gdp_name_ca_t *) _gdpr;
628
        gguid_dmr = dst_map.emplace(std::make_pair(*_gdpr_ca, dst_map_init));
629
        if (gguid_dmr.second == false)
630
        {
631
                return general_error("fd vector init");
632
        }
633
        // fd_state remains FD_NONE for fd associated with ADDR_MINE
634
        gguid_dmr.first->second.addr_state = ADDR_MINE;
635

    
636
        return 0;
637
}
638

    
639
int
640
GDPv4Router::initialize (ErrorHandler * /* errh */)
641
{
642
        int fd_connect;
643
        WritablePacket *dp;
644
        otw_dir_t *otw_dir;
645
        gdp_pname_t _tmp_pname_1;
646

    
647
        // validate click configuration
648
        if ((ninputs() != 0) || (noutputs() != 0))
649
                return general_error("configured ninputs != 0 or noutputs != 0");
650

    
651
        // SINGLE ROUTER MODE
652
        if (_single_router_mode)
653
        {
654
                // tcp bridge only
655
                if (new_tcp_listen() < 0)
656
                        return general_error("new tcp listen failed");
657

    
658
                // systemd notify
659
                sd_notify(false, "READY=1\n");
660
                return 0;
661
        }
662
        
663
        // openssl pre-requisites
664
        SSL_library_init();
665
        SSL_load_error_strings();
666
        ERR_load_BIO_strings();
667
        OpenSSL_add_all_algorithms();
668

    
669
        // SSL_CTX is shared by all SSL sessions
670
        if ((_ssl_ctx = SSL_CTX_new(DTLSv1_2_method())) == NULL)
671
                return general_error("SSL_CTX_new failed");
672

    
673
        // SSL_CTX crypto defaults 
674
        if (!(SSL_CTX_set_cipher_list(_ssl_ctx, "AECDH-NULL-SHA")))
675
                return general_error("SSL_CTX_set_cipher_list failed");
676
        // SSL_CTX_set_ecdh_auto() must be before SSL_new() (no shared cipher error)
677
        // eNULL cipher negotiation fails unless ecdh is available via set_ecdh_auto
678
        if (!(SSL_CTX_set_ecdh_auto(_ssl_ctx, 1)))
679
                return general_error("SSL_CTX_set_ecdh_auto failed");
680

    
681
        if (new_udp_listen() < 0)
682
                return general_error("new udp listen failed");
683

    
684
        if (new_tcp_listen() < 0)
685
                return general_error("new tcp listen failed");
686

    
687
        // gdp directory
688
        if ((fd_connect = new_udp_connect(&(*_sav)[_SAV_INDEX_DIR],
689
                                                                          FD_DIRECTORY)) < 0)
690
                return general_error("new connect directory failed");
691
        _fd_directory = fd_connect;
692

    
693
        // directory p
694
        dp = Packet::make(offsetof(otw_dir_t, eguid[1]));
695
        if (dp == nullptr)
696
        {
697
                return general_error("directory flush packet allocation failed");
698
        }
699
        otw_dir = (otw_dir_t *) dp->data();
700
        otw_dir->ver = GDP_CHAN_PROTO_VERSION;
701
        otw_dir->id = htons(_dr_id++);
702
        otw_dir->cmd = GDP_CMD_DIR_FLUSH;
703
        memcpy(otw_dir->eguid[0], _gdpr, sizeof(gdp_name_t));
704
        // tx directory flush request
705
        datagram_tx(_fd_directory, dp, nullptr, 0);
706
        ccdbg(DVERB, "(%d)%s tx directory flush id 0x%x\n"
707
                  "\teguid[%s]",
708
                  _fd_directory, fd_to_state_name(_fd_directory), ntohs(otw_dir->id),
709
                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1));
710
        // dp->kill() after retx below
711
        
712
        for (int index = _SAV_MIN_SIZE; index < (int) (*_sav).size(); index++)
713
        {
714
                if ((fd_connect = new_udp_connect(&(*_sav)[index], FD_UDP_CONNECT)) < 0)
715
                        ccerr("new udp connect failed at %d", index);
716
        }
717
        // track configured fds since they are recycled differently than dynamic fds
718
        _fd_last_configured = fd_connect;
719

    
720
        // retx directory flush request
721
        datagram_tx(_fd_directory, dp, nullptr, 0);
722
        ccdbg(DVERB, "(%d)%s retx directory flush id 0x%x\n"
723
                  "\teguid[%s]",
724
                  _fd_directory, fd_to_state_name(_fd_directory), ntohs(otw_dir->id),
725
                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1));
726
        dp->kill();
727
        
728
        // systemd notify
729
        sd_notify(false, "READY=1\n");
730
        return 0;
731
}
732

    
733
void
734
GDPv4Router::cleanup (CleanupStage)
735
{
736
        WritablePacket *dp;
737
        otw_dir_t *otw_dir;
738
        gdp_pname_t _tmp_pname_1;
739

    
740
        // SINGLE ROUTER MODE
741
        if (_single_router_mode || router()->initialized() == false)
742
                return;
743

    
744
        // flush
745
        dp = Packet::make(offsetof(otw_dir_t, eguid[1]));
746
        if (dp == nullptr)
747
        {
748
                ccwarn("(%d)%s packet pool exhausted on %d",
749
                           _fd_directory, fd_to_state_name(_fd_directory),
750
                           offsetof(otw_dir_t, eguid[1]));
751
                return;
752
        }
753
        otw_dir = (otw_dir_t *) dp->data();
754
        otw_dir->ver = GDP_CHAN_PROTO_VERSION;
755
        otw_dir->id = htons(_dr_id++);
756
        otw_dir->cmd = GDP_CMD_DIR_FLUSH;
757
        memcpy(otw_dir->eguid[0], _gdpr, sizeof(gdp_name_t));
758
        // tx directory flush request
759
        datagram_tx(_fd_directory, dp, nullptr, 0);
760
        ccdbg(DVERB, "(%d)%s tx directory flush id 0x%x\n"
761
                  "\teguid[%s]",
762
                  _fd_directory, fd_to_state_name(_fd_directory), ntohs(otw_dir->id),
763
                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1));
764
        // retx directory flush request
765
        datagram_tx(_fd_directory, dp, nullptr, 0);
766
        ccdbg(DVERB, "(%d)%s retx directory flush id 0x%x\n"
767
                  "\teguid[%s]",
768
                  _fd_directory, fd_to_state_name(_fd_directory), ntohs(otw_dir->id),
769
                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1));
770
        dp->kill();
771

    
772
        // close fds, but work from dynamic fds backwards so directory can be used
773
        for (uint16_t fd = (*_fdv).size(); fd > 0; --fd)
774
        {
775
                switch ((*_fdv)[fd].fd_state)
776
                {
777
                case FD_TCP_CLIENT:
778
                case FD_DTLS_ACCEPT:
779
                case FD_DTLS_DO_HANDSHAKE:
780
                case FD_DTLS_CLIENT:
781
                case FD_DTLS_SERVER:
782
                        fd_recycle(fd);
783
                        break;
784
                case FD_DIRECTORY:
785
                case FD_UDP_LISTEN:
786
                case FD_UDP_CONNECT:
787
                case FD_UDP_CONNECT_DUPE:
788
                case FD_TCP_LISTEN:
789
                        confirm_shutdown(fd);
790
                        confirm_close(fd);
791
                        break;
792
                case FD_NONE:
793
                default:
794
                        break;
795
                }
796
        }
797
        SSL_CTX_free(_ssl_ctx);
798
        
799
}
800

    
801
int
802
GDPv4Router::resolve (ErrorHandler * errh, String &arg_name, IPAddress &arg_ip)
803
{
804
        struct __res_state res_state;
805
        u_char buf[ARG_BUF_SIZE];
806
        ns_msg msg;
807
        ns_rr rr;
808
        int len;
809
        HEADER *hdr;
810
                
811
        res_ninit(&res_state);
812
        len = res_nquery(&res_state, arg_name.c_str(), C_IN, T_A,
813
                                         buf, ARG_BUF_SIZE);
814
        if (len < 0)
815
        {
816
                errh->error("dns query failed on \"%s\"", arg_name.c_str());
817
                return -1;
818
        }
819
        hdr = reinterpret_cast<HEADER*>(buf);
820
        switch (hdr->rcode)
821
        {
822
        case NOERROR:
823
                break;
824
        case FORMERR:
825
                errh->error("dns format error on query of \"%s\"", arg_name.c_str());
826
                return -1;
827
        case SERVFAIL:
828
                errh->error("dns server failure on query of \"%s\"", arg_name.c_str());
829
                return -1;
830
        case NXDOMAIN:
831
                errh->error("dns name error on query of \"%s\"", arg_name.c_str());
832
                return -1;
833
        case NOTIMP:
834
                errh->error("dns not implemented error on query of \"%s\"",
835
                                        arg_name.c_str());
836
                return -1;
837
        case REFUSED:
838
                errh->error("dns refused error on query of \"%s\"", arg_name.c_str());
839
                return -1;
840
        default:
841
                errh->error("dns unknown error on query of \"%s\"", arg_name.c_str());
842
                return -1;
843
        }
844
        if (ns_initparse(buf, len, &msg) < 0)
845
        {
846
                errh->error("dns parsing error on query of \"%s\"", arg_name.c_str());
847
                return -1;
848
        }
849
        len = ns_msg_count(msg, ns_s_an);
850
        if (len == 0)
851
        {
852
                errh->error("dns zero messages on query of \"%s\"", arg_name.c_str());
853
                return -1;
854
        }
855
        for (int i = 0; i < len; ++i)
856
        {
857
                ns_parserr(&msg, ns_s_an, i, &rr);
858
                if (ns_rr_type(rr) == T_A)
859
                {
860
                        arg_ip = ntohl(ns_get32(ns_rr_rdata(rr)));
861
                        // for now, automatic discovery always uses the first IP
862
                        break;
863
                }
864
        }
865
        res_nclose(&res_state);
866
        return 0;
867
}
868

    
869
void
870
GDPv4Router::selected (int fd, int mask)
871
{
872
        ccdbg(DVVERB, "\n(%d)%s %c%c", fd, fd_to_state_name(fd),
873
                    mask & SELECT_READ ? 'R' : '-', mask & SELECT_WRITE ? 'W' : '-');
874

    
875
        if (mask & SELECT_READ)
876
        {
877
                switch ((*_fdv)[fd].fd_state)
878
                {
879

    
880
                case FD_DIRECTORY:
881
                {
882
                        directory_rx(fd);
883
                }
884
                break;
885

    
886
                case FD_UDP_LISTEN:
887
                {
888
                        udp_listen_rx(fd);
889
                }
890
                break;
891
                
892
                case FD_UDP_CONNECT:
893
                {
894
                        udp_connect_rx(fd);
895
                }
896
                break;
897
                
898
                case FD_DTLS_ACCEPT:
899
                {
900
                        ssl_accept(fd);
901
                }
902
                break;
903

    
904
                case FD_DTLS_DO_HANDSHAKE:
905
                {
906
                        ssl_handshake(fd);
907
                }
908
                break;
909

    
910
                case FD_TCP_LISTEN:
911
                {
912
                        new_tcp_connect(fd);
913
                }
914
                break;
915

    
916
                case FD_DTLS_CLIENT:
917
                case FD_DTLS_SERVER:
918
                {
919
                        // wake up fd rx task
920
                        avow((*_fdv)[fd].rx_task != nullptr);
921
                        (*_fdv)[fd].rx_task->reschedule();
922
                        // router fd is always SELECT_READ (add/remove performs poorly)
923
                }
924
                break;
925
                
926
                case FD_TCP_CLIENT:
927
                {
928
                         avow((*_fdv)[fd].rx_task != nullptr);
929

    
930
                        ccdbg(DVERB, "\n(%d)%s selected throttle %s size %d shared %s",
931
                                  fd, fd_to_state_name(fd),
932
                                  (*_fdv)[fd].rx_throttled ? "true" : "false",
933
                                  (*_fdv)[fd].zlplist.size(),
934
                                  ((*_fdv)[fd].zlplist.begin() != (*_fdv)[fd].zlplist.end() &&
935
                                   (*(*_fdv)[fd].zlplist.begin())->shared() ?
936
                                   "true" : "false"));
937

    
938
                        // trim any unshared zlps from zlplist front
939
                        while ((*_fdv)[fd].zlplist.size() > 0 &&
940
                                   (*(*_fdv)[fd].zlplist.begin())->shared() == false)
941
                        {
942
                                (*(*_fdv)[fd].zlplist.begin())->kill();
943
                                (*_fdv)[fd].zlplist.erase((*_fdv)[fd].zlplist.begin());
944
                        }
945

    
946
                        // update rx throttled state
947
                        if ((*_fdv)[fd].rx_throttled &&
948
                                (*_fdv)[fd].zlplist.size() <= TCP_RXP_PDUS_CLR)
949
                        {
950
                                ccdbg(DPDU, "(%d)%s selected unthrottled (%d)",
951
                                           fd, fd_to_state_name(fd),
952
                                           (*_fdv)[fd].zlplist.size());
953
                                add_select(fd, SELECT_READ);
954
                                (*_fdv)[fd].rx_throttled = false;
955
                        }
956

    
957
                         // wake up fd rx task
958
                        (*_fdv)[fd].rx_task->reschedule();
959
                }
960
                break;
961

    
962
                case FD_NONE:
963
                case FD_UDP_CONNECT_DUPE:
964
                default:
965
                {
966
                        ccerr("(%d)%s unexpected select (%d)",
967
                                  fd, fd_to_state_name(fd), mask);
968
                        aver(0);
969
                }
970
                break;
971
                
972
                } // switch
973
        }
974
                
975
        if (mask & SELECT_WRITE)
976
        {
977
                switch ((*_fdv)[fd].fd_state)
978
                {
979

    
980
                case FD_DTLS_CLIENT:
981
                case FD_DTLS_SERVER:
982
                case FD_TCP_CLIENT:
983
                {
984
                        // wake up fd tx task
985
                        avow((*_fdv)[fd].tx_task != nullptr);
986
                        (*_fdv)[fd].tx_task->reschedule();
987
                        remove_select(fd, SELECT_WRITE);
988
                }
989
                break;
990

    
991
                case FD_NONE:
992
                case FD_DIRECTORY:
993
                case FD_UDP_LISTEN:
994
                case FD_UDP_CONNECT:
995
                case FD_UDP_CONNECT_DUPE:
996
                case FD_DTLS_ACCEPT:
997
                case FD_DTLS_DO_HANDSHAKE:
998
                case FD_TCP_LISTEN:
999
                {
1000
                        ccerr("(%d)%s unexpected select (%d)",
1001
                                  fd, fd_to_state_name(fd), mask);
1002
                        aver(0);
1003
                }
1004
                break;
1005

    
1006
                } // switch
1007
        }
1008

    
1009
        ccdbg(DVVERB, "\n>>> (%d)%s %c%c", fd, fd_to_state_name(fd),
1010
                  mask & SELECT_READ ? 'R' : '-', mask & SELECT_WRITE ? 'W' : '-');
1011
}
1012

    
1013
int
1014
GDPv4Router::general_error (const char *event)
1015
{
1016
        ccerr("%s", event);
1017
        return -1;
1018
}
1019

    
1020
uint16_t
1021
GDPv4Router::generate_sequence (void)
1022
{
1023
        uint16_t new_seq;
1024

    
1025
        // frequently used operation, _fd_dev_urandom remains open for quick access
1026
        if (read(_fd_dev_urandom, &new_seq, sizeof(new_seq)) != 2)
1027
        {
1028
                general_error("/dev/urandom");
1029
        }
1030
        return (new_seq & SEQ_MASK);
1031
}
1032

    
1033
int
1034
GDPv4Router::new_udp_listen (void)
1035
{
1036
        int nl_fd;
1037
        int on = 1;
1038
        //int pmtudisc_do = IP_PMTUDISC_DO;
1039

    
1040
        if (_fd_udp_listen != -1)
1041
        {
1042
                ccerr("new udp listen fd already exists");
1043
                return -1;
1044
        }
1045
        
1046
        if ((nl_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0)
1047
        {
1048
                ccerr("new udp listen socket");
1049
                return -1;
1050
        }
1051

    
1052
        if (nl_fd >= _fdv_size || (*_fdv)[nl_fd].fd_state != FD_NONE)
1053
        {
1054
                ccerr("new udp listen fd (%d) range or duplicate", nl_fd);
1055
                goto close_fd;
1056
        }
1057

    
1058
        fcntl(nl_fd, F_SETFL, O_NONBLOCK);
1059
        fcntl(nl_fd, F_SETFD, FD_CLOEXEC);
1060
        setsockopt(nl_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(int));
1061
        //setsockopt(nl_fd, IPPROTO_IP, IP_MTU_DISCOVER, &pmtudisc_do, sizeof(int));
1062

    
1063
        if (bind(nl_fd, (struct sockaddr *) &(*_sav)[_SAV_INDEX_UDP],
1064
                         sizeof((*_sav)[_SAV_INDEX_UDP])) < 0)
1065
        {
1066
                ccerr("new udp listen (%d) bind (%d)", nl_fd, errno);
1067
                goto close_fd;
1068
        }
1069

    
1070
        (*_fdv)[nl_fd].fd_state = FD_UDP_LISTEN;
1071
        _fd_udp_listen = nl_fd;
1072
        add_select(nl_fd, SELECT_READ);
1073

    
1074
        ccdbg(DINFO, "(%d)%s new udp listen", nl_fd, fd_to_state_name(nl_fd));
1075
        
1076
        return nl_fd;
1077

    
1078
close_fd:
1079

    
1080
        confirm_close(nl_fd);
1081
        return -1;
1082
}
1083

    
1084
int
1085
GDPv4Router::new_tcp_listen (void)
1086
{
1087
        int nl_fd;
1088
        int on = 1;
1089
        //int pmtudisc_do = IP_PMTUDISC_DO;
1090
 
1091
        if (_fd_tcp_listen != -1)
1092
        {
1093
                ccerr("new tcp listen fd already exists");
1094
                return -1;
1095
        }
1096
        
1097
        if ((nl_fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
1098
        {
1099
                ccerr("new tcp listen socket");
1100
                return -1;
1101
        }
1102

    
1103
        fcntl(nl_fd, F_SETFL, O_NONBLOCK);
1104
        fcntl(nl_fd, F_SETFD, FD_CLOEXEC);
1105
        setsockopt(nl_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(int));
1106
        //setsockopt(nl_fd, IPPROTO_IP, IP_MTU_DISCOVER, &pmtudisc_do, sizeof(int));
1107
        
1108
        if (bind(nl_fd, (struct sockaddr *) &(*_sav)[_SAV_INDEX_TCP],
1109
                         sizeof((*_sav)[_SAV_INDEX_TCP])) < 0)
1110
        {
1111
                ccerr("new tcp listen (%d) bind", nl_fd);
1112
                goto close_fd;
1113
        }
1114

    
1115
        if (listen(nl_fd, 5) < 0)
1116
        {
1117
                ccerr("new tcp listen (%d) error %d", nl_fd, errno);
1118
                goto close_fd;
1119
        }
1120

    
1121
        (*_fdv)[nl_fd].fd_state = FD_TCP_LISTEN;
1122
        _fd_tcp_listen = nl_fd;
1123
        add_select(nl_fd, SELECT_READ);
1124

    
1125
        ccdbg(DINFO, "(%d)%s new tcp listen", nl_fd, fd_to_state_name(nl_fd));
1126
        
1127
        return nl_fd;
1128

    
1129
close_fd:
1130

    
1131
        confirm_close(nl_fd);
1132
        return -1;
1133
}
1134

    
1135
int
1136
GDPv4Router::new_udp_connect (struct sockaddr_in *nc_sa_in, _FD_STATE fd_state)
1137
{
1138
        int nc_fd;
1139
        int on = 1;
1140
        //int pmtudisc_do = IP_PMTUDISC_DO;
1141
 
1142
        avow(nc_sa_in != nullptr);
1143
        
1144
        if (nc_sa_in->sin_addr == IPAddress("127.0.0.1") && nc_sa_in->sin_port == 0)
1145
                return -1;
1146
        
1147
        if ((nc_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0)
1148
        {
1149
                ccerr("new udp connect socket");
1150
                return -1;
1151
        }
1152

    
1153
        if (nc_fd >= _fdv_size || (*_fdv)[nc_fd].fd_state != FD_NONE)
1154
        {
1155
                ccerr("new udp connect (%d) range or duplicate", nc_fd);
1156
                goto close_fd;
1157
        }
1158

    
1159
        fcntl(nc_fd, F_SETFL, O_NONBLOCK);
1160
        fcntl(nc_fd, F_SETFD, FD_CLOEXEC);
1161
        setsockopt(nc_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(int));
1162
        //setsockopt(nc_fd, IPPROTO_IP, IP_MTU_DISCOVER, &pmtudisc_do, sizeof(int));
1163
        
1164
        if (fd_state == FD_UDP_LISTEN)
1165
        {
1166
                if (bind(nc_fd, (struct sockaddr *) &(*_sav)[_SAV_INDEX_UDP],
1167
                                 sizeof((*_sav)[_SAV_INDEX_UDP])) < 0)
1168
                {
1169
                        ccerr("new udp connect (%d) bind", nc_fd);
1170
                        goto close_fd;
1171
                }
1172
        }
1173

    
1174
        if (connect(nc_fd, (struct sockaddr *) nc_sa_in, sizeof(*nc_sa_in)) < 0)
1175
        {
1176
                ccerr("new udp connect (%d) connect", nc_fd);
1177
                goto close_fd;
1178
        }
1179

    
1180
        (*_fdv)[nc_fd].fd_state = fd_state;
1181

    
1182
        (*_fdv)[nc_fd].rxp = nullptr;
1183
        
1184
        // tasks cannot self-delete, so next fd user must delete unscheduled tasks
1185
        if ((*_fdv)[nc_fd].rx_task != nullptr)
1186
        {
1187
                delete (*_fdv)[nc_fd].rx_task;
1188
                (*_fdv)[nc_fd].rx_task = nullptr;
1189
        }
1190
        if ((*_fdv)[nc_fd].tx_task != nullptr)
1191
        {
1192
                delete (*_fdv)[nc_fd].tx_task;
1193
                (*_fdv)[nc_fd].tx_task = nullptr;                
1194
        }
1195
        
1196
        if (fd_state != FD_DIRECTORY)
1197
        {
1198
                try
1199
                {
1200
                        (*_fdv)[nc_fd].rx_task =
1201
                                new Task(static_dtls_rx, (void *)(intptr_t)nc_fd);
1202
                        (*_fdv)[nc_fd].tx_task =
1203
                                new Task(static_dtls_tx, (void *)(intptr_t)nc_fd);
1204
                }
1205
                catch (std::bad_alloc)
1206
                {
1207
                        ccerr("new udp connect (%d) tx task", nc_fd);
1208
                        delete (*_fdv)[nc_fd].rx_task;
1209
                        (*_fdv)[nc_fd].rx_task = nullptr;
1210
                        goto close_fd;
1211
                }
1212
                (*_fdv)[nc_fd].rx_task->initialize(this, false);
1213
                (*_fdv)[nc_fd].tx_task->initialize(this, false);
1214
        }
1215

    
1216
        if (fd_state == FD_UDP_CONNECT)
1217
        {
1218
                avow((*_fdv)[nc_fd].retx_timer == nullptr);
1219
                try
1220
                {
1221
                        (*_fdv)[nc_fd].retx_timer = new Timer(static_udp_connect_timer,
1222
                                                                                                  (void *)(intptr_t)nc_fd);
1223
                }
1224
                catch (std::bad_alloc)
1225
                {
1226
                        ccerr("new udp connect (%d) retx timer", nc_fd);
1227
                        delete (*_fdv)[nc_fd].tx_task;
1228
                        (*_fdv)[nc_fd].tx_task = nullptr;
1229
                        goto close_fd;
1230
                }
1231
                (*_fdv)[nc_fd].retx_timer->initialize(this, true);
1232
                (*_fdv)[nc_fd].retx_timer->schedule_after(FD_UDP_CONNECT_NOW_TS_DELTA);
1233
        }
1234

    
1235
        add_select(nc_fd, SELECT_READ);
1236

    
1237
        ccdbg(DINFO, "(%d)%s new udp connect %s:%d",
1238
                   nc_fd, fd_to_state_name(nc_fd),
1239
                   IPAddress(nc_sa_in->sin_addr).unparse().c_str(),
1240
                   ntohs(nc_sa_in->sin_port));
1241
        
1242
        return nc_fd;
1243

    
1244
close_fd:
1245

    
1246
        confirm_close(nc_fd);
1247
        return -1;
1248
}
1249

    
1250
void
1251
GDPv4Router::new_tcp_connect (int fd)
1252
{
1253
        struct sockaddr_in sa_in;
1254
        socklen_t sa_in_len = sizeof(sa_in);
1255
        int nc_fd;
1256

    
1257
        if (fd != _fd_tcp_listen)
1258
        {
1259
                ccerr("new tcp connect (%d) on unexpected fd", fd);
1260
                return;
1261
        }
1262

    
1263
        nc_fd = accept(_fd_tcp_listen, (struct sockaddr *) &sa_in, &sa_in_len);
1264
        if (nc_fd < 0)
1265
        {
1266
                ccerr("new_tcp_connect accept");
1267
                return;
1268
        }
1269
        
1270
        if (nc_fd >= _fdv_size || (*_fdv)[nc_fd].fd_state != FD_NONE)
1271
        {
1272
                ccerr("new tcp connect (%d) range or duplicate", nc_fd);
1273
                confirm_close(nc_fd);
1274
                return;
1275
        }
1276
        
1277
        fcntl(nc_fd, F_SETFL, O_NONBLOCK);
1278
        fcntl(nc_fd, F_SETFD, FD_CLOEXEC);
1279
        int enable = 1;
1280
        setsockopt(nc_fd, IPPROTO_TCP, TCP_NODELAY, (void *) &enable, sizeof enable);
1281

    
1282
        (*_fdv)[nc_fd].fd_state = FD_TCP_CLIENT;
1283

    
1284
        (*_fdv)[nc_fd].rxp = Packet::make(TCR_RXP_HDRM, nullptr, TCR_RXP_SIZE, 0);
1285
        if ((*_fdv)[nc_fd].rxp == nullptr)
1286
        {
1287
                ccerr("new tcp connect (%d) no rxp", nc_fd);
1288
                confirm_close(nc_fd);
1289
                return;
1290
        }
1291
        SET_PTYPE_ANNO((*_fdv)[nc_fd].rxp, PTYPE_GDP_PDU);
1292
        ccdbg(DVVERB, "(%d)%s make new rxp %p",
1293
                  fd, fd_to_state_name(fd), (*_fdv)[nc_fd].rxp);
1294

    
1295
        // track p rx fd
1296
        SET_RXFD_ANNO((*_fdv)[nc_fd].rxp, nc_fd);
1297

    
1298
        (*_fdv)[nc_fd].rxp_want_len = 0;
1299

    
1300
        // tasks cannot self-delete, so next fd user must delete unscheduled tasks
1301
        if ((*_fdv)[nc_fd].rx_task != nullptr)
1302
        {
1303
                delete (*_fdv)[nc_fd].rx_task;
1304
                (*_fdv)[nc_fd].rx_task = nullptr;
1305
        }
1306
        if ((*_fdv)[nc_fd].tx_task != nullptr)
1307
        {
1308
                delete (*_fdv)[nc_fd].tx_task;
1309
                (*_fdv)[nc_fd].tx_task = nullptr;                
1310
        }
1311
        
1312
        try
1313
        {
1314
                (*_fdv)[nc_fd].rx_task =
1315
                        new Task(static_tcp_client_rx, (void *)(intptr_t)nc_fd);
1316
                (*_fdv)[nc_fd].tx_task =
1317
                        new Task(static_tcp_client_tx, (void *)(intptr_t)nc_fd);
1318
        }
1319
        catch (std::bad_alloc)
1320
        {
1321
                ccerr("new tcp connect (%d) tx task", nc_fd);
1322
                (*_fdv)[nc_fd].rxp->kill();
1323
                (*_fdv)[nc_fd].rxp = nullptr;
1324
                delete (*_fdv)[nc_fd].rx_task;
1325
                (*_fdv)[nc_fd].rx_task = nullptr;
1326
                confirm_close(nc_fd);
1327
                return;
1328
        }
1329
        (*_fdv)[nc_fd].rx_task->initialize(this, false);
1330
        (*_fdv)[nc_fd].tx_task->initialize(this, false);
1331
        
1332
        add_select(nc_fd, SELECT_READ);
1333

    
1334
        ccdbg(DINFO, "(%d)%s new tcp connect %s:%d",
1335
                   nc_fd, fd_to_state_name(nc_fd),
1336
                   IPAddress(sa_in.sin_addr).unparse().c_str(),
1337
                   ntohs(sa_in.sin_port));
1338
}
1339

    
1340
void
1341
GDPv4Router::datagram_tx (int fd, WritablePacket *p,
1342
                                                  struct sockaddr_in *sa_in, size_t sa_in_size)
1343
{
1344
        ssize_t send_len;
1345
        
1346
        if (sa_in_size == 0)
1347
        {
1348
                send_len = send(fd, p->data(), p->length(), 0);
1349
                ccdbg(DVERB, "(%d)%s dgram tx len %d",
1350
                          fd, fd_to_state_name(fd), send_len);
1351
        }
1352
        else
1353
        {
1354
                send_len = sendto(fd, p->data(), p->length(), 0,
1355
                                                  (struct sockaddr *) sa_in, sa_in_size);
1356
                ccdbg(DVERB, "(%d)%s dgram tx len %d to %s:%d",
1357
                          fd, fd_to_state_name(fd), send_len,
1358
                          IPAddress(sa_in->sin_addr).unparse().c_str(),
1359
                          ntohs(sa_in->sin_port));
1360
        }
1361
        // send/sendto error check
1362
        if (send_len != p->length())
1363
        {
1364
                if (send_len < 0)
1365
                {
1366
                        switch (errno)
1367
                        {
1368
                        case EINTR:
1369
                        case EMSGSIZE:
1370
                        case ENOBUFS:
1371
                        case ENOMEM:
1372
                        case ECONNREFUSED:
1373
                                ccwarn("(%d)%s datagram_tx p %p len %d errno %d",
1374
                                           fd, fd_to_state_name(fd), p, send_len, errno);
1375
                                break;
1376
                        case EAGAIN: // connection oriented datagram socket, not expected
1377
                        default:
1378
                                ccerr("(%d)%s datagram_tx p %p len %d errno %d",
1379
                                          fd, fd_to_state_name(fd), p, send_len, errno);
1380
                                break;
1381
                        }
1382
                }
1383
                else
1384
                {
1385
                        ccerr("(%d)%s datagram_tx p %p len %d short %d",
1386
                                  fd, fd_to_state_name(fd), p, p->length(), send_len);
1387
                        aver(0);
1388
                }
1389
        }
1390
}
1391

    
1392
void
1393
GDPv4Router::udp_listen_rx (int fd)
1394
{
1395
        WritablePacket *p;
1396
        ssize_t rx_len;
1397
        struct sockaddr_in from_sa_in;
1398
        socklen_t from_len = sizeof(from_sa_in);
1399
        gdp_name_ca_t *src;
1400
        gdp_pname_t _tmp_pname_1;
1401

    
1402
        p = Packet::make(GDP_CMD_ROUTER_NAME_MAX_SZ);
1403
        if (p == nullptr)
1404
        {
1405
                ccwarn("(%d)%s packet pool exhausted on %d",
1406
                           fd, fd_to_state_name(fd), GDP_CMD_ROUTER_NAME_MAX_SZ);
1407
                return;
1408
        }
1409

    
1410
        // MSG_TRUNC discards excess but rx_len will reflect full length
1411
        rx_len = recvfrom(fd, p->data(), p->length(), MSG_TRUNC,
1412
                                          (struct sockaddr *)&from_sa_in, &from_len);
1413
        if (rx_len <= 0 || rx_len > p->length())
1414
        {
1415
                ccdbg(DVVERB, "(%d)%s recvfrom len %d p len %d unexpected, drop",
1416
                          fd, fd_to_state_name(fd), rx_len, p->length());
1417
                p->kill();
1418
                return;
1419
        }
1420

    
1421
        otw_hlo_t *otw_hlo = (otw_hlo_t *)p->data();
1422
        
1423
        if (otw_hlo->ver != GDP_CHAN_PROTO_VERSION)
1424
        {
1425
                ccdbg(DVVERB, "(%d)%s rx unknown version 0x%x",
1426
                          fd, fd_to_state_name(fd), otw_hlo->ver);
1427
                p->kill();
1428
                return;
1429
        }
1430

    
1431
        ccdbg(DVERB, "(%d)%s recvfrom %s:%d len %d p %p len %d\n"
1432
                  "\tsrc[%s]",
1433
                  fd, fd_to_state_name(fd),
1434
                  IPAddress(from_sa_in.sin_addr).unparse().c_str(),
1435
                  ntohs(from_sa_in.sin_port), rx_len, p, p->length(),
1436
                  gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1437

    
1438
        // p->kill() called at function end
1439
        switch (otw_hlo->cmd)
1440
        {
1441

    
1442
        case GDP_CMD_ROUTER_NAME_REQ:
1443
        {
1444
                if (rx_len < (ssize_t) GDP_CMD_ROUTER_NAME_REQ_SZ)
1445
                {
1446
                        ccdbg(DVVERB, "(%d)%s rx name req short len %d, drop",
1447
                                  fd, fd_to_state_name(fd), rx_len);
1448
                        break;
1449
                }
1450

    
1451
                src = (gdp_name_ca_t *) otw_hlo->src;
1452
                
1453
                if (dst_map.count(*src))
1454
                {
1455
                        ccdbg(DVERB, "(%d)%s map entry exists\n"
1456
                                  "\tsrc[%s]",
1457
                                  fd, fd_to_state_name(fd),
1458
                                  gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1459
                }
1460
                else
1461
                {
1462
                        ccdbg(DVERB, "(%d)%s map entry not found\n"
1463
                                  "\tsrc[%s]",
1464
                                  fd, fd_to_state_name(fd),
1465
                                  gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1466
                }
1467
                                
1468
                // FD_UDP_CONNECT hello reply
1469
                otw_hlo->cmd = GDP_CMD_ROUTER_NAME_REP;
1470

    
1471
                // move src to dst, fill src
1472
                std::copy(otw_hlo->src, otw_hlo->src + sizeof(gdp_name_ca_t),
1473
                                  otw_hlo->dst);
1474
                std::copy(_gdpr, _gdpr + sizeof(gdp_name_ca_t),
1475
                                  otw_hlo->src);
1476
                datagram_tx(fd, p, &from_sa_in, sizeof(from_sa_in));
1477
        }
1478
        break;
1479
                
1480
        case GDP_CMD_ROUTER_NAME_ACK:
1481
        {
1482
                int nc_fd;
1483
                gdp_name_ca_t *cguid_ca;
1484
                dst_map_it_t cguid_dmi;
1485
                gdp_pname_t _tmp_pname_1;
1486

    
1487
                if (rx_len < (ssize_t) GDP_CMD_ROUTER_NAME_ACK_SZ)
1488
                {
1489
                        ccdbg(DVVERB, "(%d)%s rx name ack len %d short, drop",
1490
                                  fd, fd_to_state_name(fd), rx_len);
1491
                        break;
1492
                }
1493

    
1494
                // validate dst
1495
                if (memcmp(_gdpr, otw_hlo->dst, sizeof(gdp_name_t)) != 0)
1496
                {
1497
                        ccdbg(DVVERB, "(%d)%s rx name ack invalid dst",
1498
                                  fd, fd_to_state_name(fd));
1499
                        break;
1500
                }
1501

    
1502
                ccdbg(DVERB, "(%d)%s rx name ack len %d",
1503
                          fd, fd_to_state_name(fd), rx_len);
1504
                
1505
                // if new peer, then map and configure
1506
                cguid_ca = (gdp_name_ca_t *) otw_hlo->src;
1507
                cguid_dmi = dst_map.lower_bound(*cguid_ca);
1508
                if (cguid_dmi == dst_map.end() ||
1509
                        dst_map.key_comp()(*cguid_ca, cguid_dmi->first))
1510
                {
1511
                        cguid_dmi =
1512
                                dst_map.emplace_hint(cguid_dmi,
1513
                                                                         dst_map_t::value_type(*cguid_ca,
1514
                                                                                                                   dst_map_init));
1515
                }
1516
                else
1517
                {
1518
                        // client src may not be unique, but gdpr src must be unique
1519
                        ccdbg(DVERB, "(%d)%s rx name ack but already in map\n"
1520
                                  "\t[%s]",
1521
                                  fd, fd_to_state_name(fd),
1522
                                  gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1523
                        break;
1524
                }
1525
                ccdbg(DVERB, "(%d)%s rx name ack map insert\n"
1526
                          "\t[%s]",
1527
                          fd, fd_to_state_name(fd),
1528
                          gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1529

    
1530
                // move connection off listen socket...
1531
                if ((nc_fd = new_udp_connect(&from_sa_in, FD_UDP_LISTEN)) < 0)
1532
                {
1533
                        ccerr("(%d)%s name ack new_udp_connect failed",
1534
                                  fd, fd_to_state_name(fd));
1535
                        dst_map_recycle(fd, cguid_dmi);
1536
                        break;
1537
                }
1538

    
1539
                // udp listen has moved the connection to nc_fd
1540
                cguid_dmi->second.fd = nc_fd;
1541
                
1542
                // no udp listen fd retx timer to delete
1543
                avow((*_fdv)[nc_fd].retx_timer == nullptr);
1544

    
1545
                // directly connected gdp router
1546
                cguid_dmi->second.addr_state = ADDR_GDPR;
1547
                avow(cguid_dmi->second.map_timer_dmip == nullptr);
1548
                avow(cguid_dmi->second.map_timer == nullptr);
1549
                try {
1550
                        cguid_dmi->second.map_timer_dmip = new dst_map_it_t(cguid_dmi);
1551
                        cguid_dmi->second.map_timer =
1552
                                new Timer(static_addr_gdpr_connect_timer,
1553
                                                  cguid_dmi->second.map_timer_dmip);
1554
                }
1555
                catch (std::bad_alloc)
1556
                {
1557
                        ccerr("(%d)%s memory exhausted on new addr gdpr connect timer",
1558
                                  nc_fd, fd_to_state_name(nc_fd));
1559
                        delete cguid_dmi->second.map_timer_dmip;
1560
                        delete cguid_dmi->second.map_timer;
1561
                        dst_map.erase(cguid_dmi);
1562
                        fd_recycle(nc_fd);
1563
                        break;
1564
                }
1565
                ccdbg(DVVERB, "(%d)%s new addr gdp connect timer %p %p",
1566
                          nc_fd, fd_to_state_name(nc_fd),
1567
                          cguid_dmi->second.map_timer,
1568
                          cguid_dmi->second.map_timer_dmip);
1569
                cguid_dmi->second.map_timer->initialize(this, true);
1570
                cguid_dmi->second.map_timer->schedule_after(ADDR_GDPR_CONNECT_TS_DELTA);
1571
                
1572
                // fd tracks related dst maps
1573
                (*_fdv)[nc_fd].dst_mil.push_back(cguid_dmi);
1574
                ccdbg(DVVERB, "(%d)%s hlo fd push dguid[%s]",
1575
                          nc_fd, fd_to_state_name(nc_fd),                          
1576
                          gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1577

    
1578
                // diagnostics
1579
                display_state(nc_fd);
1580

    
1581
                (*_fdv)[nc_fd].peer_port = htons(otw_hlo->lport);
1582
                ccdbg(DNOISY, "(%d)%s rx peer port 0x%x 0x%x",
1583
                          nc_fd, fd_to_state_name(nc_fd),
1584
                          otw_hlo->lport, (*_fdv)[nc_fd].peer_port);
1585

    
1586
                ccdbg(DINFO, "(%d)%s new udp connect (%d)%s",
1587
                           fd, fd_to_state_name(fd), nc_fd, fd_to_state_name(nc_fd));
1588

    
1589
                if (new_ssl(nc_fd) < 0)
1590
                {
1591
                        ccerr("(%d)%s name ack new_ssl failed",
1592
                                  nc_fd, fd_to_state_name(nc_fd));
1593
                        fd_recycle(nc_fd);
1594
                        break;
1595
                }
1596
        }
1597
        break;
1598
        
1599
        case GDP_CMD_ROUTER_NAME_REP:
1600
        default:
1601
        {
1602
                ccdbg(DVVERB, "(%d)%s rx invalid cmd %d",
1603
                          fd, fd_to_state_name(fd), otw_hlo->cmd);
1604
        }
1605
        break;
1606
        
1607
        } // switch
1608

    
1609
        p->kill();
1610
}
1611

    
1612
void
1613
GDPv4Router::udp_connect_rx (int fd)
1614
{
1615
        WritablePacket *p;
1616
        ssize_t rx_len;
1617
        gdp_name_ca_t *cguid_ca;
1618
        dst_map_it_t cguid_dmi;
1619
        gdp_pname_t _tmp_pname_1;
1620

    
1621
        p = Packet::make(GDP_CMD_ROUTER_NAME_MAX_SZ);
1622
        if (p == nullptr)
1623
        {
1624
                ccwarn("(%d)%s packet pool exhausted on %d",
1625
                           fd, fd_to_state_name(fd), GDP_CMD_ROUTER_NAME_MAX_SZ);
1626
                return;
1627
        }
1628

    
1629
        // MSG_TRUNC discards excess but rx_len will reflect full length
1630
        rx_len = recv(fd, p->data(), p->length(), MSG_TRUNC);
1631
        if (rx_len <= 0 || rx_len > p->length())
1632
        {
1633
                switch (errno)
1634
                {
1635

    
1636
                case ECONNREFUSED:
1637
                {
1638
                        ccdbg(DINFO, "(%d)%s peer refused udp connection (errno %d)",
1639
                                   fd, fd_to_state_name(fd), errno);
1640
                }
1641
                break;
1642

    
1643
                default:
1644
                {
1645
                        ccdbg(DINFO, "(%d)%s recv len %d p len %d errno %d",
1646
                                   fd, fd_to_state_name(fd), rx_len, p->length(), errno);
1647
                }
1648
                break;
1649

    
1650
                } // switch
1651

    
1652
                p->kill();
1653
                return;
1654
        }
1655
        
1656
        otw_hlo_t *otw_hlo = (otw_hlo_t *)p->data();
1657
        
1658
        if (otw_hlo->ver != GDP_CHAN_PROTO_VERSION)
1659
        {
1660
                ccdbg(DVVERB, "(%d)%s rx unknown version 0x%x",
1661
                          fd, fd_to_state_name(fd), otw_hlo->ver);
1662
                p->kill();
1663
                return;
1664
        }
1665

    
1666
        ccdbg(DVERB, "(%d)%s recv len %d p %p len %d\n"
1667
                  "\tsrc[%s]",
1668
                  fd, fd_to_state_name(fd),
1669
                  rx_len, p, p->length(),
1670
                  gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1671

    
1672
        // p->kill() called at function end
1673
        switch (otw_hlo->cmd)
1674
        {
1675

    
1676
        case GDP_CMD_ROUTER_NAME_REP:
1677
        {
1678
                if (rx_len < (ssize_t) GDP_CMD_ROUTER_NAME_REP_SZ)
1679
                {
1680
                        ccdbg(DVVERB, "(%d)%s rx name rep short len %d, drop",
1681
                                  fd, fd_to_state_name(fd), rx_len);
1682
                        break;
1683
                }
1684

    
1685
                // validate dst
1686
                if (memcmp(_gdpr, otw_hlo->dst, sizeof(gdp_name_ca_t)) != 0)
1687
                {
1688
                        ccdbg(DVERB, "(%d)%s rx name rep unknown dst[%s], drop",
1689
                                  fd, fd_to_state_name(fd),
1690
                                  gdp_printable_name(otw_hlo->dst, _tmp_pname_1));
1691
                        break;
1692
                }
1693

    
1694
                // insert if not in map, otherwise stop exchange here
1695
                cguid_ca = (gdp_name_ca_t *) otw_hlo->src;
1696
                cguid_dmi = dst_map.lower_bound(*cguid_ca);
1697
                if (cguid_dmi == dst_map.end() ||
1698
                        dst_map.key_comp()(*cguid_ca, cguid_dmi->first))
1699
                {
1700
                        cguid_dmi =
1701
                                dst_map.emplace_hint(cguid_dmi,
1702
                                                                         dst_map_t::value_type(*cguid_ca,
1703
                                                                                                                   dst_map_init));
1704
                }
1705
                else
1706
                {
1707
                        ccdbg(DVERB, "(%d)%s rx name rep set udp connect dupe\n"
1708
                                  "\tsrc[%s]",
1709
                                  fd, fd_to_state_name(fd),
1710
                                  gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1711
                        // delete udp connect fd retx timer (uncheduled by delete)
1712
                        avow((*_fdv)[fd].retx_timer != nullptr);
1713
                        delete (*_fdv)[fd].retx_timer;
1714
                        (*_fdv)[fd].retx_timer = nullptr;
1715
                        // udp connect dupe is dormant until reanimated
1716
                        (*_fdv)[fd].fd_state = FD_UDP_CONNECT_DUPE;
1717
                        break;
1718
                }
1719

    
1720
                // udp connect is on fd
1721
                cguid_dmi->second.fd = fd;
1722

    
1723
                // delete udp connect fd retx timer (uncheduled by delete)
1724
                avow((*_fdv)[fd].retx_timer != nullptr);
1725
                delete (*_fdv)[fd].retx_timer;
1726
                (*_fdv)[fd].retx_timer = nullptr;
1727

    
1728
                // directly connected router
1729
                cguid_dmi->second.addr_state = ADDR_GDPR;
1730
                avow(cguid_dmi->second.map_timer_dmip == nullptr);
1731
                avow(cguid_dmi->second.map_timer == nullptr);
1732
                try {
1733
                        cguid_dmi->second.map_timer_dmip = new dst_map_it_t(cguid_dmi);
1734
                        cguid_dmi->second.map_timer =
1735
                                new Timer(static_addr_gdpr_connect_timer,
1736
                                                  cguid_dmi->second.map_timer_dmip);
1737
                }
1738
                catch (std::bad_alloc)
1739
                {
1740
                        ccerr("(%d)%s memory exhausted on new addr gdpr connect timer",
1741
                                  fd, fd_to_state_name(fd));
1742
                        delete cguid_dmi->second.map_timer_dmip;
1743
                        delete cguid_dmi->second.map_timer;
1744
                        dst_map.erase(cguid_dmi);
1745
                        fd_recycle(fd);
1746
                        break;
1747
                }
1748
                ccdbg(DVVERB, "(%d)%s new addr gdp connect timer %p %p",
1749
                          fd, fd_to_state_name(fd),
1750
                          cguid_dmi->second.map_timer,
1751
                          cguid_dmi->second.map_timer_dmip);
1752
                cguid_dmi->second.map_timer->initialize(this, true);
1753
                cguid_dmi->second.map_timer->schedule_after(ADDR_GDPR_CONNECT_TS_DELTA);
1754
                
1755
                // fd tracks related map entries
1756
                (*_fdv)[fd].dst_mil.push_back(cguid_dmi);
1757
                ccdbg(DVVERB, "(%d)%s hlo fd push map\n"
1758
                          "\tsrc[%s]",
1759
                          fd, fd_to_state_name(fd),
1760
                          gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1761

    
1762
                ccdbg(DVERB, "(%d)%s rx name rep map insert\n"
1763
                          "\tsrc[%s]",
1764
                          fd, fd_to_state_name(fd),
1765
                          gdp_printable_name(otw_hlo->src, _tmp_pname_1));
1766
                // diagnostics
1767
                display_state(fd);
1768

    
1769
                // tx GDP_CMD_ROUTER_NAME_ACK msg
1770
                otw_hlo->cmd = GDP_CMD_ROUTER_NAME_ACK;
1771
                std::copy(otw_hlo->src,
1772
                                  otw_hlo->src + sizeof(gdp_name_ca_t),
1773
                                  otw_hlo->dst);
1774
                std::copy(_gdpr, _gdpr + sizeof(gdp_name_ca_t),
1775
                                  otw_hlo->src);
1776
                otw_hlo->lport = (*_sav)[_SAV_INDEX_UDP].sin_port;
1777
                datagram_tx(fd, p, nullptr, 0);
1778

    
1779
                if (new_ssl(fd) < 0)
1780
                {
1781
                        ccerr("(%d)%s name reply new_ssl error", fd, fd_to_state_name(fd));
1782
                        fd_recycle(fd);
1783
                }
1784
        }
1785
        break;
1786
        
1787
        default:
1788
        {
1789
                ccdbg(DVVERB, "(%d)%s unknown cmd %d",
1790
                          fd, fd_to_state_name(fd), otw_hlo->cmd);
1791
        }
1792
        break;
1793
        
1794
        } // switch
1795

    
1796
        p->kill();
1797
}
1798

    
1799
void
1800
GDPv4Router::udp_connect_timer (Timer *t, int fd)
1801
{
1802
        WritablePacket *p;
1803
        otw_hlo_t *otw_hlo;
1804

    
1805
        ccdbg(DVVERB, "\n(%d)%s udp connect timer %p", fd, fd_to_state_name(fd), t);
1806

    
1807
        avow(fd > _fd_directory && fd <= _fd_last_configured);
1808
        avow((*_fdv)[fd].fd_state == FD_UDP_CONNECT);
1809

    
1810
        p = Packet::make(GDP_CMD_ROUTER_NAME_REQ_SZ);
1811
        if (p != nullptr)
1812
        {
1813
                otw_hlo = (otw_hlo_t *)p->data();
1814
                otw_hlo->ver = GDP_CHAN_PROTO_VERSION;
1815
                otw_hlo->cmd = GDP_CMD_ROUTER_NAME_REQ;
1816
                otw_hlo->id = htons(_dr_id++);
1817
                memset(otw_hlo->dst, 0x00, sizeof(gdp_name_ca_t));
1818
                std::copy(_gdpr_ca->data(), _gdpr_ca->data() + sizeof(gdp_name_ca_t),
1819
                                  otw_hlo->src);
1820

    
1821
                ccdbg(DINFO, "\n(%d)%s udp connect name req", fd, fd_to_state_name(fd));
1822
                datagram_tx(fd, p, nullptr, 0);
1823
        }
1824
        
1825
        // reschedule timer
1826
        t->schedule_after(FD_UDP_CONNECT_TS_DELTA);
1827
}
1828

    
1829
void
1830
ssl_info_callback (const SSL *ssl, int where, int ret, const char *name)
1831
{
1832
        ccdbg(DVVERB, "ssl_info_callback %s 0x%x %d %s %s", name, where, ret,
1833
                  SSL_state_string(ssl), SSL_state_string_long(ssl));
1834
}
1835

    
1836
void
1837
server_ssl_info_callback (const SSL *ssl, int where, int ret)
1838
{
1839
        ssl_info_callback(ssl, where, ret, "dtls listen");
1840
}
1841

    
1842
void
1843
client_ssl_info_callback (const SSL *ssl, int where, int ret)
1844
{
1845
        ssl_info_callback(ssl, where, ret, "dtls connect");
1846
}
1847

    
1848
int GDPv4Router::new_ssl (int fd)
1849
{
1850
        BIO *new_bio_dgram;
1851
        struct timeval timeout;
1852

    
1853
        avow(fd > _fd_directory && fd < (int)(*_fdv).size());
1854
        avow((*_fdv)[fd].ssl == nullptr);
1855
        avow((*_fdv)[fd].fd_state == FD_UDP_CONNECT ||
1856
                 (*_fdv)[fd].fd_state == FD_UDP_LISTEN);
1857

    
1858
        if (((*_fdv)[fd].ssl = SSL_new(_ssl_ctx)) == nullptr)
1859
        {
1860
                // caller owns fd_recycle
1861
                return -1;
1862
        }
1863

    
1864
        if (_debug_level >= DVERB)
1865
        {
1866
                SSL_set_info_callback((*_fdv)[fd].ssl, server_ssl_info_callback);
1867
        }
1868
        // SSL_CTX_set_info_callback() diagnostics have not been required, so far
1869
        // (dated) online info say read_ahead is required, but currently not needed
1870
        // SSL_CTX_set_read_ahead (ssl_ctx, 1);
1871

    
1872
        if ((new_bio_dgram = BIO_new_dgram(fd, BIO_NOCLOSE)) == nullptr)
1873
        {
1874
                ccerr("BIO_new_dgram failed");
1875
                // caller owns fd_recycle
1876
                return -1;
1877
        }
1878
        
1879
        // timeouts
1880
        timeout.tv_sec = 0;
1881
        timeout.tv_usec = DGRAM_RCV_TIMEOUT;
1882
        BIO_ctrl(new_bio_dgram, BIO_CTRL_DGRAM_SET_RECV_TIMEOUT, 0, &timeout);
1883
        timeout.tv_usec = DGRAM_SND_TIMEOUT;
1884
        BIO_ctrl(new_bio_dgram, BIO_CTRL_DGRAM_SET_SEND_TIMEOUT, 0, &timeout);
1885
        // mtu discovery
1886
        BIO_ctrl(new_bio_dgram, BIO_CTRL_DGRAM_MTU_DISCOVER, 0, NULL);
1887
        // bind dgram bio to ssl
1888
        SSL_set_bio((*_fdv)[fd].ssl, new_bio_dgram, new_bio_dgram);
1889
        if ((*_fdv)[fd].fd_state == FD_UDP_CONNECT)
1890
        {
1891
                // peer listened for hello, however we listen for initial SSL handshake
1892
                SSL_set_accept_state((*_fdv)[fd].ssl);
1893
                // delete udp connect fd retx timer (uncheduled by delete)
1894
                delete (*_fdv)[fd].retx_timer;
1895
                (*_fdv)[fd].retx_timer = nullptr;
1896
                // udp connect moves to dtls accept state
1897
                (*_fdv)[fd].fd_state = FD_DTLS_ACCEPT;
1898
                ssl_accept(fd);
1899
        }
1900
        else // (*_fdv)[fd].fd_state == FD_UDP_LISTEN
1901
        {
1902
                struct sockaddr_in peer_sa_in;
1903
                int peer_sa_in_len = sizeof(peer_sa_in);
1904
                
1905
                if (getpeername(fd, (struct sockaddr *) &peer_sa_in,
1906
                                                (socklen_t *) &peer_sa_in_len) < 0)
1907
                {
1908
                        ccerr("getpeername %s", strerror (errno));
1909
                        // caller owns fd_recycle
1910
                        return -1;
1911
                }
1912
                ccdbg(DVVERB, "(%d)%s BIO ctrl0 set connected %s:%d",
1913
                          fd, fd_to_state_name(fd),
1914
                          IPAddress(peer_sa_in.sin_addr).unparse().c_str(),
1915
                          ntohs(peer_sa_in.sin_port));
1916
                // mandatory for ssl, but passing the wrong address is quietly accepted!
1917
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
1918
                (void)BIO_ctrl_set_connected(new_bio_dgram, &peer_sa_in);
1919
#else
1920
                (void)BIO_ctrl_set_connected(new_bio_dgram, 1, &peer_sa_in);
1921
#endif
1922
                // udp listen moves to dtls handshake state
1923
                (*_fdv)[fd].fd_state = FD_DTLS_DO_HANDSHAKE;
1924
                SSL_set_connect_state((*_fdv)[fd].ssl);
1925
                ssl_handshake(fd);
1926
        }
1927

    
1928
        ccdbg(DVERB, "(%d)%s new ssl", fd, fd_to_state_name(fd));
1929

    
1930
        return fd;
1931
}
1932

    
1933
int
1934
GDPv4Router::ssl_accept (int fd)
1935
{
1936
        int ssl_status;
1937
        int ssl_error;
1938
        dst_map_it_t cguid_dmi;
1939

    
1940
        ssl_status = SSL_accept((*_fdv)[fd].ssl);
1941
        if (ssl_status <= 0)
1942
        {
1943
                ssl_error = SSL_get_error((*_fdv)[fd].ssl, ssl_status);
1944
                // ssl_error holds an enum, not bit flags
1945
                switch (ssl_error)
1946
                {
1947
                case SSL_ERROR_WANT_READ:
1948
                        // router fd is always SELECT_READ (add/remove performs poorly)
1949
                        break;
1950
                case SSL_ERROR_WANT_WRITE:
1951
                        // this condition has not been spotted in the wild, so far...
1952
                        aver(0);
1953
                        break;
1954
                default:
1955
                        ccerr("(%d)%s ssl_accept unhandled ssl error (%d)",
1956
                                  fd, fd_to_state_name(fd), ssl_error);
1957
                        aver(0);
1958
                        break;
1959
                }
1960
                ccdbg(DVVERB, "(%d)%s ssl_accept error string %s",
1961
                          fd, fd_to_state_name(fd),
1962
                          ERR_error_string(ERR_get_error(), NULL));
1963
        }
1964
        else
1965
        {
1966
                // dtls accept moves to dtls server state
1967
                (*_fdv)[fd].fd_state = FD_DTLS_SERVER;
1968
                cguid_dmi = (*_fdv)[fd].dst_mil.front();
1969
                avow(cguid_dmi->second.map_timer != nullptr);
1970
                cguid_dmi->second.map_timer->assign(static_addr_gdpr_timer,
1971
                                                                                        cguid_dmi->second.map_timer_dmip);
1972
                cguid_dmi->second.map_timer->schedule_after(ADDR_GDPR_NOW_TS_DELTA);
1973
                ccdbg(DVVERB, "(%d)%s addr gdp connect timer reassigned to gdpr %p %p",
1974
                          fd, fd_to_state_name(fd),
1975
                          cguid_dmi->second.map_timer, cguid_dmi->second.map_timer_dmip);
1976
        }
1977

    
1978
        ccdbg(DINFO, "(%d)%s ssl_accept status %d (%s)",
1979
                   fd, fd_to_state_name(fd), ssl_status,
1980
                   (ssl_status <= 0 ? _SSL_ERROR_NAME[ssl_error] : "-"));
1981
        
1982
        return ssl_status;
1983
}
1984

    
1985
int
1986
GDPv4Router::ssl_handshake (int fd)
1987
{
1988
        int ssl_status;
1989
        int ssl_error;
1990
        dst_map_it_t cguid_dmi;
1991

    
1992
        ssl_status = SSL_do_handshake((*_fdv)[fd].ssl);
1993
        if (ssl_status <= 0)
1994
        {
1995
                ssl_error = SSL_get_error((*_fdv)[fd].ssl, ssl_status);
1996
                // ssl_error holds an enum, not bit flags
1997
                switch (ssl_error)
1998
                {
1999
                case SSL_ERROR_WANT_READ:
2000
                        // router fd is always SELECT_READ (add/remove performs poorly)
2001
                        break;
2002
                case SSL_ERROR_WANT_WRITE:
2003
                        // this condition has not been spotted in the wild, so far...
2004
                        aver(0);
2005
                        break;
2006
                default:
2007
                        ccerr("(%d)%s ssl_handshake unhandled ssl error (%d)",
2008
                                  fd, fd_to_state_name(fd), ssl_error);
2009
                        aver(0);
2010
                        break;
2011
                }
2012
                ccdbg(DVVERB, "(%d)%s ssl_handshake error string %s",
2013
                          fd, fd_to_state_name(fd),
2014
                          ERR_error_string(ERR_get_error(), NULL));
2015
        }
2016
        else
2017
        {
2018
                // dtls do handshake moves to dtls client state
2019
                (*_fdv)[fd].fd_state = FD_DTLS_CLIENT;
2020
                cguid_dmi = (*_fdv)[fd].dst_mil.front();
2021
                avow(cguid_dmi->second.map_timer != nullptr);
2022
                cguid_dmi->second.map_timer->assign(static_addr_gdpr_timer,
2023
                                                                                        cguid_dmi->second.map_timer_dmip);
2024
                cguid_dmi->second.map_timer->schedule_after(ADDR_GDPR_NOW_TS_DELTA);
2025
                ccdbg(DVVERB, "(%d)%s addr gdp connect timer reassigned to gdpr %p %p",
2026
                          fd, fd_to_state_name(fd),
2027
                          cguid_dmi->second.map_timer, cguid_dmi->second.map_timer_dmip);
2028
        }        
2029

    
2030
        ccdbg(DINFO, "(%d)%s ssl_handshake status %d (%s)",
2031
                   fd, fd_to_state_name(fd), ssl_status,
2032
                   (ssl_status <= 0 ? _SSL_ERROR_NAME[ssl_error] : "-"));
2033

    
2034
        return ssl_status;
2035
}
2036

    
2037
void
2038
GDPv4Router::advertisement_rx (int fd, WritablePacket *p)
2039
{
2040
        otw_gdp_t *otw_gdp;
2041
        uint16_t dlen;
2042
        uint32_t mlen;
2043
        WritablePacket *dp;
2044
        otw_dir_t *otw_dir;
2045
        gdp_name_ca_t *dguid_ca;
2046
        dst_map_it_t dguid_dmi;
2047
        gdp_pname_t _tmp_pname_1;
2048
        gdp_pname_t _tmp_pname_2;
2049
        bool gdpc_advert;
2050
#ifdef TEST_ADVERT_LOAD
2051
        uint32_t test_advert_load_factor = 100;
2052
#endif
2053
        
2054
        ccdbg(DVVERB, "(%d)%s advertisement rx p %p", fd, fd_to_state_name(fd), p);
2055
        
2056
#ifdef TEST_ADVERT_LOAD
2057
test_advert_load:
2058
#endif
2059
        
2060
        // read fields through on-the-wire (packed) struct layout
2061
        avow(p != nullptr);
2062
        otw_gdp = (otw_gdp_t *) p->data();
2063
        dlen = ntohs(otw_gdp->payload_len);
2064
        mlen = (otw_gdp->hdr_len * 4) + dlen;
2065
        ccdbg(DVERB, "(%d)%s advert rx hlen %d dlen %d mlen %d\n"
2066
                  "\tsrc[%s]\n"
2067
                  "\tdst[%s]",
2068
                  fd, fd_to_state_name(fd), otw_gdp->hdr_len, dlen, mlen,
2069
                  gdp_printable_name(otw_gdp->src, _tmp_pname_1),
2070
                  gdp_printable_name(otw_gdp->dst, _tmp_pname_2));
2071

    
2072
        // directory p
2073
        dp = Packet::make(offsetof(otw_dir_t, eguid[1][0]));
2074
        if (dp == nullptr)
2075
        {
2076
                ccwarn("(%d)%s packet pool exhausted on %d",
2077
                           fd, fd_to_state_name(fd), offsetof(otw_dir_t, eguid[1][0]));
2078
                goto advertisement_rx_cleanup_p;
2079
        }
2080

    
2081
        otw_dir = (otw_dir_t *) dp->data();
2082
        otw_dir->ver = GDP_CHAN_PROTO_VERSION;
2083
        otw_dir->id = htons(_dr_id++);
2084
        otw_dir->cmd = GDP_CMD_DIR_ADD;
2085
        
2086
        dguid_ca = (gdp_name_ca_t *) otw_gdp->dst;
2087
        dguid_dmi = dst_map.lower_bound(*dguid_ca);
2088
        if (dguid_dmi != dst_map.end() &&
2089
                !(dst_map.key_comp()(*dguid_ca, dguid_dmi->first)))
2090
        {
2091
                // resolve conflicts between incoming advert and map state
2092
                switch (dguid_dmi->second.addr_state)
2093
                {
2094

    
2095
                case ADDR_GDPC:
2096
                {
2097
                        if (fd != dguid_dmi->second.fd ||
2098
                                memcmp(otw_gdp->src, otw_gdp->dst, sizeof(gdp_name_ca_t)) != 0)
2099
                        {
2100
                                ccwarn("(%d)%s advert vs gdpc, drop advert\n"
2101
                                           "\tdguid[%s]",
2102
                                           fd, fd_to_state_name(fd),
2103
                                           gdp_printable_name(otw_gdp->dst, _tmp_pname_1));
2104
                                dp->kill();
2105
                                goto advertisement_rx_cleanup_p;
2106
                        }
2107
                }
2108
                break;
2109

    
2110
                case ADDR_NHOP:
2111
                {
2112
                        if (fd != dguid_dmi->second.fd ||
2113
                                memcmp(otw_gdp->src, otw_gdp->dst, sizeof(gdp_name_ca_t)) == 0)
2114
                        {
2115
                                ccdbg(DINFO, "(%d)%s advert vs nhop, recycle nhop\n"
2116
                                           "\tdguid[%s]",
2117
                                           fd, fd_to_state_name(fd),
2118
                                           gdp_printable_name(otw_gdp->dst, _tmp_pname_1));
2119
                                fd_detach_map(dguid_dmi->second.fd, dguid_dmi);
2120
                                dst_map_recycle(dguid_dmi->second.fd, dguid_dmi);
2121
                                dguid_dmi = dst_map.end();
2122

    
2123
                                // diagnostics
2124
                                display_state(fd);
2125
                        }
2126
                }
2127
                break;
2128

    
2129
                case ADDR_FIND:
2130
                {
2131
                        ccdbg(DINFO, "(%d)%s advert vs find, recycle find\n"
2132
                                   "\tdguid[%s]",
2133
                                   fd, fd_to_state_name(fd),
2134
                                   gdp_printable_name(otw_gdp->dst, _tmp_pname_1));
2135
                        map_no_route(fd, dguid_dmi);
2136
                        dst_map_recycle(fd, dguid_dmi);
2137
                        dguid_dmi = dst_map.end();
2138

    
2139
                        // diagnostics
2140
                        display_state(fd);
2141
                }
2142
                break;
2143

    
2144
                case ADDR_MINE:
2145
                case ADDR_GDPR:
2146
                default:
2147
                {
2148
                        ccwarn("(%d)%s advert conflict, drop advert\n"
2149
                                   "\tdguid[%s]",
2150
                                   fd, fd_to_state_name(fd),
2151
                                   gdp_printable_name(otw_gdp->dst, _tmp_pname_1));
2152
                        dp->kill();
2153
                        goto advertisement_rx_cleanup_p;
2154
                }
2155
                break;
2156

    
2157
                } // switch
2158
        }
2159

    
2160
    // ADDR_GDPC (src == dst)
2161
        if (memcmp(otw_gdp->src, otw_gdp->dst, sizeof(gdp_name_ca_t)) == 0)
2162
        {
2163
                if (dguid_dmi == dst_map.end() ||
2164
                        dst_map.key_comp()(*dguid_ca, dguid_dmi->first))
2165
                {
2166
                        if ((*_fdv)[fd].dst_mil.size() > 0)
2167
                        {
2168
                                ccwarn("(%d)%s reject add map[%s]\n"
2169
                                           "\tfd has owner[%s]%s",
2170
                                           fd, fd_to_state_name(fd),
2171
                                           gdp_printable_name(otw_gdp->src, _tmp_pname_1),
2172
                                           gdp_printable_name(
2173
                                                   (*_fdv)[fd].dst_mil.front()->first.data(),
2174
                                                   _tmp_pname_2),
2175
                                           addr_to_state_name(
2176
                                                   (*_fdv)[fd].dst_mil.front()->second.addr_state));
2177
                                dp->kill();
2178
                                goto advertisement_rx_cleanup_p;
2179
                        }
2180
                        
2181
                        dguid_dmi =
2182
                                dst_map.emplace_hint(dguid_dmi,
2183
                                                                         dst_map_t::value_type(*dguid_ca,
2184
                                                                                                                   dst_map_init));
2185
                        // directly connected gdp client
2186
                        dguid_dmi->second.fd = fd;
2187
                        dguid_dmi->second.addr_state = ADDR_GDPC;
2188
                        avow(dguid_dmi->second.map_timer_dmip == nullptr);
2189
                        avow(dguid_dmi->second.map_timer == nullptr);
2190
                        try {
2191
                                dguid_dmi->second.map_timer_dmip = new dst_map_it_t(dguid_dmi);
2192
                                if (!_single_router_mode)
2193
                                {
2194
                                        dguid_dmi->second.map_timer =
2195
                                                new Timer(static_addr_gdpc_timer,
2196
                                                                  dguid_dmi->second.map_timer_dmip);
2197
                                }
2198
                        }
2199
                        catch (std::bad_alloc)
2200
                        {
2201
                                ccerr("(%d)%s memory exhausted on new addr gdpc",
2202
                                          fd, fd_to_state_name(fd));
2203
                                delete dguid_dmi->second.map_timer_dmip;
2204
                                delete dguid_dmi->second.map_timer;
2205
                                dst_map.erase(dguid_dmi);
2206
                                fd_recycle(fd);
2207
                                dp->kill();
2208
                                goto advertisement_rx_cleanup_p;
2209
                        }
2210
                        if (!_single_router_mode)
2211
                        {
2212
                                ccdbg(DVVERB, "(%d)%s new addr gdpc timer %p %p",
2213
                                          fd, fd_to_state_name(fd),
2214
                                          dguid_dmi->second.map_timer,
2215
                                          dguid_dmi->second.map_timer_dmip);
2216
                                dguid_dmi->second.map_timer->initialize(this, true);
2217
                                dguid_dmi->second.map_timer->schedule_after(
2218
                                        Timestamp(0, TIMESTAMP_US(_advert_timer)));
2219
                        }
2220
                        
2221
                        ccdbg(DVVERB, "(%d)%s add map[%s]%s",
2222
                                  fd, fd_to_state_name(fd),
2223
                                  gdp_printable_name(otw_gdp->src, _tmp_pname_1),
2224
                                  addr_to_state_name(dguid_dmi->second.addr_state));
2225
                
2226
                        // fd tracks related map entries
2227
                        (*_fdv)[fd].dst_mil.push_back(dguid_dmi);
2228
                        ccdbg(DVVERB, "(%d)%s fd push map[%s]",
2229
                                  fd, fd_to_state_name(fd),
2230
                                  gdp_printable_name(otw_gdp->src, _tmp_pname_1));
2231

    
2232
                        // diagnostics
2233
                        display_state(fd);
2234
                }
2235

    
2236
                // advertise ADDR_GDPC reachability via ADDR_MINE
2237
                memcpy(otw_dir->eguid[0], _gdpr, sizeof(gdp_name_t));
2238
                memcpy(otw_dir->dguid, otw_gdp->src, sizeof(gdp_name_t));
2239
                gdpc_advert = true;
2240
        }
2241
        else
2242
        {
2243
                // SINGLE ROUTER MODE
2244
                if (_single_router_mode &&
2245
                        (dguid_dmi == dst_map.end() ||
2246
                         dst_map.key_comp()(*dguid_ca, dguid_dmi->first)))
2247
                {
2248
                        dguid_dmi =
2249
                                dst_map.emplace_hint(dguid_dmi,
2250
                                                                         dst_map_t::value_type(*dguid_ca,
2251
                                                                                                                   dst_map_init));
2252
                        dguid_dmi->second.fd = fd;
2253
                        dguid_dmi->second.addr_state = ADDR_NHOP;
2254
                        try {
2255
                                dguid_dmi->second.map_timer_dmip = new dst_map_it_t(dguid_dmi);
2256
                        }
2257
                        catch (std::bad_alloc)
2258
                        {
2259
                                ccerr("(%d)%s memory exhausted on new addr nhop",
2260
                                          fd, fd_to_state_name(fd));
2261
                                delete dguid_dmi->second.map_timer_dmip;
2262
                                dst_map.erase(dguid_dmi);
2263
                                fd_recycle(fd);
2264
                                dp->kill();
2265
                                goto advertisement_rx_cleanup_p;
2266
                        }
2267
                        // _single_router_mode ADDR_NHOP does not need to start a map timer
2268
                        ccdbg(DVVERB, "(%d)%s add map[%s]%s",
2269
                                  fd, fd_to_state_name(fd),
2270
                                  gdp_printable_name(otw_gdp->src, _tmp_pname_1),
2271
                                  addr_to_state_name(dguid_dmi->second.addr_state));
2272
                
2273
                        // fd tracks related map entries
2274
                        (*_fdv)[fd].dst_mil.push_back(dguid_dmi);
2275
                        ccdbg(DVVERB, "(%d)%s fd push map[%s]",
2276
                                  fd, fd_to_state_name(fd),
2277
                                  gdp_printable_name(otw_gdp->src, _tmp_pname_1));
2278

    
2279
                        dst_map_it_t eguid_dmi = (*_fdv)[fd].dst_mil.front();
2280
                        int fd_eguid = eguid_dmi->second.fd;
2281
                        ccdbg(DINFO, "(%d)%s single router mode map\n"
2282
                                   "\tdguid[%s]%s via\n"
2283
                                   "\teguid[%s]%s on (%d)%s",
2284
                                   fd, fd_to_state_name(fd),
2285
                                   gdp_printable_name(dguid_dmi->first.data(), _tmp_pname_1),
2286
                                   addr_to_state_name(dguid_dmi->second.addr_state),
2287
                                   gdp_printable_name(eguid_dmi->first.data(), _tmp_pname_2),
2288
                                   addr_to_state_name(eguid_dmi->second.addr_state),
2289
                                   fd_eguid, fd_to_state_name(fd_eguid));
2290
                        
2291
                        // diagnostics
2292
                        display_state(fd);
2293
                }
2294
                
2295
                // advertise ADDR_NHOP reachability via ADDR_GDPC
2296
                memcpy(otw_dir->eguid[0], otw_gdp->src, sizeof(gdp_name_t));                
2297
                memcpy(otw_dir->dguid, otw_gdp->dst, sizeof(gdp_name_t));
2298
                gdpc_advert = false;                
2299
        }
2300

    
2301
        // SINGLE ROUTER MODE
2302
        if (_single_router_mode)
2303
        {
2304
                dp->kill();
2305
                goto advertisement_rx_cleanup_p;
2306
        }
2307

    
2308
        // advertisement retx via ADDR_GDPC timer
2309
        dguid_ca = (gdp_name_ca_t *) otw_gdp->src;
2310
        dguid_dmi = dst_map.find(*dguid_ca);
2311
        if (dguid_dmi != dst_map.end())
2312
        {
2313
                if (dguid_dmi->second.dplist.size())
2314
                {        
2315
                        if (gdpc_advert)
2316
                        {
2317
                                ccwarn("(%d)%s gdpc advert period (re)started with size (%d)",
2318
                                           fd, fd_to_state_name(fd),
2319
                                           dguid_dmi->second.dplist.size());
2320
                        }
2321
                }
2322
                else
2323
                {
2324
                        // periodic full adverts and singletons (new)
2325
                        ccdbg(DINFO, "(%d)%s gdpc advert %s",
2326
                                  fd, fd_to_state_name(fd), gdpc_advert ? "periodic" : "new");
2327
                        dguid_dmi->second.dp_timer_count = 0;
2328
                        dguid_dmi->second.dp_acks = 0;
2329
                        dguid_dmi->second.dp_loss = 0;
2330
                        dguid_dmi->second.start_ts = Timestamp::now_steady();
2331
                        dguid_dmi->second.dp_algo = ADDR_GDPC_ADVERT_EVAL_END;
2332
                        dguid_dmi->second.map_timer->schedule_now();
2333
                }
2334

    
2335
                dguid_dmi->second.dplist.push_back(dp);
2336
#ifdef TEST_ADVERT_LOAD
2337
                if (--test_advert_load_factor)
2338
                {
2339
                        goto test_advert_load;
2340
                }
2341
#endif
2342
        }
2343

    
2344
advertisement_rx_cleanup_p:
2345
        
2346
        // recycle advert p chain (net4 not sending multi-advert pdus, so pedantic)
2347
        p_chain_recycle(p);
2348
}
2349

    
2350
void
2351
GDPv4Router::withdraw_rx (int fd, WritablePacket *p)
2352
{
2353
        otw_gdp_t *otw_gdp;
2354
        uint16_t dlen;
2355
        uint32_t mlen;
2356
        WritablePacket *dp;
2357
        otw_dir_t *otw_dir;
2358
        gdp_name_ca_t *dguid_ca;
2359
        dst_map_it_t dguid_dmi;
2360
        gdp_pname_t _tmp_pname_1;
2361
        gdp_pname_t _tmp_pname_2;
2362

    
2363
        ccdbg(DVVERB, "(%d)%s withdraw rx p %p", fd, fd_to_state_name(fd), p);
2364

    
2365
        // read fields through on-the-wire (packed) struct layout
2366
        avow(p != nullptr);
2367
        otw_gdp = (otw_gdp_t *) p->data();
2368
        dlen = ntohs(otw_gdp->payload_len);
2369
        mlen = (otw_gdp->hdr_len * 4) + dlen;
2370
        ccdbg(DVERB, "(%d)%s withdraw rx hlen %d dlen %d mlen %d\n"
2371
                  "\tsrc[%s]\n"
2372
                  "\tdst[%s]",
2373
                  fd, fd_to_state_name(fd), otw_gdp->hdr_len, dlen, mlen,
2374
                  gdp_printable_name(otw_gdp->src, _tmp_pname_1),
2375
                  gdp_printable_name(otw_gdp->dst, _tmp_pname_2));
2376

    
2377
        dguid_ca = (gdp_name_ca_t *) otw_gdp->dst;
2378
        dguid_dmi = dst_map.find(*dguid_ca);
2379
        if (dguid_dmi != dst_map.end() && fd == dguid_dmi->second.fd)
2380
        {
2381
                if (memcmp(otw_gdp->src, otw_gdp->dst, sizeof(gdp_name_ca_t)) == 0)
2382
                {
2383
                        // ADDR_GDPC owns the fd, so recycle fd (which will withdraw tx)
2384
                        fd_recycle(fd);
2385
                }
2386
                else
2387
                {
2388
                        if (!_single_router_mode)
2389
                        {
2390
                                // ADDR_NHOP withdraw tx
2391
                                dp = Packet::make(offsetof(otw_dir_t, eguid[1][0]));
2392
                                if (dp == nullptr)
2393
                                {
2394
                                        ccwarn("(%d)%s packet pool exhausted on %d",
2395
                                                   fd, fd_to_state_name(fd),
2396
                                                   offsetof(otw_dir_t, eguid[1][0]));
2397
                                }
2398
                                else
2399
                                {
2400
                                        otw_dir = (otw_dir_t *) dp->data();
2401
                                        otw_dir->ver = GDP_CHAN_PROTO_VERSION;
2402
                                        otw_dir->id = htons(_dr_id++);
2403
                                        otw_dir->cmd = GDP_CMD_DIR_DELETE;
2404
                                        memcpy(otw_dir->eguid[0], otw_gdp->src, sizeof(gdp_name_t));
2405
                                        memcpy(otw_dir->dguid, otw_gdp->dst, sizeof(gdp_name_t));
2406
                                        datagram_tx(_fd_directory, dp, nullptr, 0);
2407
                                        ccdbg(DVERB, "(%d)%s map withdraw tx id 0x%x\n"
2408
                                                  "\teguid[%s]\n"
2409
                                                  "\tdguid[%s]",
2410
                                                  fd, fd_to_state_name(fd),
2411
                                                  ntohs(otw_dir->id),
2412
                                                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
2413
                                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
2414
                                        dp->kill();
2415
                                }
2416
                        }
2417
                        
2418
                        // ADDR_NHOP owns one map, so detach map from fd and recycle
2419
                        fd_detach_map(dguid_dmi->second.fd, dguid_dmi);
2420
                        dst_map_recycle(dguid_dmi->second.fd, dguid_dmi);
2421
                }
2422
                
2423
                // diagnostics
2424
                display_state(fd);
2425
        }
2426

    
2427
        // recycle withdraw p (net4 not sending multi-withdraw pdus, so pedantic)
2428
        p_chain_recycle(p);
2429
}
2430

    
2431
void
2432
GDPv4Router::directory_rx (int fd)
2433
{
2434
        WritablePacket *dp;
2435
        ssize_t recv_len;
2436
        otw_dir_t *otw_dir;
2437
        otw_dir_t *otw_adv;
2438
        gdp_pname_t _tmp_pname_1;
2439
        gdp_pname_t _tmp_pname_2;
2440
        gdp_name_ca_t *dguid_ca;
2441
        dst_map_it_t dguid_dmi;
2442
        gdp_name_ca_t *eguid_ca;
2443
        dst_map_it_t eguid_dmi;
2444
        int fd_eguid;
2445
        wpl_it_t wpl_it;
2446
        otw_dir_t *or_otw_dir;
2447

    
2448
        ccdbg(DVVERB, "(%d)%s directory rx", fd, fd_to_state_name(fd));
2449
        
2450
        dp = Packet::make(sizeof(otw_dir_t));
2451
        if (dp == nullptr)
2452
        {
2453
                ccwarn("(%d)%s packet pool exhausted on %d",
2454
                           fd, fd_to_state_name(fd), sizeof(otw_dir_t));
2455
                return;
2456
        }
2457
        
2458
        // rx packet
2459
        recv_len = recv(fd, dp->data(), dp->length(), 0);
2460
        if (recv_len < 0)
2461
        {
2462
                switch (errno)
2463
                {
2464

    
2465
                case EAGAIN:
2466
                case EINTR:
2467
                {
2468
                        ccdbg(DINFO, "(%d)%s rx p %p len %d errno %d",
2469
                                   fd, fd_to_state_name(fd), dp, recv_len, errno);
2470
                }
2471
                break;
2472

    
2473
                case ECONNREFUSED:
2474
                {
2475
                        ccerr("(%d)%s rx connect refused (bad config||service down)",
2476
                                  fd, fd_to_state_name(fd), dp, recv_len, errno);
2477
                        _fd_directory_status = -1;
2478
                }
2479
                break;
2480

    
2481
                default:
2482
                {
2483
                        ccerr("(%d)%s rx p %p len %d errno %d",
2484
                                  fd, fd_to_state_name(fd), dp, recv_len, errno);
2485
                }
2486
                break;
2487

    
2488
                } // switch
2489

    
2490
                dp->kill();
2491
                return;
2492
        }
2493

    
2494
        // update directory fd status
2495
        _fd_directory_status = 0;
2496
        
2497
        // check packet short or version
2498
        if (recv_len < (ssize_t) offsetof(otw_dir_t, eguid[1]) ||
2499
                GDP_CHAN_PROTO_VERSION != *dp->data())
2500
        {
2501
                ccdbg(DVVERB, "(%d)%s rx unknown version or short len %d",
2502
                          fd, fd_to_state_name(fd), recv_len);
2503
                dp->kill();
2504
                return;
2505
        }
2506

    
2507
        // otw packed structure to access fields
2508
        otw_dir = (otw_dir_t *) dp->data();
2509
        
2510
        ccdbg(DVERB, "(%d)%s rx ver %d cmd %d id 0x%x len %d\n"
2511
                  "\teguid[%s]\n"
2512
                  "\tdguid[%s]",
2513
                  fd, fd_to_state_name(fd),
2514
                  otw_dir->ver, otw_dir->cmd, ntohs(otw_dir->id), recv_len,
2515
                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
2516
                  gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
2517

    
2518
        // dp->kill() called at function end
2519
        switch (otw_dir->cmd)
2520
        {
2521

    
2522
        case GDP_CMD_DIR_FIND:
2523
        {
2524
                // find dguid in map
2525
                dguid_ca = (gdp_name_ca_t *) otw_dir->dguid;
2526
                dguid_dmi = dst_map.find(*dguid_ca);
2527
                if (dguid_dmi == dst_map.end())
2528
                {
2529
                        ccdbg(DVERB, "(%d)%s rx id 0x%x no nhop, map not found\n"
2530
                                  "\tdguid[%s]",
2531
                                  fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2532
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_1));
2533
                        break;
2534
                }
2535

    
2536
                // map not interested in directory response
2537
                if (dguid_dmi->second.addr_state != ADDR_FIND)
2538
                {
2539
                        ccdbg(DVERB, "(%d)%s rx id 0x%x no nhop, map ignoring\n"
2540
                                  "\tdguid[%s]%s",
2541
                                  fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2542
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_1),
2543
                                  addr_to_state_name(dguid_dmi->second.addr_state));
2544
                        break;
2545
                }
2546

    
2547
                // ensure dir request id matches our request (always the first p)
2548
                wpl_it = dguid_dmi->second.dplist.begin();
2549
                avow(wpl_it != dguid_dmi->second.dplist.end());
2550
                avow(*wpl_it != nullptr);
2551
                or_otw_dir = (otw_dir_t *) (*wpl_it)->data();
2552
                if (or_otw_dir->id != otw_dir->id) // no need to ntohs() both
2553
                {
2554
                        ccdbg(DVERB, "(%d)%s rx id 0x%x no nhop, id mismatch drop\n"
2555
                                  "\tdguid[%s]",
2556
                                  fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2557
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_1));
2558
                        break;
2559
                }
2560

    
2561
                ccdbg(DINFO, "(%d)%s rx id 0x%x not found\n"
2562
                           "\tdguid[%s]",
2563
                           fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2564
                           gdp_printable_name(otw_dir->dguid, _tmp_pname_1));
2565
                map_no_route(fd, dguid_dmi);
2566
                dst_map_recycle(fd, dguid_dmi);
2567
                // dguid_dmi is invalid after dst_map_recycle
2568
                dguid_dmi = dst_map.end();
2569

    
2570
                // diagnostics
2571
                display_state(fd);
2572
        }
2573
        break;
2574
        
2575
        case GDP_CMD_DIR_FOUND:
2576
        {
2577
                // find dguid in map
2578
                dguid_ca = (gdp_name_ca_t *) otw_dir->dguid;
2579
                dguid_dmi = dst_map.find(*dguid_ca);
2580
                if (dguid_dmi == dst_map.end())
2581
                {
2582
                        ccdbg(DVERB, "(%d)%s rx id 0x%x nhop found, map not found\n"
2583
                                  "\tdguid[%s]",
2584
                                  fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2585
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_1));
2586
                        break;
2587
                }
2588

    
2589
                // map not interested in directory response
2590
                if (dguid_dmi->second.addr_state != ADDR_FIND)
2591
                {
2592
                        ccdbg(DVERB, "(%d)%s rx id 0x%x nhop found, map ignoring\n"
2593
                                  "\tdguid[%s]%s",
2594
                                  fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2595
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_1),
2596
                                  addr_to_state_name(dguid_dmi->second.addr_state));
2597
                        break;
2598
                }
2599

    
2600
                // ensure dir request id matches our request (always the first p)
2601
                wpl_it = dguid_dmi->second.dplist.begin();
2602
                avow(wpl_it != dguid_dmi->second.dplist.end());
2603
                avow(*wpl_it != nullptr);
2604
                or_otw_dir = (otw_dir_t *) (*wpl_it)->data();
2605
                if (or_otw_dir->id != otw_dir->id) // no need to ntohs() both
2606
                {
2607
                        ccdbg(DVERB, "(%d)%s rx id 0x%x nhop found, id mismatch drop\n"
2608
                                  "\tdguid[%s]",
2609
                                  fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2610
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_1));
2611
                        break;
2612
                }
2613

    
2614
                ccdbg(DVERB, "(%d)%s rx id 0x%x nhop found\n"
2615
                          "\teguid[%s]\n"
2616
                          "\tdguid[%s]",
2617
                          fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2618
                          gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
2619
                          gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
2620

    
2621
                // find eguid's fd in map
2622
                eguid_ca = (gdp_name_ca_t *) otw_dir->eguid[0];
2623
                eguid_dmi = dst_map.find(*eguid_ca);
2624
                if (eguid_dmi == dst_map.end())
2625
                {
2626
                        ccwarn("(%d)%s nhop no egress and no p memory, erase map\n"
2627
                                   "\teguid[%s]\n"
2628
                                   "\tdguid[%s]",
2629
                                   fd, fd_to_state_name(fd),
2630
                                   gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
2631
                                   gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
2632
                        map_no_route(fd, dguid_dmi);
2633
                        dst_map_recycle(fd, dguid_dmi);
2634
                        break;
2635
                }
2636
                aver(eguid_dmi->second.addr_state != ADDR_MINE);
2637
                fd_eguid = eguid_dmi->second.fd;
2638
                ccdbg(DVERB, "(%d)%s nhop egress\n"
2639
                          "\teguid[%s]%s @ (%d)%s",
2640
                          fd, fd_to_state_name(fd),
2641
                          gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
2642
                          addr_to_state_name(eguid_dmi->second.addr_state),
2643
                          fd_eguid, fd_to_state_name(fd_eguid));
2644

    
2645
                // fd tracks related map entries
2646
                (*_fdv)[fd_eguid].dst_mil.push_back(dguid_dmi);
2647
                ccdbg(DVVERB, "(%d)%s nhop egress (%d)%s push map\n"
2648
                          "\tdguid[%s]",
2649
                          fd, fd_to_state_name(fd),
2650
                          fd_eguid, fd_to_state_name(fd_eguid),
2651
                          gdp_printable_name(otw_dir->dguid, _tmp_pname_1));
2652

    
2653
                // dguid reached via eguid fd (replaces cguid's fd placeholder)
2654
                dguid_dmi->second.fd = fd_eguid;
2655
                dguid_dmi->second.addr_state = ADDR_NHOP;
2656

    
2657
                // replace addr_find_timer with addr_nhop_timer
2658
                dguid_dmi->second.map_timer->assign(static_addr_nhop_timer,
2659
                                                                                        dguid_dmi->second.map_timer_dmip);
2660
                dguid_dmi->second.map_timer->schedule_after(ADDR_NHOP_TS_DELTA);
2661
                ccdbg(DVVERB, "(%d)%s addr find timer reassigned to nhop %p %p",
2662
                          fd_eguid, fd_to_state_name(fd_eguid),
2663
                          dguid_dmi->second.map_timer, dguid_dmi->second.map_timer_dmip);
2664

    
2665
                ccdbg(DINFO, "(%d)%s rx id 0x%x found\n"
2666
                           "\tdguid[%s]%s via\n"
2667
                           "\teguid[%s]%s on (%d)%s",
2668
                           fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2669
                           gdp_printable_name(otw_dir->dguid, _tmp_pname_1),
2670
                           addr_to_state_name(dguid_dmi->second.addr_state),
2671
                           gdp_printable_name(otw_dir->eguid[0], _tmp_pname_2),
2672
                           addr_to_state_name(eguid_dmi->second.addr_state),
2673
                           fd_eguid, fd_to_state_name(fd_eguid));
2674

    
2675
                // diagnostics
2676
                display_state(fd_eguid);
2677
                
2678
                // delete stored dir request and list element
2679
                ccdbg(DVVERB, "(%d)%s recycle dir p %p next %p prev %p",
2680
                          fd, fd_to_state_name(fd),
2681
                          (*wpl_it), (*wpl_it)->next(), (*wpl_it)->prev());
2682
                (*wpl_it)->kill();
2683
                wpl_it = dguid_dmi->second.dplist.erase(wpl_it);
2684

    
2685
                // tx pending p
2686
                while (wpl_it != dguid_dmi->second.dplist.end())
2687
                {
2688
                        map_route(fd, dguid_dmi, *wpl_it);
2689

    
2690
                        // p moved out, so erase list element
2691
                        wpl_it = dguid_dmi->second.dplist.erase(wpl_it);
2692
                }
2693
        }
2694
        break;
2695

    
2696
        case GDP_CMD_DIR_ADD:
2697
        {
2698
                ccdbg(DVERB, "(%d)%s rx id 0x%x add ack\n"
2699
                          "\teguid[%s]\n"
2700
                          "\tdguid[%s]",
2701
                          fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2702
                          gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
2703
                          gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
2704

    
2705
                // gdpc location depends on advert type (src == dst vs not)
2706
                if (memcmp(_gdpr, otw_dir->eguid[0], sizeof(gdp_name_t)) == 0)
2707
                {
2708
                        dguid_ca = (gdp_name_ca_t *) otw_dir->dguid;
2709
                }
2710
                else
2711
                {
2712
                        dguid_ca = (gdp_name_ca_t *) otw_dir->eguid[0];
2713
                }
2714

    
2715
                // find dguid map
2716
                dguid_dmi = dst_map.find(*dguid_ca);
2717
                if (dguid_dmi != dst_map.end())
2718
                {
2719
                        // adverts awaiting ack are always near the front of dplist
2720
                        wpl_it = dguid_dmi->second.dplist.begin();
2721
                        while (wpl_it != dguid_dmi->second.dplist.end())
2722
                        {
2723
                                otw_adv = (otw_dir_t *) (*wpl_it)->data();
2724
                                ccdbg(DVVERB, "walk dp %p id 0x%x",
2725
                                          *wpl_it, ntohs(otw_adv->id));
2726
                                // match ack to advert by id
2727
                                if (otw_dir->id == otw_adv->id)
2728
                                {
2729
                                        (*wpl_it)->kill();
2730
                                        dguid_dmi->second.dplist.erase(wpl_it);
2731
                                        ++dguid_dmi->second.dp_acks;
2732
                                        ccdbg(DVVERB, "walk matched");
2733
                                        break;
2734
                                }
2735
                                // count each lost p once per addr_gdpc_timer interval
2736
                                if (RETRIES_ANNO(*wpl_it))
2737
                                {
2738
                                        // clear anno to trigger retx by addr_gdpc_timer
2739
                                        SET_RETRIES_ANNO(*wpl_it, 0);
2740
                                        // transfer p from front to back (not awaiting ack)
2741
                                        dguid_dmi->second.dplist.push_back(*wpl_it);
2742
                                        // erase front container after p move, implicit ++wpl_it
2743
                                        wpl_it = dguid_dmi->second.dplist.erase(wpl_it);
2744
                                        // track losses for addr_gdpc_timer eval
2745
                                        ++dguid_dmi->second.dp_loss;
2746
                                }
2747
                                else
2748
                                {
2749
                                        ++wpl_it;
2750
                                }
2751
                        }
2752
                        ccdbg(DVVERB, "walk done, moved %d", dguid_dmi->second.dp_loss);
2753
                }
2754
        }
2755
        break;
2756

    
2757
        case GDP_CMD_DIR_DELETE:
2758
        {
2759
                ccdbg(DVERB, "(%d)%s rx id 0x%x delete ack\n"
2760
                          "\tdguid[%s]",
2761
                          fd, fd_to_state_name(fd), ntohs(otw_dir->id),
2762
                          gdp_printable_name(otw_dir->dguid, _tmp_pname_1));
2763
        }
2764
        break;
2765

    
2766
        default:
2767
        {
2768
                ccdbg(DVVERB, "(%d)%s rx id 0x%x unexpected cmd %d, drop\n",
2769
                          fd, fd_to_state_name(fd), ntohs(otw_dir->id), otw_dir->cmd);
2770
        }
2771
        break;
2772

    
2773
        } // switch
2774

    
2775
        dp->kill();
2776
}
2777

    
2778
void
2779
GDPv4Router::addr_gdpr_connect_timer (Timer *t, void *v)
2780
{
2781
        dst_map_it_t *dguid_dmip;
2782
        int fd;
2783
        int ssl_status;
2784
        int ssl_error;
2785
        unsigned long get_error;
2786
        unsigned long get_reason;
2787

    
2788
        ccdbg(DVVERB, "\naddr gdp connect timer %p %p", t, v);
2789
                
2790
        dguid_dmip = static_cast<dst_map_it_t *>(v);
2791
        avow(dguid_dmip != nullptr);
2792
        avow((*dguid_dmip)->second.addr_state == ADDR_GDPR);
2793

    
2794
        fd = (*dguid_dmip)->second.fd;
2795
        avow((*_fdv)[fd].fd_state == FD_DTLS_ACCEPT ||
2796
                 (*_fdv)[fd].fd_state == FD_DTLS_DO_HANDSHAKE);
2797

    
2798
        ssl_status = SSL_heartbeat((*_fdv)[fd].ssl);
2799
        if (ssl_status <= 0)
2800
        {
2801
                ssl_error = SSL_get_error((*_fdv)[fd].ssl, ssl_status);
2802
                // ssl_error holds an enum, not bit flags
2803
                switch (ssl_error)
2804
                {
2805
                case SSL_ERROR_WANT_READ:
2806
                        // router fd is always SELECT_READ (add/remove performs poorly)
2807
                        break;
2808
                case SSL_ERROR_WANT_WRITE:
2809
                        // this condition has not been spotted in the wild, so far...
2810
                        aver(0);
2811
                        break;
2812
                default:
2813
                        get_error = ERR_get_error();
2814
                        get_reason = ERR_GET_REASON(get_error);
2815
                        switch (get_reason)
2816
                        {
2817

    
2818
                        case SSL_R_TLS_HEARTBEAT_PEER_DOESNT_ACCEPT:
2819
                        {
2820
                                ccdbg(DINFO, "(%d)%s heartbeat detected misaligned peer, recycle fd",
2821
                                           fd, fd_to_state_name(fd),
2822
                                           ERR_error_string(get_error, NULL),
2823
                                           ssl_error, get_error, get_reason);
2824
                                fd_recycle(fd);
2825
                                // vary renegotiation start by up to ~2 seconds amongst peers
2826
                                std::this_thread::sleep_for(
2827
                                        std::chrono::milliseconds(generate_sequence() & 0x1FFF));
2828
                                return;
2829
                        }
2830
                        break;
2831
                        
2832
                        case SSL_R_UNEXPECTED_MESSAGE:
2833
                        case SSL_R_TLS_HEARTBEAT_PENDING:
2834
                        default:
2835
                        {
2836
                                ccerr("(%d)%s heartbeat error %s (%d/%d/%d)",
2837
                                          fd, fd_to_state_name(fd),
2838
                                          ERR_error_string(get_error, NULL),
2839
                                          ssl_error, get_error, get_reason);
2840
                                fd_recycle(fd);
2841
                                return;
2842
                        }
2843
                        break;
2844

    
2845
                        } // switch
2846
                }
2847
        }
2848
        else
2849
        {
2850
                ccdbg(DVVERB, "(%d)%s tx heartbeat %d",
2851
                          fd, fd_to_state_name(fd), ssl_status);
2852
        }
2853

    
2854
        t->schedule_after(ADDR_GDPR_NOW_TS_DELTA);
2855
}
2856

    
2857
void
2858
GDPv4Router::addr_gdpr_timer (Timer *t, void *v)
2859
{
2860
        dst_map_it_t *dguid_dmip;
2861
        int fd;
2862
        int ssl_status;
2863
        int ssl_error;
2864
        unsigned long get_error;
2865
        unsigned long get_reason;
2866
        WritablePacket *dp;
2867
        otw_dir_t *otw_dir;
2868
        gdp_pname_t _tmp_pname_1;
2869
        gdp_pname_t _tmp_pname_2;
2870

    
2871
        ccdbg(DVVERB, "\naddr gdpr timer %p %p", t, v);
2872
                
2873
        dguid_dmip = static_cast<dst_map_it_t *>(v);
2874
        avow(dguid_dmip != nullptr);
2875
        avow((*dguid_dmip)->second.addr_state == ADDR_GDPR);
2876

    
2877
        fd = (*dguid_dmip)->second.fd;
2878
        avow((*_fdv)[fd].fd_state == FD_DTLS_CLIENT ||
2879
                 (*_fdv)[fd].fd_state == FD_DTLS_SERVER);
2880

    
2881
        if ((*_fdv)[fd].stat_tx_b || (*_fdv)[fd].stat_rx_b)
2882
        {
2883
                ccdbg(DVVERB,
2884
                          "(%d)%s dtls bytes tx %d rx %d packets tx %d rx %d",
2885
                          fd, fd_to_state_name(fd),
2886
                          (*_fdv)[fd].stat_tx_b,
2887
                          (*_fdv)[fd].stat_rx_b,
2888
                          (*_fdv)[fd].stat_tx_p,
2889
                          (*_fdv)[fd].stat_rx_p);
2890
                // move interval's bytes to total
2891
                (*_fdv)[fd].stat_tot_tx_b += (*_fdv)[fd].stat_tx_b;
2892
                (*_fdv)[fd].stat_tot_rx_b += (*_fdv)[fd].stat_rx_b;
2893
                (*_fdv)[fd].stat_tx_b = 0;
2894
                (*_fdv)[fd].stat_rx_b = 0;
2895
                // move interval's packets to total                                        
2896
                (*_fdv)[fd].stat_tot_tx_p += (*_fdv)[fd].stat_tx_p;
2897
                (*_fdv)[fd].stat_tot_rx_p += (*_fdv)[fd].stat_rx_p;
2898
                (*_fdv)[fd].stat_tx_p = 0;
2899
                (*_fdv)[fd].stat_rx_p = 0;
2900
                // reset heartbeat rx
2901
                (*_fdv)[fd].fd_state_counter = 0;
2902
        }
2903
        else if ((*_fdv)[fd].fd_state_counter)
2904
        {
2905
                ccdbg(DVVERB, "(%d)%s heartbeat rx",
2906
                          fd, fd_to_state_name(fd));
2907
                (*_fdv)[fd].fd_state_counter = 0;
2908
        }
2909
        else
2910
        {
2911
                ssl_status = SSL_heartbeat((*_fdv)[fd].ssl);
2912
                if (ssl_status <= 0)
2913
                {
2914
                        ssl_error = SSL_get_error((*_fdv)[fd].ssl, ssl_status);
2915
                        // ssl_error holds an enum, not bit flags
2916
                        switch (ssl_error)
2917
                        {
2918
                        case SSL_ERROR_WANT_READ:
2919
                                // router fd is always SELECT_READ (add/remove performs poorly)
2920
                                break;
2921
                        case SSL_ERROR_WANT_WRITE:
2922
                                // this condition has not been spotted in the wild, so far...
2923
                                aver(0);
2924
                                break;
2925
                        default:
2926
                                get_error = ERR_get_error();
2927
                                get_reason = ERR_GET_REASON(get_error);
2928
                                switch (get_reason)
2929
                                {
2930

    
2931
                                case SSL_R_UNEXPECTED_MESSAGE:
2932
                                case SSL_R_TLS_HEARTBEAT_PENDING:
2933
                                case SSL_R_TLS_HEARTBEAT_PEER_DOESNT_ACCEPT:
2934
                                default:
2935
                                {
2936
                                        ccerr("(%d)%s heartbeat error %s (%d/%d/%d)",
2937
                                                  fd, fd_to_state_name(fd),
2938
                                                  ERR_error_string(get_error, NULL),
2939
                                                  ssl_error, get_error, get_reason);
2940
                                        fd_recycle(fd);
2941
                                        return;
2942
                                }
2943
                                break;
2944

    
2945
                                } // switch
2946
                        }
2947
                }
2948
                else
2949
                {
2950
                        ccdbg(DVVERB, "(%d)%s tx heartbeat %d",
2951
                                  fd, fd_to_state_name(fd), ssl_status);
2952
                }
2953
        }
2954

    
2955
        (*_fdv)[fd].fd_timer_counter = ((*_fdv)[fd].fd_timer_counter + 1) %
2956
                ADDR_GDPR_ADVERT_INTERVAL;
2957
        ccdbg(DVVERB, "(%d)%s timer counter %d",
2958
                  fd, fd_to_state_name(fd), (*_fdv)[fd].fd_timer_counter);
2959

    
2960
        if ((*_fdv)[fd].fd_timer_counter == 1 || _fd_directory_status < 0)
2961
        {
2962
                dp = Packet::make(offsetof(otw_dir_t, eguid[1][0]));
2963
                if (dp != nullptr)
2964
                {
2965
                        otw_dir = (otw_dir_t *) dp->data();
2966
                        otw_dir->ver = GDP_CHAN_PROTO_VERSION;
2967
                        otw_dir->id = htons(_dr_id++);
2968
                        otw_dir->cmd = GDP_CMD_DIR_ADD;
2969
                        // advertise ADDR_GDPR reachability via ADDR_MINE
2970
                        memcpy(otw_dir->eguid[0], _gdpr, sizeof(gdp_name_t));
2971
                        memcpy(otw_dir->dguid, (*dguid_dmip)->first.data(),
2972
                                   sizeof(gdp_name_t));
2973
                        datagram_tx(_fd_directory, dp, nullptr, 0);
2974
                        ccdbg(DVERB, "(%d)%s advert gdpr tx id 0x%x\n"
2975
                                  "\teguid[%s]\n"
2976
                                  "\tdguid[%s]",
2977
                                  _fd_directory, fd_to_state_name(_fd_directory),
2978
                                  ntohs(otw_dir->id),
2979
                                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
2980
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
2981
                        dp->kill();
2982

    
2983
                        // speed up recovery from directory outages
2984
                        if (_fd_directory_status < 0)
2985
                        {
2986
                                ccwarn("(%d)%s fast recovery mode",
2987
                                           _fd_directory, fd_to_state_name(_fd_directory));
2988
                                t->schedule_after(ADDR_GDPR_FAST_RECOVERY_TS_DELTA);
2989
                                return;
2990
                        }
2991
                }
2992
        }
2993
        
2994
        t->schedule_after(ADDR_GDPR_TS_DELTA);
2995
}
2996

    
2997
void
2998
GDPv4Router::addr_gdpc_timer (Timer *t, void *v)
2999
{
3000
        dst_map_it_t *dguid_dmip;
3001
        int fd;
3002
        otw_dir_t *otw_dir;
3003
        gdp_pname_t _tmp_pname_1;
3004
        gdp_pname_t _tmp_pname_2;
3005
        wpl_it_t wpl_it;
3006

    
3007
        dguid_dmip = static_cast<dst_map_it_t *>(v);
3008
        avow(dguid_dmip != nullptr);
3009
        avow((*dguid_dmip)->second.addr_state == ADDR_GDPC);
3010

    
3011
        fd = (*dguid_dmip)->second.fd;
3012
        avow((*_fdv)[fd].fd_state == FD_TCP_CLIENT);
3013

    
3014
        ccdbg(DVVERB, "(%d)%s addr gdpc timer size %d acks %d loss %d",
3015
                  fd, fd_to_state_name(fd),
3016
                  (*dguid_dmip)->second.dplist.size(),
3017
                  (*dguid_dmip)->second.dp_acks, (*dguid_dmip)->second.dp_loss);
3018

    
3019
        wpl_it = (*dguid_dmip)->second.dplist.begin();
3020
        while (wpl_it != (*dguid_dmip)->second.dplist.end())
3021
        {
3022
                if (RETRIES_ANNO(*wpl_it) == 0)
3023
                {
3024
                        datagram_tx(_fd_directory, *wpl_it, nullptr, 0);
3025
                        SET_RETRIES_ANNO(*wpl_it, 1);
3026
                        otw_dir = (otw_dir_t *) (*wpl_it)->data();
3027
                        ccdbg(DINFO, "(%d)%s advert tx id 0x%x\n"
3028
                                  "\teguid[%s]\n"
3029
                                  "\tdguid[%s]",
3030
                                  fd, fd_to_state_name(fd), ntohs(otw_dir->id),
3031
                                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
3032
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
3033
                        break;
3034
                }
3035
                ++wpl_it;
3036
        }
3037

    
3038
        ++(*dguid_dmip)->second.dp_timer_count;
3039

    
3040
        // advertisement evaluation
3041
        if ((*dguid_dmip)->second.dp_algo == ADDR_GDPC_ADVERT_EVAL_EARLY)
3042
        {
3043
                // congestion
3044
                if ((*dguid_dmip)->second.dp_loss)
3045
                {
3046
                        (*dguid_dmip)->second.end_ts = Timestamp::now_steady();
3047

    
3048
                        // early evaluation per dguid
3049
                        if ((*dguid_dmip)->second.end_ts.subsec() <
3050
                                (*dguid_dmip)->second.start_ts.subsec())
3051
                        {
3052
                                ccdbg(DINFO, "(%d)%s advert timer count %d acks %d loss %d\n"
3053
                                          "\tduration " PRITIMESTAMP " (sec.subsec)",
3054
                                          fd, fd_to_state_name(fd),
3055
                                          (*dguid_dmip)->second.dp_timer_count,
3056
                                          (*dguid_dmip)->second.dp_acks,
3057
                                          (*dguid_dmip)->second.dp_loss,
3058
                                          (*dguid_dmip)->second.end_ts.sec() -
3059
                                          (*dguid_dmip)->second.start_ts.sec() - 1,
3060
                                          (TIMESTAMP_MS(1000) +
3061
                                           (*dguid_dmip)->second.end_ts.subsec()) -
3062
                                          (*dguid_dmip)->second.start_ts.subsec());
3063
                        }
3064
                        else
3065
                        {
3066
                                ccdbg(DINFO, "(%d)%s advert timer count %d acks %d loss %d\n"
3067
                                          "\tduration " PRITIMESTAMP " (sec.subsec)",
3068
                                          fd, fd_to_state_name(fd),
3069
                                          (*dguid_dmip)->second.dp_timer_count,
3070
                                          (*dguid_dmip)->second.dp_acks,
3071
                                          (*dguid_dmip)->second.dp_loss,
3072
                                          (*dguid_dmip)->second.end_ts.sec() -
3073
                                          (*dguid_dmip)->second.start_ts.sec(),
3074
                                          (*dguid_dmip)->second.end_ts.subsec() -
3075
                                          (*dguid_dmip)->second.start_ts.subsec());
3076
                        }
3077
                        // early evaluation run once per gdpc client
3078
                        (*dguid_dmip)->second.dp_algo = ADDR_GDPC_ADVERT_EVAL_END;
3079
                }
3080
        }
3081
        
3082
        if ((*dguid_dmip)->second.dplist.size())
3083
        {
3084
                ccdbg(DVVERB, "(%d)%s addr_gdpc_timer rescheduled on size %d",
3085
                          fd, fd_to_state_name(fd), (*dguid_dmip)->second.dplist.size());
3086
                t->schedule_after(Timestamp(0, TIMESTAMP_US(_advert_timer)));
3087
        }
3088
        else
3089
        {
3090
                if ((*dguid_dmip)->second.dp_algo == ADDR_GDPC_ADVERT_EVAL_END)
3091
                {
3092
                        (*dguid_dmip)->second.end_ts = Timestamp::now_steady();
3093
                
3094
                        // early evaluation per dguid
3095
                        if ((*dguid_dmip)->second.end_ts.subsec() <
3096
                                (*dguid_dmip)->second.start_ts.subsec())
3097
                        {
3098
                                ccdbg(DINFO, "(%d)%s advert timer count %d acks %d loss %d\n"
3099
                                          "\tduration " PRITIMESTAMP " (sec.subsec)",
3100
                                          fd, fd_to_state_name(fd),
3101
                                          (*dguid_dmip)->second.dp_timer_count,
3102
                                          (*dguid_dmip)->second.dp_acks,
3103
                                          (*dguid_dmip)->second.dp_loss,
3104
                                          (*dguid_dmip)->second.end_ts.sec() -
3105
                                          (*dguid_dmip)->second.start_ts.sec() - 1,
3106
                                          (TIMESTAMP_MS(1000) +
3107
                                           (*dguid_dmip)->second.end_ts.subsec()) -
3108
                                          (*dguid_dmip)->second.start_ts.subsec());
3109
                        }
3110
                        else
3111
                        {
3112
                                ccdbg(DINFO, "(%d)%s advert timer count %d acks %d loss %d\n"
3113
                                          "\tduration " PRITIMESTAMP " (sec.subsec)",
3114
                                          fd, fd_to_state_name(fd),
3115
                                          (*dguid_dmip)->second.dp_timer_count,
3116
                                          (*dguid_dmip)->second.dp_acks,
3117
                                          (*dguid_dmip)->second.dp_loss,
3118
                                          (*dguid_dmip)->second.end_ts.sec() -
3119
                                          (*dguid_dmip)->second.start_ts.sec(),
3120
                                          (*dguid_dmip)->second.end_ts.subsec() -
3121
                                          (*dguid_dmip)->second.start_ts.subsec());
3122
                        }
3123
                        (*dguid_dmip)->second.dp_algo = ADDR_GDPC_ADVERT_EVAL_OFF;
3124
                }
3125
        }
3126
        ccdbg(DVVERB, "(%d)%s addr gdpc timer done", fd, fd_to_state_name(fd));
3127
}
3128

    
3129
void
3130
GDPv4Router::addr_find_timer (Timer *t, void *v)
3131
{
3132
        dst_map_it_t *dguid_dmip;
3133
        gdp_pname_t _tmp_pname_1;
3134
        WritablePacket *p_dir;
3135
        uint retries;
3136

    
3137
        dguid_dmip = static_cast<dst_map_it_t *>(v);
3138
        avow(dguid_dmip != nullptr);
3139
        
3140
        ccdbg(DVVERB, "\naddr find timer %p %p dguid[%s]%s", t, v,
3141
                  gdp_printable_name((*dguid_dmip)->first.data(), _tmp_pname_1),
3142
                  addr_to_state_name((*dguid_dmip)->second.addr_state));
3143
        avow((*dguid_dmip)->second.addr_state == ADDR_FIND);
3144

    
3145
        // find request, retry, and reschedule (or expire and unschedule)
3146
        p_dir = (*dguid_dmip)->second.dplist.front();
3147
        retries = RETRIES_ANNO(p_dir);
3148
        if (retries > 0)
3149
        {
3150
                ccdbg(DVVERB, "find retry %d", retries);
3151
#ifdef TEST_ADDR_FIND_RETRIES
3152
                if (retries == TEST_ADDR_FIND_RETRIES_TX)
3153
#endif
3154
                {
3155
                        datagram_tx(_fd_directory, p_dir, nullptr, 0);
3156
                }
3157
                SET_RETRIES_ANNO(p_dir, --retries);
3158
                t->schedule_after(ADDR_FIND_TS_DELTA);
3159
                return;
3160
        }
3161

    
3162
    p_dir = nullptr; // map_no_route will consume dplist.front() aka p_dir
3163
        map_no_route(_fd_directory, *dguid_dmip);
3164
        dst_map_recycle(_fd_directory, *dguid_dmip);
3165
        // *dguid_dmip is invalid after dst_map_recycle
3166
}
3167

    
3168
void
3169
GDPv4Router::addr_nhop_timer (Timer *t, void *v)
3170
{
3171
        dst_map_it_t *dguid_dmip;
3172
        gdp_pname_t _tmp_pname_1;
3173

    
3174
        dguid_dmip = static_cast<dst_map_it_t *>(v);
3175
        avow(dguid_dmip != nullptr);
3176

    
3177
        ccdbg(DVVERB, "\naddr nhop timer %p %p dguid[%s]", t, v,
3178
                  gdp_printable_name((*dguid_dmip)->first.data(), _tmp_pname_1));
3179
        avow((*dguid_dmip)->second.addr_state == ADDR_NHOP);
3180

    
3181
        // free expired sguid map tasks, if any
3182
        while (!(*dguid_dmip)->second.expired_sm_tasks.empty())
3183
        {
3184
                ccdbg(DVVERB, "addr_nhop_timer delete expired sm task %p",
3185
                          (*dguid_dmip)->second.expired_sm_tasks.front());
3186
                delete (*dguid_dmip)->second.expired_sm_tasks.front();
3187
                (*dguid_dmip)->second.expired_sm_tasks.pop_front();
3188
        }
3189

    
3190
        // persist while active sguid maps
3191
        if ((*dguid_dmip)->second.src_map.size())
3192
        {
3193
                ccdbg(DVVERB, "addr_nhop_timer src map %d",
3194
                          (*dguid_dmip)->second.src_map.size());
3195
                (*dguid_dmip)->second.addr_inactive = 0;
3196
                t->schedule_after(ADDR_NHOP_TS_DELTA);
3197
                return;
3198
        }
3199

    
3200
        // reschedule timer when cplist work remains or dguid map recently active
3201
        if (++(*dguid_dmip)->second.addr_inactive < ADDR_NHOP_EXPIRE_LIMIT)
3202
        {
3203
                ccdbg(DVVERB, "addr_nhop_timer inactive %d",
3204
                          (*dguid_dmip)->second.addr_inactive);
3205
                t->schedule_after(ADDR_NHOP_TS_DELTA);
3206
                return;
3207
        }
3208

    
3209
        // expire dguid map when no work remains and dguid map has been inactive
3210
        ccdbg(DVERB, "\n(%d)%s addr nhop timer expired\n"
3211
                  "\tdguid[%s]",
3212
                  (*dguid_dmip)->second.fd, fd_to_state_name((*dguid_dmip)->second.fd),
3213
                  gdp_printable_name((*dguid_dmip)->first.data(), _tmp_pname_1));
3214

    
3215
        // delete this map's iterator from the fd's map iterator list
3216
        fd_detach_map((*dguid_dmip)->second.fd, *dguid_dmip);
3217

    
3218
        // via timer's iterator, CAREFULLY destroy addr map and this timer itself
3219
        dst_map_recycle((*dguid_dmip)->second.fd, *dguid_dmip);
3220
        // (*dguid_dmip) is invalid after dst_map_recycle
3221
}
3222

    
3223
void
3224
GDPv4Router::map_route (int fd, dst_map_it_t &dguid_dmi, WritablePacket *p)
3225
{
3226
        int fd_tx;
3227
        int fd_rx;
3228
        otw_gdp_t *otw_gdp;
3229
        WritablePacket *p_dir;
3230
        otw_dir_t *otw_dir;
3231
        gdp_name_ca_t *sguid_ca;
3232
        src_map_it_t sguid_smi;
3233
        WritablePacket *p_pkt;
3234
        WritablePacket *p_pdu;        
3235
        WritablePacket *p_clone;        
3236
        gdp_pname_t _tmp_pname_1;
3237
        gdp_pname_t _tmp_pname_2;
3238
        
3239
        avow(p != nullptr);
3240
        if (fd != _fd_directory || fd != (int)RXFD_ANNO(p))
3241
        {
3242
                ccdbg(DVVERB, "fd %d rxfd_anno %d", fd, RXFD_ANNO(p));
3243
                avow(fd == _fd_directory || fd == (int)RXFD_ANNO(p));
3244
        }
3245

    
3246
        fd_tx = dguid_dmi->second.fd;
3247

    
3248
        otw_gdp = (otw_gdp_t *) p->data();
3249
        ccdbg(DVVERB, "(%d)%s map_route p %p type %d\n"
3250
                  "\tid 0x%x mf %d off 0x%x sseq %d seqp %d",
3251
                  fd, fd_to_state_name(fd), p, PTYPE_ANNO(p),
3252
                  GET_SEQ(otw_gdp->seq_mf_off), GET_MF(otw_gdp->seq_mf_off),
3253
                  GET_OFF(otw_gdp->seq_mf_off),
3254
                  ((otw_gdp->flags & GDP_PKT_TYPE_SSEQ) ? 1 : 0),
3255
                  (((otw_gdp->flags & GDP_PKT_TYPE_MASK) ==
3256
                        GDP_PKT_TYPE_SEQ_PACKET) ? 1 : 0));
3257

    
3258
        // check dguid state
3259
        switch (dguid_dmi->second.addr_state)
3260
        {
3261

    
3262
        case ADDR_FIND:
3263
        {
3264
                if (PTYPE_ANNO(p) == PTYPE_GDP_PKT)
3265
                {
3266
                        // packed struct access to packet fields
3267
                        otw_gdp = (otw_gdp_t *) p->data();
3268

    
3269
                        // do not queue seq packets
3270
                        if ((otw_gdp->flags & GDP_PKT_TYPE_MASK) == GDP_PKT_TYPE_SEQ_PACKET)
3271
                        {
3272
                                ccdbg(DPDU, "(%d)%s map route seqp p %p len %d drop\n"
3273
                                          "\tid 0x%x mf %d off 0x%x\n"
3274
                                          "\tsrc[%s]\n"
3275
                                          "\tdst[%s]",
3276
                                          fd, fd_to_state_name(fd), p, ntohs(otw_gdp->frag_len),
3277
                                          GET_SEQ(otw_gdp->seq_mf_off), GET_MF(otw_gdp->seq_mf_off),
3278
                                          GET_OFF(otw_gdp->seq_mf_off),
3279
                                          gdp_printable_name(otw_gdp->src, _tmp_pname_1),
3280
                                          gdp_printable_name(otw_gdp->dst, _tmp_pname_2));
3281
                                p->kill();
3282
                                return;
3283
                        }
3284
                }
3285
                
3286
                p_dir = dguid_dmi->second.dplist.front();
3287
                avow(p_dir != nullptr);
3288
                
3289
                // add p chain to the existing dplist
3290
                dguid_dmi->second.dplist.push_back(p);
3291

    
3292
                // retx p_dir from front of dplist
3293
                datagram_tx(_fd_directory, p_dir, nullptr, 0);
3294

    
3295
                otw_dir = (otw_dir_t *) p_dir->data();
3296
                ccdbg(DVERB, "(%d)%s map route retx find id 0x%x dguid[%s]%s",
3297
                          fd, fd_to_state_name(fd), ntohs(otw_dir->id),
3298
                          gdp_printable_name(otw_dir->dguid, _tmp_pname_1),
3299
                          addr_to_state_name(dguid_dmi->second.addr_state));
3300
                return;
3301
        }
3302
        break;
3303

    
3304
        case ADDR_GDPR:
3305
        case ADDR_GDPC:
3306
        case ADDR_NHOP:
3307
        {
3308
                // fall through to tx
3309
        }
3310
        break;
3311

    
3312
        case ADDR_MINE:
3313
        {
3314
                ccwarn("(%d)%s addr mine drop p %p", fd, fd_to_state_name(fd), p);
3315
                p_chain_recycle(p);
3316
                return;
3317
        }
3318
        break;
3319
        
3320
        case ADDR_NONE:
3321
        default:
3322
        {
3323
                aver(0);
3324
        }
3325
        break;
3326
        
3327
        } // switch
3328

    
3329
        fd_rx = RXFD_ANNO(p);
3330
        
3331
        // find (or create) sguid src map else drop p chain and return on errors
3332
        otw_gdp = (otw_gdp_t *) p->data();
3333
        sguid_ca = (gdp_name_ca_t *) otw_gdp->src;
3334
        sguid_smi = dguid_dmi->second.src_map.lower_bound(*sguid_ca);
3335
        if (sguid_smi == dguid_dmi->second.src_map.end() ||
3336
                dguid_dmi->second.src_map.key_comp()(*sguid_ca, sguid_smi->first))
3337
        {
3338
                // create src map entry
3339
                sguid_smi =
3340
                        dguid_dmi->second.src_map.emplace_hint(
3341
                                sguid_smi, src_map_t::value_type(*sguid_ca, src_map_init));
3342
                if (sguid_smi == dguid_dmi->second.src_map.end())
3343
                {
3344
                        ccwarn("(%d)%s src map memory exhausted, drop p %p",
3345
                                   fd, fd_to_state_name(fd), p);
3346
                        p_chain_recycle(p);
3347
                        return;
3348
                }
3349
                sguid_smi->second.fd_rx = fd_rx;
3350
                if (fd_rx == dguid_dmi->second.fd)
3351
                {
3352
                        ccdbg(DINFO, "fd_rx %d dguid fd %d", fd_rx, dguid_dmi->second.fd);
3353
                        ccdbg(DINFO, "(%d)%s dump\n"
3354
                                  "\tsrc[%s]\n"
3355
                                  "\tdst[%s]",
3356
                                  fd, fd_to_state_name(fd),
3357
                                  gdp_printable_name(sguid_smi->first.data(), _tmp_pname_1),
3358
                                  gdp_printable_name(dguid_dmi->first.data(), _tmp_pname_2));
3359
                        avow(fd_rx != dguid_dmi->second.fd);
3360
                }
3361

    
3362
                // seq_map_mode is SSM_IDLE by default/init
3363
                switch ((*_fdv)[fd_tx].fd_state)
3364
                {
3365

    
3366
                case FD_TCP_CLIENT:
3367
                {
3368
                        if (PTYPE_ANNO(p) == PTYPE_GDP_PKT)
3369
                        {
3370
                                // dtls to tcp, pkt-to-pdu coalesce service
3371
                                sguid_smi->second.seq_map_mode = SMM_COAL_INIT;
3372
                        }
3373
                }
3374
                break;
3375

    
3376
                case FD_DTLS_SERVER:
3377
                case FD_DTLS_CLIENT:
3378
                {
3379
                        if (PTYPE_ANNO(p) == PTYPE_GDP_PDU)
3380
                        {
3381
                                // tcp to dtls, nack-driven pkt retx service
3382
                                sguid_smi->second.seq_map_mode = SMM_RETX_INIT;
3383
                        }
3384
                }
3385
                break;
3386

    
3387
                default:
3388
                {
3389
                        aver(0);
3390
                }
3391
                break;
3392

    
3393
                } // switch
3394

    
3395
                // create src map task (always assign task, even though map can be idle)
3396
                aver(sguid_smi->second.sm_task_smip == nullptr);
3397
                aver(sguid_smi->second.sm_task == nullptr);
3398
                try {
3399
                        sguid_smi->second.sm_task_smip = new src_map_it_t(sguid_smi);
3400
                        if (sguid_smi->second.seq_map_mode == SMM_RETX_INIT)
3401
                        {
3402
                                sguid_smi->second.sm_task =
3403
                                        new Task(static_retx_map_task,
3404
                                                         sguid_smi->second.sm_task_smip);
3405
                        }
3406
                        else if (sguid_smi->second.seq_map_mode == SMM_COAL_INIT)
3407
                        {
3408
                                sguid_smi->second.sm_task =
3409
                                        new Task(static_coal_map_task,
3410
                                                         sguid_smi->second.sm_task_smip);
3411
                        }
3412
                        else // SMM_IDLE (simple forwarding of pdus or pkts)
3413
                        {
3414
                                sguid_smi->second.sm_task =
3415
                                        new Task(static_idle_map_task,
3416
                                                         sguid_smi->second.sm_task_smip);
3417
                        }
3418
                        sguid_smi->second.sm_timer = new Timer();
3419
                }
3420
                catch (std::bad_alloc)
3421
                {
3422
                        ccwarn("(%d)%s memory exhausted on new src map",
3423
                                   fd, fd_to_state_name(fd));
3424
                        delete sguid_smi->second.sm_task_smip;
3425
                        delete sguid_smi->second.sm_task;
3426
                        dguid_dmi->second.src_map.erase(sguid_smi);
3427
                        p_chain_recycle(p);
3428
                        return;
3429
                }
3430

    
3431
                // copy of dguid's dst map iterator (dguid's dst map owns lifecycle!)
3432
                aver(dguid_dmi->second.map_timer_dmip != nullptr);
3433
                sguid_smi->second.sm_task_copy_dmip = dguid_dmi->second.map_timer_dmip;
3434

    
3435
                // fd tracks related src maps
3436
                (*_fdv)[fd_rx].src_mil.push_back(sguid_smi);
3437
                ccdbg(DINFO, "(%d)%s fd push sguid[%s]\n"
3438
                          "\towned by dguid[%s]",
3439
                          fd_rx, fd_to_state_name(fd_rx),
3440
                          gdp_printable_name(sguid_smi->first.data(), _tmp_pname_1),
3441
                          gdp_printable_name(dguid_dmi->first.data(), _tmp_pname_2));
3442
                
3443
                sguid_smi->second.sm_task->initialize(this, false);                
3444
                sguid_smi->second.sm_timer->assign(sguid_smi->second.sm_task);
3445
                sguid_smi->second.sm_timer->initialize(this, true);
3446
                sguid_smi->second.sm_timer->schedule_after_msec(SMT_WORK);
3447
                
3448
                ccdbg(DVVERB, "(%d)%s new src map task %p %p",
3449
                          fd, fd_to_state_name(fd),
3450
                          sguid_smi->second.sm_task, sguid_smi->second.sm_task_smip);
3451
        }
3452

    
3453
        switch ((*_fdv)[fd_tx].fd_state)
3454
        {
3455

    
3456
        case FD_TCP_CLIENT:
3457
        {
3458
                if (PTYPE_ANNO(p) == PTYPE_GDP_PDU)
3459
                {
3460
                        seq_assign(sguid_smi, p);
3461
                        ccdbg(DPDU, "(%d)%s rx pdu %p id 0x%x",
3462
                                  fd_rx, fd_to_state_name(fd_rx), p, SEQ_ANNO(p));
3463
                        if (RXHALT_ANNO(p))
3464
                        {
3465
                                // schedule immediate seq p
3466
                                sguid_smi->second.sm_timer->clear();
3467
                                sguid_smi->second.sm_timer->reschedule_after_msec(0);
3468
                                SET_RXHALT_ANNO(p, 0);
3469
                        }
3470

    
3471
                        // pdu in, pdu out (consumed by tcp client txplist)
3472
                        (*_fdv)[fd_tx].txplist.push_back(p);
3473

    
3474
                        // register sguid activity
3475
                        sguid_smi->second.sm_idle = 0;
3476

    
3477
                        // wake up fd tx task
3478
                        (*_fdv)[fd_tx].tx_task->reschedule();
3479
                }
3480
                else
3481
                {
3482
                        if (seq_assess(sguid_smi, p))
3483
                        {
3484
                                // FIXME unclear whether SSEQ flag may be useful to tcp client
3485
                                
3486
                                // pkt in (consumed by coalesce), (sometimes) pdu out
3487
                                p_pdu = pkt_coalesce_into_pdu(sguid_smi, p);
3488
                                if (p_pdu != nullptr)
3489
                                {
3490
                                        ccdbg(DPDU, "(%d)%s tx pdu %p id 0x%x",
3491
                                                  fd_tx, fd_to_state_name(fd_tx),
3492
                                                  p_pdu, SEQ_ANNO(p_pdu));
3493

    
3494
                                        // tx pdu (consumed by tcp client txplist)
3495
                                        (*_fdv)[fd_tx].txplist.push_back(p_pdu);
3496

    
3497
                                        // wake up fd tx task
3498
                                        (*_fdv)[fd_tx].tx_task->reschedule();
3499
                                }
3500
                        }
3501
                        else
3502
                        {
3503
                                // seq_assess wants p dropped
3504
                                p->kill();
3505
                        }
3506
                }
3507
        }
3508
        break;
3509

    
3510
        case FD_DTLS_SERVER:
3511
        case FD_DTLS_CLIENT:
3512
        {
3513
                if (PTYPE_ANNO(p) == PTYPE_GDP_PDU)
3514
                {
3515
                        seq_assign(sguid_smi, p);
3516
                        ccdbg(DPDU, "(%d)%s rx pdu %p id 0x%x",
3517
                                  fd, fd_to_state_name(fd), p, SEQ_ANNO(p));
3518
                        if (RXHALT_ANNO(p))
3519
                        {
3520
                                // schedule immediate seq p
3521
                                sguid_smi->second.sm_timer->clear();
3522
                                sguid_smi->second.sm_timer->reschedule_after_msec(0);
3523
                                SET_RXHALT_ANNO(p, 0);
3524
                        }
3525
                        
3526
                        p_pkt = pdu_fragment_into_pkts(fd_tx, p);
3527
                        // noisy fragmentation diagnostic block
3528
                        if (_debug_level >= DNOISY)
3529
                        {
3530
                                WritablePacket *l = p_pkt;
3531
                                
3532
                                ccdbg(DNOISY, "pdu %p fragmentation", p);
3533
                                while (l != nullptr)
3534
                                {
3535
                                        ccdbg(DNOISY, "\tloop p %p len %d next %p prev %p",
3536
                                                  l, l->length(), l->next(), l->prev());
3537
                                        l = static_cast<WritablePacket*>(l->next());
3538
                                }
3539
                        }
3540
                        // noisy fragmentation diagnostic block end
3541
                        if (p_pkt == nullptr)
3542
                        {
3543
                                ccwarn("(%d)%s packet pool exhausted",
3544
                                           fd, fd_to_state_name(fd));
3545
                                p_chain_recycle(p);
3546
                                break;
3547
                        }
3548

    
3549
                        // add frag chain to seq map (aka retx cache)
3550
                        ccdbg(DVERB, "(%d)%s add seq map on id 0x%x p %p",
3551
                                  fd, fd_to_state_name(fd), SEQ_ANNO(p_pkt), p_pkt);
3552
                        sguid_smi->second.seq_map.emplace(SEQ_ANNO(p_pkt), p_pkt);
3553

    
3554
                        // clone fragments, push clones on txplist
3555
                        while (p_pkt != nullptr)
3556
                        {
3557
                                p_clone = static_cast<WritablePacket*>(p_pkt->clone());
3558
                                if (p_clone == nullptr)
3559
                                {
3560
                                        ccwarn("(%d)%s packet clone exhausted",
3561
                                                   fd, fd_to_state_name(fd));
3562
                                        // no p_chain_recycle() since nacks will draw upon seq map
3563
                                        break;
3564
                                }
3565
                                else
3566
                                {
3567
                                        // disconnect p clone from next and prev annotations
3568
                                        p_clone->set_next(nullptr);
3569
                                        p_clone->set_prev(nullptr);
3570

    
3571
                                        ccdbg(DNOISY, "fd_tx %d push p %p cloned from p %p",
3572
                                                  fd_tx, p_clone, p_pkt);
3573
                                
3574
                                        // add p clone to txplist
3575
                                        (*_fdv)[fd_tx].txplist.push_back(p_clone);
3576
                                }
3577
                                // next p_pkt
3578
                                p_pkt = static_cast<WritablePacket*>(p_pkt->next());
3579

    
3580
                                // zlp
3581
                                aver(p_pkt != nullptr);
3582
                                // FIXME consider using ANNO rather than rely on zlp length
3583
                                if (p_pkt->length() == 0)
3584
                                {
3585
                                        // zlp is managed by retx map, not dtls txplist
3586
                                        break;
3587
                                }
3588
                        }
3589

    
3590
                        // pdu not currently used for fanout, recycle to free zlp
3591
                        p_chain_recycle(p);
3592
                }
3593
                else
3594
                {
3595
                        // dtls packet in, dtls packet out
3596
                        (*_fdv)[fd_tx].txplist.push_back(p);
3597

    
3598
                        // register sguid activity
3599
                        sguid_smi->second.sm_idle = 0;
3600
                }
3601

    
3602
                // wake up fd tx task
3603
                (*_fdv)[fd_tx].tx_task->reschedule();
3604
        }
3605
        break;
3606

    
3607
        default:
3608
        {
3609
                ccdbg(DVERB, "(%d)%s map route id 0x%x p %p bad ptype",
3610
                          fd, fd_to_state_name(fd));
3611
                p_chain_recycle(p);
3612
                aver(0);
3613
        }
3614
        break;
3615

    
3616
        } // switch
3617
}
3618

    
3619
void
3620
GDPv4Router::map_no_route (int fd, dst_map_it_t &dguid_dmi)
3621
{
3622
        wpl_it_t wpl_it;
3623
        WritablePacket *p;
3624
        WritablePacket *n;
3625
        otw_gdp_t *otw_gdp;
3626
        gdp_pname_t _tmp_pname_1;
3627
        gdp_pname_t _tmp_pname_2;        
3628

    
3629
        // dguid_dmi->second.fd still holds interim fd_cguid (no fd_eguid found)
3630
        
3631
        // kill dir request and erase dir request list element
3632
        wpl_it = dguid_dmi->second.dplist.begin();
3633
        ccdbg(DVVERB, "(%d)%s map recycle dir wpl_it %p p %p next %p prev %p",
3634
                  fd, fd_to_state_name(fd),
3635
                  wpl_it, (*wpl_it), (*wpl_it)->next(), (*wpl_it)->prev());
3636
        (*wpl_it)->kill();
3637
        wpl_it = dguid_dmi->second.dplist.erase(wpl_it);
3638

    
3639
        // send a noroute reply for each p chain, then kill p chain, erase elem
3640
        while (wpl_it != dguid_dmi->second.dplist.end())
3641
        {
3642
                p = static_cast<WritablePacket*>(*wpl_it);
3643
                ccdbg(DVVERB, "(%d)%s map recycle wpl_it %p p %p next %p prev %p",
3644
                          fd, fd_to_state_name(fd),
3645
                          wpl_it, (*wpl_it), (*wpl_it)->next(), (*wpl_it)->prev());
3646
                if (p != nullptr)
3647
                {
3648
                        // use first p (header only) to carry noroute reply
3649
                        otw_gdp = (otw_gdp_t *) p->data();
3650
                        // dguid_dmi->second.fd points at src fd until/unless resolved
3651
                        ccdbg(DINFO, "(%d)%s no route on p %p\n"
3652
                                   "\tdst[%s]\n"
3653
                                   "\tsrc[%s]@(%d)%s",
3654
                                   fd, fd_to_state_name(fd), p,
3655
                                   gdp_printable_name(otw_gdp->dst, _tmp_pname_1),
3656
                                   gdp_printable_name(otw_gdp->src, _tmp_pname_2),
3657
                                   dguid_dmi->second.fd,
3658
                                   fd_to_state_name(dguid_dmi->second.fd));
3659
                        std::copy(otw_gdp->src,
3660
                                          otw_gdp->src + sizeof(gdp_name_ca_t),
3661
                                          otw_gdp->dst);
3662
                        std::copy(dguid_dmi->first.data(),
3663
                                          dguid_dmi->first.data() + sizeof(gdp_name_ca_t),
3664
                                          otw_gdp->src);
3665
                        otw_gdp->flags = ((otw_gdp->flags & ~GDP_PKT_TYPE_MASK) |
3666
                                                          GDP_PKT_TYPE_NAK_NOROUTE);
3667
                        otw_gdp->payload_len = 0;
3668
                        otw_gdp->frag_len = 0;
3669
                        p->take(p->length() - GDP_HDR_SIZE);
3670
                        // disconnect repurposed p from p chain
3671
                        n = static_cast<WritablePacket*>(p->next());
3672
                        p->set_next(nullptr);
3673
                        if (n != nullptr)
3674
                        {
3675
                                n->set_prev(nullptr);
3676
                        }
3677
                        // packets need preset seq_mf_off, harmless for pdus
3678
                        SET_SEQ_PRESET_ANNO(p, 1);
3679
                        // pdu or packet in, no route pdu or packet returned
3680
                        (*_fdv)[dguid_dmi->second.fd].txplist.push_back(p);
3681
                        // wake up fd tx task
3682
                        (*_fdv)[dguid_dmi->second.fd].tx_task->reschedule();
3683
                        // skip over disconnected repurposed p
3684
                        p = n;
3685
                }
3686

    
3687
                // kill p chain remainder, if any
3688
                while (p != nullptr)
3689
                {
3690
                        n = static_cast<WritablePacket*>(p->next());
3691
                        ccdbg(DVVERB, "(%d)%s recycle dplist p %p next %p prev %p",
3692
                                  fd, fd_to_state_name(fd), p, p->next(), p->prev());
3693
                        p->kill();
3694
                        p = n;
3695
                }
3696

    
3697
                // dplist erase this element and move to next
3698
                wpl_it = dguid_dmi->second.dplist.erase(wpl_it);
3699
        }
3700

    
3701
        // be careful not to close the interim fd_cguid (no fd_eguid to close)
3702
}
3703

    
3704
void
3705
GDPv4Router::dst_map_recycle (int fd, dst_map_it_t dguid_dmi)
3706
{
3707
        wpl_it_t wpl_it;
3708
        src_map_it_t sguid_smi;
3709
        WritablePacket *p;
3710
        WritablePacket *n;
3711
        gdp_pname_t _tmp_pname_1;
3712

    
3713
        ccdbg(DINFO, "(%d)%s recycle dguid[%s]@(%d)%s",
3714
                  fd, fd_to_state_name(fd),
3715
                  gdp_printable_name(dguid_dmi->first.data(), _tmp_pname_1),
3716
                  dguid_dmi->second.fd, fd_to_state_name(dguid_dmi->second.fd));
3717

    
3718
        // clean and erase src map entries
3719
        sguid_smi = dguid_dmi->second.src_map.begin();
3720
        while (sguid_smi != dguid_dmi->second.src_map.end())
3721
        {
3722
                src_map_recycle(fd, sguid_smi, true);
3723
                sguid_smi = dguid_dmi->second.src_map.erase(sguid_smi);
3724
        }
3725
        aver(dguid_dmi->second.src_map.size() == 0);
3726
        
3727
        // delete map dplist p chains
3728
        wpl_it = dguid_dmi->second.dplist.begin();
3729
        while (wpl_it != dguid_dmi->second.dplist.end())
3730
        {
3731
                // map recycle does not send no route responses, just kill
3732
                p = static_cast<WritablePacket*>(*wpl_it);
3733
                while (p != nullptr)
3734
                {
3735
                        n = static_cast<WritablePacket*>(p->next());
3736
                        ccdbg(DVVERB, "(%d)%s recycle dplist p %p",
3737
                                  fd, fd_to_state_name(fd), p);
3738
                        p->kill();
3739
                        p = n;
3740
                }
3741

    
3742
                // erase dplist element, continue with the next element
3743
                wpl_it = dguid_dmi->second.dplist.erase(wpl_it);
3744
        }
3745
        aver(dguid_dmi->second.dplist.size() == 0);
3746

    
3747
        if (dguid_dmi->second.map_timer != nullptr ||
3748
                dguid_dmi->second.map_timer_dmip != nullptr)
3749
        {
3750
                ccdbg(DVVERB, "(%d)%s delete timer %p %p",
3751
                          fd, fd_to_state_name(fd),
3752
                          dguid_dmi->second.map_timer, dguid_dmi->second.map_timer_dmip);
3753
                delete dguid_dmi->second.map_timer;
3754
                delete dguid_dmi->second.map_timer_dmip;
3755
        }
3756

    
3757
        dst_map.erase(dguid_dmi);
3758
        display_state(fd);
3759
}
3760

    
3761
void
3762
GDPv4Router::src_map_recycle (int fd, src_map_it_t sguid_smi, bool fd_rx_detach)
3763
{
3764
        int fd_rx;
3765
        int fd_tx;
3766
        gdp_pname_t _tmp_pname_1;
3767
        gdp_pname_t _tmp_pname_2;
3768
        src_map_it_t *sguid_smip;
3769
        dst_map_it_t *dguid_dmip;
3770
        WritablePacket *p;
3771
        WritablePacket *n;
3772

    
3773
        sguid_smip = sguid_smi->second.sm_task_smip;
3774
        dguid_dmip = sguid_smi->second.sm_task_copy_dmip;
3775
        
3776
        fd_rx = sguid_smi->second.fd_rx;
3777
        fd_tx = (*dguid_dmip)->second.fd;
3778
        
3779
        ccdbg(DINFO, "(%d)%s recycle sguid[%s]@(%d)%s\n"
3780
                  "\towned by dguid[%s]@(%d)%s",
3781
                  fd, fd_to_state_name(fd),
3782
                  gdp_printable_name(sguid_smi->first.data(), _tmp_pname_1),
3783
                  fd_rx, fd_to_state_name(fd_rx),
3784
                  gdp_printable_name(
3785
                          (*sguid_smi->second.sm_task_copy_dmip)->first.data(),
3786
                          _tmp_pname_2),
3787
                  fd_tx, fd_to_state_name(fd_tx));
3788

    
3789
        // unschedule sm_task and sm_timer
3790
        sguid_smi->second.sm_timer->unschedule();
3791
        sguid_smi->second.sm_task->unschedule();
3792

    
3793
        // detach non-recycled fds (fd recycle walks and erases it's own src_mil)
3794
        if (fd_rx_detach)
3795
        {
3796
                auto src_mil_elem = (*_fdv)[fd_rx].src_mil.begin();
3797
                while (src_mil_elem != (*_fdv)[fd_rx].src_mil.end())
3798
                {
3799
                        if (*src_mil_elem == sguid_smi)
3800
                        {
3801
                                (*_fdv)[fd_rx].src_mil.erase(src_mil_elem);
3802
                                break;
3803
                        }
3804
                        ++src_mil_elem;
3805
                }
3806
        }
3807

    
3808
    // seq_map
3809
        ccdbg(DNOISY, "(%d)%s seq map size %d",
3810
                  fd, fd_to_state_name(fd), sguid_smi->second.seq_map.size());
3811
        auto seq_map_it = sguid_smi->second.seq_map.begin();
3812
        while (seq_map_it != sguid_smi->second.seq_map.end())
3813
        {
3814
                p = static_cast<WritablePacket*>(seq_map_it->second);
3815
                while (p != nullptr)
3816
                {
3817
                        ccdbg(DVVERB, "(%d)%s seq map recycle p %p",
3818
                                  fd, fd_to_state_name(fd), p);
3819
                        n = static_cast<WritablePacket*>(p->next());
3820
                        p->kill();
3821
                        p = n;
3822
                }
3823
                seq_map_it = sguid_smi->second.seq_map.erase(seq_map_it);
3824
        }
3825
        avow(sguid_smi->second.seq_map.size() == 0);
3826
        
3827
        // nplist
3828
        ccdbg(DNOISY, "(%d)%s nplist size %d",
3829
                  fd, fd_to_state_name(fd), sguid_smi->second.nplist.size());
3830
        auto npl_it = sguid_smi->second.nplist.begin();
3831
        while (npl_it != sguid_smi->second.nplist.end())
3832
        {
3833
                ccdbg(DVVERB, "(%d)%s nplist recycle p %p",
3834
                          fd, fd_to_state_name(fd), *npl_it);
3835
                (*npl_it)->kill();
3836
                npl_it = sguid_smi->second.nplist.erase(npl_it);
3837
        }
3838
        avow(sguid_smi->second.nplist.size() == 0);
3839

    
3840
        delete sguid_smi->second.sm_timer;
3841
        sguid_smi->second.sm_timer = nullptr;
3842
        (*dguid_dmip)->second.expired_sm_tasks.push_back(sguid_smi->second.sm_task);
3843
        sguid_smi->second.sm_task = nullptr;
3844
        if (!fd_rx_detach)
3845
        {
3846
                (*dguid_dmip)->second.src_map.erase(*sguid_smip);
3847
        }
3848
        delete sguid_smip;
3849
}
3850

    
3851
void
3852
GDPv4Router::fd_detach_map (int fd, dst_map_it_t &dguid_dmi)
3853
{
3854
        int index;
3855
        gdp_pname_t _tmp_pname_1;
3856

    
3857
        // walk map it elems tracked by fd to find and remove dguid_dmi
3858
        index = 0;
3859
        auto dst_mil_elem = (*_fdv)[fd].dst_mil.begin();
3860
        while (dst_mil_elem != (*_fdv)[fd].dst_mil.end())
3861
        {
3862
                ccdbg(DNOISY, "detach walk (%d)%s %d/%d %p %p dguid[%s]",
3863
                          fd, fd_to_state_name(fd), index, (*_fdv)[fd].dst_mil.size(),
3864
                          dst_mil_elem, *dst_mil_elem,
3865
                          gdp_printable_name((*dst_mil_elem)->first.data(),
3866
                                                                 _tmp_pname_1));
3867
                if (*dst_mil_elem == dguid_dmi)
3868
                {
3869
                        ccdbg(DVVERB, "(%d)%s fd detach map dguid[%s] @ (%d)%s",
3870
                                  fd, fd_to_state_name(fd),
3871
                                  gdp_printable_name((*dst_mil_elem)->first.data(),
3872
                                                                         _tmp_pname_1),
3873
                                  (*dst_mil_elem)->second.fd,
3874
                                  fd_to_state_name((*dst_mil_elem)->second.fd));
3875
                        // do not erase *dst_mil_elem aka dguid_dmi, just detach from fd
3876
                        (*_fdv)[fd].dst_mil.erase(dst_mil_elem);
3877
                        return;
3878
                }
3879
                ++dst_mil_elem;
3880
                ++index;
3881
        }
3882

    
3883
        ccerr("(%d)%s fd detach map not found dguid[%s] @ (%d)%s",
3884
                  fd, fd_to_state_name(fd),
3885
                  gdp_printable_name((*dst_mil_elem)->first.data(), _tmp_pname_1),
3886
                  (*dst_mil_elem)->second.fd,
3887
                  fd_to_state_name((*dst_mil_elem)->second.fd));
3888
        aver(0); // not behaving according to design
3889
}
3890

    
3891
void
3892
GDPv4Router::fd_recycle (int fd)
3893
{
3894
        wpl_it_t wpl_it;
3895
        WritablePacket *p;
3896
        WritablePacket *n;
3897
        WritablePacket *dp;
3898
        otw_dir_t *otw_dir;
3899
        gdp_pname_t _tmp_pname_1;
3900
        gdp_pname_t _tmp_pname_2;
3901

    
3902
        // disable all fd events
3903
        remove_select(fd, SELECT_READ | SELECT_WRITE);
3904

    
3905
        ccdbg(DNOISY, "(%d)%s dst_mil size %d",
3906
                  fd, fd_to_state_name(fd), (*_fdv)[fd].dst_mil.size());        
3907

    
3908
        // fd owner's address withdraw tx
3909
        auto dst_mil_elem = (*_fdv)[fd].dst_mil.begin();
3910
        if (!_single_router_mode &&
3911
                dst_mil_elem != (*_fdv)[fd].dst_mil.end())
3912
        {
3913
                avow((*dst_mil_elem)->second.addr_state == ADDR_GDPC ||
3914
                         (*dst_mil_elem)->second.addr_state == ADDR_GDPR);
3915

    
3916
                dp = Packet::make(offsetof(otw_dir_t, eguid[1][0]));
3917
                if (dp == nullptr)
3918
                {
3919
                        ccwarn("(%d)%s packet pool exhausted on %d",
3920
                                   _fd_directory, fd_to_state_name(_fd_directory),
3921
                                   offsetof(otw_dir_t, eguid[1][0]));
3922
                }
3923
                else
3924
                {
3925
                        otw_dir = (otw_dir_t *) dp->data();
3926
                        otw_dir->ver = GDP_CHAN_PROTO_VERSION;
3927
                        otw_dir->id = htons(_dr_id++);
3928
                        otw_dir->cmd = GDP_CMD_DIR_DELETE;
3929
                        // avoid race conditions by only deleting nexthop we own
3930
                        memcpy(otw_dir->eguid[0], _gdpr, sizeof(gdp_name_t));
3931
                        memcpy(otw_dir->dguid, (*dst_mil_elem)->first.data(),
3932
                                   sizeof(gdp_name_t));
3933
                        datagram_tx(_fd_directory, dp, nullptr, 0);
3934
                        ccdbg(DVERB, "(%d)%s fd withdraw tx id 0x%x\n"
3935
                                  "\teguid[%s]\n"
3936
                                  "\tdguid[%s]",
3937
                                  _fd_directory, fd_to_state_name(_fd_directory),
3938
                                  ntohs(otw_dir->id),
3939
                                  gdp_printable_name(otw_dir->eguid[0], _tmp_pname_1),
3940
                                  gdp_printable_name(otw_dir->dguid, _tmp_pname_2));
3941
                        dp->kill();
3942
                }
3943
        }
3944

    
3945
        // recycle any dst map referencing this fd, back to front (fd owner last)
3946
        while (!(*_fdv)[fd].dst_mil.empty())
3947
        {
3948
                dst_map_recycle(fd, (*_fdv)[fd].dst_mil.back());
3949
                (*_fdv)[fd].dst_mil.pop_back();
3950
        }
3951
        aver((*_fdv)[fd].dst_mil.size() == 0);
3952
        
3953
        // recycle other src map (held by any parent dst map) referencing this fd
3954
        auto src_mil_elem = (*_fdv)[fd].src_mil.begin();
3955
        while (src_mil_elem != (*_fdv)[fd].src_mil.end())
3956
        {
3957
                src_map_recycle(fd, *src_mil_elem, false);
3958
                src_mil_elem = (*_fdv)[fd].src_mil.erase(src_mil_elem);
3959
        }
3960
        aver((*_fdv)[fd].src_mil.size() == 0);
3961

    
3962
        switch ((*_fdv)[fd].fd_state)
3963
        {
3964
        
3965
        case FD_TCP_CLIENT:
3966
        {
3967
                // retx_timer
3968
                avow((*_fdv)[fd].retx_timer == nullptr);
3969

    
3970
                // rx_task
3971
                avow((*_fdv)[fd].rx_task != nullptr);
3972
                (*_fdv)[fd].rx_task->unschedule();
3973
                // tasks cannot self-delete, so unschedule and let next fd user delete
3974

    
3975
                // tx_task
3976
                avow((*_fdv)[fd].tx_task != nullptr);
3977
                (*_fdv)[fd].tx_task->unschedule();
3978
                // tasks cannot self-delete, so unschedule and let next fd user delete
3979

    
3980
                // ssl
3981
                avow((*_fdv)[fd].ssl == nullptr);
3982

    
3983
                // rxp
3984
                avow((*_fdv)[fd].rxp != nullptr);
3985
                (*_fdv)[fd].rxp->kill();
3986
                (*_fdv)[fd].rxp = nullptr;
3987

    
3988
                // zlplist
3989
                while (!(*_fdv)[fd].zlplist.empty())
3990
                {
3991
                        (*_fdv)[fd].zlplist.front()->kill();
3992
                        (*_fdv)[fd].zlplist.pop_front();
3993
                }
3994

    
3995
                // dst_mil handled above
3996
        // src_mil handled above
3997

    
3998
                // txplist
3999
                wpl_it = (*_fdv)[fd].txplist.begin();
4000
                while (wpl_it != (*_fdv)[fd].txplist.end())
4001
                {
4002
                        p = static_cast<WritablePacket*>(*wpl_it);
4003
                        while (p != nullptr)
4004
                        {
4005
                                n = static_cast<WritablePacket*>(p->next());
4006
                                ccdbg(DVVERB, "(%d)%s recycle txplist p %p",
4007
                                          fd, fd_to_state_name(fd), p);
4008
                                p->kill();
4009
                                p = n;
4010
                        }
4011

    
4012
                        // erase txplist element, continue with the next element
4013
                        wpl_it = (*_fdv)[fd].txplist.erase(wpl_it);
4014
                }
4015

    
4016
                // rx_throttled
4017
                (*_fdv)[fd].rx_throttled = false;
4018
                
4019
                // fd
4020
                confirm_shutdown(fd);
4021
                confirm_close(fd);
4022

    
4023
                // reset simple types only
4024
                _fdv_init_ints_only(fd);
4025

    
4026
                // recycle complete
4027
                (*_fdv)[fd].fd_state = FD_NONE;
4028
        }
4029
        break;
4030

    
4031
        case FD_DTLS_ACCEPT:
4032
        case FD_DTLS_SERVER:
4033
        {
4034
                // retx_timer
4035
                avow((*_fdv)[fd].retx_timer == nullptr);
4036

    
4037
                // rx_task
4038
                avow((*_fdv)[fd].rx_task != nullptr);
4039
                (*_fdv)[fd].rx_task->unschedule();
4040
                // tasks cannot self-delete, so unschedule and let next fd user delete
4041
                
4042
                // tx_task
4043
                avow((*_fdv)[fd].tx_task != nullptr);
4044
                (*_fdv)[fd].tx_task->unschedule();
4045
                // tasks cannot self-delete, so unschedule and let next fd user delete
4046

    
4047
        // ssl
4048
                avow((*_fdv)[fd].ssl != nullptr);
4049
                SSL_shutdown((*_fdv)[fd].ssl);
4050
                SSL_free((*_fdv)[fd].ssl);
4051
                (*_fdv)[fd].ssl = nullptr;
4052

    
4053
                // rxp
4054
                avow((*_fdv)[fd].rxp == nullptr);
4055

    
4056
                // zlplist
4057
                avow((*_fdv)[fd].zlplist.empty());
4058
                
4059
                // dst_mil handled above
4060
                // src_mil handled above
4061

    
4062
                // txplist
4063
                wpl_it = (*_fdv)[fd].txplist.begin();
4064
                while (wpl_it != (*_fdv)[fd].txplist.end())
4065
                {
4066
                        p = static_cast<WritablePacket*>(*wpl_it);
4067
                        while (p != nullptr)
4068
                        {
4069
                                n = static_cast<WritablePacket*>(p->next());
4070
                                ccdbg(DVVERB, "(%d)%s recycle txplist p %p",
4071
                                          fd, fd_to_state_name(fd), p);
4072
                                p->kill();
4073
                                p = n;
4074
                        }
4075

    
4076
                        // erase txplist element, continue with the next element
4077
                        wpl_it = (*_fdv)[fd].txplist.erase(wpl_it);
4078
                }
4079

    
4080
                // FD_DTLS_ACCEPT, FD_DTLS_SERVER can only arise from FD_UDP_CONNECT
4081
                // fd is already in connection oriented mode with configured peer
4082
                avow(fd > _fd_directory && fd <= _fd_last_configured);
4083
                avow((*_fdv)[fd].retx_timer == nullptr);
4084
                (*_fdv)[fd].fd_state = FD_UDP_CONNECT;
4085
                // reset simple types only
4086
                _fdv_init_ints_only(fd);
4087
                // udp connect attempt restarted
4088
                try
4089
                {
4090
                        (*_fdv)[fd].retx_timer = new Timer(static_udp_connect_timer,
4091
                                                                                           (void *)(intptr_t)fd);
4092
                }
4093
                catch (std::bad_alloc)
4094
                {
4095
                        ccerr("udp reconnect (%d) retx timer", fd);
4096
                        break;
4097
                }
4098
                (*_fdv)[fd].retx_timer->initialize(this, true);
4099
                (*_fdv)[fd].retx_timer->schedule_after(FD_UDP_CONNECT_NOW_TS_DELTA);
4100
                add_select(fd, SELECT_READ);
4101
        }
4102
        break;
4103

    
4104
        case FD_DTLS_DO_HANDSHAKE:
4105
        case FD_DTLS_CLIENT:
4106
        {
4107
                // retx_timer
4108
                avow((*_fdv)[fd].retx_timer == nullptr);
4109

    
4110
                // rx_task
4111
                avow((*_fdv)[fd].rx_task != nullptr);
4112
                (*_fdv)[fd].rx_task->unschedule();
4113
                // tasks cannot self-delete, so unschedule and await udp connect restart
4114

    
4115
                // tx_task
4116
                avow((*_fdv)[fd].tx_task != nullptr);
4117
                (*_fdv)[fd].tx_task->unschedule();
4118
                // tasks cannot self-delete, so unschedule and await udp connect restart
4119
                
4120
                // ssl
4121
                avow((*_fdv)[fd].ssl != nullptr);
4122
                SSL_shutdown((*_fdv)[fd].ssl);
4123
                SSL_free((*_fdv)[fd].ssl);
4124
                (*_fdv)[fd].ssl = nullptr;
4125

    
4126
                // rxp
4127
                avow((*_fdv)[fd].rxp == nullptr);
4128

    
4129
                // zlplist
4130
                avow((*_fdv)[fd].zlplist.empty());
4131
                
4132
                // dst_mil handled above
4133
                // src_mil handled above
4134
                
4135
                // txplist
4136
                wpl_it = (*_fdv)[fd].txplist.begin();
4137
                while (wpl_it != (*_fdv)[fd].txplist.end())
4138
                {
4139
                        p = static_cast<WritablePacket*>(*wpl_it);
4140
                        while (p != nullptr)
4141
                        {
4142
                                n = static_cast<WritablePacket*>(p->next());
4143
                                ccdbg(DVVERB, "(%d)%s recycle txplist p %p",
4144
                                          fd, fd_to_state_name(fd), p);
4145
                                p->kill();
4146
                                p = n;
4147
                        }
4148

    
4149
                        // erase txplist element, continue with the next element
4150
                        wpl_it = (*_fdv)[fd].txplist.erase(wpl_it);
4151
                }
4152

    
4153
                // rx_throttled
4154
                (*_fdv)[fd].rx_throttled = false;
4155

    
4156
                // fd
4157
                avow(fd > _fd_last_configured);
4158
                confirm_shutdown(fd);
4159
                confirm_close(fd);
4160

    
4161
                // reset simple types only
4162
                _fdv_init_ints_only(fd);
4163

    
4164
                // recycle complete
4165
                (*_fdv)[fd].fd_state = FD_NONE;
4166
        }
4167
        break;
4168

    
4169
    // fd in these states should not be submitted to fd_recycle
4170
        case FD_NONE:
4171
        case FD_DIRECTORY:
4172
        case FD_UDP_LISTEN:
4173
        case FD_UDP_CONNECT:
4174
        case FD_UDP_CONNECT_DUPE:
4175
        case FD_TCP_LISTEN:
4176
        default:
4177
        {
4178
                avow(0);
4179
        }
4180
        break;
4181
                
4182
        } // switch
4183

    
4184
        ccdbg(DINFO, "(%d)%s fd recycled", fd, fd_to_state_name(fd));
4185
        display_state(fd);
4186
}
4187

    
4188
void
4189
GDPv4Router::seq_assign (src_map_it_t &sguid_smi, WritablePacket *p)
4190
{
4191
        otw_gdp_t *otw_gdp;
4192
        int fd_rx;
4193
        gdp_pname_t _tmp_pname_1;
4194
        gdp_pname_t _tmp_pname_2;
4195

    
4196
        avow(p != nullptr);
4197

    
4198
        fd_rx = RXFD_ANNO(p);
4199
        avow((*_fdv)[fd_rx].fd_state == FD_TCP_CLIENT);
4200

    
4201
        ccdbg(DVVERB, "(%d)%s seq assign p %p", fd_rx, fd_to_state_name(fd_rx), p);
4202

    
4203
        otw_gdp = (otw_gdp_t *) p->data();
4204

    
4205
        if (SEQ_PRESET_ANNO(p))
4206
        {
4207
                ccdbg(DVVERB, "(%d)%s seq on p %p preset\n"
4208
                          "\tid 0x%x mf %d off 0x%x\n"
4209
                          "\tsrc[%s]\n"
4210
                          "\tdst[%s]",
4211
                          fd_rx, fd_to_state_name(fd_rx), p,
4212
                          GET_SEQ(otw_gdp->seq_mf_off),
4213
                          GET_MF(otw_gdp->seq_mf_off),
4214
                          GET_OFF(otw_gdp->seq_mf_off),
4215
                          gdp_printable_name(otw_gdp->src, _tmp_pname_1),
4216
                          gdp_printable_name(otw_gdp->dst, _tmp_pname_2));
4217
        }
4218
        else
4219
        {
4220
                // new seq
4221
                if (sguid_smi->second.seq_map_mode == SMM_RETX_INIT)
4222
                {
4223
                        sguid_smi->second.seq_map_mode = SMM_RETX_LIVE;
4224
                                
4225
                        otw_gdp->flags |= GDP_PKT_TYPE_SSEQ;
4226
                        // new seq assigned to first p
4227
                        SET_SEQ(otw_gdp->seq_mf_off, generate_sequence());
4228
                        // calculate so_next seq and off
4229
                        if (GET_MF(otw_gdp->seq_mf_off) == 0)
4230
                        {
4231
                                // wrap(seq++), mf ignored, off 0
4232
                                SET_SEQ_MF_OFF(sguid_smi->second.so_next,
4233
                                                           ((GET_SEQ(otw_gdp->seq_mf_off) + 1) &
4234
                                                                SEQ_MASK), 0, 0);
4235
                        }
4236
                        else
4237
                        {
4238
                                // same seq, mf ignored, off+=frag_len
4239
                                SET_SEQ_MF_OFF(sguid_smi->second.so_next,
4240
                                                           GET_SEQ(otw_gdp->seq_mf_off), 0,
4241
                                                           (GET_OFF(otw_gdp->seq_mf_off) +
4242
                                                                ntohs(otw_gdp->frag_len)));
4243
                        }
4244
                        // FIXME need to review with removal of duo?
4245
                        // so_last used to detect idle duo
4246
                        sguid_smi->second.so_last = sguid_smi->second.so_next;
4247
                }
4248
                else
4249
                {
4250
                        avow(sguid_smi->second.fd_rx == fd_rx);
4251
                        
4252
                        // so_next seq assigned to p
4253
                        SET_SEQ(otw_gdp->seq_mf_off,
4254
                                        GET_SEQ(sguid_smi->second.so_next));
4255
                        // calculate so_next seq and off
4256
                        if (GET_MF(otw_gdp->seq_mf_off) == 0)
4257
                        {
4258
                                // wrap(seq++), mf ignored, off 0
4259
                                SET_SEQ_MF_OFF(sguid_smi->second.so_next,
4260
                                                           ((GET_SEQ(otw_gdp->seq_mf_off) + 1) &
4261
                                                                SEQ_MASK), 0, 0);
4262
                        }
4263
                        else
4264
                        {
4265
                                // same seq, mf ignored, off+=frag_len
4266
                                SET_SEQ_MF_OFF(sguid_smi->second.so_next,
4267
                                                           GET_SEQ(otw_gdp->seq_mf_off), 0,
4268
                                                           (GET_OFF(otw_gdp->seq_mf_off) +
4269
                                                                ntohs(otw_gdp->frag_len)));
4270
                        }
4271
                }
4272
                        
4273
                // copy assigned seq into p anno
4274
                SET_SEQ_ANNO(p, GET_SEQ(otw_gdp->seq_mf_off));
4275
                ccdbg(DVVERB, "(%d)%s seq on p %p origin\n"
4276
                          "\tid 0x%x mf %d off 0x%x sseq %d seqp %d nnr %d\n"
4277
                          "\tsrc[%s]\n"
4278
                          "\tdst[%s]",
4279
                          fd_rx, fd_to_state_name(fd_rx), p,
4280
                          GET_SEQ(otw_gdp->seq_mf_off),
4281
                          GET_MF(otw_gdp->seq_mf_off),
4282
                          GET_OFF(otw_gdp->seq_mf_off),
4283
                          ((otw_gdp->flags & GDP_PKT_TYPE_SSEQ) ? 1 : 0),
4284
                          (((otw_gdp->flags & GDP_PKT_TYPE_MASK) ==
4285
                                GDP_PKT_TYPE_SEQ_PACKET) ? 1 : 0),
4286
                          (((otw_gdp->flags & GDP_PKT_TYPE_MASK) ==
4287
                                GDP_PKT_TYPE_NAK_NOROUTE) ? 1 : 0),
4288
                          gdp_printable_name(otw_gdp->src, _tmp_pname_1),
4289
                          gdp_printable_name(otw_gdp->dst, _tmp_pname_2));
4290
        }
4291
}
4292

    
4293
bool
4294
GDPv4Router::seq_assess (src_map_it_t &sguid_smi, WritablePacket *p)
4295
{
4296
        int fd_rx;
4297
        otw_gdp_t *otw_gdp;
4298
        uint32_t nack_seq_mf_off;
4299
        bool valid;
4300
        
4301
        avow(p != nullptr);
4302

    
4303
        fd_rx = RXFD_ANNO(p);
4304
        avow((*_fdv)[fd_rx].fd_state == FD_DTLS_SERVER ||
4305
                 (*_fdv)[fd_rx].fd_state == FD_DTLS_CLIENT);
4306

    
4307
        ccdbg(DVVERB, "(%d)%s seq assess p %p", fd_rx, fd_to_state_name(fd_rx), p);
4308

    
4309
        otw_gdp = (otw_gdp_t *) p->data();
4310

    
4311
        SET_SEQ_ANNO(p, GET_SEQ(otw_gdp->seq_mf_off));
4312

    
4313
        ccdbg(DVVERB, "(%d)%s seq on p %p check\n"
4314
                  "\tid 0x%x mf %d off 0x%x sseq %d seqp %d nnr %d\n",
4315
                  fd_rx, fd_to_state_name(fd_rx), p,
4316
                  GET_SEQ(otw_gdp->seq_mf_off),
4317
                  GET_MF(otw_gdp->seq_mf_off),
4318
                  GET_OFF(otw_gdp->seq_mf_off),
4319
                  ((otw_gdp->flags & GDP_PKT_TYPE_SSEQ) ? 1 : 0),
4320
                  (((otw_gdp->flags & GDP_PKT_TYPE_MASK) ==
4321
                        GDP_PKT_TYPE_SEQ_PACKET) ? 1 : 0),
4322
                  (((otw_gdp->flags & GDP_PKT_TYPE_MASK) ==
4323
                        GDP_PKT_TYPE_NAK_NOROUTE) ? 1 : 0));
4324

    
4325
        // new seq
4326
        if (sguid_smi->second.seq_map_mode == SMM_COAL_INIT)
4327
        {
4328
                sguid_smi->second.seq_map_mode = SMM_COAL_LIVE;
4329

    
4330
                // process p seq info
4331
                if ((otw_gdp->flags & (GDP_PKT_TYPE_MASK | GDP_PKT_TYPE_SSEQ)) ==
4332
                        GDP_PKT_TYPE_SSEQ)
4333
                {
4334
                        // case 1 normal: new, start sequence (not seq packet)
4335
                        ccdbg(DVVERB, "(%d)%s seq start at p %p\n"
4336
                                  "\tid 0x%x mf %d off 0x%x\n",
4337
                                  fd_rx, fd_to_state_name(fd_rx), p,
4338
                                  GET_SEQ(otw_gdp->seq_mf_off), GET_MF(otw_gdp->seq_mf_off),
4339
                                  GET_OFF(otw_gdp->seq_mf_off));
4340
            // init nack to the nearest pdu (one or more frags) boundary
4341
                        if (GET_SEQ(otw_gdp->seq_mf_off) != 0)
4342
                        {
4343
                                // so_nacked seq--
4344
                                SET_SEQ_MF_OFF(sguid_smi->second.so_nacked,
4345
                                                           ((GET_SEQ(otw_gdp->seq_mf_off) - 1) &
4346
                                                                SEQ_MASK), 0, 0);
4347
                        }
4348
                        else
4349
                        {
4350
                                // so_nacked wraps backwards to seq max
4351
                                SET_SEQ_MF_OFF(sguid_smi->second.so_nacked, SEQ_MASK, 0, 0);
4352
                        }
4353
                }
4354
                else
4355
                {
4356
                        // case 2 loss: new, not start sequence nor seq packet
4357
                        // case 3 loss: new, seq packet (not start sequence)
4358
                        //
4359
                        // nack backwards to the nearest pdu (one or more frags) boundary
4360
                        if (GET_OFF(otw_gdp->seq_mf_off) == 0)
4361
                        {
4362
                                // request missed pdu preceding this pdu boundary
4363
                                if (GET_SEQ(otw_gdp->seq_mf_off) != 0)
4364
                                {
4365
                                        // nack seq--
4366
                                        SET_SEQ_MF_OFF(nack_seq_mf_off,
4367
                                                                   ((GET_SEQ(otw_gdp->seq_mf_off) - 1) &
4368
                                                                        SEQ_MASK), 0, 0);
4369
                                }
4370
                                else
4371
                                {
4372
                                        // nack seq wraps backwards to seq max
4373
                                        SET_SEQ_MF_OFF(nack_seq_mf_off, SEQ_MASK, 0, 0);
4374
                                }
4375
                        }
4376
                        else
4377
                        {
4378
                                // request missed fragment(s) preceding this fragment
4379
                                SET_SEQ_MF_OFF(nack_seq_mf_off, GET_SEQ(otw_gdp->seq_mf_off),
4380
                                                           1, 0);
4381
                        }
4382
                        ccdbg(DVVERB, "(%d)%s seq start nack at p %p\n"
4383
                                  "\tid 0x%x mf %d off 0x%x\n"
4384
                                  "\tid 0x%x mf %d off 0x%x\n",
4385
                                  fd_rx, fd_to_state_name(fd_rx), p,
4386
                                  GET_SEQ(nack_seq_mf_off), GET_MF(nack_seq_mf_off),
4387
                                  GET_OFF(nack_seq_mf_off),
4388
                                  GET_SEQ(otw_gdp->seq_mf_off),
4389
                                  GET_MF(otw_gdp->seq_mf_off), GET_OFF(otw_gdp->seq_mf_off));
4390
                        sguid_smi->second.so_nacked = nack_seq_mf_off;
4391
                        seq_hole(sguid_smi, p, nack_seq_mf_off, otw_gdp->seq_mf_off);
4392
                        seq_nack(sguid_smi, false);
4393
                }
4394
        }                        
4395
        else
4396
        {
4397
                if ((otw_gdp->flags & GDP_PKT_TYPE_SSEQ) != GDP_PKT_TYPE_SSEQ)
4398
                {
4399
                        // case 3 normal: known, not start sequence nor seq packet
4400
                        // case 4 normal: known, seq packet (not start sequence)
4401
                        //
4402
                        // check predicted seq and off is a match or (if <) a retx
4403
                        if (GET_SEQ_OFF(otw_gdp->seq_mf_off) ==
4404
                                GET_SEQ_OFF(sguid_smi->second.so_test))
4405
                        {
4406
                                if ((otw_gdp->flags & GDP_PKT_TYPE_MASK) !=
4407
                                        GDP_PKT_TYPE_SEQ_PACKET)
4408
                                {
4409
                                        ccdbg(DVVERB, "(%d)%s seq ok at p %p\n",
4410
                                                  fd_rx, fd_to_state_name(fd_rx), p);
4411
                                }
4412
                                else
4413
                                {
4414
                                        ccdbg(DPDU, "(%d)%s seq ok seqp at p %p\n",
4415
                                                  fd_rx, fd_to_state_name(fd_rx), p);
4416
                                        // schedule nack (front or trim) evaluation
4417
                                        sguid_smi->second.nack_trim = 2;
4418
                                        sguid_smi->second.sm_task->reschedule();
4419
                                }
4420
                        }
4421
                        // if so_test in first half of window, a retx is ! in half above,
4422
                        // or if so_test in second half of window, a retx is in half below
4423
                        else if ((GET_SEQ(sguid_smi->second.so_test) <= SEQ_MP &&
4424
                                          !(GET_SEQ_OFF(otw_gdp->seq_mf_off) >=
4425
                                                GET_SEQ_OFF(sguid_smi->second.so_test) &&
4426
                                                GET_SEQ_OFF(otw_gdp->seq_mf_off) <
4427
                                                GET_SEQ_OFF(sguid_smi->second.so_test) + SEQ_OFF_MP)) ||
4428
                                         (GET_SEQ(sguid_smi->second.so_test) > SEQ_MP &&
4429
                                          GET_SEQ_OFF(otw_gdp->seq_mf_off) >=
4430
                                          GET_SEQ_OFF(sguid_smi->second.so_test) - SEQ_OFF_MP &&
4431
                                          GET_SEQ_OFF(otw_gdp->seq_mf_off) <
4432
                                          GET_SEQ_OFF(sguid_smi->second.so_test)))
4433
                        {
4434
                                ccdbg(DVVERB, "(%d)%s seq assess retx at p %p",
4435
                                          fd_rx, fd_to_state_name(fd_rx), p);
4436
                            valid = seq_retx(sguid_smi, p);
4437
                                return valid;
4438
                        }
4439
                        else
4440
                        {
4441
                                // nack from lower bound to upper bound
4442
                                if ((GET_SEQ(sguid_smi->second.so_test) <= SEQ_MP &&
4443
                                         !(GET_SEQ_OFF(sguid_smi->second.so_nacked) >
4444
                                           GET_SEQ_OFF(sguid_smi->second.so_test) &&
4445
                                           GET_SEQ_OFF(sguid_smi->second.so_nacked) <
4446
                                           GET_SEQ_OFF(sguid_smi->second.so_test) + SEQ_OFF_MP)) ||
4447
                                        (GET_SEQ(sguid_smi->second.so_test) > SEQ_MP &&
4448
                                         GET_SEQ_OFF(sguid_smi->second.so_nacked) >
4449
                                         GET_SEQ_OFF(sguid_smi->second.so_test) - SEQ_OFF_MP &&
4450
                                         GET_SEQ_OFF(sguid_smi->second.so_nacked) <=
4451
                                         GET_SEQ_OFF(sguid_smi->second.so_test)))
4452
                                {
4453
                                        ccdbg(DVVERB, "(%d)%s seq check nack at p %p\n"
4454
                                                  "\tid 0x%x mf %d off 0x%x\n"
4455
                                                  "\tid 0x%x mf %d off 0x%x\n",
4456
                                                  fd_rx, fd_to_state_name(fd_rx), p,
4457
                                                  GET_SEQ(sguid_smi->second.so_test),
4458
                                                  GET_MF(sguid_smi->second.so_test),
4459
                                                  GET_OFF(sguid_smi->second.so_test),
4460
                                                  GET_SEQ(otw_gdp->seq_mf_off),
4461
                                                  GET_MF(otw_gdp->seq_mf_off),
4462
                                                  GET_OFF(otw_gdp->seq_mf_off));
4463
                                        // so_nacked moves to new upper bound (end of this p)
4464
                                        SET_SEQ_MF_OFF(sguid_smi->second.so_nacked,
4465
                                                                   GET_SEQ(otw_gdp->seq_mf_off), 0,
4466
                                                                   (GET_OFF(otw_gdp->seq_mf_off) +
4467
                                                                        ntohs(otw_gdp->frag_len)));
4468
                                        // nack packets between so_test and new upper bound
4469
                                        seq_hole(sguid_smi, p, sguid_smi->second.so_test,
4470
                                                         otw_gdp->seq_mf_off);
4471
                                        seq_nack(sguid_smi, false);
4472
                                }
4473
                                else
4474
                                {
4475
                                        ccdbg(DVVERB, "(%d)%s seq already nacked at p %p\n"
4476
                                                  "\ttest id 0x%x mf %d off 0x%x\n"
4477
                                                  "\tpast id 0x%x mf %d off 0x%x\n"
4478
                                                  "\tcurr id 0x%x mf %d off 0x%x\n",
4479
                                                  fd_rx, fd_to_state_name(fd_rx), p,
4480
                                                  GET_SEQ(sguid_smi->second.so_test),
4481
                                                  GET_MF(sguid_smi->second.so_test),
4482
                                                  GET_OFF(sguid_smi->second.so_test),
4483
                                                  GET_SEQ(sguid_smi->second.so_nacked),
4484
                                                  GET_MF(sguid_smi->second.so_nacked),
4485
                                                  GET_OFF(sguid_smi->second.so_nacked),
4486
                                                  GET_SEQ(otw_gdp->seq_mf_off),
4487
                                                  GET_MF(otw_gdp->seq_mf_off),
4488
                                                  GET_OFF(otw_gdp->seq_mf_off));
4489
                                }
4490
                        }
4491
                }
4492
                else
4493
                {
4494
                        // case 5: known, start sequence (implies not seq packet)
4495
                        //
4496
                        // tcp client disconnected then reconnected before state aged out
4497
                        ccdbg(DVVERB, "(%d)%s seq restart at p %p\n"
4498
                                  "\tid 0x%x mf %d off 0x%x\n",
4499
                                  fd_rx, fd_to_state_name(fd_rx), p,
4500
                                  GET_SEQ(otw_gdp->seq_mf_off), GET_MF(otw_gdp->seq_mf_off),
4501
                                  GET_OFF(otw_gdp->seq_mf_off));
4502
                }
4503
        }
4504

    
4505
        // seq packet carries no payload, seq packet seq_mf_off is new upper bound
4506
        if ((otw_gdp->flags & GDP_PKT_TYPE_MASK) == GDP_PKT_TYPE_SEQ_PACKET)
4507
        {
4508
                SET_SEQ_MF_OFF(sguid_smi->second.so_test,
4509
                                           GET_SEQ(otw_gdp->seq_mf_off), 0,
4510
                                           GET_OFF(otw_gdp->seq_mf_off));
4511
                ccdbg(DPDU, "(%d)%s seq packet update id 0x%x off 0x%x",
4512
                          fd_rx, fd_to_state_name(fd_rx),
4513
                          GET_SEQ(otw_gdp->seq_mf_off), GET_OFF(otw_gdp->seq_mf_off));
4514
                return false;
4515
        }
4516
        else
4517
        {
4518
                // predict next seq
4519
                if (GET_MF(otw_gdp->seq_mf_off) == 0)
4520
                {
4521
                        // whole p or last frag upper bound is wrap(seq++), mf ignore, off 0
4522
                        SET_SEQ_MF_OFF(sguid_smi->second.so_test,
4523
                                                   ((GET_SEQ(otw_gdp->seq_mf_off) + 1) & SEQ_MASK),
4524
                                                   0, 0);
4525
                        ccdbg(DVVERB, "(%d)%s seq++ off 0", fd_rx, fd_to_state_name(fd_rx));
4526
                }
4527
                else
4528
                {
4529
                        // otherwise upper bound is same seq, mf ignore, off+=frag_len
4530
                        SET_SEQ_MF_OFF(sguid_smi->second.so_test,
4531
                                                   GET_SEQ(otw_gdp->seq_mf_off), 0,
4532
                                                   (GET_OFF(otw_gdp->seq_mf_off) +
4533
                                                        ntohs(otw_gdp->frag_len)));
4534
                        ccdbg(DVVERB, "(%d)%s seq off+=frag_len",
4535
                                  fd_rx, fd_to_state_name(fd_rx));
4536
                }
4537
        }
4538
        return true;
4539
}
4540

    
4541
void
4542
GDPv4Router::seq_hole (src_map_it_t &sguid_smi,
4543
                                           WritablePacket *p_ref,
4544
                                           uint32_t predict_seq_mf_off,
4545
                                           uint32_t current_seq_mf_off)
4546
{
4547
        int fd;
4548
        uint32_t split_seq_mf_off;
4549
        otw_gdp_t *otw_gdp_ref;
4550
        WritablePacket *p_nack;
4551
        otw_gdp_nack_t *otw_gdp_nack;
4552

    
4553
        fd = sguid_smi->second.fd_rx;
4554

    
4555
        ccdbg(DVERB, "(%d)%s seq hole p %p", fd, fd_to_state_name(fd), p_ref);
4556

    
4557
        // nack window wrap is always split, simplifies nack intercept handler
4558
        if (GET_SEQ(predict_seq_mf_off) > GET_SEQ(current_seq_mf_off))
4559
        {
4560
                ccdbg(DVVERB, "(%d)%s seq hole split\n"
4561
                          "\tid 0x%x mf %d off 0x%x lower\n"
4562
                          "\tid 0x%x mf %d off 0x%x upper",
4563
                          fd, fd_to_state_name(fd),
4564
                          GET_SEQ(predict_seq_mf_off),
4565
                          GET_MF(predict_seq_mf_off),
4566
                          GET_OFF(predict_seq_mf_off),                           
4567
                          GET_SEQ(current_seq_mf_off),
4568
                          GET_MF(current_seq_mf_off),
4569
                          GET_OFF(current_seq_mf_off));
4570

    
4571
                // match p end using impossible (i.e. mf 1 with off max) END_SEQ_MF_OFF
4572
                split_seq_mf_off = END_SEQ_MF_OFF;
4573
                seq_hole(sguid_smi, p_ref, predict_seq_mf_off, split_seq_mf_off);
4574

    
4575
                // second half of split, unless new lower bound would be on upper bound
4576
                if (GET_SEQ_OFF(current_seq_mf_off) != 0)
4577
                {
4578
                        SET_SEQ_MF_OFF(split_seq_mf_off, 0, 0, 0);
4579
                        seq_hole(sguid_smi, p_ref, split_seq_mf_off, current_seq_mf_off);
4580
                }
4581
                return;
4582
        }
4583

    
4584
        // CAUTION: do not alter p_ref (FIXME add protection to p_ref)
4585
        otw_gdp_ref = (otw_gdp_t *) p_ref->data();
4586
        
4587
        // create new p from p_ref (hdr only) with opaque space for nack upper bound
4588
        p_nack = Packet::make(0, p_ref->data(), GDP_HDR_SIZE, sizeof(uint32_t));
4589
        if (p_nack == nullptr)
4590
        {
4591
                ccwarn("packet pool exhausted on seq hole");
4592
                // do not kill p_ref
4593
                return;
4594
        }
4595
        put_same_p(p_nack == p_nack->put(sizeof(uint32_t)));
4596

    
4597
        // preset seq_mf_off (do not overwrite!)
4598
        SET_SEQ_PRESET_ANNO(p_nack, 1);
4599
        SET_RXFD_ANNO(p_nack, RXFD_ANNO(p_ref));
4600
        SET_PTYPE_ANNO(p_nack, PTYPE_GDP_PKT);
4601

    
4602
        otw_gdp_nack = (otw_gdp_nack_t *) p_nack->data();
4603
        
4604
        // rewrite src and dst into p_nack with positions swapped
4605
        std::copy(otw_gdp_ref->dst, otw_gdp_ref->dst + sizeof(gdp_name_ca_t),
4606
                          otw_gdp_nack->src);
4607
        std::copy(otw_gdp_ref->src, otw_gdp_ref->src + sizeof(gdp_name_ca_t),
4608
                          otw_gdp_nack->dst);
4609

    
4610
        // upper bound rides in payload
4611
        otw_gdp_nack->frag_len = sizeof(uint32_t);
4612
        otw_gdp_nack->payload_len = sizeof(uint32_t);
4613
        
4614
        otw_gdp_nack->lo_seq_mf_off = predict_seq_mf_off;
4615
        otw_gdp_nack->up_seq_mf_off = current_seq_mf_off;
4616

    
4617
        otw_gdp_nack->flags = ((otw_gdp_nack->flags & ~GDP_PKT_TYPE_MASK) |
4618
                                                   GDP_PKT_TYPE_NAK_PACKET);
4619

    
4620
        ccdbg(DVVERB, "seq hole push back p %p\n"
4621
                  "\tid 0x%x mf %d off 0x%x lower\n"
4622
                  "\tid 0x%x mf %d off 0x%x upper",
4623
                  p_nack,
4624
                  GET_SEQ(otw_gdp_nack->lo_seq_mf_off),
4625
                  GET_MF(otw_gdp_nack->lo_seq_mf_off),
4626
                  GET_OFF(otw_gdp_nack->lo_seq_mf_off),
4627
                  GET_SEQ(otw_gdp_nack->up_seq_mf_off),
4628
                  GET_MF(otw_gdp_nack->up_seq_mf_off),
4629
                  GET_OFF(otw_gdp_nack->up_seq_mf_off));
4630

    
4631
        // push p_nack onto back of nplist
4632
        sguid_smi->second.nplist.push_back(p_nack);
4633
}
4634

    
4635
void
4636
GDPv4Router::seq_nack (src_map_it_t &sguid_smi, bool trim)
4637
{
4638
        int fd_rx;
4639
        WritablePacket *p;
4640
        otw_gdp_nack_t *otw_gdp_nack;
4641

    
4642
        fd_rx = sguid_smi->second.fd_rx;
4643

    
4644
        if (trim)
4645
        {
4646
                // sguid and dguid are used to fill nack trim with src and dst swapped
4647
                dst_map_it_t *dguid_dmip;
4648

    
4649
                p = Packet::make(sizeof(otw_gdp_nack_t));
4650
                if (p == nullptr)
4651
                {
4652
                        ccwarn("packet pool exhausted on seq trim");
4653
                        return;
4654
                }
4655
                // preset seq_mf_off (do not overwrite!)
4656
                SET_SEQ_PRESET_ANNO(p, 1);
4657
                SET_RXFD_ANNO(p, fd_rx);
4658
                SET_PTYPE_ANNO(p, PTYPE_GDP_PKT);
4659

    
4660
                otw_gdp_nack = (otw_gdp_nack_t *) p->data();
4661

    
4662
                otw_gdp_nack->ver = GDP_CHAN_PROTO_VERSION;
4663
                otw_gdp_nack->hdr_len = GDP_HDR_SIZE / 4;
4664
                otw_gdp_nack->flags = GDP_PKT_TYPE_NAK_PACKET;
4665
                otw_gdp_nack->ttl = GDP_TTL_DEFAULT;
4666
                otw_gdp_nack->lo_seq_mf_off = sguid_smi->second.so_test;
4667
                otw_gdp_nack->frag_len = 0;
4668
                otw_gdp_nack->payload_len = 0;
4669
                otw_gdp_nack->up_seq_mf_off = sguid_smi->second.so_test;
4670

    
4671
                dguid_dmip = sguid_smi->second.sm_task_copy_dmip;
4672
                avow(dguid_dmip != nullptr);
4673

    
4674
                // nack trim goes in the reverse direction
4675
                std::copy((*dguid_dmip)->first.data(),
4676
                                  (*dguid_dmip)->first.data() + sizeof(gdp_name_ca_t),
4677
                                  otw_gdp_nack->src);
4678
                std::copy(sguid_smi->first.data(),
4679
                                  sguid_smi->first.data() + sizeof(gdp_name_ca_t),
4680
                                  otw_gdp_nack->dst);
4681
        }
4682
        else
4683
        {
4684
                // peek at front nack
4685
                aver(sguid_smi->second.nplist.size());
4686
                p = static_cast<WritablePacket*>(sguid_smi->second.nplist.front());
4687
                if (p->shared())
4688
                {
4689
                        otw_gdp_nack = (otw_gdp_nack_t *) p->data();
4690
                        ccdbg(DVVERB, "(%d)%s seq %s p %p already enqueued\n"
4691
                                  "\tid 0x%x mf %d off 0x%x lower\n"
4692
                                  "\tid 0x%x mf %d off 0x%x upper",
4693
                                  fd_rx, fd_to_state_name(fd_rx),
4694
                                  (trim ? "trim" : "nack"), p,
4695
                                  GET_SEQ(otw_gdp_nack->lo_seq_mf_off),
4696
                                  GET_MF(otw_gdp_nack->lo_seq_mf_off),
4697
                                  GET_OFF(otw_gdp_nack->lo_seq_mf_off),
4698
                                  GET_SEQ(otw_gdp_nack->up_seq_mf_off),
4699
                                  GET_MF(otw_gdp_nack->up_seq_mf_off),
4700
                                  GET_OFF(otw_gdp_nack->up_seq_mf_off));
4701
                        return;
4702
                }
4703

    
4704
                // clone front nack p
4705
                p = static_cast<WritablePacket*>(p->clone());
4706
                if (p == nullptr)
4707
                {
4708
                        ccwarn("packet pool exhausted on seq nack clone");
4709
                        return;
4710
                }
4711
                // NOTE: retx handler (elsewhere) may split nplist.front()
4712
                // nack p (write new upper bound) or kill it, but neither
4713
                // action is harmful to an already enqueued nack p clone. Not
4714
                // uniqueifying enables p->shared(), which indicates the nack
4715
                // p is already enqueued on the txplist.
4716
                //
4717
                // DEPRECATED:
4718
                // // uniqueify kills original input p whenever uniqueify fails
4719
                // p = p->uniqueify();
4720
                // if (p == nullptr)
4721
                // {
4722
                //         ccwarn("packet pool exhausted on seq nack uniqueify");
4723
                //         return;
4724
                // }
4725

    
4726
                // trim rest of front nack p chain (holding splits), if any, local anno
4727
                p->set_next(nullptr);
4728

    
4729
                otw_gdp_nack = (otw_gdp_nack_t *) p->data();
4730
        }
4731
        aver(SEQ_PRESET_ANNO(p) == 1);
4732

    
4733
        ccdbg(DVVERB, "(%d)%s seq %s p %p\n"
4734
                  "\tid 0x%x mf %d off 0x%x lower\n"
4735
                  "\tid 0x%x mf %d off 0x%x upper",
4736
                  fd_rx, fd_to_state_name(fd_rx),
4737
                  (trim ? "trim" : "nack"), p,
4738
                  GET_SEQ(otw_gdp_nack->lo_seq_mf_off),
4739
                  GET_MF(otw_gdp_nack->lo_seq_mf_off),
4740
                  GET_OFF(otw_gdp_nack->lo_seq_mf_off),
4741
                  GET_SEQ(otw_gdp_nack->up_seq_mf_off),
4742
                  GET_MF(otw_gdp_nack->up_seq_mf_off),
4743
                  GET_OFF(otw_gdp_nack->up_seq_mf_off));
4744

    
4745
        // tx nack via hairpin turn (multicast nacks must not generate map entries)
4746
        (*_fdv)[fd_rx].txplist.push_back(p);
4747
        (*_fdv)[fd_rx].tx_task->reschedule();
4748
        return;
4749
}
4750

    
4751
bool
4752
GDPv4Router::seq_retx (src_map_it_t &sguid_smi, WritablePacket *p_retx)
4753
{
4754
        int fd;
4755
        otw_gdp_t *otw_gdp_retx;
4756
        WritablePacket *p_nack;
4757
        otw_gdp_nack_t *otw_gdp_nack;
4758
        WritablePacket *p_nack_last;
4759
        WritablePacket *n;        
4760
        WritablePacket *p_split;
4761
        otw_gdp_nack_t *otw_gdp_split;
4762

    
4763
        fd = sguid_smi->second.fd_rx;
4764

    
4765
        otw_gdp_retx = (otw_gdp_t *) p_retx->data();
4766

    
4767
        ccdbg(DVERB, "(%d)%s seq retx p %p\n"
4768
                  "\tid 0x%x mf %d off 0x%x",
4769
                  fd, fd_to_state_name(fd), p_retx,
4770
                  GET_SEQ(otw_gdp_retx->seq_mf_off),
4771
                  GET_MF(otw_gdp_retx->seq_mf_off),
4772
                  GET_OFF(otw_gdp_retx->seq_mf_off));
4773

    
4774
        if (sguid_smi->second.nplist.size() == 0)
4775
        {
4776
                ccdbg(DVERB, "### retx drop on empty nplist");
4777
                return false;
4778
        }
4779

    
4780
        // search nplist front nack chain for retx permit, else return false (drop)
4781
        p_nack_last = nullptr;
4782
        p_nack = static_cast<WritablePacket*>(sguid_smi->second.nplist.front());
4783
        do
4784
        {
4785
                otw_gdp_nack = (otw_gdp_nack_t *) p_nack->data();
4786
                ccdbg(DVERB, "### retx check nack p %p\n"
4787
                          "\tnack id 0x%x mf %d off 0x%x lower\n"
4788
                          "\tnack id 0x%x mf %d off 0x%x upper",
4789
                          p_nack,
4790
                          GET_SEQ(otw_gdp_nack->lo_seq_mf_off),
4791
                          GET_MF(otw_gdp_nack->lo_seq_mf_off),
4792
                          GET_OFF(otw_gdp_nack->lo_seq_mf_off),
4793
                          GET_SEQ(otw_gdp_nack->up_seq_mf_off),
4794
                          GET_MF(otw_gdp_nack->up_seq_mf_off),
4795
                          GET_OFF(otw_gdp_nack->up_seq_mf_off));
4796

    
4797
                // retx pemitted by this p_nack
4798
                if (GET_SEQ_OFF(otw_gdp_retx->seq_mf_off) >=
4799
                        GET_SEQ_OFF(otw_gdp_nack->lo_seq_mf_off) &&
4800
                        GET_SEQ_OFF(otw_gdp_retx->seq_mf_off) +
4801
                        ntohs(otw_gdp_retx->frag_len) <=
4802
                        GET_SEQ_OFF(otw_gdp_nack->up_seq_mf_off))
4803
                {
4804
                        // retx seq/mf/off on lo
4805
                        if (GET_SEQ_OFF(otw_gdp_retx->seq_mf_off) ==
4806
                                GET_SEQ_OFF(otw_gdp_nack->lo_seq_mf_off))
4807
                        {
4808
                                // retx seq/*/off+frag_len on up
4809
                                //  or
4810
                                // retx mf == 0 and
4811
                                //     (retx seq+1/*/0x0 in up or (retx max/0/* && end in up))
4812
                                if (GET_SEQ_OFF(otw_gdp_retx->seq_mf_off) +
4813
                                        ntohs(otw_gdp_retx->frag_len) ==
4814
                                        GET_SEQ_OFF(otw_gdp_nack->up_seq_mf_off) ||
4815
                                        (GET_MF(otw_gdp_retx->seq_mf_off) == 0 &&
4816
                                         (GET_SEQ_NEXT(otw_gdp_retx->seq_mf_off) ==
4817
                                          GET_SEQ_OFF(otw_gdp_nack->up_seq_mf_off) ||
4818
                                          (GET_SEQ(otw_gdp_retx->seq_mf_off) == SEQ_MASK &&